1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "hencoder.h"
17 #include <map>
18 #include <utility>
19 #include "utils/hdf_base.h"
20 #include "OMX_VideoExt.h"
21 #include "media_description.h"  // foundation/multimedia/av_codec/interfaces/inner_api/native/
22 #include "type_converter.h"
23 #include "hcodec_log.h"
24 #include "hcodec_dfx.h"
25 #include "hcodec_utils.h"
26 #include "v3_0/codec_ext_types.h"
27 
28 namespace OHOS::MediaAVCodec {
29 using namespace std;
30 
~HEncoder()31 HEncoder::~HEncoder()
32 {
33     MsgHandleLoop::Stop();
34 }
35 
OnConfigure(const Format & format)36 int32_t HEncoder::OnConfigure(const Format &format)
37 {
38     configFormat_ = make_shared<Format>(format);
39     int32_t ret = ConfigureBufferType();
40     if (ret != AVCS_ERR_OK) {
41         return ret;
42     }
43 
44     optional<double> frameRate = GetFrameRateFromUser(format);
45     ret = SetupPort(format, frameRate);
46     if (ret != AVCS_ERR_OK) {
47         return ret;
48     }
49     ConfigureProtocol(format, frameRate);
50 
51     ret = ConfigureOutputBitrate(format);
52     if (ret != AVCS_ERR_OK) {
53         HLOGW("ConfigureOutputBitrate failed");
54     }
55     ret = SetColorAspects(format);
56     if (ret != AVCS_ERR_OK) {
57         HLOGW("set color aspect failed");
58     }
59     (void)SetProcessName();
60     (void)SetFrameRateAdaptiveMode(format);
61     CheckIfEnableCb(format);
62     ret = SetTemperalLayer(format);
63     if (ret != AVCS_ERR_OK) {
64         return ret;
65     }
66     ret = SetLTRParam(format);
67     if (ret != AVCS_ERR_OK) {
68         return ret;
69     }
70     ret = SetQpRange(format, false);
71     if (ret != AVCS_ERR_OK) {
72         return ret;
73     }
74     ret = SetLowLatency(format);
75     if (ret != AVCS_ERR_OK) {
76         return ret;
77     }
78     ret = SetRepeat(format);
79     if (ret != AVCS_ERR_OK) {
80         return ret;
81     }
82     (void)EnableEncoderParamsFeedback(format);
83     return AVCS_ERR_OK;
84 }
85 
ConfigureBufferType()86 int32_t HEncoder::ConfigureBufferType()
87 {
88     UseBufferType useBufferTypes;
89     InitOMXParamExt(useBufferTypes);
90     useBufferTypes.portIndex = OMX_DirInput;
91     useBufferTypes.bufferType = CODEC_BUFFER_TYPE_DYNAMIC_HANDLE;
92     if (!SetParameter(OMX_IndexParamUseBufferType, useBufferTypes)) {
93         HLOGE("component don't support CODEC_BUFFER_TYPE_DYNAMIC_HANDLE");
94         return AVCS_ERR_INVALID_VAL;
95     }
96     return AVCS_ERR_OK;
97 }
98 
CheckIfEnableCb(const Format & format)99 void HEncoder::CheckIfEnableCb(const Format &format)
100 {
101     int32_t enableCb = 0;
102     if (format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_ENABLE_SURFACE_INPUT_CALLBACK, enableCb)) {
103         HLOGI("enable surface mode callback flag %d", enableCb);
104         enableSurfaceModeInputCb_ = static_cast<bool>(enableCb);
105     }
106 }
107 
SetRepeat(const Format & format)108 int32_t HEncoder::SetRepeat(const Format &format)
109 {
110     int repeatMs = 0;
111     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_REPEAT_PREVIOUS_FRAME_AFTER, repeatMs)) {
112         return AVCS_ERR_OK;
113     }
114     if (repeatMs <= 0) {
115         HLOGW("invalid repeatMs %d", repeatMs);
116         return AVCS_ERR_INVALID_VAL;
117     }
118     repeatUs_ = static_cast<uint64_t>(repeatMs * TIME_RATIO_S_TO_MS);
119 
120     int repeatMaxCnt = 0;
121     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_REPEAT_PREVIOUS_MAX_COUNT, repeatMaxCnt)) {
122         return AVCS_ERR_OK;
123     }
124     if (repeatMaxCnt == 0) {
125         HLOGW("invalid repeatMaxCnt %d", repeatMaxCnt);
126         return AVCS_ERR_INVALID_VAL;
127     }
128     repeatMaxCnt_ = repeatMaxCnt;
129     return AVCS_ERR_OK;
130 }
131 
SetLTRParam(const Format & format)132 int32_t HEncoder::SetLTRParam(const Format &format)
133 {
134     int32_t ltrFrameNum = -1;
135     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_LTR_FRAME_COUNT, ltrFrameNum)) {
136         return AVCS_ERR_OK;
137     }
138     if (!caps_.port.video.isSupportLTR) {
139         HLOGW("platform not support LTR");
140         return AVCS_ERR_OK;
141     }
142     if (ltrFrameNum <= 0 || ltrFrameNum > caps_.port.video.maxLTRFrameNum) {
143         HLOGE("invalid ltrFrameNum %d", ltrFrameNum);
144         return AVCS_ERR_INVALID_VAL;
145     }
146     if (enableTSVC_) {
147         HLOGW("user has enabled temporal scale, can not set LTR param");
148         return AVCS_ERR_INVALID_VAL;
149     }
150     CodecLTRParam info;
151     InitOMXParamExt(info);
152     info.ltrFrameListLen = static_cast<uint32_t>(ltrFrameNum);
153     if (!SetParameter(OMX_IndexParamLTR, info)) {
154         HLOGE("configure LTR failed");
155         return AVCS_ERR_INVALID_VAL;
156     }
157     enableLTR_ = true;
158     return AVCS_ERR_OK;
159 }
160 
EnableEncoderParamsFeedback(const Format & format)161 int32_t HEncoder::EnableEncoderParamsFeedback(const Format &format)
162 {
163     int32_t enableParamsFeedback {};
164     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_ENABLE_PARAMS_FEEDBACK, enableParamsFeedback)) {
165         return AVCS_ERR_OK;
166     }
167     OMX_CONFIG_BOOLEANTYPE param {};
168     InitOMXParam(param);
169     param.bEnabled = enableParamsFeedback ? OMX_TRUE : OMX_FALSE;
170     if (!SetParameter(OMX_IndexParamEncParamsFeedback, param)) {
171         HLOGE("configure encoder params feedback[%d] failed", enableParamsFeedback);
172         return AVCS_ERR_INVALID_VAL;
173     }
174     HLOGI("configure encoder params feedback[%d] success", enableParamsFeedback);
175     return AVCS_ERR_OK;
176 }
177 
SetQpRange(const Format & format,bool isCfg)178 int32_t HEncoder::SetQpRange(const Format &format, bool isCfg)
179 {
180     int32_t minQp;
181     int32_t maxQp;
182     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_QP_MIN, minQp) ||
183         !format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_QP_MAX, maxQp)) {
184         return AVCS_ERR_OK;
185     }
186 
187     CodecQPRangeParam QPRangeParam;
188     InitOMXParamExt(QPRangeParam);
189     QPRangeParam.minQp = static_cast<uint32_t>(minQp);
190     QPRangeParam.maxQp = static_cast<uint32_t>(maxQp);
191     if (!SetParameter(OMX_IndexParamQPRange, QPRangeParam, isCfg)) {
192         HLOGE("set qp range (%d~%d) failed", minQp, maxQp);
193         return AVCS_ERR_UNKNOWN;
194     }
195     HLOGI("set qp range (%d~%d) succ", minQp, maxQp);
196     return AVCS_ERR_OK;
197 }
198 
SetTemperalLayer(const Format & format)199 int32_t HEncoder::SetTemperalLayer(const Format &format)
200 {
201     int32_t enableTemporalScale = 0;
202     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_ENABLE_TEMPORAL_SCALABILITY, enableTemporalScale) ||
203         (enableTemporalScale == 0)) {
204         return AVCS_ERR_OK;
205     }
206     if (!caps_.port.video.isSupportTSVC) {
207         HLOGW("platform not support temporal scale");
208         return AVCS_ERR_OK;
209     }
210     Media::Plugins::TemporalGopReferenceMode GopReferenceMode{};
211     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_TEMPORAL_GOP_REFERENCE_MODE,
212         *reinterpret_cast<int *>(&GopReferenceMode)) ||
213         static_cast<int32_t>(GopReferenceMode) != 2) { // 2: gop mode
214         HLOGE("user enable temporal scalability but not set invalid temporal gop mode");
215         return AVCS_ERR_INVALID_VAL;
216     }
217     int32_t temporalGopSize = 0;
218     if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_ENCODER_TEMPORAL_GOP_SIZE, temporalGopSize)) {
219         HLOGE("user enable temporal scalability but not set invalid temporal gop size");
220         return AVCS_ERR_INVALID_VAL;
221     }
222 
223     CodecTemperalLayerParam temperalLayerParam;
224     InitOMXParamExt(temperalLayerParam);
225     switch (temporalGopSize) {
226         case 2: // 2: picture size of the temporal group
227             temperalLayerParam.layerCnt = 2; // 2: layer of the temporal group
228             break;
229         case 4: // 4: picture size of the temporal group
230             temperalLayerParam.layerCnt = 3; // 3: layer of the temporal group
231             break;
232         default:
233             HLOGE("user set invalid temporal gop size %d", temporalGopSize);
234             return AVCS_ERR_INVALID_VAL;
235     }
236 
237     if (!SetParameter(OMX_IndexParamTemperalLayer, temperalLayerParam)) {
238         HLOGE("set temporal layer param failed");
239         return AVCS_ERR_UNKNOWN;
240     }
241     HLOGI("set temporal layer param %d succ", temperalLayerParam.layerCnt);
242     enableTSVC_ = true;
243     return AVCS_ERR_OK;
244 }
245 
OnConfigureBuffer(std::shared_ptr<AVBuffer> buffer)246 int32_t HEncoder::OnConfigureBuffer(std::shared_ptr<AVBuffer> buffer)
247 {
248     if (!caps_.port.video.isSupportWaterMark) {
249         HLOGE("this device dont support water mark");
250         return AVCS_ERR_UNSUPPORT;
251     }
252     if (buffer == nullptr || buffer->memory_ == nullptr || buffer->meta_ == nullptr) {
253         HLOGE("invalid buffer");
254         return AVCS_ERR_INVALID_VAL;
255     }
256     sptr<SurfaceBuffer> waterMarkBuffer = buffer->memory_->GetSurfaceBuffer();
257     if (waterMarkBuffer == nullptr) {
258         HLOGE("null surfacebuffer");
259         return AVCS_ERR_INVALID_VAL;
260     }
261     if (waterMarkBuffer->GetFormat() != GRAPHIC_PIXEL_FMT_RGBA_8888) {
262         HLOGE("pixel fmt should be RGBA8888");
263         return AVCS_ERR_INVALID_VAL;
264     }
265     bool enableWaterMark = false;
266     int32_t x = 0;
267     int32_t y = 0;
268     int32_t w = 0;
269     int32_t h = 0;
270     if (!buffer->meta_->GetData(OHOS::Media::Tag::VIDEO_ENCODER_ENABLE_WATERMARK, enableWaterMark) ||
271         !buffer->meta_->GetData(OHOS::Media::Tag::VIDEO_COORDINATE_X, x) ||
272         !buffer->meta_->GetData(OHOS::Media::Tag::VIDEO_COORDINATE_Y, y) ||
273         !buffer->meta_->GetData(OHOS::Media::Tag::VIDEO_COORDINATE_W, w) ||
274         !buffer->meta_->GetData(OHOS::Media::Tag::VIDEO_COORDINATE_H, h)) {
275         HLOGE("invalid value");
276         return AVCS_ERR_INVALID_VAL;
277     }
278     if (x < 0 || y < 0 || w <= 0 || h <= 0) {
279         HLOGE("invalid coordinate, x %d, y %d, w %d, h %d", x, y, w, h);
280         return AVCS_ERR_INVALID_VAL;
281     }
282     CodecHDI::CodecParamOverlay param {
283         .size = sizeof(param), .enable = enableWaterMark, .dstX = static_cast<uint32_t>(x),
284         .dstY = static_cast<uint32_t>(y), .dstW = static_cast<uint32_t>(w), .dstH = static_cast<uint32_t>(h),
285     };
286     int8_t* p = reinterpret_cast<int8_t*>(&param);
287     std::vector<int8_t> inVec(p, p + sizeof(param));
288     CodecHDI::OmxCodecBuffer omxbuffer {};
289     omxbuffer.bufferhandle = new HDI::Base::NativeBuffer(waterMarkBuffer->GetBufferHandle());
290     int32_t ret = compNode_->SetParameterWithBuffer(CodecHDI::Codec_IndexParamOverlayBuffer, inVec, omxbuffer);
291     if (ret != HDF_SUCCESS) {
292         HLOGE("SetParameterWithBuffer failed");
293         return AVCS_ERR_INVALID_VAL;
294     }
295     HLOGI("SetParameterWithBuffer succ");
296     return AVCS_ERR_OK;
297 }
298 
SetColorAspects(const Format & format)299 int32_t HEncoder::SetColorAspects(const Format &format)
300 {
301     int range = 0;
302     int primary = static_cast<int>(COLOR_PRIMARY_UNSPECIFIED);
303     int transfer = static_cast<int>(TRANSFER_CHARACTERISTIC_UNSPECIFIED);
304     int matrix = static_cast<int>(MATRIX_COEFFICIENT_UNSPECIFIED);
305 
306     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_RANGE_FLAG, range)) {
307         HLOGI("user set range flag %d", range);
308     }
309     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_COLOR_PRIMARIES, primary)) {
310         HLOGI("user set primary %d", primary);
311     }
312     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_TRANSFER_CHARACTERISTICS, transfer)) {
313         HLOGI("user set transfer %d", transfer);
314     }
315     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_MATRIX_COEFFICIENTS, matrix)) {
316         HLOGI("user set matrix %d", matrix);
317     }
318     if (primary < 0 || primary > UINT8_MAX ||
319         transfer < 0 || transfer > UINT8_MAX ||
320         matrix < 0 || matrix > UINT8_MAX) {
321         HLOGW("invalid color");
322         return AVCS_ERR_INVALID_VAL;
323     }
324 
325     CodecVideoColorspace param;
326     InitOMXParamExt(param);
327     param.portIndex = OMX_DirInput;
328     param.aspects.range = static_cast<bool>(range);
329     param.aspects.primaries = static_cast<uint8_t>(primary);
330     param.aspects.transfer = static_cast<uint8_t>(transfer);
331     param.aspects.matrixCoeffs = static_cast<uint8_t>(matrix);
332 
333     if (!SetParameter(OMX_IndexColorAspects, param, true)) {
334         HLOGE("failed to set CodecVideoColorSpace");
335         return AVCS_ERR_UNKNOWN;
336     }
337     HLOGI("set color aspects (isFullRange %d, primary %u, transfer %u, matrix %u) succ",
338           param.aspects.range, param.aspects.primaries,
339           param.aspects.transfer, param.aspects.matrixCoeffs);
340     return AVCS_ERR_OK;
341 }
342 
CalcInputBufSize(PortInfo & info,VideoPixelFormat pixelFmt)343 void HEncoder::CalcInputBufSize(PortInfo &info, VideoPixelFormat pixelFmt)
344 {
345     uint32_t inSize = AlignTo(info.width, 128u) * AlignTo(info.height, 128u); // 128: block size
346     if (pixelFmt == VideoPixelFormat::RGBA) {
347         inSize = inSize * 4; // 4 byte per pixel
348     } else {
349         inSize = inSize * 3 / 2; // 3: nom, 2: denom
350     }
351     info.inputBufSize = inSize;
352 }
353 
SetupPort(const Format & format,std::optional<double> frameRate)354 int32_t HEncoder::SetupPort(const Format &format, std::optional<double> frameRate)
355 {
356     constexpr int32_t MAX_ENCODE_WIDTH = 10000;
357     constexpr int32_t MAX_ENCODE_HEIGHT = 10000;
358     int32_t width;
359     if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width) || width <= 0 || width > MAX_ENCODE_WIDTH) {
360         HLOGE("format should contain width");
361         return AVCS_ERR_INVALID_VAL;
362     }
363     int32_t height;
364     if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height) || height <= 0 || height > MAX_ENCODE_HEIGHT) {
365         HLOGE("format should contain height");
366         return AVCS_ERR_INVALID_VAL;
367     }
368     width_ = static_cast<uint32_t>(width);
369     height_ = static_cast<uint32_t>(height);
370     HLOGI("user set width %d, height %d", width, height);
371     if (!GetPixelFmtFromUser(format)) {
372         return AVCS_ERR_INVALID_VAL;
373     }
374 
375     if (!frameRate.has_value()) {
376         HLOGI("user don't set valid frame rate, use default 60.0");
377         frameRate = 60.0; // default frame rate 60.0
378     }
379 
380     PortInfo inputPortInfo = {static_cast<uint32_t>(width), static_cast<uint32_t>(height),
381                               OMX_VIDEO_CodingUnused, configuredFmt_, frameRate.value()};
382     CalcInputBufSize(inputPortInfo, configuredFmt_.innerFmt);
383     int32_t ret = SetVideoPortInfo(OMX_DirInput, inputPortInfo);
384     if (ret != AVCS_ERR_OK) {
385         return ret;
386     }
387 
388     PortInfo outputPortInfo = {static_cast<uint32_t>(width), static_cast<uint32_t>(height),
389                                codingType_, std::nullopt, frameRate.value()};
390     ret = SetVideoPortInfo(OMX_DirOutput, outputPortInfo);
391     if (ret != AVCS_ERR_OK) {
392         return ret;
393     }
394     return AVCS_ERR_OK;
395 }
396 
UpdateInPortFormat()397 int32_t HEncoder::UpdateInPortFormat()
398 {
399     OMX_PARAM_PORTDEFINITIONTYPE def;
400     InitOMXParam(def);
401     def.nPortIndex = OMX_DirInput;
402     if (!GetParameter(OMX_IndexParamPortDefinition, def)) {
403         HLOGE("get input port definition failed");
404         return AVCS_ERR_UNKNOWN;
405     }
406     PrintPortDefinition(def);
407     uint32_t w = def.format.video.nFrameWidth;
408     uint32_t h = def.format.video.nFrameHeight;
409     inBufferCnt_ = def.nBufferCountActual;
410 
411     // save into member variable
412     requestCfg_.timeout = 0;
413     requestCfg_.width = static_cast<int32_t>(w);
414     requestCfg_.height = static_cast<int32_t>(h);
415     requestCfg_.strideAlignment = STRIDE_ALIGNMENT;
416     requestCfg_.format = configuredFmt_.graphicFmt;
417     requestCfg_.usage = BUFFER_MODE_REQUEST_USAGE;
418 
419     if (inputFormat_ == nullptr) {
420         inputFormat_ = make_shared<Format>();
421     }
422     inputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, w);
423     inputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, h);
424     inputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT,
425         static_cast<int32_t>(configuredFmt_.innerFmt));
426     return AVCS_ERR_OK;
427 }
428 
UpdateOutPortFormat()429 int32_t HEncoder::UpdateOutPortFormat()
430 {
431     OMX_PARAM_PORTDEFINITIONTYPE def;
432     InitOMXParam(def);
433     def.nPortIndex = OMX_DirOutput;
434     if (!GetParameter(OMX_IndexParamPortDefinition, def)) {
435         HLOGE("get output port definition failed");
436         return AVCS_ERR_UNKNOWN;
437     }
438     PrintPortDefinition(def);
439     if (outputFormat_ == nullptr) {
440         outputFormat_ = make_shared<Format>();
441     }
442     outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, def.format.video.nFrameWidth);
443     outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, def.format.video.nFrameHeight);
444     return AVCS_ERR_OK;
445 }
446 
SetPFramesSpacing(int32_t iFramesIntervalInMs,double frameRate,uint32_t bFramesSpacing=0)447 static uint32_t SetPFramesSpacing(int32_t iFramesIntervalInMs, double frameRate, uint32_t bFramesSpacing = 0)
448 {
449     if (iFramesIntervalInMs < 0) { // IPPPP...
450         return UINT32_MAX - 1;
451     }
452     if (iFramesIntervalInMs == 0) { // IIIII...
453         return 0;
454     }
455     uint32_t iFramesInterval = iFramesIntervalInMs * frameRate / TIME_RATIO_S_TO_MS;
456     uint32_t pFramesSpacing = iFramesInterval / (bFramesSpacing + 1);
457     return pFramesSpacing > 0 ? pFramesSpacing - 1 : 0;
458 }
459 
GetBitRateFromUser(const Format & format)460 std::optional<uint32_t> HEncoder::GetBitRateFromUser(const Format &format)
461 {
462     int64_t bitRateLong;
463     if (format.GetLongValue(MediaDescriptionKey::MD_KEY_BITRATE, bitRateLong) && bitRateLong > 0 &&
464         bitRateLong <= UINT32_MAX) {
465         LOGI("user set bit rate %" PRId64 "", bitRateLong);
466         return static_cast<uint32_t>(bitRateLong);
467     }
468     int32_t bitRateInt;
469     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_BITRATE, bitRateInt) && bitRateInt > 0) {
470         LOGI("user set bit rate %d", bitRateInt);
471         return static_cast<uint32_t>(bitRateInt);
472     }
473     return nullopt;
474 }
475 
GetBitRateModeFromUser(const Format & format)476 std::optional<VideoEncodeBitrateMode> HEncoder::GetBitRateModeFromUser(const Format &format)
477 {
478     VideoEncodeBitrateMode mode;
479     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_VIDEO_ENCODE_BITRATE_MODE, *reinterpret_cast<int *>(&mode))) {
480         return mode;
481     }
482     return nullopt;
483 }
484 
SetConstantQualityMode(int32_t quality)485 int32_t HEncoder::SetConstantQualityMode(int32_t quality)
486 {
487     ControlRateConstantQuality bitrateType;
488     InitOMXParamExt(bitrateType);
489     bitrateType.portIndex = OMX_DirOutput;
490     bitrateType.qualityValue = static_cast<uint32_t>(quality);
491     if (!SetParameter(OMX_IndexParamControlRateConstantQuality, bitrateType)) {
492         HLOGE("failed to set OMX_IndexParamControlRateConstantQuality");
493         return AVCS_ERR_UNKNOWN;
494     }
495     HLOGI("set CQ mode and target quality %u succ", bitrateType.qualityValue);
496     outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_VIDEO_ENCODE_BITRATE_MODE, CQ);
497     outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_QUALITY, quality);
498     return AVCS_ERR_OK;
499 }
500 
ConfigureOutputBitrate(const Format & format)501 int32_t HEncoder::ConfigureOutputBitrate(const Format &format)
502 {
503     OMX_VIDEO_PARAM_BITRATETYPE bitrateType;
504     InitOMXParam(bitrateType);
505     bitrateType.nPortIndex = OMX_DirOutput;
506     if (!GetParameter(OMX_IndexParamVideoBitrate, bitrateType)) {
507         HLOGE("get OMX_IndexParamVideoBitrate failed");
508         return AVCS_ERR_UNKNOWN;
509     }
510 
511     optional<VideoEncodeBitrateMode> bitRateMode = GetBitRateModeFromUser(format);
512     int32_t quality;
513     if (bitRateMode.has_value() && bitRateMode.value() == CQ &&
514         format.GetIntValue(MediaDescriptionKey::MD_KEY_QUALITY, quality) && quality >= 0) {
515         return SetConstantQualityMode(quality);
516     }
517     optional<uint32_t> bitRate = GetBitRateFromUser(format);
518     if (bitRate.has_value()) {
519         bitrateType.nTargetBitrate = bitRate.value();
520     }
521     if (bitRateMode.has_value() && (bitRateMode.value() == VBR || bitRateMode.value() == CBR)) {
522         bitrateType.eControlRate = (bitRateMode.value() == CBR) ?
523             OMX_Video_ControlRateConstant : OMX_Video_ControlRateVariable;
524     }
525     if (!SetParameter(OMX_IndexParamVideoBitrate, bitrateType)) {
526         HLOGE("failed to set OMX_IndexParamVideoBitrate");
527         return AVCS_ERR_UNKNOWN;
528     }
529     outputFormat_->PutLongValue(MediaDescriptionKey::MD_KEY_BITRATE,
530         static_cast<int64_t>(bitrateType.nTargetBitrate));
531     if (bitrateType.eControlRate == OMX_Video_ControlRateConstant ||
532         bitrateType.eControlRate == OMX_Video_ControlRateVariable) {
533         bitRateMode = bitrateType.eControlRate == OMX_Video_ControlRateConstant ? CBR : VBR;
534         outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_VIDEO_ENCODE_BITRATE_MODE,
535             static_cast<int32_t>(bitRateMode.value()));
536         HLOGI("set %s mode and target bitrate %u bps succ", (bitRateMode.value() == CBR) ? "CBR" : "VBR",
537             bitrateType.nTargetBitrate);
538     } else {
539         HLOGI("set default bitratemode and target bitrate %u bps succ", bitrateType.nTargetBitrate);
540     }
541     return AVCS_ERR_OK;
542 }
543 
ConfigureProtocol(const Format & format,std::optional<double> frameRate)544 void HEncoder::ConfigureProtocol(const Format &format, std::optional<double> frameRate)
545 {
546     int32_t ret = AVCS_ERR_OK;
547     switch (static_cast<int>(codingType_)) {
548         case OMX_VIDEO_CodingAVC:
549             ret = SetupAVCEncoderParameters(format, frameRate);
550             break;
551         case CODEC_OMX_VIDEO_CodingHEVC:
552             ret = SetupHEVCEncoderParameters(format, frameRate);
553             break;
554         default:
555             break;
556     }
557     if (ret != AVCS_ERR_OK) {
558         HLOGW("set protocol param failed");
559     }
560 }
561 
SetupAVCEncoderParameters(const Format & format,std::optional<double> frameRate)562 int32_t HEncoder::SetupAVCEncoderParameters(const Format &format, std::optional<double> frameRate)
563 {
564     OMX_VIDEO_PARAM_AVCTYPE avcType;
565     InitOMXParam(avcType);
566     avcType.nPortIndex = OMX_DirOutput;
567     if (!GetParameter(OMX_IndexParamVideoAvc, avcType)) {
568         HLOGE("get OMX_IndexParamVideoAvc parameter fail");
569         return AVCS_ERR_UNKNOWN;
570     }
571     avcType.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
572     avcType.nBFrames = 0;
573 
574     AVCProfile profile;
575     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_PROFILE, *reinterpret_cast<int *>(&profile))) {
576         optional<OMX_VIDEO_AVCPROFILETYPE> omxAvcProfile = TypeConverter::InnerAvcProfileToOmxProfile(profile);
577         if (omxAvcProfile.has_value()) {
578             avcType.eProfile = omxAvcProfile.value();
579         }
580     }
581     int32_t iFrameInterval;
582     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_I_FRAME_INTERVAL, iFrameInterval) && frameRate.has_value()) {
583         SetAvcFields(avcType, iFrameInterval, frameRate.value());
584     }
585 
586     if (avcType.nBFrames != 0) {
587         avcType.nAllowedPictureTypes |= OMX_VIDEO_PictureTypeB;
588     }
589     avcType.bEnableUEP = OMX_FALSE;
590     avcType.bEnableFMO = OMX_FALSE;
591     avcType.bEnableASO = OMX_FALSE;
592     avcType.bEnableRS = OMX_FALSE;
593     avcType.bFrameMBsOnly = OMX_TRUE;
594     avcType.bMBAFF = OMX_FALSE;
595     avcType.eLoopFilterMode = OMX_VIDEO_AVCLoopFilterEnable;
596 
597     if (!SetParameter(OMX_IndexParamVideoAvc, avcType)) {
598         HLOGE("failed to set OMX_IndexParamVideoAvc");
599         return AVCS_ERR_UNKNOWN;
600     }
601     return AVCS_ERR_OK;
602 }
603 
SetAvcFields(OMX_VIDEO_PARAM_AVCTYPE & avcType,int32_t iFrameInterval,double frameRate)604 void HEncoder::SetAvcFields(OMX_VIDEO_PARAM_AVCTYPE &avcType, int32_t iFrameInterval, double frameRate)
605 {
606     HLOGI("iFrameInterval:%d, frameRate:%.2f, eProfile:0x%x, eLevel:0x%x",
607           iFrameInterval, frameRate, avcType.eProfile, avcType.eLevel);
608 
609     if (avcType.eProfile == OMX_VIDEO_AVCProfileBaseline) {
610         avcType.nSliceHeaderSpacing = 0;
611         avcType.bUseHadamard = OMX_TRUE;
612         avcType.nRefFrames = 1;
613         avcType.nPFrames = SetPFramesSpacing(iFrameInterval, frameRate, avcType.nBFrames);
614         if (avcType.nPFrames == 0) {
615             avcType.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI;
616         }
617         avcType.nRefIdx10ActiveMinus1 = 0;
618         avcType.nRefIdx11ActiveMinus1 = 0;
619         avcType.bEntropyCodingCABAC = OMX_FALSE;
620         avcType.bWeightedPPrediction = OMX_FALSE;
621         avcType.bconstIpred = OMX_FALSE;
622         avcType.bDirect8x8Inference = OMX_FALSE;
623         avcType.bDirectSpatialTemporal = OMX_FALSE;
624         avcType.nCabacInitIdc = 0;
625     } else if (avcType.eProfile == OMX_VIDEO_AVCProfileMain || avcType.eProfile == OMX_VIDEO_AVCProfileHigh) {
626         avcType.nSliceHeaderSpacing = 0;
627         avcType.bUseHadamard = OMX_TRUE;
628         avcType.nRefFrames = avcType.nBFrames == 0 ? 1 : 2; // 2 is number of reference frames
629         avcType.nPFrames = SetPFramesSpacing(iFrameInterval, frameRate, avcType.nBFrames);
630         avcType.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;
631         avcType.nRefIdx10ActiveMinus1 = 0;
632         avcType.nRefIdx11ActiveMinus1 = 0;
633         avcType.bEntropyCodingCABAC = OMX_TRUE;
634         avcType.bWeightedPPrediction = OMX_TRUE;
635         avcType.bconstIpred = OMX_TRUE;
636         avcType.bDirect8x8Inference = OMX_TRUE;
637         avcType.bDirectSpatialTemporal = OMX_TRUE;
638         avcType.nCabacInitIdc = 1;
639     }
640 }
641 
SetupHEVCEncoderParameters(const Format & format,std::optional<double> frameRate)642 int32_t HEncoder::SetupHEVCEncoderParameters(const Format &format, std::optional<double> frameRate)
643 {
644     CodecVideoParamHevc hevcType;
645     InitOMXParamExt(hevcType);
646     hevcType.portIndex = OMX_DirOutput;
647     if (!GetParameter(OMX_IndexParamVideoHevc, hevcType)) {
648         HLOGE("get OMX_IndexParamVideoHevc parameter fail");
649         return AVCS_ERR_UNKNOWN;
650     }
651 
652     HEVCProfile profile;
653     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_PROFILE, *reinterpret_cast<int *>(&profile))) {
654         optional<CodecHevcProfile> omxHevcProfile = TypeConverter::InnerHevcProfileToOmxProfile(profile);
655         if (omxHevcProfile.has_value()) {
656             hevcType.profile = omxHevcProfile.value();
657             HLOGI("HEVCProfile %d, CodecHevcProfile 0x%x", profile, hevcType.profile);
658         }
659     }
660 
661     int32_t iFrameInterval;
662     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_I_FRAME_INTERVAL, iFrameInterval) && frameRate.has_value()) {
663         if (iFrameInterval < 0) { // IPPPP...
664             hevcType.keyFrameInterval = UINT32_MAX - 1;
665         } else if (iFrameInterval == 0) { // all intra
666             hevcType.keyFrameInterval = 1;
667         } else {
668             hevcType.keyFrameInterval = iFrameInterval * frameRate.value() / TIME_RATIO_S_TO_MS;
669         }
670         HLOGI("frameRate %.2f, iFrameInterval %d, keyFrameInterval %u", frameRate.value(),
671               iFrameInterval, hevcType.keyFrameInterval);
672     }
673 
674     if (!SetParameter(OMX_IndexParamVideoHevc, hevcType)) {
675         HLOGE("failed to set OMX_IndexParamVideoHevc");
676         return AVCS_ERR_INVALID_VAL;
677     }
678     return AVCS_ERR_OK;
679 }
680 
RequestIDRFrame()681 int32_t HEncoder::RequestIDRFrame()
682 {
683     OMX_CONFIG_INTRAREFRESHVOPTYPE params;
684     InitOMXParam(params);
685     params.nPortIndex = OMX_DirOutput;
686     params.IntraRefreshVOP = OMX_TRUE;
687     if (!SetParameter(OMX_IndexConfigVideoIntraVOPRefresh, params, true)) {
688         HLOGE("failed to request IDR frame");
689         return AVCS_ERR_UNKNOWN;
690     }
691     HLOGI("Set IDR Frame success");
692     return AVCS_ERR_OK;
693 }
694 
OnSetParameters(const Format & format)695 int32_t HEncoder::OnSetParameters(const Format &format)
696 {
697     optional<uint32_t> bitRate = GetBitRateFromUser(format);
698     if (bitRate.has_value()) {
699         OMX_VIDEO_CONFIG_BITRATETYPE bitrateCfgType;
700         InitOMXParam(bitrateCfgType);
701         bitrateCfgType.nPortIndex = OMX_DirOutput;
702         bitrateCfgType.nEncodeBitrate = bitRate.value();
703         if (!SetParameter(OMX_IndexConfigVideoBitrate, bitrateCfgType, true)) {
704             HLOGW("failed to config OMX_IndexConfigVideoBitrate");
705         }
706     }
707 
708     optional<double> frameRate = GetFrameRateFromUser(format);
709     if (frameRate.has_value()) {
710         OMX_CONFIG_FRAMERATETYPE framerateCfgType;
711         InitOMXParam(framerateCfgType);
712         framerateCfgType.nPortIndex = OMX_DirInput;
713         framerateCfgType.xEncodeFramerate = frameRate.value() * FRAME_RATE_COEFFICIENT;
714         if (!SetParameter(OMX_IndexConfigVideoFramerate, framerateCfgType, true)) {
715             HLOGW("failed to config OMX_IndexConfigVideoFramerate");
716         }
717     }
718 
719     int32_t requestIdr;
720     if (format.GetIntValue(MediaDescriptionKey::MD_KEY_REQUEST_I_FRAME, requestIdr) && requestIdr != 0) {
721         int32_t ret = RequestIDRFrame();
722         if (ret != AVCS_ERR_OK) {
723             return ret;
724         }
725     }
726 
727     int32_t ret = SetQpRange(format, true);
728     if (ret != AVCS_ERR_OK) {
729         return ret;
730     }
731     return AVCS_ERR_OK;
732 }
733 
SubmitOutputBuffersToOmxNode()734 int32_t HEncoder::SubmitOutputBuffersToOmxNode()
735 {
736     for (BufferInfo &info : outputBufferPool_) {
737         if (info.owner == BufferOwner::OWNED_BY_US) {
738             int32_t ret = NotifyOmxToFillThisOutBuffer(info);
739             if (ret != AVCS_ERR_OK) {
740                 return ret;
741             }
742         } else {
743             HLOGE("buffer should be owned by us");
744             return AVCS_ERR_UNKNOWN;
745         }
746     }
747     return AVCS_ERR_OK;
748 }
749 
ReadyToStart()750 bool HEncoder::ReadyToStart()
751 {
752     if (callback_ == nullptr || outputFormat_ == nullptr || inputFormat_ == nullptr) {
753         return false;
754     }
755     if (inputSurface_) {
756         HLOGI("surface mode");
757     } else {
758         HLOGI("buffer mode");
759     }
760     return true;
761 }
762 
AllocateBuffersOnPort(OMX_DIRTYPE portIndex)763 int32_t HEncoder::AllocateBuffersOnPort(OMX_DIRTYPE portIndex)
764 {
765     if (portIndex == OMX_DirOutput) {
766         return AllocateAvLinearBuffers(portIndex);
767     }
768     if (inputSurface_) {
769         return AllocInBufsForDynamicSurfaceBuf();
770     } else {
771         int32_t ret = AllocateAvSurfaceBuffers(portIndex);
772         if (ret == AVCS_ERR_OK) {
773             UpdateFormatFromSurfaceBuffer();
774         }
775         return ret;
776     }
777 }
778 
UpdateFormatFromSurfaceBuffer()779 void HEncoder::UpdateFormatFromSurfaceBuffer()
780 {
781     if (inputBufferPool_.empty()) {
782         return;
783     }
784     sptr<SurfaceBuffer> surfaceBuffer = inputBufferPool_.front().surfaceBuffer;
785     if (surfaceBuffer == nullptr) {
786         return;
787     }
788     int32_t stride = surfaceBuffer->GetStride();
789     HLOGI("input stride = %d", stride);
790     inputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_STRIDE, stride);
791 }
792 
ClearDirtyList()793 void HEncoder::ClearDirtyList()
794 {
795     sptr<SurfaceBuffer> buffer;
796     sptr<SyncFence> fence;
797     int64_t pts = -1;
798     OHOS::Rect damage;
799     while (true) {
800         GSError ret = inputSurface_->AcquireBuffer(buffer, fence, pts, damage);
801         if (ret != GSERROR_OK || buffer == nullptr) {
802             return;
803         }
804         HLOGI("return stale buffer to surface, seq = %u, pts = %" PRId64 "", buffer->GetSeqNum(), pts);
805         inputSurface_->ReleaseBuffer(buffer, -1);
806     }
807 }
808 
SubmitAllBuffersOwnedByUs()809 int32_t HEncoder::SubmitAllBuffersOwnedByUs()
810 {
811     HLOGI(">>");
812     if (isBufferCirculating_) {
813         HLOGI("buffer is already circulating, no need to do again");
814         return AVCS_ERR_OK;
815     }
816     int32_t ret = SubmitOutputBuffersToOmxNode();
817     if (ret != AVCS_ERR_OK) {
818         return ret;
819     }
820     if (inputSurface_) {
821         ClearDirtyList();
822         sptr<IBufferConsumerListener> listener = new EncoderBuffersConsumerListener(this);
823         inputSurface_->RegisterConsumerListener(listener);
824         SendAsyncMsg(MsgWhat::GET_BUFFER_FROM_SURFACE, nullptr);
825     } else {
826         for (BufferInfo &info : inputBufferPool_) {
827             if (info.owner == BufferOwner::OWNED_BY_US) {
828                 NotifyUserToFillThisInBuffer(info);
829             }
830         }
831     }
832 
833     isBufferCirculating_ = true;
834     return AVCS_ERR_OK;
835 }
836 
OnCreateInputSurface()837 sptr<Surface> HEncoder::OnCreateInputSurface()
838 {
839     if (inputSurface_) {
840         HLOGE("inputSurface_ already exists");
841         return nullptr;
842     }
843 
844     sptr<Surface> consumerSurface = Surface::CreateSurfaceAsConsumer("HEncoderSurface");
845     if (consumerSurface == nullptr) {
846         HLOGE("Create the surface consummer fail");
847         return nullptr;
848     }
849     GSError err = consumerSurface->SetDefaultUsage(SURFACE_MODE_CONSUMER_USAGE);
850     if (err == GSERROR_OK) {
851         HLOGI("set consumer usage 0x%x succ", SURFACE_MODE_CONSUMER_USAGE);
852     } else {
853         HLOGW("set consumer usage 0x%x failed", SURFACE_MODE_CONSUMER_USAGE);
854     }
855 
856     sptr<IBufferProducer> producer = consumerSurface->GetProducer();
857     if (producer == nullptr) {
858         HLOGE("Get the surface producer fail");
859         return nullptr;
860     }
861 
862     sptr<Surface> producerSurface = Surface::CreateSurfaceAsProducer(producer);
863     if (producerSurface == nullptr) {
864         HLOGE("CreateSurfaceAsProducer fail");
865         return nullptr;
866     }
867 
868     inputSurface_ = consumerSurface;
869     if (inBufferCnt_ > inputSurface_->GetQueueSize()) {
870         inputSurface_->SetQueueSize(inBufferCnt_);
871     }
872     HLOGI("queue size %u", inputSurface_->GetQueueSize());
873     return producerSurface;
874 }
875 
OnSetInputSurface(sptr<Surface> & inputSurface)876 int32_t HEncoder::OnSetInputSurface(sptr<Surface> &inputSurface)
877 {
878     if (inputSurface_) {
879         HLOGW("inputSurface_ already exists");
880     }
881 
882     if (inputSurface == nullptr) {
883         HLOGE("surface is null");
884         return AVCS_ERR_INVALID_VAL;
885     }
886     if (!inputSurface->IsConsumer()) {
887         HLOGE("expect consumer surface");
888         return AVCS_ERR_INVALID_VAL;
889     }
890 
891     inputSurface_ = inputSurface;
892     if (inBufferCnt_ > inputSurface_->GetQueueSize()) {
893         inputSurface_->SetQueueSize(inBufferCnt_);
894     }
895     HLOGI("succ");
896     return AVCS_ERR_OK;
897 }
898 
WrapPerFrameParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)899 void HEncoder::WrapPerFrameParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
900                                               const shared_ptr<Media::Meta> &meta)
901 {
902     omxBuffer->alongParam.clear();
903     WrapLTRParamIntoOmxBuffer(omxBuffer, meta);
904     WrapRequestIFrameParamIntoOmxBuffer(omxBuffer, meta);
905     WrapQPRangeParamIntoOmxBuffer(omxBuffer, meta);
906     WrapStartQPIntoOmxBuffer(omxBuffer, meta);
907     WrapIsSkipFrameIntoOmxBuffer(omxBuffer, meta);
908     meta->Clear();
909 }
910 
WrapLTRParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)911 void HEncoder::WrapLTRParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
912                                          const shared_ptr<Media::Meta> &meta)
913 {
914     if (!enableLTR_) {
915         return;
916     }
917     AppendToVector(omxBuffer->alongParam, OMX_IndexParamLTR);
918     CodecLTRPerFrameParam param;
919     bool markLTR = false;
920     meta->GetData(OHOS::Media::Tag::VIDEO_ENCODER_PER_FRAME_MARK_LTR, markLTR);
921     param.markAsLTR = markLTR;
922     int32_t useLtrPoc = 0;
923     param.useLTR = meta->GetData(OHOS::Media::Tag::VIDEO_ENCODER_PER_FRAME_USE_LTR, useLtrPoc);
924     param.useLTRPoc = static_cast<uint32_t>(useLtrPoc);
925     AppendToVector(omxBuffer->alongParam, param);
926 }
927 
WrapRequestIFrameParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)928 void HEncoder::WrapRequestIFrameParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
929                                                    const shared_ptr<Media::Meta> &meta)
930 {
931     bool requestIFrame = false;
932     meta->GetData(OHOS::Media::Tag::VIDEO_REQUEST_I_FRAME, requestIFrame);
933     if (!requestIFrame) {
934         return;
935     }
936     AppendToVector(omxBuffer->alongParam, OMX_IndexConfigVideoIntraVOPRefresh);
937     OMX_CONFIG_INTRAREFRESHVOPTYPE params;
938     InitOMXParam(params);
939     params.nPortIndex = OMX_DirOutput;
940     params.IntraRefreshVOP = OMX_TRUE;
941     AppendToVector(omxBuffer->alongParam, params);
942     HLOGI("pts=%" PRId64 ", requestIFrame", omxBuffer->pts);
943 }
944 
WrapQPRangeParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)945 void HEncoder::WrapQPRangeParamIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
946                                              const shared_ptr<Media::Meta> &meta)
947 {
948     int32_t minQp;
949     int32_t maxQp;
950     if (!meta->GetData(OHOS::Media::Tag::VIDEO_ENCODER_QP_MIN, minQp) ||
951         !meta->GetData(OHOS::Media::Tag::VIDEO_ENCODER_QP_MAX, maxQp)) {
952         return;
953     }
954     AppendToVector(omxBuffer->alongParam, OMX_IndexParamQPRange);
955     CodecQPRangeParam param;
956     InitOMXParamExt(param);
957     param.minQp = static_cast<uint32_t>(minQp);
958     param.maxQp = static_cast<uint32_t>(maxQp);
959     AppendToVector(omxBuffer->alongParam, param);
960     HLOGI("pts=%" PRId64 ", qp=(%d~%d)", omxBuffer->pts, minQp, maxQp);
961 }
962 
WrapStartQPIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)963 void HEncoder::WrapStartQPIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
964                                         const shared_ptr<Media::Meta> &meta)
965 {
966     int32_t startQp {};
967     if (!meta->GetData(OHOS::Media::Tag::VIDEO_ENCODER_QP_START, startQp)) {
968         return;
969     }
970     AppendToVector(omxBuffer->alongParam, OMX_IndexParamQPStsart);
971     AppendToVector(omxBuffer->alongParam, startQp);
972 }
973 
WrapIsSkipFrameIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,const shared_ptr<Media::Meta> & meta)974 void HEncoder::WrapIsSkipFrameIntoOmxBuffer(shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer,
975                                             const shared_ptr<Media::Meta> &meta)
976 {
977     bool isSkip {};
978     if (!meta->GetData(OHOS::Media::Tag::VIDEO_PER_FRAME_IS_SKIP, isSkip)) {
979         return;
980     }
981     AppendToVector(omxBuffer->alongParam, OMX_IndexParamSkipFrame);
982     AppendToVector(omxBuffer->alongParam, isSkip);
983 }
984 
DealWithResolutionChange(uint32_t newWidth,uint32_t newHeight)985 void HEncoder::DealWithResolutionChange(uint32_t newWidth, uint32_t newHeight)
986 {
987     if (width_ != newWidth || height_ != newHeight) {
988         HLOGI("resolution changed, %ux%u -> %ux%u", width_, height_, newWidth, newHeight);
989         width_ = newWidth;
990         height_ = newHeight;
991         outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width_);
992         outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height_);
993         outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_WIDTH, width_);
994         outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_HEIGHT, height_);
995         HLOGI("output format changed: %s", outputFormat_->Stringify().c_str());
996         callback_->OnOutputFormatChanged(*(outputFormat_.get()));
997     }
998 }
999 
ExtractPerFrameParamFromOmxBuffer(const shared_ptr<CodecHDI::OmxCodecBuffer> & omxBuffer,shared_ptr<Media::Meta> & meta)1000 void HEncoder::ExtractPerFrameParamFromOmxBuffer(
1001     const shared_ptr<CodecHDI::OmxCodecBuffer> &omxBuffer, shared_ptr<Media::Meta> &meta)
1002 {
1003     meta->Clear();
1004     BinaryReader reader(static_cast<uint8_t*>(omxBuffer->alongParam.data()), omxBuffer->alongParam.size());
1005     int* index = nullptr;
1006     while ((index = reader.Read<int>()) != nullptr) {
1007         switch (*index) {
1008             case OMX_IndexConfigCommonOutputSize: {
1009                 auto *param = reader.Read<OMX_FRAMESIZETYPE>();
1010                 IF_TRUE_RETURN_VOID(param == nullptr);
1011                 DealWithResolutionChange(param->nWidth, param->nHeight);
1012                 break;
1013             }
1014             case OMX_IndexParamEncOutQp: {
1015                 auto *averageQp = reader.Read<OMX_S32>();
1016                 IF_TRUE_RETURN_VOID(averageQp == nullptr);
1017                 HLOGD("pts=%" PRId64 ", averageQp=(%d)", omxBuffer->pts, *averageQp);
1018                 meta->SetData(OHOS::Media::Tag::VIDEO_ENCODER_QP_AVERAGE, *averageQp);
1019                 break;
1020             }
1021             case OMX_IndexParamEncOutMse: {
1022                 auto *averageMseLcu = reader.Read<double>();
1023                 IF_TRUE_RETURN_VOID(averageMseLcu == nullptr);
1024                 HLOGD("pts=%" PRId64 ", averageMseLcu=(%f)", omxBuffer->pts, *averageMseLcu);
1025                 meta->SetData(OHOS::Media::Tag::VIDEO_ENCODER_MSE, *averageMseLcu);
1026                 break;
1027             }
1028             case OMX_IndexParamEncOutLTR: {
1029                 auto *encOutLtrParam = reader.Read<CodecEncOutLTRParam>();
1030                 ExtractPerFrameLTRParam(encOutLtrParam, meta);
1031                 break;
1032             }
1033             case OMX_IndexParamEncOutFrameLayer: {
1034                 auto *frameLayer = reader.Read<OMX_S32>();
1035                 IF_TRUE_RETURN_VOID(frameLayer == nullptr);
1036                 HLOGD("pts=%" PRId64 ", frameLayer=(%d)", omxBuffer->pts, *frameLayer);
1037                 meta->SetData(OHOS::Media::Tag::VIDEO_ENCODER_FRAME_TEMPORAL_ID, *frameLayer);
1038                 break;
1039             }
1040             default: {
1041                 break;
1042             }
1043         }
1044     }
1045     omxBuffer->alongParam.clear();
1046 }
1047 
ExtractPerFrameLTRParam(const CodecEncOutLTRParam * ltrParam,shared_ptr<Media::Meta> & meta)1048 void HEncoder::ExtractPerFrameLTRParam(const CodecEncOutLTRParam* ltrParam, shared_ptr<Media::Meta> &meta)
1049 {
1050     if (ltrParam == nullptr) {
1051         return;
1052     }
1053     meta->SetData(OHOS::Media::Tag::VIDEO_PER_FRAME_IS_LTR, ltrParam->isLTR);
1054     meta->SetData(OHOS::Media::Tag::VIDEO_PER_FRAME_POC, static_cast<int32_t>(ltrParam->poc));
1055 }
1056 
AllocInBufsForDynamicSurfaceBuf()1057 int32_t HEncoder::AllocInBufsForDynamicSurfaceBuf()
1058 {
1059     inputBufferPool_.clear();
1060     for (uint32_t i = 0; i < inBufferCnt_; ++i) {
1061         shared_ptr<CodecHDI::OmxCodecBuffer> omxBuffer = DynamicSurfaceBufferToOmxBuffer();
1062         shared_ptr<CodecHDI::OmxCodecBuffer> outBuffer = make_shared<CodecHDI::OmxCodecBuffer>();
1063         int32_t ret = compNode_->UseBuffer(OMX_DirInput, *omxBuffer, *outBuffer);
1064         if (ret != HDF_SUCCESS) {
1065             HLOGE("Failed to UseBuffer on input port");
1066             return AVCS_ERR_UNKNOWN;
1067         }
1068         BufferInfo info {};
1069         info.isInput = true;
1070         info.owner = BufferOwner::OWNED_BY_SURFACE;
1071         info.surfaceBuffer = nullptr;
1072         info.avBuffer = AVBuffer::CreateAVBuffer();
1073         info.omxBuffer = outBuffer;
1074         info.bufferId = outBuffer->bufferId;
1075         inputBufferPool_.push_back(info);
1076     }
1077 
1078     return AVCS_ERR_OK;
1079 }
1080 
EraseBufferFromPool(OMX_DIRTYPE portIndex,size_t i)1081 void HEncoder::EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i)
1082 {
1083     vector<BufferInfo> &pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_;
1084     if (i >= pool.size()) {
1085         return;
1086     }
1087     const BufferInfo &info = pool[i];
1088     FreeOmxBuffer(portIndex, info);
1089     ReduceOwner((portIndex == OMX_DirInput), info.owner);
1090     pool.erase(pool.begin() + i);
1091 }
1092 
OnQueueInputBuffer(const MsgInfo & msg,BufferOperationMode mode)1093 void HEncoder::OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode)
1094 {
1095     if (inputSurface_ && !enableSurfaceModeInputCb_) {
1096         HLOGE("cannot queue input on surface mode");
1097         ReplyErrorCode(msg.id, AVCS_ERR_INVALID_OPERATION);
1098         return;
1099     }
1100     // buffer mode or surface callback mode
1101     uint32_t bufferId = 0;
1102     (void)msg.param->GetValue(BUFFER_ID, bufferId);
1103     SCOPED_TRACE_WITH_ID(bufferId);
1104     BufferInfo* bufferInfo = FindBufferInfoByID(OMX_DirInput, bufferId);
1105     if (bufferInfo == nullptr) {
1106         ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL);
1107         return;
1108     }
1109     if (bufferInfo->owner != BufferOwner::OWNED_BY_USER) {
1110         HLOGE("wrong ownership: buffer id=%d, owner=%s", bufferId, ToString(bufferInfo->owner));
1111         ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL);
1112         return;
1113     }
1114 
1115     bool discard = false;
1116     if (inputSurface_ && bufferInfo->avBuffer->meta_->GetData(
1117         OHOS::Media::Tag::VIDEO_ENCODER_PER_FRAME_DISCARD, discard) && discard) {
1118         HLOGI("inBufId = %u, discard by user, pts = %" PRId64, bufferId, bufferInfo->avBuffer->pts_);
1119         bufferInfo->avBuffer->meta_->Clear();
1120         ResetSlot(*bufferInfo);
1121         ReplyErrorCode(msg.id, AVCS_ERR_OK);
1122         return;
1123     }
1124     ChangeOwner(*bufferInfo, BufferOwner::OWNED_BY_US);
1125     WrapSurfaceBufferToSlot(*bufferInfo, bufferInfo->surfaceBuffer, bufferInfo->avBuffer->pts_,
1126         UserFlagToOmxFlag(static_cast<AVCodecBufferFlag>(bufferInfo->avBuffer->flag_)));
1127     WrapPerFrameParamIntoOmxBuffer(bufferInfo->omxBuffer, bufferInfo->avBuffer->meta_);
1128     ReplyErrorCode(msg.id, AVCS_ERR_OK);
1129     int32_t err = HCodec::OnQueueInputBuffer(mode, bufferInfo);
1130     if (err != AVCS_ERR_OK) {
1131         ResetSlot(*bufferInfo);
1132         callback_->OnError(AVCODEC_ERROR_INTERNAL, AVCS_ERR_INPUT_DATA_ERROR);
1133     }
1134 }
1135 
OnGetBufferFromSurface(const ParamSP & param)1136 void HEncoder::OnGetBufferFromSurface(const ParamSP& param)
1137 {
1138     if (GetOneBufferFromSurface()) {
1139         TraverseAvaliableBuffers();
1140     }
1141 }
1142 
GetOneBufferFromSurface()1143 bool HEncoder::GetOneBufferFromSurface()
1144 {
1145     SCOPED_TRACE();
1146     InSurfaceBufferEntry entry{};
1147     entry.item = make_shared<BufferItem>();
1148     GSError ret = inputSurface_->AcquireBuffer(
1149         entry.item->buffer, entry.item->fence, entry.pts, entry.item->damage);
1150     if (ret != GSERROR_OK || entry.item->buffer == nullptr) {
1151         return false;
1152     }
1153     if (!CheckBufPixFmt(entry.item->buffer)) {
1154         return false;
1155     }
1156     entry.item->generation = ++currGeneration_;
1157     entry.item->surface = inputSurface_;
1158     avaliableBuffers_.push_back(entry);
1159     newestBuffer_ = entry;
1160     HLOGD("generation = %" PRIu64 ", seq = %u, pts = %" PRId64 ", now list size = %zu",
1161           entry.item->generation, entry.item->buffer->GetSeqNum(), entry.pts, avaliableBuffers_.size());
1162     if (repeatUs_ != 0) {
1163         SendRepeatMsg(entry.item->generation);
1164     }
1165     return true;
1166 }
1167 
SendRepeatMsg(uint64_t generation)1168 void HEncoder::SendRepeatMsg(uint64_t generation)
1169 {
1170     ParamSP param = make_shared<ParamBundle>();
1171     param->SetValue("generation", generation);
1172     SendAsyncMsg(MsgWhat::CHECK_IF_REPEAT, param, repeatUs_);
1173 }
1174 
RepeatIfNecessary(const ParamSP & param)1175 void HEncoder::RepeatIfNecessary(const ParamSP& param)
1176 {
1177     uint64_t generation = 0;
1178     param->GetValue("generation", generation);
1179     if (inputPortEos_ || (repeatUs_ == 0) || newestBuffer_.item == nullptr ||
1180         newestBuffer_.item->generation != generation) {
1181         return;
1182     }
1183     if (repeatMaxCnt_ > 0 && newestBuffer_.repeatTimes >= repeatMaxCnt_) {
1184         HLOGD("stop repeat generation = %" PRIu64 ", seq = %u, pts = %" PRId64 ", which has been repeated %d times",
1185               generation, newestBuffer_.item->buffer->GetSeqNum(), newestBuffer_.pts, newestBuffer_.repeatTimes);
1186         return;
1187     }
1188     if (avaliableBuffers_.size() >= MAX_LIST_SIZE) {
1189         HLOGW("stop repeat, list size to big: %zu", avaliableBuffers_.size());
1190         return;
1191     }
1192     int64_t newPts = newestBuffer_.pts + static_cast<int64_t>(repeatUs_);
1193     HLOGD("generation = %" PRIu64 ", seq = %u, pts %" PRId64 " -> %" PRId64,
1194           generation, newestBuffer_.item->buffer->GetSeqNum(), newestBuffer_.pts, newPts);
1195     newestBuffer_.pts = newPts;
1196     newestBuffer_.repeatTimes++;
1197     avaliableBuffers_.push_back(newestBuffer_);
1198     SendRepeatMsg(generation);
1199     TraverseAvaliableBuffers();
1200 }
1201 
TraverseAvaliableBuffers()1202 void HEncoder::TraverseAvaliableBuffers()
1203 {
1204     while (!avaliableBuffers_.empty()) {
1205         auto it = find_if(inputBufferPool_.begin(), inputBufferPool_.end(),
1206                           [](const BufferInfo &info) { return info.owner == BufferOwner::OWNED_BY_SURFACE; });
1207         if (it == inputBufferPool_.end()) {
1208             HLOGD("buffer cnt = %zu, but no avaliable slot", avaliableBuffers_.size());
1209             return;
1210         }
1211         InSurfaceBufferEntry entry = avaliableBuffers_.front();
1212         avaliableBuffers_.pop_front();
1213         SubmitOneBuffer(entry, *it);
1214     }
1215 }
1216 
TraverseAvaliableSlots()1217 void HEncoder::TraverseAvaliableSlots()
1218 {
1219     for (BufferInfo& info : inputBufferPool_) {
1220         if (info.owner != BufferOwner::OWNED_BY_SURFACE) {
1221             continue;
1222         }
1223         if (avaliableBuffers_.empty() && !GetOneBufferFromSurface()) {
1224             HLOGD("slot %u is avaliable, but no buffer", info.bufferId);
1225             return;
1226         }
1227         InSurfaceBufferEntry entry = avaliableBuffers_.front();
1228         avaliableBuffers_.pop_front();
1229         SubmitOneBuffer(entry, info);
1230     }
1231 }
1232 
SubmitOneBuffer(InSurfaceBufferEntry & entry,BufferInfo & info)1233 void HEncoder::SubmitOneBuffer(InSurfaceBufferEntry& entry, BufferInfo &info)
1234 {
1235     if (entry.item == nullptr) {
1236         ChangeOwner(info, BufferOwner::OWNED_BY_US);
1237         HLOGI("got input eos");
1238         inputPortEos_ = true;
1239         info.omxBuffer->flag = OMX_BUFFERFLAG_EOS;
1240         info.omxBuffer->bufferhandle = nullptr;
1241         info.omxBuffer->filledLen = 0;
1242         info.surfaceBuffer = nullptr;
1243         NotifyOmxToEmptyThisInBuffer(info);
1244         return;
1245     }
1246     if (!WaitFence(entry.item->fence)) {
1247         return;
1248     }
1249     ChangeOwner(info, BufferOwner::OWNED_BY_US);
1250     WrapSurfaceBufferToSlot(info, entry.item->buffer, entry.pts, 0);
1251     encodingBuffers_[info.bufferId] = entry;
1252     if (enableSurfaceModeInputCb_) {
1253         info.avBuffer->pts_ = entry.pts;
1254         NotifyUserToFillThisInBuffer(info);
1255     } else {
1256         int32_t err = NotifyOmxToEmptyThisInBuffer(info);
1257         if (err != AVCS_ERR_OK) {
1258             ResetSlot(info);
1259             callback_->OnError(AVCODEC_ERROR_INTERNAL, AVCS_ERR_INPUT_DATA_ERROR);
1260         }
1261     }
1262 }
1263 
OnOMXEmptyBufferDone(uint32_t bufferId,BufferOperationMode mode)1264 void HEncoder::OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode)
1265 {
1266     SCOPED_TRACE_WITH_ID(bufferId);
1267     BufferInfo *info = FindBufferInfoByID(OMX_DirInput, bufferId);
1268     if (info == nullptr) {
1269         HLOGE("unknown buffer id %u", bufferId);
1270         return;
1271     }
1272     if (info->owner != BufferOwner::OWNED_BY_OMX) {
1273         HLOGE("wrong ownership: buffer id=%d, owner=%s", bufferId, ToString(info->owner));
1274         return;
1275     }
1276     if (inputSurface_) {
1277         ResetSlot(*info);
1278         if (mode == RESUBMIT_BUFFER && !inputPortEos_) {
1279             TraverseAvaliableSlots();
1280         }
1281     } else {
1282         ChangeOwner(*info, BufferOwner::OWNED_BY_US);
1283         if (mode == RESUBMIT_BUFFER && !inputPortEos_) {
1284             NotifyUserToFillThisInBuffer(*info);
1285         }
1286     }
1287 }
1288 
ResetSlot(BufferInfo & info)1289 void HEncoder::ResetSlot(BufferInfo& info)
1290 {
1291     ChangeOwner(info, BufferOwner::OWNED_BY_SURFACE);
1292     encodingBuffers_.erase(info.bufferId);
1293     info.surfaceBuffer = nullptr;
1294 }
1295 
OnBufferAvailable()1296 void HEncoder::EncoderBuffersConsumerListener::OnBufferAvailable()
1297 {
1298     codec_->SendAsyncMsg(MsgWhat::GET_BUFFER_FROM_SURFACE, nullptr);
1299 }
1300 
OnSignalEndOfInputStream(const MsgInfo & msg)1301 void HEncoder::OnSignalEndOfInputStream(const MsgInfo &msg)
1302 {
1303     if (inputSurface_ == nullptr) {
1304         HLOGE("can only be called in surface mode");
1305         ReplyErrorCode(msg.id, AVCS_ERR_INVALID_OPERATION);
1306         return;
1307     }
1308     ReplyErrorCode(msg.id, AVCS_ERR_OK);
1309     avaliableBuffers_.push_back(InSurfaceBufferEntry {});
1310     TraverseAvaliableBuffers();
1311 }
1312 
OnEnterUninitializedState()1313 void HEncoder::OnEnterUninitializedState()
1314 {
1315     if (inputSurface_) {
1316         inputSurface_->UnregisterConsumerListener();
1317     }
1318     avaliableBuffers_.clear();
1319     newestBuffer_.item.reset();
1320     encodingBuffers_.clear();
1321 }
1322 
~BufferItem()1323 HEncoder::BufferItem::~BufferItem()
1324 {
1325     if (surface && buffer) {
1326         LOGD("release seq = %u", buffer->GetSeqNum());
1327         surface->ReleaseBuffer(buffer, -1);
1328     }
1329 }
1330 
1331 } // namespace OHOS::MediaAVCodec
1332