1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "hardware/imagecodec/image_decoder.h"
17 #include "hardware/imagecodec/image_codec_log.h"
18 #include <cassert>
19 #include "codec_omx_ext.h" // drivers/peripheral/codec/interfaces/include/codec_omx_ext.h
20 #include "OMX_VideoExt.h" // third_party/openmax/api/1.1.2/OMX_VideoExt.h
21 #include "surface_buffer.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_buffer.h
22 
23 namespace OHOS::ImagePlugin {
24 using namespace std;
25 
ImageDecoder()26 ImageDecoder::ImageDecoder()
27     : ImageCodec(static_cast<OMX_VIDEO_CODINGTYPE>(CODEC_OMX_VIDEO_CodingHEVC), false)
28 {}
29 
OnConfigure(const Format & format)30 int32_t ImageDecoder::OnConfigure(const Format &format)
31 {
32     configFormat_ = make_shared<Format>(format);
33 
34     (void)format.GetValue(ImageCodecDescriptionKey::ENABLE_HEIF_GRID, enableHeifGrid_);
35 
36     UseBufferType useBufferTypes;
37     InitOMXParamExt(useBufferTypes);
38     useBufferTypes.portIndex = OMX_DirOutput;
39     useBufferTypes.bufferType = CODEC_BUFFER_TYPE_HANDLE;
40     if (!SetParameter(OMX_IndexParamUseBufferType, useBufferTypes)) {
41         HLOGE("component don't support CODEC_BUFFER_TYPE_HANDLE");
42         return IC_ERR_INVALID_VAL;
43     }
44 
45     (void)SetProcessName(format);
46     (void)SetFrameRateAdaptiveMode(format);
47     return SetupPort(format);
48 }
49 
SetupPort(const Format & format)50 int32_t ImageDecoder::SetupPort(const Format &format)
51 {
52     uint32_t width;
53     if (!format.GetValue(ImageCodecDescriptionKey::WIDTH, width) || width <= 0) {
54         HLOGE("format should contain width");
55         return IC_ERR_INVALID_VAL;
56     }
57     uint32_t height;
58     if (!format.GetValue(ImageCodecDescriptionKey::HEIGHT, height) || height <= 0) {
59         HLOGE("format should contain height");
60         return IC_ERR_INVALID_VAL;
61     }
62     HLOGI("user set width %{public}u, height %{public}u", width, height);
63     if (!GetPixelFmtFromUser(format)) {
64         return IC_ERR_INVALID_VAL;
65     }
66     uint32_t inputBufferCnt = 0;
67     (void)format.GetValue(ImageCodecDescriptionKey::INPUT_BUFFER_COUNT, inputBufferCnt);
68     uint32_t outputBufferCnt = 0;
69     (void)format.GetValue(ImageCodecDescriptionKey::OUTPUT_BUFFER_COUNT, outputBufferCnt);
70 
71     optional<double> frameRate = GetFrameRateFromUser(format);
72     if (!frameRate.has_value()) {
73         HLOGI("user don't set valid frame rate, use default 30.0");
74         frameRate = 30.0;  // default frame rate 30.0
75     }
76 
77     PortInfo inputPortInfo {width, height, codingType_, std::nullopt, frameRate.value()};
78     int32_t maxInputSize = 0;
79     (void)format.GetValue(ImageCodecDescriptionKey::MAX_INPUT_SIZE, maxInputSize);
80     if (maxInputSize > 0) {
81         inputPortInfo.inputBufSize = static_cast<uint32_t>(maxInputSize);
82     }
83     if (inputBufferCnt > 0) {
84         inputPortInfo.bufferCnt = inputBufferCnt;
85     }
86     int32_t ret = SetVideoPortInfo(OMX_DirInput, inputPortInfo);
87     if (ret != IC_ERR_OK) {
88         return ret;
89     }
90 
91     PortInfo outputPortInfo = {width, height, OMX_VIDEO_CodingUnused, configuredFmt_, frameRate.value()};
92     if (outputBufferCnt > 0) {
93         outputPortInfo.bufferCnt = outputBufferCnt;
94     }
95     ret = SetVideoPortInfo(OMX_DirOutput, outputPortInfo);
96     if (ret != IC_ERR_OK) {
97         return ret;
98     }
99 
100     return IC_ERR_OK;
101 }
102 
UpdateConfiguredFmt(OMX_COLOR_FORMATTYPE portFmt)103 bool ImageDecoder::UpdateConfiguredFmt(OMX_COLOR_FORMATTYPE portFmt)
104 {
105     auto graphicFmt = static_cast<GraphicPixelFormat>(portFmt);
106     if (graphicFmt != configuredFmt_.graphicFmt) {
107         optional<PixelFmt> fmt = TypeConverter::GraphicFmtToFmt(graphicFmt);
108         if (!fmt.has_value()) {
109             return false;
110         }
111         HLOGI("GraphicPixelFormat need update: configured(%{public}s) -> portdefinition(%{public}s)",
112             configuredFmt_.strFmt.c_str(), fmt->strFmt.c_str());
113         configuredFmt_ = fmt.value();
114     }
115     return true;
116 }
117 
UpdateInPortFormat()118 int32_t ImageDecoder::UpdateInPortFormat()
119 {
120     OMX_PARAM_PORTDEFINITIONTYPE def;
121     InitOMXParam(def);
122     def.nPortIndex = OMX_DirInput;
123     if (!GetParameter(OMX_IndexParamPortDefinition, def)) {
124         HLOGE("get input port definition failed");
125         return IC_ERR_UNKNOWN;
126     }
127     PrintPortDefinition(def);
128     if (inputFormat_ == nullptr) {
129         inputFormat_ = make_shared<Format>();
130     }
131     inputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, def.format.video.nFrameWidth);
132     inputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, def.format.video.nFrameHeight);
133     return IC_ERR_OK;
134 }
135 
UpdateOutPortFormat()136 int32_t ImageDecoder::UpdateOutPortFormat()
137 {
138     OMX_PARAM_PORTDEFINITIONTYPE def;
139     InitOMXParam(def);
140     def.nPortIndex = OMX_DirOutput;
141     if (!GetParameter(OMX_IndexParamPortDefinition, def)) {
142         HLOGE("get output port definition failed");
143         return IC_ERR_UNKNOWN;
144     }
145     PrintPortDefinition(def);
146     if (def.nBufferCountActual == 0) {
147         HLOGE("invalid bufferCount");
148         return IC_ERR_UNKNOWN;
149     }
150     (void)UpdateConfiguredFmt(def.format.video.eColorFormat);
151 
152     uint32_t w = def.format.video.nFrameWidth;
153     uint32_t h = def.format.video.nFrameHeight;
154 
155     // save into member variable
156     requestCfg_.timeout = 0;
157     requestCfg_.width = static_cast<int32_t>(w);
158     requestCfg_.height = static_cast<int32_t>(h);
159     requestCfg_.strideAlignment = STRIDE_ALIGNMENT;
160     requestCfg_.format = configuredFmt_.graphicFmt;
161     requestCfg_.usage = GetProducerUsage();
162     UpdateDisplaySizeByCrop();
163 
164     // save into format
165     if (outputFormat_ == nullptr) {
166         outputFormat_ = make_shared<Format>();
167     }
168     if (!outputFormat_->ContainKey(ImageCodecDescriptionKey::WIDTH)) {
169         outputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, w);
170     }
171     if (!outputFormat_->ContainKey(ImageCodecDescriptionKey::HEIGHT)) {
172         outputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, h);
173     }
174     outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_WIDTH, requestCfg_.width);
175     outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_HEIGHT, requestCfg_.height);
176     outputFormat_->SetValue(ImageCodecDescriptionKey::PIXEL_FORMAT,
177         static_cast<int32_t>(configuredFmt_.graphicFmt));
178     return IC_ERR_OK;
179 }
180 
UpdateColorAspects()181 void ImageDecoder::UpdateColorAspects()
182 {
183     CodecVideoColorspace param;
184     InitOMXParamExt(param);
185     param.portIndex = OMX_DirOutput;
186     if (!GetParameter(OMX_IndexColorAspects, param, true)) {
187         return;
188     }
189     HLOGI("range:%{public}d, primary:%{public}d, transfer:%{public}d, matrix:%{public}d)",
190         param.aspects.range, param.aspects.primaries, param.aspects.transfer, param.aspects.matrixCoeffs);
191     if (outputFormat_) {
192         outputFormat_->SetValue(ImageCodecDescriptionKey::RANGE_FLAG, param.aspects.range);
193         outputFormat_->SetValue(ImageCodecDescriptionKey::COLOR_PRIMARIES, param.aspects.primaries);
194         outputFormat_->SetValue(ImageCodecDescriptionKey::TRANSFER_CHARACTERISTICS, param.aspects.transfer);
195         outputFormat_->SetValue(ImageCodecDescriptionKey::MATRIX_COEFFICIENTS, param.aspects.matrixCoeffs);
196         callback_->OnOutputFormatChanged(*(outputFormat_.get()));
197     }
198 }
199 
UpdateDisplaySizeByCrop()200 void ImageDecoder::UpdateDisplaySizeByCrop()
201 {
202     OMX_CONFIG_RECTTYPE rect;
203     InitOMXParam(rect);
204     rect.nPortIndex = OMX_DirOutput;
205     if (!GetParameter(OMX_IndexConfigCommonOutputCrop, rect, true)) {
206         HLOGW("get crop failed, use default");
207         return;
208     }
209     if (rect.nLeft < 0 || rect.nTop < 0 || rect.nWidth == 0 || rect.nHeight == 0 ||
210         static_cast<int32_t>(rect.nLeft) + static_cast<int32_t>(rect.nWidth) > requestCfg_.width ||
211         static_cast<int32_t>(rect.nTop) + static_cast<int32_t>(rect.nHeight) > requestCfg_.height) {
212         HLOGW("wrong crop rect (%{public}d, %{public}d, %{public}u, %{public}u), use default",
213               rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight);
214         return;
215     }
216     HLOGI("crop rect (%{public}d, %{public}d, %{public}u, %{public}u)",
217           rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight);
218     requestCfg_.width = static_cast<int32_t>(rect.nWidth);
219     requestCfg_.height = static_cast<int32_t>(rect.nHeight);
220 }
221 
ReConfigureOutputBufferCnt()222 int32_t ImageDecoder::ReConfigureOutputBufferCnt()
223 {
224     IF_TRUE_RETURN_VAL(enableHeifGrid_, IC_ERR_OK);
225     OMX_PARAM_PORTDEFINITIONTYPE def;
226     InitOMXParam(def);
227     def.nPortIndex = OMX_DirOutput;
228     IF_TRUE_RETURN_VAL_WITH_MSG(!GetParameter(OMX_IndexParamPortDefinition, def), IC_ERR_UNKNOWN,
229                                 "failed to get output port def");
230     uint32_t outputBufferCnt = 0;
231     (void)configFormat_->GetValue(ImageCodecDescriptionKey::OUTPUT_BUFFER_COUNT, outputBufferCnt);
232     IF_TRUE_RETURN_VAL_WITH_MSG(outputBufferCnt == 0, IC_ERR_UNKNOWN, "failed to get output buffer cnt");
233     def.nBufferCountActual = outputBufferCnt;
234     IF_TRUE_RETURN_VAL_WITH_MSG(!SetParameter(OMX_IndexParamPortDefinition, def), IC_ERR_UNKNOWN,
235                                 "failed to set output port def");
236     return IC_ERR_OK;
237 }
238 
GetProducerUsage()239 uint64_t ImageDecoder::GetProducerUsage()
240 {
241     uint64_t producerUsage = BUFFER_MODE_REQUEST_USAGE;
242 
243     GetBufferHandleUsageParams vendorUsage;
244     InitOMXParamExt(vendorUsage);
245     vendorUsage.portIndex = static_cast<uint32_t>(OMX_DirOutput);
246     if (GetParameter(OMX_IndexParamGetBufferHandleUsage, vendorUsage)) {
247         HLOGI("vendor producer usage = 0x%" PRIx64 "", vendorUsage.usage);
248         producerUsage |= vendorUsage.usage;
249     }
250     HLOGI("decoder producer usage = 0x%" PRIx64 "", producerUsage);
251     return producerUsage;
252 }
253 
OnGetOutputBufferUsage()254 uint64_t ImageDecoder::OnGetOutputBufferUsage()
255 {
256     uint64_t usage = GetProducerUsage();
257     usage |= BUFFER_USAGE_CPU_WRITE;
258     return usage;
259 }
260 
OnSetOutputBuffer(sptr<SurfaceBuffer> output)261 int32_t ImageDecoder::OnSetOutputBuffer(sptr<SurfaceBuffer> output)
262 {
263     if (output == nullptr) {
264         HLOGE("invalid output buffer");
265         return IC_ERR_INVALID_VAL;
266     }
267     outputBuffer_ = output;
268     return IC_ERR_OK;
269 }
270 
OnGetPackedInputFlag()271 bool ImageDecoder::OnGetPackedInputFlag()
272 {
273     OMX_CONFIG_BOOLEANTYPE packedInputFlag;
274     InitOMXParam(packedInputFlag);
275     if (GetParameter(OMX_IndexParamSupportPackInput, packedInputFlag)) {
276         isPackedInputSupported_ = static_cast<bool>(packedInputFlag.bEnabled);
277     } else {
278         HLOGW("get packed input flag failed, use false as default");
279         isPackedInputSupported_ = false;
280     }
281     return isPackedInputSupported_;
282 }
283 
ReadyToStart()284 bool ImageDecoder::ReadyToStart()
285 {
286     if (callback_ == nullptr || outputFormat_ == nullptr || inputFormat_ == nullptr) {
287         HLOGE("callback not set or format is not configured, can't start");
288         return false;
289     }
290     if (enableHeifGrid_ && outputBuffer_ != nullptr) {
291         HLOGE("can not set output buffer when heif grid is enabled");
292         return false;
293     }
294     if (!enableHeifGrid_ && outputBuffer_ == nullptr) {
295         HLOGE("must set output buffer when heif grid is not enabled");
296         return false;
297     }
298     return true;
299 }
300 
AllocateBuffersOnPort(OMX_DIRTYPE portIndex,bool isOutputPortSettingChanged)301 int32_t ImageDecoder::AllocateBuffersOnPort(OMX_DIRTYPE portIndex, bool isOutputPortSettingChanged)
302 {
303     if (portIndex == OMX_DirInput) {
304         return AllocateHardwareBuffers(portIndex);
305     }
306     int32_t ret = AllocateSurfaceBuffers(portIndex, isOutputPortSettingChanged, outputBuffer_);
307     if (ret == IC_ERR_OK) {
308         UpdateFormatFromSurfaceBuffer();
309     }
310     return ret;
311 }
312 
UpdateFormatFromSurfaceBuffer()313 void ImageDecoder::UpdateFormatFromSurfaceBuffer()
314 {
315     if (outputBufferPool_.empty()) {
316         return;
317     }
318     sptr<SurfaceBuffer> surfaceBuffer = outputBufferPool_.front().surfaceBuffer;
319     if (surfaceBuffer == nullptr) {
320         return;
321     }
322     outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_WIDTH, surfaceBuffer->GetWidth());
323     outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_HEIGHT, surfaceBuffer->GetHeight());
324     outputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, surfaceBuffer->GetStride());
325 
326     OMX_PARAM_PORTDEFINITIONTYPE def;
327     int32_t ret = GetPortDefinition(OMX_DirOutput, def);
328     int32_t sliceHeight = static_cast<int32_t>(def.format.video.nSliceHeight);
329     if (ret == IC_ERR_OK && sliceHeight >= surfaceBuffer->GetHeight()) {
330         outputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, sliceHeight);
331     }
332 }
333 
SubmitAllBuffersOwnedByUs()334 int32_t ImageDecoder::SubmitAllBuffersOwnedByUs()
335 {
336     HLOGI(">>");
337     if (isBufferCirculating_) {
338         HLOGI("buffer is already circulating, no need to do again");
339         return IC_ERR_OK;
340     }
341     int32_t ret = SubmitOutputBuffersToOmxNode();
342     if (ret != IC_ERR_OK) {
343         return ret;
344     }
345     for (BufferInfo& info : inputBufferPool_) {
346         if (info.owner == BufferOwner::OWNED_BY_US) {
347             NotifyUserToFillThisInBuffer(info);
348         }
349     }
350     isBufferCirculating_ = true;
351     return IC_ERR_OK;
352 }
353 
SubmitOutputBuffersToOmxNode()354 int32_t ImageDecoder::SubmitOutputBuffersToOmxNode()
355 {
356     for (BufferInfo& info : outputBufferPool_) {
357         switch (info.owner) {
358             case BufferOwner::OWNED_BY_US: {
359                 int32_t ret = NotifyOmxToFillThisOutBuffer(info);
360                 if (ret != IC_ERR_OK) {
361                     return ret;
362                 }
363                 continue;
364             }
365             case BufferOwner::OWNED_BY_OMX: {
366                 continue;
367             }
368             default: {
369                 HLOGE("buffer id %{public}u has invalid owner %{public}d", info.bufferId, info.owner);
370                 return IC_ERR_UNKNOWN;
371             }
372         }
373     }
374     return IC_ERR_OK;
375 }
376 
OnOMXEmptyBufferDone(uint32_t bufferId,BufferOperationMode mode)377 void ImageDecoder::OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode)
378 {
379     BufferInfo *info = FindBufferInfoByID(OMX_DirInput, bufferId);
380     if (info == nullptr) {
381         HLOGE("unknown buffer id %{public}u", bufferId);
382         return;
383     }
384     if (info->owner != BufferOwner::OWNED_BY_OMX) {
385         HLOGE("wrong ownership: buffer id=%{public}d, owner=%{public}s", bufferId, ToString(info->owner));
386         return;
387     }
388     ChangeOwner(*info, BufferOwner::OWNED_BY_US);
389 
390     switch (mode) {
391         case KEEP_BUFFER:
392             return;
393         case RESUBMIT_BUFFER: {
394             if (!inputPortEos_) {
395                 NotifyUserToFillThisInBuffer(*info);
396             }
397             return;
398         }
399         default: {
400             HLOGE("SHOULD NEVER BE HERE");
401             return;
402         }
403     }
404 }
405 
EraseBufferFromPool(OMX_DIRTYPE portIndex,size_t i)406 void ImageDecoder::EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i)
407 {
408     vector<BufferInfo>& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_;
409     if (i >= pool.size()) {
410         HLOGE("EraseBufferFromPool invalid params i: %{public}lu, pool size: %{public}lu", i, pool.size());
411         return;
412     }
413     BufferInfo& info = pool[i];
414     FreeOmxBuffer(portIndex, info);
415     pool.erase(pool.begin() + i);
416 }
417 
418 
419 } // namespace OHOS::ImagePlugin