1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef IMAGE_CODEC_H
17 #define IMAGE_CODEC_H
18 
19 #include <queue>
20 #include <array>
21 #include <functional>
22 #include "securec.h"
23 #include "OMX_Component.h"  // third_party/openmax/api/1.1.2
24 #include "param_bundle.h"
25 #include "image_codec_buffer.h"
26 #include "image_codec_common.h"
27 #include "format.h"
28 #include "state_machine.h"
29 #include "type_converter.h"
30 #include "codec_omx_ext.h"
31 #include "hdi_define.h"
32 
33 namespace OHOS::ImagePlugin {
34 inline constexpr int TIME_RATIO_S_TO_MS = 1000;
35 inline constexpr double US_TO_MS = 1000.0;
36 inline constexpr double US_TO_S = 1000000.0;
37 inline constexpr uint32_t STRIDE_ALIGNMENT = 32;
38 // for demo: "/data/misc/imagecodecdump"
39 inline constexpr char DUMP_PATH[] = "/data/storage/el2/base/files";
40 
GetYuv420Size(uint32_t w,uint32_t h)41 inline uint32_t GetYuv420Size(uint32_t w, uint32_t h)
42 {
43     return w * h * 3 / 2;  // 3: nom of ratio, 2: denom of ratio
44 }
45 
46 class ImageCodec : protected StateMachine {
47 public:
48     static std::shared_ptr<ImageCodec> Create();
GetComponentName()49     std::string GetComponentName() const { return componentName_; }
50     int32_t SetCallback(const std::shared_ptr<ImageCodecCallback> &callback);
51     int32_t Configure(const Format &format);
52     int32_t QueueInputBuffer(uint32_t index);
53     int32_t ReleaseOutputBuffer(uint32_t index);
54     int32_t GetInputFormat(Format& format);
55     int32_t GetOutputFormat(Format& format);
56     int32_t Start();
57     int32_t Release();
58     int32_t GetOutputBufferUsage(uint64_t& usage);
59     int32_t SetOutputBuffer(sptr<SurfaceBuffer> output);
60     int32_t GetPackedInputFlag(bool& flag);
61 protected:
62     enum MsgWhat : MsgType {
63         INIT,
64         SET_CALLBACK,
65         CONFIGURE,
66         START,
67         GET_INPUT_FORMAT,
68         GET_OUTPUT_FORMAT,
69         QUEUE_INPUT_BUFFER,
70         RELEASE_OUTPUT_BUFFER,
71         RELEASE,
72         GET_OUTPUT_BUFFER_USAGE,
73         SET_OUTPUT_BUFFER,
74         GET_PACKED_INPUT_FLAG,
75 
76         INNER_MSG_BEGIN = 1000,
77         CODEC_EVENT,
78         OMX_EMPTY_BUFFER_DONE,
79         OMX_FILL_BUFFER_DONE,
80         CHECK_IF_STUCK,
81         FORCE_SHUTDOWN,
82     };
83 
84     enum BufferOperationMode {
85         KEEP_BUFFER,
86         RESUBMIT_BUFFER,
87         FREE_BUFFER,
88     };
89 
90     enum BufferOwner {
91         OWNED_BY_US = 0,
92         OWNED_BY_USER = 1,
93         OWNED_BY_OMX = 2,
94         OWNER_CNT = 3,
95     };
96 
97     struct PortInfo {
98         uint32_t width;
99         uint32_t height;
100         OMX_VIDEO_CODINGTYPE codingType;
101         std::optional<PixelFmt> pixelFmt;
102         double frameRate;
103         std::optional<uint32_t> inputBufSize;
104         std::optional<uint32_t> bufferCnt;
105     };
106 
107     struct BufferInfo {
BufferInfoBufferInfo108         BufferInfo() : lastOwnerChangeTime(std::chrono::steady_clock::now()) {}
109         bool isInput = true;
110         BufferOwner owner = OWNED_BY_US;
111         std::chrono::time_point<std::chrono::steady_clock> lastOwnerChangeTime;
112         uint32_t bufferId = 0;
113         std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> omxBuffer;
114         sptr<SurfaceBuffer> surfaceBuffer;
115         std::shared_ptr<ImageCodecBuffer> imgCodecBuffer;
116 
117         void CleanUpUnusedInfo();
118         void BeginCpuAccess();
119         void EndCpuAccess();
120         bool IsValidFrame() const;
121         void Dump(const std::string& prefix, bool dumpMode) const;
122 
123     private:
124         void Dump(const std::string& prefix) const;
125         void DumpSurfaceBuffer(const std::string& prefix) const;
126         void DumpLinearBuffer(const std::string& prefix) const;
127     };
128 protected:
129     ImageCodec(OMX_VIDEO_CODINGTYPE codingType, bool isEncoder);
130     ~ImageCodec() override;
131     static const char* ToString(MsgWhat what);
132     static const char* ToString(BufferOwner owner);
133     void ReplyErrorCode(MsgId id, int32_t err);
134     void PrintAllBufferInfo();
135     std::array<uint32_t, OWNER_CNT> CountOwner(bool isInput);
136     void ChangeOwner(BufferInfo& info, BufferOwner newOwner);
137     void UpdateInputRecord(const BufferInfo& info, std::chrono::time_point<std::chrono::steady_clock> now);
138     void UpdateOutputRecord(const BufferInfo& info, std::chrono::time_point<std::chrono::steady_clock> now);
139 
140     // configure
141     virtual int32_t OnConfigure(const Format &format) = 0;
142     bool GetPixelFmtFromUser(const Format &format);
143     static std::optional<double> GetFrameRateFromUser(const Format &format);
144     int32_t SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info);
145     virtual int32_t UpdateInPortFormat() = 0;
146     virtual int32_t UpdateOutPortFormat() = 0;
UpdateColorAspects()147     virtual void UpdateColorAspects() {}
148     void PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def);
149     int32_t SetFrameRateAdaptiveMode(const Format &format);
150     int32_t SetProcessName(const Format &format);
151     virtual int32_t ReConfigureOutputBufferCnt() = 0;
152     virtual uint64_t OnGetOutputBufferUsage() = 0;
153     virtual int32_t OnSetOutputBuffer(sptr<SurfaceBuffer> output) = 0;
154     virtual bool OnGetPackedInputFlag() = 0;
155 
156     // start
157     virtual bool ReadyToStart() = 0;
158     virtual int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex, bool isOutputPortSettingChanged) = 0;
159     virtual void UpdateFormatFromSurfaceBuffer() = 0;
160     int32_t GetPortDefinition(OMX_DIRTYPE portIndex, OMX_PARAM_PORTDEFINITIONTYPE& def);
161     int32_t AllocateSurfaceBuffers(OMX_DIRTYPE portIndex, bool isOutputPortSettingChanged,
162                                    sptr<SurfaceBuffer> output = nullptr);
163     int32_t AllocateHardwareBuffers(OMX_DIRTYPE portIndex);
164     std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> SurfaceBufferToOmxBuffer(
165         const sptr<SurfaceBuffer>& surfaceBuffer);
166     std::shared_ptr<HdiCodecNamespace::OmxCodecBuffer> DynamicSurfaceBufferToOmxBuffer();
167 
168     virtual int32_t SubmitAllBuffersOwnedByUs() = 0;
SubmitOutputBuffersToOmxNode()169     virtual int32_t SubmitOutputBuffersToOmxNode() { return IC_ERR_UNSUPPORT; }
170     BufferInfo* FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId);
171     std::optional<size_t> FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId);
172 
173     // input buffer circulation
174     virtual void NotifyUserToFillThisInBuffer(BufferInfo &info);
175     virtual void OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode);
176     void OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info);
177     int32_t NotifyOmxToEmptyThisInBuffer(BufferInfo& info);
178     virtual void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) = 0;
179 
180     // output buffer circulation
181     int32_t NotifyOmxToFillThisOutBuffer(BufferInfo &info);
182     void OnOMXFillBufferDone(const HdiCodecNamespace::OmxCodecBuffer& omxBuffer, BufferOperationMode mode);
183     void OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx);
184     void NotifyUserOutBufferAvaliable(BufferInfo &info);
185     void OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode);
186 
187     // // stop/release
188     void ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner, bool erase = false);
189     bool IsAllBufferOwnedByUs(OMX_DIRTYPE portIndex);
190     bool IsAllBufferOwnedByUs();
191     void EraseOutBuffersOwnedByUs();
192     void ClearBufferPool(OMX_DIRTYPE portIndex);
193     virtual void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) = 0;
194     void FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info);
195 
196     // template
197     template <typename T>
InitOMXParam(T & param)198     static inline void InitOMXParam(T& param)
199     {
200         (void)memset_s(&param, sizeof(T), 0x0, sizeof(T));
201         param.nSize = sizeof(T);
202         param.nVersion.s.nVersionMajor = 1;
203     }
204 
205     template <typename T>
InitOMXParamExt(T & param)206     static inline void InitOMXParamExt(T& param)
207     {
208         (void)memset_s(&param, sizeof(T), 0x0, sizeof(T));
209         param.size = sizeof(T);
210         param.version.s.nVersionMajor = 1;
211     }
212 
213     template <typename T>
214     bool GetParameter(uint32_t index, T& param, bool isCfg = false)
215     {
216         int8_t* p = reinterpret_cast<int8_t*>(&param);
217         std::vector<int8_t> inVec(p, p + sizeof(T));
218         std::vector<int8_t> outVec;
219         int32_t ret = isCfg ? compNode_->GetConfig(index, inVec, outVec) :
220                               compNode_->GetParameter(index, inVec, outVec);
221         if (ret != HDF_SUCCESS) {
222             return false;
223         }
224         if (outVec.size() != sizeof(T)) {
225             return false;
226         }
227         ret = memcpy_s(&param, sizeof(T), outVec.data(), outVec.size());
228         if (ret != EOK) {
229             return false;
230         }
231         return true;
232     }
233 
234     template <typename T>
235     bool SetParameter(uint32_t index, const T& param, bool isCfg = false)
236     {
237         const int8_t* p = reinterpret_cast<const int8_t*>(&param);
238         std::vector<int8_t> inVec(p, p + sizeof(T));
239         int32_t ret = isCfg ? compNode_->SetConfig(index, inVec) :
240                               compNode_->SetParameter(index, inVec);
241         if (ret != HDF_SUCCESS) {
242             return false;
243         }
244         return true;
245     }
246 
247 protected:
248     bool isEncoder_;
249     OMX_VIDEO_CODINGTYPE codingType_;
250     uint32_t componentId_ = 0;
251     std::string componentName_;
252     std::string compUniqueStr_;
253     bool is10Bit_ = false;
254     bool debugMode_ = false;
255     bool dumpMode_ = false;
256     bool isPackedInputSupported_ = false;
257     sptr<HdiCodecNamespace::ICodecCallback> compCb_ = nullptr;
258     sptr<HdiCodecNamespace::ICodecComponent> compNode_ = nullptr;
259     sptr<HdiCodecNamespace::ICodecComponentManager> compMgr_ = nullptr;
260 
261     std::shared_ptr<ImageCodecCallback> callback_;
262     PixelFmt configuredFmt_{};
263     BufferRequestConfig requestCfg_{};
264     std::shared_ptr<Format> configFormat_;
265     std::shared_ptr<Format> inputFormat_;
266     std::shared_ptr<Format> outputFormat_;
267 
268     std::vector<BufferInfo> inputBufferPool_;
269     std::vector<BufferInfo> outputBufferPool_;
270     bool isBufferCirculating_ = false;
271     bool inputPortEos_ = false;
272     bool outputPortEos_ = false;
273 
274     struct TotalCntAndCost {
275         uint64_t totalCnt = 0;
276         uint64_t totalCostUs = 0;
277     };
278     std::array<std::array<TotalCntAndCost, OWNER_CNT>, OWNER_CNT> inputHoldTimeRecord_;
279     std::chrono::time_point<std::chrono::steady_clock> firstInTime_;
280     uint64_t inTotalCnt_ = 0;
281     std::array<std::array<TotalCntAndCost, OWNER_CNT>, OWNER_CNT> outputHoldTimeRecord_;
282     std::chrono::time_point<std::chrono::steady_clock> firstOutTime_;
283     TotalCntAndCost outRecord_;
284     std::unordered_map<int64_t, std::chrono::time_point<std::chrono::steady_clock>> inTimeMap_;
285 
286     static constexpr char BUFFER_ID[] = "buffer-id";
287     static constexpr uint32_t WAIT_FENCE_MS = 1000;
288 private:
289     struct BaseState : State {
290     protected:
291         BaseState(ImageCodec *codec, const std::string &stateName,
292                   BufferOperationMode inputMode = KEEP_BUFFER, BufferOperationMode outputMode = KEEP_BUFFER)
293             : State(stateName), codec_(codec), inputMode_(inputMode), outputMode_(outputMode) {}
294         void OnMsgReceived(const MsgInfo &info) override;
295         void ReplyErrorCode(MsgId id, int32_t err);
296         void OnCodecEvent(const MsgInfo &info);
297         virtual void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2);
298         void OnGetFormat(const MsgInfo &info);
299         virtual void OnShutDown(const MsgInfo &info) = 0;
300         void OnCheckIfStuck(const MsgInfo &info);
301         void OnForceShutDown(const MsgInfo &info);
OnStateExitedBaseState302         void OnStateExited() override { codec_->stateGeneration_++; }
303 
304     protected:
305         ImageCodec *codec_;
306         BufferOperationMode inputMode_;
307         BufferOperationMode outputMode_;
308     };
309 
310     struct UninitializedState : BaseState {
UninitializedStateUninitializedState311         explicit UninitializedState(ImageCodec *codec) : BaseState(codec, "Uninitialized") {}
312     private:
313         void OnStateEntered() override;
314         void OnMsgReceived(const MsgInfo &info) override;
315         int32_t OnAllocateComponent(const std::string &name);
316         void OnShutDown(const MsgInfo &info) override;
317     };
318 
319     struct InitializedState : BaseState {
InitializedStateInitializedState320         explicit InitializedState(ImageCodec *codec) : BaseState(codec, "Initialized") {}
321     private:
322         void OnStateEntered() override;
323         void ProcessShutDownFromRunning();
324         void OnMsgReceived(const MsgInfo &info) override;
325         void OnSetCallBack(const MsgInfo &info);
326         void OnConfigure(const MsgInfo &info);
327         void OnGetOutputBufferUsage(const MsgInfo &info);
328         void OnSetOutputBuffer(const MsgInfo &info);
329         void OnGetPackedInputFlag(const MsgInfo &info);
330         void OnStart(const MsgInfo &info);
331         void OnShutDown(const MsgInfo &info) override;
332     };
333 
334     struct StartingState : BaseState {
StartingStateStartingState335         explicit StartingState(ImageCodec *codec) : BaseState(codec, "Starting") {}
336     private:
337         void OnStateEntered() override;
338         void OnStateExited() override;
339         void OnMsgReceived(const MsgInfo &info) override;
340         int32_t AllocateBuffers();
341         void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
342         void OnShutDown(const MsgInfo &info) override;
343         void ReplyStartMsg(int32_t errCode);
344         bool hasError_ = false;
345     };
346 
347     struct RunningState : BaseState {
RunningStateRunningState348         explicit RunningState(ImageCodec *codec) : BaseState(codec, "Running", RESUBMIT_BUFFER, RESUBMIT_BUFFER) {}
349     private:
350         void OnStateEntered() override;
351         void OnMsgReceived(const MsgInfo &info) override;
352         void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
353         void OnShutDown(const MsgInfo &info) override;
354     };
355 
356     struct OutputPortChangedState : BaseState {
OutputPortChangedStateOutputPortChangedState357         explicit OutputPortChangedState(ImageCodec *codec)
358             : BaseState(codec, "OutputPortChanged", RESUBMIT_BUFFER, FREE_BUFFER) {}
359     private:
360         void OnStateEntered() override;
361         void OnMsgReceived(const MsgInfo &info) override;
362         void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
363         void OnShutDown(const MsgInfo &info) override;
364         void HandleOutputPortDisabled();
365         void HandleOutputPortEnabled();
366     };
367 
368     struct StoppingState : BaseState {
StoppingStateStoppingState369         explicit StoppingState(ImageCodec *codec) : BaseState(codec, "Stopping"),
370             omxNodeInIdleState_(false),
371             omxNodeIsChangingToLoadedState_(false) {}
372     private:
373         void OnStateEntered() override;
374         void OnMsgReceived(const MsgInfo &info) override;
375         void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override;
376         void OnShutDown(const MsgInfo &info) override;
377         void ChangeStateIfWeOwnAllBuffers();
378         void ChangeOmxNodeToLoadedState(bool forceToFreeBuffer);
379         bool omxNodeInIdleState_;
380         bool omxNodeIsChangingToLoadedState_;
381     };
382 
383     class HdiCallback : public HdiCodecNamespace::ICodecCallback {
384     public:
HdiCallback(ImageCodec * codec)385         explicit HdiCallback(ImageCodec* codec) : codec_(codec) { }
386         virtual ~HdiCallback() = default;
387         int32_t EventHandler(HdiCodecNamespace::CodecEventType event, const HdiCodecNamespace::EventInfo& info);
388         int32_t EmptyBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer);
389         int32_t FillBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer);
390     private:
391         ImageCodec* codec_;
392     };
393 private:
394     int32_t DoSyncCall(MsgWhat msgType, std::function<void(ParamSP)> oper);
395     int32_t DoSyncCallAndGetReply(MsgWhat msgType, std::function<void(ParamSP)> oper, ParamSP &reply);
396     int32_t InitWithName(const std::string &name);
397     void ReleaseComponent();
398     void CleanUpOmxNode();
399     void ChangeOmxToTargetState(HdiCodecNamespace::CodecStateType &state,
400                                 HdiCodecNamespace::CodecStateType targetState);
401     bool RollOmxBackToLoaded();
402 
403     int32_t ForceShutdown(int32_t generation);
404     void SignalError(ImageCodecError err);
405     void DeferMessage(const MsgInfo &info);
406     void ProcessDeferredMessages();
407     void ReplyToSyncMsgLater(const MsgInfo& msg);
408     bool GetFirstSyncMsgToReply(MsgInfo& msg);
409 
410 private:
411     static constexpr size_t MAX_IMAGE_CODEC_BUFFER_SIZE = 8192 * 4096 * 4; // 8K RGBA
412     static constexpr uint32_t THREE_SECONDS_IN_US = 3'000'000;
413     static constexpr double FRAME_RATE_COEFFICIENT = 65536.0;
414 
415     std::shared_ptr<UninitializedState> uninitializedState_;
416     std::shared_ptr<InitializedState> initializedState_;
417     std::shared_ptr<StartingState> startingState_;
418     std::shared_ptr<RunningState> runningState_;
419     std::shared_ptr<OutputPortChangedState> outputPortChangedState_;
420     std::shared_ptr<StoppingState> stoppingState_;
421 
422     int32_t stateGeneration_ = 0;
423     bool isShutDownFromRunning_ = false;
424     bool notifyCallerAfterShutdownComplete_ = false;
425     bool hasFatalError_ = false;
426     std::list<MsgInfo> deferredQueue_;
427     std::map<MsgType, std::queue<std::pair<MsgId, ParamSP>>> syncMsgToReply_;
428 }; // class ImageCodec
429 } // namespace OHOS::ImagePlugin
430 
431 #endif // IMAGE_CODEC_H