/* * Copyright (C) 2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef HCODEC_HCODEC_H #define HCODEC_HCODEC_H #include #include #include #include #include "securec.h" #include "OMX_Component.h" // third_party/openmax/api/1.1.2 #include "codecbase.h" #include "avcodec_errors.h" #include "state_machine.h" #include "codec_hdi.h" #include "type_converter.h" #include "buffer/avbuffer.h" #include "meta/meta_key.h" // foundation/multimedia/histreamer/interface/inner_api/ namespace OHOS::MediaAVCodec { class HCodec : public CodecBase, protected StateMachine, public std::enable_shared_from_this { public: static std::shared_ptr Create(const std::string &name); int32_t Init(Media::Meta &callerInfo) override; int32_t SetCallback(const std::shared_ptr &callback) override; int32_t Configure(const Format &format) override; int32_t SetCustomBuffer(std::shared_ptr buffer) override; sptr CreateInputSurface() override; int32_t SetInputSurface(sptr surface) override; int32_t SetOutputSurface(sptr surface) override; int32_t QueueInputBuffer(uint32_t index) override; int32_t NotifyEos() override; int32_t ReleaseOutputBuffer(uint32_t index) override; int32_t RenderOutputBuffer(uint32_t index) override; int32_t SignalRequestIDRFrame() override; int32_t SetParameter(const Format& format) override; int32_t GetInputFormat(Format& format) override; int32_t GetOutputFormat(Format& format) override; std::string GetHidumperInfo() override; int32_t Start() override; int32_t Stop() override; int32_t Flush() override; int32_t Reset() override; int32_t Release() override; virtual GSError OnBufferReleasedByConsumer(uint64_t surfaceId) { return GSERROR_OK; } protected: enum MsgWhat : MsgType { INIT, SET_CALLBACK, CONFIGURE, CONFIGURE_BUFFER, CREATE_INPUT_SURFACE, SET_INPUT_SURFACE, SET_OUTPUT_SURFACE, START, GET_INPUT_FORMAT, GET_OUTPUT_FORMAT, SET_PARAMETERS, REQUEST_IDR_FRAME, FLUSH, QUEUE_INPUT_BUFFER, NOTIFY_EOS, RELEASE_OUTPUT_BUFFER, RENDER_OUTPUT_BUFFER, STOP, RELEASE, GET_HIDUMPER_INFO, PRINT_ALL_BUFFER_OWNER, INNER_MSG_BEGIN = 1000, CODEC_EVENT, OMX_EMPTY_BUFFER_DONE, OMX_FILL_BUFFER_DONE, GET_BUFFER_FROM_SURFACE, CHECK_IF_REPEAT, CHECK_IF_STUCK, FORCE_SHUTDOWN, }; enum BufferOperationMode { KEEP_BUFFER, RESUBMIT_BUFFER, FREE_BUFFER, }; enum BufferOwner { OWNED_BY_US = 0, OWNED_BY_USER = 1, OWNED_BY_OMX = 2, OWNED_BY_SURFACE = 3, OWNER_CNT = 4, }; struct PortInfo { uint32_t width = 0; uint32_t height = 0; OMX_VIDEO_CODINGTYPE codingType; std::optional pixelFmt; double frameRate; std::optional inputBufSize; }; enum DumpMode : unsigned long { DUMP_NONE = 0, DUMP_ENCODER_INPUT = 0b1000, DUMP_ENCODER_OUTPUT = 0b0100, DUMP_DECODER_INPUT = 0b0010, DUMP_DECODER_OUTPUT = 0b0001, }; struct BufferInfo { BufferInfo() : lastOwnerChangeTime(std::chrono::steady_clock::now()) {} bool isInput = true; BufferOwner owner = OWNED_BY_US; std::chrono::time_point lastOwnerChangeTime; int64_t lastFlushTime = 0; uint32_t bufferId = 0; std::shared_ptr omxBuffer; std::shared_ptr avBuffer; sptr surfaceBuffer; bool needDealWithCache = false; void CleanUpUnusedInfo(); void BeginCpuAccess(); void EndCpuAccess(); bool IsValidFrame() const; #ifdef BUILD_ENG_VERSION void Dump(const std::string& prefix, uint64_t cnt, DumpMode dumpMode, bool isEncoder) const; private: void Dump(const std::string& prefix, uint64_t cnt) const; void DumpSurfaceBuffer(const std::string& prefix, uint64_t cnt) const; void DecideDumpInfo(int& alignedH, uint32_t& totalSize, std::string& suffix, bool& dumpAsVideo) const; void DumpLinearBuffer(const std::string& prefix) const; static constexpr char DUMP_PATH[] = "/data/misc/hcodecdump"; #endif }; protected: HCodec(CodecHDI::CodecCompCapability caps, OMX_VIDEO_CODINGTYPE codingType, bool isEncoder); ~HCodec() override; static const char* ToString(MsgWhat what); static const char* ToString(BufferOwner owner); void ReplyErrorCode(MsgId id, int32_t err); void OnPrintAllBufferOwner(const MsgInfo& msg); void PrintAllBufferInfo(); void PrintAllBufferInfo(bool isInput, std::chrono::time_point now); void PrintStatistic(bool isInput, std::chrono::time_point now); std::string OnGetHidumperInfo(); void UpdateOwner(); void UpdateOwner(bool isInput); void ReduceOwner(bool isInput, BufferOwner owner); void ChangeOwner(BufferInfo& info, BufferOwner newOwner); void ChangeOwnerNormal(BufferInfo& info, BufferOwner newOwner); void ChangeOwnerDebug(BufferInfo& info, BufferOwner newOwner); void UpdateInputRecord(const BufferInfo& info, std::chrono::time_point now); void UpdateOutputRecord(const BufferInfo& info, std::chrono::time_point now); // configure virtual int32_t OnConfigure(const Format &format) = 0; virtual int32_t OnConfigureBuffer(std::shared_ptr buffer) { return AVCS_ERR_UNSUPPORT; } bool GetPixelFmtFromUser(const Format &format); static std::optional GetFrameRateFromUser(const Format &format); int32_t SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info); virtual int32_t UpdateInPortFormat() = 0; virtual int32_t UpdateOutPortFormat() = 0; virtual void UpdateColorAspects() {} void PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def); int32_t SetFrameRateAdaptiveMode(const Format &format); int32_t SetProcessName(); int32_t SetLowLatency(const Format &format); virtual int32_t OnSetOutputSurface(const sptr &surface, bool cfg) { return AVCS_ERR_UNSUPPORT; } virtual int32_t OnSetParameters(const Format &format) { return AVCS_ERR_OK; } virtual sptr OnCreateInputSurface() { return nullptr; } virtual int32_t OnSetInputSurface(sptr &inputSurface) { return AVCS_ERR_UNSUPPORT; } virtual int32_t RequestIDRFrame() { return AVCS_ERR_UNSUPPORT; } // start virtual bool ReadyToStart() = 0; virtual int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex) = 0; virtual void SetCallerToBuffer(int fd) {} virtual void UpdateFormatFromSurfaceBuffer() = 0; int32_t GetPortDefinition(OMX_DIRTYPE portIndex, OMX_PARAM_PORTDEFINITIONTYPE& def); int32_t AllocateAvSurfaceBuffers(OMX_DIRTYPE portIndex); int32_t AllocateAvLinearBuffers(OMX_DIRTYPE portIndex); int32_t AllocateAvHardwareBuffers(OMX_DIRTYPE portIndex, const OMX_PARAM_PORTDEFINITIONTYPE& def); int32_t AllocateAvSharedBuffers(OMX_DIRTYPE portIndex, const OMX_PARAM_PORTDEFINITIONTYPE& def); std::shared_ptr SurfaceBufferToOmxBuffer( const sptr& surfaceBuffer); std::shared_ptr DynamicSurfaceBufferToOmxBuffer(); virtual int32_t SubmitAllBuffersOwnedByUs() = 0; virtual int32_t SubmitOutputBuffersToOmxNode() { return AVCS_ERR_UNSUPPORT; } BufferInfo* FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId); std::optional FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId); virtual void OnGetBufferFromSurface(const ParamSP& param) = 0; uint32_t UserFlagToOmxFlag(AVCodecBufferFlag userFlag); AVCodecBufferFlag OmxFlagToUserFlag(uint32_t omxFlag); bool WaitFence(const sptr& fence); void WrapSurfaceBufferToSlot(BufferInfo &info, const sptr& surfaceBuffer, int64_t pts, uint32_t flag); // input buffer circulation virtual void NotifyUserToFillThisInBuffer(BufferInfo &info); virtual void OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode); int32_t OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info); virtual void OnSignalEndOfInputStream(const MsgInfo &msg); int32_t NotifyOmxToEmptyThisInBuffer(BufferInfo& info); virtual void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) = 0; virtual void RepeatIfNecessary(const ParamSP& param) {} bool CheckBufPixFmt(const sptr& buffer); // output buffer circulation virtual void SubmitDynamicBufferIfPossible() {} int32_t NotifyOmxToFillThisOutBuffer(BufferInfo &info); void OnOMXFillBufferDone(const CodecHDI::OmxCodecBuffer& omxBuffer, BufferOperationMode mode); void OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx); void NotifyUserOutBufferAvaliable(BufferInfo &info); void OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode); virtual void OnReleaseOutputBuffer(const BufferInfo &info) {} virtual void OnRenderOutputBuffer(const MsgInfo &msg, BufferOperationMode mode); virtual void ExtractPerFrameParamFromOmxBuffer( const std::shared_ptr &omxBuffer, std::shared_ptr &meta) {} // stop/release void ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner, bool erase = false); bool IsAllBufferOwnedByUsOrSurface(OMX_DIRTYPE portIndex); bool IsAllBufferOwnedByUsOrSurface(); void EraseOutBuffersOwnedByUsOrSurface(); void ClearBufferPool(OMX_DIRTYPE portIndex); virtual void OnClearBufferPool(OMX_DIRTYPE portIndex) {} virtual void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) = 0; void FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info); virtual void OnEnterUninitializedState() {} // template template static inline void InitOMXParam(T& param) { (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); param.nSize = sizeof(T); param.nVersion.s.nVersionMajor = 1; } template static inline void InitOMXParamExt(T& param) { (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); param.size = sizeof(T); param.version.s.nVersionMajor = 1; } template bool GetParameter(uint32_t index, T& param, bool isCfg = false) { int8_t* p = reinterpret_cast(¶m); std::vector inVec(p, p + sizeof(T)); std::vector outVec; int32_t ret = isCfg ? compNode_->GetConfig(index, inVec, outVec) : compNode_->GetParameter(index, inVec, outVec); if (ret != HDF_SUCCESS) { return false; } if (outVec.size() != sizeof(T)) { return false; } ret = memcpy_s(¶m, sizeof(T), outVec.data(), outVec.size()); if (ret != EOK) { return false; } return true; } template bool SetParameter(uint32_t index, const T& param, bool isCfg = false) { const int8_t* p = reinterpret_cast(¶m); std::vector inVec(p, p + sizeof(T)); int32_t ret = isCfg ? compNode_->SetConfig(index, inVec) : compNode_->SetParameter(index, inVec); if (ret != HDF_SUCCESS) { return false; } return true; } static inline uint32_t AlignTo(uint32_t side, uint32_t align) { if (align == 0) { return side; } return (side + align - 1) / align * align; } protected: CodecHDI::CodecCompCapability caps_; OMX_VIDEO_CODINGTYPE codingType_; bool isEncoder_; bool isSecure_ = false; std::string shortName_; uint32_t componentId_ = 0; std::string compUniqueStr_; struct CallerInfo { int32_t pid = -1; std::string processName; } playerCaller_, avcodecCaller_; bool calledByAvcodec_ = true; bool debugMode_ = false; DumpMode dumpMode_ = DUMP_NONE; sptr compCb_ = nullptr; sptr compNode_ = nullptr; sptr compMgr_ = nullptr; std::shared_ptr callback_; PixelFmt configuredFmt_{}; BufferRequestConfig requestCfg_{}; std::shared_ptr configFormat_; std::shared_ptr inputFormat_; std::shared_ptr outputFormat_; std::vector inputBufferPool_; std::vector outputBufferPool_; bool isBufferCirculating_ = false; bool inputPortEos_ = false; bool outputPortEos_ = false; bool gotFirstInput_ = false; bool gotFirstOutput_ = false; bool outPortHasChanged_ = false; // VRR #ifdef USE_VIDEO_PROCESSING_ENGINE bool isVrrInitialized_ = false; virtual int32_t VrrPrediction(BufferInfo &info) { return AVCS_ERR_UNSUPPORT; } #endif struct TotalCntAndCost { uint64_t totalCnt = 0; uint64_t totalCostUs = 0; }; // whole lift time uint64_t inTotalCnt_ = 0; TotalCntAndCost outRecord_; std::unordered_map> inTimeMap_; // normal: every 200 frames, debug: whole life time static constexpr uint64_t PRINT_PER_FRAME = 200; std::array inputHoldTimeRecord_; std::array outputHoldTimeRecord_; std::chrono::time_point firstInTime_; std::chrono::time_point firstOutTime_; // used when buffer circulation stoped static constexpr char KEY_LAST_OWNER_CHANGE_TIME[] = "lastOwnerChangeTime"; std::chrono::time_point lastOwnerChangeTime_; uint32_t circulateWarnPrintedTimes_ = 0; static constexpr uint32_t MAX_CIRCULATE_WARN_TIMES = 3; std::array inputOwner_ {}; std::array inputOwnerStr_ {}; std::array outputOwner_ {}; std::array outputOwnerStr_ {}; static constexpr char BUFFER_ID[] = "buffer-id"; static constexpr uint32_t WAIT_FENCE_MS = 1000; static constexpr uint32_t WARN_FENCE_MS = 30; static constexpr uint32_t STRIDE_ALIGNMENT = 32; static constexpr double FRAME_RATE_COEFFICIENT = 65536.0; private: struct BaseState : State { protected: BaseState(HCodec *codec, const std::string &stateName, BufferOperationMode inputMode = KEEP_BUFFER, BufferOperationMode outputMode = KEEP_BUFFER) : State(stateName), codec_(codec), inputMode_(inputMode), outputMode_(outputMode) {} void OnMsgReceived(const MsgInfo &info) override; void ReplyErrorCode(MsgId id, int32_t err); void OnCodecEvent(const MsgInfo &info); virtual void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2); void OnGetFormat(const MsgInfo &info); virtual void OnShutDown(const MsgInfo &info) = 0; virtual void OnCheckIfStuck(const MsgInfo &info); void OnForceShutDown(const MsgInfo &info); void OnStateExited() override { codec_->stateGeneration_++; } void OnSetParameters(const MsgInfo &info); protected: HCodec *codec_; BufferOperationMode inputMode_; BufferOperationMode outputMode_; }; struct UninitializedState : BaseState { explicit UninitializedState(HCodec *codec) : BaseState(codec, "Uninitialized") {} private: void OnStateEntered() override; void OnMsgReceived(const MsgInfo &info) override; void OnShutDown(const MsgInfo &info) override; }; struct InitializedState : BaseState { explicit InitializedState(HCodec *codec) : BaseState(codec, "Initialized") {} private: void OnStateEntered() override; void ProcessShutDownFromRunning(); void OnMsgReceived(const MsgInfo &info) override; void OnSetCallBack(const MsgInfo &info); void OnConfigure(const MsgInfo &info); void OnStart(const MsgInfo &info); void OnShutDown(const MsgInfo &info) override; }; struct StartingState : BaseState { explicit StartingState(HCodec *codec) : BaseState(codec, "Starting") {} private: void OnStateEntered() override; void OnStateExited() override; void OnMsgReceived(const MsgInfo &info) override; int32_t AllocateBuffers(); void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2) override; void OnShutDown(const MsgInfo &info) override; void ReplyStartMsg(int32_t errCode); bool hasError_ = false; }; struct RunningState : BaseState { explicit RunningState(HCodec *codec) : BaseState(codec, "Running", RESUBMIT_BUFFER, RESUBMIT_BUFFER) {} private: void OnStateEntered() override; void OnMsgReceived(const MsgInfo &info) override; void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2) override; void OnShutDown(const MsgInfo &info) override; void OnFlush(const MsgInfo &info); }; struct OutputPortChangedState : BaseState { explicit OutputPortChangedState(HCodec *codec) : BaseState(codec, "OutputPortChanged", RESUBMIT_BUFFER, FREE_BUFFER) {} private: void OnStateEntered() override; void OnMsgReceived(const MsgInfo &info) override; void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2) override; void OnShutDown(const MsgInfo &info) override; void HandleOutputPortDisabled(); void HandleOutputPortEnabled(); void OnFlush(const MsgInfo &info); void OnCheckIfStuck(const MsgInfo &info) override; }; struct FlushingState : BaseState { explicit FlushingState(HCodec *codec) : BaseState(codec, "Flushing") {} private: void OnStateEntered() override; void OnMsgReceived(const MsgInfo &info) override; void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2) override; void OnShutDown(const MsgInfo &info) override; void ChangeStateIfWeOwnAllBuffers(); bool IsFlushCompleteOnAllPorts(); int32_t UpdateFlushStatusOnPorts(uint32_t data1, uint32_t data2); bool flushCompleteFlag_[2] {false, false}; }; struct StoppingState : BaseState { explicit StoppingState(HCodec *codec) : BaseState(codec, "Stopping"), omxNodeInIdleState_(false), omxNodeIsChangingToLoadedState_(false) {} private: void OnStateEntered() override; void OnMsgReceived(const MsgInfo &info) override; void OnCodecEvent(CodecHDI::CodecEventType event, uint32_t data1, uint32_t data2) override; void OnShutDown(const MsgInfo &info) override; void ChangeStateIfWeOwnAllBuffers(); void ChangeOmxNodeToLoadedState(bool forceToFreeBuffer); bool omxNodeInIdleState_; bool omxNodeIsChangingToLoadedState_; }; class HdiCallback : public CodecHDI::ICodecCallback { public: explicit HdiCallback(std::weak_ptr codec) : codec_(codec) { } virtual ~HdiCallback() = default; int32_t EventHandler(CodecHDI::CodecEventType event, const CodecHDI::EventInfo& info); int32_t EmptyBufferDone(int64_t appData, const CodecHDI::OmxCodecBuffer& buffer); int32_t FillBufferDone(int64_t appData, const CodecHDI::OmxCodecBuffer& buffer); private: std::weak_ptr codec_; }; private: int32_t DoSyncCall(MsgWhat msgType, std::function oper); int32_t DoSyncCallAndGetReply(MsgWhat msgType, std::function oper, ParamSP &reply); void PrintCaller(); int32_t OnAllocateComponent(); void ReleaseComponent(); void CleanUpOmxNode(); void ChangeOmxToTargetState(CodecHDI::CodecStateType &state, CodecHDI::CodecStateType targetState); bool RollOmxBackToLoaded(); int32_t ForceShutdown(int32_t generation, bool isNeedNotifyCaller); void SignalError(AVCodecErrorType errorType, int32_t errorCode); void DeferMessage(const MsgInfo &info); void ProcessDeferredMessages(); void ReplyToSyncMsgLater(const MsgInfo& msg); bool GetFirstSyncMsgToReply(MsgInfo& msg); private: static constexpr size_t MAX_HCODEC_BUFFER_SIZE = 8192 * 4096 * 4; // 8K RGBA static constexpr uint32_t THREE_SECONDS_IN_US = 3'000'000; static constexpr uint32_t ONE_SECONDS_IN_US = 1'000'000; static constexpr uint32_t FIVE_SECONDS_IN_MS = 5'000; std::shared_ptr uninitializedState_; std::shared_ptr initializedState_; std::shared_ptr startingState_; std::shared_ptr runningState_; std::shared_ptr outputPortChangedState_; std::shared_ptr flushingState_; std::shared_ptr stoppingState_; int32_t stateGeneration_ = 0; bool isShutDownFromRunning_ = false; bool notifyCallerAfterShutdownComplete_ = false; bool keepComponentAllocated_ = false; bool hasFatalError_ = false; std::list deferredQueue_; std::map>> syncMsgToReply_; }; // class HCodec } // namespace OHOS::MediaAVCodec #endif // HCODEC_HCODEC_H