1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "codec_server.h"
17 #include <functional>
18 #include <malloc.h>
19 #include <map>
20 #include <unistd.h>
21 #include <limits>
22 #include <vector>
23 #include "avcodec_codec_name.h"
24 #include "avcodec_dump_utils.h"
25 #include "avcodec_errors.h"
26 #include "avcodec_log.h"
27 #include "avcodec_sysevent.h"
28 #include "buffer/avbuffer.h"
29 #include "codec_ability_singleton.h"
30 #include "codec_factory.h"
31 #include "media_description.h"
32 #include "meta/meta_key.h"
33 #include "surface_type.h"
34 
35 namespace {
36 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN_FRAMEWORK, "CodecServer"};
37 enum DumpIndex : uint32_t {
38     DUMP_INDEX_FORWARD_CALLER_PID          = 0x01'01'01'00,
39     DUMP_INDEX_FORWARD_CALLER_UID          = 0x01'01'02'00,
40     DUMP_INDEX_FORWARD_CALLER_PROCESS_NAME = 0x01'01'03'00,
41     DUMP_INDEX_CALLER_PID                  = 0x01'01'04'00,
42     DUMP_INDEX_CALLER_UID                  = 0x01'01'05'00,
43     DUMP_INDEX_CALLER_PROCESS_NAME         = 0x01'01'06'00,
44     DUMP_INDEX_STATUS                      = 0x01'01'07'00,
45     DUMP_INDEX_LAST_ERROR                  = 0x01'01'08'00,
46     DUMP_INDEX_CODEC_INFO_START            = 0x01'02'00'00,
47 };
48 constexpr uint32_t DUMP_OFFSET_8 = 8;
49 
50 const std::map<OHOS::MediaAVCodec::CodecServer::CodecStatus, std::string> CODEC_STATE_MAP = {
51     {OHOS::MediaAVCodec::CodecServer::UNINITIALIZED, "uninitialized"},
52     {OHOS::MediaAVCodec::CodecServer::INITIALIZED, "initialized"},
53     {OHOS::MediaAVCodec::CodecServer::CONFIGURED, "configured"},
54     {OHOS::MediaAVCodec::CodecServer::RUNNING, "running"},
55     {OHOS::MediaAVCodec::CodecServer::FLUSHED, "flushed"},
56     {OHOS::MediaAVCodec::CodecServer::END_OF_STREAM, "end of stream"},
57     {OHOS::MediaAVCodec::CodecServer::ERROR, "error"},
58 };
59 
60 const std::vector<std::pair<std::string_view, const std::string>> VIDEO_DUMP_TABLE = {
61     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_CODEC_NAME, "Codec_Name"},
62     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_WIDTH, "Width"},
63     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_HEIGHT, "Height"},
64     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_FRAME_RATE, "Frame_Rate"},
65     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_BITRATE, "Bit_Rate"},
66     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, "Pixel_Format"},
67     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_SCALE_TYPE, "Scale_Type"},
68     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_ROTATION_ANGLE, "Rotation_Angle"},
69     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_MAX_INPUT_SIZE, "Max_Input_Size"},
70     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_MAX_INPUT_BUFFER_COUNT, "Max_Input_Buffer_Count"},
71     {OHOS::MediaAVCodec::MediaDescriptionKey::MD_KEY_MAX_OUTPUT_BUFFER_COUNT, "Max_Output_Buffer_Count"},
72 };
73 
74 const std::map<int32_t, const std::string> PIXEL_FORMAT_STRING_MAP = {
75     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::YUV420P), "YUV420P"},
76     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::YUVI420), "YUVI420"},
77     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::NV12), "NV12"},
78     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::NV21), "NV21"},
79     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::SURFACE_FORMAT), "SURFACE_FORMAT"},
80     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::RGBA), "RGBA"},
81     {static_cast<int32_t>(OHOS::MediaAVCodec::VideoPixelFormat::UNKNOWN), "UNKNOWN_FORMAT"},
82 };
83 
84 const std::map<int32_t, const std::string> SCALE_TYPE_STRING_MAP = {
85     {OHOS::ScalingMode::SCALING_MODE_FREEZE, "Freeze"},
86     {OHOS::ScalingMode::SCALING_MODE_SCALE_TO_WINDOW, "Scale_to_window"},
87     {OHOS::ScalingMode::SCALING_MODE_SCALE_CROP, "Scale_crop"},
88     {OHOS::ScalingMode::SCALING_MODE_NO_SCALE_CROP, "No_scale_crop"},
89 };
90 
GetAudioCodecName(const OHOS::MediaAVCodec::AVCodecType type,std::string & name)91 int32_t GetAudioCodecName(const OHOS::MediaAVCodec::AVCodecType type, std::string &name)
92 {
93     using namespace OHOS::MediaAVCodec;
94     if (name.compare(AVCodecCodecName::AUDIO_DECODER_API9_AAC_NAME) == 0) {
95         name = AVCodecCodecName::AUDIO_DECODER_AAC_NAME;
96     } else if (name.compare(AVCodecCodecName::AUDIO_ENCODER_API9_AAC_NAME) == 0) {
97         name = AVCodecCodecName::AUDIO_ENCODER_AAC_NAME;
98     }
99     if (name.find("Audio") != name.npos) {
100         if ((name.find("Decoder") != name.npos && type != AVCODEC_TYPE_AUDIO_DECODER) ||
101             (name.find("Encoder") != name.npos && type != AVCODEC_TYPE_AUDIO_ENCODER)) {
102             AVCODEC_LOGE("AudioCodec name:%{public}s invalid", name.c_str());
103             return AVCS_ERR_INVALID_OPERATION;
104         }
105     }
106     return AVCS_ERR_OK;
107 }
108 
109 struct PostProcessingCallbackUserData {
110     std::shared_ptr<OHOS::MediaAVCodec::CodecServer> codecServer;
111 };
112 
PostProcessingCallbackOnError(int32_t errorCode,void * userData)113 void PostProcessingCallbackOnError(int32_t errorCode, void* userData)
114 {
115     CHECK_AND_RETURN_LOG(userData != nullptr, "Post processing callback's userData is nullptr");
116     auto callbackUserData = static_cast<PostProcessingCallbackUserData*>(userData);
117     auto codecServer = callbackUserData->codecServer;
118     CHECK_AND_RETURN_LOG(codecServer != nullptr, "Codec server dose not exit");
119     codecServer->PostProcessingOnError(errorCode);
120 }
121 
PostProcessingCallbackOnOutputBufferAvailable(uint32_t index,int32_t flag,void * userData)122 void PostProcessingCallbackOnOutputBufferAvailable(uint32_t index, int32_t flag, void* userData)
123 {
124     CHECK_AND_RETURN_LOG(userData != nullptr, "Post processing callback's userData is nullptr");
125     auto callbackUserData = static_cast<PostProcessingCallbackUserData*>(userData);
126     auto codecServer = callbackUserData->codecServer;
127     CHECK_AND_RETURN_LOG(codecServer != nullptr, "Codec server dose not exit");
128     codecServer->PostProcessingOnOutputBufferAvailable(index, flag);
129 }
130 
PostProcessingCallbackOnOutputFormatChanged(const OHOS::Media::Format & format,void * userData)131 void PostProcessingCallbackOnOutputFormatChanged(const OHOS::Media::Format& format, void* userData)
132 {
133     CHECK_AND_RETURN_LOG(userData != nullptr, "Post processing callback's userData is nullptr");
134     auto callbackUserData = static_cast<PostProcessingCallbackUserData*>(userData);
135     auto codecServer = callbackUserData->codecServer;
136     CHECK_AND_RETURN_LOG(codecServer != nullptr, "Codec server dose not exit");
137     codecServer->PostProcessingOnOutputFormatChanged(format);
138 }
139 } // namespace
140 
141 namespace OHOS {
142 namespace MediaAVCodec {
143 using namespace Media;
Create()144 std::shared_ptr<ICodecService> CodecServer::Create()
145 {
146     std::shared_ptr<CodecServer> server = std::make_shared<CodecServer>();
147 
148     int32_t ret = server->InitServer();
149     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, nullptr, "Server init failed, ret: %{public}d!", ret);
150     return server;
151 }
152 
CodecServer()153 CodecServer::CodecServer()
154 {
155     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances create", FAKE_POINTER(this));
156 }
157 
~CodecServer()158 CodecServer::~CodecServer()
159 {
160     std::unique_ptr<std::thread> thread = std::make_unique<std::thread>(&CodecServer::ExitProcessor, this);
161     CHECK_AND_RETURN_LOG(thread != nullptr, "0x%{public}06" PRIXPTR " create exit thread failed", FAKE_POINTER(this));
162     if (thread->joinable()) {
163         thread->join();
164     }
165     shareBufCallback_ = nullptr;
166     avBufCallback_ = nullptr;
167     (void)mallopt(M_FLUSH_THREAD_CACHE, 0);
168 
169     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances destroy", FAKE_POINTER(this));
170 }
171 
ExitProcessor()172 void CodecServer::ExitProcessor()
173 {
174     codecBase_ = nullptr;
175 }
176 
InitServer()177 int32_t CodecServer::InitServer()
178 {
179     return AVCS_ERR_OK;
180 }
181 
Init(AVCodecType type,bool isMimeType,const std::string & name,Meta & callerInfo,API_VERSION apiVersion)182 int32_t CodecServer::Init(AVCodecType type, bool isMimeType, const std::string &name,
183                           Meta &callerInfo, API_VERSION apiVersion)
184 {
185     std::lock_guard<std::shared_mutex> lock(mutex_);
186     (void)mallopt(M_SET_THREAD_CACHE, M_THREAD_CACHE_DISABLE);
187     (void)mallopt(M_DELAYED_FREE, M_DELAYED_FREE_DISABLE);
188     codecType_ = type;
189     codecName_ = name;
190     int32_t ret = isMimeType ? InitByMime(callerInfo, apiVersion) : InitByName(callerInfo, apiVersion);
191     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret,
192                              "Init failed. isMimeType:(%{public}d), name:(%{public}s), error:(%{public}d)", isMimeType,
193                              name.c_str(), ret);
194     SetCallerInfo(callerInfo);
195 
196     shareBufCallback_ = std::make_shared<CodecBaseCallback>(shared_from_this());
197     ret = codecBase_->SetCallback(shareBufCallback_);
198     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "SetCallback failed.");
199 
200     avBufCallback_ = std::make_shared<VCodecBaseCallback>(shared_from_this());
201     ret = codecBase_->SetCallback(avBufCallback_);
202     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "SetCallback failed.");
203 
204     StatusChanged(INITIALIZED);
205     return AVCS_ERR_OK;
206 }
207 
InitByName(Meta & callerInfo,API_VERSION apiVersion)208 int32_t CodecServer::InitByName(Meta &callerInfo, API_VERSION apiVersion)
209 {
210     int32_t ret = GetAudioCodecName(codecType_, codecName_);
211     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "CodecName get failed.");
212 
213     codecBase_ = CodecFactory::Instance().CreateCodecByName(codecName_, apiVersion);
214     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "CodecBase is nullptr.");
215     return codecBase_->Init(callerInfo);
216 }
217 
InitByMime(Meta & callerInfo,API_VERSION apiVersion)218 int32_t CodecServer::InitByMime(Meta &callerInfo, API_VERSION apiVersion)
219 {
220     int32_t ret = AVCS_ERR_NO_MEMORY;
221     bool isEncoder = (codecType_ == AVCODEC_TYPE_VIDEO_ENCODER) || (codecType_ == AVCODEC_TYPE_AUDIO_ENCODER);
222     std::vector<std::string> nameArray = CodecFactory::Instance().GetCodecNameArrayByMime(codecName_, isEncoder);
223     std::vector<std::string>::iterator iter;
224     for (iter = nameArray.begin(); iter != nameArray.end(); ++iter) {
225         ret = AVCS_ERR_NO_MEMORY;
226         codecBase_ = CodecFactory::Instance().CreateCodecByName(*iter, apiVersion);
227         CHECK_AND_CONTINUE_LOG(codecBase_ != nullptr, "Skip creation failure. name:(%{public}s)", iter->c_str());
228         ret = codecBase_->Init(callerInfo);
229         CHECK_AND_CONTINUE_LOG(ret == AVCS_ERR_OK, "Skip initialization failure. name:(%{public}s)", iter->c_str());
230         codecName_ = *iter;
231         break;
232     }
233     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "CodecBase is nullptr.");
234     return iter == nameArray.end() ? ret : AVCS_ERR_OK;
235 }
236 
Configure(const Format & format)237 int32_t CodecServer::Configure(const Format &format)
238 {
239     std::lock_guard<std::shared_mutex> lock(mutex_);
240     CHECK_AND_RETURN_RET_LOG(status_ == INITIALIZED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
241                              GetStatusDescription(status_).data());
242     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
243     Format config = format;
244 
245     int32_t isSetParameterCb = 0;
246     format.GetIntValue(Tag::VIDEO_ENCODER_ENABLE_SURFACE_INPUT_CALLBACK, isSetParameterCb);
247     isSetParameterCb_ = isSetParameterCb != 0;
248 
249     int32_t paramCheckRet = AVCS_ERR_OK;
250     if (codecType_ == AVCODEC_TYPE_VIDEO_ENCODER || codecType_ == AVCODEC_TYPE_VIDEO_DECODER) {
251         auto scenario = CodecParamChecker::CheckCodecScenario(config, codecType_, codecName_);
252         CHECK_AND_RETURN_RET_LOG(scenario != std::nullopt, AVCS_ERR_INVALID_VAL, "Failed to get codec scenario");
253         scenario_ = scenario.value();
254         paramCheckRet = CodecParamChecker::CheckConfigureValid(config, codecName_, scenario_);
255         CHECK_AND_RETURN_RET_LOG(paramCheckRet == AVCS_ERR_OK || paramCheckRet == AVCS_ERR_CODEC_PARAM_INCORRECT,
256             paramCheckRet, "Params in format is not valid.");
257         CodecScenarioInit(config);
258     }
259 
260     int32_t ret = codecBase_->Configure(config);
261     if (ret != AVCS_ERR_OK) {
262         StatusChanged(ERROR);
263         return ret;
264     }
265     ret = CreatePostProcessing(config);
266     if (ret != AVCS_ERR_OK) {
267         StatusChanged(ERROR);
268         return ret;
269     }
270     StatusChanged(CONFIGURED);
271     return paramCheckRet;
272 }
273 
SetCustomBuffer(std::shared_ptr<AVBuffer> buffer)274 int32_t CodecServer::SetCustomBuffer(std::shared_ptr<AVBuffer> buffer)
275 {
276     std::lock_guard<std::shared_mutex> lock(mutex_);
277     CHECK_AND_RETURN_RET_LOG(status_ == CONFIGURED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
278                              GetStatusDescription(status_).data());
279     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
280     return codecBase_->SetCustomBuffer(buffer);
281 }
282 
CodecScenarioInit(Format & config)283 int32_t CodecServer::CodecScenarioInit(Format &config)
284 {
285     switch (scenario_) {
286         case CodecScenario::CODEC_SCENARIO_ENC_TEMPORAL_SCALABILITY:
287             temporalScalability_ = std::make_shared<TemporalScalability>(codecName_);
288             temporalScalability_->ValidateTemporalGopParam(config);
289             if (!temporalScalability_->svcLTR_) {
290                 temporalScalability_ = nullptr;
291             }
292             break;
293         default:
294             break;
295     }
296 
297     return AVCS_ERR_OK;
298 }
299 
StartInputParamTask()300 void CodecServer::StartInputParamTask()
301 {
302     inputParamTask_ = std::make_shared<TaskThread>("InputParamTask");
303     std::weak_ptr<CodecServer> weakThis = weak_from_this();
304     inputParamTask_->RegisterHandler([weakThis] {
305         std::shared_ptr<CodecServer> cs = weakThis.lock();
306         if (cs) {
307             uint32_t index = cs->temporalScalability_->GetFirstBufferIndex();
308             AVCodecBufferInfo info;
309             AVCodecBufferFlag flag = AVCODEC_BUFFER_FLAG_NONE;
310             CHECK_AND_RETURN_LOG(cs->QueueInputBuffer(index, info, flag) == AVCS_ERR_OK, "QueueInputBuffer failed");
311         }
312     });
313     inputParamTask_->Start();
314 }
315 
Start()316 int32_t CodecServer::Start()
317 {
318     SetFreeStatus(false);
319     std::lock_guard<std::shared_mutex> lock(mutex_);
320     CHECK_AND_RETURN_RET_LOG(status_ == FLUSHED || status_ == CONFIGURED, AVCS_ERR_INVALID_STATE,
321                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
322     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
323     if (temporalScalability_ != nullptr && isCreateSurface_ && !isSetParameterCb_) {
324         StartInputParamTask();
325     }
326     int32_t ret = StartPostProcessing();
327     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Start post processing failed");
328     ret = codecBase_->Start();
329     if (ret != AVCS_ERR_OK) {
330         (void)StopPostProcessing();
331         StatusChanged(ERROR);
332     } else {
333         StatusChanged(RUNNING);
334         isModeConfirmed_ = true;
335         CodecDfxInfo codecDfxInfo;
336         GetCodecDfxInfo(codecDfxInfo);
337         CodecStartEventWrite(codecDfxInfo);
338     }
339     return ret;
340 }
341 
Stop()342 int32_t CodecServer::Stop()
343 {
344     SetFreeStatus(true);
345     std::lock_guard<std::shared_mutex> lock(mutex_);
346     CHECK_AND_RETURN_RET_LOGW(status_ != CONFIGURED, AVCS_ERR_OK,
347                               "Already in %{public}s state", GetStatusDescription(status_).data());
348     CHECK_AND_RETURN_RET_LOG(status_ == RUNNING || status_ == END_OF_STREAM || status_ == FLUSHED,
349                              AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
350                              GetStatusDescription(status_).data());
351     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
352     int32_t retPostProcessing = StopPostProcessing();
353     int32_t retCodec = codecBase_->Stop();
354     CodecStopEventWrite(caller_.pid, caller_.uid, FAKE_POINTER(this));
355     if ((retPostProcessing + retCodec) != AVCS_ERR_OK) {
356         StatusChanged(ERROR);
357         return (retCodec == AVCS_ERR_OK) ? retPostProcessing : retCodec;
358     }
359     StatusChanged(CONFIGURED);
360     return AVCS_ERR_OK;
361 }
362 
Flush()363 int32_t CodecServer::Flush()
364 {
365     SetFreeStatus(true);
366     std::lock_guard<std::shared_mutex> lock(mutex_);
367     CHECK_AND_RETURN_RET_LOGW(status_ != FLUSHED, AVCS_ERR_OK,
368                               "Already in %{public}s state", GetStatusDescription(status_).data());
369     CHECK_AND_RETURN_RET_LOG(status_ == RUNNING || status_ == END_OF_STREAM, AVCS_ERR_INVALID_STATE,
370                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
371     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
372     int32_t retPostProcessing = FlushPostProcessing();
373     int32_t retCodec = codecBase_->Flush();
374     CodecStopEventWrite(caller_.pid, caller_.uid, FAKE_POINTER(this));
375     if (retPostProcessing + retCodec != AVCS_ERR_OK) {
376         StatusChanged(ERROR);
377         return (retPostProcessing != AVCS_ERR_OK) ? retPostProcessing : retCodec;
378     }
379     StatusChanged(FLUSHED);
380     return AVCS_ERR_OK;
381 }
382 
NotifyEos()383 int32_t CodecServer::NotifyEos()
384 {
385     std::lock_guard<std::shared_mutex> lock(mutex_);
386     CHECK_AND_RETURN_RET_LOG(status_ == RUNNING, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
387                              GetStatusDescription(status_).data());
388     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
389     int32_t ret = codecBase_->NotifyEos();
390     if (ret == AVCS_ERR_OK) {
391         CodecStatus newStatus = END_OF_STREAM;
392         StatusChanged(newStatus);
393         CodecStopEventWrite(caller_.pid, caller_.uid, FAKE_POINTER(this));
394     }
395     return ret;
396 }
397 
Reset()398 int32_t CodecServer::Reset()
399 {
400     SetFreeStatus(true);
401     std::lock_guard<std::shared_mutex> lock(mutex_);
402     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
403     drmDecryptor_ = nullptr;
404     if (temporalScalability_ != nullptr) {
405         if (inputParamTask_ != nullptr) {
406             temporalScalability_->SetBlockQueueActive();
407             inputParamTask_->Stop();
408             inputParamTask_ = nullptr;
409         }
410         temporalScalability_ = nullptr;
411     }
412     int32_t ret = codecBase_->Reset();
413     CodecStatus newStatus = (ret == AVCS_ERR_OK ? INITIALIZED : ERROR);
414     StatusChanged(newStatus);
415     ret = ReleasePostProcessing();
416     if (ret != AVCS_ERR_OK) {
417         StatusChanged(ERROR);
418     }
419     CodecStopEventWrite(caller_.pid, caller_.uid, FAKE_POINTER(this));
420     lastErrMsg_.clear();
421     if (ret == AVCS_ERR_OK) {
422         isSurfaceMode_ = false;
423         isModeConfirmed_ = false;
424     }
425     return ret;
426 }
427 
Release()428 int32_t CodecServer::Release()
429 {
430     SetFreeStatus(true);
431     std::lock_guard<std::shared_mutex> lock(mutex_);
432     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
433     drmDecryptor_ = nullptr;
434     if (temporalScalability_ != nullptr) {
435         if (inputParamTask_ != nullptr) {
436             temporalScalability_->SetBlockQueueActive();
437             inputParamTask_->Stop();
438             inputParamTask_ = nullptr;
439         }
440         temporalScalability_ = nullptr;
441     }
442     int32_t ret = codecBase_->Release();
443     CodecStopEventWrite(caller_.pid, caller_.uid, FAKE_POINTER(this));
444     std::unique_ptr<std::thread> thread = std::make_unique<std::thread>(&CodecServer::ExitProcessor, this);
445     if (thread && thread->joinable()) {
446         thread->join();
447     }
448     shareBufCallback_ = nullptr;
449     avBufCallback_ = nullptr;
450     (void)ReleasePostProcessing();
451     if (ret == AVCS_ERR_OK) {
452         isSurfaceMode_ = false;
453         isModeConfirmed_ = false;
454     }
455     return ret;
456 }
457 
CreateInputSurface()458 sptr<Surface> CodecServer::CreateInputSurface()
459 {
460     std::lock_guard<std::shared_mutex> lock(mutex_);
461     CHECK_AND_RETURN_RET_LOG(status_ == CONFIGURED, nullptr, "In invalid state, %{public}s",
462                              GetStatusDescription(status_).data());
463     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, nullptr, "Codecbase is nullptr");
464     sptr<Surface> surface = codecBase_->CreateInputSurface();
465     if (surface != nullptr) {
466         isSurfaceMode_ = true;
467         isCreateSurface_ = true;
468     }
469     return surface;
470 }
471 
SetInputSurface(sptr<Surface> surface)472 int32_t CodecServer::SetInputSurface(sptr<Surface> surface)
473 {
474     std::lock_guard<std::shared_mutex> lock(mutex_);
475     CHECK_AND_RETURN_RET_LOG(status_ == CONFIGURED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
476                              GetStatusDescription(status_).data());
477     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
478     if (surface != nullptr) {
479         isSurfaceMode_ = true;
480     }
481     return codecBase_->SetInputSurface(surface);
482 }
483 
SetOutputSurface(sptr<Surface> surface)484 int32_t CodecServer::SetOutputSurface(sptr<Surface> surface)
485 {
486     std::lock_guard<std::shared_mutex> lock(mutex_);
487     bool isBufferMode = isModeConfirmed_ && !isSurfaceMode_;
488     CHECK_AND_RETURN_RET_LOG(!isBufferMode, AVCS_ERR_INVALID_OPERATION, "In buffer mode.");
489 
490     bool isValidState = isModeConfirmed_ ? isSurfaceMode_ && (status_ == CONFIGURED || status_ == RUNNING ||
491                                                               status_ == FLUSHED    || status_ == END_OF_STREAM)
492                                          : status_ == CONFIGURED;
493     CHECK_AND_RETURN_RET_LOG(isValidState, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
494                              GetStatusDescription(status_).data());
495     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
496     GSError gsRet = surface->SetSurfaceSourceType(OHSurfaceSource::OH_SURFACE_SOURCE_VIDEO);
497     EXPECT_AND_LOGW(gsRet != GSERROR_OK, "Set surface source type failed, %{public}s", GSErrorStr(gsRet).c_str());
498     int32_t ret = AVCS_ERR_OK;
499     if (postProcessing_) {
500         ret = SetOutputSurfaceForPostProcessing(surface);
501     } else {
502         ret = codecBase_->SetOutputSurface(surface);
503     }
504     isSurfaceMode_ = true;
505 #ifdef EMULATOR_ENABLED
506     Format config_emulator;
507     config_emulator.PutIntValue(Tag::VIDEO_PIXEL_FORMAT, static_cast<int32_t>(VideoPixelFormat::RGBA));
508     codecBase_->SetParameter(config_emulator);
509 #endif
510     return ret;
511 }
512 
DrmVideoCencDecrypt(uint32_t index)513 int32_t CodecServer::DrmVideoCencDecrypt(uint32_t index)
514 {
515     int32_t ret = AVCS_ERR_OK;
516     if (drmDecryptor_ != nullptr) {
517         if (decryptVideoBufs_.find(index) != decryptVideoBufs_.end()) {
518             uint32_t dataSize = static_cast<uint32_t>(decryptVideoBufs_[index].inBuf->memory_->GetSize());
519             decryptVideoBufs_[index].outBuf->pts_ = decryptVideoBufs_[index].inBuf->pts_;
520             decryptVideoBufs_[index].outBuf->dts_ = decryptVideoBufs_[index].inBuf->dts_;
521             decryptVideoBufs_[index].outBuf->duration_ = decryptVideoBufs_[index].inBuf->duration_;
522             decryptVideoBufs_[index].outBuf->flag_ = decryptVideoBufs_[index].inBuf->flag_;
523             if (decryptVideoBufs_[index].inBuf->meta_ != nullptr) {
524                 *(decryptVideoBufs_[index].outBuf->meta_) = *(decryptVideoBufs_[index].inBuf->meta_);
525             }
526             if (dataSize == 0) {
527                 decryptVideoBufs_[index].outBuf->memory_->SetSize(dataSize);
528                 return ret;
529             }
530             drmDecryptor_->SetCodecName(codecName_);
531             ret = drmDecryptor_->DrmVideoCencDecrypt(decryptVideoBufs_[index].inBuf,
532                 decryptVideoBufs_[index].outBuf, dataSize);
533             decryptVideoBufs_[index].outBuf->memory_->SetSize(dataSize);
534         }
535     }
536     return ret;
537 }
538 
QueueInputBuffer(uint32_t index,AVCodecBufferInfo info,AVCodecBufferFlag flag)539 int32_t CodecServer::QueueInputBuffer(uint32_t index, AVCodecBufferInfo info, AVCodecBufferFlag flag)
540 {
541     std::shared_lock<std::shared_mutex> freeLock(freeMutex_);
542     if (isFree_) {
543         AVCODEC_LOGE("In invalid state, free out");
544         return AVCS_ERR_INVALID_STATE;
545     }
546 
547     int32_t ret = AVCS_ERR_OK;
548     if (flag & AVCODEC_BUFFER_FLAG_EOS) {
549         std::lock_guard<std::shared_mutex> lock(mutex_);
550         ret = QueueInputBufferIn(index, info, flag);
551         if (ret == AVCS_ERR_OK) {
552             CodecStatus newStatus = END_OF_STREAM;
553             StatusChanged(newStatus);
554         }
555     } else {
556         std::shared_lock<std::shared_mutex> lock(mutex_);
557         ret = QueueInputBufferIn(index, info, flag);
558     }
559     return ret;
560 }
561 
QueueInputBufferIn(uint32_t index,AVCodecBufferInfo info,AVCodecBufferFlag flag)562 int32_t CodecServer::QueueInputBufferIn(uint32_t index, AVCodecBufferInfo info, AVCodecBufferFlag flag)
563 {
564     int32_t ret = AVCS_ERR_OK;
565     CHECK_AND_RETURN_RET_LOG(
566         status_ == RUNNING || ((isSetParameterCb_ || inputParamTask_ != nullptr) && status_ == END_OF_STREAM),
567         AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s", GetStatusDescription(status_).data());
568     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
569     if (temporalScalability_ != nullptr) {
570         temporalScalability_->ConfigureLTR(index);
571     }
572     if (videoCb_ != nullptr) {
573         ret = DrmVideoCencDecrypt(index);
574         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, AVCS_ERR_DECRYPT_FAILED, "CodecServer decrypt failed");
575         ret = codecBase_->QueueInputBuffer(index);
576     }
577     if (codecCb_ != nullptr) {
578         ret = codecBase_->QueueInputBuffer(index, info, flag);
579     }
580     return ret;
581 }
582 
QueueInputBuffer(uint32_t index)583 int32_t CodecServer::QueueInputBuffer(uint32_t index)
584 {
585     (void)index;
586     return AVCS_ERR_UNSUPPORT;
587 }
588 
QueueInputParameter(uint32_t index)589 int32_t CodecServer::QueueInputParameter(uint32_t index)
590 {
591     (void)index;
592     return AVCS_ERR_UNSUPPORT;
593 }
594 
GetOutputFormat(Format & format)595 int32_t CodecServer::GetOutputFormat(Format &format)
596 {
597     std::lock_guard<std::shared_mutex> lock(mutex_);
598     CHECK_AND_RETURN_RET_LOG(status_ != UNINITIALIZED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
599                              GetStatusDescription(status_).data());
600     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
601     if (postProcessing_) {
602         int32_t ret = codecBase_->GetOutputFormat(format);
603         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "GetOutputFormat failed");
604         return GetPostProcessingOutputFormat(format);
605     } else {
606         return codecBase_->GetOutputFormat(format);
607     }
608 }
609 
CheckDrmSvpConsistency(const sptr<DrmStandard::IMediaKeySessionService> & keySession,bool svpFlag)610 int32_t CodecServer::CheckDrmSvpConsistency(const sptr<DrmStandard::IMediaKeySessionService> &keySession,
611     bool svpFlag)
612 {
613     AVCODEC_LOGI("CheckDrmSvpConsistency");
614     CHECK_AND_RETURN_RET_LOG(keySession != nullptr, AVCS_ERR_INVALID_VAL, "keySession is nullptr");
615     std::string tmpName = codecName_;
616     transform(tmpName.begin(), tmpName.end(), tmpName.begin(), ::tolower);
617 
618     // check codec name when secure video path is false
619     if (svpFlag == false) {
620         if (tmpName.find(".secure") != std::string::npos) {
621             AVCODEC_LOGE("CheckDrmSvpConsistency failed, svpFlag is false but the decoder is secure!");
622             return AVCS_ERR_INVALID_VAL;
623         }
624         return AVCS_ERR_OK;
625     }
626 
627     // check codec name when secure video path is true
628     if (tmpName.find(".secure") == std::string::npos) {
629         AVCODEC_LOGE("CheckDrmSvpConsistency failed, svpFlag is true but the decoder is not secure!");
630         return AVCS_ERR_INVALID_VAL;
631     }
632 
633     // check session level when secure video path is true
634 #ifdef SUPPORT_DRM
635     DrmStandard::IMediaKeySessionService::ContentProtectionLevel sessionLevel;
636     int ret = keySession->GetContentProtectionLevel(&sessionLevel);
637     CHECK_AND_RETURN_RET_LOG(ret == 0, AVCS_ERR_INVALID_VAL, "GetContentProtectionLevel failed");
638     if (sessionLevel <
639         DrmStandard::IMediaKeySessionService::ContentProtectionLevel::CONTENT_PROTECTION_LEVEL_HW_CRYPTO) {
640         AVCODEC_LOGE("CheckDrmSvpConsistency failed, key session's content protection level is too low!");
641         return AVCS_ERR_INVALID_VAL;
642     }
643 #endif
644 
645     return AVCS_ERR_OK;
646 }
647 
648 #ifdef SUPPORT_DRM
SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> & keySession,const bool svpFlag)649 int32_t CodecServer::SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySession, const bool svpFlag)
650 {
651     std::lock_guard<std::shared_mutex> lock(mutex_);
652     AVCODEC_LOGI("CodecServer::SetDecryptConfig");
653     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
654 
655     int32_t ret = CheckDrmSvpConsistency(keySession, svpFlag);
656     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, AVCS_ERR_INVALID_VAL, "check svp failed");
657 
658     if (drmDecryptor_ == nullptr) {
659         drmDecryptor_ = std::make_shared<CodecDrmDecrypt>();
660     }
661     CHECK_AND_RETURN_RET_LOG(drmDecryptor_ != nullptr, AVCS_ERR_NO_MEMORY, "drmDecryptor is nullptr");
662     drmDecryptor_->SetDecryptionConfig(keySession, svpFlag);
663     return AVCS_ERR_OK;
664 }
665 #endif
666 
ReleaseOutputBuffer(uint32_t index,bool render)667 int32_t CodecServer::ReleaseOutputBuffer(uint32_t index, bool render)
668 {
669     std::shared_lock<std::shared_mutex> freeLock(freeMutex_);
670     if (isFree_) {
671         AVCODEC_LOGE("In invalid state, free out");
672         return AVCS_ERR_INVALID_STATE;
673     }
674     std::shared_lock<std::shared_mutex> lock(mutex_);
675     CHECK_AND_RETURN_RET_LOG(status_ == RUNNING || status_ == END_OF_STREAM, AVCS_ERR_INVALID_STATE,
676                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
677 
678     if (postProcessing_) {
679         return ReleaseOutputBufferOfPostProcessing(index, render);
680     } else {
681         return ReleaseOutputBufferOfCodec(index, render);
682     }
683 }
684 
ReleaseOutputBufferOfCodec(uint32_t index,bool render)685 int32_t CodecServer::ReleaseOutputBufferOfCodec(uint32_t index, bool render)
686 {
687     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
688     int32_t ret;
689     if (render) {
690         ret = codecBase_->RenderOutputBuffer(index);
691     } else {
692         ret = codecBase_->ReleaseOutputBuffer(index);
693     }
694     return ret;
695 }
696 
RenderOutputBufferAtTime(uint32_t index,int64_t renderTimestampNs)697 int32_t CodecServer::RenderOutputBufferAtTime(uint32_t index, int64_t renderTimestampNs)
698 {
699     (void)renderTimestampNs;
700     std::shared_lock<std::shared_mutex> freeLock(freeMutex_);
701     if (isFree_) {
702         AVCODEC_LOGE("In invalid state, free out");
703         return AVCS_ERR_INVALID_STATE;
704     }
705     std::shared_lock<std::shared_mutex> lock(mutex_);
706     CHECK_AND_RETURN_RET_LOG(status_ == RUNNING || status_ == END_OF_STREAM, AVCS_ERR_INVALID_STATE,
707                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
708     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
709     if (postProcessing_) {
710         return postProcessing_->ReleaseOutputBuffer(index, true);
711     } else {
712         return codecBase_->RenderOutputBuffer(index);
713     }
714 }
715 
SetParameter(const Format & format)716 int32_t CodecServer::SetParameter(const Format &format)
717 {
718     std::lock_guard<std::shared_mutex> lock(mutex_);
719     CHECK_AND_RETURN_RET_LOG(status_ != INITIALIZED && status_ != CONFIGURED, AVCS_ERR_INVALID_STATE,
720                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
721     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
722 
723     if (codecType_ == AVCODEC_TYPE_VIDEO_ENCODER || codecType_ == AVCODEC_TYPE_VIDEO_DECODER) {
724         Format oldFormat;
725         int32_t ret = codecBase_->GetOutputFormat(oldFormat);
726         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, AVCS_ERR_INVALID_OPERATION, "Failed to get codec format");
727         ret = CodecParamChecker::CheckParameterValid(format, oldFormat, codecName_, scenario_);
728         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Params in format is not valid.");
729     }
730 
731     return codecBase_->SetParameter(format);
732 }
733 
SetCallback(const std::shared_ptr<AVCodecCallback> & callback)734 int32_t CodecServer::SetCallback(const std::shared_ptr<AVCodecCallback> &callback)
735 {
736     std::lock_guard<std::shared_mutex> cbLock(cbMutex_);
737     codecCb_ = callback;
738     return AVCS_ERR_OK;
739 }
740 
SetCallback(const std::shared_ptr<MediaCodecCallback> & callback)741 int32_t CodecServer::SetCallback(const std::shared_ptr<MediaCodecCallback> &callback)
742 {
743     std::lock_guard<std::shared_mutex> cbLock(cbMutex_);
744     videoCb_ = callback;
745     return AVCS_ERR_OK;
746 }
747 
SetCallback(const std::shared_ptr<MediaCodecParameterCallback> & callback)748 int32_t CodecServer::SetCallback(const std::shared_ptr<MediaCodecParameterCallback> &callback)
749 {
750     (void)callback;
751     return AVCS_ERR_UNSUPPORT;
752 }
753 
SetCallback(const std::shared_ptr<MediaCodecParameterWithAttrCallback> & callback)754 int32_t CodecServer::SetCallback(const std::shared_ptr<MediaCodecParameterWithAttrCallback> &callback)
755 {
756     (void)callback;
757     return AVCS_ERR_UNSUPPORT;
758 }
759 
GetInputFormat(Format & format)760 int32_t CodecServer::GetInputFormat(Format &format)
761 {
762     std::lock_guard<std::shared_mutex> lock(mutex_);
763     CHECK_AND_RETURN_RET_LOG(
764         status_ == CONFIGURED || status_ == RUNNING || status_ == FLUSHED || status_ == END_OF_STREAM,
765         AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s", GetStatusDescription(status_).data());
766     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
767     return codecBase_->GetInputFormat(format);
768 }
769 
DumpInfo(int32_t fd)770 int32_t CodecServer::DumpInfo(int32_t fd)
771 {
772     CHECK_AND_RETURN_RET_LOG(fd >= 0, AVCS_ERR_OK, "Get a invalid fd");
773     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
774     Format codecFormat;
775     int32_t ret = codecBase_->GetOutputFormat(codecFormat);
776     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Get codec format failed.");
777 
778     AVCodecDumpControler dumpControler;
779     auto statusIt = CODEC_STATE_MAP.find(status_);
780     if (forwardCaller_.pid != -1 || !forwardCaller_.processName.empty()) {
781         dumpControler.AddInfo(DUMP_INDEX_FORWARD_CALLER_PID, "Forward_Caller_Pid", std::to_string(forwardCaller_.pid));
782         dumpControler.AddInfo(DUMP_INDEX_FORWARD_CALLER_UID, "Forward_Caller_Uid", std::to_string(forwardCaller_.uid));
783         dumpControler.AddInfo(DUMP_INDEX_FORWARD_CALLER_PROCESS_NAME,
784             "Forward_Caller_Process_Name", forwardCaller_.processName);
785     }
786     dumpControler.AddInfo(DUMP_INDEX_CALLER_PID, "Caller_Pid", std::to_string(caller_.pid));
787     dumpControler.AddInfo(DUMP_INDEX_CALLER_UID, "Caller_Uid", std::to_string(caller_.uid));
788     dumpControler.AddInfo(DUMP_INDEX_CALLER_PROCESS_NAME, "Caller_Process_Name", caller_.processName);
789     dumpControler.AddInfo(DUMP_INDEX_STATUS, "Status", statusIt != CODEC_STATE_MAP.end() ? statusIt->second : "");
790     dumpControler.AddInfo(DUMP_INDEX_LAST_ERROR, "Last_Error", lastErrMsg_.size() ? lastErrMsg_ : "Null");
791 
792     uint32_t infoIndex = 1;
793     for (auto iter : VIDEO_DUMP_TABLE) {
794         uint32_t dumpIndex = DUMP_INDEX_CODEC_INFO_START + (infoIndex++ << DUMP_OFFSET_8);
795         if (iter.first == MediaDescriptionKey::MD_KEY_PIXEL_FORMAT) {
796             dumpControler.AddInfoFromFormatWithMapping(dumpIndex, codecFormat,
797                                                        iter.first, iter.second, PIXEL_FORMAT_STRING_MAP);
798         } else if (iter.first == MediaDescriptionKey::MD_KEY_SCALE_TYPE) {
799             dumpControler.AddInfoFromFormatWithMapping(dumpIndex, codecFormat,
800                                                        iter.first, iter.second, SCALE_TYPE_STRING_MAP);
801         } else {
802             dumpControler.AddInfoFromFormat(dumpIndex, codecFormat, iter.first, iter.second);
803         }
804     }
805     std::string dumpString;
806     dumpControler.GetDumpString(dumpString);
807     dumpString += codecBase_->GetHidumperInfo();
808     dumpString += "\n";
809     write(fd, dumpString.c_str(), dumpString.size());
810     return AVCS_ERR_OK;
811 }
812 
SetCallerInfo(const Meta & callerInfo)813 void CodecServer::SetCallerInfo(const Meta &callerInfo)
814 {
815     (void)callerInfo.GetData(Tag::AV_CODEC_CALLER_PID, caller_.pid);
816     (void)callerInfo.GetData(Tag::AV_CODEC_CALLER_UID, caller_.uid);
817     (void)callerInfo.GetData(Tag::AV_CODEC_CALLER_PROCESS_NAME, caller_.processName);
818     (void)callerInfo.GetData(Tag::AV_CODEC_FORWARD_CALLER_PID, forwardCaller_.pid);
819     (void)callerInfo.GetData(Tag::AV_CODEC_FORWARD_CALLER_UID, forwardCaller_.uid);
820     (void)callerInfo.GetData(Tag::AV_CODEC_FORWARD_CALLER_PROCESS_NAME, forwardCaller_.processName);
821 
822     if (caller_.pid == -1) {
823         caller_.pid = getprocpid();
824         caller_.uid = getuid();
825     }
826 
827     EXPECT_AND_LOGI((forwardCaller_.pid >= 0) || (!forwardCaller_.processName.empty()),
828         "Forward caller pid: %{public}d, process name: %{public}s",
829         forwardCaller_.pid, forwardCaller_.processName.c_str());
830     AVCODEC_LOGI("Caller pid: %{public}d, process name: %{public}s", caller_.pid, caller_.processName.c_str());
831 }
832 
GetStatusDescription(CodecStatus status)833 inline const std::string &CodecServer::GetStatusDescription(CodecStatus status)
834 {
835     if (status < UNINITIALIZED || status >= ERROR) {
836         return CODEC_STATE_MAP.at(ERROR);
837     }
838     return CODEC_STATE_MAP.at(status);
839 }
840 
StatusChanged(CodecStatus newStatus)841 inline void CodecServer::StatusChanged(CodecStatus newStatus)
842 {
843     if (status_ == newStatus) {
844         return;
845     }
846     AVCODEC_LOGI("Status %{public}s -> %{public}s",
847         GetStatusDescription(status_).data(), GetStatusDescription(newStatus).data());
848     status_ = newStatus;
849 }
850 
OnError(int32_t errorType,int32_t errorCode)851 void CodecServer::OnError(int32_t errorType, int32_t errorCode)
852 {
853     std::lock_guard<std::shared_mutex> lock(cbMutex_);
854     lastErrMsg_ = AVCSErrorToString(static_cast<AVCodecServiceErrCode>(errorCode));
855     FaultEventWrite(FaultType::FAULT_TYPE_INNER_ERROR, lastErrMsg_, "Codec");
856     if (videoCb_ != nullptr) {
857         videoCb_->OnError(static_cast<AVCodecErrorType>(errorType), errorCode);
858     }
859     if (codecCb_ != nullptr) {
860         codecCb_->OnError(static_cast<AVCodecErrorType>(errorType), errorCode);
861     }
862 }
863 
OnOutputFormatChanged(const Format & format)864 void CodecServer::OnOutputFormatChanged(const Format &format)
865 {
866     std::lock_guard<std::shared_mutex> lock(cbMutex_);
867     if (postProcessing_) {
868         outputFormatChanged_ = format;
869         return;
870     }
871     if (videoCb_ != nullptr) {
872         videoCb_->OnOutputFormatChanged(format);
873     }
874     if (codecCb_ != nullptr) {
875         codecCb_->OnOutputFormatChanged(format);
876     }
877 }
878 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVSharedMemory> buffer)879 void CodecServer::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVSharedMemory> buffer)
880 {
881     std::shared_lock<std::shared_mutex> lock(cbMutex_);
882     if (codecCb_ == nullptr || (isCreateSurface_ && !isSetParameterCb_)) {
883         return;
884     }
885     codecCb_->OnInputBufferAvailable(index, buffer);
886 }
887 
OnOutputBufferAvailable(uint32_t index,AVCodecBufferInfo info,AVCodecBufferFlag flag,std::shared_ptr<AVSharedMemory> buffer)888 void CodecServer::OnOutputBufferAvailable(uint32_t index, AVCodecBufferInfo info, AVCodecBufferFlag flag,
889                                           std::shared_ptr<AVSharedMemory> buffer)
890 {
891     std::shared_lock<std::shared_mutex> lock(cbMutex_);
892     if (codecCb_ == nullptr) {
893         return;
894     }
895     codecCb_->OnOutputBufferAvailable(index, info, flag, buffer);
896 }
897 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)898 void CodecServer::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
899 {
900     std::shared_lock<std::shared_mutex> lock(cbMutex_);
901     if (temporalScalability_ != nullptr) {
902         temporalScalability_->StoreAVBuffer(index, buffer);
903     }
904     if (videoCb_ == nullptr || (isCreateSurface_ && !isSetParameterCb_)) {
905         return;
906     }
907     if (drmDecryptor_ != nullptr) {
908         if (decryptVideoBufs_.find(index) != decryptVideoBufs_.end()) {
909             videoCb_->OnInputBufferAvailable(index, decryptVideoBufs_[index].inBuf);
910             return;
911         }
912         DrmDecryptVideoBuf decryptVideoBuf;
913         MemoryFlag memFlag = MEMORY_READ_WRITE;
914         std::shared_ptr<AVAllocator> avAllocator = AVAllocatorFactory::CreateSharedAllocator(memFlag);
915         if (avAllocator == nullptr) {
916             AVCODEC_LOGE("CreateSharedAllocator failed");
917             return;
918         }
919         decryptVideoBuf.inBuf = AVBuffer::CreateAVBuffer(avAllocator,
920             static_cast<int32_t>(buffer->memory_->GetCapacity()));
921         if (decryptVideoBuf.inBuf == nullptr || decryptVideoBuf.inBuf->memory_ == nullptr ||
922             decryptVideoBuf.inBuf->memory_->GetCapacity() != static_cast<int32_t>(buffer->memory_->GetCapacity())) {
923             AVCODEC_LOGE("CreateAVBuffer failed");
924             return;
925         }
926         decryptVideoBuf.outBuf = buffer;
927         decryptVideoBufs_.insert({index, decryptVideoBuf});
928         videoCb_->OnInputBufferAvailable(index, decryptVideoBuf.inBuf);
929     } else {
930         videoCb_->OnInputBufferAvailable(index, buffer);
931     }
932 }
933 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)934 void CodecServer::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
935 {
936     CHECK_AND_RETURN_LOG(buffer != nullptr, "buffer is nullptr!");
937 
938     if (temporalScalability_ != nullptr && !(buffer->flag_ == AVCODEC_BUFFER_FLAG_CODEC_DATA)) {
939         temporalScalability_->SetDisposableFlag(buffer);
940     }
941 
942     std::shared_lock<std::shared_mutex> lock(cbMutex_);
943     CHECK_AND_RETURN_LOG(videoCb_ != nullptr, "videoCb_ is nullptr!");
944     if (postProcessing_) {
945         /*
946             If post processing is configured, this callback flow is intercepted here. Just push the decoded buffer info
947             {index, buffer} into the decodedBufferInfoQueue_ which is monitored by another task thread. Once the queue
948             has data, the thread will pop the data from the queue and calls "CodecServer::ReleaseOutputBuffer" to flush
949             it into video processing engine. The video processing engine will automatically processing the frame
950             according to the index. The callback ipc proxy's function "videoCb_->OnOutputBufferAvailable" is called
951             later in "PostProcessingOnOutputBufferAvailable" by video processing engine when the frame is processed
952             to notify application that the frame is ready. At last, application calls
953             "OH_VideoDecoder_RenderOutputBuffer" or "OH_VideoDecoder_FreeOutputBuffer" to flush the frame.
954         */
955         (void)PushDecodedBufferInfo(index, buffer);
956     } else {
957         videoCb_->OnOutputBufferAvailable(index, buffer);
958     }
959 }
960 
CodecBaseCallback(const std::shared_ptr<CodecServer> & codec)961 CodecBaseCallback::CodecBaseCallback(const std::shared_ptr<CodecServer> &codec) : codec_(codec)
962 {
963     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances create", FAKE_POINTER(this));
964 }
965 
~CodecBaseCallback()966 CodecBaseCallback::~CodecBaseCallback()
967 {
968     codec_ = nullptr;
969     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances destroy", FAKE_POINTER(this));
970 }
971 
OnError(AVCodecErrorType errorType,int32_t errorCode)972 void CodecBaseCallback::OnError(AVCodecErrorType errorType, int32_t errorCode)
973 {
974     if (codec_ != nullptr) {
975         codec_->OnError(errorType, errorCode);
976     }
977 }
978 
OnOutputFormatChanged(const Format & format)979 void CodecBaseCallback::OnOutputFormatChanged(const Format &format)
980 {
981     if (codec_ != nullptr) {
982         codec_->OnOutputFormatChanged(format);
983     }
984 }
985 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVSharedMemory> buffer)986 void CodecBaseCallback::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVSharedMemory> buffer)
987 {
988     if (codec_ != nullptr) {
989         codec_->OnInputBufferAvailable(index, buffer);
990     }
991 }
992 
OnOutputBufferAvailable(uint32_t index,AVCodecBufferInfo info,AVCodecBufferFlag flag,std::shared_ptr<AVSharedMemory> buffer)993 void CodecBaseCallback::OnOutputBufferAvailable(uint32_t index, AVCodecBufferInfo info, AVCodecBufferFlag flag,
994                                                 std::shared_ptr<AVSharedMemory> buffer)
995 {
996     if (codec_ != nullptr) {
997         codec_->OnOutputBufferAvailable(index, info, flag, buffer);
998     }
999 }
1000 
VCodecBaseCallback(const std::shared_ptr<CodecServer> & codec)1001 VCodecBaseCallback::VCodecBaseCallback(const std::shared_ptr<CodecServer> &codec) : codec_(codec)
1002 {
1003     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances create", FAKE_POINTER(this));
1004 }
1005 
~VCodecBaseCallback()1006 VCodecBaseCallback::~VCodecBaseCallback()
1007 {
1008     codec_ = nullptr;
1009     AVCODEC_LOGD("0x%{public}06" PRIXPTR " Instances destroy", FAKE_POINTER(this));
1010 }
1011 
OnError(AVCodecErrorType errorType,int32_t errorCode)1012 void VCodecBaseCallback::OnError(AVCodecErrorType errorType, int32_t errorCode)
1013 {
1014     if (codec_ != nullptr) {
1015         codec_->OnError(errorType, errorCode);
1016     }
1017 }
1018 
OnOutputFormatChanged(const Format & format)1019 void VCodecBaseCallback::OnOutputFormatChanged(const Format &format)
1020 {
1021     if (codec_ != nullptr) {
1022         codec_->OnOutputFormatChanged(format);
1023     }
1024 }
1025 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)1026 void VCodecBaseCallback::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
1027 {
1028     if (codec_ != nullptr) {
1029         codec_->OnInputBufferAvailable(index, buffer);
1030     }
1031 }
1032 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)1033 void VCodecBaseCallback::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
1034 {
1035     if (codec_ != nullptr) {
1036         codec_->OnOutputBufferAvailable(index, buffer);
1037     }
1038 }
1039 
GetCodecDfxInfo(CodecDfxInfo & codecDfxInfo)1040 int32_t CodecServer::GetCodecDfxInfo(CodecDfxInfo &codecDfxInfo)
1041 {
1042     Format format;
1043     codecBase_->GetOutputFormat(format);
1044     int32_t videoPixelFormat = static_cast<int32_t>(VideoPixelFormat::UNKNOWN);
1045     format.GetIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, videoPixelFormat);
1046     videoPixelFormat = PIXEL_FORMAT_STRING_MAP.find(videoPixelFormat) != PIXEL_FORMAT_STRING_MAP.end()
1047                            ? videoPixelFormat
1048                            : static_cast<int32_t>(VideoPixelFormat::UNKNOWN);
1049     int32_t codecIsVendor = 0;
1050     format.GetIntValue("IS_VENDOR", codecIsVendor);
1051 
1052     codecDfxInfo.clientPid = caller_.pid;
1053     codecDfxInfo.clientUid = caller_.uid;
1054     codecDfxInfo.codecInstanceId = FAKE_POINTER(this);
1055     format.GetStringValue(MediaDescriptionKey::MD_KEY_CODEC_NAME, codecDfxInfo.codecName);
1056     codecDfxInfo.codecIsVendor = codecIsVendor == 1 ? "True" : "False";
1057     codecDfxInfo.codecMode = isSurfaceMode_ ? "Surface mode" : "Buffer Mode";
1058     format.GetLongValue(MediaDescriptionKey::MD_KEY_BITRATE, codecDfxInfo.encoderBitRate);
1059     format.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, codecDfxInfo.videoWidth);
1060     format.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, codecDfxInfo.videoHeight);
1061     format.GetDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, codecDfxInfo.videoFrameRate);
1062     format.GetIntValue(MediaDescriptionKey::MD_KEY_CHANNEL_COUNT, codecDfxInfo.audioChannelCount);
1063     codecDfxInfo.videoPixelFormat =
1064         codecDfxInfo.audioChannelCount == 0 ? PIXEL_FORMAT_STRING_MAP.at(videoPixelFormat) : "";
1065     format.GetIntValue(MediaDescriptionKey::MD_KEY_SAMPLE_RATE, codecDfxInfo.audioSampleRate);
1066     return 0;
1067 }
1068 
Configure(const std::shared_ptr<Media::Meta> & meta)1069 int32_t CodecServer::Configure(const std::shared_ptr<Media::Meta> &meta)
1070 {
1071     std::lock_guard<std::shared_mutex> lock(mutex_);
1072     CHECK_AND_RETURN_RET_LOG(meta != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
1073 
1074     CHECK_AND_RETURN_RET_LOG(status_ == INITIALIZED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
1075                              GetStatusDescription(status_).data());
1076     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
1077 
1078     int32_t ret = codecBase_->Configure(meta);
1079 
1080     CodecStatus newStatus = (ret == AVCS_ERR_OK ? CONFIGURED : ERROR);
1081     StatusChanged(newStatus);
1082     return ret;
1083 }
SetParameter(const std::shared_ptr<Media::Meta> & parameter)1084 int32_t CodecServer::SetParameter(const std::shared_ptr<Media::Meta> &parameter)
1085 {
1086     std::lock_guard<std::shared_mutex> lock(mutex_);
1087     CHECK_AND_RETURN_RET_LOG(parameter != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
1088     CHECK_AND_RETURN_RET_LOG(status_ != INITIALIZED && status_ != CONFIGURED, AVCS_ERR_INVALID_STATE,
1089                              "In invalid state, %{public}s", GetStatusDescription(status_).data());
1090     return codecBase_->SetParameter(parameter);
1091 }
GetOutputFormat(std::shared_ptr<Media::Meta> & parameter)1092 int32_t CodecServer::GetOutputFormat(std::shared_ptr<Media::Meta> &parameter)
1093 {
1094     std::lock_guard<std::shared_mutex> lock(mutex_);
1095     CHECK_AND_RETURN_RET_LOG(status_ != UNINITIALIZED, AVCS_ERR_INVALID_STATE, "In invalid state, %{public}s",
1096                              GetStatusDescription(status_).data());
1097     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "codecBase is nullptr");
1098     return codecBase_->GetOutputFormat(parameter);
1099 }
1100 
SetOutputBufferQueue(const sptr<Media::AVBufferQueueProducer> & bufferQueueProducer)1101 int32_t CodecServer::SetOutputBufferQueue(const sptr<Media::AVBufferQueueProducer> &bufferQueueProducer)
1102 {
1103     std::lock_guard<std::shared_mutex> lock(mutex_);
1104     CHECK_AND_RETURN_RET_LOG(bufferQueueProducer != nullptr, AVCS_ERR_NO_MEMORY, "Codecbase is nullptr");
1105     return codecBase_->SetOutputBufferQueue(bufferQueueProducer);
1106 }
Prepare()1107 int32_t CodecServer::Prepare()
1108 {
1109     std::lock_guard<std::shared_mutex> lock(mutex_);
1110     switch (codecType_) {
1111         case AVCODEC_TYPE_VIDEO_DECODER:
1112             // Post processing is only available for video decoder.
1113             return PreparePostProcessing();
1114         case AVCODEC_TYPE_VIDEO_ENCODER:
1115             return AVCS_ERR_OK;
1116         default:
1117             // Audio's interface "Prepare"
1118             return codecBase_->Prepare();
1119     }
1120 }
GetInputBufferQueue()1121 sptr<Media::AVBufferQueueProducer> CodecServer::GetInputBufferQueue()
1122 {
1123     std::lock_guard<std::shared_mutex> lock(mutex_);
1124     return codecBase_->GetInputBufferQueue();
1125 }
ProcessInputBuffer()1126 void CodecServer::ProcessInputBuffer()
1127 {
1128     std::lock_guard<std::shared_mutex> lock(mutex_);
1129     return codecBase_->ProcessInputBuffer();
1130 }
1131 
1132 #ifdef SUPPORT_DRM
SetAudioDecryptionConfig(const sptr<DrmStandard::IMediaKeySessionService> & keySession,const bool svpFlag)1133 int32_t CodecServer::SetAudioDecryptionConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySession,
1134     const bool svpFlag)
1135 {
1136     std::lock_guard<std::shared_mutex> lock(mutex_);
1137     AVCODEC_LOGI("CodecServer::SetAudioDecryptionConfig");
1138     CHECK_AND_RETURN_RET_LOG(codecBase_ != nullptr, AVCS_ERR_NO_MEMORY, "codecBase is nullptr");
1139     return codecBase_->SetAudioDecryptionConfig(keySession, svpFlag);
1140 }
1141 #endif
1142 
CheckRunning()1143 bool CodecServer::CheckRunning()
1144 {
1145     if (status_ == CodecServer::RUNNING) {
1146         return true;
1147     }
1148     return false;
1149 }
1150 
SetFreeStatus(bool isFree)1151 void CodecServer::SetFreeStatus(bool isFree)
1152 {
1153     std::lock_guard<std::shared_mutex> lock(freeMutex_);
1154     isFree_ = isFree;
1155 }
1156 
CreatePostProcessing(const Format & format)1157 int32_t CodecServer::CreatePostProcessing(const Format& format)
1158 {
1159     if (codecType_ != AVCODEC_TYPE_VIDEO_DECODER) {
1160         return AVCS_ERR_OK;
1161     }
1162     int32_t colorSpaceType;
1163     if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_VIDEO_DECODER_OUTPUT_COLOR_SPACE, colorSpaceType)) {
1164         return AVCS_ERR_OK;
1165     }
1166     auto capData = CodecAbilitySingleton::GetInstance().GetCapabilityByName(codecName_);
1167     CHECK_AND_RETURN_RET_LOG(capData != std::nullopt && capData->isVendor, AVCS_ERR_UNKNOWN,
1168         "Get codec capability from codec list failed");
1169     CHECK_AND_RETURN_RET_LOG(codecBase_, AVCS_ERR_UNKNOWN, "Decoder is not found");
1170     int32_t ret;
1171     postProcessing_ = PostProcessingType::Create(codecBase_, format, ret);
1172     if (postProcessing_) {
1173         AVCODEC_LOGI("Post processing is configured");
1174     }
1175     return ret;
1176 }
1177 
SetCallbackForPostProcessing()1178 int32_t CodecServer::SetCallbackForPostProcessing()
1179 {
1180     using namespace std::placeholders;
1181     postProcessingCallback_.onError = std::bind(PostProcessingCallbackOnError, _1, _2);
1182     postProcessingCallback_.onOutputBufferAvailable =
1183         std::bind(PostProcessingCallbackOnOutputBufferAvailable, _1, _2, _3);
1184     postProcessingCallback_.onOutputFormatChanged = std::bind(PostProcessingCallbackOnOutputFormatChanged, _1, _2);
1185     auto userData = new PostProcessingCallbackUserData;
1186     CHECK_AND_RETURN_RET_LOG(userData != nullptr, AVCS_ERR_NO_MEMORY,
1187         "Failed to create post processing callback userdata");
1188     userData->codecServer = shared_from_this();
1189     postProcessingUserData_ = static_cast<void*>(userData);
1190     return postProcessing_->SetCallback(postProcessingCallback_, postProcessingUserData_);
1191 }
1192 
ClearCallbackForPostProcessing()1193 void CodecServer::ClearCallbackForPostProcessing()
1194 {
1195     std::shared_lock<std::shared_mutex> lock(cbMutex_);
1196     postProcessingCallback_.onError = nullptr;
1197     postProcessingCallback_.onOutputBufferAvailable = nullptr;
1198 }
1199 
SetOutputSurfaceForPostProcessing(sptr<Surface> surface)1200 int32_t CodecServer::SetOutputSurfaceForPostProcessing(sptr<Surface> surface)
1201 {
1202     int32_t ret = postProcessing_->SetOutputSurface(surface);
1203     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set output surface failed");
1204     return ret;
1205 }
1206 
PreparePostProcessing()1207 int32_t CodecServer::PreparePostProcessing()
1208 {
1209     if (!postProcessing_) {
1210         return AVCS_ERR_OK;
1211     }
1212 
1213     int32_t ret{AVCS_ERR_OK};
1214     if (postProcessingUserData_ == nullptr) {
1215         ret = SetCallbackForPostProcessing();
1216         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set callback for post post processing failed");
1217     }
1218 
1219     if (decodedBufferInfoQueue_ == nullptr) {
1220         decodedBufferInfoQueue_ = DecodedBufferInfoQueue::Create("DecodedBufferInfoQueue");
1221         CHECK_AND_RETURN_RET_LOG(decodedBufferInfoQueue_, AVCS_ERR_NO_MEMORY,
1222             "Create decoded buffer info queue failed");
1223     }
1224 
1225     if (postProcessingInputBufferInfoQueue_ == nullptr) {
1226         postProcessingInputBufferInfoQueue_ = DecodedBufferInfoQueue::Create("PostProcessingInputBufferInfoQueue");
1227         CHECK_AND_RETURN_RET_LOG(postProcessingInputBufferInfoQueue_, AVCS_ERR_NO_MEMORY,
1228             "Create post processing input buffer info queue failed");
1229     }
1230 
1231     if (postProcessingOutputBufferInfoQueue_ == nullptr) {
1232         postProcessingOutputBufferInfoQueue_ = DecodedBufferInfoQueue::Create("PostProcessingOutputBufferInfoQueue");
1233         CHECK_AND_RETURN_RET_LOG(postProcessingOutputBufferInfoQueue_, AVCS_ERR_NO_MEMORY,
1234             "Create post processing output buffer info queue failed");
1235     }
1236 
1237     ret = postProcessing_->Prepare();
1238     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare post processing failed");
1239 
1240     AVCODEC_LOGI("Post processing is prepared");
1241     return AVCS_ERR_OK;
1242 }
1243 
StartPostProcessing()1244 int32_t CodecServer::StartPostProcessing()
1245 {
1246     if (!postProcessing_) {
1247         return AVCS_ERR_OK;
1248     }
1249     int32_t ret = postProcessing_->Start();
1250     if (ret != AVCS_ERR_OK) {
1251         StatusChanged(ERROR);
1252         return ret;
1253     }
1254     ret = StartPostProcessingTask();
1255     if (ret != AVCS_ERR_OK) {
1256         StatusChanged(ERROR);
1257         return ret;
1258     }
1259     AVCODEC_LOGI("Post processing is started");
1260     return ret;
1261 }
1262 
StopPostProcessing()1263 int32_t CodecServer::StopPostProcessing()
1264 {
1265     DeactivatePostProcessingQueue();
1266     if (postProcessingTask_) {
1267         postProcessingTask_->Stop();
1268     }
1269     AVCODEC_LOGD("Post processing task stopped");
1270     if (postProcessing_) {
1271         int32_t ret = postProcessing_->Stop();
1272         CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Stop post processing failed");
1273     }
1274     if (decodedBufferInfoQueue_) {
1275         decodedBufferInfoQueue_->Clear();
1276     }
1277     if (postProcessingInputBufferInfoQueue_) {
1278         postProcessingInputBufferInfoQueue_->Clear();
1279     }
1280     if (postProcessingOutputBufferInfoQueue_) {
1281         postProcessingOutputBufferInfoQueue_->Clear();
1282     }
1283 
1284     AVCODEC_LOGD("reset frame count");
1285     decodedFrameCount_.store(0);
1286     processedFrameCount_.store(0);
1287     decoderIsEOS_.store(false);
1288 
1289     AVCODEC_LOGI("Post processing is stopped");
1290     return AVCS_ERR_OK;
1291 }
1292 
FlushPostProcessing()1293 int32_t CodecServer::FlushPostProcessing()
1294 {
1295     if (!postProcessing_) {
1296         return AVCS_ERR_OK;
1297     }
1298     DeactivatePostProcessingQueue();
1299     if (postProcessingTask_) {
1300         postProcessingTask_->Pause();
1301     }
1302     auto ret = postProcessing_->Flush();
1303     if (decodedBufferInfoQueue_) {
1304         decodedBufferInfoQueue_->Clear();
1305     }
1306     if (postProcessingInputBufferInfoQueue_) {
1307         postProcessingInputBufferInfoQueue_->Clear();
1308     }
1309     if (postProcessingOutputBufferInfoQueue_) {
1310         postProcessingOutputBufferInfoQueue_->Clear();
1311     }
1312 
1313     AVCODEC_LOGD("reset frame count");
1314     decodedFrameCount_.store(0);
1315     processedFrameCount_.store(0);
1316     decoderIsEOS_.store(false);
1317 
1318     CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Flush post processing failed");
1319     AVCODEC_LOGI("Post processing is flushed");
1320     return AVCS_ERR_OK;
1321 }
1322 
ResetPostProcessing()1323 int32_t CodecServer::ResetPostProcessing()
1324 {
1325     if (postProcessing_) {
1326         DeactivatePostProcessingQueue();
1327         if (postProcessingTask_) {
1328             postProcessingTask_->Stop();
1329         }
1330         postProcessing_->Reset();
1331         CleanPostProcessingResource();
1332         postProcessing_.reset();
1333     }
1334     AVCODEC_LOGI("Post processing is reset");
1335     return AVCS_ERR_OK;
1336 }
1337 
ReleasePostProcessing()1338 int32_t CodecServer::ReleasePostProcessing()
1339 {
1340     if (postProcessing_) {
1341         DeactivatePostProcessingQueue();
1342         if (postProcessingTask_) {
1343             postProcessingTask_->Stop();
1344         }
1345         postProcessing_->Release();
1346         CleanPostProcessingResource();
1347         postProcessing_.reset();
1348         AVCODEC_LOGI("Post processing is released");
1349     }
1350 
1351     if (postProcessingUserData_ != nullptr) {
1352         auto p = static_cast<PostProcessingCallbackUserData*>(postProcessingUserData_);
1353         p->codecServer.reset();
1354         delete p;
1355         postProcessingUserData_ = nullptr;
1356     }
1357 
1358     return AVCS_ERR_OK;
1359 }
1360 
ReleaseOutputBufferOfPostProcessing(uint32_t index,bool render)1361 int32_t CodecServer::ReleaseOutputBufferOfPostProcessing(uint32_t index, bool render)
1362 {
1363     CHECK_AND_RETURN_RET_LOG(postProcessing_, AVCS_ERR_UNKNOWN, "Post processing is null");
1364     if (index == std::numeric_limits<uint32_t>::max()) {
1365         AVCODEC_LOGD("EOS is got");
1366         return AVCS_ERR_OK;
1367     }
1368     std::shared_ptr<DecodedBufferInfo> info{nullptr};
1369     CHECK_AND_RETURN_RET_LOG(postProcessingOutputBufferInfoQueue_, AVCS_ERR_UNKNOWN, "Queue is null");
1370     auto ret = postProcessingOutputBufferInfoQueue_->PopWait(info);
1371     CHECK_AND_RETURN_RET_LOG(ret == QueueResult::OK, AVCS_ERR_UNKNOWN,
1372         "Failed to get data, %{public}s", QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1373     CHECK_AND_RETURN_RET_LOG(info && info->buffer, AVCS_ERR_UNKNOWN, "Data is null");
1374     return postProcessing_->ReleaseOutputBuffer(index, render);
1375 }
1376 
GetPostProcessingOutputFormat(Format & format)1377 int32_t CodecServer::GetPostProcessingOutputFormat(Format& format)
1378 {
1379     postProcessing_->GetOutputFormat(format);
1380     return AVCS_ERR_OK;
1381 }
1382 
PushDecodedBufferInfo(uint32_t index,std::shared_ptr<AVBuffer> buffer)1383 int32_t CodecServer::PushDecodedBufferInfo(uint32_t index, std::shared_ptr<AVBuffer> buffer)
1384 {
1385     auto info = std::make_shared<DecodedBufferInfo>();
1386     CHECK_AND_RETURN_RET_LOG(info, AVCS_ERR_NO_MEMORY, "Failed to allocate info");
1387     info->index = index;
1388     info->buffer = buffer;
1389     CHECK_AND_RETURN_RET_LOG(decodedBufferInfoQueue_, AVCS_ERR_UNKNOWN, "Queue is null");
1390     auto ret = decodedBufferInfoQueue_->PushWait(info);
1391     CHECK_AND_RETURN_RET_LOG(ret == QueueResult::OK, AVCS_ERR_UNKNOWN, "Push data failed, %{public}s",
1392         QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1393     return AVCS_ERR_OK;
1394 }
1395 
PostProcessingOnError(int32_t errorCode)1396 void CodecServer::PostProcessingOnError(int32_t errorCode)
1397 {
1398     std::lock_guard<std::shared_mutex> lock(cbMutex_);
1399     if (videoCb_ == nullptr) {
1400         AVCODEC_LOGD("Missing video callback");
1401         return;
1402     }
1403     int32_t ret = VPEErrorToAVCSError(errorCode);
1404     AVCODEC_LOGD("PostProcessingOnError, errorCodec:%{public}d -> %{public}d", errorCode, ret);
1405     videoCb_->OnError(AVCodecErrorType::AVCODEC_ERROR_INTERNAL, ret);
1406 }
1407 
1408 void CodecServer::PostProcessingOnOutputBufferAvailable(uint32_t index, [[maybe_unused]] int32_t flag)
1409 {
1410     std::lock_guard<std::shared_mutex> lock(cbMutex_);
1411     if (videoCb_ == nullptr) {
1412         AVCODEC_LOGD("Missing video callback");
1413         return;
1414     }
1415     CHECK_AND_RETURN_LOG(postProcessingInputBufferInfoQueue_ && postProcessingOutputBufferInfoQueue_, "Queue is null");
1416     std::shared_ptr<DecodedBufferInfo> info{nullptr};
1417     auto ret = postProcessingInputBufferInfoQueue_->PopWait(info);
1418     CHECK_AND_RETURN_LOG(ret == QueueResult::OK, "Get data failed, %{public}s",
1419         QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1420     CHECK_AND_RETURN_LOG(info && info->buffer, "Invalid data");
1421     info->index = index;
1422     ret = postProcessingOutputBufferInfoQueue_->PushWait(info);
1423     CHECK_AND_RETURN_LOG(ret == QueueResult::OK, "Push data failed, %{public}s",
1424         QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1425     videoCb_->OnOutputBufferAvailable(index, info->buffer);
1426     processedFrameCount_++;
1427 }
1428 
PostProcessingOnOutputFormatChanged(const Format & format)1429 void CodecServer::PostProcessingOnOutputFormatChanged(const Format& format)
1430 {
1431     std::lock_guard<std::shared_mutex> lock(cbMutex_);
1432     if (videoCb_ == nullptr) {
1433         AVCODEC_LOGD("Missing video callback");
1434         return;
1435     }
1436     int32_t width = 0;
1437     if (format.GetIntValue(Media::Tag::VIDEO_WIDTH, width)) {
1438         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_WIDTH, width);
1439         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_PIC_WIDTH, width);
1440     }
1441     int32_t height = 0;
1442     if (format.GetIntValue(Media::Tag::VIDEO_HEIGHT, height)) {
1443         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_HEIGHT, height);
1444         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_PIC_HEIGHT, height);
1445     }
1446     int32_t stride = 0;
1447     if (format.GetIntValue(Media::Tag::VIDEO_STRIDE, stride)) {
1448         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_STRIDE, stride);
1449     }
1450     int32_t sliceHeight = 0;
1451     if (format.GetIntValue(Media::Tag::VIDEO_SLICE_HEIGHT, sliceHeight)) {
1452         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_SLICE_HEIGHT, sliceHeight);
1453     }
1454     int32_t outputColorSpace = 0;
1455     if (format.GetIntValue(Media::Tag::VIDEO_DECODER_OUTPUT_COLOR_SPACE, outputColorSpace)) {
1456         outputFormatChanged_.PutIntValue(Media::Tag::VIDEO_DECODER_OUTPUT_COLOR_SPACE, outputColorSpace);
1457     }
1458     videoCb_->OnOutputFormatChanged(outputFormatChanged_);
1459 }
1460 
StartPostProcessingTask()1461 int32_t CodecServer::StartPostProcessingTask()
1462 {
1463     if (!postProcessingTask_) {
1464         postProcessingTask_ = std::make_unique<TaskThread>("PostProcessing");
1465         CHECK_AND_RETURN_RET_LOG(postProcessingTask_, AVCS_ERR_UNKNOWN, "Create task post processing failed");
1466         std::function<void()> task = std::bind(&CodecServer::PostProcessingTask, this);
1467         postProcessingTask_->RegisterHandler(task);
1468     }
1469     if (decodedBufferInfoQueue_) {
1470         decodedBufferInfoQueue_->Activate();
1471     }
1472     if (postProcessingInputBufferInfoQueue_) {
1473         postProcessingInputBufferInfoQueue_->Activate();
1474     }
1475     if (postProcessingOutputBufferInfoQueue_) {
1476         postProcessingOutputBufferInfoQueue_->Activate();
1477     }
1478     postProcessingTask_->Start();
1479 
1480     return AVCS_ERR_OK;
1481 }
1482 
PostProcessingTask()1483 void CodecServer::PostProcessingTask()
1484 {
1485     CHECK_AND_RETURN_LOG(decodedBufferInfoQueue_ && postProcessingInputBufferInfoQueue_, "Queue is null");
1486     if (decoderIsEOS_.load() == false) {
1487         std::shared_ptr<DecodedBufferInfo> info{nullptr};
1488         auto ret = decodedBufferInfoQueue_->PopWait(info);
1489         CHECK_AND_RETURN_LOG(ret == QueueResult::OK, "Get data failed, %{public}s",
1490             QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1491         CHECK_AND_RETURN_LOG(info && info->buffer, "Invalid data");
1492         ret = postProcessingInputBufferInfoQueue_->PushWait(info);
1493         CHECK_AND_RETURN_LOG(ret == QueueResult::OK, "Push data failed, %{public}s",
1494             QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(ret)]);
1495         decodedFrameCount_++;
1496         AVCODEC_LOGD("Decoded frame count = %{public}" PRIu64, decodedFrameCount_.load());
1497         if (info->buffer->flag_ == AVCODEC_BUFFER_FLAG_EOS) {
1498             AVCODEC_LOGI("index = %{public}u, EOS flag", info->index);
1499             decoderIsEOS_.store(true);
1500         } else {
1501             (void)ReleaseOutputBufferOfCodec(info->index, true);
1502             return;
1503         }
1504     }
1505 
1506     if (decodedFrameCount_.load() == processedFrameCount_.load() + 1) {
1507         std::shared_ptr<DecodedBufferInfo> eosInfo{nullptr};
1508         auto eosRet = postProcessingInputBufferInfoQueue_->PopWait(eosInfo);
1509         CHECK_AND_RETURN_LOG(eosRet == QueueResult::OK, "Get data failed, %{public}s",
1510             QUEUE_RESULT_DESCRIPTION[static_cast<int32_t>(eosRet)]);
1511         CHECK_AND_RETURN_LOG(eosInfo && eosInfo->buffer, "Invalid data");
1512         if (eosInfo->buffer->flag_ != AVCODEC_BUFFER_FLAG_EOS) {
1513             AVCODEC_LOGE("The cache info is not EOS frame");
1514             std::this_thread::sleep_for(std::chrono::seconds(1)); // 1: sleep for 1s
1515             return;
1516         }
1517         AVCODEC_LOGD("EOS flag got, frame count = %{public}" PRIu64, decodedFrameCount_.load());
1518         processedFrameCount_++;
1519         std::lock_guard<std::shared_mutex> lock(cbMutex_);
1520         if (videoCb_ == nullptr) {
1521             AVCODEC_LOGD("Missing video callback");
1522         } else {
1523             videoCb_->OnOutputBufferAvailable(std::numeric_limits<uint32_t>::max(), eosInfo->buffer);
1524             return;
1525         }
1526     } else {
1527         std::this_thread::sleep_for(std::chrono::milliseconds(20)); // 20: sleep for 20ms
1528     }
1529 }
1530 
DeactivatePostProcessingQueue()1531 void CodecServer::DeactivatePostProcessingQueue()
1532 {
1533     if (decodedBufferInfoQueue_) {
1534         decodedBufferInfoQueue_->Deactivate();
1535     }
1536     if (postProcessingInputBufferInfoQueue_) {
1537         postProcessingInputBufferInfoQueue_->Deactivate();
1538     }
1539     if (postProcessingOutputBufferInfoQueue_) {
1540         postProcessingOutputBufferInfoQueue_->Deactivate();
1541     }
1542 }
1543 
CleanPostProcessingResource()1544 void CodecServer::CleanPostProcessingResource()
1545 {
1546     ClearCallbackForPostProcessing();
1547     if (postProcessingTask_) {
1548         postProcessingTask_.reset();
1549     }
1550     if (decodedBufferInfoQueue_) {
1551         decodedBufferInfoQueue_.reset();
1552     }
1553     if (postProcessingInputBufferInfoQueue_) {
1554         postProcessingInputBufferInfoQueue_.reset();
1555     }
1556     if (postProcessingOutputBufferInfoQueue_) {
1557         postProcessingOutputBufferInfoQueue_.reset();
1558     }
1559 
1560     decodedFrameCount_.store(0);
1561     processedFrameCount_.store(0);
1562     decoderIsEOS_.store(false);
1563 }
1564 
1565 } // namespace MediaAVCodec
1566 } // namespace OHOS
1567