1 /*
2  * Copyright (c) 2023-2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "surface_encoder_adapter.h"
17 #include <ctime>
18 #include "avcodec_info.h"
19 #include "avcodec_common.h"
20 #include "codec_server.h"
21 #include "meta/format.h"
22 #include "media_description.h"
23 #include "native_avcapability.h"
24 #include "native_avcodec_base.h"
25 #include "avcodec_trace.h"
26 #include "avcodec_sysevent.h"
27 #include "common/log.h"
28 
29 namespace {
30 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "SurfaceEncoderAdapter" };
31 }
32 
33 constexpr uint32_t TIME_OUT_MS = 1000;
34 constexpr uint32_t NS_PER_US = 1000;
35 constexpr int64_t SEC_TO_NS = 1000000000;
36 namespace OHOS {
37 namespace Media {
38 
39 using namespace OHOS::MediaAVCodec;
40 class SurfaceEncoderAdapterCallback : public MediaAVCodec::MediaCodecCallback {
41 public:
SurfaceEncoderAdapterCallback(std::shared_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter)42     explicit SurfaceEncoderAdapterCallback(std::shared_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter)
43         : surfaceEncoderAdapter_(std::move(surfaceEncoderAdapter))
44     {
45     }
46 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)47     void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override
48     {
49         if (auto surfaceEncoderAdapter = surfaceEncoderAdapter_.lock()) {
50             surfaceEncoderAdapter->encoderAdapterCallback_->OnError(errorType, errorCode);
51         } else {
52             MEDIA_LOG_I("invalid surfaceEncoderAdapter");
53         }
54     }
55 
OnOutputFormatChanged(const MediaAVCodec::Format & format)56     void OnOutputFormatChanged(const MediaAVCodec::Format &format) override
57     {
58     }
59 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)60     void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
61     {
62     }
63 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)64     void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
65     {
66         if (auto surfaceEncoderAdapter = surfaceEncoderAdapter_.lock()) {
67             surfaceEncoderAdapter->OnOutputBufferAvailable(index, buffer);
68         } else {
69             MEDIA_LOG_I("invalid surfaceEncoderAdapter");
70         }
71     }
72 
73 private:
74     std::weak_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter_;
75 };
76 
77 class DroppedFramesCallback : public MediaAVCodec::MediaCodecParameterWithAttrCallback {
78 public:
DroppedFramesCallback(std::shared_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter)79     explicit DroppedFramesCallback(std::shared_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter)
80         : surfaceEncoderAdapter_(std::move(surfaceEncoderAdapter))
81     {
82     }
83 
OnInputParameterWithAttrAvailable(uint32_t index,std::shared_ptr<Format> attribute,std::shared_ptr<Format> parameter)84     void OnInputParameterWithAttrAvailable(uint32_t index, std::shared_ptr<Format> attribute,
85         std::shared_ptr<Format> parameter) override
86     {
87         if (auto surfaceEncoderAdapter = surfaceEncoderAdapter_.lock()) {
88             surfaceEncoderAdapter->OnInputParameterWithAttrAvailable(index, attribute, parameter);
89         } else {
90             MEDIA_LOG_I("invalid surfaceEncoderAdapter");
91         }
92     }
93 
94 private:
95     std::weak_ptr<SurfaceEncoderAdapter> surfaceEncoderAdapter_;
96 };
97 
SurfaceEncoderAdapter()98 SurfaceEncoderAdapter::SurfaceEncoderAdapter()
99 {
100     MEDIA_LOG_I("encoder adapter create");
101 }
102 
~SurfaceEncoderAdapter()103 SurfaceEncoderAdapter::~SurfaceEncoderAdapter()
104 {
105     MEDIA_LOG_I("encoder adapter destroy");
106     if (codecServer_) {
107         codecServer_->Release();
108     }
109     codecServer_ = nullptr;
110 }
111 
Init(const std::string & mime,bool isEncoder)112 Status SurfaceEncoderAdapter::Init(const std::string &mime, bool isEncoder)
113 {
114     MEDIA_LOG_I("Init mime: " PUBLIC_LOG_S, mime.c_str());
115     codecMimeType_ = mime;
116     Format format;
117     std::shared_ptr<Media::Meta> callerInfo = std::make_shared<Media::Meta>();
118     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_PID, appPid_);
119     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_UID, appUid_);
120     callerInfo->SetData(Media::Tag::AV_CODEC_FORWARD_CALLER_PROCESS_NAME, bundleName_);
121     format.SetMeta(callerInfo);
122     int32_t ret = MediaAVCodec::VideoEncoderFactory::CreateByMime(mime, format, codecServer_);
123     MEDIA_LOG_I("AVCodecVideoEncoderImpl::Init CreateByMime errorCode %{public}d", ret);
124     if (!codecServer_) {
125         MEDIA_LOG_I("Create codecServer failed");
126         SetFaultEvent("SurfaceEncoderAdapter::Init Create codecServer failed", ret);
127         return Status::ERROR_UNKNOWN;
128     }
129     if (!releaseBufferTask_) {
130         releaseBufferTask_ = std::make_shared<Task>("SurfaceEncoder");
131         releaseBufferTask_->RegisterJob([this] {
132             ReleaseBuffer();
133             return 0;
134         });
135     }
136     return Status::OK;
137 }
138 
ConfigureGeneralFormat(MediaAVCodec::Format & format,const std::shared_ptr<Meta> & meta)139 void SurfaceEncoderAdapter::ConfigureGeneralFormat(MediaAVCodec::Format &format, const std::shared_ptr<Meta> &meta)
140 {
141     MEDIA_LOG_I("ConfigureGeneralFormat");
142     if (meta->Find(Tag::VIDEO_WIDTH) != meta->end()) {
143         int32_t videoWidth;
144         meta->Get<Tag::VIDEO_WIDTH>(videoWidth);
145         format.PutIntValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_WIDTH, videoWidth);
146     }
147     if (meta->Find(Tag::VIDEO_HEIGHT) != meta->end()) {
148         int32_t videoHeight;
149         meta->Get<Tag::VIDEO_HEIGHT>(videoHeight);
150         format.PutIntValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_HEIGHT, videoHeight);
151     }
152     if (meta->Find(Tag::VIDEO_CAPTURE_RATE) != meta->end()) {
153         double videoCaptureRate;
154         meta->Get<Tag::VIDEO_CAPTURE_RATE>(videoCaptureRate);
155         format.PutDoubleValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_CAPTURE_RATE, videoCaptureRate);
156     }
157     if (meta->Find(Tag::MEDIA_BITRATE) != meta->end()) {
158         int64_t mediaBitrate;
159         meta->Get<Tag::MEDIA_BITRATE>(mediaBitrate);
160         format.PutLongValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_BITRATE, mediaBitrate);
161     }
162     if (meta->Find(Tag::VIDEO_FRAME_RATE) != meta->end()) {
163         double videoFrameRate;
164         meta->Get<Tag::VIDEO_FRAME_RATE>(videoFrameRate);
165         format.PutDoubleValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_FRAME_RATE, videoFrameRate);
166     }
167     if (meta->Find(Tag::MIME_TYPE) != meta->end()) {
168         std::string mimeType;
169         meta->Get<Tag::MIME_TYPE>(mimeType);
170         format.PutStringValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_CODEC_MIME, mimeType);
171     }
172     if (meta->Find(Tag::VIDEO_H265_PROFILE) != meta->end()) {
173         Plugins::HEVCProfile h265Profile;
174         meta->Get<Tag::VIDEO_H265_PROFILE>(h265Profile);
175         format.PutIntValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_PROFILE, h265Profile);
176     }
177 }
178 
ConfigureEnableFormat(MediaAVCodec::Format & format,const std::shared_ptr<Meta> & meta)179 void SurfaceEncoderAdapter::ConfigureEnableFormat(MediaAVCodec::Format &format, const std::shared_ptr<Meta> &meta)
180 {
181     MEDIA_LOG_I("ConfigureEnableFormat");
182     if (meta->Find(Tag::VIDEO_ENCODER_ENABLE_WATERMARK) != meta->end()) {
183         bool enableWatermark = false;
184         meta->Get<Tag::VIDEO_ENCODER_ENABLE_WATERMARK>(enableWatermark);
185         format.PutIntValue(Tag::VIDEO_ENCODER_ENABLE_WATERMARK, enableWatermark);
186     }
187 }
188 
Configure(const std::shared_ptr<Meta> & meta)189 Status SurfaceEncoderAdapter::Configure(const std::shared_ptr<Meta> &meta)
190 {
191     MEDIA_LOG_I("Configure");
192     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Configure");
193     MediaAVCodec::Format format = MediaAVCodec::Format();
194     ConfigureGeneralFormat(format, meta);
195     ConfigureAboutRGBA(format, meta);
196     ConfigureAboutEnableTemporalScale(format, meta);
197     ConfigureEnableFormat(format, meta);
198     if (!codecServer_) {
199         SetFaultEvent("SurfaceEncoderAdapter::Configure, CodecServer is null");
200         return Status::ERROR_UNKNOWN;
201     }
202     int32_t ret = static_cast<int32_t>(Status::OK);
203     if (!isTransCoderMode) {
204         std::shared_ptr<MediaAVCodec::MediaCodecParameterWithAttrCallback> droppedFramesCallback =
205         std::make_shared<DroppedFramesCallback>(shared_from_this());
206         ret = codecServer_->SetCallback(droppedFramesCallback);
207         if (ret != 0) {
208             MEDIA_LOG_I("Set dropped Frames Callback failed");
209             SetFaultEvent("DroppedFramesCallback::DroppedFramesCallback error", ret);
210             return Status::ERROR_UNKNOWN;
211         }
212     }
213     if (isTransCoderMode) {
214         format.PutIntValue(Tag::VIDEO_FRAME_RATE_ADAPTIVE_MODE, true);
215     }
216     ret = codecServer_->Configure(format);
217     if (ret != 0) {
218         SetFaultEvent("SurfaceEncoderAdapter::Configure error", ret);
219     }
220     return ret == 0 ? Status::OK : Status::ERROR_UNKNOWN;
221 }
222 
SetWatermark(std::shared_ptr<AVBuffer> & waterMarkBuffer)223 Status SurfaceEncoderAdapter::SetWatermark(std::shared_ptr<AVBuffer> &waterMarkBuffer)
224 {
225     MEDIA_LOG_I("SetWaterMark");
226     if (!codecServer_) {
227         MEDIA_LOG_I("CodecServer is null");
228         SetFaultEvent("SurfaceEncoderAdapter::setWatermark, CodecServer is null");
229         return Status::ERROR_UNKNOWN;
230     }
231     int ret = codecServer_->SetCustomBuffer(waterMarkBuffer);
232     if (ret != 0) {
233         MEDIA_LOG_E("SetCustomBuffer error");
234         return Status::ERROR_UNKNOWN;
235     }
236     return Status::OK;
237 }
238 
SetOutputBufferQueue(const sptr<AVBufferQueueProducer> & bufferQueueProducer)239 Status SurfaceEncoderAdapter::SetOutputBufferQueue(const sptr<AVBufferQueueProducer> &bufferQueueProducer)
240 {
241     MEDIA_LOG_I("SetOutputBufferQueue");
242     outputBufferQueueProducer_ = bufferQueueProducer;
243     return Status::OK;
244 }
245 
SetEncoderAdapterCallback(const std::shared_ptr<EncoderAdapterCallback> & encoderAdapterCallback)246 Status SurfaceEncoderAdapter::SetEncoderAdapterCallback(
247     const std::shared_ptr<EncoderAdapterCallback> &encoderAdapterCallback)
248 {
249     MEDIA_LOG_I("SetEncoderAdapterCallback");
250     std::shared_ptr<MediaAVCodec::MediaCodecCallback> surfaceEncoderAdapterCallback =
251         std::make_shared<SurfaceEncoderAdapterCallback>(shared_from_this());
252     encoderAdapterCallback_ = encoderAdapterCallback;
253     if (!codecServer_) {
254         SetFaultEvent("SurfaceEncoderAdapter::SetEncoderAdapterCallback, CodecServer is null");
255         return Status::ERROR_UNKNOWN;
256     }
257     int32_t ret = codecServer_->SetCallback(surfaceEncoderAdapterCallback);
258     if (ret == 0) {
259         return Status::OK;
260     } else {
261         SetFaultEvent("SurfaceEncoderAdapter::SetEncoderAdapterCallback error", ret);
262         return Status::ERROR_UNKNOWN;
263     }
264 }
265 
SetEncoderAdapterKeyFramePtsCallback(const std::shared_ptr<EncoderAdapterKeyFramePtsCallback> & encoderAdapterKeyFramePtsCallback)266 Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback(
267     const std::shared_ptr<EncoderAdapterKeyFramePtsCallback> &encoderAdapterKeyFramePtsCallback)
268 {
269     MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback");
270     encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback;
271     return Status::OK;
272 }
273 
SetInputSurface(sptr<Surface> surface)274 Status SurfaceEncoderAdapter::SetInputSurface(sptr<Surface> surface)
275 {
276     MEDIA_LOG_I("GetInputSurface");
277     if (!codecServer_) {
278         SetFaultEvent("SurfaceEncoderAdapter::SetInputSurface, CodecServer is null");
279         return Status::ERROR_UNKNOWN;
280     }
281     MediaAVCodec::CodecServer *codecServerPtr = (MediaAVCodec::CodecServer *)(codecServer_.get());
282     int32_t ret = codecServerPtr->SetInputSurface(surface);
283     if (ret == 0) {
284         return Status::OK;
285     } else {
286         SetFaultEvent("SurfaceEncoderAdapter::SetInputSurface error", ret);
287         return Status::ERROR_UNKNOWN;
288     }
289 }
290 
SetTransCoderMode()291 Status SurfaceEncoderAdapter::SetTransCoderMode()
292 {
293     MEDIA_LOG_I("SetTransCoderMode");
294     isTransCoderMode = true;
295     return Status::OK;
296 }
297 
GetInputSurface()298 sptr<Surface> SurfaceEncoderAdapter::GetInputSurface()
299 {
300     return codecServer_->CreateInputSurface();
301 }
302 
Start()303 Status SurfaceEncoderAdapter::Start()
304 {
305     MEDIA_LOG_I("Start");
306     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Start");
307     if (!codecServer_) {
308         SetFaultEvent("SurfaceEncoderAdapter::Start, CodecServer is null");
309         return Status::ERROR_UNKNOWN;
310     }
311     int32_t ret;
312     isThreadExit_ = false;
313     if (releaseBufferTask_) {
314         releaseBufferTask_->Start();
315     }
316     ret = codecServer_->Start();
317     isStart_ = true;
318     isStartKeyFramePts_ = true;
319     if (ret == 0) {
320         return Status::OK;
321     } else {
322         SetFaultEvent("SurfaceEncoderAdapter::Start error", ret);
323         return Status::ERROR_UNKNOWN;
324     }
325 }
326 
Stop()327 Status SurfaceEncoderAdapter::Stop()
328 {
329     MEDIA_LOG_I("Stop");
330     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Stop");
331     GetCurrentTime(stopTime_);
332     isStopKeyFramePts_ = true;
333     MEDIA_LOG_I("Stop time: " PUBLIC_LOG_D64, stopTime_);
334 
335     if (isStart_ && !isTransCoderMode) {
336         std::unique_lock<std::mutex> lock(stopMutex_);
337         stopCondition_.wait_for(lock, std::chrono::milliseconds(TIME_OUT_MS));
338         AddStopPts();
339     }
340     if (releaseBufferTask_) {
341         isThreadExit_ = true;
342         releaseBufferCondition_.notify_all();
343         releaseBufferTask_->Stop();
344         MEDIA_LOG_I("releaseBufferTask_ Stop");
345     }
346     if (!codecServer_) {
347         return Status::OK;
348     }
349     int32_t ret = codecServer_->Stop();
350     MEDIA_LOG_I("codecServer_ Stop");
351     isStart_ = false;
352     if (ret == 0) {
353         return Status::OK;
354     } else {
355         SetFaultEvent("SurfaceEncoderAdapter::Stop error", ret);
356         return Status::ERROR_UNKNOWN;
357     }
358 }
359 
Pause()360 Status SurfaceEncoderAdapter::Pause()
361 {
362     MEDIA_LOG_I("Pause");
363     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Pause");
364     if (isTransCoderMode) {
365         return Status::OK;
366     }
367     std::lock_guard<std::mutex> lock(checkFramesMutex_);
368     int64_t pauseTime = 0;
369     GetCurrentTime(pauseTime);
370     MEDIA_LOG_I("Pause time: " PUBLIC_LOG_D64, pauseTime);
371     if (pauseResumeQueue_.empty() ||
372         (pauseResumeQueue_.back().second == StateCode::RESUME && pauseResumeQueue_.back().first <= pauseTime)) {
373         pauseResumeQueue_.push_back({pauseTime, StateCode::PAUSE});
374         pauseResumeQueue_.push_back({std::numeric_limits<int64_t>::max(), StateCode::RESUME});
375         pauseResumePts_.push_back({pauseTime, StateCode::PAUSE});
376         pauseResumePts_.push_back({std::numeric_limits<int64_t>::max(), StateCode::RESUME});
377     }
378     return Status::OK;
379 }
380 
Resume()381 Status SurfaceEncoderAdapter::Resume()
382 {
383     MEDIA_LOG_I("Resume");
384     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Resume");
385     if (isTransCoderMode) {
386         isResume_ = true;
387         return Status::OK;
388     }
389     std::lock_guard<std::mutex> lock(checkFramesMutex_);
390     int64_t resumeTime = 0;
391     GetCurrentTime(resumeTime);
392     MEDIA_LOG_I("resume time: " PUBLIC_LOG_D64, resumeTime);
393     if (pauseResumeQueue_.empty()) {
394         MEDIA_LOG_I("Status Error, no pause before resume");
395         return Status::ERROR_UNKNOWN;
396     }
397     if (pauseResumeQueue_.back().second == StateCode::RESUME) {
398         pauseResumeQueue_.back().first = std::min(resumeTime, pauseResumeQueue_.back().first);
399         pauseResumePts_.back().first = std::min(resumeTime, pauseResumePts_.back().first);
400     }
401     return Status::OK;
402 }
403 
Flush()404 Status SurfaceEncoderAdapter::Flush()
405 {
406     MEDIA_LOG_I("Flush");
407     if (!codecServer_) {
408         SetFaultEvent("SurfaceEncoderAdapter::Flush, CodecServer is null");
409         return Status::ERROR_UNKNOWN;
410     }
411     int32_t ret = codecServer_->Flush();
412     if (ret == 0) {
413         return Status::OK;
414     } else {
415         SetFaultEvent("SurfaceEncoderAdapter::Flush error", ret);
416         return Status::ERROR_UNKNOWN;
417     }
418 }
419 
Reset()420 Status SurfaceEncoderAdapter::Reset()
421 {
422     MEDIA_LOG_I("Reset");
423     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Reset");
424     if (!codecServer_) {
425         return Status::OK;
426     }
427     int32_t ret = codecServer_->Reset();
428     startBufferTime_ = -1;
429     stopTime_ = -1;
430     totalPauseTime_ = 0;
431     isStart_ = false;
432     isStartKeyFramePts_ = false;
433     mappingTimeQueue_.clear();
434     pauseResumeQueue_.clear();
435     pauseResumePts_.clear();
436     if (ret == 0) {
437         return Status::OK;
438     } else {
439         SetFaultEvent("SurfaceEncoderAdapter::Reset error", ret);
440         return Status::ERROR_UNKNOWN;
441     }
442 }
443 
Release()444 Status SurfaceEncoderAdapter::Release()
445 {
446     MEDIA_LOG_I("Release");
447     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Release");
448     if (!codecServer_) {
449         return Status::OK;
450     }
451     int32_t ret = codecServer_->Release();
452     if (ret == 0) {
453         return Status::OK;
454     } else {
455         SetFaultEvent("SurfaceEncoderAdapter::Release error", ret);
456         return Status::ERROR_UNKNOWN;
457     }
458 }
459 
NotifyEos(int64_t pts)460 Status SurfaceEncoderAdapter::NotifyEos(int64_t pts)
461 {
462     MEDIA_LOG_I("NotifyEos");
463     if (!codecServer_) {
464         SetFaultEvent("SurfaceEncoderAdapter::NotifyEos, CodecServer is null");
465         return Status::ERROR_UNKNOWN;
466     }
467     int32_t ret = 0;
468     MEDIA_LOG_I("lastBuffer PTS: " PUBLIC_LOG_D64, pts);
469     eosPts_ = pts;
470     if (!isTransCoderMode || currentPts_.load() >= eosPts_.load()) {
471         ret = codecServer_->NotifyEos();
472     }
473     if (ret == 0) {
474         return Status::OK;
475     } else {
476         SetFaultEvent("SurfaceEncoderAdapter::NotifyEos error", ret);
477         return Status::ERROR_UNKNOWN;
478     }
479 }
480 
SetParameter(const std::shared_ptr<Meta> & parameter)481 Status SurfaceEncoderAdapter::SetParameter(const std::shared_ptr<Meta> &parameter)
482 {
483     MEDIA_LOG_I("SetParameter");
484     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::SetParameter");
485     if (!codecServer_) {
486         SetFaultEvent("SurfaceEncoderAdapter::SetParameter, CodecServer is null");
487         return Status::ERROR_UNKNOWN;
488     }
489     MediaAVCodec::Format format = MediaAVCodec::Format();
490     int32_t ret = codecServer_->SetParameter(format);
491     if (ret == 0) {
492         return Status::OK;
493     } else {
494         SetFaultEvent("SurfaceEncoderAdapter::SetParameter error", ret);
495         return Status::ERROR_UNKNOWN;
496     }
497 }
498 
GetOutputFormat()499 std::shared_ptr<Meta> SurfaceEncoderAdapter::GetOutputFormat()
500 {
501     MEDIA_LOG_I("GetOutputFormat is not supported");
502     return nullptr;
503 }
504 
TransCoderOnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)505 void SurfaceEncoderAdapter::TransCoderOnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
506 {
507     if (buffer->pts_ >= eosPts_.load() && codecServer_) {
508         codecServer_->NotifyEos();
509     }
510     if (stopTime_ != -1 && buffer->pts_ > stopTime_) {
511         MEDIA_LOG_I("buffer->pts > stopTime, ready to stop");
512         std::unique_lock<std::mutex> lock(stopMutex_);
513         stopCondition_.notify_all();
514     }
515     if (startBufferTime_ == -1 && buffer->pts_ != 0) {
516         startBufferTime_ = buffer->pts_;
517     }
518 
519     int32_t size = buffer->memory_->GetSize();
520     std::shared_ptr<AVBuffer> emptyOutputBuffer;
521     AVBufferConfig avBufferConfig;
522     avBufferConfig.size = size;
523     avBufferConfig.memoryType = MemoryType::SHARED_MEMORY;
524     avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE;
525     Status status = outputBufferQueueProducer_->RequestBuffer(emptyOutputBuffer, avBufferConfig, TIME_OUT_MS);
526     if (status != Status::OK) {
527         MEDIA_LOG_I("RequestBuffer fail.");
528         return;
529     }
530     std::shared_ptr<AVMemory> &bufferMem = emptyOutputBuffer->memory_;
531     if (emptyOutputBuffer->memory_ == nullptr) {
532         MEDIA_LOG_I("emptyOutputBuffer->memory_ is nullptr");
533         return;
534     }
535     bufferMem->Write(buffer->memory_->GetAddr(), size, 0);
536     *(emptyOutputBuffer->meta_) = *(buffer->meta_);
537     if (isResume_) {
538         const int64_t MS_TO_NS = 1000000;
539         totalPauseTime_ = totalPauseTime_ + buffer->pts_ - lastBufferTime_ - MS_TO_NS;
540         isResume_ = false;
541     }
542     lastBufferTime_ = buffer->pts_;
543     emptyOutputBuffer->pts_ = buffer->pts_ - startBufferTime_ - totalPauseTime_;
544     if (!isTransCoderMode) {
545         emptyOutputBuffer->pts_ = emptyOutputBuffer->pts_ / NS_PER_US;
546     }
547     emptyOutputBuffer->flag_ = buffer->flag_;
548     outputBufferQueueProducer_->PushBuffer(emptyOutputBuffer, true);
549     {
550         std::lock_guard<std::mutex> lock(releaseBufferMutex_);
551         indexs_.push_back(index);
552     }
553     releaseBufferCondition_.notify_all();
554     MEDIA_LOG_D("OnOutputBufferAvailable end");
555 }
556 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)557 void SurfaceEncoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
558 {
559     MEDIA_LOG_D("OnOutputBufferAvailable buffer->pts" PUBLIC_LOG_D64, buffer->pts_);
560     currentPts_ = currentPts_.load() < buffer->pts_? buffer->pts_ : currentPts_.load();
561     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::OnOutputBufferAvailable");
562     if (isTransCoderMode) {
563         TransCoderOnOutputBufferAvailable(index, buffer);
564         return;
565     }
566     if (stopTime_ != -1 && buffer->pts_ > stopTime_) {
567         MEDIA_LOG_I("buffer->pts > stopTime, ready to stop");
568         std::unique_lock<std::mutex> lock(stopMutex_);
569         stopCondition_.notify_all();
570     }
571 
572     int64_t mappingTime = -1;
573     if (!(buffer->flag_ & AVCODEC_BUFFER_FLAG_CODEC_DATA)) {
574         std::lock_guard<std::mutex> mappingLock(mappingPtsMutex_);
575         if (mappingTimeQueue_.empty() || mappingTimeQueue_.front().first != buffer->pts_) {
576             MEDIA_LOG_D("buffer->pts fail");
577         } else {
578             mappingTime = mappingTimeQueue_.front().second;
579             mappingTimeQueue_.pop_front();
580         }
581         if (startBufferTime_ == -1) {
582             startBufferTime_ = buffer->pts_;
583         }
584         // cache recent 2 pts
585         preKeyFramePts_ = currentKeyFramePts_;
586         currentKeyFramePts_ = buffer->pts_;
587         AddStartPts(buffer->pts_);
588         AddPauseResumePts(buffer->pts_);
589     } else {
590         MEDIA_LOG_D("OnOutputBufferAvailable buffer->flag_" PUBLIC_LOG_U32, buffer->flag_);
591         mappingTime = startBufferTime_ + buffer->pts_;
592     }
593     int32_t size = buffer->memory_->GetSize();
594     std::shared_ptr<AVBuffer> emptyOutputBuffer;
595     AVBufferConfig avBufferConfig;
596     avBufferConfig.size = size;
597     avBufferConfig.memoryType = MemoryType::SHARED_MEMORY;
598     avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE;
599     Status status = outputBufferQueueProducer_->RequestBuffer(emptyOutputBuffer, avBufferConfig, TIME_OUT_MS);
600     FALSE_RETURN_MSG(status == Status::OK, "RequestBuffer fail.");
601     std::shared_ptr<AVMemory> &bufferMem = emptyOutputBuffer->memory_;
602     FALSE_RETURN_MSG(emptyOutputBuffer->memory_ != nullptr, "emptyOutputBuffer->memory_ is nullptr");
603     bufferMem->Write(buffer->memory_->GetAddr(), size, 0);
604     *(emptyOutputBuffer->meta_) = *(buffer->meta_);
605     emptyOutputBuffer->pts_ = (mappingTime - startBufferTime_) / NS_PER_US;
606     emptyOutputBuffer->flag_ = buffer->flag_;
607     outputBufferQueueProducer_->PushBuffer(emptyOutputBuffer, true);
608     {
609         std::lock_guard<std::mutex> lock(releaseBufferMutex_);
610         indexs_.push_back(index);
611     }
612     releaseBufferCondition_.notify_all();
613 }
614 
ReleaseBuffer()615 void SurfaceEncoderAdapter::ReleaseBuffer()
616 {
617     MEDIA_LOG_I("ReleaseBuffer");
618     while (true) {
619         if (isThreadExit_) {
620             MEDIA_LOG_I("Exit ReleaseBuffer thread.");
621             break;
622         }
623         std::vector<uint32_t> indexs;
624         {
625             std::unique_lock<std::mutex> lock(releaseBufferMutex_);
626             releaseBufferCondition_.wait(lock, [this] {
627                 return isThreadExit_ || !indexs_.empty();
628             });
629             indexs = indexs_;
630             indexs_.clear();
631         }
632         for (auto &index : indexs) {
633             codecServer_->ReleaseOutputBuffer(index);
634         }
635     }
636     MEDIA_LOG_I("ReleaseBuffer end");
637 }
638 
ConfigureAboutRGBA(MediaAVCodec::Format & format,const std::shared_ptr<Meta> & meta)639 void SurfaceEncoderAdapter::ConfigureAboutRGBA(MediaAVCodec::Format &format, const std::shared_ptr<Meta> &meta)
640 {
641     Plugins::VideoPixelFormat pixelFormat = Plugins::VideoPixelFormat::NV12;
642     if (meta->Find(Tag::VIDEO_PIXEL_FORMAT) != meta->end()) {
643         meta->Get<Tag::VIDEO_PIXEL_FORMAT>(pixelFormat);
644     }
645     format.PutIntValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, static_cast<int32_t>(pixelFormat));
646 
647     if (meta->Find(Tag::VIDEO_ENCODE_BITRATE_MODE) != meta->end()) {
648         Plugins::VideoEncodeBitrateMode videoEncodeBitrateMode;
649         meta->Get<Tag::VIDEO_ENCODE_BITRATE_MODE>(videoEncodeBitrateMode);
650         format.PutIntValue(MediaAVCodec::MediaDescriptionKey::MD_KEY_VIDEO_ENCODE_BITRATE_MODE, videoEncodeBitrateMode);
651     }
652 }
653 
ConfigureAboutEnableTemporalScale(MediaAVCodec::Format & format,const std::shared_ptr<Meta> & meta)654 void SurfaceEncoderAdapter::ConfigureAboutEnableTemporalScale(MediaAVCodec::Format &format,
655     const std::shared_ptr<Meta> &meta)
656 {
657     if (meta->Find(Tag::VIDEO_ENCODER_ENABLE_TEMPORAL_SCALABILITY) != meta->end()) {
658         bool enableTemporalScale;
659         meta->Get<Tag::VIDEO_ENCODER_ENABLE_TEMPORAL_SCALABILITY>(enableTemporalScale);
660         if (!enableTemporalScale) {
661             MEDIA_LOG_I("video encoder enableTemporalScale is false!");
662             return;
663         }
664 
665         bool isSupported = true;
666         if (isSupported) {
667             MEDIA_LOG_I("VIDEO_ENCODER_TEMPORAL_SCALABILITY is supported!");
668             format.PutIntValue(MediaAVCodec::MediaDescriptionKey::OH_MD_KEY_VIDEO_ENCODER_ENABLE_TEMPORAL_SCALABILITY,
669                 1);
670         } else {
671             MEDIA_LOG_I("VIDEO_ENCODER_TEMPORAL_SCALABILITY is not supported!");
672         }
673     }
674 }
675 
SetFaultEvent(const std::string & errMsg,int32_t ret)676 void SurfaceEncoderAdapter::SetFaultEvent(const std::string &errMsg, int32_t ret)
677 {
678     SetFaultEvent(errMsg + ", ret = " + std::to_string(ret));
679 }
680 
SetFaultEvent(const std::string & errMsg)681 void SurfaceEncoderAdapter::SetFaultEvent(const std::string &errMsg)
682 {
683     VideoCodecFaultInfo videoCodecFaultInfo;
684     videoCodecFaultInfo.appName = bundleName_;
685     videoCodecFaultInfo.instanceId = std::to_string(instanceId_);
686     videoCodecFaultInfo.callerType = "player_framework";
687     videoCodecFaultInfo.videoCodec = codecMimeType_;
688     videoCodecFaultInfo.errMsg = errMsg;
689     FaultVideoCodecEventWrite(videoCodecFaultInfo);
690 }
691 
SetCallingInfo(int32_t appUid,int32_t appPid,const std::string & bundleName,uint64_t instanceId)692 void SurfaceEncoderAdapter::SetCallingInfo(int32_t appUid, int32_t appPid,
693     const std::string &bundleName, uint64_t instanceId)
694 {
695     appUid_ = appUid;
696     appPid_ = appPid;
697     bundleName_ = bundleName;
698     instanceId_ = instanceId;
699 }
700 
OnInputParameterWithAttrAvailable(uint32_t index,std::shared_ptr<Format> & attribute,std::shared_ptr<Format> & parameter)701 void SurfaceEncoderAdapter::OnInputParameterWithAttrAvailable(uint32_t index, std::shared_ptr<Format> &attribute,
702     std::shared_ptr<Format> &parameter)
703 {
704     MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::OnInputParameterWithAttrAvailable");
705     if (isTransCoderMode) {
706         MEDIA_LOG_D("isTransCoderMode");
707         parameter->PutIntValue(Tag::VIDEO_ENCODER_PER_FRAME_DISCARD, false);
708         codecServer_->QueueInputParameter(index);
709         return;
710     }
711     std::lock_guard<std::mutex> lock(checkFramesMutex_);
712     int64_t currentPts = 0;
713     attribute->GetLongValue(Tag::MEDIA_TIME_STAMP, currentPts);
714     MEDIA_LOG_D("OnInputParameterWithAttrAvailable currentPts " PUBLIC_LOG_D64, currentPts);
715     int64_t checkFramesPauseTime = 0;
716     bool isDroppedFrames = CheckFrames(currentPts, checkFramesPauseTime);
717     {
718         std::lock_guard<std::mutex> mappingLock(mappingPtsMutex_);
719         if (isDroppedFrames) {
720             totalPauseTime_ = totalPauseTime_ + currentPts - lastBufferTime_;
721         } else {
722             int64_t frameDifference = 1000000; // Frame Difference less 1000000 ns
723             if (checkFramesPauseTime + frameDifference < currentPts - lastBufferTime_) {
724                 totalPauseTime_ = totalPauseTime_ + checkFramesPauseTime;
725             }
726             mappingTimeQueue_.push_back({currentPts, currentPts - totalPauseTime_});
727         }
728         lastBufferTime_ = currentPts;
729     }
730     parameter->PutIntValue(Tag::VIDEO_ENCODER_PER_FRAME_DISCARD, isDroppedFrames);
731     codecServer_->QueueInputParameter(index);
732 }
733 
CheckFrames(int64_t currentPts,int64_t & checkFramesPauseTime)734 bool SurfaceEncoderAdapter::CheckFrames(int64_t currentPts, int64_t &checkFramesPauseTime)
735 {
736     if (pauseResumeQueue_.empty()) {
737         return false;
738     }
739     auto stateCode = pauseResumeQueue_[0].second;
740     MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32
741         " time:" PUBLIC_LOG_D64, static_cast<int32_t>(stateCode), pauseResumeQueue_[0].first);
742     // means not dropped frames when less than pause time
743     if (stateCode == StateCode::PAUSE && currentPts < pauseResumeQueue_[0].first) {
744         return false;
745     }
746     // means dropped frames when less than resume time
747     if (stateCode == StateCode::RESUME && currentPts < pauseResumeQueue_[0].first) {
748         return true;
749     }
750     if (stateCode == StateCode::PAUSE) {
751         checkFramesPauseTime -= (pauseResumeQueue_[0].first - lastBufferTime_);
752     } else {
753         checkFramesPauseTime += (pauseResumeQueue_[0].first - lastBufferTime_);
754     }
755     pauseResumeQueue_.pop_front();
756     return CheckFrames(currentPts, checkFramesPauseTime);
757 }
758 
GetCurrentTime(int64_t & currentTime)759 void SurfaceEncoderAdapter::GetCurrentTime(int64_t &currentTime)
760 {
761     struct timespec timestamp = {0, 0};
762     clock_gettime(CLOCK_MONOTONIC, &timestamp);
763     currentTime = static_cast<int64_t>(timestamp.tv_sec) * SEC_TO_NS + static_cast<int64_t>(timestamp.tv_nsec);
764 }
765 
AddStartPts(int64_t currentPts)766 void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts)
767 {
768     // start time
769     if (isStartKeyFramePts_) {
770         keyFramePts_ += std::to_string(currentPts / NS_PER_US) + ",";
771         isStartKeyFramePts_ = false;
772         MEDIA_LOG_I("AddStartPts success %{public}s end", keyFramePts_.c_str());
773     }
774 }
775 
AddStopPts()776 void SurfaceEncoderAdapter::AddStopPts()
777 {
778     // stop time
779     MEDIA_LOG_D("AddStopPts enter");
780     if (isStopKeyFramePts_) {
781         if (currentKeyFramePts_ > stopTime_) {
782             keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US);
783             MEDIA_LOG_I("AddStopPts preKeyFramePts_ %{public}s end", keyFramePts_.c_str());
784         } else {
785             keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US);
786             MEDIA_LOG_I("AddStopPts currentKeyFramePts_ %{public}s end", keyFramePts_.c_str());
787         }
788         isStopKeyFramePts_ = false;
789         encoderAdapterKeyFramePtsCallback_->OnReportKeyFramePts(keyFramePts_);
790         keyFramePts_.clear();
791     }
792 }
793 
AddPauseResumePts(int64_t currentPts)794 bool SurfaceEncoderAdapter::AddPauseResumePts(int64_t currentPts)
795 {
796     if (pauseResumePts_.empty()) {
797         return false;
798     }
799     auto stateCode = pauseResumePts_[0].second;
800     MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32
801         " time:" PUBLIC_LOG_D64, static_cast<int32_t>(stateCode), pauseResumePts_[0].first);
802     // means not dropped frames when less than pause time
803     if (stateCode == StateCode::PAUSE && currentPts < pauseResumePts_[0].first) {
804         return false;
805     }
806     // means dropped frames when less than resume time
807     if (stateCode == StateCode::RESUME && currentPts < pauseResumePts_[0].first) {
808         return true;
809     }
810     if (stateCode == StateCode::PAUSE) {
811         MEDIA_LOG_D("AddPausePts %{public}s start", keyFramePts_.c_str());
812         keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US) + ",";
813         MEDIA_LOG_D("AddPausePts %{public}s end", keyFramePts_.c_str());
814     }
815     if (stateCode == StateCode::RESUME) {
816         MEDIA_LOG_D("AddResumePts %{public}s start", keyFramePts_.c_str());
817         keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US) + ",";
818         MEDIA_LOG_D("AddResumePts %{public}s end", keyFramePts_.c_str());
819     }
820     pauseResumePts_.pop_front();
821     return AddPauseResumePts(currentPts);
822 }
823 } // namespace MEDIA
824 } // namespace OHOS