1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "av_thumbnail_generator.h"
17 
18 #include "buffer/avbuffer_common.h"
19 #include "common/media_source.h"
20 #include "ibuffer_consumer_listener.h"
21 #include "graphic_common_c.h"
22 #include "media_errors.h"
23 #include "media_log.h"
24 #include "media_description.h"
25 #include "meta/meta.h"
26 #include "meta/meta_key.h"
27 #include "plugin/plugin_time.h"
28 #include "sync_fence.h"
29 #include "uri_helper.h"
30 
31 #include "v1_0/cm_color_space.h"
32 #include "v1_0/hdr_static_metadata.h"
33 #include "v1_0/buffer_handle_meta_key_type.h"
34 
35 namespace {
36 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_METADATA, "AVThumbnailGenerator" };
37 }
38 
39 namespace OHOS {
40 namespace Media {
41 using namespace OHOS::HDI::Display::Graphic::Common::V1_0;
42 constexpr float BYTES_PER_PIXEL_YUV = 1.5;
43 constexpr int32_t RATE_UV = 2;
44 constexpr int32_t SHIFT_BITS_P010_2_NV12 = 8;
45 constexpr double VIDEO_FRAME_RATE = 2000.0;
46 constexpr int32_t MAX_WAIT_TIME_SECOND = 3;
47 
48 class ThumnGeneratorCodecCallback : public OHOS::MediaAVCodec::MediaCodecCallback {
49 public:
ThumnGeneratorCodecCallback(AVThumbnailGenerator * generator)50     explicit ThumnGeneratorCodecCallback(AVThumbnailGenerator *generator) : generator_(generator) {}
51 
52     ~ThumnGeneratorCodecCallback() = default;
53 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)54     void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override
55     {
56         generator_->OnError(errorType, errorCode);
57     }
58 
OnOutputFormatChanged(const MediaAVCodec::Format & format)59     void OnOutputFormatChanged(const MediaAVCodec::Format &format) override
60     {
61         generator_->OnOutputFormatChanged(format);
62     }
63 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)64     void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
65     {
66         generator_->OnInputBufferAvailable(index, buffer);
67     }
68 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)69     void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
70     {
71         generator_->OnOutputBufferAvailable(index, buffer);
72     }
73 
74 private:
75     AVThumbnailGenerator *generator_;
76 };
77 
AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> & mediaDemuxer)78 AVThumbnailGenerator::AVThumbnailGenerator(std::shared_ptr<MediaDemuxer> &mediaDemuxer) : mediaDemuxer_(mediaDemuxer)
79 {
80     MEDIA_LOGI("Constructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
81 }
82 
~AVThumbnailGenerator()83 AVThumbnailGenerator::~AVThumbnailGenerator()
84 {
85     MEDIA_LOGI("Destructor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
86     Destroy();
87 }
88 
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)89 void AVThumbnailGenerator::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
90 {
91     MEDIA_LOGE("OnError errorType:%{public}d, errorCode:%{public}d", static_cast<int32_t>(errorType), errorCode);
92     stopProcessing_ = true;
93 }
94 
InitDecoder()95 Status AVThumbnailGenerator::InitDecoder()
96 {
97     MEDIA_LOGD("Init decoder start.");
98     if (videoDecoder_ != nullptr) {
99         MEDIA_LOGD("AVThumbnailGenerator InitDecoder already.");
100         Format format;
101         format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
102         videoDecoder_->SetParameter(format);
103         videoDecoder_->Start();
104         return Status::OK;
105     }
106     videoDecoder_ = MediaAVCodec::VideoDecoderFactory::CreateByMime(trackMime_);
107     CHECK_AND_RETURN_RET_LOG(videoDecoder_ != nullptr, Status::ERROR_NO_MEMORY, "Create videoDecoder_ is nullptr");
108     Format trackFormat{};
109     trackFormat.SetMeta(GetVideoTrackInfo());
110     trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width_);
111     trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height_);
112     MEDIA_LOGI("0x%{public}06" PRIXPTR " Init decoder trackFormat width:%{public}d, height:%{public}d",
113                FAKE_POINTER(this), width_, height_);
114     trackFormat.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT,
115                             static_cast<int32_t>(Plugins::VideoPixelFormat::NV12));
116     trackFormat.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
117     videoDecoder_->Configure(trackFormat);
118     std::shared_ptr<MediaAVCodec::MediaCodecCallback> mediaCodecCallback =
119         std::make_shared<ThumnGeneratorCodecCallback>(this);
120     videoDecoder_->SetCallback(mediaCodecCallback);
121     videoDecoder_->Prepare();
122     auto res = videoDecoder_->Start();
123     CHECK_AND_RETURN_RET(res == MSERR_OK, Status::ERROR_WRONG_STATE);
124     return Status::OK;
125 }
126 
GetVideoTrackInfo()127 std::shared_ptr<Meta> AVThumbnailGenerator::GetVideoTrackInfo()
128 {
129     CHECK_AND_RETURN_RET(trackInfo_ == nullptr, trackInfo_);
130     CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "GetTargetTrackInfo demuxer is nullptr");
131     std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
132     size_t trackCount = trackInfos.size();
133     CHECK_AND_RETURN_RET_LOG(trackCount > 0, nullptr, "GetTargetTrackInfo trackCount is invalid");
134     for (size_t index = 0; index < trackCount; index++) {
135         if (!(trackInfos[index]->GetData(Tag::MIME_TYPE, trackMime_))) {
136             MEDIA_LOGW("GetTargetTrackInfo get mime type failed %{public}s", trackMime_.c_str());
137             continue;
138         }
139         if (trackMime_.find("video/") == 0) {
140             Plugins::MediaType mediaType;
141             CHECK_AND_RETURN_RET_LOG(trackInfos[index]->GetData(Tag::MEDIA_TYPE, mediaType), nullptr,
142                                      "GetTargetTrackInfo failed to get mediaType, index:%{public}d", index);
143             CHECK_AND_RETURN_RET_LOG(
144                 mediaType == Plugins::MediaType::VIDEO, nullptr,
145                 "GetTargetTrackInfo mediaType is not video, index:%{public}d, mediaType:%{public}d", index,
146                 static_cast<int32_t>(mediaType));
147             CHECK_AND_RETURN_RET_LOG(trackInfos[index]->Get<Tag::VIDEO_FRAME_RATE>(frameRate_) && frameRate_ > 0,
148                 nullptr, "failed to get video frame rate");
149             trackIndex_ = index;
150             MEDIA_LOGI("0x%{public}06" PRIXPTR " GetTrackInfo success trackIndex_:%{public}d, trackMime_:%{public}s",
151                        FAKE_POINTER(this), trackIndex_, trackMime_.c_str());
152             if (trackInfos[index]->Get<Tag::VIDEO_ROTATION>(rotation_)) {
153                 MEDIA_LOGI("rotation %{public}d", static_cast<int32_t>(rotation_));
154             }
155             return trackInfos[trackIndex_];
156         }
157     }
158     MEDIA_LOGW("GetTargetTrackInfo FAILED.");
159     return nullptr;
160 }
161 
OnOutputFormatChanged(const MediaAVCodec::Format & format)162 void AVThumbnailGenerator::OnOutputFormatChanged(const MediaAVCodec::Format &format)
163 {
164     MEDIA_LOGD("OnOutputFormatChanged");
165     outputFormat_ = format;
166     int32_t width = 0;
167     int32_t height = 0;
168     bool hasWidth = format.GetIntValue(Tag::VIDEO_PIC_WIDTH, width);
169     bool hasHeight = format.GetIntValue(Tag::VIDEO_PIC_HEIGHT, height);
170     CHECK_AND_RETURN_LOG(hasWidth && hasHeight, "OutputFormat doesn't have width or height");
171     width_ = width;
172     height_ = height;
173 }
174 
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)175 void AVThumbnailGenerator::OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
176 {
177     MEDIA_LOGD("OnInputBufferAvailable index:%{public}u", index);
178     if (stopProcessing_.load() || hasFetchedFrame_.load() || readErrorFlag_.load()) {
179         MEDIA_LOGD("stop or has fetched frame, need not queue input buffer");
180         return;
181     }
182     CHECK_AND_RETURN_LOG(mediaDemuxer_ != nullptr, "OnInputBufferAvailable demuxer is nullptr.");
183     CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "OnInputBufferAvailable decoder is nullptr.");
184     auto readSampleRes = mediaDemuxer_->ReadSample(trackIndex_, buffer);
185     if (readSampleRes != Status::OK && readSampleRes != Status::END_OF_STREAM && readSampleRes != Status::ERROR_AGAIN) {
186         std::unique_lock<std::mutex> lock(mutex_);
187         readErrorFlag_ = true;
188         cond_.notify_all();
189     }
190     videoDecoder_->QueueInputBuffer(index);
191 }
192 
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)193 void AVThumbnailGenerator::OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer)
194 {
195     MEDIA_LOGD("OnOutputBufferAvailable index:%{public}u , pts %{public}ld", index, buffer->pts_);
196     CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "Video decoder not exist");
197     bool isEosBuffer = buffer->flag_ & (uint32_t)(AVBufferFlag::EOS);
198     bool isValidBuffer = buffer != nullptr && buffer->memory_ != nullptr &&
199          (buffer->memory_->GetSize() != 0 || buffer->memory_->GetSurfaceBuffer() != nullptr || isEosBuffer);
200     bool isValidState = !hasFetchedFrame_.load() && !stopProcessing_.load();
201     if (!isValidBuffer || !isValidState) {
202         MEDIA_LOGW("isValidBuffer %{public}d isValidState %{public}d", isValidBuffer, isValidState);
203         videoDecoder_->ReleaseOutputBuffer(index, false);
204         return;
205     }
206     bool isClosest = seekMode_ == Plugins::SeekMode::SEEK_CLOSEST;
207     bool isAvailableFrame = !isClosest || buffer->pts_ >= seekTime_ || isEosBuffer;
208     if (!isAvailableFrame) {
209         videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
210         bufferIndex_ = index;
211         avBuffer_ = buffer;
212         return;
213     }
214     if (isAvailableFrame) {
215         hasFetchedFrame_ = true;
216         if (isClosest && avBuffer_ != nullptr) {
217             int64_t preDiff = seekTime_ - avBuffer_->pts_;
218             int64_t nextDiff = buffer->pts_ - seekTime_;
219             if (preDiff > nextDiff && !(buffer->flag_ & (uint32_t)(AVBufferFlag::EOS))) {
220                 videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
221                 bufferIndex_ = index;
222                 avBuffer_ = buffer;
223             } else {
224                 videoDecoder_->ReleaseOutputBuffer(index, false);
225             }
226         } else {
227             bufferIndex_ = index;
228             avBuffer_ = buffer;
229         }
230         MEDIA_LOGI("dstTime %{public}ld resTime %{public}ld", seekTime_, buffer->pts_);
231         PauseFetchFrame();
232         cond_.notify_all();
233         return;
234     }
235     videoDecoder_->ReleaseOutputBuffer(index, false);
236 }
237 
FetchFrameAtTime(int64_t timeUs,int32_t option,const OutputConfiguration & param)238 std::shared_ptr<AVSharedMemory> AVThumbnailGenerator::FetchFrameAtTime(int64_t timeUs, int32_t option,
239                                                                        const OutputConfiguration &param)
240 {
241     MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
242                "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
243                FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
244                static_cast<int32_t>(param.colorFormat));
245     CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
246 
247     readErrorFlag_ = false;
248     hasFetchedFrame_ = false;
249     outputConfig_ = param;
250     seekTime_ = timeUs;
251     trackInfo_ = GetVideoTrackInfo();
252     CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
253     mediaDemuxer_->Resume();
254     mediaDemuxer_->SelectTrack(trackIndex_);
255     int64_t realSeekTime = timeUs;
256     auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
257     CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
258     CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
259     {
260         std::unique_lock<std::mutex> lock(mutex_);
261 
262         // wait up to 3s to fetch frame AVSharedMemory at time.
263         if (cond_.wait_for(lock, std::chrono::seconds(MAX_WAIT_TIME_SECOND),
264             [this] { return hasFetchedFrame_.load() || readErrorFlag_.load(); })) {
265             HandleFetchFrameAtTimeRes();
266         } else {
267             PauseFetchFrame();
268         }
269     }
270     return fetchedFrameAtTime_;
271 }
272 
HandleFetchFrameAtTimeRes()273 void AVThumbnailGenerator::HandleFetchFrameAtTimeRes()
274 {
275     CHECK_AND_RETURN_RET_LOG(!readErrorFlag_.load(), PauseFetchFrame(), "ReadSample error, exit fetchFrame");
276     MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
277             FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
278     ConvertToAVSharedMemory();
279     videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
280 }
281 
FetchFrameYuv(int64_t timeUs,int32_t option,const OutputConfiguration & param)282 std::shared_ptr<AVBuffer> AVThumbnailGenerator::FetchFrameYuv(int64_t timeUs, int32_t option,
283                                                               const OutputConfiguration &param)
284 {
285     MEDIA_LOGI("Fetch frame 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64 ", option:%{public}d,"
286                "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d",
287                FAKE_POINTER(this), timeUs, option, param.dstWidth, param.dstHeight,
288                static_cast<int32_t>(param.colorFormat));
289     CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr");
290     avBuffer_ = nullptr;
291     readErrorFlag_ = false;
292     hasFetchedFrame_ = false;
293     outputConfig_ = param;
294     seekTime_ = timeUs;
295     trackInfo_ = GetVideoTrackInfo();
296     CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr.");
297     mediaDemuxer_->SelectTrack(trackIndex_);
298     int64_t realSeekTime = timeUs;
299     auto res = SeekToTime(Plugins::Us2Ms(timeUs), static_cast<Plugins::SeekMode>(option), realSeekTime);
300     CHECK_AND_RETURN_RET_LOG(res == Status::OK, nullptr, "Seek fail");
301     CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed.");
302     {
303         std::unique_lock<std::mutex> lock(mutex_);
304 
305         // wait up to 3s to fetch frame AVSharedMemory at time.
306         if (cond_.wait_for(lock, std::chrono::seconds(MAX_WAIT_TIME_SECOND),
307             [this] { return hasFetchedFrame_.load() || readErrorFlag_.load(); })) {
308             HandleFetchFrameYuvRes();
309         } else {
310             HandleFetchFrameYuvFailed();
311         }
312     }
313     return avBuffer_;
314 }
315 
HandleFetchFrameYuvRes()316 void AVThumbnailGenerator::HandleFetchFrameYuvRes()
317 {
318     CHECK_AND_RETURN_RET_LOG(!readErrorFlag_.load(), HandleFetchFrameYuvFailed(), "ReadSample error, exit fetchFrame");
319     MEDIA_LOGI("0x%{public}06" PRIXPTR " Fetch frame OK width:%{public}d, height:%{public}d",
320                 FAKE_POINTER(this), outputConfig_.dstWidth, outputConfig_.dstHeight);
321     avBuffer_ = GenerateAlignmentAvBuffer();
322     if (avBuffer_ != nullptr) {
323         avBuffer_->meta_->Set<Tag::VIDEO_WIDTH>(width_);
324         avBuffer_->meta_->Set<Tag::VIDEO_HEIGHT>(height_);
325         avBuffer_->meta_->Set<Tag::VIDEO_ROTATION>(rotation_);
326     }
327 }
328 
HandleFetchFrameYuvFailed()329 void AVThumbnailGenerator::HandleFetchFrameYuvFailed()
330 {
331     videoDecoder_->ReleaseOutputBuffer(bufferIndex_, false);
332     PauseFetchFrame();
333 }
334 
SeekToTime(int64_t timeMs,Plugins::SeekMode option,int64_t realSeekTime)335 Status AVThumbnailGenerator::SeekToTime(int64_t timeMs, Plugins::SeekMode option, int64_t realSeekTime)
336 {
337     seekMode_ = option;
338     if (option == Plugins::SeekMode::SEEK_CLOSEST) {
339         option = Plugins::SeekMode::SEEK_PREVIOUS_SYNC;
340     }
341     auto res = mediaDemuxer_->SeekTo(timeMs, option, realSeekTime);
342     /* SEEK_NEXT_SYNC or SEEK_PREVIOUS_SYNC may cant find I frame and return seek failed
343        if seek failed, use SEEK_CLOSEST_SYNC seek again */
344     if (res != Status::OK && option != Plugins::SeekMode::SEEK_CLOSEST_SYNC) {
345         res = mediaDemuxer_->SeekTo(timeMs, Plugins::SeekMode::SEEK_CLOSEST_SYNC, realSeekTime);
346         seekMode_ = Plugins::SeekMode::SEEK_CLOSEST_SYNC;
347     }
348     return res;
349 }
350 
ConvertToAVSharedMemory()351 void AVThumbnailGenerator::ConvertToAVSharedMemory()
352 {
353     auto surfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
354     if (surfaceBuffer != nullptr) {
355         auto ret = GetYuvDataAlignStride(surfaceBuffer);
356         CHECK_AND_RETURN_LOG(ret == MSERR_OK, "Copy frame failed");
357         OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
358         frame->width_ = surfaceBuffer->GetWidth();
359         frame->height_ = surfaceBuffer->GetHeight();
360         frame->stride_ = frame->width_ * RATE_UV;
361         frame->bytesPerPixel_ = RATE_UV;
362         frame->size_ = frame->width_ * frame->height_ * BYTES_PER_PIXEL_YUV;
363         frame->rotation_ = static_cast<int32_t>(rotation_);
364         return;
365     }
366 
367     int32_t width;
368     int32_t height;
369     outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width);
370     outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height);
371     if (width == 0 || height == 0) {
372         width = width_;
373         height = height_;
374     }
375 
376     fetchedFrameAtTime_ = std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + avBuffer_->memory_->GetSize(),
377         AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
378     int32_t ret = fetchedFrameAtTime_->Init();
379     CHECK_AND_RETURN_LOG(ret == static_cast<int32_t>(Status::OK), "Create AVSharedmemory failed, ret:%{public}d", ret);
380     OutputFrame *frame = reinterpret_cast<OutputFrame *>(fetchedFrameAtTime_->GetBase());
381     frame->width_ = width;
382     frame->height_ = height;
383     frame->stride_ = width * RATE_UV;
384     frame->bytesPerPixel_ = RATE_UV;
385     frame->size_ = avBuffer_->memory_->GetSize();
386     frame->rotation_ = static_cast<int32_t>(rotation_);
387     fetchedFrameAtTime_->Write(avBuffer_->memory_->GetAddr(), frame->size_, sizeof(OutputFrame));
388 }
389 
ConvertP010ToNV12(const sptr<SurfaceBuffer> & surfaceBuffer,uint8_t * dstNV12,int32_t strideWidth,int32_t strideHeight)390 void AVThumbnailGenerator::ConvertP010ToNV12(const sptr<SurfaceBuffer> &surfaceBuffer, uint8_t *dstNV12,
391                                              int32_t strideWidth, int32_t strideHeight)
392 {
393     int32_t width = surfaceBuffer->GetWidth();
394     int32_t height = surfaceBuffer->GetHeight();
395     uint8_t *srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
396 
397     // copy src Y component to dst
398     for (int32_t i = 0; i < height; i++) {
399         uint16_t *srcY = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
400         uint8_t *dstY = dstNV12 + width * i;
401         for (int32_t j = 0; j < width; j++) {
402             *dstY = static_cast<uint8_t>(*srcY >> SHIFT_BITS_P010_2_NV12);
403             srcY++;
404             dstY++;
405         }
406     }
407 
408     uint8_t *maxDstAddr = dstNV12 + width * height + width * height / RATE_UV;
409     uint8_t *originSrcAddr = srcP010;
410     uint16_t *maxSrcAddr = reinterpret_cast<uint16_t *>(originSrcAddr) + surfaceBuffer->GetSize();
411 
412     srcP010 = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + strideWidth * strideHeight;
413     dstNV12 = dstNV12 + width * height;
414 
415     // copy src UV component to dst, height(UV) = height(Y) / 2;
416     for (int32_t i = 0; i < height / 2; i++) {
417         uint16_t *srcUV = reinterpret_cast<uint16_t *>(srcP010 + strideWidth * i);
418         uint8_t *dstUV = dstNV12 + width * i;
419         for (int32_t j = 0; j < width && srcUV < maxSrcAddr && dstUV < maxDstAddr; j++) {
420             *dstUV = static_cast<uint8_t>(*srcUV >> SHIFT_BITS_P010_2_NV12);
421             *(dstUV + 1) = static_cast<uint8_t>(*(srcUV + 1) >> SHIFT_BITS_P010_2_NV12);
422             srcUV += 2;  // srcUV move by 2 to process U and V component
423             dstUV += 2;  // dstUV move by 2 to process U and V component
424         }
425     }
426 }
427 
GenerateAlignmentAvBuffer()428 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAlignmentAvBuffer()
429 {
430     if (avBuffer_ == nullptr) {
431         MEDIA_LOGE("Generate Alignment AvBuffer failed, avBuffer_ is nullptr.");
432         return nullptr;
433     }
434     if (avBuffer_->memory_->GetSize() != 0 && avBuffer_->memory_->GetSurfaceBuffer() == nullptr) {
435         return GenerateAvBufferFromFCodec();
436     }
437     auto srcSurfaceBuffer = avBuffer_->memory_->GetSurfaceBuffer();
438     auto width = srcSurfaceBuffer->GetWidth();
439     auto height = srcSurfaceBuffer->GetHeight();
440     bool isHdr = srcSurfaceBuffer->GetFormat() ==
441                  static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010);
442     if (isHdr) {
443         sptr<SurfaceBuffer> dstSurfaceBuffer = SurfaceBuffer::Create();
444         BufferRequestConfig requestConfig = {
445             .width = width,
446             .height = height,
447             .strideAlignment = 0x2,
448             .format = srcSurfaceBuffer->GetFormat(),  // always yuv
449             .usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA | BUFFER_USAGE_MEM_MMZ_CACHE,
450             .timeout = 0,
451         };
452         GSError allocRes = dstSurfaceBuffer->Alloc(requestConfig);
453         CHECK_AND_RETURN_RET_LOG(allocRes == 0, nullptr, "Alloc surfaceBuffer failed, ecode %{public}d", allocRes);
454 
455         // create avBuffer from surfaceBuffer
456         std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(dstSurfaceBuffer);
457         bool ret = targetAvBuffer && targetAvBuffer->memory_ && targetAvBuffer->memory_->GetSurfaceBuffer() != nullptr;
458         CHECK_AND_RETURN_RET_LOG(ret, nullptr, "create avBuffer failed");
459         targetAvBuffer->meta_->Set<Tag::VIDEO_IS_HDR_VIVID>(true);
460         CopySurfaceBufferInfo(srcSurfaceBuffer, dstSurfaceBuffer);
461         CopySurfaceBufferPixels(srcSurfaceBuffer, targetAvBuffer);
462         return targetAvBuffer;
463     }
464 
465     // not hdr, just copy pixels
466     std::shared_ptr<AVAllocator> allocator = AVAllocatorFactory::CreateSharedAllocator(MemoryFlag::MEMORY_READ_WRITE);
467     CHECK_AND_RETURN_RET_LOG(allocator != nullptr, nullptr, "create avBuffer allocator failed");
468     int32_t bufferSize = width * height * BYTES_PER_PIXEL_YUV;
469     std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(allocator, bufferSize);
470     CHECK_AND_RETURN_RET_LOG(targetAvBuffer, nullptr, "create avBuffer failed");
471     CopySurfaceBufferPixels(srcSurfaceBuffer, targetAvBuffer);
472     return targetAvBuffer;
473 }
474 
GenerateAvBufferFromFCodec()475 std::shared_ptr<AVBuffer> AVThumbnailGenerator::GenerateAvBufferFromFCodec()
476 {
477     AVBufferConfig avBufferConfig;
478     avBufferConfig.size = avBuffer_->memory_->GetSize();
479     avBufferConfig.memoryType = MemoryType::SHARED_MEMORY;
480     avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE;
481     std::shared_ptr<AVBuffer> targetAvBuffer = AVBuffer::CreateAVBuffer(avBufferConfig);
482     CHECK_AND_RETURN_RET_LOG(targetAvBuffer != nullptr, nullptr, "Create avBuffer failed");
483     targetAvBuffer->memory_->Write(avBuffer_->memory_->GetAddr(), avBuffer_->memory_->GetSize(), 0);
484     return targetAvBuffer;
485 }
486 
GetYuvDataAlignStride(const sptr<SurfaceBuffer> & surfaceBuffer)487 int32_t AVThumbnailGenerator::GetYuvDataAlignStride(const sptr<SurfaceBuffer> &surfaceBuffer)
488 {
489     int32_t width = surfaceBuffer->GetWidth();
490     int32_t height = surfaceBuffer->GetHeight();
491     int32_t stride = surfaceBuffer->GetStride();
492     int32_t outputHeight;
493     auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
494     if (!hasSliceHeight || outputHeight < height) {
495         outputHeight = height;
496     }
497     MEDIA_LOGD("GetYuvDataAlignStride stride:%{public}d, strideWidth:%{public}d, outputHeight:%{public}d", stride,
498                stride, outputHeight);
499 
500     fetchedFrameAtTime_ =
501         std::make_shared<AVSharedMemoryBase>(sizeof(OutputFrame) + width * height * BYTES_PER_PIXEL_YUV,
502             AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory");
503     auto ret = fetchedFrameAtTime_->Init();
504     CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "Create AVSharedmemory failed, ret:%{public}d");
505     uint8_t *dstPtr = static_cast<uint8_t *>(sizeof(OutputFrame) + fetchedFrameAtTime_->GetBase());
506     uint8_t *srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
507     int32_t format = surfaceBuffer->GetFormat();
508     if (format == static_cast<int32_t>(GraphicPixelFormat::GRAPHIC_PIXEL_FMT_YCBCR_P010)) {
509         ConvertP010ToNV12(surfaceBuffer, dstPtr, stride, outputHeight);
510         return MSERR_OK;
511     }
512 
513     // copy src Y component to dst
514     for (int32_t y = 0; y < height; y++) {
515         auto ret = memcpy_s(dstPtr, width, srcPtr, width);
516         if (ret != EOK) {
517             MEDIA_LOGW("Memcpy Y component failed.");
518         }
519         srcPtr += stride;
520         dstPtr += width;
521     }
522 
523     srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + stride * outputHeight;
524 
525     // copy src UV component to dst, height(UV) = height(Y) / 2
526     for (int32_t uv = 0; uv < height / 2; uv++) {
527         auto ret = memcpy_s(dstPtr, width, srcPtr, width);
528         if (ret != EOK) {
529             MEDIA_LOGW("Memcpy UV component failed.");
530         }
531         srcPtr += stride;
532         dstPtr += width;
533     }
534     return MSERR_OK;
535 }
536 
CopySurfaceBufferPixels(const sptr<SurfaceBuffer> & surfaceBuffer,std::shared_ptr<AVBuffer> & avBuffer)537 int32_t AVThumbnailGenerator::CopySurfaceBufferPixels(const sptr<SurfaceBuffer> &surfaceBuffer,
538                                                       std::shared_ptr<AVBuffer> &avBuffer)
539 {
540     CHECK_AND_RETURN_RET(surfaceBuffer != nullptr && avBuffer != nullptr && avBuffer->memory_ != nullptr,
541                          MSERR_INVALID_VAL);
542     int32_t width = surfaceBuffer->GetWidth();
543     int32_t height = surfaceBuffer->GetHeight();
544     int32_t stride = surfaceBuffer->GetStride();
545 
546     bool isHdr = false;
547     (void)avBuffer->meta_->Get<Tag::VIDEO_IS_HDR_VIVID>(isHdr);
548 
549     int32_t outputHeight;
550     auto hasSliceHeight = outputFormat_.GetIntValue(Tag::VIDEO_SLICE_HEIGHT, outputHeight);
551     if (!hasSliceHeight || outputHeight < height) {
552         outputHeight = height;
553     }
554     MEDIA_LOGI("width %{public}d stride %{public}d outputHeight %{public}d", width, stride, outputHeight);
555 
556     uint8_t *srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr());
557     uint8_t *dstPtr = nullptr;
558     if (avBuffer->memory_->GetSurfaceBuffer() != nullptr) {
559         dstPtr = static_cast<uint8_t *>(avBuffer->memory_->GetSurfaceBuffer()->GetVirAddr());
560     } else {
561         dstPtr = avBuffer->memory_->GetAddr();
562     }
563 
564     // copy src Y component to dst
565     int32_t lineByteCount = isHdr ? stride : width;
566     for (int32_t y = 0; y < height; y++) {
567         auto ret = memcpy_s(dstPtr, lineByteCount, srcPtr, lineByteCount);
568         TRUE_LOG(ret != EOK, MEDIA_LOGW, "Memcpy UV component failed.");
569         srcPtr += stride;
570         dstPtr += lineByteCount;
571     }
572 
573     srcPtr = static_cast<uint8_t *>(surfaceBuffer->GetVirAddr()) + stride * outputHeight;
574 
575     // copy src UV component to dst, height(UV) = height(Y) / 2
576     for (int32_t uv = 0; uv < height / RATE_UV; uv++) {
577         auto ret = memcpy_s(dstPtr, lineByteCount, srcPtr, lineByteCount);
578         TRUE_LOG(ret != EOK, MEDIA_LOGW, "Memcpy UV component failed.");
579         srcPtr += stride;
580         dstPtr += lineByteCount;
581     }
582     return MSERR_OK;
583 }
584 
Reset()585 void AVThumbnailGenerator::Reset()
586 {
587     if (mediaDemuxer_ != nullptr) {
588         mediaDemuxer_->Reset();
589     }
590 
591     if (videoDecoder_ != nullptr) {
592         videoDecoder_->Reset();
593     }
594 
595     hasFetchedFrame_ = false;
596     trackInfo_ = nullptr;
597 }
598 
CopySurfaceBufferInfo(sptr<SurfaceBuffer> & source,sptr<SurfaceBuffer> & dst)599 void AVThumbnailGenerator::CopySurfaceBufferInfo(sptr<SurfaceBuffer> &source, sptr<SurfaceBuffer> &dst)
600 {
601     if (source == nullptr || dst == nullptr) {
602         MEDIA_LOGI("CopySurfaceBufferInfo failed, source or dst is nullptr");
603         return;
604     }
605     std::vector<uint8_t> hdrMetadataTypeVec;
606     std::vector<uint8_t> colorSpaceInfoVec;
607     std::vector<uint8_t> staticData;
608     std::vector<uint8_t> dynamicData;
609 
610     if (source->GetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec) == GSERROR_OK) {
611         dst->SetMetadata(ATTRKEY_HDR_METADATA_TYPE, hdrMetadataTypeVec);
612     }
613     if (source->GetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec) == GSERROR_OK) {
614         dst->SetMetadata(ATTRKEY_COLORSPACE_INFO, colorSpaceInfoVec);
615     }
616     if (GetSbStaticMetadata(source, staticData) && (staticData.size() > 0)) {
617         SetSbStaticMetadata(dst, staticData);
618     }
619     if (GetSbDynamicMetadata(source, dynamicData) && (dynamicData.size()) > 0) {
620         SetSbDynamicMetadata(dst, dynamicData);
621     }
622 }
623 
GetSbStaticMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & staticMetadata)624 bool AVThumbnailGenerator::GetSbStaticMetadata(const sptr<SurfaceBuffer> &buffer, std::vector<uint8_t> &staticMetadata)
625 {
626     return buffer->GetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
627 }
628 
GetSbDynamicMetadata(const sptr<SurfaceBuffer> & buffer,std::vector<uint8_t> & dynamicMetadata)629 bool AVThumbnailGenerator::GetSbDynamicMetadata(const sptr<SurfaceBuffer> &buffer,
630                                                 std::vector<uint8_t> &dynamicMetadata)
631 {
632     return buffer->GetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
633 }
634 
SetSbStaticMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & staticMetadata)635 bool AVThumbnailGenerator::SetSbStaticMetadata(sptr<SurfaceBuffer> &buffer, const std::vector<uint8_t> &staticMetadata)
636 {
637     return buffer->SetMetadata(ATTRKEY_HDR_STATIC_METADATA, staticMetadata) == GSERROR_OK;
638 }
639 
SetSbDynamicMetadata(sptr<SurfaceBuffer> & buffer,const std::vector<uint8_t> & dynamicMetadata)640 bool AVThumbnailGenerator::SetSbDynamicMetadata(sptr<SurfaceBuffer> &buffer,
641                                                 const std::vector<uint8_t> &dynamicMetadata)
642 {
643     return buffer->SetMetadata(ATTRKEY_HDR_DYNAMIC_METADATA, dynamicMetadata) == GSERROR_OK;
644 }
645 
Destroy()646 void AVThumbnailGenerator::Destroy()
647 {
648     stopProcessing_ = true;
649     if (videoDecoder_ != nullptr) {
650         videoDecoder_->Stop();
651         videoDecoder_->Release();
652     }
653     mediaDemuxer_ = nullptr;
654     videoDecoder_ = nullptr;
655     MEDIA_LOGI("0x%{public}06" PRIXPTR " Finish Destroy.", FAKE_POINTER(this));
656 }
657 
PauseFetchFrame()658 void AVThumbnailGenerator::PauseFetchFrame()
659 {
660     hasFetchedFrame_ = true;
661     mediaDemuxer_->Pause();
662     mediaDemuxer_->Flush();
663     videoDecoder_->Flush();
664     Format format;
665     format.PutDoubleValue(MediaDescriptionKey::MD_KEY_FRAME_RATE, frameRate_);
666     videoDecoder_->SetParameter(format);
667 }
668 }  // namespace Media
669 }  // namespace OHOS