1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define MEDIA_PLUGIN
17 #define HST_LOG_TAG "FfmpegDemuxerPlugin"
18 #include <unistd.h>
19 #include <algorithm>
20 #include <malloc.h>
21 #include <string>
22 #include <sstream>
23 #include <iomanip>
24 #include <map>
25 #include <fstream>
26 #include <chrono>
27 #include <limits>
28 #include "avcodec_trace.h"
29 #include "securec.h"
30 #include "ffmpeg_format_helper.h"
31 #include "ffmpeg_utils.h"
32 #include "buffer/avbuffer.h"
33 #include "plugin/plugin_buffer.h"
34 #include "plugin/plugin_definition.h"
35 #include "common/log.h"
36 #include "meta/video_types.h"
37 #include "avcodec_sysevent.h"
38 #include "demuxer_log_compressor.h"
39 #include "ffmpeg_demuxer_plugin.h"
40 #include "meta/format.h"
41 #include "syspara/parameters.h"
42 
43 namespace {
44 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_DEMUXER, "FfmpegDemuxerPlugin" };
45 }
46 
47 namespace OHOS {
48 namespace Media {
49 namespace Plugins {
50 namespace Ffmpeg {
51 const uint32_t DEFAULT_READ_SIZE = 4096;
52 const uint32_t DEFAULT_SNIFF_SIZE = 4096 * 4;
53 const int32_t MP3_PROBE_SCORE_LIMIT = 5;
54 const uint32_t STR_MAX_LEN = 4;
55 const uint32_t RANK_MAX = 100;
56 const uint32_t NAL_START_CODE_SIZE = 4;
57 const uint32_t INIT_DOWNLOADS_DATA_SIZE_THRESHOLD = 2 * 1024 * 1024;
58 const int64_t LIVE_FLV_PROBE_SIZE = 100 * 1024 * 2;
59 const uint32_t DEFAULT_CACHE_LIMIT = 50 * 1024 * 1024; // 50M
60 const int64_t INIT_TIME_THRESHOLD = 1000;
61 const uint32_t ID3V2_HEADER_SIZE = 10;
62 const int32_t MS_TO_NS = 1000 * 1000;
63 const uint32_t REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT = 200000;
64 
65 // id3v2 tag position
66 const int32_t POS_0 = 0;
67 const int32_t POS_1 = 1;
68 const int32_t POS_2 = 2;
69 const int32_t POS_3 = 3;
70 const int32_t POS_4 = 4;
71 const int32_t POS_5 = 5;
72 const int32_t POS_6 = 6;
73 const int32_t POS_7 = 7;
74 const int32_t POS_8 = 8;
75 const int32_t POS_9 = 9;
76 const int32_t POS_14 = 14;
77 const int32_t POS_21 = 21;
78 const int32_t POS_FF = 0xff;
79 const int32_t LEN_MASK = 0x7f;
80 const int32_t TAG_MASK = 0x80;
81 const int32_t TAG_VERSION_MASK = 0x10;
82 
83 namespace {
84 std::map<std::string, std::shared_ptr<AVInputFormat>> g_pluginInputFormat;
85 std::mutex g_mtx;
86 
87 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource);
88 
89 Status RegisterPlugins(const std::shared_ptr<Register>& reg);
90 
91 void ReplaceDelimiter(const std::string &delmiters, char newDelimiter, std::string &str);
92 
93 static const std::map<SeekMode, int32_t>  g_seekModeToFFmpegSeekFlags = {
94     { SeekMode::SEEK_PREVIOUS_SYNC, AVSEEK_FLAG_BACKWARD },
95     { SeekMode::SEEK_NEXT_SYNC, AVSEEK_FLAG_FRAME },
96     { SeekMode::SEEK_CLOSEST_SYNC, AVSEEK_FLAG_FRAME | AVSEEK_FLAG_BACKWARD }
97 };
98 
99 static const std::map<AVCodecID, std::string> g_bitstreamFilterMap = {
100     { AV_CODEC_ID_H264, "h264_mp4toannexb" },
101 };
102 
103 static const std::map<AVCodecID, StreamType> g_streamParserMap = {
104     { AV_CODEC_ID_HEVC, StreamType::HEVC },
105     { AV_CODEC_ID_VVC,  StreamType::VVC },
106 };
107 
108 static const std::vector<AVMediaType> g_streamMediaTypeVec = {
109     AVMEDIA_TYPE_AUDIO,
110     AVMEDIA_TYPE_VIDEO,
111     AVMEDIA_TYPE_SUBTITLE,
112     AVMEDIA_TYPE_TIMEDMETA
113 };
114 
115 static std::vector<AVCodecID> g_imageCodecID = {
116     AV_CODEC_ID_MJPEG,
117     AV_CODEC_ID_PNG,
118     AV_CODEC_ID_PAM,
119     AV_CODEC_ID_BMP,
120     AV_CODEC_ID_JPEG2000,
121     AV_CODEC_ID_TARGA,
122     AV_CODEC_ID_TIFF,
123     AV_CODEC_ID_GIF,
124     AV_CODEC_ID_PCX,
125     AV_CODEC_ID_XWD,
126     AV_CODEC_ID_XBM,
127     AV_CODEC_ID_WEBP,
128     AV_CODEC_ID_APNG,
129     AV_CODEC_ID_XPM,
130     AV_CODEC_ID_SVG,
131 };
132 
HaveValidParser(const AVCodecID codecId)133 bool HaveValidParser(const AVCodecID codecId)
134 {
135     return g_streamParserMap.count(codecId) != 0;
136 }
137 
GetFileDuration(const AVFormatContext & avFormatContext)138 int64_t GetFileDuration(const AVFormatContext& avFormatContext)
139 {
140     int64_t duration = 0;
141     const AVDictionaryEntry *metaDuration = av_dict_get(avFormatContext.metadata, "DURATION", NULL, 0);
142     int64_t us;
143     if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
144         if (us > duration) {
145             MEDIA_LOG_D("Get duration from file");
146             duration = us;
147         }
148     }
149 
150     if (duration <= 0) {
151         for (uint32_t i = 0; i < avFormatContext.nb_streams; ++i) {
152             auto streamDuration = (ConvertTimeFromFFmpeg(avFormatContext.streams[i]->duration,
153                 avFormatContext.streams[i]->time_base)) / 1000; // us
154             if (streamDuration > duration) {
155                 MEDIA_LOG_D("Get duration from stream " PUBLIC_LOG_U32, i);
156                 duration = streamDuration;
157             }
158         }
159     }
160     return duration;
161 }
162 
GetStreamDuration(const AVStream & avStream)163 int64_t GetStreamDuration(const AVStream& avStream)
164 {
165     int64_t duration = 0;
166     const AVDictionaryEntry *metaDuration = av_dict_get(avStream.metadata, "DURATION", NULL, 0);
167     int64_t us;
168     if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
169         if (us > duration) {
170             MEDIA_LOG_D("Get duration from stream");
171             duration = us;
172         }
173     }
174     return duration;
175 }
176 
CheckStartTime(const AVFormatContext * formatContext,const AVStream * stream,int64_t & timeStamp,int64_t seekTime)177 bool CheckStartTime(const AVFormatContext *formatContext, const AVStream *stream, int64_t &timeStamp, int64_t seekTime)
178 {
179     int64_t startTime = 0;
180     int64_t num = 1000; // ms convert us
181     FALSE_RETURN_V_MSG_E(stream != nullptr, false, "String is nullptr");
182     if (stream->start_time != AV_NOPTS_VALUE) {
183         startTime = stream->start_time;
184         if (timeStamp > 0 && startTime > INT64_MAX - timeStamp) {
185             MEDIA_LOG_E("Seek value overflow with start time: " PUBLIC_LOG_D64 " timeStamp: " PUBLIC_LOG_D64,
186                 startTime, timeStamp);
187             return false;
188         }
189     }
190     MEDIA_LOG_D("StartTime: " PUBLIC_LOG_D64, startTime);
191     int64_t fileDuration = formatContext->duration;
192     int64_t streamDuration = stream->duration;
193     if (fileDuration == AV_NOPTS_VALUE || fileDuration <= 0) {
194         fileDuration = GetFileDuration(*formatContext);
195     }
196     if (streamDuration == AV_NOPTS_VALUE || streamDuration <= 0) {
197         streamDuration = GetStreamDuration(*stream);
198     }
199     MEDIA_LOG_D("File duration=" PUBLIC_LOG_D64 ", stream duration=" PUBLIC_LOG_D64, fileDuration, streamDuration);
200     // when timestemp out of file duration, return error
201     if (fileDuration > 0 && seekTime * num > fileDuration) { // fileDuration us
202         MEDIA_LOG_E("Seek to timestamp=" PUBLIC_LOG_D64 " failed, max=" PUBLIC_LOG_D64, timeStamp, fileDuration);
203         return false;
204     }
205     // when timestemp out of stream duration, seek to end of stream
206     if (streamDuration > 0 && timeStamp > streamDuration) {
207         MEDIA_LOG_W("Out of stream, seek to " PUBLIC_LOG_D64, timeStamp);
208         timeStamp = streamDuration;
209     }
210     if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
211         MEDIA_LOG_D("Reset timeStamp by start time [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
212             timeStamp, timeStamp + startTime);
213         timeStamp += startTime;
214     }
215     return true;
216 }
217 
ConvertFlagsToFFmpeg(AVStream * avStream,int64_t ffTime,SeekMode mode,int64_t seekTime)218 int ConvertFlagsToFFmpeg(AVStream *avStream, int64_t ffTime, SeekMode mode, int64_t seekTime)
219 {
220     FALSE_RETURN_V_MSG_E(avStream != nullptr && avStream->codecpar != nullptr, -1, "AVStream is nullptr");
221     if (avStream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ffTime == 0) {
222         return AVSEEK_FLAG_FRAME;
223     }
224     if (avStream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO || seekTime == 0) {
225         return AVSEEK_FLAG_BACKWARD;
226     }
227     if (mode == SeekMode::SEEK_NEXT_SYNC || mode == SeekMode::SEEK_PREVIOUS_SYNC) {
228         return g_seekModeToFFmpegSeekFlags.at(mode);
229     }
230     // find closest time in next and prev
231     int keyFrameNext = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_FRAME);
232     FALSE_RETURN_V_MSG_E(keyFrameNext >= 0, AVSEEK_FLAG_BACKWARD, "Not next key frame");
233 
234     int keyFramePrev = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_BACKWARD);
235     FALSE_RETURN_V_MSG_E(keyFramePrev >= 0, AVSEEK_FLAG_FRAME, "Not pre key frame");
236 
237     int64_t ffTimePrev = CalculateTimeByFrameIndex(avStream, keyFramePrev);
238     int64_t ffTimeNext = CalculateTimeByFrameIndex(avStream, keyFrameNext);
239     MEDIA_LOG_D("FfTime=" PUBLIC_LOG_D64 ", ffTimePrev=" PUBLIC_LOG_D64 ", ffTimeNext=" PUBLIC_LOG_D64,
240         ffTime, ffTimePrev, ffTimeNext);
241     if (ffTimePrev == ffTimeNext || (ffTimeNext - ffTime < ffTime - ffTimePrev)) {
242         return AVSEEK_FLAG_FRAME;
243     } else {
244         return AVSEEK_FLAG_BACKWARD;
245     }
246 }
247 
IsSupportedTrack(const AVStream & avStream)248 bool IsSupportedTrack(const AVStream& avStream)
249 {
250     FALSE_RETURN_V_MSG_E(avStream.codecpar != nullptr, false, "Codecpar is nullptr");
251     if (std::find(g_streamMediaTypeVec.cbegin(), g_streamMediaTypeVec.cend(),
252         avStream.codecpar->codec_type) == g_streamMediaTypeVec.cend()) {
253         MEDIA_LOG_E("Unsupport track type: " PUBLIC_LOG_S,
254             ConvertFFmpegMediaTypeToString(avStream.codecpar->codec_type).data());
255         return false;
256     }
257     if (avStream.codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
258         if (avStream.codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
259             MEDIA_LOG_E("Unsupport raw video track");
260             return false;
261         }
262         if (std::count(g_imageCodecID.begin(), g_imageCodecID.end(), avStream.codecpar->codec_id) > 0) {
263             MEDIA_LOG_E("Unsupport image track");
264             return false;
265         }
266     }
267     return true;
268 }
269 } // namespace
270 
271 std::atomic<int> FFmpegDemuxerPlugin::readatIndex_ = 0;
FFmpegDemuxerPlugin(std::string name)272 FFmpegDemuxerPlugin::FFmpegDemuxerPlugin(std::string name)
273     : DemuxerPlugin(std::move(name)),
274       seekable_(Seekable::SEEKABLE),
275       ioContext_(),
276       selectedTrackIds_(),
277       cacheQueue_("cacheQueue"),
278       streamParserInited_(false),
279       parserRefIoContext_()
280 {
281     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
282     MEDIA_LOG_D("In");
283 #ifndef _WIN32
284     (void)mallopt(M_SET_THREAD_CACHE, M_THREAD_CACHE_DISABLE);
285     (void)mallopt(M_DELAYED_FREE, M_DELAYED_FREE_DISABLE);
286 #endif
287     av_log_set_callback(FfmpegLogPrint);
288 #ifdef BUILD_ENG_VERSION
289     std::string dumpModeStr = OHOS::system::GetParameter("FFmpegDemuxerPlugin.dump", "0");
290     dumpMode_ = static_cast<DumpMode>(strtoul(dumpModeStr.c_str(), nullptr, 2)); // 2 is binary
291     MEDIA_LOG_D("Dump mode = %s(%lu)", dumpModeStr.c_str(), dumpMode_);
292 #endif
293     MEDIA_LOG_D("Out");
294 }
295 
~FFmpegDemuxerPlugin()296 FFmpegDemuxerPlugin::~FFmpegDemuxerPlugin()
297 {
298     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
299     MEDIA_LOG_D("In");
300 #ifndef _WIN32
301     (void)mallopt(M_FLUSH_THREAD_CACHE, 0);
302 #endif
303     formatContext_ = nullptr;
304     pluginImpl_ = nullptr;
305     avbsfContext_ = nullptr;
306     streamParser_ = nullptr;
307     referenceParser_ = nullptr;
308     parserRefFormatContext_ = nullptr;
309     selectedTrackIds_.clear();
310     if (firstFrame_ != nullptr) {
311         av_packet_free(&firstFrame_);
312         av_free(firstFrame_);
313         firstFrame_ = nullptr;
314     }
315     MEDIA_LOG_D("Out");
316 }
317 
Dump(const DumpParam & dumpParam)318 void FFmpegDemuxerPlugin::Dump(const DumpParam &dumpParam)
319 {
320     std::string path;
321     switch (dumpParam.mode) {
322         case DUMP_READAT_INPUT:
323             path = "Readat_index." + std::to_string(dumpParam.index) + "_offset." + std::to_string(dumpParam.offset) +
324                 "_size." + std::to_string(dumpParam.size);
325             break;
326         case DUMP_AVPACKET_OUTPUT:
327             path = "AVPacket_index." + std::to_string(dumpParam.index) + "_track." +
328                 std::to_string(dumpParam.trackId) + "_pts." + std::to_string(dumpParam.pts) + "_pos." +
329                 std::to_string(dumpParam.pos);
330             break;
331         case DUMP_AVBUFFER_OUTPUT:
332             path = "AVBuffer_track." + std::to_string(dumpParam.trackId) + "_index." +
333                 std::to_string(dumpParam.index) + "_pts." + std::to_string(dumpParam.pts);
334             break;
335         default:
336             return;
337     }
338     std::ofstream ofs;
339     path = "/data/ff_dump/" + path;
340     ofs.open(path, std::ios::out); //  | std::ios::app
341     if (ofs.is_open()) {
342         ofs.write(reinterpret_cast<char*>(dumpParam.buf), dumpParam.size);
343         ofs.close();
344     }
345     MEDIA_LOG_D("Dump path:" PUBLIC_LOG_S, path.c_str());
346 }
347 
Reset()348 Status FFmpegDemuxerPlugin::Reset()
349 {
350     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
351     MEDIA_LOG_D("In");
352     readatIndex_ = 0;
353     avpacketIndex_ = 0;
354     ioContext_.offset = 0;
355     ioContext_.eos = false;
356     for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
357         cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
358     }
359     selectedTrackIds_.clear();
360     pluginImpl_.reset();
361     formatContext_.reset();
362     avbsfContext_.reset();
363     trackMtx_.clear();
364     trackDfxInfoMap_.clear();
365     return Status::OK;
366 }
367 
InitBitStreamContext(const AVStream & avStream)368 void FFmpegDemuxerPlugin::InitBitStreamContext(const AVStream& avStream)
369 {
370     FALSE_RETURN_MSG(avStream.codecpar != nullptr, "Codecpar is nullptr");
371     AVCodecID codecID = avStream.codecpar->codec_id;
372     MEDIA_LOG_D("For track " PUBLIC_LOG_D32, avStream.index);
373     FALSE_RETURN_MSG(g_bitstreamFilterMap.count(codecID) != 0, "Can not match any BitStreamContext");
374     const AVBitStreamFilter* avBitStreamFilter = av_bsf_get_by_name(g_bitstreamFilterMap.at(codecID).c_str());
375 
376     FALSE_RETURN_MSG((avBitStreamFilter != nullptr), "Call av_bsf_get_by_name failed for" PUBLIC_LOG_S,
377             g_bitstreamFilterMap.at(codecID).c_str());
378 
379     if (!avbsfContext_) {
380         AVBSFContext* avbsfContext {nullptr};
381         int ret = av_bsf_alloc(avBitStreamFilter, &avbsfContext);
382         FALSE_RETURN_MSG((ret >= 0 && avbsfContext != nullptr),
383             "Call av_bsf_alloc failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
384 
385         ret = avcodec_parameters_copy(avbsfContext->par_in, avStream.codecpar);
386         FALSE_RETURN_MSG((ret >= 0), "Call avcodec_parameters_copy failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
387 
388         ret = av_bsf_init(avbsfContext);
389         FALSE_RETURN_MSG((ret >= 0), "Call av_bsf_init failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
390 
391         avbsfContext_ = std::shared_ptr<AVBSFContext>(avbsfContext, [](AVBSFContext* ptr) {
392             if (ptr) {
393                 av_bsf_free(&ptr);
394             }
395         });
396     }
397     FALSE_RETURN_MSG(avbsfContext_ != nullptr, "Stream " PUBLIC_LOG_S " will not be converted to annexb",
398             g_bitstreamFilterMap.at(codecID).c_str());
399     MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will convert to annexb", avStream.index);
400 }
401 
ConvertAvcToAnnexb(AVPacket & pkt)402 Status FFmpegDemuxerPlugin::ConvertAvcToAnnexb(AVPacket& pkt)
403 {
404     int ret = av_bsf_send_packet(avbsfContext_.get(), &pkt);
405     FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
406         "Call av_bsf_send_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
407     av_packet_unref(&pkt);
408 
409     ret = av_bsf_receive_packet(avbsfContext_.get(), &pkt);
410     FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
411         "Call av_bsf_receive_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
412     return Status::OK;
413 }
414 
ConvertHevcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)415 Status FFmpegDemuxerPlugin::ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
416 {
417     size_t cencInfoSize = 0;
418     uint8_t *cencInfo = av_packet_get_side_data(samplePacket->pkts[0], AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
419     streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, cencInfo, cencInfoSize, false);
420     if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) &&
421         streamParser_->IsSyncFrame(pkt.data, pkt.size)) {
422         pkt.flags = static_cast<int32_t>(static_cast<uint32_t>(pkt.flags) | static_cast<uint32_t>(AV_PKT_FLAG_KEY));
423     }
424     return Status::OK;
425 }
426 
ConvertVvcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)427 Status FFmpegDemuxerPlugin::ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
428 {
429     streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, nullptr, 0, false);
430     return Status::OK;
431 }
432 
WriteBuffer(std::shared_ptr<AVBuffer> outBuffer,const uint8_t * writeData,int32_t writeSize)433 Status FFmpegDemuxerPlugin::WriteBuffer(
434     std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize)
435 {
436     FALSE_RETURN_V_MSG_E(outBuffer != nullptr, Status::ERROR_NULL_POINTER, "Buffer is nullptr");
437     if (writeData != nullptr && writeSize > 0) {
438         FALSE_RETURN_V_MSG_E(outBuffer->memory_ != nullptr, Status::ERROR_NULL_POINTER, "Memory is nullptr");
439         int32_t ret = outBuffer->memory_->Write(writeData, writeSize, 0);
440         FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_INVALID_OPERATION, "Memory write failed");
441     }
442 
443     MEDIA_LOG_D("CurrentBuffer: [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_U32 "]",
444         outBuffer->pts_, outBuffer->duration_, outBuffer->flag_);
445     return Status::OK;
446 }
447 
SetDrmCencInfo(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)448 Status FFmpegDemuxerPlugin::SetDrmCencInfo(
449     std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
450 {
451     FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
452         "Sample is nullptr");
453     FALSE_RETURN_V_MSG_E((samplePacket != nullptr && samplePacket->pkts.size() > 0), Status::ERROR_INVALID_OPERATION,
454         "Packet is nullptr");
455     FALSE_RETURN_V_MSG_E((samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0),
456         Status::ERROR_INVALID_OPERATION, "Packet empty");
457 
458     size_t cencInfoSize = 0;
459     MetaDrmCencInfo *cencInfo = (MetaDrmCencInfo *)av_packet_get_side_data(samplePacket->pkts[0],
460         AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
461     if ((cencInfo != nullptr) && (cencInfoSize != 0)) {
462         std::vector<uint8_t> drmCencVec(reinterpret_cast<uint8_t *>(cencInfo),
463             (reinterpret_cast<uint8_t *>(cencInfo)) + sizeof(MetaDrmCencInfo));
464         sample->meta_->SetData(Media::Tag::DRM_CENC_INFO, std::move(drmCencVec));
465     }
466     return Status::OK;
467 }
468 
GetNextFrame(const uint8_t * data,const uint32_t size)469 bool FFmpegDemuxerPlugin::GetNextFrame(const uint8_t *data, const uint32_t size)
470 {
471     if (size < NAL_START_CODE_SIZE) {
472         return false;
473     }
474     bool hasShortStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 1); // 001
475     bool hasLongStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1); // 0001
476     return hasShortStartCode || hasLongStartCode;
477 }
478 
NeedCombineFrame(uint32_t trackId)479 bool FFmpegDemuxerPlugin::NeedCombineFrame(uint32_t trackId)
480 {
481     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, false, "AVFormatContext is nullptr");
482     if (FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MPEGTS &&
483         formatContext_->streams[trackId]->codecpar->codec_id == AV_CODEC_ID_HEVC) {
484         return true;
485     }
486     return false;
487 }
488 
CombinePackets(std::shared_ptr<SamplePacket> samplePacket)489 AVPacket* FFmpegDemuxerPlugin::CombinePackets(std::shared_ptr<SamplePacket> samplePacket)
490 {
491     AVPacket *tempPkt = nullptr;
492     if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) && samplePacket->pkts.size() > 1) {
493         int totalSize = 0;
494         for (auto pkt : samplePacket->pkts) {
495             FALSE_RETURN_V_MSG_E(pkt != nullptr, nullptr, "AVPacket is nullptr");
496             totalSize += pkt->size;
497         }
498         tempPkt = av_packet_alloc();
499         FALSE_RETURN_V_MSG_E(tempPkt != nullptr, nullptr, "Temp packet is nullptr");
500         int ret = av_new_packet(tempPkt, totalSize);
501         FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "Call av_new_packet failed");
502         av_packet_copy_props(tempPkt, samplePacket->pkts[0]);
503         int offset = 0;
504         bool copySuccess = true;
505         for (auto pkt : samplePacket->pkts) {
506             if (pkt == nullptr) {
507                 copySuccess = false;
508                 MEDIA_LOG_E("Cache packet is nullptr");
509                 break;
510             }
511             ret = memcpy_s(tempPkt->data + offset, pkt->size, pkt->data, pkt->size);
512             if (ret != EOK) {
513                 copySuccess = false;
514                 MEDIA_LOG_E("Memcpy failed, ret:" PUBLIC_LOG_D32, ret);
515                 break;
516             }
517             offset += pkt->size;
518         }
519         if (!copySuccess) {
520             av_packet_free(&tempPkt);
521             av_free(tempPkt);
522             tempPkt = nullptr;
523             return nullptr;
524         }
525         tempPkt->size = totalSize;
526         MEDIA_LOG_D("Combine " PUBLIC_LOG_ZU " packets, total size=" PUBLIC_LOG_D32,
527             samplePacket->pkts.size(), totalSize);
528     } else {
529         tempPkt = samplePacket->pkts[0];
530     }
531     return tempPkt;
532 }
533 
ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample,AVPacket * srcAVPacket,std::shared_ptr<SamplePacket> dstSamplePacket)534 Status FFmpegDemuxerPlugin::ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* srcAVPacket,
535     std::shared_ptr<SamplePacket> dstSamplePacket)
536 {
537     Status ret = Status::OK;
538     auto codecId = formatContext_->streams[srcAVPacket->stream_index]->codecpar->codec_id;
539     if (codecId == AV_CODEC_ID_HEVC && streamParser_ != nullptr && streamParserInited_) {
540         ret = ConvertHevcToAnnexb(*srcAVPacket, dstSamplePacket);
541         SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_HEVC);
542     } else if (codecId == AV_CODEC_ID_VVC && streamParser_ != nullptr && streamParserInited_) {
543         ret = ConvertVvcToAnnexb(*srcAVPacket, dstSamplePacket);
544     } else if (codecId == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
545         ret = ConvertAvcToAnnexb(*srcAVPacket);
546         SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_H264);
547     }
548     if (ioContext_.retry) {
549         ioContext_.retry = false;
550         formatContext_->pb->eof_reached = 0;
551         formatContext_->pb->error = 0;
552         cacheQueue_.Pop(dstSamplePacket->pkts[0]->stream_index);
553         return Status::ERROR_AGAIN;
554     }
555     return ret;
556 }
557 
WriteBufferAttr(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)558 void FFmpegDemuxerPlugin::WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
559 {
560     AVStream *avStream = formatContext_->streams[samplePacket->pkts[0]->stream_index];
561     if (samplePacket->pkts[0]->pts != AV_NOPTS_VALUE) {
562         sample->pts_ = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->pts, avStream->time_base));
563     }
564     // durantion dts
565     if (samplePacket->pkts[0]->duration != AV_NOPTS_VALUE) {
566         int64_t duration = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->duration, avStream->time_base));
567         sample->duration_ = duration;
568         sample->meta_->SetData(Media::Tag::BUFFER_DURATION, duration);
569     }
570     if (samplePacket->pkts[0]->dts != AV_NOPTS_VALUE) {
571         int64_t dts = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->dts, avStream->time_base));
572         sample->dts_ = dts;
573         sample->meta_->SetData(Media::Tag::BUFFER_DECODING_TIMESTAMP, dts);
574     }
575     if (avStream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
576         avStream->codecpar->codec_id != AV_CODEC_ID_H264 &&
577         firstFrame_ && samplePacket->pkts[0]->dts == firstFrame_->dts) {
578         if (streamParser_ != nullptr) {
579             streamParser_->ResetXPSSendStatus();
580         }
581     }
582 }
583 
ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)584 Status FFmpegDemuxerPlugin::ConvertAVPacketToSample(
585     std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
586 {
587     FALSE_RETURN_V_MSG_E(samplePacket != nullptr && samplePacket->pkts.size() > 0 &&
588         samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0,
589         Status::ERROR_INVALID_OPERATION, "Input packet is nullptr or empty");
590     MEDIA_LOG_D("Convert packet info for track " PUBLIC_LOG_D32, samplePacket->pkts[0]->stream_index);
591     FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
592         "Input sample is nullptr");
593 
594     WriteBufferAttr(sample, samplePacket);
595 
596     // convert
597     AVPacket *tempPkt = CombinePackets(samplePacket);
598     FALSE_RETURN_V_MSG_E(tempPkt != nullptr, Status::ERROR_INVALID_OPERATION, "Temp packet is empty");
599     Status ret = ConvertPacketToAnnexb(sample, tempPkt, samplePacket);
600     FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Convert annexb failed");
601 
602     // flag\copy
603     int32_t remainSize = tempPkt->size - static_cast<int32_t>(samplePacket->offset);
604     int32_t copySize = remainSize < sample->memory_->GetCapacity() ? remainSize : sample->memory_->GetCapacity();
605     MEDIA_LOG_D("Convert size [" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
606         tempPkt->size, remainSize, copySize, samplePacket->offset);
607     uint32_t flag = ConvertFlagsFromFFmpeg(*tempPkt, (copySize != tempPkt->size));
608     SetDrmCencInfo(sample, samplePacket);
609 
610     sample->flag_ = flag;
611     ret = WriteBuffer(sample, tempPkt->data + samplePacket->offset, copySize);
612     FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
613 
614     if (!samplePacket->isEOS) {
615         trackDfxInfoMap_[tempPkt->stream_index].lastPts = sample->pts_;
616         trackDfxInfoMap_[tempPkt->stream_index].lastDurantion = sample->duration_;
617         trackDfxInfoMap_[tempPkt->stream_index].lastPos = tempPkt->pos;
618     }
619 #ifdef BUILD_ENG_VERSION
620     DumpParam dumpParam {DumpMode(DUMP_AVBUFFER_OUTPUT & dumpMode_), tempPkt->data + samplePacket->offset,
621         tempPkt->stream_index, -1, copySize, trackDfxInfoMap_[tempPkt->stream_index].frameIndex++, tempPkt->pts, -1};
622     Dump(dumpParam);
623 #endif
624     if (tempPkt != nullptr && tempPkt->size != samplePacket->pkts[0]->size) {
625         av_packet_free(&tempPkt);
626         av_free(tempPkt);
627         tempPkt = nullptr;
628     }
629 
630     if (copySize < remainSize) {
631         samplePacket->offset += static_cast<uint32_t>(copySize);
632         MEDIA_LOG_D("Buffer is not enough, next buffer to copy remain data");
633         return Status::ERROR_NOT_ENOUGH_DATA;
634     }
635     return Status::OK;
636 }
637 
PushEOSToAllCache()638 Status FFmpegDemuxerPlugin::PushEOSToAllCache()
639 {
640     Status ret = Status::OK;
641     for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
642         auto streamIndex = selectedTrackIds_[i];
643         MEDIA_LOG_I("Track " PUBLIC_LOG_D32, streamIndex);
644         std::shared_ptr<SamplePacket> eosSample = std::make_shared<SamplePacket>();
645         eosSample->isEOS = true;
646         cacheQueue_.Push(streamIndex, eosSample);
647         ret = CheckCacheDataLimit(streamIndex);
648     }
649     return ret;
650 }
651 
WebvttPktProcess(AVPacket * pkt)652 bool FFmpegDemuxerPlugin::WebvttPktProcess(AVPacket *pkt)
653 {
654     auto trackId = pkt->stream_index;
655     if (pkt->size > 0) {    // vttc
656         return false;
657     } else {    // vtte
658         if (cacheQueue_.HasCache(trackId)) {
659             std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
660             if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts.size() > 0 &&
661                 cacheSamplePacket->pkts[0] != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
662                 cacheSamplePacket->pkts[0]->duration = pkt->pts - cacheSamplePacket->pkts[0]->pts;
663             }
664         }
665     }
666     av_packet_free(&pkt);
667     return true;
668 }
669 
IsWebvttMP4(const AVStream * avStream)670 bool FFmpegDemuxerPlugin::IsWebvttMP4(const AVStream *avStream)
671 {
672     if (avStream->codecpar->codec_id == AV_CODEC_ID_WEBVTT &&
673         FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4) {
674         return true;
675     }
676     return false;
677 }
678 
WebvttMP4EOSProcess(const AVPacket * pkt)679 void FFmpegDemuxerPlugin::WebvttMP4EOSProcess(const AVPacket *pkt)
680 {
681     if (pkt != nullptr) {
682         auto trackId = pkt->stream_index;
683         AVStream *avStream = formatContext_->streams[trackId];
684         if (IsWebvttMP4(avStream) && pkt->size == 0 && cacheQueue_.HasCache(trackId)) {
685             std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
686             if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
687                 cacheSamplePacket->pkts[0]->duration =
688                     formatContext_->streams[pkt->stream_index]->duration - cacheSamplePacket->pkts[0]->pts;
689             }
690         }
691     }
692 }
693 
ReadPacketToCacheQueue(const uint32_t readId)694 Status FFmpegDemuxerPlugin::ReadPacketToCacheQueue(const uint32_t readId)
695 {
696     std::lock_guard<std::mutex> lock(mutex_);
697     AVPacket *pkt = nullptr;
698     bool continueRead = true;
699     Status ret = Status::OK;
700     while (continueRead) {
701         if (pkt == nullptr) {
702             pkt = av_packet_alloc();
703             FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
704         }
705         std::unique_lock<std::mutex> sLock(syncMutex_);
706         int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
707         sLock.unlock();
708         if (ffmpegRet == AVERROR_EOF) { // eos
709             WebvttMP4EOSProcess(pkt);
710             av_packet_free(&pkt);
711             ret = PushEOSToAllCache();
712             FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Push eos failed");
713             return Status::END_OF_STREAM;
714         }
715         if (ffmpegRet < 0) { // fail
716             av_packet_free(&pkt);
717             MEDIA_LOG_E("Call av_read_frame failed:" PUBLIC_LOG_S ", retry: " PUBLIC_LOG_D32,
718                 AVStrError(ffmpegRet).c_str(), int(ioContext_.retry));
719             if (ioContext_.retry) {
720                 formatContext_->pb->eof_reached = 0;
721                 formatContext_->pb->error = 0;
722                 ioContext_.retry = false;
723                 return Status::ERROR_AGAIN;
724             }
725             return Status::ERROR_UNKNOWN;
726         }
727         auto trackId = pkt->stream_index;
728         if (!TrackIsSelected(trackId)) {
729             av_packet_unref(pkt);
730             continue;
731         }
732         AVStream *avStream = formatContext_->streams[trackId];
733         if (IsWebvttMP4(avStream) && WebvttPktProcess(pkt)) {
734             break;
735         } else if (!IsWebvttMP4(avStream) && (!NeedCombineFrame(readId) ||
736             (cacheQueue_.HasCache(static_cast<uint32_t>(trackId)) && GetNextFrame(pkt->data, pkt->size)))) {
737             continueRead = false;
738         }
739         ret = AddPacketToCacheQueue(pkt);
740         FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Add cache failed");
741         pkt = nullptr;
742     }
743     return ret;
744 }
745 
SetEosSample(std::shared_ptr<AVBuffer> sample)746 Status FFmpegDemuxerPlugin::SetEosSample(std::shared_ptr<AVBuffer> sample)
747 {
748     MEDIA_LOG_D("In");
749     sample->pts_ = 0;
750     sample->flag_ =  (uint32_t)(AVBufferFlag::EOS);
751     Status ret = WriteBuffer(sample, nullptr, 0);
752     FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
753     MEDIA_LOG_I("Out");
754     return Status::OK;
755 }
756 
Start()757 Status FFmpegDemuxerPlugin::Start()
758 {
759     return Status::OK;
760 }
761 
Stop()762 Status FFmpegDemuxerPlugin::Stop()
763 {
764     return Status::OK;
765 }
766 
767 // Write packet unimplemented, return 0
AVWritePacket(void * opaque,uint8_t * buf,int bufSize)768 int FFmpegDemuxerPlugin::AVWritePacket(void* opaque, uint8_t* buf, int bufSize)
769 {
770     (void)opaque;
771     (void)buf;
772     (void)bufSize;
773     return 0;
774 }
775 
CheckContextIsValid(void * opaque,int & bufSize)776 int FFmpegDemuxerPlugin::CheckContextIsValid(void* opaque, int &bufSize)
777 {
778     int ret = -1;
779     auto ioContext = static_cast<IOContext*>(opaque);
780     FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
781     FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, ret, "DataSource is nullptr");
782     FALSE_RETURN_V_MSG_E(ioContext->offset <= INT64_MAX - static_cast<int64_t>(bufSize), ret, "Offset invalid");
783 
784     if (ioContext->dataSource->IsDash() && ioContext->eos == true) {
785         MEDIA_LOG_I("Read eos");
786         return AVERROR_EOF;
787     }
788 
789     MEDIA_LOG_D("Offset: " PUBLIC_LOG_D64 ", totalSize: " PUBLIC_LOG_U64, ioContext->offset, ioContext->fileSize);
790     if (ioContext->fileSize > 0) {
791         FALSE_RETURN_V_MSG_E(static_cast<uint64_t>(ioContext->offset) <= ioContext->fileSize, ret, "Out of file size");
792         if (static_cast<size_t>(ioContext->offset + bufSize) > ioContext->fileSize) {
793             bufSize = static_cast<int64_t>(ioContext->fileSize) - ioContext->offset;
794         }
795     }
796     return 0;
797 }
798 
799 // Write packet data into the buffer provided by ffmpeg
AVReadPacket(void * opaque,uint8_t * buf,int bufSize)800 int FFmpegDemuxerPlugin::AVReadPacket(void* opaque, uint8_t* buf, int bufSize)
801 {
802     int ret = CheckContextIsValid(opaque, bufSize);
803     FALSE_RETURN_V(ret == 0, ret);
804 
805     ret = -1;
806     auto ioContext = static_cast<IOContext*>(opaque);
807     FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
808     auto buffer = std::make_shared<Buffer>();
809     FALSE_RETURN_V_MSG_E(buffer != nullptr, ret, "Buffer is nullptr");
810     auto bufData = buffer->WrapMemory(buf, bufSize, 0);
811     FALSE_RETURN_V_MSG_E(buffer->GetMemory() != nullptr, ret, "Memory is nullptr");
812 
813     MediaAVCodec::AVCodecTrace trace("AVReadPacket_ReadAt");
814     auto result = ioContext->dataSource->ReadAt(ioContext->offset, buffer, static_cast<size_t>(bufSize));
815     int dataSize = static_cast<int>(buffer->GetMemory()->GetSize());
816     MEDIA_LOG_D("Want:" PUBLIC_LOG_D32 ", Get:" PUBLIC_LOG_D32 ", offset:" PUBLIC_LOG_D64 ", index:" PUBLIC_LOG_D32,
817         bufSize, dataSize, ioContext->offset, readatIndex_.load());
818 #ifdef BUILD_ENG_VERSION
819     DumpParam dumpParam {DumpMode(DUMP_READAT_INPUT & ioContext->dumpMode), buf, -1, ioContext->offset,
820         dataSize, readatIndex_++, -1, -1};
821     Dump(dumpParam);
822 #endif
823     switch (result) {
824         case Status::OK:
825         case Status::ERROR_AGAIN:
826             if (dataSize == 0) {
827                 MEDIA_LOG_I("Read again");
828                 ioContext->retry = true;
829             } else {
830                 ioContext->offset += dataSize;
831                 ret = dataSize;
832             }
833             break;
834         case Status::END_OF_STREAM:
835             MEDIA_LOG_I("Read end");
836             ioContext->eos = true;
837             ret = AVERROR_EOF;
838             break;
839         default:
840             MEDIA_LOG_I("Read failed " PUBLIC_LOG_D32, static_cast<int>(result));
841             break;
842     }
843 
844     if (!ioContext->initCompleted) {
845         if (ioContext->initDownloadDataSize <= UINT32_MAX - static_cast<uint32_t>(dataSize)) {
846             ioContext->initDownloadDataSize += static_cast<uint32_t>(dataSize);
847         } else {
848             MEDIA_LOG_W("DataSize " PUBLIC_LOG_U32 " is invalid", static_cast<uint32_t>(dataSize));
849         }
850     }
851 
852     return ret;
853 }
854 
AVSeek(void * opaque,int64_t offset,int whence)855 int64_t FFmpegDemuxerPlugin::AVSeek(void* opaque, int64_t offset, int whence)
856 {
857     auto ioContext = static_cast<IOContext*>(opaque);
858     uint64_t newPos = 0;
859     FALSE_RETURN_V_MSG_E(ioContext != nullptr, newPos, "IOContext is nullptr");
860     switch (whence) {
861         case SEEK_SET:
862             newPos = static_cast<uint64_t>(offset);
863             ioContext->offset = newPos;
864             MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
865                 whence, offset, newPos);
866             break;
867         case SEEK_CUR:
868             newPos = ioContext->offset + offset;
869             MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
870                 whence, offset, newPos);
871             break;
872         case SEEK_END:
873         case AVSEEK_SIZE: {
874             FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, newPos, "DataSource is nullptr");
875             if (ioContext->dataSource->IsDash()) {
876                 return -1;
877             }
878             uint64_t mediaDataSize = 0;
879             if (ioContext->dataSource->GetSize(mediaDataSize) == Status::OK && (mediaDataSize > 0)) {
880                 newPos = mediaDataSize + offset;
881                 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
882                     whence, offset, newPos);
883             }
884             break;
885         }
886         default:
887             MEDIA_LOG_E("Unexpected whence " PUBLIC_LOG_D32, whence);
888             break;
889     }
890     if (whence != AVSEEK_SIZE) {
891         ioContext->offset = newPos;
892     }
893     MEDIA_LOG_D("Current offset: " PUBLIC_LOG_D64 ", new pos: " PUBLIC_LOG_U64, ioContext->offset, newPos);
894     return newPos;
895 }
896 
AllocAVIOContext(int flags,IOContext * ioContext)897 AVIOContext* FFmpegDemuxerPlugin::AllocAVIOContext(int flags, IOContext *ioContext)
898 {
899     auto buffer = static_cast<unsigned char*>(av_malloc(DEFAULT_READ_SIZE));
900     FALSE_RETURN_V_MSG_E(buffer != nullptr, nullptr, "Call av_malloc failed");
901 
902     AVIOContext* avioContext = avio_alloc_context(
903         buffer, DEFAULT_READ_SIZE, flags & AVIO_FLAG_WRITE, static_cast<void*>(ioContext),
904         AVReadPacket, AVWritePacket, AVSeek);
905     if (avioContext == nullptr) {
906         MEDIA_LOG_E("Call avio_alloc_context failed");
907         av_free(buffer);
908         return nullptr;
909     }
910     avioContext->seekable = (seekable_ == Seekable::SEEKABLE) ? AVIO_SEEKABLE_NORMAL : 0;
911     if (!(static_cast<uint32_t>(flags) & static_cast<uint32_t>(AVIO_FLAG_WRITE))) {
912         avioContext->buf_ptr = avioContext->buf_end;
913         avioContext->write_flag = 0;
914     }
915     return avioContext;
916 }
917 
FreeContext(AVFormatContext * formatContext,AVIOContext * avioContext)918 void FreeContext(AVFormatContext* formatContext, AVIOContext* avioContext)
919 {
920     if (formatContext) {
921         avformat_close_input(&formatContext);
922     }
923     if (avioContext) {
924         if (avioContext->buffer) {
925             av_freep(&(avioContext->buffer));
926         }
927         avio_context_free(&avioContext);
928     }
929 }
930 
ParseHeader(AVFormatContext * formatContext,std::shared_ptr<AVInputFormat> pluginImpl,AVDictionary ** options)931 int32_t ParseHeader(AVFormatContext* formatContext, std::shared_ptr<AVInputFormat> pluginImpl, AVDictionary **options)
932 {
933     FALSE_RETURN_V_MSG_E(formatContext && pluginImpl, -1, "AVFormatContext is nullptr");
934     MediaAVCodec::AVCodecTrace trace("ffmpeg_init");
935 
936     AVIOContext* avioContext = formatContext->pb;
937     auto begin = std::chrono::steady_clock::now();
938     int ret = avformat_open_input(&formatContext, nullptr, pluginImpl.get(), options);
939     if (ret < 0) {
940         FreeContext(formatContext, avioContext);
941         MEDIA_LOG_E("Call avformat_open_input failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
942             pluginImpl->name, AVStrError(ret).c_str());
943         return ret;
944     }
945 
946     auto open = std::chrono::steady_clock::now();
947     if (FFmpegFormatHelper::GetFileTypeByName(*formatContext) == FileType::FLV) { // Fix init live-flv-source too slow
948         formatContext->probesize = LIVE_FLV_PROBE_SIZE;
949     }
950 
951     ret = avformat_find_stream_info(formatContext, NULL);
952     auto parse = std::chrono::steady_clock::now();
953     int64_t openSpend = static_cast<int64_t>(
954             std::chrono::duration_cast<std::chrono::milliseconds>(open - begin).count());
955     int64_t parseSpend = static_cast<int64_t>(
956             std::chrono::duration_cast<std::chrono::milliseconds>(parse - open).count());
957     if ((parseSpend < 0) || (openSpend > INT64_MAX - parseSpend) || (openSpend + parseSpend > INIT_TIME_THRESHOLD)) {
958         MEDIA_LOG_W("Spend [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]", openSpend, parseSpend);
959     }
960     if (ret < 0) {
961         FreeContext(formatContext, avioContext);
962         MEDIA_LOG_E("Parse stream info failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
963             pluginImpl->name, AVStrError(ret).c_str());
964         return ret;
965     }
966     return 0;
967 }
968 
InitAVFormatContext(IOContext * ioContext)969 std::shared_ptr<AVFormatContext> FFmpegDemuxerPlugin::InitAVFormatContext(IOContext *ioContext)
970 {
971     AVFormatContext* formatContext = avformat_alloc_context();
972     FALSE_RETURN_V_MSG_E(formatContext != nullptr, nullptr, "AVFormatContext is nullptr");
973 
974     formatContext->pb = AllocAVIOContext(AVIO_FLAG_READ, ioContext);
975     if (formatContext->pb == nullptr) {
976         FreeContext(formatContext, nullptr);
977         return nullptr;
978     }
979 
980     formatContext->flags = static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_CUSTOM_IO);
981     if (std::string(pluginImpl_->name) == "mp3") {
982         formatContext->flags =
983             static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_FAST_SEEK);
984     }
985     AVDictionary *options = nullptr;
986     if (ioContext_.dataSource->IsDash()) {
987         av_dict_set(&options, "use_tfdt", "true", 0);
988     }
989 
990     int ret = ParseHeader(formatContext, pluginImpl_, &options);
991     av_dict_free(&options);
992     FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "ParseHeader failed");
993 
994     std::shared_ptr<AVFormatContext> retFormatContext =
995         std::shared_ptr<AVFormatContext>(formatContext, [](AVFormatContext *ptr) {
996             if (ptr) {
997                 auto ctx = ptr->pb;
998                 avformat_close_input(&ptr);
999                 if (ctx) {
1000                     ctx->opaque = nullptr;
1001                     av_freep(&(ctx->buffer));
1002                     av_opt_free(ctx);
1003                     avio_context_free(&ctx);
1004                     ctx = nullptr;
1005                 }
1006             }
1007         });
1008     return retFormatContext;
1009 }
1010 
NotifyInitializationCompleted()1011 void FFmpegDemuxerPlugin::NotifyInitializationCompleted()
1012 {
1013     ioContext_.initCompleted = true;
1014     if (ioContext_.initDownloadDataSize >= INIT_DOWNLOADS_DATA_SIZE_THRESHOLD) {
1015         MEDIA_LOG_I("Large init size %{public}u", ioContext_.initDownloadDataSize);
1016         MediaAVCodec::DemuxerInitEventWrite(ioContext_.initDownloadDataSize, pluginName_);
1017     }
1018 }
1019 
SetDataSource(const std::shared_ptr<DataSource> & source)1020 Status FFmpegDemuxerPlugin::SetDataSource(const std::shared_ptr<DataSource>& source)
1021 {
1022     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1023     FALSE_RETURN_V_MSG_E(formatContext_ == nullptr, Status::ERROR_WRONG_STATE, "AVFormatContext is nullptr");
1024     FALSE_RETURN_V_MSG_E(source != nullptr, Status::ERROR_INVALID_PARAMETER, "DataSource is nullptr");
1025     ioContext_.dataSource = source;
1026     ioContext_.offset = 0;
1027     ioContext_.eos = false;
1028     ioContext_.dumpMode = dumpMode_;
1029     seekable_ = ioContext_.dataSource->IsDash() ? Plugins::Seekable::UNSEEKABLE : source->GetSeekable();
1030     if (seekable_ == Plugins::Seekable::SEEKABLE) {
1031         ioContext_.dataSource->GetSize(ioContext_.fileSize);
1032     } else {
1033         ioContext_.fileSize = -1;
1034     }
1035     MEDIA_LOG_I("FileSize: " PUBLIC_LOG_U64 ", seekable: " PUBLIC_LOG_D32, ioContext_.fileSize, seekable_);
1036     {
1037         std::lock_guard<std::mutex> glock(g_mtx);
1038         pluginImpl_ = g_pluginInputFormat[pluginName_];
1039     }
1040     FALSE_RETURN_V_MSG_E(pluginImpl_ != nullptr, Status::ERROR_UNSUPPORTED_FORMAT, "No match inputformat");
1041     formatContext_ = InitAVFormatContext(&ioContext_);
1042     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1043     InitParser();
1044 
1045     NotifyInitializationCompleted();
1046     MEDIA_LOG_I("Out");
1047     cachelimitSize_ = DEFAULT_CACHE_LIMIT;
1048     return Status::OK;
1049 }
1050 
InitParser()1051 void FFmpegDemuxerPlugin::InitParser()
1052 {
1053     FALSE_RETURN_MSG(formatContext_ != nullptr, "AVFormatContext is nullptr");
1054     ParserBoxInfo();
1055     for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1056         if (g_bitstreamFilterMap.count(formatContext_->streams[trackIndex]->codecpar->codec_id) != 0) {
1057             InitBitStreamContext(*(formatContext_->streams[trackIndex]));
1058             break;
1059         }
1060         if (HaveValidParser(formatContext_->streams[trackIndex]->codecpar->codec_id)) {
1061             streamParser_ = StreamParserManager::Create(g_streamParserMap.at(
1062                 formatContext_->streams[trackIndex]->codecpar->codec_id));
1063             if (streamParser_ == nullptr) {
1064                 MEDIA_LOG_W("Init failed");
1065             } else {
1066                 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will be converted", trackIndex);
1067             }
1068             break;
1069         }
1070     }
1071 }
1072 
GetSeiInfo()1073 Status FFmpegDemuxerPlugin::GetSeiInfo()
1074 {
1075     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1076     Status ret = Status::OK;
1077     if (streamParser_ != nullptr && !streamParserInited_) {
1078         for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1079             auto avStream = formatContext_->streams[trackIndex];
1080             if (HaveValidParser(avStream->codecpar->codec_id)) {
1081                 ret = GetVideoFirstKeyFrame(trackIndex);
1082                 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, Status::ERROR_NO_MEMORY, "No memory");
1083                 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr && firstFrame_->data != nullptr,
1084                     Status::ERROR_WRONG_STATE, "Get first frame failed");
1085                 bool convertRet = streamParser_->ConvertExtraDataToAnnexb(
1086                     avStream->codecpar->extradata, avStream->codecpar->extradata_size);
1087                 FALSE_RETURN_V_MSG_E(convertRet, Status::ERROR_INVALID_DATA, "ConvertExtraDataToAnnexb failed");
1088                 streamParserInited_ = true;
1089                 break;
1090             }
1091         }
1092     }
1093     return ret;
1094 }
1095 
GetMediaInfo(MediaInfo & mediaInfo)1096 Status FFmpegDemuxerPlugin::GetMediaInfo(MediaInfo& mediaInfo)
1097 {
1098     MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetMediaInfo");
1099     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1100     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1101 
1102     Status ret = GetSeiInfo();
1103     FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "GetSeiInfo failed");
1104 
1105     FFmpegFormatHelper::ParseMediaInfo(*formatContext_, mediaInfo.general);
1106     DemuxerLogCompressor::StringifyMeta(mediaInfo.general, -1); // source meta
1107     for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1108         Meta meta;
1109         auto avStream = formatContext_->streams[trackIndex];
1110         if (avStream == nullptr) {
1111             MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " info is nullptr", trackIndex);
1112             mediaInfo.tracks.push_back(meta);
1113             continue;
1114         }
1115         FFmpegFormatHelper::ParseTrackInfo(*avStream, meta, *formatContext_);
1116         if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC) {
1117             if (streamParser_ != nullptr && streamParserInited_ && firstFrame_ != nullptr) {
1118                 streamParser_->ConvertPacketToAnnexb(&(firstFrame_->data), firstFrame_->size, nullptr, 0, false);
1119                 streamParser_->ParseAnnexbExtraData(firstFrame_->data, firstFrame_->size);
1120                 // Parser only sends xps info when first call ConvertPacketToAnnexb
1121                 // readSample will call ConvertPacketToAnnexb again, so rest here
1122                 streamParser_->ResetXPSSendStatus();
1123                 ParseHEVCMetadataInfo(*avStream, meta);
1124             } else {
1125                 MEDIA_LOG_W("Parse hevc info failed");
1126             }
1127         }
1128         if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC ||
1129             avStream->codecpar->codec_id == AV_CODEC_ID_H264 ||
1130             avStream->codecpar->codec_id == AV_CODEC_ID_VVC) {
1131             ConvertCsdToAnnexb(*avStream, meta);
1132         }
1133         mediaInfo.tracks.push_back(meta);
1134         DemuxerLogCompressor::StringifyMeta(meta, trackIndex);
1135     }
1136     return Status::OK;
1137 }
1138 
GetUserMeta(std::shared_ptr<Meta> meta)1139 Status FFmpegDemuxerPlugin::GetUserMeta(std::shared_ptr<Meta> meta)
1140 {
1141     MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetUserMeta");
1142     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1143     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1144     FALSE_RETURN_V_MSG_E(meta != nullptr, Status::ERROR_NULL_POINTER, "Meta is nullptr");
1145 
1146     FFmpegFormatHelper::ParseUserMeta(*formatContext_, meta);
1147     return Status::OK;
1148 }
1149 
ParseDrmInfo(const MetaDrmInfo * const metaDrmInfo,size_t drmInfoSize,std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1150 void FFmpegDemuxerPlugin::ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize,
1151     std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1152 {
1153     MEDIA_LOG_D("In");
1154     size_t infoCount = drmInfoSize / sizeof(MetaDrmInfo);
1155     for (size_t index = 0; index < infoCount; index++) {
1156         std::stringstream ssConverter;
1157         std::string uuid;
1158         for (uint32_t i = 0; i < metaDrmInfo[index].uuidLen; i++) {
1159             int32_t singleUuid = static_cast<int32_t>(metaDrmInfo[index].uuid[i]);
1160             ssConverter << std::hex << std::setfill('0') << std::setw(2) << singleUuid; // 2:w
1161             uuid = ssConverter.str();
1162         }
1163         drmInfo.insert({ uuid, std::vector<uint8_t>(metaDrmInfo[index].pssh,
1164             metaDrmInfo[index].pssh + metaDrmInfo[index].psshLen) });
1165     }
1166 }
1167 
GetDrmInfo(std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1168 Status FFmpegDemuxerPlugin::GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1169 {
1170     MEDIA_LOG_D("In");
1171     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1172     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1173 
1174     for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1175         Meta meta;
1176         AVStream *avStream = formatContext_->streams[trackIndex];
1177         if (avStream == nullptr) {
1178             MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " is nullptr", trackIndex);
1179             continue;
1180         }
1181         MEDIA_LOG_D("GetDrmInfo by stream side data");
1182         size_t drmInfoSize = 0;
1183         MetaDrmInfo *tmpDrmInfo = (MetaDrmInfo *)av_stream_get_side_data(avStream,
1184             AV_PKT_DATA_ENCRYPTION_INIT_INFO, &drmInfoSize);
1185         if (tmpDrmInfo != nullptr && drmInfoSize != 0) {
1186             ParseDrmInfo(tmpDrmInfo, drmInfoSize, drmInfo);
1187         }
1188     }
1189     return Status::OK;
1190 }
1191 
ConvertCsdToAnnexb(const AVStream & avStream,Meta & format)1192 void FFmpegDemuxerPlugin::ConvertCsdToAnnexb(const AVStream& avStream, Meta &format)
1193 {
1194     uint8_t *extradata = avStream.codecpar->extradata;
1195     int32_t extradataSize = avStream.codecpar->extradata_size;
1196     if (HaveValidParser(avStream.codecpar->codec_id) && streamParser_ != nullptr && streamParserInited_) {
1197         streamParser_->ConvertPacketToAnnexb(&(extradata), extradataSize, nullptr, 0, true);
1198     } else if (avStream.codecpar->codec_id == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
1199         if (avbsfContext_->par_out->extradata != nullptr && avbsfContext_->par_out->extradata_size > 0) {
1200             extradata = avbsfContext_->par_out->extradata;
1201             extradataSize = avbsfContext_->par_out->extradata_size;
1202         }
1203     }
1204     if (extradata != nullptr && extradataSize > 0) {
1205         std::vector<uint8_t> extra(extradataSize);
1206         extra.assign(extradata, extradata + extradataSize);
1207         format.Set<Tag::MEDIA_CODEC_CONFIG>(extra);
1208     }
1209 }
1210 
AddPacketToCacheQueue(AVPacket * pkt)1211 Status FFmpegDemuxerPlugin::AddPacketToCacheQueue(AVPacket *pkt)
1212 {
1213     FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Pkt is nullptr");
1214 #ifdef BUILD_ENG_VERSION
1215     DumpParam dumpParam {DumpMode(DUMP_AVPACKET_OUTPUT & dumpMode_), pkt->data, pkt->stream_index, -1, pkt->size,
1216         avpacketIndex_++, pkt->pts, pkt->pos};
1217     Dump(dumpParam);
1218 #endif
1219     auto trackId = pkt->stream_index;
1220     Status ret = Status::OK;
1221     if (NeedCombineFrame(trackId) && !GetNextFrame(pkt->data, pkt->size) && cacheQueue_.HasCache(trackId)) {
1222         std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
1223         if (cacheSamplePacket != nullptr) {
1224             cacheSamplePacket->pkts.push_back(pkt);
1225         }
1226     } else {
1227         std::shared_ptr<SamplePacket> cacheSamplePacket = std::make_shared<SamplePacket>();
1228         if (cacheSamplePacket != nullptr) {
1229             cacheSamplePacket->pkts.push_back(pkt);
1230             cacheSamplePacket->offset = 0;
1231             cacheQueue_.Push(static_cast<uint32_t>(trackId), cacheSamplePacket);
1232             ret = CheckCacheDataLimit(static_cast<uint32_t>(trackId));
1233         }
1234     }
1235     return ret;
1236 }
1237 
GetVideoFirstKeyFrame(uint32_t trackIndex)1238 Status FFmpegDemuxerPlugin::GetVideoFirstKeyFrame(uint32_t trackIndex)
1239 {
1240     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1241     AVPacket *pkt = nullptr;
1242     Status ret = Status::OK;
1243     while (1) {
1244         if (pkt == nullptr) {
1245             pkt = av_packet_alloc();
1246             FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1247         }
1248 
1249         std::unique_lock<std::mutex> sLock(syncMutex_);
1250         int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
1251         sLock.unlock();
1252         if (ffmpegRet < 0) {
1253             MEDIA_LOG_E("Call av_read_frame failed, ret:" PUBLIC_LOG_D32, ffmpegRet);
1254             av_packet_unref(pkt);
1255             break;
1256         }
1257         cacheQueue_.AddTrackQueue(pkt->stream_index);
1258         ret = AddPacketToCacheQueue(pkt);
1259         if (ret != Status::OK) {
1260             return ret;
1261         }
1262 
1263         if (static_cast<uint32_t>(pkt->stream_index) == trackIndex) {
1264             firstFrame_ = av_packet_alloc();
1265             FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1266             int avRet = av_new_packet(firstFrame_, pkt->size);
1267             FALSE_RETURN_V_MSG_E(avRet >= 0, Status::ERROR_INVALID_DATA, "Call av_new_packet failed");
1268             av_packet_copy_props(firstFrame_, pkt);
1269             memcpy_s(firstFrame_->data, pkt->size, pkt->data, pkt->size);
1270             break;
1271         }
1272         pkt = nullptr;
1273     }
1274     return ret;
1275 }
1276 
ParseHEVCMetadataInfo(const AVStream & avStream,Meta & format)1277 void FFmpegDemuxerPlugin::ParseHEVCMetadataInfo(const AVStream& avStream, Meta& format)
1278 {
1279     HevcParseFormat parse;
1280     parse.isHdrVivid = streamParser_->IsHdrVivid();
1281     parse.colorRange = streamParser_->GetColorRange();
1282     parse.colorPrimaries = streamParser_->GetColorPrimaries();
1283     parse.colorTransfer = streamParser_->GetColorTransfer();
1284     parse.colorMatrixCoeff = streamParser_->GetColorMatrixCoeff();
1285     parse.profile = streamParser_->GetProfileIdc();
1286     parse.level = streamParser_->GetLevelIdc();
1287     parse.chromaLocation = streamParser_->GetChromaLocation();
1288     parse.picWidInLumaSamples = streamParser_->GetPicWidInLumaSamples();
1289     parse.picHetInLumaSamples = streamParser_->GetPicHetInLumaSamples();
1290 
1291     FFmpegFormatHelper::ParseHevcInfo(*formatContext_, parse, format);
1292 }
1293 
TrackIsSelected(const uint32_t trackId)1294 bool FFmpegDemuxerPlugin::TrackIsSelected(const uint32_t trackId)
1295 {
1296     return std::any_of(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1297                        [trackId](uint32_t id) { return id == trackId; });
1298 }
1299 
SelectTrack(uint32_t trackId)1300 Status FFmpegDemuxerPlugin::SelectTrack(uint32_t trackId)
1301 {
1302     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1303     MEDIA_LOG_I("Select " PUBLIC_LOG_D32, trackId);
1304     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1305     if (trackId >= static_cast<uint32_t>(formatContext_.get()->nb_streams)) {
1306         MEDIA_LOG_E("Track is invalid, just have " PUBLIC_LOG_D32 " tracks in file", formatContext_.get()->nb_streams);
1307         return Status::ERROR_INVALID_PARAMETER;
1308     }
1309 
1310     AVStream* avStream = formatContext_->streams[trackId];
1311     FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1312     if (!IsSupportedTrack(*avStream)) {
1313         MEDIA_LOG_E("Track type is unsupport");
1314         return Status::ERROR_INVALID_PARAMETER;
1315     }
1316 
1317     if (!TrackIsSelected(trackId)) {
1318         selectedTrackIds_.push_back(trackId);
1319         trackMtx_[trackId] = std::make_shared<std::mutex>();
1320         trackDfxInfoMap_[trackId] = {0, -1, -1};
1321         return cacheQueue_.AddTrackQueue(trackId);
1322     } else {
1323         MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " has been selected", trackId);
1324     }
1325     return Status::OK;
1326 }
1327 
UnselectTrack(uint32_t trackId)1328 Status FFmpegDemuxerPlugin::UnselectTrack(uint32_t trackId)
1329 {
1330     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1331     MEDIA_LOG_I("Unselect " PUBLIC_LOG_D32, trackId);
1332     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1333     auto index = std::find_if(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1334                               [trackId](uint32_t selectedId) {return trackId == selectedId; });
1335     if (TrackIsSelected(trackId)) {
1336         selectedTrackIds_.erase(index);
1337         trackMtx_.erase(trackId);
1338         trackDfxInfoMap_.erase(trackId);
1339         return cacheQueue_.RemoveTrackQueue(trackId);
1340     } else {
1341         MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " is not in selected list", trackId);
1342     }
1343     return Status::OK;
1344 }
1345 
SeekTo(int32_t trackId,int64_t seekTime,SeekMode mode,int64_t & realSeekTime)1346 Status FFmpegDemuxerPlugin::SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime)
1347 {
1348     (void) trackId;
1349     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1350     MediaAVCodec::AVCodecTrace trace("SeekTo");
1351     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1352     FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1353 
1354     FALSE_RETURN_V_MSG_E(seekTime >= 0, Status::ERROR_INVALID_PARAMETER,
1355         "Seek time " PUBLIC_LOG_D64 " is not unsupported", seekTime);
1356     FALSE_RETURN_V_MSG_E(g_seekModeToFFmpegSeekFlags.count(mode) != 0, Status::ERROR_INVALID_PARAMETER,
1357         "Seek mode " PUBLIC_LOG_D32 " is not unsupported", static_cast<uint32_t>(mode));
1358 
1359     int trackIndex = static_cast<int>(selectedTrackIds_[0]);
1360     for (size_t i = 1; i < selectedTrackIds_.size(); i++) {
1361         int index = static_cast<int>(selectedTrackIds_[i]);
1362         if (formatContext_->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
1363             trackIndex = index;
1364             break;
1365         }
1366     }
1367     MEDIA_LOG_D("Seek based on track " PUBLIC_LOG_D32, trackIndex);
1368     auto avStream = formatContext_->streams[trackIndex];
1369     FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1370     int64_t ffTime = ConvertTimeToFFmpeg(seekTime * MS_TO_NS, avStream->time_base);
1371     if (!CheckStartTime(formatContext_.get(), avStream, ffTime, seekTime)) {
1372         MEDIA_LOG_E("Get start time from track " PUBLIC_LOG_D32 " failed", trackIndex);
1373         return Status::ERROR_INVALID_OPERATION;
1374     }
1375     realSeekTime = ConvertTimeFromFFmpeg(ffTime, avStream->time_base);
1376     int flag = ConvertFlagsToFFmpeg(avStream, ffTime, mode, seekTime);
1377     MEDIA_LOG_I("Time [" PUBLIC_LOG_U64 "/" PUBLIC_LOG_U64 "/" PUBLIC_LOG_D64 "] flag ["
1378                 PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
1379                 seekTime, ffTime, realSeekTime, static_cast<int32_t>(mode), flag);
1380     auto ret = av_seek_frame(formatContext_.get(), trackIndex, ffTime, flag);
1381     if (formatContext_->pb->error) {
1382         formatContext_->pb->error = 0;
1383     }
1384     FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
1385         "Call av_seek_frame failed, err: " PUBLIC_LOG_S, AVStrError(ret).c_str());
1386     for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1387         cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1388         cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1389     }
1390     return Status::OK;
1391 }
1392 
Flush()1393 Status FFmpegDemuxerPlugin::Flush()
1394 {
1395     Status ret = Status::OK;
1396     std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1397     MEDIA_LOG_I("In");
1398     for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1399         ret = cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1400         ret = cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1401     }
1402     if (formatContext_) {
1403         avio_flush(formatContext_.get()->pb);
1404         avformat_flush(formatContext_.get());
1405     }
1406     return ret;
1407 }
1408 
ResetEosStatus()1409 void FFmpegDemuxerPlugin::ResetEosStatus()
1410 {
1411     MEDIA_LOG_I("In");
1412     formatContext_->pb->eof_reached = 0;
1413     formatContext_->pb->error = 0;
1414 }
1415 
ReadSample(uint32_t trackId,std::shared_ptr<AVBuffer> sample)1416 Status FFmpegDemuxerPlugin::ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample)
1417 {
1418     std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1419     MediaAVCodec::AVCodecTrace trace("ReadSample");
1420     MEDIA_LOG_D("In");
1421     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1422     FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1423     FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_INVALID_PARAMETER, "Track has not been selected");
1424     FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_!=nullptr, Status::ERROR_INVALID_PARAMETER,
1425         "AVBuffer or memory is nullptr");
1426     Status ret;
1427     if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1428         ret = ReadPacketToCacheQueue(trackId);
1429     }
1430     while (!cacheQueue_.HasCache(trackId)) {
1431         ret = ReadPacketToCacheQueue(trackId);
1432         if (ret == Status::END_OF_STREAM) {
1433             MEDIA_LOG_D("Read to end");
1434         }
1435         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1436         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1437         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1438     }
1439     std::lock_guard<std::mutex> lockTrack(*trackMtx_[trackId].get());
1440     auto samplePacket = cacheQueue_.Front(trackId);
1441     FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_NULL_POINTER, "Cache packet is nullptr");
1442     if (samplePacket->isEOS) {
1443         ret = SetEosSample(sample);
1444         if (ret == Status::OK) {
1445             MEDIA_LOG_I("Track:" PUBLIC_LOG_D32 " eos [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
1446                 trackId, trackDfxInfoMap_[trackId].lastPts,
1447                 trackDfxInfoMap_[trackId].lastDurantion, trackDfxInfoMap_[trackId].lastPos);
1448             cacheQueue_.Pop(trackId);
1449         }
1450         return ret;
1451     }
1452     ret = ConvertAVPacketToSample(sample, samplePacket);
1453     if (ret == Status::ERROR_NOT_ENOUGH_DATA) {
1454         return Status::OK;
1455     } else if (ret == Status::OK) {
1456         MEDIA_LOG_D("All partial sample has been copied");
1457         cacheQueue_.Pop(trackId);
1458     }
1459     return ret;
1460 }
1461 
GetNextSampleSize(uint32_t trackId,int32_t & size)1462 Status FFmpegDemuxerPlugin::GetNextSampleSize(uint32_t trackId, int32_t& size)
1463 {
1464     std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1465     MediaAVCodec::AVCodecTrace trace("GetNextSampleSize");
1466     MEDIA_LOG_D("In, track " PUBLIC_LOG_D32, trackId);
1467     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1468     FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_UNKNOWN, "Track has not been selected");
1469 
1470     Status ret;
1471     if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1472         ret = ReadPacketToCacheQueue(trackId);
1473     }
1474     while (!cacheQueue_.HasCache(trackId)) {
1475         ret = ReadPacketToCacheQueue(trackId);
1476         if (ret == Status::END_OF_STREAM) {
1477             MEDIA_LOG_D("Read to end");
1478         }
1479         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1480         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1481         FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1482     }
1483     std::shared_ptr<SamplePacket> samplePacket = cacheQueue_.Front(trackId);
1484     FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_UNKNOWN, "Cache sample is nullptr");
1485     if (samplePacket->isEOS) {
1486         MEDIA_LOG_I("Track " PUBLIC_LOG_D32 " eos", trackId);
1487         return Status::END_OF_STREAM;
1488     }
1489     FALSE_RETURN_V_MSG_E(samplePacket->pkts.size() > 0, Status::ERROR_UNKNOWN, "Cache sample is empty");
1490     int totalSize = 0;
1491     for (auto pkt : samplePacket->pkts) {
1492         FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_UNKNOWN, "Packet in sample is nullptr");
1493         totalSize += pkt->size;
1494     }
1495     size = totalSize;
1496     return Status::OK;
1497 }
1498 
InitPTSandIndexConvert()1499 void FFmpegDemuxerPlugin::InitPTSandIndexConvert()
1500 {
1501     indexToRelativePTSFrameCount_ = 0; // init IndexToRelativePTSFrameCount_
1502     relativePTSToIndexPosition_ = 0; // init RelativePTSToIndexPosition_
1503     indexToRelativePTSMaxHeap_ = std::priority_queue<int64_t>(); // init IndexToRelativePTSMaxHeap_
1504     relativePTSToIndexPTSMin_ = INT64_MAX;
1505     relativePTSToIndexPTSMax_ = INT64_MIN;
1506     relativePTSToIndexRightDiff_ = INT64_MAX;
1507     relativePTSToIndexLeftDiff_ = INT64_MAX;
1508     relativePTSToIndexTempDiff_ = INT64_MAX;
1509 }
1510 
GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,const uint64_t relativePresentationTimeUs,uint32_t & index)1511 Status FFmpegDemuxerPlugin::GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
1512     const uint64_t relativePresentationTimeUs, uint32_t &index)
1513 {
1514     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1515 
1516     FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1517 
1518     InitPTSandIndexConvert();
1519 
1520     auto avStream = formatContext_->streams[trackIndex];
1521     FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1522 
1523     FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1524         Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1525 
1526     Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1527         static_cast<int64_t>(relativePresentationTimeUs), index);
1528     FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1529 
1530     int64_t absolutePTS = static_cast<int64_t>(relativePresentationTimeUs) + absolutePTSIndexZero_;
1531 
1532     ret = GetPresentationTimeUsFromFfmpegMOV(RELATIVEPTS_TO_INDEX, trackIndex,
1533         absolutePTS, index);
1534     FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1535 
1536     if (absolutePTS < relativePTSToIndexPTSMin_ || absolutePTS > relativePTSToIndexPTSMax_) {
1537         MEDIA_LOG_E("Pts is out of range");
1538         return Status::ERROR_INVALID_DATA;
1539     }
1540 
1541     if (relativePTSToIndexLeftDiff_ == 0 || relativePTSToIndexRightDiff_ == 0) {
1542         index = relativePTSToIndexPosition_;
1543     } else {
1544         index = relativePTSToIndexLeftDiff_ < relativePTSToIndexRightDiff_ ?
1545         relativePTSToIndexPosition_ - 1 : relativePTSToIndexPosition_;
1546     }
1547     return Status::OK;
1548 }
1549 
GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,const uint32_t index,uint64_t & relativePresentationTimeUs)1550 Status FFmpegDemuxerPlugin::GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
1551     const uint32_t index, uint64_t &relativePresentationTimeUs)
1552 {
1553     FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1554 
1555     FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1556 
1557     InitPTSandIndexConvert();
1558 
1559     auto avStream = formatContext_->streams[trackIndex];
1560     FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1561 
1562     FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1563         Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1564 
1565     Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1566         static_cast<int64_t>(relativePresentationTimeUs), index);
1567     FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1568 
1569     GetPresentationTimeUsFromFfmpegMOV(INDEX_TO_RELATIVEPTS, trackIndex,
1570         static_cast<int64_t>(relativePresentationTimeUs), index);
1571     FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1572 
1573     if (index + 1 > indexToRelativePTSFrameCount_) {
1574         MEDIA_LOG_E("Index is out of range");
1575         return Status::ERROR_INVALID_DATA;
1576     }
1577 
1578     int64_t relativepts = indexToRelativePTSMaxHeap_.top() - absolutePTSIndexZero_;
1579     FALSE_RETURN_V_MSG_E(relativepts >= 0, Status::ERROR_INVALID_DATA, "Existence of calculation results less than 0");
1580     relativePresentationTimeUs = static_cast<uint64_t>(relativepts);
1581 
1582     return Status::OK;
1583 }
1584 
PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1585 Status FFmpegDemuxerPlugin::PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,
1586     const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1587 {
1588     uint32_t sttsIndex = 0;
1589     uint32_t cttsIndex = 0;
1590     int64_t pts = 0; // init pts
1591     int64_t dts = 0; // init dts
1592 
1593     int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1594     int32_t cttsCurNum = 0;
1595 
1596     cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1597     while (sttsIndex < avStream->stts_count && cttsIndex < avStream->ctts_count &&
1598             cttsCurNum >= 0 && sttsCurNum >= 0) {
1599         if (cttsCurNum == 0) {
1600             cttsIndex++;
1601             if (cttsIndex >= avStream->ctts_count) {
1602                 break;
1603             }
1604             cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1605         }
1606         cttsCurNum--;
1607         if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1608             ((dts + static_cast<int64_t>(avStream->ctts_data[cttsIndex].duration)) /
1609             static_cast<int64_t>(avStream->time_scale))) {
1610                 MEDIA_LOG_E("pts overflow");
1611                 return Status::ERROR_INVALID_DATA;
1612         }
1613         double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1614         double ptsTemp = static_cast<double>(dts) + static_cast<double>(avStream->ctts_data[cttsIndex].duration);
1615         pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1616         if (mode == GET_ALL_FRAME_PTS &&
1617             static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1618             MEDIA_LOG_I("PTS list has reached the maximum limit");
1619             break;
1620         }
1621         PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1622         sttsCurNum--;
1623         if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1624             MEDIA_LOG_E("dts overflow");
1625             return Status::ERROR_INVALID_DATA;
1626         }
1627         dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1628         if (sttsCurNum == 0) {
1629             sttsIndex++;
1630             sttsCurNum = sttsIndex < avStream->stts_count ?
1631                          static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1632         }
1633     }
1634     return Status::OK;
1635 }
1636 
PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1637 Status FFmpegDemuxerPlugin::PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,
1638     const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1639 {
1640     uint32_t sttsIndex = 0;
1641     int64_t pts = 0; // init pts
1642     int64_t dts = 0; // init dts
1643 
1644     int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1645 
1646     while (sttsIndex < avStream->stts_count && sttsCurNum >= 0) {
1647         if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1648             (dts / static_cast<int64_t>(avStream->time_scale))) {
1649                 MEDIA_LOG_E("pts overflow");
1650                 return Status::ERROR_INVALID_DATA;
1651         }
1652         double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1653         double ptsTemp = static_cast<double>(dts);
1654         pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1655         if (mode == GET_ALL_FRAME_PTS &&
1656             static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1657             MEDIA_LOG_I("PTS list has reached the maximum limit");
1658             break;
1659         }
1660         PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1661         sttsCurNum--;
1662         if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1663             MEDIA_LOG_E("dts overflow");
1664             return Status::ERROR_INVALID_DATA;
1665         }
1666         dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1667         if (sttsCurNum == 0) {
1668             sttsIndex++;
1669             sttsCurNum = sttsIndex < avStream->stts_count ?
1670                          static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1671         }
1672     }
1673     return Status::OK;
1674 }
1675 
GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,uint32_t trackIndex,int64_t absolutePTS,uint32_t index)1676 Status FFmpegDemuxerPlugin::GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,
1677     uint32_t trackIndex, int64_t absolutePTS, uint32_t index)
1678 {
1679     auto avStream = formatContext_->streams[trackIndex];
1680     FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1681     FALSE_RETURN_V_MSG_E(avStream->stts_data != nullptr && avStream->stts_count != 0,
1682         Status::ERROR_NULL_POINTER, "AVStream->stts_data is empty");
1683     FALSE_RETURN_V_MSG_E(avStream->time_scale != 0, Status::ERROR_INVALID_DATA, "AVStream->time_scale is zero");
1684     if (mode == GET_ALL_FRAME_PTS) {
1685         ptsListOrg_.clear();
1686     }
1687     return avStream->ctts_data == nullptr ?
1688         PTSAndIndexConvertOnlySttsProcess(mode, avStream, absolutePTS, index) :
1689         PTSAndIndexConvertSttsAndCttsProcess(mode, avStream, absolutePTS, index);
1690 }
1691 
PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,int64_t pts,int64_t absolutePTS,uint32_t index)1692 void FFmpegDemuxerPlugin::PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,
1693     int64_t pts, int64_t absolutePTS, uint32_t index)
1694 {
1695     switch (mode) {
1696         case GET_FIRST_PTS:
1697             absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1698             break;
1699         case INDEX_TO_RELATIVEPTS:
1700             IndexToRelativePTSProcess(pts, index);
1701             break;
1702         case RELATIVEPTS_TO_INDEX:
1703             RelativePTSToIndexProcess(pts, absolutePTS);
1704             break;
1705         case GET_ALL_FRAME_PTS:
1706             absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1707             ptsListOrg_.emplace_back(pts);
1708             break;
1709         default:
1710             MEDIA_LOG_E("Wrong mode");
1711             break;
1712     }
1713 }
1714 
IndexToRelativePTSProcess(int64_t pts,uint32_t index)1715 void FFmpegDemuxerPlugin::IndexToRelativePTSProcess(int64_t pts, uint32_t index)
1716 {
1717     if (indexToRelativePTSMaxHeap_.size() < index + 1) {
1718         indexToRelativePTSMaxHeap_.push(pts);
1719     } else {
1720         if (pts < indexToRelativePTSMaxHeap_.top()) {
1721             indexToRelativePTSMaxHeap_.pop();
1722             indexToRelativePTSMaxHeap_.push(pts);
1723         }
1724     }
1725     indexToRelativePTSFrameCount_++;
1726 }
1727 
RelativePTSToIndexProcess(int64_t pts,int64_t absolutePTS)1728 void FFmpegDemuxerPlugin::RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS)
1729 {
1730     if (relativePTSToIndexPTSMin_ > pts) {
1731         relativePTSToIndexPTSMin_ = pts;
1732     }
1733     if (relativePTSToIndexPTSMax_ < pts) {
1734         relativePTSToIndexPTSMax_ = pts;
1735     }
1736     relativePTSToIndexTempDiff_ = abs(pts - absolutePTS);
1737     if (pts < absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexLeftDiff_) {
1738         relativePTSToIndexLeftDiff_ = relativePTSToIndexTempDiff_;
1739     }
1740     if (pts >= absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexRightDiff_) {
1741         relativePTSToIndexRightDiff_ = relativePTSToIndexTempDiff_;
1742     }
1743     if (pts < absolutePTS) {
1744         relativePTSToIndexPosition_++;
1745     }
1746 }
1747 
CheckCacheDataLimit(uint32_t trackId)1748 Status FFmpegDemuxerPlugin::CheckCacheDataLimit(uint32_t trackId)
1749 {
1750     if (!outOfLimit_) {
1751         auto cacheDataSize = cacheQueue_.GetCacheDataSize(trackId);
1752         if (cacheDataSize > cachelimitSize_) {
1753             MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " cache out of limit: " PUBLIC_LOG_U32 "/" PUBLIC_LOG_U32 ", by user "
1754                 PUBLIC_LOG_D32, trackId, cacheDataSize, cachelimitSize_, static_cast<int32_t>(setLimitByUser));
1755             outOfLimit_ = true;
1756         }
1757     }
1758     return Status::OK;
1759 }
1760 
SetCacheLimit(uint32_t limitSize)1761 void FFmpegDemuxerPlugin::SetCacheLimit(uint32_t limitSize)
1762 {
1763     setLimitByUser = true;
1764     cachelimitSize_ = limitSize;
1765 }
1766 
1767 namespace { // plugin set
1768 
IsStartWithID3(const uint8_t * buf,const char * tagName)1769 int IsStartWithID3(const uint8_t *buf, const char *tagName)
1770 {
1771     return buf[POS_0] == tagName[POS_0] &&
1772            buf[POS_1] == tagName[POS_1] &&
1773            buf[POS_2] == tagName[POS_2] &&
1774            buf[POS_3] != POS_FF &&
1775            buf[POS_4] != POS_FF &&
1776            (buf[POS_6] & TAG_MASK) == 0 &&
1777            (buf[POS_7] & TAG_MASK) == 0 &&
1778            (buf[POS_8] & TAG_MASK) == 0 &&
1779            (buf[POS_9] & TAG_MASK) == 0;
1780 }
1781 
GetID3TagLen(const uint8_t * buf)1782 int GetID3TagLen(const uint8_t *buf)
1783 {
1784     int32_t len = ((buf[POS_6] & LEN_MASK) << POS_21) + ((buf[POS_7] & LEN_MASK) << POS_14) +
1785                   ((buf[POS_8] & LEN_MASK) << POS_7) + (buf[POS_9] & LEN_MASK) +
1786                   static_cast<int32_t>(ID3V2_HEADER_SIZE);
1787     if (buf[POS_5] & TAG_VERSION_MASK) {
1788         len += static_cast<int32_t>(ID3V2_HEADER_SIZE);
1789     }
1790     return len;
1791 }
1792 
GetConfidence(std::shared_ptr<AVInputFormat> plugin,const std::string & pluginName,std::shared_ptr<DataSource> dataSource,size_t & getData)1793 int32_t GetConfidence(std::shared_ptr<AVInputFormat> plugin, const std::string& pluginName,
1794     std::shared_ptr<DataSource> dataSource, size_t &getData)
1795 {
1796     size_t bufferSize = DEFAULT_SNIFF_SIZE;
1797     uint64_t fileSize = 0;
1798     Status getFileSize = dataSource->GetSize(fileSize);
1799     if (getFileSize == Status::OK) {
1800         bufferSize = (bufferSize < fileSize) ? bufferSize : fileSize;
1801     }
1802     std::vector<uint8_t> buff(bufferSize + AVPROBE_PADDING_SIZE); // fix ffmpeg probe crash, refer to tools/probetest.c
1803     auto bufferInfo = std::make_shared<Buffer>();
1804     auto bufData = bufferInfo->WrapMemory(buff.data(), bufferSize, bufferSize);
1805     FALSE_RETURN_V_MSG_E(bufferInfo->GetMemory() != nullptr, -1,
1806         "Alloc buffer failed for " PUBLIC_LOG_S, pluginName.c_str());
1807     Status ret = Status::OK;
1808     {
1809         std::string traceName = "Sniff_" + pluginName + "_Readat";
1810         MediaAVCodec::AVCodecTrace trace(traceName.c_str());
1811         ret = dataSource->ReadAt(0, bufferInfo, bufferSize);
1812     }
1813     FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1814     getData = bufferInfo->GetMemory()->GetSize();
1815     FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1816     if (getFileSize == Status::OK && getData > ID3V2_HEADER_SIZE && IsStartWithID3(buff.data(), "ID3")) {
1817         int32_t id3Len = GetID3TagLen(buff.data());
1818         // id3 tag length is out of file, or file just contains id3 tag, no valid data.
1819         FALSE_RETURN_V_MSG_E(id3Len >= 0 && static_cast<uint64_t>(id3Len) < fileSize, -1,
1820             "File data error for " PUBLIC_LOG_S, pluginName.c_str());
1821         if (id3Len > 0) {
1822             uint64_t remainSize = fileSize - static_cast<uint64_t>(id3Len);
1823             bufferSize = (bufferSize < remainSize) ? bufferSize : remainSize;
1824             int resetRet = memset_s(buff.data(), bufferSize, 0, bufferSize);
1825             FALSE_RETURN_V_MSG_E(resetRet == EOK, -1, "Reset buff failed for " PUBLIC_LOG_S, pluginName.c_str());
1826             ret = dataSource->ReadAt(id3Len, bufferInfo, bufferSize);
1827             FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1828             getData = bufferInfo->GetMemory()->GetSize();
1829             FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1830         }
1831     }
1832     AVProbeData probeData{"", buff.data(), static_cast<int32_t>(getData), ""};
1833     return plugin->read_probe(&probeData);
1834 }
1835 
Sniff(const std::string & pluginName,std::shared_ptr<DataSource> dataSource)1836 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource)
1837 {
1838     FALSE_RETURN_V_MSG_E(!pluginName.empty(), 0, "Plugin name is empty");
1839     FALSE_RETURN_V_MSG_E(dataSource != nullptr, 0, "DataSource is nullptr");
1840     std::shared_ptr<AVInputFormat> plugin;
1841     {
1842         std::lock_guard<std::mutex> lock(g_mtx);
1843         plugin = g_pluginInputFormat[pluginName];
1844     }
1845     FALSE_RETURN_V_MSG_E((plugin != nullptr && plugin->read_probe), 0,
1846         "Get plugin for " PUBLIC_LOG_S " failed", pluginName.c_str());
1847     size_t getData = 0;
1848     int confidence = GetConfidence(plugin, pluginName, dataSource, getData);
1849     if (confidence < 0) {
1850         return 0;
1851     }
1852     if (StartWith(plugin->name, "mp3") && confidence > 0 && confidence <= MP3_PROBE_SCORE_LIMIT) {
1853         MEDIA_LOG_W("Score " PUBLIC_LOG_D32 " is too low", confidence);
1854         confidence = 0;
1855     }
1856     if (getData < DEFAULT_SNIFF_SIZE || confidence > 0) {
1857         MEDIA_LOG_I("Sniff:" PUBLIC_LOG_S "[" PUBLIC_LOG_ZU "/" PUBLIC_LOG_D32 "]", plugin->name, getData, confidence);
1858     }
1859     return confidence;
1860 }
1861 
ReplaceDelimiter(const std::string & delmiters,char newDelimiter,std::string & str)1862 void ReplaceDelimiter(const std::string& delmiters, char newDelimiter, std::string& str)
1863 {
1864     MEDIA_LOG_D("Reset from [" PUBLIC_LOG_S "]", str.c_str());
1865     for (auto it = str.begin(); it != str.end(); ++it) {
1866         if (delmiters.find(newDelimiter) != std::string::npos) {
1867             *it = newDelimiter;
1868         }
1869     }
1870     MEDIA_LOG_D("Reset to [" PUBLIC_LOG_S "]", str.c_str());
1871 };
1872 
RegisterPlugins(const std::shared_ptr<Register> & reg)1873 Status RegisterPlugins(const std::shared_ptr<Register>& reg)
1874 {
1875     MEDIA_LOG_I("In");
1876     FALSE_RETURN_V_MSG_E(reg != nullptr, Status::ERROR_INVALID_PARAMETER, "Register is nullptr");
1877     std::lock_guard<std::mutex> lock(g_mtx);
1878     const AVInputFormat* plugin = nullptr;
1879     void* i = nullptr;
1880     while ((plugin = av_demuxer_iterate(&i))) {
1881         if (plugin == nullptr) {
1882             continue;
1883         }
1884         MEDIA_LOG_D("Check ffmpeg demuxer " PUBLIC_LOG_S "[" PUBLIC_LOG_S "]", plugin->name, plugin->long_name);
1885         if (plugin->long_name != nullptr &&
1886             !strncmp(plugin->long_name, "pcm ", STR_MAX_LEN)) {
1887             continue;
1888         }
1889         if (!IsInputFormatSupported(plugin->name)) {
1890             continue;
1891         }
1892 
1893         std::string pluginName = "avdemux_" + std::string(plugin->name);
1894         ReplaceDelimiter(".,|-<> ", '_', pluginName);
1895 
1896         DemuxerPluginDef regInfo;
1897         regInfo.name = pluginName;
1898         regInfo.description = "ffmpeg demuxer plugin";
1899         regInfo.rank = RANK_MAX;
1900         regInfo.AddExtensions(SplitString(plugin->extensions, ','));
1901         g_pluginInputFormat[pluginName] =
1902             std::shared_ptr<AVInputFormat>(const_cast<AVInputFormat*>(plugin), [](void*) {});
1903         auto func = [](const std::string& name) -> std::shared_ptr<DemuxerPlugin> {
1904             return std::make_shared<FFmpegDemuxerPlugin>(name);
1905         };
1906         regInfo.SetCreator(func);
1907         regInfo.SetSniffer(Sniff);
1908         auto ret = reg->AddPlugin(regInfo);
1909         if (ret != Status::OK) {
1910             MEDIA_LOG_E("Add plugin failed, err=" PUBLIC_LOG_D32, static_cast<int>(ret));
1911         } else {
1912             MEDIA_LOG_D("Add plugin " PUBLIC_LOG_S, pluginName.c_str());
1913         }
1914     }
1915     FALSE_RETURN_V_MSG_E(!g_pluginInputFormat.empty(), Status::ERROR_UNKNOWN, "Can not load any ffmpeg demuxer");
1916     return Status::OK;
1917 }
1918 } // namespace
__anonf5d992460a02null1919 PLUGIN_DEFINITION(FFmpegDemuxer, LicenseType::LGPL, RegisterPlugins, [] {});
1920 } // namespace Ffmpeg
1921 } // namespace Plugins
1922 } // namespace Media
1923 } // namespace OHOS
1924