1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #define MEDIA_PLUGIN
17 #define HST_LOG_TAG "FfmpegDemuxerPlugin"
18 #include <unistd.h>
19 #include <algorithm>
20 #include <malloc.h>
21 #include <string>
22 #include <sstream>
23 #include <iomanip>
24 #include <map>
25 #include <fstream>
26 #include <chrono>
27 #include <limits>
28 #include "avcodec_trace.h"
29 #include "securec.h"
30 #include "ffmpeg_format_helper.h"
31 #include "ffmpeg_utils.h"
32 #include "buffer/avbuffer.h"
33 #include "plugin/plugin_buffer.h"
34 #include "plugin/plugin_definition.h"
35 #include "common/log.h"
36 #include "meta/video_types.h"
37 #include "avcodec_sysevent.h"
38 #include "demuxer_log_compressor.h"
39 #include "ffmpeg_demuxer_plugin.h"
40 #include "meta/format.h"
41 #include "syspara/parameters.h"
42
43 namespace {
44 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_DEMUXER, "FfmpegDemuxerPlugin" };
45 }
46
47 namespace OHOS {
48 namespace Media {
49 namespace Plugins {
50 namespace Ffmpeg {
51 const uint32_t DEFAULT_READ_SIZE = 4096;
52 const uint32_t DEFAULT_SNIFF_SIZE = 4096 * 4;
53 const int32_t MP3_PROBE_SCORE_LIMIT = 5;
54 const uint32_t STR_MAX_LEN = 4;
55 const uint32_t RANK_MAX = 100;
56 const uint32_t NAL_START_CODE_SIZE = 4;
57 const uint32_t INIT_DOWNLOADS_DATA_SIZE_THRESHOLD = 2 * 1024 * 1024;
58 const int64_t LIVE_FLV_PROBE_SIZE = 100 * 1024 * 2;
59 const uint32_t DEFAULT_CACHE_LIMIT = 50 * 1024 * 1024; // 50M
60 const int64_t INIT_TIME_THRESHOLD = 1000;
61 const uint32_t ID3V2_HEADER_SIZE = 10;
62 const int32_t MS_TO_NS = 1000 * 1000;
63 const uint32_t REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT = 200000;
64
65 // id3v2 tag position
66 const int32_t POS_0 = 0;
67 const int32_t POS_1 = 1;
68 const int32_t POS_2 = 2;
69 const int32_t POS_3 = 3;
70 const int32_t POS_4 = 4;
71 const int32_t POS_5 = 5;
72 const int32_t POS_6 = 6;
73 const int32_t POS_7 = 7;
74 const int32_t POS_8 = 8;
75 const int32_t POS_9 = 9;
76 const int32_t POS_14 = 14;
77 const int32_t POS_21 = 21;
78 const int32_t POS_FF = 0xff;
79 const int32_t LEN_MASK = 0x7f;
80 const int32_t TAG_MASK = 0x80;
81 const int32_t TAG_VERSION_MASK = 0x10;
82
83 namespace {
84 std::map<std::string, std::shared_ptr<AVInputFormat>> g_pluginInputFormat;
85 std::mutex g_mtx;
86
87 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource);
88
89 Status RegisterPlugins(const std::shared_ptr<Register>& reg);
90
91 void ReplaceDelimiter(const std::string &delmiters, char newDelimiter, std::string &str);
92
93 static const std::map<SeekMode, int32_t> g_seekModeToFFmpegSeekFlags = {
94 { SeekMode::SEEK_PREVIOUS_SYNC, AVSEEK_FLAG_BACKWARD },
95 { SeekMode::SEEK_NEXT_SYNC, AVSEEK_FLAG_FRAME },
96 { SeekMode::SEEK_CLOSEST_SYNC, AVSEEK_FLAG_FRAME | AVSEEK_FLAG_BACKWARD }
97 };
98
99 static const std::map<AVCodecID, std::string> g_bitstreamFilterMap = {
100 { AV_CODEC_ID_H264, "h264_mp4toannexb" },
101 };
102
103 static const std::map<AVCodecID, StreamType> g_streamParserMap = {
104 { AV_CODEC_ID_HEVC, StreamType::HEVC },
105 { AV_CODEC_ID_VVC, StreamType::VVC },
106 };
107
108 static const std::vector<AVMediaType> g_streamMediaTypeVec = {
109 AVMEDIA_TYPE_AUDIO,
110 AVMEDIA_TYPE_VIDEO,
111 AVMEDIA_TYPE_SUBTITLE,
112 AVMEDIA_TYPE_TIMEDMETA
113 };
114
115 static std::vector<AVCodecID> g_imageCodecID = {
116 AV_CODEC_ID_MJPEG,
117 AV_CODEC_ID_PNG,
118 AV_CODEC_ID_PAM,
119 AV_CODEC_ID_BMP,
120 AV_CODEC_ID_JPEG2000,
121 AV_CODEC_ID_TARGA,
122 AV_CODEC_ID_TIFF,
123 AV_CODEC_ID_GIF,
124 AV_CODEC_ID_PCX,
125 AV_CODEC_ID_XWD,
126 AV_CODEC_ID_XBM,
127 AV_CODEC_ID_WEBP,
128 AV_CODEC_ID_APNG,
129 AV_CODEC_ID_XPM,
130 AV_CODEC_ID_SVG,
131 };
132
HaveValidParser(const AVCodecID codecId)133 bool HaveValidParser(const AVCodecID codecId)
134 {
135 return g_streamParserMap.count(codecId) != 0;
136 }
137
GetFileDuration(const AVFormatContext & avFormatContext)138 int64_t GetFileDuration(const AVFormatContext& avFormatContext)
139 {
140 int64_t duration = 0;
141 const AVDictionaryEntry *metaDuration = av_dict_get(avFormatContext.metadata, "DURATION", NULL, 0);
142 int64_t us;
143 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
144 if (us > duration) {
145 MEDIA_LOG_D("Get duration from file");
146 duration = us;
147 }
148 }
149
150 if (duration <= 0) {
151 for (uint32_t i = 0; i < avFormatContext.nb_streams; ++i) {
152 auto streamDuration = (ConvertTimeFromFFmpeg(avFormatContext.streams[i]->duration,
153 avFormatContext.streams[i]->time_base)) / 1000; // us
154 if (streamDuration > duration) {
155 MEDIA_LOG_D("Get duration from stream " PUBLIC_LOG_U32, i);
156 duration = streamDuration;
157 }
158 }
159 }
160 return duration;
161 }
162
GetStreamDuration(const AVStream & avStream)163 int64_t GetStreamDuration(const AVStream& avStream)
164 {
165 int64_t duration = 0;
166 const AVDictionaryEntry *metaDuration = av_dict_get(avStream.metadata, "DURATION", NULL, 0);
167 int64_t us;
168 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
169 if (us > duration) {
170 MEDIA_LOG_D("Get duration from stream");
171 duration = us;
172 }
173 }
174 return duration;
175 }
176
CheckStartTime(const AVFormatContext * formatContext,const AVStream * stream,int64_t & timeStamp,int64_t seekTime)177 bool CheckStartTime(const AVFormatContext *formatContext, const AVStream *stream, int64_t &timeStamp, int64_t seekTime)
178 {
179 int64_t startTime = 0;
180 int64_t num = 1000; // ms convert us
181 FALSE_RETURN_V_MSG_E(stream != nullptr, false, "String is nullptr");
182 if (stream->start_time != AV_NOPTS_VALUE) {
183 startTime = stream->start_time;
184 if (timeStamp > 0 && startTime > INT64_MAX - timeStamp) {
185 MEDIA_LOG_E("Seek value overflow with start time: " PUBLIC_LOG_D64 " timeStamp: " PUBLIC_LOG_D64,
186 startTime, timeStamp);
187 return false;
188 }
189 }
190 MEDIA_LOG_D("StartTime: " PUBLIC_LOG_D64, startTime);
191 int64_t fileDuration = formatContext->duration;
192 int64_t streamDuration = stream->duration;
193 if (fileDuration == AV_NOPTS_VALUE || fileDuration <= 0) {
194 fileDuration = GetFileDuration(*formatContext);
195 }
196 if (streamDuration == AV_NOPTS_VALUE || streamDuration <= 0) {
197 streamDuration = GetStreamDuration(*stream);
198 }
199 MEDIA_LOG_D("File duration=" PUBLIC_LOG_D64 ", stream duration=" PUBLIC_LOG_D64, fileDuration, streamDuration);
200 // when timestemp out of file duration, return error
201 if (fileDuration > 0 && seekTime * num > fileDuration) { // fileDuration us
202 MEDIA_LOG_E("Seek to timestamp=" PUBLIC_LOG_D64 " failed, max=" PUBLIC_LOG_D64, timeStamp, fileDuration);
203 return false;
204 }
205 // when timestemp out of stream duration, seek to end of stream
206 if (streamDuration > 0 && timeStamp > streamDuration) {
207 MEDIA_LOG_W("Out of stream, seek to " PUBLIC_LOG_D64, timeStamp);
208 timeStamp = streamDuration;
209 }
210 if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
211 MEDIA_LOG_D("Reset timeStamp by start time [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
212 timeStamp, timeStamp + startTime);
213 timeStamp += startTime;
214 }
215 return true;
216 }
217
ConvertFlagsToFFmpeg(AVStream * avStream,int64_t ffTime,SeekMode mode,int64_t seekTime)218 int ConvertFlagsToFFmpeg(AVStream *avStream, int64_t ffTime, SeekMode mode, int64_t seekTime)
219 {
220 FALSE_RETURN_V_MSG_E(avStream != nullptr && avStream->codecpar != nullptr, -1, "AVStream is nullptr");
221 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ffTime == 0) {
222 return AVSEEK_FLAG_FRAME;
223 }
224 if (avStream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO || seekTime == 0) {
225 return AVSEEK_FLAG_BACKWARD;
226 }
227 if (mode == SeekMode::SEEK_NEXT_SYNC || mode == SeekMode::SEEK_PREVIOUS_SYNC) {
228 return g_seekModeToFFmpegSeekFlags.at(mode);
229 }
230 // find closest time in next and prev
231 int keyFrameNext = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_FRAME);
232 FALSE_RETURN_V_MSG_E(keyFrameNext >= 0, AVSEEK_FLAG_BACKWARD, "Not next key frame");
233
234 int keyFramePrev = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_BACKWARD);
235 FALSE_RETURN_V_MSG_E(keyFramePrev >= 0, AVSEEK_FLAG_FRAME, "Not pre key frame");
236
237 int64_t ffTimePrev = CalculateTimeByFrameIndex(avStream, keyFramePrev);
238 int64_t ffTimeNext = CalculateTimeByFrameIndex(avStream, keyFrameNext);
239 MEDIA_LOG_D("FfTime=" PUBLIC_LOG_D64 ", ffTimePrev=" PUBLIC_LOG_D64 ", ffTimeNext=" PUBLIC_LOG_D64,
240 ffTime, ffTimePrev, ffTimeNext);
241 if (ffTimePrev == ffTimeNext || (ffTimeNext - ffTime < ffTime - ffTimePrev)) {
242 return AVSEEK_FLAG_FRAME;
243 } else {
244 return AVSEEK_FLAG_BACKWARD;
245 }
246 }
247
IsSupportedTrack(const AVStream & avStream)248 bool IsSupportedTrack(const AVStream& avStream)
249 {
250 FALSE_RETURN_V_MSG_E(avStream.codecpar != nullptr, false, "Codecpar is nullptr");
251 if (std::find(g_streamMediaTypeVec.cbegin(), g_streamMediaTypeVec.cend(),
252 avStream.codecpar->codec_type) == g_streamMediaTypeVec.cend()) {
253 MEDIA_LOG_E("Unsupport track type: " PUBLIC_LOG_S,
254 ConvertFFmpegMediaTypeToString(avStream.codecpar->codec_type).data());
255 return false;
256 }
257 if (avStream.codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
258 if (avStream.codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
259 MEDIA_LOG_E("Unsupport raw video track");
260 return false;
261 }
262 if (std::count(g_imageCodecID.begin(), g_imageCodecID.end(), avStream.codecpar->codec_id) > 0) {
263 MEDIA_LOG_E("Unsupport image track");
264 return false;
265 }
266 }
267 return true;
268 }
269 } // namespace
270
271 std::atomic<int> FFmpegDemuxerPlugin::readatIndex_ = 0;
FFmpegDemuxerPlugin(std::string name)272 FFmpegDemuxerPlugin::FFmpegDemuxerPlugin(std::string name)
273 : DemuxerPlugin(std::move(name)),
274 seekable_(Seekable::SEEKABLE),
275 ioContext_(),
276 selectedTrackIds_(),
277 cacheQueue_("cacheQueue"),
278 streamParserInited_(false),
279 parserRefIoContext_()
280 {
281 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
282 MEDIA_LOG_D("In");
283 #ifndef _WIN32
284 (void)mallopt(M_SET_THREAD_CACHE, M_THREAD_CACHE_DISABLE);
285 (void)mallopt(M_DELAYED_FREE, M_DELAYED_FREE_DISABLE);
286 #endif
287 av_log_set_callback(FfmpegLogPrint);
288 #ifdef BUILD_ENG_VERSION
289 std::string dumpModeStr = OHOS::system::GetParameter("FFmpegDemuxerPlugin.dump", "0");
290 dumpMode_ = static_cast<DumpMode>(strtoul(dumpModeStr.c_str(), nullptr, 2)); // 2 is binary
291 MEDIA_LOG_D("Dump mode = %s(%lu)", dumpModeStr.c_str(), dumpMode_);
292 #endif
293 MEDIA_LOG_D("Out");
294 }
295
~FFmpegDemuxerPlugin()296 FFmpegDemuxerPlugin::~FFmpegDemuxerPlugin()
297 {
298 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
299 MEDIA_LOG_D("In");
300 #ifndef _WIN32
301 (void)mallopt(M_FLUSH_THREAD_CACHE, 0);
302 #endif
303 formatContext_ = nullptr;
304 pluginImpl_ = nullptr;
305 avbsfContext_ = nullptr;
306 streamParser_ = nullptr;
307 referenceParser_ = nullptr;
308 parserRefFormatContext_ = nullptr;
309 selectedTrackIds_.clear();
310 if (firstFrame_ != nullptr) {
311 av_packet_free(&firstFrame_);
312 av_free(firstFrame_);
313 firstFrame_ = nullptr;
314 }
315 MEDIA_LOG_D("Out");
316 }
317
Dump(const DumpParam & dumpParam)318 void FFmpegDemuxerPlugin::Dump(const DumpParam &dumpParam)
319 {
320 std::string path;
321 switch (dumpParam.mode) {
322 case DUMP_READAT_INPUT:
323 path = "Readat_index." + std::to_string(dumpParam.index) + "_offset." + std::to_string(dumpParam.offset) +
324 "_size." + std::to_string(dumpParam.size);
325 break;
326 case DUMP_AVPACKET_OUTPUT:
327 path = "AVPacket_index." + std::to_string(dumpParam.index) + "_track." +
328 std::to_string(dumpParam.trackId) + "_pts." + std::to_string(dumpParam.pts) + "_pos." +
329 std::to_string(dumpParam.pos);
330 break;
331 case DUMP_AVBUFFER_OUTPUT:
332 path = "AVBuffer_track." + std::to_string(dumpParam.trackId) + "_index." +
333 std::to_string(dumpParam.index) + "_pts." + std::to_string(dumpParam.pts);
334 break;
335 default:
336 return;
337 }
338 std::ofstream ofs;
339 path = "/data/ff_dump/" + path;
340 ofs.open(path, std::ios::out); // | std::ios::app
341 if (ofs.is_open()) {
342 ofs.write(reinterpret_cast<char*>(dumpParam.buf), dumpParam.size);
343 ofs.close();
344 }
345 MEDIA_LOG_D("Dump path:" PUBLIC_LOG_S, path.c_str());
346 }
347
Reset()348 Status FFmpegDemuxerPlugin::Reset()
349 {
350 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
351 MEDIA_LOG_D("In");
352 readatIndex_ = 0;
353 avpacketIndex_ = 0;
354 ioContext_.offset = 0;
355 ioContext_.eos = false;
356 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
357 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
358 }
359 selectedTrackIds_.clear();
360 pluginImpl_.reset();
361 formatContext_.reset();
362 avbsfContext_.reset();
363 trackMtx_.clear();
364 trackDfxInfoMap_.clear();
365 return Status::OK;
366 }
367
InitBitStreamContext(const AVStream & avStream)368 void FFmpegDemuxerPlugin::InitBitStreamContext(const AVStream& avStream)
369 {
370 FALSE_RETURN_MSG(avStream.codecpar != nullptr, "Codecpar is nullptr");
371 AVCodecID codecID = avStream.codecpar->codec_id;
372 MEDIA_LOG_D("For track " PUBLIC_LOG_D32, avStream.index);
373 FALSE_RETURN_MSG(g_bitstreamFilterMap.count(codecID) != 0, "Can not match any BitStreamContext");
374 const AVBitStreamFilter* avBitStreamFilter = av_bsf_get_by_name(g_bitstreamFilterMap.at(codecID).c_str());
375
376 FALSE_RETURN_MSG((avBitStreamFilter != nullptr), "Call av_bsf_get_by_name failed for" PUBLIC_LOG_S,
377 g_bitstreamFilterMap.at(codecID).c_str());
378
379 if (!avbsfContext_) {
380 AVBSFContext* avbsfContext {nullptr};
381 int ret = av_bsf_alloc(avBitStreamFilter, &avbsfContext);
382 FALSE_RETURN_MSG((ret >= 0 && avbsfContext != nullptr),
383 "Call av_bsf_alloc failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
384
385 ret = avcodec_parameters_copy(avbsfContext->par_in, avStream.codecpar);
386 FALSE_RETURN_MSG((ret >= 0), "Call avcodec_parameters_copy failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
387
388 ret = av_bsf_init(avbsfContext);
389 FALSE_RETURN_MSG((ret >= 0), "Call av_bsf_init failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
390
391 avbsfContext_ = std::shared_ptr<AVBSFContext>(avbsfContext, [](AVBSFContext* ptr) {
392 if (ptr) {
393 av_bsf_free(&ptr);
394 }
395 });
396 }
397 FALSE_RETURN_MSG(avbsfContext_ != nullptr, "Stream " PUBLIC_LOG_S " will not be converted to annexb",
398 g_bitstreamFilterMap.at(codecID).c_str());
399 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will convert to annexb", avStream.index);
400 }
401
ConvertAvcToAnnexb(AVPacket & pkt)402 Status FFmpegDemuxerPlugin::ConvertAvcToAnnexb(AVPacket& pkt)
403 {
404 int ret = av_bsf_send_packet(avbsfContext_.get(), &pkt);
405 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
406 "Call av_bsf_send_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
407 av_packet_unref(&pkt);
408
409 ret = av_bsf_receive_packet(avbsfContext_.get(), &pkt);
410 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
411 "Call av_bsf_receive_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
412 return Status::OK;
413 }
414
ConvertHevcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)415 Status FFmpegDemuxerPlugin::ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
416 {
417 size_t cencInfoSize = 0;
418 uint8_t *cencInfo = av_packet_get_side_data(samplePacket->pkts[0], AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
419 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, cencInfo, cencInfoSize, false);
420 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) &&
421 streamParser_->IsSyncFrame(pkt.data, pkt.size)) {
422 pkt.flags = static_cast<int32_t>(static_cast<uint32_t>(pkt.flags) | static_cast<uint32_t>(AV_PKT_FLAG_KEY));
423 }
424 return Status::OK;
425 }
426
ConvertVvcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)427 Status FFmpegDemuxerPlugin::ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
428 {
429 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, nullptr, 0, false);
430 return Status::OK;
431 }
432
WriteBuffer(std::shared_ptr<AVBuffer> outBuffer,const uint8_t * writeData,int32_t writeSize)433 Status FFmpegDemuxerPlugin::WriteBuffer(
434 std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize)
435 {
436 FALSE_RETURN_V_MSG_E(outBuffer != nullptr, Status::ERROR_NULL_POINTER, "Buffer is nullptr");
437 if (writeData != nullptr && writeSize > 0) {
438 FALSE_RETURN_V_MSG_E(outBuffer->memory_ != nullptr, Status::ERROR_NULL_POINTER, "Memory is nullptr");
439 int32_t ret = outBuffer->memory_->Write(writeData, writeSize, 0);
440 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_INVALID_OPERATION, "Memory write failed");
441 }
442
443 MEDIA_LOG_D("CurrentBuffer: [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_U32 "]",
444 outBuffer->pts_, outBuffer->duration_, outBuffer->flag_);
445 return Status::OK;
446 }
447
SetDrmCencInfo(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)448 Status FFmpegDemuxerPlugin::SetDrmCencInfo(
449 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
450 {
451 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
452 "Sample is nullptr");
453 FALSE_RETURN_V_MSG_E((samplePacket != nullptr && samplePacket->pkts.size() > 0), Status::ERROR_INVALID_OPERATION,
454 "Packet is nullptr");
455 FALSE_RETURN_V_MSG_E((samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0),
456 Status::ERROR_INVALID_OPERATION, "Packet empty");
457
458 size_t cencInfoSize = 0;
459 MetaDrmCencInfo *cencInfo = (MetaDrmCencInfo *)av_packet_get_side_data(samplePacket->pkts[0],
460 AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
461 if ((cencInfo != nullptr) && (cencInfoSize != 0)) {
462 std::vector<uint8_t> drmCencVec(reinterpret_cast<uint8_t *>(cencInfo),
463 (reinterpret_cast<uint8_t *>(cencInfo)) + sizeof(MetaDrmCencInfo));
464 sample->meta_->SetData(Media::Tag::DRM_CENC_INFO, std::move(drmCencVec));
465 }
466 return Status::OK;
467 }
468
GetNextFrame(const uint8_t * data,const uint32_t size)469 bool FFmpegDemuxerPlugin::GetNextFrame(const uint8_t *data, const uint32_t size)
470 {
471 if (size < NAL_START_CODE_SIZE) {
472 return false;
473 }
474 bool hasShortStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 1); // 001
475 bool hasLongStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1); // 0001
476 return hasShortStartCode || hasLongStartCode;
477 }
478
NeedCombineFrame(uint32_t trackId)479 bool FFmpegDemuxerPlugin::NeedCombineFrame(uint32_t trackId)
480 {
481 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, false, "AVFormatContext is nullptr");
482 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MPEGTS &&
483 formatContext_->streams[trackId]->codecpar->codec_id == AV_CODEC_ID_HEVC) {
484 return true;
485 }
486 return false;
487 }
488
CombinePackets(std::shared_ptr<SamplePacket> samplePacket)489 AVPacket* FFmpegDemuxerPlugin::CombinePackets(std::shared_ptr<SamplePacket> samplePacket)
490 {
491 AVPacket *tempPkt = nullptr;
492 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) && samplePacket->pkts.size() > 1) {
493 int totalSize = 0;
494 for (auto pkt : samplePacket->pkts) {
495 FALSE_RETURN_V_MSG_E(pkt != nullptr, nullptr, "AVPacket is nullptr");
496 totalSize += pkt->size;
497 }
498 tempPkt = av_packet_alloc();
499 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, nullptr, "Temp packet is nullptr");
500 int ret = av_new_packet(tempPkt, totalSize);
501 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "Call av_new_packet failed");
502 av_packet_copy_props(tempPkt, samplePacket->pkts[0]);
503 int offset = 0;
504 bool copySuccess = true;
505 for (auto pkt : samplePacket->pkts) {
506 if (pkt == nullptr) {
507 copySuccess = false;
508 MEDIA_LOG_E("Cache packet is nullptr");
509 break;
510 }
511 ret = memcpy_s(tempPkt->data + offset, pkt->size, pkt->data, pkt->size);
512 if (ret != EOK) {
513 copySuccess = false;
514 MEDIA_LOG_E("Memcpy failed, ret:" PUBLIC_LOG_D32, ret);
515 break;
516 }
517 offset += pkt->size;
518 }
519 if (!copySuccess) {
520 av_packet_free(&tempPkt);
521 av_free(tempPkt);
522 tempPkt = nullptr;
523 return nullptr;
524 }
525 tempPkt->size = totalSize;
526 MEDIA_LOG_D("Combine " PUBLIC_LOG_ZU " packets, total size=" PUBLIC_LOG_D32,
527 samplePacket->pkts.size(), totalSize);
528 } else {
529 tempPkt = samplePacket->pkts[0];
530 }
531 return tempPkt;
532 }
533
ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample,AVPacket * srcAVPacket,std::shared_ptr<SamplePacket> dstSamplePacket)534 Status FFmpegDemuxerPlugin::ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* srcAVPacket,
535 std::shared_ptr<SamplePacket> dstSamplePacket)
536 {
537 Status ret = Status::OK;
538 auto codecId = formatContext_->streams[srcAVPacket->stream_index]->codecpar->codec_id;
539 if (codecId == AV_CODEC_ID_HEVC && streamParser_ != nullptr && streamParserInited_) {
540 ret = ConvertHevcToAnnexb(*srcAVPacket, dstSamplePacket);
541 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_HEVC);
542 } else if (codecId == AV_CODEC_ID_VVC && streamParser_ != nullptr && streamParserInited_) {
543 ret = ConvertVvcToAnnexb(*srcAVPacket, dstSamplePacket);
544 } else if (codecId == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
545 ret = ConvertAvcToAnnexb(*srcAVPacket);
546 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_H264);
547 }
548 if (ioContext_.retry) {
549 ioContext_.retry = false;
550 formatContext_->pb->eof_reached = 0;
551 formatContext_->pb->error = 0;
552 cacheQueue_.Pop(dstSamplePacket->pkts[0]->stream_index);
553 return Status::ERROR_AGAIN;
554 }
555 return ret;
556 }
557
WriteBufferAttr(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)558 void FFmpegDemuxerPlugin::WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
559 {
560 AVStream *avStream = formatContext_->streams[samplePacket->pkts[0]->stream_index];
561 if (samplePacket->pkts[0]->pts != AV_NOPTS_VALUE) {
562 sample->pts_ = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->pts, avStream->time_base));
563 }
564 // durantion dts
565 if (samplePacket->pkts[0]->duration != AV_NOPTS_VALUE) {
566 int64_t duration = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->duration, avStream->time_base));
567 sample->duration_ = duration;
568 sample->meta_->SetData(Media::Tag::BUFFER_DURATION, duration);
569 }
570 if (samplePacket->pkts[0]->dts != AV_NOPTS_VALUE) {
571 int64_t dts = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->dts, avStream->time_base));
572 sample->dts_ = dts;
573 sample->meta_->SetData(Media::Tag::BUFFER_DECODING_TIMESTAMP, dts);
574 }
575 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
576 avStream->codecpar->codec_id != AV_CODEC_ID_H264 &&
577 firstFrame_ && samplePacket->pkts[0]->dts == firstFrame_->dts) {
578 if (streamParser_ != nullptr) {
579 streamParser_->ResetXPSSendStatus();
580 }
581 }
582 }
583
ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)584 Status FFmpegDemuxerPlugin::ConvertAVPacketToSample(
585 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
586 {
587 FALSE_RETURN_V_MSG_E(samplePacket != nullptr && samplePacket->pkts.size() > 0 &&
588 samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0,
589 Status::ERROR_INVALID_OPERATION, "Input packet is nullptr or empty");
590 MEDIA_LOG_D("Convert packet info for track " PUBLIC_LOG_D32, samplePacket->pkts[0]->stream_index);
591 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
592 "Input sample is nullptr");
593
594 WriteBufferAttr(sample, samplePacket);
595
596 // convert
597 AVPacket *tempPkt = CombinePackets(samplePacket);
598 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, Status::ERROR_INVALID_OPERATION, "Temp packet is empty");
599 Status ret = ConvertPacketToAnnexb(sample, tempPkt, samplePacket);
600 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Convert annexb failed");
601
602 // flag\copy
603 int32_t remainSize = tempPkt->size - static_cast<int32_t>(samplePacket->offset);
604 int32_t copySize = remainSize < sample->memory_->GetCapacity() ? remainSize : sample->memory_->GetCapacity();
605 MEDIA_LOG_D("Convert size [" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
606 tempPkt->size, remainSize, copySize, samplePacket->offset);
607 uint32_t flag = ConvertFlagsFromFFmpeg(*tempPkt, (copySize != tempPkt->size));
608 SetDrmCencInfo(sample, samplePacket);
609
610 sample->flag_ = flag;
611 ret = WriteBuffer(sample, tempPkt->data + samplePacket->offset, copySize);
612 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
613
614 if (!samplePacket->isEOS) {
615 trackDfxInfoMap_[tempPkt->stream_index].lastPts = sample->pts_;
616 trackDfxInfoMap_[tempPkt->stream_index].lastDurantion = sample->duration_;
617 trackDfxInfoMap_[tempPkt->stream_index].lastPos = tempPkt->pos;
618 }
619 #ifdef BUILD_ENG_VERSION
620 DumpParam dumpParam {DumpMode(DUMP_AVBUFFER_OUTPUT & dumpMode_), tempPkt->data + samplePacket->offset,
621 tempPkt->stream_index, -1, copySize, trackDfxInfoMap_[tempPkt->stream_index].frameIndex++, tempPkt->pts, -1};
622 Dump(dumpParam);
623 #endif
624 if (tempPkt != nullptr && tempPkt->size != samplePacket->pkts[0]->size) {
625 av_packet_free(&tempPkt);
626 av_free(tempPkt);
627 tempPkt = nullptr;
628 }
629
630 if (copySize < remainSize) {
631 samplePacket->offset += static_cast<uint32_t>(copySize);
632 MEDIA_LOG_D("Buffer is not enough, next buffer to copy remain data");
633 return Status::ERROR_NOT_ENOUGH_DATA;
634 }
635 return Status::OK;
636 }
637
PushEOSToAllCache()638 Status FFmpegDemuxerPlugin::PushEOSToAllCache()
639 {
640 Status ret = Status::OK;
641 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
642 auto streamIndex = selectedTrackIds_[i];
643 MEDIA_LOG_I("Track " PUBLIC_LOG_D32, streamIndex);
644 std::shared_ptr<SamplePacket> eosSample = std::make_shared<SamplePacket>();
645 eosSample->isEOS = true;
646 cacheQueue_.Push(streamIndex, eosSample);
647 ret = CheckCacheDataLimit(streamIndex);
648 }
649 return ret;
650 }
651
WebvttPktProcess(AVPacket * pkt)652 bool FFmpegDemuxerPlugin::WebvttPktProcess(AVPacket *pkt)
653 {
654 auto trackId = pkt->stream_index;
655 if (pkt->size > 0) { // vttc
656 return false;
657 } else { // vtte
658 if (cacheQueue_.HasCache(trackId)) {
659 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
660 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts.size() > 0 &&
661 cacheSamplePacket->pkts[0] != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
662 cacheSamplePacket->pkts[0]->duration = pkt->pts - cacheSamplePacket->pkts[0]->pts;
663 }
664 }
665 }
666 av_packet_free(&pkt);
667 return true;
668 }
669
IsWebvttMP4(const AVStream * avStream)670 bool FFmpegDemuxerPlugin::IsWebvttMP4(const AVStream *avStream)
671 {
672 if (avStream->codecpar->codec_id == AV_CODEC_ID_WEBVTT &&
673 FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4) {
674 return true;
675 }
676 return false;
677 }
678
WebvttMP4EOSProcess(const AVPacket * pkt)679 void FFmpegDemuxerPlugin::WebvttMP4EOSProcess(const AVPacket *pkt)
680 {
681 if (pkt != nullptr) {
682 auto trackId = pkt->stream_index;
683 AVStream *avStream = formatContext_->streams[trackId];
684 if (IsWebvttMP4(avStream) && pkt->size == 0 && cacheQueue_.HasCache(trackId)) {
685 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
686 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
687 cacheSamplePacket->pkts[0]->duration =
688 formatContext_->streams[pkt->stream_index]->duration - cacheSamplePacket->pkts[0]->pts;
689 }
690 }
691 }
692 }
693
ReadPacketToCacheQueue(const uint32_t readId)694 Status FFmpegDemuxerPlugin::ReadPacketToCacheQueue(const uint32_t readId)
695 {
696 std::lock_guard<std::mutex> lock(mutex_);
697 AVPacket *pkt = nullptr;
698 bool continueRead = true;
699 Status ret = Status::OK;
700 while (continueRead) {
701 if (pkt == nullptr) {
702 pkt = av_packet_alloc();
703 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
704 }
705 std::unique_lock<std::mutex> sLock(syncMutex_);
706 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
707 sLock.unlock();
708 if (ffmpegRet == AVERROR_EOF) { // eos
709 WebvttMP4EOSProcess(pkt);
710 av_packet_free(&pkt);
711 ret = PushEOSToAllCache();
712 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Push eos failed");
713 return Status::END_OF_STREAM;
714 }
715 if (ffmpegRet < 0) { // fail
716 av_packet_free(&pkt);
717 MEDIA_LOG_E("Call av_read_frame failed:" PUBLIC_LOG_S ", retry: " PUBLIC_LOG_D32,
718 AVStrError(ffmpegRet).c_str(), int(ioContext_.retry));
719 if (ioContext_.retry) {
720 formatContext_->pb->eof_reached = 0;
721 formatContext_->pb->error = 0;
722 ioContext_.retry = false;
723 return Status::ERROR_AGAIN;
724 }
725 return Status::ERROR_UNKNOWN;
726 }
727 auto trackId = pkt->stream_index;
728 if (!TrackIsSelected(trackId)) {
729 av_packet_unref(pkt);
730 continue;
731 }
732 AVStream *avStream = formatContext_->streams[trackId];
733 if (IsWebvttMP4(avStream) && WebvttPktProcess(pkt)) {
734 break;
735 } else if (!IsWebvttMP4(avStream) && (!NeedCombineFrame(readId) ||
736 (cacheQueue_.HasCache(static_cast<uint32_t>(trackId)) && GetNextFrame(pkt->data, pkt->size)))) {
737 continueRead = false;
738 }
739 ret = AddPacketToCacheQueue(pkt);
740 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Add cache failed");
741 pkt = nullptr;
742 }
743 return ret;
744 }
745
SetEosSample(std::shared_ptr<AVBuffer> sample)746 Status FFmpegDemuxerPlugin::SetEosSample(std::shared_ptr<AVBuffer> sample)
747 {
748 MEDIA_LOG_D("In");
749 sample->pts_ = 0;
750 sample->flag_ = (uint32_t)(AVBufferFlag::EOS);
751 Status ret = WriteBuffer(sample, nullptr, 0);
752 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
753 MEDIA_LOG_I("Out");
754 return Status::OK;
755 }
756
Start()757 Status FFmpegDemuxerPlugin::Start()
758 {
759 return Status::OK;
760 }
761
Stop()762 Status FFmpegDemuxerPlugin::Stop()
763 {
764 return Status::OK;
765 }
766
767 // Write packet unimplemented, return 0
AVWritePacket(void * opaque,uint8_t * buf,int bufSize)768 int FFmpegDemuxerPlugin::AVWritePacket(void* opaque, uint8_t* buf, int bufSize)
769 {
770 (void)opaque;
771 (void)buf;
772 (void)bufSize;
773 return 0;
774 }
775
CheckContextIsValid(void * opaque,int & bufSize)776 int FFmpegDemuxerPlugin::CheckContextIsValid(void* opaque, int &bufSize)
777 {
778 int ret = -1;
779 auto ioContext = static_cast<IOContext*>(opaque);
780 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
781 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, ret, "DataSource is nullptr");
782 FALSE_RETURN_V_MSG_E(ioContext->offset <= INT64_MAX - static_cast<int64_t>(bufSize), ret, "Offset invalid");
783
784 if (ioContext->dataSource->IsDash() && ioContext->eos == true) {
785 MEDIA_LOG_I("Read eos");
786 return AVERROR_EOF;
787 }
788
789 MEDIA_LOG_D("Offset: " PUBLIC_LOG_D64 ", totalSize: " PUBLIC_LOG_U64, ioContext->offset, ioContext->fileSize);
790 if (ioContext->fileSize > 0) {
791 FALSE_RETURN_V_MSG_E(static_cast<uint64_t>(ioContext->offset) <= ioContext->fileSize, ret, "Out of file size");
792 if (static_cast<size_t>(ioContext->offset + bufSize) > ioContext->fileSize) {
793 bufSize = static_cast<int64_t>(ioContext->fileSize) - ioContext->offset;
794 }
795 }
796 return 0;
797 }
798
799 // Write packet data into the buffer provided by ffmpeg
AVReadPacket(void * opaque,uint8_t * buf,int bufSize)800 int FFmpegDemuxerPlugin::AVReadPacket(void* opaque, uint8_t* buf, int bufSize)
801 {
802 int ret = CheckContextIsValid(opaque, bufSize);
803 FALSE_RETURN_V(ret == 0, ret);
804
805 ret = -1;
806 auto ioContext = static_cast<IOContext*>(opaque);
807 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
808 auto buffer = std::make_shared<Buffer>();
809 FALSE_RETURN_V_MSG_E(buffer != nullptr, ret, "Buffer is nullptr");
810 auto bufData = buffer->WrapMemory(buf, bufSize, 0);
811 FALSE_RETURN_V_MSG_E(buffer->GetMemory() != nullptr, ret, "Memory is nullptr");
812
813 MediaAVCodec::AVCodecTrace trace("AVReadPacket_ReadAt");
814 auto result = ioContext->dataSource->ReadAt(ioContext->offset, buffer, static_cast<size_t>(bufSize));
815 int dataSize = static_cast<int>(buffer->GetMemory()->GetSize());
816 MEDIA_LOG_D("Want:" PUBLIC_LOG_D32 ", Get:" PUBLIC_LOG_D32 ", offset:" PUBLIC_LOG_D64 ", index:" PUBLIC_LOG_D32,
817 bufSize, dataSize, ioContext->offset, readatIndex_.load());
818 #ifdef BUILD_ENG_VERSION
819 DumpParam dumpParam {DumpMode(DUMP_READAT_INPUT & ioContext->dumpMode), buf, -1, ioContext->offset,
820 dataSize, readatIndex_++, -1, -1};
821 Dump(dumpParam);
822 #endif
823 switch (result) {
824 case Status::OK:
825 case Status::ERROR_AGAIN:
826 if (dataSize == 0) {
827 MEDIA_LOG_I("Read again");
828 ioContext->retry = true;
829 } else {
830 ioContext->offset += dataSize;
831 ret = dataSize;
832 }
833 break;
834 case Status::END_OF_STREAM:
835 MEDIA_LOG_I("Read end");
836 ioContext->eos = true;
837 ret = AVERROR_EOF;
838 break;
839 default:
840 MEDIA_LOG_I("Read failed " PUBLIC_LOG_D32, static_cast<int>(result));
841 break;
842 }
843
844 if (!ioContext->initCompleted) {
845 if (ioContext->initDownloadDataSize <= UINT32_MAX - static_cast<uint32_t>(dataSize)) {
846 ioContext->initDownloadDataSize += static_cast<uint32_t>(dataSize);
847 } else {
848 MEDIA_LOG_W("DataSize " PUBLIC_LOG_U32 " is invalid", static_cast<uint32_t>(dataSize));
849 }
850 }
851
852 return ret;
853 }
854
AVSeek(void * opaque,int64_t offset,int whence)855 int64_t FFmpegDemuxerPlugin::AVSeek(void* opaque, int64_t offset, int whence)
856 {
857 auto ioContext = static_cast<IOContext*>(opaque);
858 uint64_t newPos = 0;
859 FALSE_RETURN_V_MSG_E(ioContext != nullptr, newPos, "IOContext is nullptr");
860 switch (whence) {
861 case SEEK_SET:
862 newPos = static_cast<uint64_t>(offset);
863 ioContext->offset = newPos;
864 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
865 whence, offset, newPos);
866 break;
867 case SEEK_CUR:
868 newPos = ioContext->offset + offset;
869 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
870 whence, offset, newPos);
871 break;
872 case SEEK_END:
873 case AVSEEK_SIZE: {
874 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, newPos, "DataSource is nullptr");
875 if (ioContext->dataSource->IsDash()) {
876 return -1;
877 }
878 uint64_t mediaDataSize = 0;
879 if (ioContext->dataSource->GetSize(mediaDataSize) == Status::OK && (mediaDataSize > 0)) {
880 newPos = mediaDataSize + offset;
881 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
882 whence, offset, newPos);
883 }
884 break;
885 }
886 default:
887 MEDIA_LOG_E("Unexpected whence " PUBLIC_LOG_D32, whence);
888 break;
889 }
890 if (whence != AVSEEK_SIZE) {
891 ioContext->offset = newPos;
892 }
893 MEDIA_LOG_D("Current offset: " PUBLIC_LOG_D64 ", new pos: " PUBLIC_LOG_U64, ioContext->offset, newPos);
894 return newPos;
895 }
896
AllocAVIOContext(int flags,IOContext * ioContext)897 AVIOContext* FFmpegDemuxerPlugin::AllocAVIOContext(int flags, IOContext *ioContext)
898 {
899 auto buffer = static_cast<unsigned char*>(av_malloc(DEFAULT_READ_SIZE));
900 FALSE_RETURN_V_MSG_E(buffer != nullptr, nullptr, "Call av_malloc failed");
901
902 AVIOContext* avioContext = avio_alloc_context(
903 buffer, DEFAULT_READ_SIZE, flags & AVIO_FLAG_WRITE, static_cast<void*>(ioContext),
904 AVReadPacket, AVWritePacket, AVSeek);
905 if (avioContext == nullptr) {
906 MEDIA_LOG_E("Call avio_alloc_context failed");
907 av_free(buffer);
908 return nullptr;
909 }
910 avioContext->seekable = (seekable_ == Seekable::SEEKABLE) ? AVIO_SEEKABLE_NORMAL : 0;
911 if (!(static_cast<uint32_t>(flags) & static_cast<uint32_t>(AVIO_FLAG_WRITE))) {
912 avioContext->buf_ptr = avioContext->buf_end;
913 avioContext->write_flag = 0;
914 }
915 return avioContext;
916 }
917
FreeContext(AVFormatContext * formatContext,AVIOContext * avioContext)918 void FreeContext(AVFormatContext* formatContext, AVIOContext* avioContext)
919 {
920 if (formatContext) {
921 avformat_close_input(&formatContext);
922 }
923 if (avioContext) {
924 if (avioContext->buffer) {
925 av_freep(&(avioContext->buffer));
926 }
927 avio_context_free(&avioContext);
928 }
929 }
930
ParseHeader(AVFormatContext * formatContext,std::shared_ptr<AVInputFormat> pluginImpl,AVDictionary ** options)931 int32_t ParseHeader(AVFormatContext* formatContext, std::shared_ptr<AVInputFormat> pluginImpl, AVDictionary **options)
932 {
933 FALSE_RETURN_V_MSG_E(formatContext && pluginImpl, -1, "AVFormatContext is nullptr");
934 MediaAVCodec::AVCodecTrace trace("ffmpeg_init");
935
936 AVIOContext* avioContext = formatContext->pb;
937 auto begin = std::chrono::steady_clock::now();
938 int ret = avformat_open_input(&formatContext, nullptr, pluginImpl.get(), options);
939 if (ret < 0) {
940 FreeContext(formatContext, avioContext);
941 MEDIA_LOG_E("Call avformat_open_input failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
942 pluginImpl->name, AVStrError(ret).c_str());
943 return ret;
944 }
945
946 auto open = std::chrono::steady_clock::now();
947 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext) == FileType::FLV) { // Fix init live-flv-source too slow
948 formatContext->probesize = LIVE_FLV_PROBE_SIZE;
949 }
950
951 ret = avformat_find_stream_info(formatContext, NULL);
952 auto parse = std::chrono::steady_clock::now();
953 int64_t openSpend = static_cast<int64_t>(
954 std::chrono::duration_cast<std::chrono::milliseconds>(open - begin).count());
955 int64_t parseSpend = static_cast<int64_t>(
956 std::chrono::duration_cast<std::chrono::milliseconds>(parse - open).count());
957 if ((parseSpend < 0) || (openSpend > INT64_MAX - parseSpend) || (openSpend + parseSpend > INIT_TIME_THRESHOLD)) {
958 MEDIA_LOG_W("Spend [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]", openSpend, parseSpend);
959 }
960 if (ret < 0) {
961 FreeContext(formatContext, avioContext);
962 MEDIA_LOG_E("Parse stream info failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
963 pluginImpl->name, AVStrError(ret).c_str());
964 return ret;
965 }
966 return 0;
967 }
968
InitAVFormatContext(IOContext * ioContext)969 std::shared_ptr<AVFormatContext> FFmpegDemuxerPlugin::InitAVFormatContext(IOContext *ioContext)
970 {
971 AVFormatContext* formatContext = avformat_alloc_context();
972 FALSE_RETURN_V_MSG_E(formatContext != nullptr, nullptr, "AVFormatContext is nullptr");
973
974 formatContext->pb = AllocAVIOContext(AVIO_FLAG_READ, ioContext);
975 if (formatContext->pb == nullptr) {
976 FreeContext(formatContext, nullptr);
977 return nullptr;
978 }
979
980 formatContext->flags = static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_CUSTOM_IO);
981 if (std::string(pluginImpl_->name) == "mp3") {
982 formatContext->flags =
983 static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_FAST_SEEK);
984 }
985 AVDictionary *options = nullptr;
986 if (ioContext_.dataSource->IsDash()) {
987 av_dict_set(&options, "use_tfdt", "true", 0);
988 }
989
990 int ret = ParseHeader(formatContext, pluginImpl_, &options);
991 av_dict_free(&options);
992 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "ParseHeader failed");
993
994 std::shared_ptr<AVFormatContext> retFormatContext =
995 std::shared_ptr<AVFormatContext>(formatContext, [](AVFormatContext *ptr) {
996 if (ptr) {
997 auto ctx = ptr->pb;
998 avformat_close_input(&ptr);
999 if (ctx) {
1000 ctx->opaque = nullptr;
1001 av_freep(&(ctx->buffer));
1002 av_opt_free(ctx);
1003 avio_context_free(&ctx);
1004 ctx = nullptr;
1005 }
1006 }
1007 });
1008 return retFormatContext;
1009 }
1010
NotifyInitializationCompleted()1011 void FFmpegDemuxerPlugin::NotifyInitializationCompleted()
1012 {
1013 ioContext_.initCompleted = true;
1014 if (ioContext_.initDownloadDataSize >= INIT_DOWNLOADS_DATA_SIZE_THRESHOLD) {
1015 MEDIA_LOG_I("Large init size %{public}u", ioContext_.initDownloadDataSize);
1016 MediaAVCodec::DemuxerInitEventWrite(ioContext_.initDownloadDataSize, pluginName_);
1017 }
1018 }
1019
SetDataSource(const std::shared_ptr<DataSource> & source)1020 Status FFmpegDemuxerPlugin::SetDataSource(const std::shared_ptr<DataSource>& source)
1021 {
1022 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1023 FALSE_RETURN_V_MSG_E(formatContext_ == nullptr, Status::ERROR_WRONG_STATE, "AVFormatContext is nullptr");
1024 FALSE_RETURN_V_MSG_E(source != nullptr, Status::ERROR_INVALID_PARAMETER, "DataSource is nullptr");
1025 ioContext_.dataSource = source;
1026 ioContext_.offset = 0;
1027 ioContext_.eos = false;
1028 ioContext_.dumpMode = dumpMode_;
1029 seekable_ = ioContext_.dataSource->IsDash() ? Plugins::Seekable::UNSEEKABLE : source->GetSeekable();
1030 if (seekable_ == Plugins::Seekable::SEEKABLE) {
1031 ioContext_.dataSource->GetSize(ioContext_.fileSize);
1032 } else {
1033 ioContext_.fileSize = -1;
1034 }
1035 MEDIA_LOG_I("FileSize: " PUBLIC_LOG_U64 ", seekable: " PUBLIC_LOG_D32, ioContext_.fileSize, seekable_);
1036 {
1037 std::lock_guard<std::mutex> glock(g_mtx);
1038 pluginImpl_ = g_pluginInputFormat[pluginName_];
1039 }
1040 FALSE_RETURN_V_MSG_E(pluginImpl_ != nullptr, Status::ERROR_UNSUPPORTED_FORMAT, "No match inputformat");
1041 formatContext_ = InitAVFormatContext(&ioContext_);
1042 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1043 InitParser();
1044
1045 NotifyInitializationCompleted();
1046 MEDIA_LOG_I("Out");
1047 cachelimitSize_ = DEFAULT_CACHE_LIMIT;
1048 return Status::OK;
1049 }
1050
InitParser()1051 void FFmpegDemuxerPlugin::InitParser()
1052 {
1053 FALSE_RETURN_MSG(formatContext_ != nullptr, "AVFormatContext is nullptr");
1054 ParserBoxInfo();
1055 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1056 if (g_bitstreamFilterMap.count(formatContext_->streams[trackIndex]->codecpar->codec_id) != 0) {
1057 InitBitStreamContext(*(formatContext_->streams[trackIndex]));
1058 break;
1059 }
1060 if (HaveValidParser(formatContext_->streams[trackIndex]->codecpar->codec_id)) {
1061 streamParser_ = StreamParserManager::Create(g_streamParserMap.at(
1062 formatContext_->streams[trackIndex]->codecpar->codec_id));
1063 if (streamParser_ == nullptr) {
1064 MEDIA_LOG_W("Init failed");
1065 } else {
1066 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will be converted", trackIndex);
1067 }
1068 break;
1069 }
1070 }
1071 }
1072
GetSeiInfo()1073 Status FFmpegDemuxerPlugin::GetSeiInfo()
1074 {
1075 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1076 Status ret = Status::OK;
1077 if (streamParser_ != nullptr && !streamParserInited_) {
1078 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1079 auto avStream = formatContext_->streams[trackIndex];
1080 if (HaveValidParser(avStream->codecpar->codec_id)) {
1081 ret = GetVideoFirstKeyFrame(trackIndex);
1082 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, Status::ERROR_NO_MEMORY, "No memory");
1083 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr && firstFrame_->data != nullptr,
1084 Status::ERROR_WRONG_STATE, "Get first frame failed");
1085 bool convertRet = streamParser_->ConvertExtraDataToAnnexb(
1086 avStream->codecpar->extradata, avStream->codecpar->extradata_size);
1087 FALSE_RETURN_V_MSG_E(convertRet, Status::ERROR_INVALID_DATA, "ConvertExtraDataToAnnexb failed");
1088 streamParserInited_ = true;
1089 break;
1090 }
1091 }
1092 }
1093 return ret;
1094 }
1095
GetMediaInfo(MediaInfo & mediaInfo)1096 Status FFmpegDemuxerPlugin::GetMediaInfo(MediaInfo& mediaInfo)
1097 {
1098 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetMediaInfo");
1099 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1100 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1101
1102 Status ret = GetSeiInfo();
1103 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "GetSeiInfo failed");
1104
1105 FFmpegFormatHelper::ParseMediaInfo(*formatContext_, mediaInfo.general);
1106 DemuxerLogCompressor::StringifyMeta(mediaInfo.general, -1); // source meta
1107 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1108 Meta meta;
1109 auto avStream = formatContext_->streams[trackIndex];
1110 if (avStream == nullptr) {
1111 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " info is nullptr", trackIndex);
1112 mediaInfo.tracks.push_back(meta);
1113 continue;
1114 }
1115 FFmpegFormatHelper::ParseTrackInfo(*avStream, meta, *formatContext_);
1116 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC) {
1117 if (streamParser_ != nullptr && streamParserInited_ && firstFrame_ != nullptr) {
1118 streamParser_->ConvertPacketToAnnexb(&(firstFrame_->data), firstFrame_->size, nullptr, 0, false);
1119 streamParser_->ParseAnnexbExtraData(firstFrame_->data, firstFrame_->size);
1120 // Parser only sends xps info when first call ConvertPacketToAnnexb
1121 // readSample will call ConvertPacketToAnnexb again, so rest here
1122 streamParser_->ResetXPSSendStatus();
1123 ParseHEVCMetadataInfo(*avStream, meta);
1124 } else {
1125 MEDIA_LOG_W("Parse hevc info failed");
1126 }
1127 }
1128 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC ||
1129 avStream->codecpar->codec_id == AV_CODEC_ID_H264 ||
1130 avStream->codecpar->codec_id == AV_CODEC_ID_VVC) {
1131 ConvertCsdToAnnexb(*avStream, meta);
1132 }
1133 mediaInfo.tracks.push_back(meta);
1134 DemuxerLogCompressor::StringifyMeta(meta, trackIndex);
1135 }
1136 return Status::OK;
1137 }
1138
GetUserMeta(std::shared_ptr<Meta> meta)1139 Status FFmpegDemuxerPlugin::GetUserMeta(std::shared_ptr<Meta> meta)
1140 {
1141 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetUserMeta");
1142 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1143 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1144 FALSE_RETURN_V_MSG_E(meta != nullptr, Status::ERROR_NULL_POINTER, "Meta is nullptr");
1145
1146 FFmpegFormatHelper::ParseUserMeta(*formatContext_, meta);
1147 return Status::OK;
1148 }
1149
ParseDrmInfo(const MetaDrmInfo * const metaDrmInfo,size_t drmInfoSize,std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1150 void FFmpegDemuxerPlugin::ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize,
1151 std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1152 {
1153 MEDIA_LOG_D("In");
1154 size_t infoCount = drmInfoSize / sizeof(MetaDrmInfo);
1155 for (size_t index = 0; index < infoCount; index++) {
1156 std::stringstream ssConverter;
1157 std::string uuid;
1158 for (uint32_t i = 0; i < metaDrmInfo[index].uuidLen; i++) {
1159 int32_t singleUuid = static_cast<int32_t>(metaDrmInfo[index].uuid[i]);
1160 ssConverter << std::hex << std::setfill('0') << std::setw(2) << singleUuid; // 2:w
1161 uuid = ssConverter.str();
1162 }
1163 drmInfo.insert({ uuid, std::vector<uint8_t>(metaDrmInfo[index].pssh,
1164 metaDrmInfo[index].pssh + metaDrmInfo[index].psshLen) });
1165 }
1166 }
1167
GetDrmInfo(std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1168 Status FFmpegDemuxerPlugin::GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1169 {
1170 MEDIA_LOG_D("In");
1171 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1172 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1173
1174 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1175 Meta meta;
1176 AVStream *avStream = formatContext_->streams[trackIndex];
1177 if (avStream == nullptr) {
1178 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " is nullptr", trackIndex);
1179 continue;
1180 }
1181 MEDIA_LOG_D("GetDrmInfo by stream side data");
1182 size_t drmInfoSize = 0;
1183 MetaDrmInfo *tmpDrmInfo = (MetaDrmInfo *)av_stream_get_side_data(avStream,
1184 AV_PKT_DATA_ENCRYPTION_INIT_INFO, &drmInfoSize);
1185 if (tmpDrmInfo != nullptr && drmInfoSize != 0) {
1186 ParseDrmInfo(tmpDrmInfo, drmInfoSize, drmInfo);
1187 }
1188 }
1189 return Status::OK;
1190 }
1191
ConvertCsdToAnnexb(const AVStream & avStream,Meta & format)1192 void FFmpegDemuxerPlugin::ConvertCsdToAnnexb(const AVStream& avStream, Meta &format)
1193 {
1194 uint8_t *extradata = avStream.codecpar->extradata;
1195 int32_t extradataSize = avStream.codecpar->extradata_size;
1196 if (HaveValidParser(avStream.codecpar->codec_id) && streamParser_ != nullptr && streamParserInited_) {
1197 streamParser_->ConvertPacketToAnnexb(&(extradata), extradataSize, nullptr, 0, true);
1198 } else if (avStream.codecpar->codec_id == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
1199 if (avbsfContext_->par_out->extradata != nullptr && avbsfContext_->par_out->extradata_size > 0) {
1200 extradata = avbsfContext_->par_out->extradata;
1201 extradataSize = avbsfContext_->par_out->extradata_size;
1202 }
1203 }
1204 if (extradata != nullptr && extradataSize > 0) {
1205 std::vector<uint8_t> extra(extradataSize);
1206 extra.assign(extradata, extradata + extradataSize);
1207 format.Set<Tag::MEDIA_CODEC_CONFIG>(extra);
1208 }
1209 }
1210
AddPacketToCacheQueue(AVPacket * pkt)1211 Status FFmpegDemuxerPlugin::AddPacketToCacheQueue(AVPacket *pkt)
1212 {
1213 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Pkt is nullptr");
1214 #ifdef BUILD_ENG_VERSION
1215 DumpParam dumpParam {DumpMode(DUMP_AVPACKET_OUTPUT & dumpMode_), pkt->data, pkt->stream_index, -1, pkt->size,
1216 avpacketIndex_++, pkt->pts, pkt->pos};
1217 Dump(dumpParam);
1218 #endif
1219 auto trackId = pkt->stream_index;
1220 Status ret = Status::OK;
1221 if (NeedCombineFrame(trackId) && !GetNextFrame(pkt->data, pkt->size) && cacheQueue_.HasCache(trackId)) {
1222 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
1223 if (cacheSamplePacket != nullptr) {
1224 cacheSamplePacket->pkts.push_back(pkt);
1225 }
1226 } else {
1227 std::shared_ptr<SamplePacket> cacheSamplePacket = std::make_shared<SamplePacket>();
1228 if (cacheSamplePacket != nullptr) {
1229 cacheSamplePacket->pkts.push_back(pkt);
1230 cacheSamplePacket->offset = 0;
1231 cacheQueue_.Push(static_cast<uint32_t>(trackId), cacheSamplePacket);
1232 ret = CheckCacheDataLimit(static_cast<uint32_t>(trackId));
1233 }
1234 }
1235 return ret;
1236 }
1237
GetVideoFirstKeyFrame(uint32_t trackIndex)1238 Status FFmpegDemuxerPlugin::GetVideoFirstKeyFrame(uint32_t trackIndex)
1239 {
1240 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1241 AVPacket *pkt = nullptr;
1242 Status ret = Status::OK;
1243 while (1) {
1244 if (pkt == nullptr) {
1245 pkt = av_packet_alloc();
1246 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1247 }
1248
1249 std::unique_lock<std::mutex> sLock(syncMutex_);
1250 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
1251 sLock.unlock();
1252 if (ffmpegRet < 0) {
1253 MEDIA_LOG_E("Call av_read_frame failed, ret:" PUBLIC_LOG_D32, ffmpegRet);
1254 av_packet_unref(pkt);
1255 break;
1256 }
1257 cacheQueue_.AddTrackQueue(pkt->stream_index);
1258 ret = AddPacketToCacheQueue(pkt);
1259 if (ret != Status::OK) {
1260 return ret;
1261 }
1262
1263 if (static_cast<uint32_t>(pkt->stream_index) == trackIndex) {
1264 firstFrame_ = av_packet_alloc();
1265 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1266 int avRet = av_new_packet(firstFrame_, pkt->size);
1267 FALSE_RETURN_V_MSG_E(avRet >= 0, Status::ERROR_INVALID_DATA, "Call av_new_packet failed");
1268 av_packet_copy_props(firstFrame_, pkt);
1269 memcpy_s(firstFrame_->data, pkt->size, pkt->data, pkt->size);
1270 break;
1271 }
1272 pkt = nullptr;
1273 }
1274 return ret;
1275 }
1276
ParseHEVCMetadataInfo(const AVStream & avStream,Meta & format)1277 void FFmpegDemuxerPlugin::ParseHEVCMetadataInfo(const AVStream& avStream, Meta& format)
1278 {
1279 HevcParseFormat parse;
1280 parse.isHdrVivid = streamParser_->IsHdrVivid();
1281 parse.colorRange = streamParser_->GetColorRange();
1282 parse.colorPrimaries = streamParser_->GetColorPrimaries();
1283 parse.colorTransfer = streamParser_->GetColorTransfer();
1284 parse.colorMatrixCoeff = streamParser_->GetColorMatrixCoeff();
1285 parse.profile = streamParser_->GetProfileIdc();
1286 parse.level = streamParser_->GetLevelIdc();
1287 parse.chromaLocation = streamParser_->GetChromaLocation();
1288 parse.picWidInLumaSamples = streamParser_->GetPicWidInLumaSamples();
1289 parse.picHetInLumaSamples = streamParser_->GetPicHetInLumaSamples();
1290
1291 FFmpegFormatHelper::ParseHevcInfo(*formatContext_, parse, format);
1292 }
1293
TrackIsSelected(const uint32_t trackId)1294 bool FFmpegDemuxerPlugin::TrackIsSelected(const uint32_t trackId)
1295 {
1296 return std::any_of(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1297 [trackId](uint32_t id) { return id == trackId; });
1298 }
1299
SelectTrack(uint32_t trackId)1300 Status FFmpegDemuxerPlugin::SelectTrack(uint32_t trackId)
1301 {
1302 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1303 MEDIA_LOG_I("Select " PUBLIC_LOG_D32, trackId);
1304 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1305 if (trackId >= static_cast<uint32_t>(formatContext_.get()->nb_streams)) {
1306 MEDIA_LOG_E("Track is invalid, just have " PUBLIC_LOG_D32 " tracks in file", formatContext_.get()->nb_streams);
1307 return Status::ERROR_INVALID_PARAMETER;
1308 }
1309
1310 AVStream* avStream = formatContext_->streams[trackId];
1311 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1312 if (!IsSupportedTrack(*avStream)) {
1313 MEDIA_LOG_E("Track type is unsupport");
1314 return Status::ERROR_INVALID_PARAMETER;
1315 }
1316
1317 if (!TrackIsSelected(trackId)) {
1318 selectedTrackIds_.push_back(trackId);
1319 trackMtx_[trackId] = std::make_shared<std::mutex>();
1320 trackDfxInfoMap_[trackId] = {0, -1, -1};
1321 return cacheQueue_.AddTrackQueue(trackId);
1322 } else {
1323 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " has been selected", trackId);
1324 }
1325 return Status::OK;
1326 }
1327
UnselectTrack(uint32_t trackId)1328 Status FFmpegDemuxerPlugin::UnselectTrack(uint32_t trackId)
1329 {
1330 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1331 MEDIA_LOG_I("Unselect " PUBLIC_LOG_D32, trackId);
1332 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1333 auto index = std::find_if(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1334 [trackId](uint32_t selectedId) {return trackId == selectedId; });
1335 if (TrackIsSelected(trackId)) {
1336 selectedTrackIds_.erase(index);
1337 trackMtx_.erase(trackId);
1338 trackDfxInfoMap_.erase(trackId);
1339 return cacheQueue_.RemoveTrackQueue(trackId);
1340 } else {
1341 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " is not in selected list", trackId);
1342 }
1343 return Status::OK;
1344 }
1345
SeekTo(int32_t trackId,int64_t seekTime,SeekMode mode,int64_t & realSeekTime)1346 Status FFmpegDemuxerPlugin::SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime)
1347 {
1348 (void) trackId;
1349 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1350 MediaAVCodec::AVCodecTrace trace("SeekTo");
1351 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1352 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1353
1354 FALSE_RETURN_V_MSG_E(seekTime >= 0, Status::ERROR_INVALID_PARAMETER,
1355 "Seek time " PUBLIC_LOG_D64 " is not unsupported", seekTime);
1356 FALSE_RETURN_V_MSG_E(g_seekModeToFFmpegSeekFlags.count(mode) != 0, Status::ERROR_INVALID_PARAMETER,
1357 "Seek mode " PUBLIC_LOG_D32 " is not unsupported", static_cast<uint32_t>(mode));
1358
1359 int trackIndex = static_cast<int>(selectedTrackIds_[0]);
1360 for (size_t i = 1; i < selectedTrackIds_.size(); i++) {
1361 int index = static_cast<int>(selectedTrackIds_[i]);
1362 if (formatContext_->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
1363 trackIndex = index;
1364 break;
1365 }
1366 }
1367 MEDIA_LOG_D("Seek based on track " PUBLIC_LOG_D32, trackIndex);
1368 auto avStream = formatContext_->streams[trackIndex];
1369 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1370 int64_t ffTime = ConvertTimeToFFmpeg(seekTime * MS_TO_NS, avStream->time_base);
1371 if (!CheckStartTime(formatContext_.get(), avStream, ffTime, seekTime)) {
1372 MEDIA_LOG_E("Get start time from track " PUBLIC_LOG_D32 " failed", trackIndex);
1373 return Status::ERROR_INVALID_OPERATION;
1374 }
1375 realSeekTime = ConvertTimeFromFFmpeg(ffTime, avStream->time_base);
1376 int flag = ConvertFlagsToFFmpeg(avStream, ffTime, mode, seekTime);
1377 MEDIA_LOG_I("Time [" PUBLIC_LOG_U64 "/" PUBLIC_LOG_U64 "/" PUBLIC_LOG_D64 "] flag ["
1378 PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
1379 seekTime, ffTime, realSeekTime, static_cast<int32_t>(mode), flag);
1380 auto ret = av_seek_frame(formatContext_.get(), trackIndex, ffTime, flag);
1381 if (formatContext_->pb->error) {
1382 formatContext_->pb->error = 0;
1383 }
1384 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
1385 "Call av_seek_frame failed, err: " PUBLIC_LOG_S, AVStrError(ret).c_str());
1386 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1387 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1388 cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1389 }
1390 return Status::OK;
1391 }
1392
Flush()1393 Status FFmpegDemuxerPlugin::Flush()
1394 {
1395 Status ret = Status::OK;
1396 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1397 MEDIA_LOG_I("In");
1398 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1399 ret = cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1400 ret = cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1401 }
1402 if (formatContext_) {
1403 avio_flush(formatContext_.get()->pb);
1404 avformat_flush(formatContext_.get());
1405 }
1406 return ret;
1407 }
1408
ResetEosStatus()1409 void FFmpegDemuxerPlugin::ResetEosStatus()
1410 {
1411 MEDIA_LOG_I("In");
1412 formatContext_->pb->eof_reached = 0;
1413 formatContext_->pb->error = 0;
1414 }
1415
ReadSample(uint32_t trackId,std::shared_ptr<AVBuffer> sample)1416 Status FFmpegDemuxerPlugin::ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample)
1417 {
1418 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1419 MediaAVCodec::AVCodecTrace trace("ReadSample");
1420 MEDIA_LOG_D("In");
1421 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1422 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1423 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_INVALID_PARAMETER, "Track has not been selected");
1424 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_!=nullptr, Status::ERROR_INVALID_PARAMETER,
1425 "AVBuffer or memory is nullptr");
1426 Status ret;
1427 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1428 ret = ReadPacketToCacheQueue(trackId);
1429 }
1430 while (!cacheQueue_.HasCache(trackId)) {
1431 ret = ReadPacketToCacheQueue(trackId);
1432 if (ret == Status::END_OF_STREAM) {
1433 MEDIA_LOG_D("Read to end");
1434 }
1435 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1436 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1437 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1438 }
1439 std::lock_guard<std::mutex> lockTrack(*trackMtx_[trackId].get());
1440 auto samplePacket = cacheQueue_.Front(trackId);
1441 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_NULL_POINTER, "Cache packet is nullptr");
1442 if (samplePacket->isEOS) {
1443 ret = SetEosSample(sample);
1444 if (ret == Status::OK) {
1445 MEDIA_LOG_I("Track:" PUBLIC_LOG_D32 " eos [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
1446 trackId, trackDfxInfoMap_[trackId].lastPts,
1447 trackDfxInfoMap_[trackId].lastDurantion, trackDfxInfoMap_[trackId].lastPos);
1448 cacheQueue_.Pop(trackId);
1449 }
1450 return ret;
1451 }
1452 ret = ConvertAVPacketToSample(sample, samplePacket);
1453 if (ret == Status::ERROR_NOT_ENOUGH_DATA) {
1454 return Status::OK;
1455 } else if (ret == Status::OK) {
1456 MEDIA_LOG_D("All partial sample has been copied");
1457 cacheQueue_.Pop(trackId);
1458 }
1459 return ret;
1460 }
1461
GetNextSampleSize(uint32_t trackId,int32_t & size)1462 Status FFmpegDemuxerPlugin::GetNextSampleSize(uint32_t trackId, int32_t& size)
1463 {
1464 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1465 MediaAVCodec::AVCodecTrace trace("GetNextSampleSize");
1466 MEDIA_LOG_D("In, track " PUBLIC_LOG_D32, trackId);
1467 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1468 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_UNKNOWN, "Track has not been selected");
1469
1470 Status ret;
1471 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1472 ret = ReadPacketToCacheQueue(trackId);
1473 }
1474 while (!cacheQueue_.HasCache(trackId)) {
1475 ret = ReadPacketToCacheQueue(trackId);
1476 if (ret == Status::END_OF_STREAM) {
1477 MEDIA_LOG_D("Read to end");
1478 }
1479 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1480 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1481 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1482 }
1483 std::shared_ptr<SamplePacket> samplePacket = cacheQueue_.Front(trackId);
1484 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_UNKNOWN, "Cache sample is nullptr");
1485 if (samplePacket->isEOS) {
1486 MEDIA_LOG_I("Track " PUBLIC_LOG_D32 " eos", trackId);
1487 return Status::END_OF_STREAM;
1488 }
1489 FALSE_RETURN_V_MSG_E(samplePacket->pkts.size() > 0, Status::ERROR_UNKNOWN, "Cache sample is empty");
1490 int totalSize = 0;
1491 for (auto pkt : samplePacket->pkts) {
1492 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_UNKNOWN, "Packet in sample is nullptr");
1493 totalSize += pkt->size;
1494 }
1495 size = totalSize;
1496 return Status::OK;
1497 }
1498
InitPTSandIndexConvert()1499 void FFmpegDemuxerPlugin::InitPTSandIndexConvert()
1500 {
1501 indexToRelativePTSFrameCount_ = 0; // init IndexToRelativePTSFrameCount_
1502 relativePTSToIndexPosition_ = 0; // init RelativePTSToIndexPosition_
1503 indexToRelativePTSMaxHeap_ = std::priority_queue<int64_t>(); // init IndexToRelativePTSMaxHeap_
1504 relativePTSToIndexPTSMin_ = INT64_MAX;
1505 relativePTSToIndexPTSMax_ = INT64_MIN;
1506 relativePTSToIndexRightDiff_ = INT64_MAX;
1507 relativePTSToIndexLeftDiff_ = INT64_MAX;
1508 relativePTSToIndexTempDiff_ = INT64_MAX;
1509 }
1510
GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,const uint64_t relativePresentationTimeUs,uint32_t & index)1511 Status FFmpegDemuxerPlugin::GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
1512 const uint64_t relativePresentationTimeUs, uint32_t &index)
1513 {
1514 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1515
1516 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1517
1518 InitPTSandIndexConvert();
1519
1520 auto avStream = formatContext_->streams[trackIndex];
1521 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1522
1523 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1524 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1525
1526 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1527 static_cast<int64_t>(relativePresentationTimeUs), index);
1528 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1529
1530 int64_t absolutePTS = static_cast<int64_t>(relativePresentationTimeUs) + absolutePTSIndexZero_;
1531
1532 ret = GetPresentationTimeUsFromFfmpegMOV(RELATIVEPTS_TO_INDEX, trackIndex,
1533 absolutePTS, index);
1534 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1535
1536 if (absolutePTS < relativePTSToIndexPTSMin_ || absolutePTS > relativePTSToIndexPTSMax_) {
1537 MEDIA_LOG_E("Pts is out of range");
1538 return Status::ERROR_INVALID_DATA;
1539 }
1540
1541 if (relativePTSToIndexLeftDiff_ == 0 || relativePTSToIndexRightDiff_ == 0) {
1542 index = relativePTSToIndexPosition_;
1543 } else {
1544 index = relativePTSToIndexLeftDiff_ < relativePTSToIndexRightDiff_ ?
1545 relativePTSToIndexPosition_ - 1 : relativePTSToIndexPosition_;
1546 }
1547 return Status::OK;
1548 }
1549
GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,const uint32_t index,uint64_t & relativePresentationTimeUs)1550 Status FFmpegDemuxerPlugin::GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
1551 const uint32_t index, uint64_t &relativePresentationTimeUs)
1552 {
1553 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1554
1555 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1556
1557 InitPTSandIndexConvert();
1558
1559 auto avStream = formatContext_->streams[trackIndex];
1560 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1561
1562 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1563 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1564
1565 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1566 static_cast<int64_t>(relativePresentationTimeUs), index);
1567 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1568
1569 GetPresentationTimeUsFromFfmpegMOV(INDEX_TO_RELATIVEPTS, trackIndex,
1570 static_cast<int64_t>(relativePresentationTimeUs), index);
1571 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1572
1573 if (index + 1 > indexToRelativePTSFrameCount_) {
1574 MEDIA_LOG_E("Index is out of range");
1575 return Status::ERROR_INVALID_DATA;
1576 }
1577
1578 int64_t relativepts = indexToRelativePTSMaxHeap_.top() - absolutePTSIndexZero_;
1579 FALSE_RETURN_V_MSG_E(relativepts >= 0, Status::ERROR_INVALID_DATA, "Existence of calculation results less than 0");
1580 relativePresentationTimeUs = static_cast<uint64_t>(relativepts);
1581
1582 return Status::OK;
1583 }
1584
PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1585 Status FFmpegDemuxerPlugin::PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,
1586 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1587 {
1588 uint32_t sttsIndex = 0;
1589 uint32_t cttsIndex = 0;
1590 int64_t pts = 0; // init pts
1591 int64_t dts = 0; // init dts
1592
1593 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1594 int32_t cttsCurNum = 0;
1595
1596 cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1597 while (sttsIndex < avStream->stts_count && cttsIndex < avStream->ctts_count &&
1598 cttsCurNum >= 0 && sttsCurNum >= 0) {
1599 if (cttsCurNum == 0) {
1600 cttsIndex++;
1601 if (cttsIndex >= avStream->ctts_count) {
1602 break;
1603 }
1604 cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1605 }
1606 cttsCurNum--;
1607 if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1608 ((dts + static_cast<int64_t>(avStream->ctts_data[cttsIndex].duration)) /
1609 static_cast<int64_t>(avStream->time_scale))) {
1610 MEDIA_LOG_E("pts overflow");
1611 return Status::ERROR_INVALID_DATA;
1612 }
1613 double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1614 double ptsTemp = static_cast<double>(dts) + static_cast<double>(avStream->ctts_data[cttsIndex].duration);
1615 pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1616 if (mode == GET_ALL_FRAME_PTS &&
1617 static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1618 MEDIA_LOG_I("PTS list has reached the maximum limit");
1619 break;
1620 }
1621 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1622 sttsCurNum--;
1623 if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1624 MEDIA_LOG_E("dts overflow");
1625 return Status::ERROR_INVALID_DATA;
1626 }
1627 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1628 if (sttsCurNum == 0) {
1629 sttsIndex++;
1630 sttsCurNum = sttsIndex < avStream->stts_count ?
1631 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1632 }
1633 }
1634 return Status::OK;
1635 }
1636
PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1637 Status FFmpegDemuxerPlugin::PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,
1638 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1639 {
1640 uint32_t sttsIndex = 0;
1641 int64_t pts = 0; // init pts
1642 int64_t dts = 0; // init dts
1643
1644 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1645
1646 while (sttsIndex < avStream->stts_count && sttsCurNum >= 0) {
1647 if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1648 (dts / static_cast<int64_t>(avStream->time_scale))) {
1649 MEDIA_LOG_E("pts overflow");
1650 return Status::ERROR_INVALID_DATA;
1651 }
1652 double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1653 double ptsTemp = static_cast<double>(dts);
1654 pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1655 if (mode == GET_ALL_FRAME_PTS &&
1656 static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1657 MEDIA_LOG_I("PTS list has reached the maximum limit");
1658 break;
1659 }
1660 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1661 sttsCurNum--;
1662 if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1663 MEDIA_LOG_E("dts overflow");
1664 return Status::ERROR_INVALID_DATA;
1665 }
1666 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1667 if (sttsCurNum == 0) {
1668 sttsIndex++;
1669 sttsCurNum = sttsIndex < avStream->stts_count ?
1670 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1671 }
1672 }
1673 return Status::OK;
1674 }
1675
GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,uint32_t trackIndex,int64_t absolutePTS,uint32_t index)1676 Status FFmpegDemuxerPlugin::GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,
1677 uint32_t trackIndex, int64_t absolutePTS, uint32_t index)
1678 {
1679 auto avStream = formatContext_->streams[trackIndex];
1680 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1681 FALSE_RETURN_V_MSG_E(avStream->stts_data != nullptr && avStream->stts_count != 0,
1682 Status::ERROR_NULL_POINTER, "AVStream->stts_data is empty");
1683 FALSE_RETURN_V_MSG_E(avStream->time_scale != 0, Status::ERROR_INVALID_DATA, "AVStream->time_scale is zero");
1684 if (mode == GET_ALL_FRAME_PTS) {
1685 ptsListOrg_.clear();
1686 }
1687 return avStream->ctts_data == nullptr ?
1688 PTSAndIndexConvertOnlySttsProcess(mode, avStream, absolutePTS, index) :
1689 PTSAndIndexConvertSttsAndCttsProcess(mode, avStream, absolutePTS, index);
1690 }
1691
PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,int64_t pts,int64_t absolutePTS,uint32_t index)1692 void FFmpegDemuxerPlugin::PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,
1693 int64_t pts, int64_t absolutePTS, uint32_t index)
1694 {
1695 switch (mode) {
1696 case GET_FIRST_PTS:
1697 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1698 break;
1699 case INDEX_TO_RELATIVEPTS:
1700 IndexToRelativePTSProcess(pts, index);
1701 break;
1702 case RELATIVEPTS_TO_INDEX:
1703 RelativePTSToIndexProcess(pts, absolutePTS);
1704 break;
1705 case GET_ALL_FRAME_PTS:
1706 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1707 ptsListOrg_.emplace_back(pts);
1708 break;
1709 default:
1710 MEDIA_LOG_E("Wrong mode");
1711 break;
1712 }
1713 }
1714
IndexToRelativePTSProcess(int64_t pts,uint32_t index)1715 void FFmpegDemuxerPlugin::IndexToRelativePTSProcess(int64_t pts, uint32_t index)
1716 {
1717 if (indexToRelativePTSMaxHeap_.size() < index + 1) {
1718 indexToRelativePTSMaxHeap_.push(pts);
1719 } else {
1720 if (pts < indexToRelativePTSMaxHeap_.top()) {
1721 indexToRelativePTSMaxHeap_.pop();
1722 indexToRelativePTSMaxHeap_.push(pts);
1723 }
1724 }
1725 indexToRelativePTSFrameCount_++;
1726 }
1727
RelativePTSToIndexProcess(int64_t pts,int64_t absolutePTS)1728 void FFmpegDemuxerPlugin::RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS)
1729 {
1730 if (relativePTSToIndexPTSMin_ > pts) {
1731 relativePTSToIndexPTSMin_ = pts;
1732 }
1733 if (relativePTSToIndexPTSMax_ < pts) {
1734 relativePTSToIndexPTSMax_ = pts;
1735 }
1736 relativePTSToIndexTempDiff_ = abs(pts - absolutePTS);
1737 if (pts < absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexLeftDiff_) {
1738 relativePTSToIndexLeftDiff_ = relativePTSToIndexTempDiff_;
1739 }
1740 if (pts >= absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexRightDiff_) {
1741 relativePTSToIndexRightDiff_ = relativePTSToIndexTempDiff_;
1742 }
1743 if (pts < absolutePTS) {
1744 relativePTSToIndexPosition_++;
1745 }
1746 }
1747
CheckCacheDataLimit(uint32_t trackId)1748 Status FFmpegDemuxerPlugin::CheckCacheDataLimit(uint32_t trackId)
1749 {
1750 if (!outOfLimit_) {
1751 auto cacheDataSize = cacheQueue_.GetCacheDataSize(trackId);
1752 if (cacheDataSize > cachelimitSize_) {
1753 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " cache out of limit: " PUBLIC_LOG_U32 "/" PUBLIC_LOG_U32 ", by user "
1754 PUBLIC_LOG_D32, trackId, cacheDataSize, cachelimitSize_, static_cast<int32_t>(setLimitByUser));
1755 outOfLimit_ = true;
1756 }
1757 }
1758 return Status::OK;
1759 }
1760
SetCacheLimit(uint32_t limitSize)1761 void FFmpegDemuxerPlugin::SetCacheLimit(uint32_t limitSize)
1762 {
1763 setLimitByUser = true;
1764 cachelimitSize_ = limitSize;
1765 }
1766
1767 namespace { // plugin set
1768
IsStartWithID3(const uint8_t * buf,const char * tagName)1769 int IsStartWithID3(const uint8_t *buf, const char *tagName)
1770 {
1771 return buf[POS_0] == tagName[POS_0] &&
1772 buf[POS_1] == tagName[POS_1] &&
1773 buf[POS_2] == tagName[POS_2] &&
1774 buf[POS_3] != POS_FF &&
1775 buf[POS_4] != POS_FF &&
1776 (buf[POS_6] & TAG_MASK) == 0 &&
1777 (buf[POS_7] & TAG_MASK) == 0 &&
1778 (buf[POS_8] & TAG_MASK) == 0 &&
1779 (buf[POS_9] & TAG_MASK) == 0;
1780 }
1781
GetID3TagLen(const uint8_t * buf)1782 int GetID3TagLen(const uint8_t *buf)
1783 {
1784 int32_t len = ((buf[POS_6] & LEN_MASK) << POS_21) + ((buf[POS_7] & LEN_MASK) << POS_14) +
1785 ((buf[POS_8] & LEN_MASK) << POS_7) + (buf[POS_9] & LEN_MASK) +
1786 static_cast<int32_t>(ID3V2_HEADER_SIZE);
1787 if (buf[POS_5] & TAG_VERSION_MASK) {
1788 len += static_cast<int32_t>(ID3V2_HEADER_SIZE);
1789 }
1790 return len;
1791 }
1792
GetConfidence(std::shared_ptr<AVInputFormat> plugin,const std::string & pluginName,std::shared_ptr<DataSource> dataSource,size_t & getData)1793 int32_t GetConfidence(std::shared_ptr<AVInputFormat> plugin, const std::string& pluginName,
1794 std::shared_ptr<DataSource> dataSource, size_t &getData)
1795 {
1796 size_t bufferSize = DEFAULT_SNIFF_SIZE;
1797 uint64_t fileSize = 0;
1798 Status getFileSize = dataSource->GetSize(fileSize);
1799 if (getFileSize == Status::OK) {
1800 bufferSize = (bufferSize < fileSize) ? bufferSize : fileSize;
1801 }
1802 std::vector<uint8_t> buff(bufferSize + AVPROBE_PADDING_SIZE); // fix ffmpeg probe crash, refer to tools/probetest.c
1803 auto bufferInfo = std::make_shared<Buffer>();
1804 auto bufData = bufferInfo->WrapMemory(buff.data(), bufferSize, bufferSize);
1805 FALSE_RETURN_V_MSG_E(bufferInfo->GetMemory() != nullptr, -1,
1806 "Alloc buffer failed for " PUBLIC_LOG_S, pluginName.c_str());
1807 Status ret = Status::OK;
1808 {
1809 std::string traceName = "Sniff_" + pluginName + "_Readat";
1810 MediaAVCodec::AVCodecTrace trace(traceName.c_str());
1811 ret = dataSource->ReadAt(0, bufferInfo, bufferSize);
1812 }
1813 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1814 getData = bufferInfo->GetMemory()->GetSize();
1815 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1816 if (getFileSize == Status::OK && getData > ID3V2_HEADER_SIZE && IsStartWithID3(buff.data(), "ID3")) {
1817 int32_t id3Len = GetID3TagLen(buff.data());
1818 // id3 tag length is out of file, or file just contains id3 tag, no valid data.
1819 FALSE_RETURN_V_MSG_E(id3Len >= 0 && static_cast<uint64_t>(id3Len) < fileSize, -1,
1820 "File data error for " PUBLIC_LOG_S, pluginName.c_str());
1821 if (id3Len > 0) {
1822 uint64_t remainSize = fileSize - static_cast<uint64_t>(id3Len);
1823 bufferSize = (bufferSize < remainSize) ? bufferSize : remainSize;
1824 int resetRet = memset_s(buff.data(), bufferSize, 0, bufferSize);
1825 FALSE_RETURN_V_MSG_E(resetRet == EOK, -1, "Reset buff failed for " PUBLIC_LOG_S, pluginName.c_str());
1826 ret = dataSource->ReadAt(id3Len, bufferInfo, bufferSize);
1827 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1828 getData = bufferInfo->GetMemory()->GetSize();
1829 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1830 }
1831 }
1832 AVProbeData probeData{"", buff.data(), static_cast<int32_t>(getData), ""};
1833 return plugin->read_probe(&probeData);
1834 }
1835
Sniff(const std::string & pluginName,std::shared_ptr<DataSource> dataSource)1836 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource)
1837 {
1838 FALSE_RETURN_V_MSG_E(!pluginName.empty(), 0, "Plugin name is empty");
1839 FALSE_RETURN_V_MSG_E(dataSource != nullptr, 0, "DataSource is nullptr");
1840 std::shared_ptr<AVInputFormat> plugin;
1841 {
1842 std::lock_guard<std::mutex> lock(g_mtx);
1843 plugin = g_pluginInputFormat[pluginName];
1844 }
1845 FALSE_RETURN_V_MSG_E((plugin != nullptr && plugin->read_probe), 0,
1846 "Get plugin for " PUBLIC_LOG_S " failed", pluginName.c_str());
1847 size_t getData = 0;
1848 int confidence = GetConfidence(plugin, pluginName, dataSource, getData);
1849 if (confidence < 0) {
1850 return 0;
1851 }
1852 if (StartWith(plugin->name, "mp3") && confidence > 0 && confidence <= MP3_PROBE_SCORE_LIMIT) {
1853 MEDIA_LOG_W("Score " PUBLIC_LOG_D32 " is too low", confidence);
1854 confidence = 0;
1855 }
1856 if (getData < DEFAULT_SNIFF_SIZE || confidence > 0) {
1857 MEDIA_LOG_I("Sniff:" PUBLIC_LOG_S "[" PUBLIC_LOG_ZU "/" PUBLIC_LOG_D32 "]", plugin->name, getData, confidence);
1858 }
1859 return confidence;
1860 }
1861
ReplaceDelimiter(const std::string & delmiters,char newDelimiter,std::string & str)1862 void ReplaceDelimiter(const std::string& delmiters, char newDelimiter, std::string& str)
1863 {
1864 MEDIA_LOG_D("Reset from [" PUBLIC_LOG_S "]", str.c_str());
1865 for (auto it = str.begin(); it != str.end(); ++it) {
1866 if (delmiters.find(newDelimiter) != std::string::npos) {
1867 *it = newDelimiter;
1868 }
1869 }
1870 MEDIA_LOG_D("Reset to [" PUBLIC_LOG_S "]", str.c_str());
1871 };
1872
RegisterPlugins(const std::shared_ptr<Register> & reg)1873 Status RegisterPlugins(const std::shared_ptr<Register>& reg)
1874 {
1875 MEDIA_LOG_I("In");
1876 FALSE_RETURN_V_MSG_E(reg != nullptr, Status::ERROR_INVALID_PARAMETER, "Register is nullptr");
1877 std::lock_guard<std::mutex> lock(g_mtx);
1878 const AVInputFormat* plugin = nullptr;
1879 void* i = nullptr;
1880 while ((plugin = av_demuxer_iterate(&i))) {
1881 if (plugin == nullptr) {
1882 continue;
1883 }
1884 MEDIA_LOG_D("Check ffmpeg demuxer " PUBLIC_LOG_S "[" PUBLIC_LOG_S "]", plugin->name, plugin->long_name);
1885 if (plugin->long_name != nullptr &&
1886 !strncmp(plugin->long_name, "pcm ", STR_MAX_LEN)) {
1887 continue;
1888 }
1889 if (!IsInputFormatSupported(plugin->name)) {
1890 continue;
1891 }
1892
1893 std::string pluginName = "avdemux_" + std::string(plugin->name);
1894 ReplaceDelimiter(".,|-<> ", '_', pluginName);
1895
1896 DemuxerPluginDef regInfo;
1897 regInfo.name = pluginName;
1898 regInfo.description = "ffmpeg demuxer plugin";
1899 regInfo.rank = RANK_MAX;
1900 regInfo.AddExtensions(SplitString(plugin->extensions, ','));
1901 g_pluginInputFormat[pluginName] =
1902 std::shared_ptr<AVInputFormat>(const_cast<AVInputFormat*>(plugin), [](void*) {});
1903 auto func = [](const std::string& name) -> std::shared_ptr<DemuxerPlugin> {
1904 return std::make_shared<FFmpegDemuxerPlugin>(name);
1905 };
1906 regInfo.SetCreator(func);
1907 regInfo.SetSniffer(Sniff);
1908 auto ret = reg->AddPlugin(regInfo);
1909 if (ret != Status::OK) {
1910 MEDIA_LOG_E("Add plugin failed, err=" PUBLIC_LOG_D32, static_cast<int>(ret));
1911 } else {
1912 MEDIA_LOG_D("Add plugin " PUBLIC_LOG_S, pluginName.c_str());
1913 }
1914 }
1915 FALSE_RETURN_V_MSG_E(!g_pluginInputFormat.empty(), Status::ERROR_UNKNOWN, "Can not load any ffmpeg demuxer");
1916 return Status::OK;
1917 }
1918 } // namespace
__anonf5d992460a02null1919 PLUGIN_DEFINITION(FFmpegDemuxer, LicenseType::LGPL, RegisterPlugins, [] {});
1920 } // namespace Ffmpeg
1921 } // namespace Plugins
1922 } // namespace Media
1923 } // namespace OHOS
1924