1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef FFMPEG_DEMUXER_PLUGIN_H
17 #define FFMPEG_DEMUXER_PLUGIN_H
18 
19 #include <atomic>
20 #include <vector>
21 #include <thread>
22 #include <map>
23 #include <queue>
24 #include <shared_mutex>
25 #include <list>
26 #include "buffer/avbuffer.h"
27 #include "plugin/demuxer_plugin.h"
28 #include "block_queue_pool.h"
29 #include "stream_parser_manager.h"
30 #include "reference_parser_manager.h"
31 #include "meta/meta.h"
32 
33 #ifdef __cplusplus
34 extern "C" {
35 #endif
36 #include "libavformat/avformat.h"
37 #include "libavcodec/avcodec.h"
38 #include "libavutil/dict.h"
39 #include "libavutil/opt.h"
40 #include "libavutil/parseutils.h"
41 #include "libavcodec/bsf.h"
42 #ifdef __cplusplus
43 }
44 #endif
45 
46 namespace OHOS {
47 namespace Media {
48 namespace Plugins {
49 namespace Ffmpeg {
50 class FFmpegDemuxerPlugin : public DemuxerPlugin {
51 public:
52     explicit FFmpegDemuxerPlugin(std::string name);
53     ~FFmpegDemuxerPlugin() override;
54     Status Reset() override;
55     Status Start() override;
56     Status Stop() override;
57     Status Flush() override;
58     Status SetDataSource(const std::shared_ptr<DataSource>& source) override;
59     Status GetMediaInfo(MediaInfo& mediaInfo) override;
60     Status GetUserMeta(std::shared_ptr<Meta> meta) override;
61     Status SelectTrack(uint32_t trackId) override;
62     Status UnselectTrack(uint32_t trackId) override;
63     Status SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime) override;
64     Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample) override;
65     Status GetNextSampleSize(uint32_t trackId, int32_t& size) override;
66     Status GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo) override;
67     void ResetEosStatus() override;
68     bool IsRefParserSupported() override;
69     Status ParserRefUpdatePos(int64_t timeStampMs, bool isForward = true) override;
70     Status ParserRefInfo() override;
71     Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo) override;
72     Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo) override;
73     Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo) override;
74     Status GetIFramePos(std::vector<uint32_t> &IFramePos) override;
75     Status Dts2FrameId(int64_t dts, uint32_t &frameId, bool offset = true) override;
76 
77     Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
78         const uint64_t relativePresentationTimeUs, uint32_t &index) override;
79     Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
80         const uint32_t index, uint64_t &relativePresentationTimeUs) override;
81     void SetCacheLimit(uint32_t limitSize) override;
82 
83 private:
84     enum DumpMode : unsigned long {
85         DUMP_NONE = 0,
86         DUMP_READAT_INPUT = 0b001,
87         DUMP_AVPACKET_OUTPUT = 0b010,
88         DUMP_AVBUFFER_OUTPUT = 0b100,
89     };
90     enum IndexAndPTSConvertMode : unsigned int {
91         GET_FIRST_PTS,
92         INDEX_TO_RELATIVEPTS,
93         RELATIVEPTS_TO_INDEX,
94         GET_ALL_FRAME_PTS,
95     };
96     struct IOContext {
97         std::shared_ptr<DataSource> dataSource {nullptr};
98         int64_t offset {0};
99         uint64_t fileSize {0};
100         bool eos {false};
101         std::atomic<bool> retry {false};
102         uint32_t initDownloadDataSize {0};
103         std::atomic<bool> initCompleted {false};
104         DumpMode dumpMode {DUMP_NONE};
105     };
106     void ConvertCsdToAnnexb(const AVStream& avStream, Meta &format);
107     int64_t GetFileDuration(const AVFormatContext& avFormatContext);
108     int64_t GetStreamDuration(const AVStream& avStream);
109 
110     static int AVReadPacket(void* opaque, uint8_t* buf, int bufSize);
111     static int AVWritePacket(void* opaque, uint8_t* buf, int bufSize);
112     static int64_t AVSeek(void* opaque, int64_t offset, int whence);
113     AVIOContext* AllocAVIOContext(int flags, IOContext *ioContext);
114     std::shared_ptr<AVFormatContext> InitAVFormatContext(IOContext *ioContext);
115     static int CheckContextIsValid(void* opaque, int &bufSize);
116     void NotifyInitializationCompleted();
117 
118     void InitParser();
119     void InitBitStreamContext(const AVStream& avStream);
120     Status ConvertAvcToAnnexb(AVPacket& pkt);
121     Status PushEOSToAllCache();
122     bool TrackIsSelected(const uint32_t trackId);
123     Status ReadPacketToCacheQueue(const uint32_t readId);
124     Status AddPacketToCacheQueue(AVPacket *pkt);
125     Status SetDrmCencInfo(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket);
126     void WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket);
127     Status ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket);
128     Status ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* avpacket,
129         std::shared_ptr<SamplePacket> dstSamplePacket);
130     Status SetEosSample(std::shared_ptr<AVBuffer> sample);
131     Status WriteBuffer(std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize);
132     void ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize,
133         std::multimap<std::string, std::vector<uint8_t>>& drmInfo);
134     bool GetNextFrame(const uint8_t *data, const uint32_t size);
135     bool NeedCombineFrame(uint32_t trackId);
136     AVPacket* CombinePackets(std::shared_ptr<SamplePacket> samplePacket);
137     Status ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket);
138     Status ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket);
139     Status GetSeiInfo();
140 
141     int FindNaluSpliter(int size, const uint8_t *data);
142     bool CanDropAvcPkt(const AVPacket& pkt);
143     bool CanDropHevcPkt(const AVPacket& pkt);
144     bool WebvttPktProcess(AVPacket *pkt);
145     bool IsWebvttMP4(const AVStream *avStream);
146     void WebvttMP4EOSProcess(const AVPacket *pkt);
147     Status CheckCacheDataLimit(uint32_t trackId);
148 
149     Status GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,
150         uint32_t trackIndex, int64_t absolutePTS, uint32_t index);
151     Status PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,
152         const AVStream* avStream, int64_t absolutePTS, uint32_t index);
153     Status PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,
154         const AVStream* avStream, int64_t absolutePTS, uint32_t index);
155     void InitPTSandIndexConvert();
156     void IndexToRelativePTSProcess(int64_t pts, uint32_t index);
157     void RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS);
158     void PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,
159         int64_t pts, int64_t absolutePTS, uint32_t index);
160     int64_t absolutePTSIndexZero_ = INT64_MAX;
161     std::priority_queue<int64_t> indexToRelativePTSMaxHeap_;
162     uint32_t indexToRelativePTSFrameCount_ = 0;
163     uint32_t relativePTSToIndexPosition_ = 0;
164     int64_t relativePTSToIndexPTSMin_ = INT64_MAX;
165     int64_t relativePTSToIndexPTSMax_ = INT64_MIN;
166     int64_t relativePTSToIndexRightDiff_ = INT64_MAX;
167     int64_t relativePTSToIndexLeftDiff_ = INT64_MAX;
168     int64_t relativePTSToIndexTempDiff_ = INT64_MAX;
169     void ParserFirstDts();
170     Status InitIoContext();
171     Status ParserRefInit();
172     Status ParserRefInfoLoop(AVPacket *pkt, uint32_t curStreamId);
173     Status SelectProGopId();
174     void ParserBoxInfo();
175     AVStream *GetVideoStream();
176 
177     std::mutex mutex_ {};
178     std::shared_mutex sharedMutex_;
179     std::unordered_map<uint32_t, std::shared_ptr<std::mutex>> trackMtx_;
180     Seekable seekable_;
181     IOContext ioContext_;
182     std::vector<uint32_t> selectedTrackIds_;
183     BlockQueuePool cacheQueue_;
184 
185     std::shared_ptr<AVInputFormat> pluginImpl_ {nullptr};
186     std::shared_ptr<AVFormatContext> formatContext_ {nullptr};
187     std::shared_ptr<AVBSFContext> avbsfContext_ {nullptr};
188     std::shared_ptr<StreamParserManager> streamParser_ {nullptr};
189     bool streamParserInited_ {false};
190 
191     Status GetVideoFirstKeyFrame(uint32_t trackIndex);
192     void ParseHEVCMetadataInfo(const AVStream& avStream, Meta &format);
193     AVPacket *firstFrame_ = nullptr;
194 
195     std::atomic<bool> parserState_ = true;
196     IOContext parserRefIoContext_;
197     std::shared_ptr<AVFormatContext> parserRefFormatContext_{nullptr};
198     int parserRefVideoStreamIdx_ = -1;
199     std::shared_ptr<ReferenceParserManager> referenceParser_{nullptr};
200     int32_t parserCurGopId_ = 0;
201     int64_t pendingSeekMsTime_ = -1;
202     int64_t parserRefStartTime_ = -1;
203     std::list<uint32_t> processingIFrame_;
204     std::vector<uint32_t> IFramePos_;
205     double fps_{0};
206     int64_t firstDts_ = 0;
207     uint32_t dtsOffset_ = 0;
208     bool isSdtpExist_ = false;
209     std::mutex syncMutex_;
210     bool updatePosIsForward_ = true;
211     bool isInit_ = false;
212     uint32_t cachelimitSize_ = 0;
213     bool outOfLimit_ = false;
214     bool setLimitByUser = false;
215 
216     // dfx
217     struct TrackDfxInfo {
218         int frameIndex = 0; // for each track
219         int64_t lastPts;
220         int64_t lastPos;
221         int64_t lastDurantion;
222     };
223     struct DumpParam {
224         DumpMode mode;
225         uint8_t* buf;
226         int trackId;
227         int64_t offset;
228         int size;
229         int index;
230         int64_t pts;
231         int64_t pos;
232     };
233     std::unordered_map<int, TrackDfxInfo> trackDfxInfoMap_;
234     DumpMode dumpMode_ {DUMP_NONE};
235     static std::atomic<int> readatIndex_;
236     int avpacketIndex_ {0};
237 
238     static void Dump(const DumpParam &dumpParam);
239 
240     std::vector<int64_t> ptsListOrg_;
241     std::vector<int64_t> ptsListFromZero_;
242     std::unordered_map<int32_t, int64_t> iFramePtsMap_;
243     Status GetGopIdFromSeekPos(int64_t seekMs, int32_t &gopId);
244     Status ParserRefCheckVideoValid(const AVStream *videoStream);
245 };
246 } // namespace Ffmpeg
247 } // namespace Plugins
248 } // namespace Media
249 } // namespace OHOS
250 #endif // FFMPEG_DEMUXER_PLUGIN_H
251