1 /*
2  * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef MEDIA_DEMUXER_H
17 #define MEDIA_DEMUXER_H
18 
19 #include <atomic>
20 #include <limits>
21 #include <string>
22 #include <shared_mutex>
23 #include <unordered_set>
24 
25 #include "osal/task/condition_variable.h"
26 #include "avcodec_common.h"
27 #include "buffer/avbuffer.h"
28 #include "common/media_source.h"
29 #include "common/seek_callback.h"
30 #include "demuxer/type_finder.h"
31 #include "filter/filter.h"
32 #include "meta/media_types.h"
33 #include "osal/task/autolock.h"
34 #include "osal/task/task.h"
35 #include "plugin/plugin_base.h"
36 #include "plugin/plugin_info.h"
37 #include "plugin/plugin_time.h"
38 #include "plugin/demuxer_plugin.h"
39 
40 namespace OHOS {
41 namespace Media {
42 namespace {
43     constexpr uint32_t TRACK_ID_DUMMY = std::numeric_limits<uint32_t>::max();
44     constexpr int32_t DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT = 120;
45 }
46 
47 using MediaSource = OHOS::Media::Plugins::MediaSource;
48 class BaseStreamDemuxer;
49 class DemuxerPluginManager;
50 class Source;
51 
52 class AVBufferQueueProducer;
53 
54 class MediaDemuxer : public std::enable_shared_from_this<MediaDemuxer>, public Plugins::Callback {
55 public:
56     explicit MediaDemuxer();
57     ~MediaDemuxer() override;
58 
59     Status SetDataSource(const std::shared_ptr<MediaSource> &source);
60     Status SetSubtitleSource(const std::shared_ptr<MediaSource> &source);
61     void SetBundleName(const std::string& bundleName);
62     Status SetOutputBufferQueue(int32_t trackId, const sptr<AVBufferQueueProducer>& producer);
63 
64     std::shared_ptr<Meta> GetGlobalMetaInfo() const;
65     std::vector<std::shared_ptr<Meta>> GetStreamMetaInfo() const;
66     std::shared_ptr<Meta> GetUserMeta();
67 
68     Status SeekTo(int64_t seekTime, Plugins::SeekMode mode, int64_t& realSeekTime);
69     Status Reset();
70     Status Start();
71     Status Stop();
72     Status Pause();
73     Status PauseDragging();
74     Status PauseAudioAlign();
75     Status Resume();
76     Status ResumeDragging();
77     Status ResumeAudioAlign();
78     Status Flush();
79     Status Preroll();
80     Status PausePreroll();
81 
82     Status StartTask(int32_t trackId);
83     Status SelectTrack(int32_t trackId);
84     Status UnselectTrack(int32_t trackId);
85     Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample);
86     Status GetBitRates(std::vector<uint32_t> &bitRates);
87     Status SelectBitRate(uint32_t bitRate);
88     Status GetDownloadInfo(DownloadInfo& downloadInfo);
89     Status GetPlaybackInfo(PlaybackInfo& playbackInfo);
90     Status GetMediaKeySystemInfo(std::multimap<std::string, std::vector<uint8_t>> &infos);
91     void SetDrmCallback(const std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> &callback);
92     void OnEvent(const Plugins::PluginEvent &event) override;
93     void OnSeekReadyEvent(const Plugins::PluginEvent &event);
94 
95     std::map<uint32_t, sptr<AVBufferQueueProducer>> GetBufferQueueProducerMap();
96     Status PauseTaskByTrackId(int32_t trackId);
97     bool IsRenderNextVideoFrameSupported();
98 
99     void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver);
100     bool GetDuration(int64_t& durationMs);
101     void SetPlayerId(const std::string &playerId);
102     void SetInterruptState(bool isInterruptNeeded);
103     void SetDumpInfo(bool isDump, uint64_t instanceId);
104 
105     Status OptimizeDecodeSlow(bool isDecodeOptimizationEnabled);
106     Status SetDecoderFramerateUpperLimit(int32_t decoderFramerateUpperLimit, uint32_t trackId);
107     Status SetSpeed(float speed);
108     Status SetFrameRate(double frameRate, uint32_t trackId);
109 
110     bool IsLocalDrmInfosExisted();
111     Status DisableMediaTrack(Plugins::MediaType mediaType);
112     void OnBufferAvailable(uint32_t trackId);
113     void SetSelectBitRateFlag(bool flag, uint32_t desBitRate) override;
114     bool CanAutoSelectBitRate() override;
115     void OnDumpInfo(int32_t fd);
116 
117     bool IsRefParserSupported();
118     Status StartReferenceParser(int64_t startTimeMs, bool isForward = true);
119     Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo);
120     Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo);
121     Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo);
122     bool IsVideoEos();
123     bool HasEosTrack();
124     Status GetIFramePos(std::vector<uint32_t> &IFramePos);
125     Status Dts2FrameId(int64_t dts, uint32_t &frameId, bool offset = true);
126     void RegisterVideoStreamReadyCallback(const std::shared_ptr<VideoStreamReadyCallback> &callback);
127     void DeregisterVideoStreamReadyCallback();
128 
129     Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
130         const uint64_t relativePresentationTimeUs, uint32_t &index);
131     Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
132         const uint32_t index, uint64_t &relativePresentationTimeUs);
133 
134     Status ResumeDemuxerReadLoop();
135     Status PauseDemuxerReadLoop();
136     void SetCacheLimit(uint32_t limitSize);
137     void SetEnableOnlineFdCache(bool isEnableFdCache);
138     void WaitForBufferingEnd();
139     int32_t GetCurrentVideoTrackId();
140     uint32_t GetTargetVideoTrackId(std::vector<std::shared_ptr<Meta>> trackInfos);
141     void SetIsEnableReselectVideoTrack(bool isEnable);
142     bool IsHasMultiVideoTrack();
143 private:
144     class AVBufferQueueProducerListener;
145     class TrackWrapper;
146 
147     struct MediaMetaData {
148         std::vector<std::shared_ptr<Meta>> trackMetas;
149         std::shared_ptr<Meta> globalMeta;
150     };
151 
152     struct MaintainBaseInfo {
153         int64_t segmentOffset = -1;
154         int64_t basePts = -1;
155         int64_t lastPts = 0;
156     };
157     bool isHttpSource_ = false;
158     std::string videoMime_{};
159 
160     void InitMediaMetaData(const Plugins::MediaInfo& mediaInfo);
161     void InitDefaultTrack(const Plugins::MediaInfo& mediaInfo, uint32_t& videoTrackId,
162         uint32_t& audioTrackId, uint32_t& subtitleTrackId, std::string& videoMime);
163     std::shared_ptr<Meta> GetTrackMeta(uint32_t trackId);
164     Status AddDemuxerCopyTask(uint32_t trackId, TaskType type);
165 
166     Status StopAllTask();
167     Status PauseAllTask();
168     Status ResumeAllTask();
169     void AccelerateTrackTask(uint32_t trackId);
170     void SetTrackNotifyFlag(uint32_t trackId, bool isNotifyNeeded);
171     void ResetInner();
172 
173     bool GetDrmInfosUpdated(const std::multimap<std::string, std::vector<uint8_t>> &newInfos,
174         std::multimap<std::string, std::vector<uint8_t>> &result);
175     Status ProcessDrmInfos();
176     void HandleSourceDrmInfoEvent(const std::multimap<std::string, std::vector<uint8_t>> &info);
177     Status ReportDrmInfos(const std::multimap<std::string, std::vector<uint8_t>> &info);
178 
179     bool HasVideo();
180     void DumpBufferToFile(uint32_t trackId, std::shared_ptr<AVBuffer> buffer);
181     bool IsBufferDroppable(std::shared_ptr<AVBuffer> sample, uint32_t trackId);
182     void CheckDropAudioFrame(std::shared_ptr<AVBuffer> sample, uint32_t trackId);
183     bool IsTrackDisabled(Plugins::MediaType mediaType);
184     bool CheckTrackEnabledById(uint32_t trackId);
185     bool HandleDashChangeStream(uint32_t trackId);
186 
187     Status SeekToTimeAfter();
188     bool SelectBitRateChangeStream(uint32_t trackId);
189     bool SelectTrackChangeStream(uint32_t trackId);
190     bool HandleSelectTrackChangeStream(int32_t trackId, int32_t newStreamID, int32_t& newTrackId);
191     std::shared_ptr<Plugins::DemuxerPlugin> GetCurFFmpegPlugin();
192 
193     Plugins::Seekable seekable_;
194     Plugins::Seekable subSeekable_;
195     std::string uri_;
196     std::string SubtitleUri_;
197     uint64_t mediaDataSize_ = 0;
198     uint64_t subMediaDataSize_;
199 
200     std::shared_ptr<MediaSource> mediaSource_;
201     std::shared_ptr<Source> source_;
202     std::shared_ptr<Source> subtitleSource_;
203     MediaMetaData mediaMetaData_;
204 
205     int64_t ReadLoop(uint32_t trackId);
206     Status CopyFrameToUserQueue(uint32_t trackId);
207     bool GetBufferFromUserQueue(uint32_t queueIndex, uint32_t size = 0);
208     Status InnerReadSample(uint32_t trackId, std::shared_ptr<AVBuffer>);
209     Status InnerSelectTrack(int32_t trackId);
210     Status HandleReadSample(uint32_t trackId);
211     int64_t ParserRefInfo();
212     void TryRecvParserTask();
213 
214     Status HandleSelectTrack(int32_t trackId);
215     Status HandleDashSelectTrack(int32_t trackId);
216     Status DoSelectTrack(int32_t trackId, int32_t curTrackId);
217     Status HandleRebootPlugin(int32_t trackId, bool& isRebooted);
218     bool DashCheckChangeStream(uint32_t trackId);
219 
220     bool IsSubtitleMime(const std::string& mime);
221     Status InnerPrepare();
222     Status HandleAutoMaintainPts(uint32_t trackId, std::shared_ptr<AVBuffer> sample);
223     Status InitPtsInfo();
224 
225     Mutex mapMutex_{};
226     std::map<uint32_t, std::shared_ptr<TrackWrapper>> trackMap_;
227     std::map<uint32_t, sptr<AVBufferQueueProducer>> bufferQueueMap_;
228     std::map<uint32_t, std::shared_ptr<AVBuffer>> bufferMap_;
229     std::map<uint32_t, bool> eosMap_;
230     std::map<uint32_t, uint32_t> requestBufferErrorCountMap_;
231     std::atomic<bool> isThreadExit_ = true;
232     bool useBufferQueue_ = false;
233     bool isAccurateSeekForHLS_ = false;
234     int64_t videoStartTime_{0};
235 
236     std::shared_mutex drmMutex{};
237     std::mutex isSelectTrackMutex_{};
238     std::mutex rebootPluginMutex_{};
239     std::multimap<std::string, std::vector<uint8_t>> localDrmInfos_;
240     std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> drmCallback_;
241 
242     std::map<uint32_t, std::unique_ptr<Task>> taskMap_;
243     std::shared_ptr<Pipeline::EventReceiver> eventReceiver_;
244     int64_t lastSeekTime_{Plugins::HST_TIME_NONE};
245     bool isSeeked_{false};
246     uint32_t videoTrackId_{TRACK_ID_DUMMY};
247 
248     uint32_t audioTrackId_{TRACK_ID_DUMMY};
249     uint32_t subtitleTrackId_{TRACK_ID_DUMMY};
250     bool firstAudio_{true};
251 
252     std::atomic<bool> isStopped_ = true;
253     std::atomic<bool> isPaused_ = false;
254     std::shared_ptr<BaseStreamDemuxer> streamDemuxer_;
255     std::shared_ptr<BaseStreamDemuxer> subStreamDemuxer_;
256     std::string bundleName_ {};
257 
258     std::string playerId_;
259     int64_t duration_ {0};
260 
261     std::atomic<bool> isDecodeOptimizationEnabled_ {false};
262     std::atomic<float> speed_ {1.0f};
263     std::atomic<double> framerate_ {0.0};
264     std::atomic<int32_t> decoderFramerateUpperLimit_ {DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT};
265     std::unordered_set<Plugins::MediaType> disabledMediaTracks_ {};
266     std::string subtitlePluginName_;
267     std::shared_ptr<Plugins::DemuxerPlugin> subtitlePlugin_;
268     std::shared_ptr<MediaSource> subtitleMediaSource_;
269     bool isDump_ = false;
270     std::shared_ptr<DemuxerPluginManager> demuxerPluginManager_;
271     std::atomic<bool> isSelectBitRate_ = false;
272     uint32_t targetBitRate_ = 0;
273     std::string dumpPrefix_ = "";
274 
275     std::unique_ptr<Task> parserRefInfoTask_;
276     bool isFirstParser_ = true;
277     bool isParserTaskEnd_ = false;
278     std::atomic<bool> isOnEventNoMemory_ = false;
279     std::atomic<bool> isSeekError_ = false;
280     std::shared_ptr<VideoStreamReadyCallback> VideoStreamReadyCallback_ = nullptr;
281     std::mutex draggingMutex_ {};
282 
283     std::atomic<bool> isDemuxerLoopExecuting_ {false};
284     std::atomic<bool> isFirstFrameAfterSeek_ {false};
285 
286     std::mutex prerollMutex_ {};
287     std::atomic<bool> inPreroll_ = false;
288 
289     std::map<int32_t, int32_t> inSelectTrackType_{};
290     std::map<int32_t, std::pair<int32_t, int32_t>> seekReadyStreamInfo_{};
291     std::condition_variable rebootPluginCondition_;
292     std::atomic<bool> isSelectTrack_ = false;
293     std::atomic<bool> shouldCheckAudioFramePts_ = false;
294     int64_t lastAudioPts_ = 0;
295     std::atomic<bool> shouldCheckSubtitleFramePts_ = false;
296     int64_t lastSubtitlePts_ = 0;
297     std::atomic<bool> isInterruptNeeded_ {false};
298     bool isAutoMaintainPts_ = false;
299     std::map<uint32_t, std::shared_ptr<MaintainBaseInfo>> maintainBaseInfos_;
300     bool isEnableReselectVideoTrack_ {false};
301     int32_t videoTrackCount_ = 0;
302     uint32_t targetVideoTrackId_ {TRACK_ID_DUMMY};
303 };
304 } // namespace Media
305 } // namespace OHOS
306 #endif // MEDIA_DEMUXER_H
307