1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef HISTREAMER_AUDIO_SINK_H
17 #define HISTREAMER_AUDIO_SINK_H
18 #include <mutex>
19 #include "common/status.h"
20 #include "meta/meta.h"
21 #include "sink/media_synchronous_sink.h"
22 #include "media_sync_manager.h"
23 #include "buffer/avbuffer_queue.h"
24 #include "buffer/avbuffer_queue_define.h"
25 #include "plugin/audio_sink_plugin.h"
26 #include "filter/filter.h"
27 #include "plugin/plugin_time.h"
28 
29 namespace OHOS {
30 namespace Media {
31 using namespace OHOS::Media::Plugins;
32 class AudioSink : public std::enable_shared_from_this<AudioSink>, public Pipeline::MediaSynchronousSink {
33 public:
34     AudioSink();
35     ~AudioSink();
36     Status Init(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver);
37     sptr<AVBufferQueueProducer> GetBufferQueueProducer();
38     sptr<AVBufferQueueConsumer> GetBufferQueueConsumer();
39     Status SetParameter(const std::shared_ptr<Meta>& meta);
40     Status GetParameter(std::shared_ptr<Meta>& meta);
41     Status Prepare();
42     Status Start();
43     Status Stop();
44     Status Pause();
45     Status Resume();
46     Status Flush();
47     Status Release();
48     Status SetPlayRange(int64_t start, int64_t end);
49     Status SetVolume(float volume);
50     void DrainOutputBuffer(bool flushed);
51     void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver>& receiver);
52     Status GetLatency(uint64_t& nanoSec);
53     void SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter);
54     int64_t DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer) override;
55     void ResetSyncInfo() override;
56     Status SetSpeed(float speed);
57     Status SetAudioEffectMode(int32_t effectMode);
58     Status GetAudioEffectMode(int32_t &effectMode);
59     int32_t SetVolumeWithRamp(float targetVolume, int32_t duration);
60     void SetThreadGroupId(const std::string& groupId);
61     Status SetIsTransitent(bool isTransitent);
62     Status ChangeTrack(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver);
63     Status SetMuted(bool isMuted);
64     Status SetSeekTime(int64_t seekTime);
65     bool GetSyncCenterClockTime(int64_t &clockTime);
66 
67     float GetMaxAmplitude();
68     int32_t SetMaxAmplitudeCbStatus(bool status);
69     static const int64_t kMinAudioClockUpdatePeriodUs = 20 * HST_USECOND;
70 
71     static const int64_t kMaxAllowedAudioSinkDelayUs = 1500 * HST_MSECOND;
72 protected:
73     std::atomic<OHOS::Media::Pipeline::FilterState> state_;
74 private:
75     Status PrepareInputBufferQueue();
76     std::shared_ptr<Plugins::AudioSinkPlugin> CreatePlugin();
77     int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs);
78     int64_t getDurationUsPlayedAtSampleRate(uint32_t numFrames);
79     void UpdateAudioWriteTimeMayWait();
80     bool UpdateTimeAnchorIfNeeded(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer);
81     void DrainAndReportEosEvent();
82     void HandleEosInner(bool drain);
83     bool DropApeBuffer(std::shared_ptr<AVBuffer> filledOutputBuffer);
84     void CalcMaxAmplitude(std::shared_ptr<AVBuffer> filledOutputBuffer);
85     void CheckUpdateState(char *frame, uint64_t replyBytes, int32_t format);
86     int64_t CalcBufferDuration(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer);
87 
88     class UnderrunDetector {
89     public:
90         void DetectAudioUnderrun(int64_t clkTime, int64_t latency);
91         void SetEventReceiver(std::weak_ptr<Pipeline::EventReceiver> eventReceiver);
92         void UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency);
93         void SetLastAudioBufferDuration(int64_t durationUs);
94         void Reset();
95     private:
96         std::weak_ptr<Pipeline::EventReceiver> eventReceiver_;
97         Mutex mutex_ {};
98         int64_t lastClkTime_ {HST_TIME_NONE};
99         int64_t lastLatency_ {HST_TIME_NONE};
100         int64_t lastBufferDuration_ {HST_TIME_NONE};
101     };
102 
103     class AudioLagDetector : public Pipeline::LagDetector {
104     public:
105         void Reset() override;
106 
107         bool CalcLag(std::shared_ptr<AVBuffer> buffer) override;
108 
SetLatency(int64_t latency)109         void SetLatency(int64_t latency)
110         {
111             latency_ = latency;
112         }
113 
114         struct AudioDrainTimeGroup {
115             int64_t lastAnchorPts = 0;
116             int64_t anchorDuration = 0;
117             int64_t writeDuration = 0;
118             int64_t nowClockTime = 0;
119 
120             AudioDrainTimeGroup() = default;
AudioDrainTimeGroupAudioDrainTimeGroup121             AudioDrainTimeGroup(int64_t anchorPts, int64_t duration, int64_t writeDuration, int64_t clockTime)
122                 : lastAnchorPts(anchorPts),
123                   anchorDuration(duration),
124                   writeDuration(writeDuration),
125                   nowClockTime(clockTime) {}
126         };
127 
128         void UpdateDrainTimeGroup(AudioDrainTimeGroup group);
129     private:
130         int64_t latency_ = 0;
131         int64_t lastDrainTimeMs_ = 0;
132         AudioDrainTimeGroup lastDrainTimeGroup_ {};
133     };
134 
135     std::shared_ptr<Plugins::AudioSinkPlugin> plugin_ {};
136     std::shared_ptr<Pipeline::EventReceiver> playerEventReceiver_;
137     int32_t appUid_{0};
138     int32_t appPid_{0};
139     int64_t numFramesWritten_ {0};
140     int64_t firstAudioAnchorTimeMediaUs_ {HST_TIME_NONE};
141     int64_t nextAudioClockUpdateTimeUs_ {HST_TIME_NONE};
142     int64_t lastAnchorClockTime_  {HST_TIME_NONE};
143     int64_t latestBufferPts_ {HST_TIME_NONE};
144     int64_t latestBufferDuration_ {0};
145     int64_t bufferDurationSinceLastAnchor_ {0};
146     std::atomic<bool> forceUpdateTimeAnchorNextTime_ {true};
147     const std::string INPUT_BUFFER_QUEUE_NAME = "AudioSinkInputBufferQueue";
148     std::shared_ptr<AVBufferQueue> inputBufferQueue_;
149     sptr<AVBufferQueueProducer> inputBufferQueueProducer_;
150     sptr<AVBufferQueueConsumer> inputBufferQueueConsumer_;
151     int64_t firstPts_ {HST_TIME_NONE};
152     int32_t sampleRate_ {0};
153     int32_t samplePerFrame_ {0};
154     int32_t audioChannelCount_ = 0;
155     int64_t fixDelay_ {0};
156     bool isTransitent_ {false};
157     bool isEos_ {false};
158     std::unique_ptr<Task> eosTask_ {nullptr};
159     enum class EosInterruptState : int {
160         NONE,
161         INITIAL,
162         PAUSE,
163         RESUME,
164         STOP,
165     };
166     Mutex eosMutex_ {};
167     std::atomic<bool> eosDraining_ {false};
168     std::atomic<EosInterruptState> eosInterruptType_ {EosInterruptState::NONE};
169     bool isApe_ {false};
170     int64_t playRangeStartTime_ = -1;
171     int64_t playRangeEndTime_ = -1;
172     // vars for audio progress optimization
173     int64_t playingBufferDurationUs_ {0};
174     int64_t lastBufferWriteTime_ {0};
175     bool lastBufferWriteSuccess_ {true};
176     float speed_ {1.0f};
177     std::mutex pluginMutex_;
178     float volume_ {-1.0f};
179     int32_t effectMode_ {-1};
180     bool isMuted_ = false;
181     Mutex amplitudeMutex_ {};
182     float maxAmplitude_ = 0;
183 
184     bool calMaxAmplitudeCbStatus_ = false;
185     UnderrunDetector underrunDetector_;
186     AudioLagDetector lagDetector_;
187     std::atomic<int64_t> seekTimeUs_ {HST_TIME_NONE};
188 };
189 }
190 }
191 
192 #endif // HISTREAMER_AUDIO_SINK_H
193