1 /*
2  * Copyright (c) 2021-2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define HST_LOG_TAG "AudioSinkFilter"
17 #include "pipeline/filters/sink/audio_sink/audio_sink_filter.h"
18 #include "foundation/log.h"
19 #include "foundation/osal/utils/util.h"
20 #include "foundation/utils/dump_buffer.h"
21 #include "foundation/utils/steady_clock.h"
22 #include "pipeline/factory/filter_factory.h"
23 #include "pipeline/filters/common/plugin_settings.h"
24 #include "pipeline/filters/common/plugin_utils.h"
25 #include "plugin/common/plugin_attr_desc.h"
26 #include "plugin/common/plugin_time.h"
27 #include "plugin/common/plugin_types.h"
28 
29 namespace OHOS {
30 namespace Media {
31 namespace Pipeline {
32 using namespace Plugin;
33 namespace {
34 constexpr int REPORT_DURATION = 20 * HST_MSECOND; // 20 ms
35 constexpr int WAIT_PREROLLED_TIMEOUT = 80 * HST_MSECOND; // 80ms
36 }
37 static AutoRegisterFilter<AudioSinkFilter> g_registerFilterHelper("builtin.player.audiosink");
38 
AudioSinkFilter(const std::string & name)39 AudioSinkFilter::AudioSinkFilter(const std::string& name) : MediaSynchronousSink(name)
40 {
41     filterType_ = FilterType::AUDIO_SINK;
42     syncerPriority_ = IMediaSynchronizer::AUDIO_SINK;
43     reportAnchorDuration_ = REPORT_DURATION;
44     waitPrerolledTimeout_ = WAIT_PREROLLED_TIMEOUT;
45     MEDIA_LOG_I("audio sink ctor called");
46 }
~AudioSinkFilter()47 AudioSinkFilter::~AudioSinkFilter()
48 {
49     MEDIA_LOG_D("audio sink dtor called");
50     if (plugin_) {
51         plugin_->Stop();
52         plugin_->Deinit();
53     }
54 }
55 
Init(EventReceiver * receiver,FilterCallback * callback)56 void AudioSinkFilter::Init(EventReceiver* receiver, FilterCallback* callback)
57 {
58     MediaSynchronousSink::Init(receiver, callback);
59     outPorts_.clear();
60 }
61 
SetPluginParameter(Tag tag,const Plugin::ValueType & value)62 ErrorCode AudioSinkFilter::SetPluginParameter(Tag tag, const Plugin::ValueType& value)
63 {
64     return TranslatePluginStatus(plugin_->SetParameter(tag, value));
65 }
66 
SetParameter(int32_t key,const Plugin::Any & value)67 ErrorCode AudioSinkFilter::SetParameter(int32_t key, const Plugin::Any& value)
68 {
69     Tag tag = Tag::INVALID;
70     if (!TranslateIntoParameter(key, tag)) {
71         MEDIA_LOG_I("SetParameter key " PUBLIC_LOG_D32 " is out of boundary", key);
72         return ErrorCode::ERROR_INVALID_PARAMETER_VALUE;
73     }
74     switch (tag) {
75         case Tag::APP_PID:
76             FALSE_RETURN_V_MSG_E(Any::IsSameTypeWith<int32_t>(value), ErrorCode::ERROR_INVALID_PARAMETER_TYPE,
77                                  "APP_PID type should be int32_t");
78             appPid_ = Plugin::AnyCast<int32_t>(value);
79             break;
80         case Tag::APP_UID:
81             FALSE_RETURN_V_MSG_E(Any::IsSameTypeWith<int32_t>(value), ErrorCode::ERROR_INVALID_PARAMETER_TYPE,
82                                  "APP_UID type should be int32_t");
83             appUid_ = Plugin::AnyCast<int32_t>(value);
84             break;
85         case Tag::AUDIO_RENDER_INFO:
86             FALSE_RETURN_V_MSG_E(Any::IsSameTypeWith<AudioRenderInfo>(value), ErrorCode::ERROR_INVALID_PARAMETER_TYPE,
87                                  "AUDIO_RENDER_INFO type should be AudioRenderInfo");
88             audioRenderInfo_ = Plugin::AnyCast<AudioRenderInfo>(value);
89             break;
90         case Tag::AUDIO_INTERRUPT_MODE:
91             FALSE_RETURN_V_MSG_E(Any::IsSameTypeWith<int32_t>(value), ErrorCode::ERROR_INVALID_PARAMETER_TYPE,
92                                  "AUDIO_INTERRUPT_MODE type should be int32_t");
93             audioInterruptMode_ = static_cast<AudioInterruptMode>(Plugin::AnyCast<int32_t>(value));
94             if (plugin_) {
95                 (void)plugin_->SetParameter(Tag::AUDIO_INTERRUPT_MODE, audioInterruptMode_);
96             }
97             break;
98         default:
99             break;
100     }
101     return ErrorCode::SUCCESS;
102 }
103 
GetParameter(int32_t key,Plugin::Any & value)104 ErrorCode AudioSinkFilter::GetParameter(int32_t key, Plugin::Any& value)
105 {
106     if (state_.load() == FilterState::CREATED) {
107         return ErrorCode::ERROR_AGAIN;
108     }
109     Tag tag = Tag::INVALID;
110     if (!TranslateIntoParameter(key, tag)) {
111         MEDIA_LOG_I("GetParameter key " PUBLIC_LOG_D32 " is out of boundary", key);
112         return ErrorCode::ERROR_INVALID_PARAMETER_VALUE;
113     }
114     RETURN_AGAIN_IF_NULL(plugin_);
115     return TranslatePluginStatus(plugin_->GetParameter(tag, value));
116 }
117 
Negotiate(const std::string & inPort,const std::shared_ptr<const Plugin::Capability> & upstreamCap,Plugin::Capability & negotiatedCap,const Plugin::Meta & upstreamParams,Plugin::Meta & downstreamParams)118 bool AudioSinkFilter::Negotiate(const std::string& inPort,
119                                 const std::shared_ptr<const Plugin::Capability>& upstreamCap,
120                                 Plugin::Capability& negotiatedCap,
121                                 const Plugin::Meta& upstreamParams,
122                                 Plugin::Meta& downstreamParams)
123 {
124     MEDIA_LOG_I("audio sink negotiate started");
125     FALSE_LOG(const_cast<Plugin::Meta&>(upstreamParams).Get<Tag::MEDIA_SEEKABLE>(seekable_));
126     PROFILE_BEGIN("Audio Sink Negotiate begin");
127     auto candidatePlugins = FindAvailablePlugins(*upstreamCap, Plugin::PluginType::AUDIO_SINK);
128     if (candidatePlugins.empty()) {
129         MEDIA_LOG_E("no available audio sink plugin");
130         return false;
131     }
132     // always use first one
133     std::shared_ptr<Plugin::PluginInfo> selectedPluginInfo = candidatePlugins[0].first;
134     for (const auto& onCap : selectedPluginInfo->inCaps) {
135         if (onCap.keys.count(CapabilityID::AUDIO_SAMPLE_FORMAT) == 0) {
136             MEDIA_LOG_E("each in caps of sink must contains valid audio sample format");
137             return false;
138         }
139     }
140     negotiatedCap = candidatePlugins[0].second;
141     MEDIA_LOG_I("use plugin " PUBLIC_LOG_S " with negotiated " PUBLIC_LOG_S, selectedPluginInfo->name.c_str(),
142                 Capability2String(negotiatedCap).c_str());
143     auto res = UpdateAndInitPluginByInfo<Plugin::AudioSink>(plugin_, pluginInfo_, selectedPluginInfo,
144         [this](const std::string& name) -> std::shared_ptr<Plugin::AudioSink> {
145         auto plugin = Plugin::PluginManager::Instance().CreateAudioSinkPlugin(name);
146         (void)plugin->SetParameter(Tag::APP_PID, appPid_);
147         (void)plugin->SetParameter(Tag::APP_UID, appUid_);
148         (void)plugin->SetParameter(Tag::AUDIO_RENDER_INFO, audioRenderInfo_);
149         (void)plugin->SetParameter(Tag::AUDIO_INTERRUPT_MODE, audioInterruptMode_);
150         (void)plugin->SetCallback(this);
151         return plugin;
152     });
153     NOK_LOG(plugin_->SetParameter(Tag::MEDIA_SEEKABLE, seekable_));
154     Plugin::ValueType pluginValue;
155     if (plugin_->GetParameter(Tag::AUDIO_OUTPUT_CHANNELS, pluginValue) == Plugin::Status::OK) {
156         auto outputChannels = Plugin::AnyCast<uint32_t>(pluginValue);
157         downstreamParams.Set<Tag::AUDIO_OUTPUT_CHANNELS>(outputChannels);
158         MEDIA_LOG_D("Get support outputChannels: " PUBLIC_LOG_U32, outputChannels);
159     }
160     if (plugin_->GetParameter(Tag::AUDIO_OUTPUT_CHANNEL_LAYOUT, pluginValue) == Plugin::Status::OK) {
161         auto outputChanLayout = Plugin::AnyCast<Plugin::AudioChannelLayout>(pluginValue);
162         downstreamParams.Set<Tag::AUDIO_OUTPUT_CHANNEL_LAYOUT>(outputChanLayout);
163         MEDIA_LOG_D("Get support outputChannelLayout: " PUBLIC_LOG_U64, outputChanLayout);
164     }
165     PROFILE_END("Audio Sink Negotiate end");
166     return res;
167 }
168 
Configure(const std::string & inPort,const std::shared_ptr<const Plugin::Meta> & upstreamMeta,Plugin::Meta & upstreamParams,Plugin::Meta & downstreamParams)169 bool AudioSinkFilter::Configure(const std::string& inPort, const std::shared_ptr<const Plugin::Meta>& upstreamMeta,
170                                 Plugin::Meta& upstreamParams, Plugin::Meta& downstreamParams)
171 {
172     PROFILE_BEGIN("Audio sink configure begin");
173     MEDIA_LOG_I("receive upstream meta " PUBLIC_LOG_S, Meta2String(*upstreamMeta).c_str());
174     if (plugin_ == nullptr || pluginInfo_ == nullptr) {
175         MEDIA_LOG_E("cannot configure decoder when no plugin available");
176         return false;
177     }
178     SetVolumeToPlugin();
179     auto err = ConfigureToPreparePlugin(upstreamMeta);
180     if (err != ErrorCode::SUCCESS) {
181         MEDIA_LOG_E("sink configure error");
182         FilterBase::OnEvent({name_, EventType::EVENT_ERROR, err});
183         return false;
184     }
185     UpdateMediaTimeRange(*upstreamMeta);
186     state_ = FilterState::READY;
187     FilterBase::OnEvent({name_, EventType::EVENT_READY});
188     MEDIA_LOG_I("audio sink send EVENT_READY");
189     PROFILE_END("Audio sink configure end");
190     return true;
191 }
192 
ConfigureToPreparePlugin(const std::shared_ptr<const Plugin::Meta> & meta)193 ErrorCode AudioSinkFilter::ConfigureToPreparePlugin(const std::shared_ptr<const Plugin::Meta>& meta)
194 {
195     FAIL_RETURN_MSG(ConfigPluginWithMeta(*plugin_, *meta), "sink configuration failed.");
196     FAIL_RETURN_MSG(TranslatePluginStatus(plugin_->Prepare()), "sink prepare failed");
197     return ErrorCode::SUCCESS;
198 }
199 
PushData(const std::string & inPort,const AVBufferPtr & buffer,int64_t offset)200 ErrorCode AudioSinkFilter::PushData(const std::string& inPort, const AVBufferPtr& buffer, int64_t offset)
201 {
202     MEDIA_LOG_DD("audio sink push data started, state: " PUBLIC_LOG_D32, state_.load());
203     if (isFlushing || state_.load() == FilterState::INITIALIZED) {
204         MEDIA_LOG_DD("audio sink is flushing ignore this buffer");
205         return ErrorCode::SUCCESS;
206     }
207     if (state_.load() != FilterState::RUNNING) {
208         pushThreadIsBlocking = true;
209         MEDIA_LOG_I("audio sink push data wait.");
210         OSAL::ScopedLock lock(mutex_);
211         startWorkingCondition_.Wait(lock, [this] {
212             return state_ == FilterState::RUNNING || state_ == FilterState::INITIALIZED || isFlushing;
213         });
214         pushThreadIsBlocking = false;
215     }
216     if (isFlushing || state_.load() == FilterState::INITIALIZED) {
217         MEDIA_LOG_I("PushData return due to: isFlushing = " PUBLIC_LOG_D32 ", state = " PUBLIC_LOG_D32,
218                     isFlushing, static_cast<int>(state_.load()));
219         return ErrorCode::SUCCESS;
220     }
221     DUMP_BUFFER2LOG("AudioSink Write", buffer, offset);
222     FAIL_RETURN_MSG(WriteToPluginRefTimeSync(buffer), "audio sink write failed");
223     if ((buffer->flag & BUFFER_FLAG_EOS) != 0) {
224         plugin_->Drain();
225         Event event{
226             .srcFilter = name_,
227             .type = EventType::EVENT_COMPLETE,
228         };
229         MEDIA_LOG_D("audio sink push data send event_complete");
230         FilterBase::OnEvent(event);
231     }
232     MEDIA_LOG_DD("audio sink push data end");
233     return ErrorCode::SUCCESS;
234 }
235 
Start()236 ErrorCode AudioSinkFilter::Start()
237 {
238     MEDIA_LOG_I("start called");
239     if (state_ != FilterState::READY && state_ != FilterState::PAUSED) {
240         MEDIA_LOG_W("sink is not ready when start, state: " PUBLIC_LOG_D32, static_cast<int32_t>(state_.load()));
241         return ErrorCode::ERROR_INVALID_OPERATION;
242     }
243     forceUpdateTimeAnchorNextTime_ = true;
244     auto err = FilterBase::Start();
245     if (err != ErrorCode::SUCCESS) {
246         MEDIA_LOG_E("audio sink filter start error");
247         return err;
248     }
249     err = TranslatePluginStatus(plugin_->Start());
250     FAIL_RETURN_MSG(err, "audio sink plugin start failed");
251     if (pushThreadIsBlocking.load()) {
252         startWorkingCondition_.NotifyOne();
253     }
254     frameCnt_ = 0;
255     return ErrorCode::SUCCESS;
256 }
257 
Stop()258 ErrorCode AudioSinkFilter::Stop()
259 {
260     MEDIA_LOG_I("audio sink stop start");
261     FilterBase::Stop();
262     if (plugin_ != nullptr) {
263         plugin_->Stop();
264     }
265     if (pushThreadIsBlocking.load()) {
266         startWorkingCondition_.NotifyOne();
267     }
268     MEDIA_LOG_I("audio sink stop finish");
269     return ErrorCode::SUCCESS;
270 }
271 
Pause()272 ErrorCode AudioSinkFilter::Pause()
273 {
274     MEDIA_LOG_I("audio sink filter pause start");
275     // only worked when state is working
276     if (state_ != FilterState::READY && state_ != FilterState::RUNNING) {
277         MEDIA_LOG_W("audio sink cannot pause when not working");
278         return ErrorCode::ERROR_INVALID_OPERATION;
279     }
280     auto err = FilterBase::Pause();
281     FAIL_RETURN_MSG(err, "audio sink pause failed");
282     err = TranslatePluginStatus(plugin_->Pause());
283     MEDIA_LOG_D("audio sink filter pause end");
284     return err;
285 }
Resume()286 ErrorCode AudioSinkFilter::Resume()
287 {
288     MEDIA_LOG_I("audio sink filter resume");
289     // only worked when state is paused
290     if (state_ == FilterState::PAUSED) {
291         forceUpdateTimeAnchorNextTime_ = true;
292         state_ = FilterState::RUNNING;
293         if (pushThreadIsBlocking) {
294             startWorkingCondition_.NotifyOne();
295         }
296         if (frameCnt_ > 0) {
297             frameCnt_ = 0;
298         }
299         return TranslatePluginStatus(plugin_->Resume());
300     }
301     return ErrorCode::SUCCESS;
302 }
303 
FlushStart()304 void AudioSinkFilter::FlushStart()
305 {
306     MEDIA_LOG_I("audio sink flush start entered");
307     isFlushing = true;
308     if (pushThreadIsBlocking) {
309         startWorkingCondition_.NotifyOne();
310     }
311 }
312 
FlushEnd()313 void AudioSinkFilter::FlushEnd()
314 {
315     MEDIA_LOG_I("audio sink flush end entered");
316     plugin_->Resume();
317     isFlushing = false;
318     ResetSyncInfo();
319 }
SetVolumeToPlugin()320 ErrorCode AudioSinkFilter::SetVolumeToPlugin()
321 {
322     if (plugin_ == nullptr) {
323         return ErrorCode::SUCCESS;
324     }
325     if (volume_ < 0) {
326         MEDIA_LOG_D("No need to set volume because upper layer hasn't set it.");
327         return ErrorCode::SUCCESS;
328     }
329     return TranslatePluginStatus(plugin_->SetVolume(volume_));
330 }
SetVolume(float volume)331 ErrorCode AudioSinkFilter::SetVolume(float volume)
332 {
333     volume_ = volume;
334     MEDIA_LOG_I("set volume " PUBLIC_LOG ".3f", volume);
335     return SetVolumeToPlugin();
336 }
337 
DoSyncWrite(const AVBufferPtr & buffer)338 ErrorCode AudioSinkFilter::DoSyncWrite(const AVBufferPtr &buffer)
339 {
340     bool render = true;
341     if ((buffer->flag & BUFFER_FLAG_EOS) == 0) { // only need to update when not eos
342         // audio sink always report time anchor and do not drop
343         int64_t nowCt = 0;
344         auto syncCenter = syncCenter_.lock();
345         if (syncCenter) {
346             nowCt = syncCenter->GetClockTimeNow();
347         }
348         if (lastReportedClockTime_ == HST_TIME_NONE || lastReportedClockTime_ + reportAnchorDuration_ <= nowCt ||
349             forceUpdateTimeAnchorNextTime_) {
350             uint64_t latency = 0;
351             if (plugin_->GetLatency(latency) != Plugin::Status::OK) {
352                 MEDIA_LOG_W("failed to get latency");
353             }
354             if (syncCenter) {
355                 render = syncCenter->UpdateTimeAnchor(nowCt + latency, buffer->pts, this);
356             }
357             lastReportedClockTime_ = nowCt;
358             forceUpdateTimeAnchorNextTime_ = true;
359         }
360         latestBufferPts_ = buffer->pts;
361         latestBufferDuration_ = buffer->duration;
362     }
363     if (!render) {
364         MEDIA_LOG_DD("drop buffer with pts " PUBLIC_LOG_D64 " due to seek not need to render", buffer->pts);
365         return ErrorCode::SUCCESS;
366     } else {
367         return TranslatePluginStatus(plugin_->Write(buffer));
368     }
369 }
370 
ResetSyncInfo()371 void AudioSinkFilter::ResetSyncInfo()
372 {
373     ResetPrerollReported();
374     lastReportedClockTime_ = HST_TIME_NONE;
375     latestBufferPts_ = HST_TIME_NONE;
376     latestBufferDuration_ = HST_TIME_NONE;
377 }
378 
OnEvent(const Plugin::PluginEvent & event)379 void AudioSinkFilter::OnEvent(const Plugin::PluginEvent& event)
380 {
381     FilterBase::OnEvent(Event{name_, EventType::EVENT_PLUGIN_EVENT, event});
382 }
383 } // namespace Pipeline
384 } // namespace Media
385 } // namespace OHOS
386