1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "subtitle_sink.h"
17 
18 #include "common/log.h"
19 #include "syspara/parameters.h"
20 #include "meta/format.h"
21 
22 namespace {
23 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "SubtitleSink" };
24 }
25 
26 namespace OHOS {
27 namespace Media {
28 namespace {
29 constexpr bool SUBTITME_LOOP_RUNNING = true;
30 }
31 
SubtitleSink()32 SubtitleSink::SubtitleSink()
33 {
34     MEDIA_LOG_I("SubtitleSink ctor");
35     syncerPriority_ = IMediaSynchronizer::SUBTITLE_SINK;
36 }
37 
~SubtitleSink()38 SubtitleSink::~SubtitleSink()
39 {
40     MEDIA_LOG_I("SubtitleSink dtor");
41     {
42         std::unique_lock<std::mutex> lock(mutex_);
43         isThreadExit_ = true;
44     }
45     updateCond_.notify_all();
46     if (readThread_ != nullptr && readThread_->joinable()) {
47         readThread_->join();
48         readThread_ = nullptr;
49     }
50 
51     if (inputBufferQueueProducer_ != nullptr) {
52         for (auto &buffer : inputBufferVector_) {
53             inputBufferQueueProducer_->DetachBuffer(buffer);
54         }
55         inputBufferVector_.clear();
56         inputBufferQueueProducer_->SetQueueSize(0);
57     }
58 }
59 
NotifySeek()60 void SubtitleSink::NotifySeek()
61 {
62     Flush();
63 }
64 
GetTargetSubtitleIndex(int64_t currentTime)65 void SubtitleSink::GetTargetSubtitleIndex(int64_t currentTime)
66 {
67     int32_t left = 0;
68     int32_t right = static_cast<int32_t>(subtitleInfoVec_.size());
69     while (left < right) {
70         int32_t mid = (left + right) / 2;
71         int64_t startTime = subtitleInfoVec_.at(mid).pts_;
72         int64_t endTime = subtitleInfoVec_.at(mid).duration_ + startTime;
73         if (startTime > currentTime) {
74             right = mid;
75             continue;
76         } else if (endTime < currentTime) {
77             left = mid + 1;
78             continue;
79         } else {
80             left = mid;
81             break;
82         }
83     }
84     currentInfoIndex_ = static_cast<uint32_t>(left);
85 }
86 
Init(std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver)87 Status SubtitleSink::Init(std::shared_ptr<Meta> &meta, const std::shared_ptr<Pipeline::EventReceiver> &receiver)
88 {
89     state_ = Pipeline::FilterState::INITIALIZED;
90     if (meta != nullptr) {
91         meta->SetData(Tag::APP_PID, appPid_);
92         meta->SetData(Tag::APP_UID, appUid_);
93     }
94     return Status::OK;
95 }
96 
GetBufferQueueProducer()97 sptr<AVBufferQueueProducer> SubtitleSink::GetBufferQueueProducer()
98 {
99     if (state_ != Pipeline::FilterState::READY) {
100         return nullptr;
101     }
102     return inputBufferQueueProducer_;
103 }
104 
GetBufferQueueConsumer()105 sptr<AVBufferQueueConsumer> SubtitleSink::GetBufferQueueConsumer()
106 {
107     if (state_ != Pipeline::FilterState::READY) {
108         return nullptr;
109     }
110     return inputBufferQueueConsumer_;
111 }
112 
SetParameter(const std::shared_ptr<Meta> & meta)113 Status SubtitleSink::SetParameter(const std::shared_ptr<Meta> &meta)
114 {
115     return Status::OK;
116 }
117 
GetParameter(std::shared_ptr<Meta> & meta)118 Status SubtitleSink::GetParameter(std::shared_ptr<Meta> &meta)
119 {
120     return Status::OK;
121 }
122 
Prepare()123 Status SubtitleSink::Prepare()
124 {
125     state_ = Pipeline::FilterState::PREPARING;
126     Status ret = PrepareInputBufferQueue();
127     if (ret != Status::OK) {
128         state_ = Pipeline::FilterState::INITIALIZED;
129         return ret;
130     }
131     state_ = Pipeline::FilterState::READY;
132     return ret;
133 }
134 
Start()135 Status SubtitleSink::Start()
136 {
137     isEos_ = false;
138     state_ = Pipeline::FilterState::RUNNING;
139     readThread_ = std::make_unique<std::thread>(&SubtitleSink::RenderLoop, this);
140     pthread_setname_np(readThread_->native_handle(), "SubtitleRenderLoop");
141     return Status::OK;
142 }
143 
Stop()144 Status SubtitleSink::Stop()
145 {
146     updateCond_.notify_all();
147     state_ = Pipeline::FilterState::INITIALIZED;
148     return Status::OK;
149 }
150 
Pause()151 Status SubtitleSink::Pause()
152 {
153     state_ = Pipeline::FilterState::PAUSED;
154     return Status::OK;
155 }
156 
Resume()157 Status SubtitleSink::Resume()
158 {
159     {
160         std::unique_lock<std::mutex> lock(mutex_);
161         isEos_ = false;
162         state_ = Pipeline::FilterState::RUNNING;
163     }
164     updateCond_.notify_all();
165     return Status::OK;
166 }
167 
Flush()168 Status SubtitleSink::Flush()
169 {
170     {
171         std::unique_lock<std::mutex> lock(mutex_);
172         shouldUpdate_ = true;
173         if (subtitleInfoVec_.size() > 0) {
174             inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
175             subtitleInfoVec_.clear();
176         }
177     }
178     isFlush_.store(true);
179     updateCond_.notify_all();
180     return Status::OK;
181 }
182 
Release()183 Status SubtitleSink::Release()
184 {
185     return Status::OK;
186 }
187 
SetIsTransitent(bool isTransitent)188 Status SubtitleSink::SetIsTransitent(bool isTransitent)
189 {
190     isTransitent_ = isTransitent;
191     return Status::OK;
192 }
193 
PrepareInputBufferQueue()194 Status SubtitleSink::PrepareInputBufferQueue()
195 {
196     if (inputBufferQueue_ != nullptr && inputBufferQueue_->GetQueueSize() > 0) {
197         MEDIA_LOG_I("InputBufferQueue already create");
198         return Status::ERROR_INVALID_OPERATION;
199     }
200     int32_t inputBufferNum = 1;
201     int32_t capacity = 1024;
202     MemoryType memoryType;
203 #ifndef MEDIA_OHOS
204     memoryType = MemoryType::VIRTUAL_MEMORY;
205 #else
206     memoryType = MemoryType::SHARED_MEMORY;
207 #endif
208     MEDIA_LOG_I("PrepareInputBufferQueue");
209     if (inputBufferQueue_ == nullptr) {
210         inputBufferQueue_ = AVBufferQueue::Create(inputBufferNum, memoryType, INPUT_BUFFER_QUEUE_NAME);
211     }
212     FALSE_RETURN_V_MSG_E(inputBufferQueue_ != nullptr, Status::ERROR_UNKNOWN, "inputBufferQueue_ is nullptr");
213     inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
214     inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
215 
216     for (int i = 0; i < inputBufferNum; i++) {
217         std::shared_ptr<AVAllocator> avAllocator;
218 #ifndef MEDIA_OHOS
219         MEDIA_LOG_D("CreateVirtualAllocator,i=%{public}d capacity=%{public}d", i, capacity);
220         avAllocator = AVAllocatorFactory::CreateVirtualAllocator();
221 #else
222         MEDIA_LOG_D("CreateSharedAllocator,i=%{public}d capacity=%{public}d", i, capacity);
223         avAllocator = AVAllocatorFactory::CreateSharedAllocator(MemoryFlag::MEMORY_READ_WRITE);
224 #endif
225         std::shared_ptr<AVBuffer> inputBuffer = AVBuffer::CreateAVBuffer(avAllocator, capacity);
226         FALSE_RETURN_V_MSG_E(inputBuffer != nullptr, Status::ERROR_UNKNOWN,
227                              "inputBuffer is nullptr");
228         FALSE_RETURN_V_MSG_E(inputBufferQueueProducer_ != nullptr, Status::ERROR_UNKNOWN,
229                              "inputBufferQueueProducer_ is nullptr");
230         inputBufferQueueProducer_->AttachBuffer(inputBuffer, false);
231         MEDIA_LOG_I("Attach intput buffer. index: %{public}d, bufferId: %{public}" PRIu64,
232             i, inputBuffer->GetUniqueId());
233         inputBufferVector_.push_back(inputBuffer);
234     }
235     return Status::OK;
236 }
237 
DrainOutputBuffer(bool flushed)238 void SubtitleSink::DrainOutputBuffer(bool flushed)
239 {
240     Status ret;
241     FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
242     FALSE_RETURN(!isEos_.load());
243     ret = inputBufferQueueConsumer_->AcquireBuffer(filledOutputBuffer_);
244     if (filledOutputBuffer_->flag_ & BUFFER_FLAG_EOS) {
245         isEos_ = true;
246     }
247     if (ret != Status::OK || filledOutputBuffer_ == nullptr || filledOutputBuffer_->memory_ == nullptr) {
248         return;
249     }
250     std::string subtitleText(reinterpret_cast<const char *>(filledOutputBuffer_->memory_->GetAddr()),
251                              filledOutputBuffer_->memory_->GetSize());
252     SubtitleInfo subtitleInfo{ subtitleText, filledOutputBuffer_->pts_, filledOutputBuffer_->duration_ };
253     {
254         std::unique_lock<std::mutex> lock(mutex_);
255         subtitleInfoVec_.push_back(subtitleInfo);
256     }
257     updateCond_.notify_all();
258 }
259 
RenderLoop()260 void SubtitleSink::RenderLoop()
261 {
262     while (SUBTITME_LOOP_RUNNING) {
263         std::unique_lock<std::mutex> lock(mutex_);
264         updateCond_.wait(lock, [this] {
265             return isThreadExit_.load() ||
266                    (subtitleInfoVec_.size() > 0 && state_ == Pipeline::FilterState::RUNNING);
267         });
268         if (isFlush_) {
269             MEDIA_LOG_I("SubtitleSink RenderLoop flush");
270             isFlush_.store(false);
271             continue;
272         }
273         FALSE_RETURN(!isThreadExit_.load());
274         // wait timeout, seek or stop
275         SubtitleInfo tempSubtitleInfo = subtitleInfoVec_.back();
276         SubtitleInfo subtitleInfo{ tempSubtitleInfo.text_, tempSubtitleInfo.pts_, tempSubtitleInfo.duration_ };
277         int64_t waitTime = static_cast<int64_t>(CalcWaitTime(subtitleInfo));
278         updateCond_.wait_for(lock, std::chrono::microseconds(waitTime),
279                              [this] { return isThreadExit_.load() || shouldUpdate_; });
280         if (isFlush_) {
281             MEDIA_LOG_I("SubtitleSink RenderLoop flush");
282             isFlush_.store(false);
283             continue;
284         }
285         FALSE_RETURN(!isThreadExit_.load());
286         auto actionToDo = ActionToDo(subtitleInfo);
287         if (actionToDo == SubtitleBufferState::DROP) {
288             inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
289             subtitleInfoVec_.clear();
290             continue;
291         } else if (actionToDo == SubtitleBufferState::WAIT) {
292             continue;
293         } else {}
294         NotifyRender(subtitleInfo);
295         inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
296         subtitleInfoVec_.clear();
297     }
298 }
299 
ResetSyncInfo()300 void SubtitleSink::ResetSyncInfo()
301 {
302     auto syncCenter = syncCenter_.lock();
303     if (syncCenter) {
304         syncCenter->Reset();
305     }
306     lastReportedClockTime_ = HST_TIME_NONE;
307 }
308 
CalcWaitTime(SubtitleInfo & subtitleInfo)309 uint64_t SubtitleSink::CalcWaitTime(SubtitleInfo &subtitleInfo)
310 {
311     int64_t curTime;
312     if (shouldUpdate_.load()) {
313         shouldUpdate_ = false;
314     }
315     curTime = GetMediaTime();
316     if (subtitleInfo.pts_ < curTime) {
317         return -1;
318     }
319     return (subtitleInfo.pts_ - curTime) / speed_;
320 }
321 
ActionToDo(SubtitleInfo & subtitleInfo)322 uint32_t SubtitleSink::ActionToDo(SubtitleInfo &subtitleInfo)
323 {
324     auto curTime = GetMediaTime();
325     if (shouldUpdate_ || subtitleInfo.pts_ + subtitleInfo.duration_ < curTime) {
326         return SubtitleBufferState::DROP;
327     }
328     if (subtitleInfo.pts_ > curTime || state_ != Pipeline::FilterState::RUNNING) {
329         return SubtitleBufferState::WAIT;
330     }
331     subtitleInfo.duration_ -= curTime - subtitleInfo.pts_;
332     return SubtitleBufferState::SHOW;
333 }
334 
DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer> & buffer)335 int64_t SubtitleSink::DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer> &buffer)
336 {
337     (void)buffer;
338     return 0;
339 }
340 
NotifyRender(SubtitleInfo & subtitleInfo)341 void SubtitleSink::NotifyRender(SubtitleInfo &subtitleInfo)
342 {
343     Format format;
344     (void)format.PutStringValue(Tag::SUBTITLE_TEXT, subtitleInfo.text_);
345     (void)format.PutIntValue(Tag::SUBTITLE_PTS, Plugins::Us2Ms(subtitleInfo.pts_));
346     (void)format.PutIntValue(Tag::SUBTITLE_DURATION, Plugins::Us2Ms(subtitleInfo.duration_));
347     Event event{ .srcFilter = "SubtitleSink", .type = EventType::EVENT_SUBTITLE_TEXT_UPDATE, .param = format };
348     FALSE_RETURN(playerEventReceiver_ != nullptr);
349     playerEventReceiver_->OnEvent(event);
350 }
351 
SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> & receiver)352 void SubtitleSink::SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver)
353 {
354     FALSE_RETURN(receiver != nullptr);
355     playerEventReceiver_ = receiver;
356 }
357 
SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)358 void SubtitleSink::SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)
359 {
360     syncCenter_ = syncCenter;
361     MediaSynchronousSink::Init();
362 }
363 
SetSpeed(float speed)364 Status SubtitleSink::SetSpeed(float speed)
365 {
366     FALSE_RETURN_V_MSG_W(speed > 0, Status::OK, "Invalid speed %{public}f", speed);
367     {
368         std::unique_lock<std::mutex> lock(mutex_);
369         speed_ = speed;
370         shouldUpdate_ = true;
371     }
372     updateCond_.notify_all();
373     return Status::OK;
374 }
375 
GetMediaTime()376 int64_t SubtitleSink::GetMediaTime()
377 {
378     auto syncCenter = syncCenter_.lock();
379     if (!syncCenter) {
380         return 0;
381     }
382     return syncCenter->GetMediaTimeNow();
383 }
384 }  // namespace MEDIA
385 }  // namespace OHOS