1 /*
2 * Copyright (c) 2023-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "muxer_filter.h"
17 #include <sys/timeb.h>
18 #include <unordered_map>
19 #include "common/log.h"
20 #include "filter/filter_factory.h"
21 #include "muxer/media_muxer.h"
22 #include "avcodec_trace.h"
23 #include "avcodec_sysevent.h"
24
25 namespace {
26 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "MuxerFilter" };
27 static const std::unordered_map<OHOS::Media::Plugins::OutputFormat, std::string> FORMAT_TABLE = {
28 {OHOS::Media::Plugins::OutputFormat::DEFAULT, OHOS::Media::Plugins::MimeType::MEDIA_MP4},
29 {OHOS::Media::Plugins::OutputFormat::MPEG_4, OHOS::Media::Plugins::MimeType::MEDIA_MP4},
30 {OHOS::Media::Plugins::OutputFormat::M4A, OHOS::Media::Plugins::MimeType::MEDIA_M4A},
31 {OHOS::Media::Plugins::OutputFormat::AMR, OHOS::Media::Plugins::MimeType::MEDIA_AMR},
32 {OHOS::Media::Plugins::OutputFormat::MP3, OHOS::Media::Plugins::MimeType::MEDIA_MP3},
33 {OHOS::Media::Plugins::OutputFormat::WAV, OHOS::Media::Plugins::MimeType::MEDIA_WAV},
34 };
35 }
36
37 namespace OHOS {
38 namespace Media {
39 namespace Pipeline {
40 using namespace OHOS::MediaAVCodec;
41 constexpr int64_t WAIT_TIME_OUT_NS = 3000000000;
42 constexpr int64_t US_TO_MS = 1000;
43 constexpr uint32_t BUFFER_IS_EOS = 1;
44 static AutoRegisterFilter<MuxerFilter> g_registerMuxerFilter("builtin.recorder.muxer", FilterType::FILTERTYPE_MUXER,
__anon1cfe904a0202(const std::string& name, const FilterType type) 45 [](const std::string& name, const FilterType type) {
46 return std::make_shared<MuxerFilter>(name, FilterType::FILTERTYPE_MUXER);
47 });
48
49 class MuxerBrokerListener : public IBrokerListener {
50 public:
MuxerBrokerListener(std::shared_ptr<MuxerFilter> muxerFilter,int32_t trackIndex,StreamType streamType,sptr<AVBufferQueueProducer> inputBufferQueue)51 MuxerBrokerListener(std::shared_ptr<MuxerFilter> muxerFilter, int32_t trackIndex,
52 StreamType streamType, sptr<AVBufferQueueProducer> inputBufferQueue)
53 : muxerFilter_(std::move(muxerFilter)), trackIndex_(trackIndex), streamType_(streamType),
54 inputBufferQueue_(inputBufferQueue)
55 {
56 }
57
AsObject()58 sptr<IRemoteObject> AsObject() override
59 {
60 return nullptr;
61 }
62
OnBufferFilled(std::shared_ptr<AVBuffer> & avBuffer)63 void OnBufferFilled(std::shared_ptr<AVBuffer> &avBuffer) override
64 {
65 if (inputBufferQueue_ != nullptr) {
66 if (auto muxerFilter = muxerFilter_.lock()) {
67 muxerFilter->OnBufferFilled(avBuffer, trackIndex_, streamType_, inputBufferQueue_.promote());
68 } else {
69 MEDIA_LOG_I("invalid muxerFilter");
70 }
71 }
72 }
73
74 private:
75 std::weak_ptr<MuxerFilter> muxerFilter_;
76 int32_t trackIndex_;
77 StreamType streamType_;
78 wptr<AVBufferQueueProducer> inputBufferQueue_;
79 };
80
MuxerFilter(std::string name,FilterType type)81 MuxerFilter::MuxerFilter(std::string name, FilterType type): Filter(name, type)
82 {
83 MEDIA_LOG_I("MuxerFilter create");
84 }
85
~MuxerFilter()86 MuxerFilter::~MuxerFilter()
87 {
88 MEDIA_LOG_I("MuxerFilter destroy");
89 }
90
SetOutputParameter(int32_t appUid,int32_t appPid,int32_t fd,int32_t format)91 Status MuxerFilter::SetOutputParameter(int32_t appUid, int32_t appPid, int32_t fd, int32_t format)
92 {
93 MEDIA_LOG_I("SetOutputParameter, appUid:" PUBLIC_LOG_D32 ", appPid:" PUBLIC_LOG_D32 ", format:" PUBLIC_LOG_D32,
94 static_cast<int32_t>(appUid), static_cast<int32_t>(appPid), static_cast<int32_t>(format));
95 mediaMuxer_ = std::make_shared<MediaMuxer>(appUid, appPid);
96 Status ret = mediaMuxer_->Init(fd, (Plugins::OutputFormat)format);
97 outputFormat_ = (Plugins::OutputFormat)format;
98 if (ret != Status::OK) {
99 SetFaultEvent("MuxerFilter::SetOutputParameter, muxerFilter init error", (int32_t)ret);
100 }
101 return ret;
102 }
103
SetTransCoderMode()104 Status MuxerFilter::SetTransCoderMode()
105 {
106 MEDIA_LOG_I("SetTransCoderMode");
107 isTransCoderMode = true;
108 return Status::OK;
109 }
110
GetCurrentPtsMs()111 int64_t MuxerFilter::GetCurrentPtsMs()
112 {
113 if (lastVideoPts_ != 0) {
114 return lastVideoPts_ / US_TO_MS;
115 } else {
116 return lastAudioPts_ / US_TO_MS;
117 }
118 }
119
Init(const std::shared_ptr<EventReceiver> & receiver,const std::shared_ptr<FilterCallback> & callback)120 void MuxerFilter::Init(const std::shared_ptr<EventReceiver> &receiver,
121 const std::shared_ptr<FilterCallback> &callback)
122 {
123 MEDIA_LOG_I("Init");
124 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Init");
125 eventReceiver_ = receiver;
126 filterCallback_ = callback;
127 }
128
DoPrepare()129 Status MuxerFilter::DoPrepare()
130 {
131 MEDIA_LOG_I("Prepare");
132 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Prepare");
133 return Status::OK;
134 }
135
DoStart()136 Status MuxerFilter::DoStart()
137 {
138 MEDIA_LOG_I("Start");
139 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Start");
140 startCount_++;
141 if (startCount_ == preFilterCount_) {
142 startCount_ = 0;
143 Status ret = mediaMuxer_->Start();
144 if (ret != Status::OK) {
145 SetFaultEvent("MuxerFilter::DoStart error", (int32_t)ret);
146 }
147 return ret;
148 } else {
149 return Status::OK;
150 }
151 }
152
DoPause()153 Status MuxerFilter::DoPause()
154 {
155 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Pause");
156 MEDIA_LOG_I("Pause");
157 return Status::OK;
158 }
159
DoResume()160 Status MuxerFilter::DoResume()
161 {
162 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Resume");
163 MEDIA_LOG_I("Resume");
164 return Status::OK;
165 }
166
DoStop()167 Status MuxerFilter::DoStop()
168 {
169 MEDIA_LOG_I("Stop");
170 MediaAVCodec::AVCodecTrace trace("MuxerFilter::Stop");
171 stopCount_++;
172 Status ret = Status::OK;
173 if (stopCount_ == preFilterCount_) {
174 stopCount_ = 0;
175 ret = mediaMuxer_->Stop();
176 if (ret == Status::ERROR_WRONG_STATE) {
177 return Status::OK;
178 }
179 }
180 if (ret != Status::OK) {
181 SetFaultEvent("MuxerFilter::DoStop error", (int32_t)ret);
182 }
183 return ret;
184 }
185
DoFlush()186 Status MuxerFilter::DoFlush()
187 {
188 return Status::OK;
189 }
190
DoRelease()191 Status MuxerFilter::DoRelease()
192 {
193 MEDIA_LOG_I("Release");
194 return Status::OK;
195 }
196
SetParameter(const std::shared_ptr<Meta> & parameter)197 void MuxerFilter::SetParameter(const std::shared_ptr<Meta> ¶meter)
198 {
199 MEDIA_LOG_I("SetParameter");
200 MediaAVCodec::AVCodecTrace trace("MuxerFilter::SetParameter");
201 mediaMuxer_->SetParameter(parameter);
202 }
203
SetUserMeta(const std::shared_ptr<Meta> & userMeta)204 void MuxerFilter::SetUserMeta(const std::shared_ptr<Meta> &userMeta)
205 {
206 MEDIA_LOG_I("SetUserMeta enter");
207 Status ret = mediaMuxer_->SetUserMeta(userMeta);
208 if (ret != Status::OK) {
209 MEDIA_LOG_I("SetUserMeta failed");
210 }
211 }
212
GetParameter(std::shared_ptr<Meta> & parameter)213 void MuxerFilter::GetParameter(std::shared_ptr<Meta> ¶meter)
214 {
215 MEDIA_LOG_I("GetParameter");
216 MediaAVCodec::AVCodecTrace trace("MuxerFilter::GetParameter");
217 }
218
LinkNext(const std::shared_ptr<Filter> & nextFilter,StreamType outType)219 Status MuxerFilter::LinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType)
220 {
221 return Status::OK;
222 }
223
UpdateNext(const std::shared_ptr<Filter> & nextFilter,StreamType outType)224 Status MuxerFilter::UpdateNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType)
225 {
226 MEDIA_LOG_I("UpdateNext");
227 return Status::OK;
228 }
229
UnLinkNext(const std::shared_ptr<Filter> & nextFilter,StreamType outType)230 Status MuxerFilter::UnLinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType)
231 {
232 MEDIA_LOG_I("UnLinkNext");
233 return Status::OK;
234 }
235
GetFilterType()236 FilterType MuxerFilter::GetFilterType()
237 {
238 MEDIA_LOG_I("GetFilterType");
239 return FilterType::FILTERTYPE_MUXER;
240 }
241
OnLinked(StreamType inType,const std::shared_ptr<Meta> & meta,const std::shared_ptr<FilterLinkCallback> & callback)242 Status MuxerFilter::OnLinked(StreamType inType, const std::shared_ptr<Meta> &meta,
243 const std::shared_ptr<FilterLinkCallback> &callback)
244 {
245 MEDIA_LOG_I("OnLinked");
246 MediaAVCodec::AVCodecTrace trace("MuxerFilter::OnLinked");
247 int32_t trackIndex;
248 std::string mimeType;
249 meta->Get<Tag::MIME_TYPE>(mimeType);
250 if (mimeType.find("audio/") == 0) {
251 audioCodecMimeType_ = mimeType;
252 } else if (mimeType.find("video/") == 0) {
253 videoCodecMimeType_ = mimeType;
254 } else if (mimeType.find("meta/") == 0) {
255 metaDataCodecMimeType_ = mimeType;
256 std::string srcMimeType;
257 meta->Get<Tag::TIMED_METADATA_SRC_TRACK_MIME>(srcMimeType);
258 if (trackIndexMap_.find(srcMimeType) != trackIndexMap_.end()) {
259 auto sourceTrackIndex = trackIndexMap_.at(videoCodecMimeType_);
260 meta->Set<Tag::TIMED_METADATA_SRC_TRACK>(sourceTrackIndex);
261 }
262 }
263 auto ret = mediaMuxer_->AddTrack(trackIndex, meta);
264 if (ret != Status::OK) {
265 eventReceiver_->OnEvent({"muxer_filter", EventType::EVENT_ERROR, ret});
266 SetFaultEvent("MuxerFilter::OnLinked error", (int32_t)ret);
267 return ret;
268 }
269 trackIndexMap_.emplace(std::make_pair(mimeType, trackIndex));
270 sptr<AVBufferQueueProducer> inputBufferQueue = mediaMuxer_->GetInputBufferQueue(trackIndex);
271 callback->OnLinkedResult(inputBufferQueue, const_cast<std::shared_ptr<Meta> &>(meta));
272 sptr<IBrokerListener> listener = new MuxerBrokerListener(shared_from_this(), trackIndex,
273 inType, inputBufferQueue);
274 inputBufferQueue->SetBufferFilledListener(listener);
275 preFilterCount_++;
276 bufferPtsMap_.insert(std::pair<int32_t, int64_t>(trackIndex, 0));
277 return Status::OK;
278 }
279
OnUpdated(StreamType inType,const std::shared_ptr<Meta> & meta,const std::shared_ptr<FilterLinkCallback> & callback)280 Status MuxerFilter::OnUpdated(StreamType inType, const std::shared_ptr<Meta> &meta,
281 const std::shared_ptr<FilterLinkCallback> &callback)
282 {
283 MEDIA_LOG_I("OnUpdated");
284 return Status::OK;
285 }
286
287
OnUnLinked(StreamType inType,const std::shared_ptr<FilterLinkCallback> & callback)288 Status MuxerFilter::OnUnLinked(StreamType inType, const std::shared_ptr<FilterLinkCallback> &callback)
289 {
290 MEDIA_LOG_I("OnUnLinked");
291 return Status::OK;
292 }
293
OnBufferFilled(std::shared_ptr<AVBuffer> & inputBuffer,int32_t trackIndex,StreamType streamType,sptr<AVBufferQueueProducer> inputBufferQueue)294 void MuxerFilter::OnBufferFilled(std::shared_ptr<AVBuffer> &inputBuffer, int32_t trackIndex,
295 StreamType streamType, sptr<AVBufferQueueProducer> inputBufferQueue)
296 {
297 MEDIA_LOG_D("OnBufferFilled");
298 MediaAVCodec::AVCodecTrace trace("MuxerFilter::OnBufferFilled");
299 if (!isTransCoderMode) {
300 int64_t currentBufferPts = inputBuffer->pts_;
301 int64_t anotherBufferPts = 0;
302 for (auto mapInterator = bufferPtsMap_.begin(); mapInterator != bufferPtsMap_.end(); mapInterator++) {
303 if (mapInterator->first != trackIndex) {
304 anotherBufferPts = mapInterator->second;
305 }
306 }
307 bufferPtsMap_[trackIndex] = currentBufferPts;
308 if (preFilterCount_ != 1 && std::abs(currentBufferPts - anotherBufferPts) >= WAIT_TIME_OUT_NS) {
309 MEDIA_LOG_I("OnBufferFilled pts time interval is greater than 3 seconds");
310 }
311 MEDIA_LOG_D("OnBufferFilled buffer->pts" PUBLIC_LOG_D64, inputBuffer->pts_);
312 inputBufferQueue->ReturnBuffer(inputBuffer, true);
313 return;
314 }
315 OnTransCoderBufferFilled(inputBuffer, trackIndex, streamType, inputBufferQueue);
316 }
317
OnTransCoderBufferFilled(std::shared_ptr<AVBuffer> & inputBuffer,int32_t trackIndex,StreamType streamType,sptr<AVBufferQueueProducer> inputBufferQueue)318 void MuxerFilter::OnTransCoderBufferFilled(std::shared_ptr<AVBuffer> &inputBuffer, int32_t trackIndex,
319 StreamType streamType, sptr<AVBufferQueueProducer> inputBufferQueue)
320 {
321 MEDIA_LOG_D("OnTransCoderBufferFilled");
322 if ((inputBuffer->flag_ & BUFFER_IS_EOS) == 1) {
323 eosCount_++;
324 if (streamType == StreamType::STREAMTYPE_ENCODED_VIDEO) {
325 MEDIA_LOG_I("video is eos");
326 videoIsEos = true;
327 } else if (streamType == StreamType::STREAMTYPE_ENCODED_AUDIO) {
328 MEDIA_LOG_I("audio is eos");
329 audioIsEos = true;
330 }
331 }
332 if ((eosCount_ == preFilterCount_) || (videoIsEos && audioIsEos)) {
333 eventReceiver_->OnEvent({"muxer_filter", EventType::EVENT_COMPLETE, Status::OK});
334 }
335 if (streamType == StreamType::STREAMTYPE_ENCODED_AUDIO) {
336 lastAudioPts_ = inputBuffer->pts_;
337 if (videoCodecMimeType_.empty()) {
338 inputBufferQueue->ReturnBuffer(inputBuffer, true);
339 } else if (inputBuffer->pts_ <= lastVideoPts_ || videoIsEos) {
340 inputBufferQueue->ReturnBuffer(inputBuffer, true);
341 } else {
342 std::unique_lock<std::mutex> lock(stopMutex_);
343 stopCondition_.wait_for(lock, std::chrono::milliseconds(US_TO_MS));
344 inputBufferQueue->ReturnBuffer(inputBuffer, true);
345 }
346 } else if (streamType == StreamType::STREAMTYPE_ENCODED_VIDEO) {
347 lastVideoPts_ = inputBuffer->pts_;
348 std::unique_lock<std::mutex> lock(stopMutex_);
349 stopCondition_.notify_all();
350 inputBufferQueue->ReturnBuffer(inputBuffer, true);
351 } else {
352 inputBufferQueue->ReturnBuffer(inputBuffer, true);
353 }
354 }
355
SetFaultEvent(const std::string & errMsg,int32_t ret)356 void MuxerFilter::SetFaultEvent(const std::string &errMsg, int32_t ret)
357 {
358 SetFaultEvent(errMsg + ", ret = " + std::to_string(ret));
359 }
360
SetFaultEvent(const std::string & errMsg)361 void MuxerFilter::SetFaultEvent(const std::string &errMsg)
362 {
363 MuxerFaultInfo muxerFaultInfo;
364 muxerFaultInfo.appName = bundleName_;
365 muxerFaultInfo.instanceId = std::to_string(instanceId_);
366 muxerFaultInfo.callerType = "player_framework";
367 muxerFaultInfo.videoCodec = videoCodecMimeType_;
368 muxerFaultInfo.audioCodec = audioCodecMimeType_;
369 muxerFaultInfo.metaCodec = metaDataCodecMimeType_;
370 muxerFaultInfo.containerFormat = GetContainerFormat(outputFormat_);
371 muxerFaultInfo.errMsg = errMsg;
372 FaultMuxerEventWrite(muxerFaultInfo);
373 }
374
GetContainerFormat(Plugins::OutputFormat format)375 const std::string &MuxerFilter::GetContainerFormat(Plugins::OutputFormat format)
376 {
377 static std::string emptyFormat = "";
378 FALSE_RETURN_V_MSG_E(FORMAT_TABLE.find(format) != FORMAT_TABLE.end(), emptyFormat,
379 "The output format %{public}d is not supported!", format);
380 return FORMAT_TABLE.at(format);
381 }
382
SetCallingInfo(int32_t appUid,int32_t appPid,const std::string & bundleName,uint64_t instanceId)383 void MuxerFilter::SetCallingInfo(int32_t appUid, int32_t appPid,
384 const std::string &bundleName, uint64_t instanceId)
385 {
386 appUid_ = appUid;
387 appPid_ = appPid;
388 bundleName_ = bundleName;
389 instanceId_ = instanceId;
390 }
391 } // namespace Pipeline
392 } // namespace MEDIA
393 } // namespace OHOS
394