1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaAppender"
19 
20 #include <media/stagefright/MediaAppender.h>
21 #include <media/stagefright/MediaCodec.h>
22 #include <media/stagefright/foundation/ABuffer.h>
23 #include <utils/Log.h>
24 // TODO : check if this works for NDK apps without JVM
25 // #include <media/ndk/NdkJavaVMHelperPriv.h>
26 
27 namespace android {
28 
29 struct MediaAppender::sampleDataInfo {
30     size_t size;
31     int64_t time;
32     size_t exTrackIndex;
33     sp<MetaData> meta;
34 };
35 
create(int fd,AppendMode mode)36 sp<MediaAppender> MediaAppender::create(int fd, AppendMode mode) {
37     if (fd < 0) {
38         ALOGE("invalid file descriptor");
39         return nullptr;
40     }
41     if (!(mode >= APPEND_MODE_FIRST && mode <= APPEND_MODE_LAST)) {
42         ALOGE("invalid mode %d", mode);
43         return nullptr;
44     }
45     sp<MediaAppender> ma = new (std::nothrow) MediaAppender(fd, mode);
46     if (ma->init() != OK) {
47         return nullptr;
48     }
49     return ma;
50 }
51 
52 // TODO: inject mediamuxer and mediaextractor objects.
53 // TODO: @format is not required as an input if we can sniff the file and find the format of
54 //       the existing content.
55 // TODO: Code it to the interface(MediaAppender), and have a separate MediaAppender NDK
MediaAppender(int fd,AppendMode mode)56 MediaAppender::MediaAppender(int fd, AppendMode mode)
57     : mFd(fd),
58       mMode(mode),
59       // TODO : check if this works for NDK apps without JVM
60       // mExtractor(new NuMediaExtractor(NdkJavaVMHelper::getJNIEnv() != nullptr
61       //           ? NuMediaExtractor::EntryPoint::NDK_WITH_JVM
62       //           : NuMediaExtractor::EntryPoint::NDK_NO_JVM)),
63       mExtractor(new (std::nothrow) NuMediaExtractor(NuMediaExtractor::EntryPoint::NDK_WITH_JVM)),
64       mTrackCount(0),
65       mState(UNINITIALIZED) {
66           ALOGV("MediaAppender::MediaAppender mode:%d", mode);
67       }
68 
init()69 status_t MediaAppender::init() {
70     std::scoped_lock lock(mMutex);
71     ALOGV("MediaAppender::init");
72     status_t status = mExtractor->setDataSource(mFd, 0, lseek(mFd, 0, SEEK_END));
73     if (status != OK) {
74         ALOGE("extractor_setDataSource failed, status :%d", status);
75         return status;
76     }
77 
78     sp<AMessage> fileFormat;
79     status = mExtractor->getFileFormat(&fileFormat);
80     if (status != OK) {
81         ALOGE("extractor_getFileFormat failed, status :%d", status);
82         return status;
83     }
84 
85     AString fileMime;
86     fileFormat->findString("mime", &fileMime);
87     // only compare the end of the file MIME type to allow for vendor customized mime type
88     if (fileMime.endsWith("mp4")){
89         mFormat = MediaMuxer::OUTPUT_FORMAT_MPEG_4;
90     } else {
91         ALOGE("Unsupported file format, extractor name:%s, fileformat %s",
92               mExtractor->getName(), fileMime.c_str());
93         return ERROR_UNSUPPORTED;
94     }
95 
96     mTrackCount = mExtractor->countTracks();
97     ALOGV("mTrackCount:%zu", mTrackCount);
98     if (mTrackCount == 0) {
99         ALOGE("no tracks are present");
100         return ERROR_MALFORMED;
101     }
102     size_t exTrackIndex = 0;
103     ssize_t audioTrackIndex = -1, videoTrackIndex = -1;
104     bool audioSyncSampleTimeSet = false;
105 
106     while (exTrackIndex < mTrackCount) {
107         sp<AMessage> fmt;
108         status = mExtractor->getTrackFormat(exTrackIndex, &fmt, 0);
109         if (status != OK) {
110             ALOGE("getTrackFormat failed for trackIndex:%zu, status:%d", exTrackIndex, status);
111             return status;
112         }
113         AString mime;
114         if (fmt->findString("mime", &mime)) {
115             if (!strncasecmp(mime.c_str(), "video/", 6)) {
116                 ALOGV("VideoTrack");
117                 if (videoTrackIndex != -1) {
118                     ALOGE("Not more than one video track is supported");
119                     return ERROR_UNSUPPORTED;
120                 }
121                 videoTrackIndex = exTrackIndex;
122             } else if (!strncasecmp(mime.c_str(), "audio/", 6)) {
123                 ALOGV("AudioTrack");
124                 if (audioTrackIndex != -1) {
125                     ALOGE("Not more than one audio track is supported");
126                 }
127                 audioTrackIndex = exTrackIndex;
128             } else {
129                 ALOGV("Neither Video nor Audio track");
130             }
131         }
132         mFmtIndexMap.emplace(exTrackIndex, fmt);
133         mSampleCountVect.emplace_back(0);
134         mMaxTimestampVect.emplace_back(0);
135         mLastSyncSampleTimeVect.emplace_back(0);
136         status = mExtractor->selectTrack(exTrackIndex);
137         if (status != OK) {
138             ALOGE("selectTrack failed for trackIndex:%zu, status:%d", exTrackIndex, status);
139             return status;
140         }
141         ++exTrackIndex;
142     }
143 
144     ALOGV("AudioTrackIndex:%zu, VideoTrackIndex:%zu", audioTrackIndex, videoTrackIndex);
145 
146     do {
147         sampleDataInfo tmpSDI;
148         // TODO: read info into members of the struct sampleDataInfo directly
149         size_t sampleSize;
150         status = mExtractor->getSampleSize(&sampleSize);
151         if (status != OK) {
152             ALOGE("getSampleSize failed, status:%d", status);
153             return status;
154         }
155         mSampleSizeVect.emplace_back(sampleSize);
156         tmpSDI.size = sampleSize;
157         int64_t sampleTime = 0;
158         status = mExtractor->getSampleTime(&sampleTime);
159         if (status != OK) {
160             ALOGE("getSampleTime failed, status:%d", status);
161             return status;
162         }
163         mSampleTimeVect.emplace_back(sampleTime);
164         tmpSDI.time = sampleTime;
165         status = mExtractor->getSampleTrackIndex(&exTrackIndex);
166         if (status != OK) {
167             ALOGE("getSampleTrackIndex failed, status:%d", status);
168             return status;
169         }
170         mSampleIndexVect.emplace_back(exTrackIndex);
171         tmpSDI.exTrackIndex = exTrackIndex;
172         ++mSampleCountVect[exTrackIndex];
173         mMaxTimestampVect[exTrackIndex] = std::max(mMaxTimestampVect[exTrackIndex], sampleTime);
174         sp<MetaData> sampleMeta;
175         status = mExtractor->getSampleMeta(&sampleMeta);
176         if (status != OK) {
177             ALOGE("getSampleMeta failed, status:%d", status);
178             return status;
179         }
180         mSampleMetaVect.emplace_back(sampleMeta);
181         int32_t val = 0;
182         if (sampleMeta->findInt32(kKeyIsSyncFrame, &val) && val != 0) {
183             mLastSyncSampleTimeVect[exTrackIndex] = sampleTime;
184         }
185         tmpSDI.meta = sampleMeta;
186         mSDI.emplace_back(tmpSDI);
187     } while (mExtractor->advance() == OK);
188 
189     mExtractor.clear();
190 
191     std::sort(mSDI.begin(), mSDI.end(), [](sampleDataInfo& a, sampleDataInfo& b) {
192         int64_t aOffset, bOffset;
193         a.meta->findInt64(kKeySampleFileOffset, &aOffset);
194         b.meta->findInt64(kKeySampleFileOffset, &bOffset);
195         return aOffset < bOffset;
196     });
197     for (int64_t syncSampleTime : mLastSyncSampleTimeVect) {
198         ALOGV("before ignoring frames, mLastSyncSampleTimeVect:%lld", (long long)syncSampleTime);
199     }
200     ALOGV("mMode:%u", mMode);
201     if (mMode == APPEND_MODE_IGNORE_LAST_VIDEO_GOP && videoTrackIndex != -1 ) {
202         ALOGV("Video track is present");
203         bool lastVideoIframe = false;
204         size_t lastVideoIframeOffset = 0;
205         int64_t lastVideoSampleTime = -1;
206         for (auto rItr = mSDI.rbegin(); rItr != mSDI.rend(); ++rItr) {
207             if (rItr->exTrackIndex != videoTrackIndex) {
208                 continue;
209             }
210             if (lastVideoSampleTime == -1) {
211                 lastVideoSampleTime = rItr->time;
212             }
213             int64_t offset = 0;
214             if (!rItr->meta->findInt64(kKeySampleFileOffset, &offset) || offset == 0) {
215                 ALOGE("Missing offset");
216                 return ERROR_MALFORMED;
217             }
218             ALOGV("offset:%lld", (long long)offset);
219             int32_t val = 0;
220             if (rItr->meta->findInt32(kKeyIsSyncFrame, &val) && val != 0) {
221                 ALOGV("sampleTime:%lld", (long long)rItr->time);
222                 ALOGV("lastVideoSampleTime:%lld", (long long)lastVideoSampleTime);
223                 if (lastVideoIframe == false && (lastVideoSampleTime - rItr->time) >
224                                 1000000/* Track interleaving duration in MPEG4Writer*/) {
225                     ALOGV("lastVideoIframe got chosen");
226                     lastVideoIframe = true;
227                     mLastSyncSampleTimeVect[videoTrackIndex] = rItr->time;
228                     lastVideoIframeOffset = offset;
229                     ALOGV("lastVideoIframeOffset:%lld", (long long)offset);
230                     break;
231                 }
232             }
233         }
234         if (lastVideoIframe == false) {
235             ALOGV("Need to rewrite all samples");
236             mLastSyncSampleTimeVect[videoTrackIndex] = 0;
237             lastVideoIframeOffset = 0;
238         }
239         unsigned int framesIgnoredCount = 0;
240         for (auto itr = mSDI.begin(); itr != mSDI.end();) {
241             int64_t offset = 0;
242             ALOGV("trackIndex:%zu, %" PRId64 "", itr->exTrackIndex, itr->time);
243             if (itr->meta->findInt64(kKeySampleFileOffset, &offset) &&
244                                         offset >= lastVideoIframeOffset) {
245                 ALOGV("offset:%lld", (long long)offset);
246                 if (!audioSyncSampleTimeSet && audioTrackIndex != -1 &&
247                                             audioTrackIndex == itr->exTrackIndex) {
248                     mLastSyncSampleTimeVect[audioTrackIndex] = itr->time;
249                     audioSyncSampleTimeSet = true;
250                 }
251                 itr = mSDI.erase(itr);
252                 ++framesIgnoredCount;
253             } else {
254                 ++itr;
255             }
256         }
257         ALOGV("framesIgnoredCount:%u", framesIgnoredCount);
258     }
259 
260     if (mMode == APPEND_MODE_IGNORE_LAST_VIDEO_GOP && videoTrackIndex == -1 &&
261                             audioTrackIndex != -1) {
262         ALOGV("Only AudioTrack is present");
263         for (auto rItr = mSDI.rbegin(); rItr != mSDI.rend();  ++rItr) {
264             int32_t val = 0;
265             if (rItr->meta->findInt32(kKeyIsSyncFrame, &val) && val != 0) {
266                     mLastSyncSampleTimeVect[audioTrackIndex] = rItr->time;
267                     break;
268             }
269         }
270         unsigned int framesIgnoredCount = 0;
271         for (auto itr = mSDI.begin(); itr != mSDI.end();) {
272             if (itr->time >= mLastSyncSampleTimeVect[audioTrackIndex]) {
273                 itr = mSDI.erase(itr);
274                 ++framesIgnoredCount;
275             } else {
276                 ++itr;
277             }
278         }
279         ALOGV("framesIgnoredCount :%u", framesIgnoredCount);
280     }
281 
282     for (size_t i = 0; i < mLastSyncSampleTimeVect.size(); ++i) {
283         ALOGV("mLastSyncSampleTimeVect[%zu]:%lld", i, (long long)mLastSyncSampleTimeVect[i]);
284         mFmtIndexMap[i]->setInt64(
285                 "sample-time-before-append" /*AMEDIAFORMAT_KEY_SAMPLE_TIME_BEFORE_APPEND*/,
286                 mLastSyncSampleTimeVect[i]);
287     }
288     for (size_t i = 0; i < mMaxTimestampVect.size(); ++i) {
289         ALOGV("mMaxTimestamp[%zu]:%lld", i, (long long)mMaxTimestampVect[i]);
290     }
291     for (size_t i = 0; i < mSampleCountVect.size(); ++i) {
292         ALOGV("SampleCountVect[%zu]:%zu", i, mSampleCountVect[i]);
293     }
294     mState = INITIALIZED;
295     return OK;
296 }
297 
~MediaAppender()298 MediaAppender::~MediaAppender() {
299     ALOGV("MediaAppender::~MediaAppender");
300     mMuxer.clear();
301     mExtractor.clear();
302 }
303 
start()304 status_t MediaAppender::start() {
305     std::scoped_lock lock(mMutex);
306     ALOGV("MediaAppender::start");
307     if (mState != INITIALIZED) {
308         ALOGE("MediaAppender::start() is called in invalid state %d", mState);
309         return INVALID_OPERATION;
310     }
311     mMuxer = new (std::nothrow) MediaMuxer(mFd, mFormat);
312     for (const auto& n : mFmtIndexMap) {
313         ssize_t muxIndex = mMuxer->addTrack(n.second);
314         if (muxIndex < 0) {
315             ALOGE("addTrack failed");
316             return UNKNOWN_ERROR;
317         }
318         mTrackIndexMap.emplace(n.first, muxIndex);
319     }
320     ALOGV("trackIndexmap size:%zu", mTrackIndexMap.size());
321 
322     status_t status = mMuxer->start();
323     if (status != OK) {
324         ALOGE("muxer start failed:%d", status);
325         return status;
326     }
327 
328     ALOGV("Sorting samples based on their offsets");
329     for (int i = 0; i < mSDI.size(); ++i) {
330         ALOGV("i:%d", i + 1);
331         /* TODO : Allocate a single allocation of the max size, and reuse it across ABuffers if
332          * using new ABuffer(void *, size_t).
333          */
334         sp<ABuffer> data = new (std::nothrow) ABuffer(mSDI[i].size);
335         if (data == nullptr) {
336             ALOGE("memory allocation failed");
337             return NO_MEMORY;
338         }
339         data->setRange(0, mSDI[i].size);
340         int32_t val = 0;
341         int sampleFlags = 0;
342         if (mSDI[i].meta->findInt32(kKeyIsSyncFrame, &val) && val != 0) {
343             sampleFlags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
344         }
345 
346         int64_t val64;
347         if (mSDI[i].meta->findInt64(kKeySampleFileOffset, &val64)) {
348             ALOGV("SampleFileOffset Found :%zu:%lld:%lld", mSDI[i].exTrackIndex,
349                   (long long)mSampleCountVect[mSDI[i].exTrackIndex], (long long)val64);
350             sp<AMessage> bufMeta = data->meta();
351             bufMeta->setInt64("sample-file-offset" /*AMEDIAFORMAT_KEY_SAMPLE_TIME_BEFORE_APPEND*/,
352                               val64);
353         }
354         if (mSDI[i].meta->findInt64(kKeyLastSampleIndexInChunk, &val64)) {
355             ALOGV("kKeyLastSampleIndexInChunk Found %lld:%lld",
356                   (long long)mSampleCountVect[mSDI[i].exTrackIndex], (long long)val64);
357             sp<AMessage> bufMeta = data->meta();
358             bufMeta->setInt64(
359                     "last-sample-index-in-chunk" /*AMEDIAFORMAT_KEY_LAST_SAMPLE_INDEX_IN_CHUNK*/,
360                     val64);
361         }
362         status = mMuxer->writeSampleData(data, mTrackIndexMap[mSDI[i].exTrackIndex], mSDI[i].time,
363                                          sampleFlags);
364         if (status != OK) {
365             ALOGE("muxer writeSampleData failed:%d", status);
366             return status;
367         }
368     }
369     mState = STARTED;
370     return OK;
371 }
372 
stop()373 status_t MediaAppender::stop() {
374     std::scoped_lock lock(mMutex);
375     ALOGV("MediaAppender::stop");
376     if (mState == STARTED) {
377         status_t status = mMuxer->stop();
378         if (status != OK) {
379             mState = ERROR;
380         } else {
381             mState = STOPPED;
382         }
383         return status;
384     } else {
385         ALOGE("stop() is called in invalid state %d", mState);
386         return INVALID_OPERATION;
387     }
388 }
389 
getTrackCount()390 ssize_t MediaAppender::getTrackCount() {
391     std::scoped_lock lock(mMutex);
392     ALOGV("MediaAppender::getTrackCount");
393     if (mState != INITIALIZED && mState != STARTED) {
394         ALOGE("getTrackCount() is called in invalid state %d", mState);
395         return -1;
396     }
397     return mTrackCount;
398 }
399 
getTrackFormat(size_t idx)400 sp<AMessage> MediaAppender::getTrackFormat(size_t idx) {
401     std::scoped_lock lock(mMutex);
402     ALOGV("MediaAppender::getTrackFormat");
403     if (mState != INITIALIZED && mState != STARTED) {
404         ALOGE("getTrackFormat() is called in invalid state %d", mState);
405         return nullptr;
406     }
407     if (idx < 0 || idx >= mTrackCount) {
408         ALOGE("getTrackFormat() idx is out of range");
409         return nullptr;
410     }
411     return mFmtIndexMap[idx];
412 }
413 
writeSampleData(const sp<ABuffer> & buffer,size_t trackIndex,int64_t timeUs,uint32_t flags)414 status_t MediaAppender::writeSampleData(const sp<ABuffer>& buffer, size_t trackIndex,
415                                         int64_t timeUs, uint32_t flags) {
416     std::scoped_lock lock(mMutex);
417     ALOGV("writeSampleData:trackIndex:%zu, time:%" PRId64 "", trackIndex, timeUs);
418     return mMuxer->writeSampleData(buffer, trackIndex, timeUs, flags);
419 }
420 
421 status_t MediaAppender::setOrientationHint([[maybe_unused]] int degrees) {
422     ALOGE("setOrientationHint not supported. Has to be called prior to start on initial muxer");
423     return ERROR_UNSUPPORTED;
424 };
425 
426 status_t MediaAppender::setLocation([[maybe_unused]] int latit, [[maybe_unused]] int longit) {
427     ALOGE("setLocation not supported. Has to be called prior to start on initial muxer");
428     return ERROR_UNSUPPORTED;
429 }
430 
431 ssize_t MediaAppender::addTrack([[maybe_unused]] const sp<AMessage> &format) {
432     ALOGE("addTrack not supported");
433     return ERROR_UNSUPPORTED;
434 }
435 
436 }  // namespace android
437