1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "avmetadata_collector.h"
17 
18 #include <string>
19 
20 #include "avmetadatahelper.h"
21 #include "buffer/avsharedmemorybase.h"
22 #include "media_log.h"
23 #include "meta/video_types.h"
24 #include "meta/any.h"
25 #include "time_format_utils.h"
26 
27 namespace {
28 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_METADATA, "AVMetaDataCollector" };
29 }  // namespace
30 
31 namespace OHOS {
32 namespace Media {
33 static constexpr int PICTURE_MAX_SIZE = 1024 * 1024;
34 static constexpr int SECOND_DEVIDE_MS = 1000;
35 
36 static const std::unordered_map<Plugins::FileType, std::string> fileTypeMap = {
37     { Plugins::FileType::UNKNOW, "uknown" },
38     { Plugins::FileType::MP4, "mp4" },
39     { Plugins::FileType::MPEGTS, "mpeg" },
40     { Plugins::FileType::MKV, "mkv" },
41     { Plugins::FileType::AMR, "amr" },
42     { Plugins::FileType::AAC, "aac-adts" },
43     { Plugins::FileType::MP3, "mpeg" },
44     { Plugins::FileType::FLAC, "flac" },
45     { Plugins::FileType::OGG, "ogg" },
46     { Plugins::FileType::M4A, "mp4" },
47     { Plugins::FileType::WAV, "wav" }
48 };
49 
50 static const std::unordered_map<int32_t, std::string> AVMETA_KEY_TO_X_MAP = {
51     { AV_KEY_ALBUM, Tag::MEDIA_ALBUM },
52     { AV_KEY_ALBUM_ARTIST, Tag::MEDIA_ALBUM_ARTIST },
53     { AV_KEY_ARTIST, Tag::MEDIA_ARTIST },
54     { AV_KEY_AUTHOR, Tag::MEDIA_AUTHOR },
55     { AV_KEY_COMPOSER, Tag::MEDIA_COMPOSER },
56     { AV_KEY_DATE_TIME, Tag::MEDIA_DATE },
57     { AV_KEY_DATE_TIME_FORMAT, Tag::MEDIA_CREATION_TIME },
58     { AV_KEY_DURATION, Tag::MEDIA_DURATION },
59     { AV_KEY_GENRE, Tag::MEDIA_GENRE },
60     { AV_KEY_HAS_AUDIO, Tag::MEDIA_HAS_AUDIO },
61     { AV_KEY_HAS_VIDEO, Tag::MEDIA_HAS_VIDEO },
62     { AV_KEY_MIME_TYPE, Tag::MIME_TYPE },
63     { AV_KEY_NUM_TRACKS, Tag::MEDIA_TRACK_COUNT },
64     { AV_KEY_SAMPLE_RATE, Tag::AUDIO_SAMPLE_RATE },
65     { AV_KEY_TITLE, Tag::MEDIA_TITLE },
66     { AV_KEY_VIDEO_HEIGHT, Tag::VIDEO_HEIGHT },
67     { AV_KEY_VIDEO_WIDTH, Tag::VIDEO_WIDTH },
68     { AV_KEY_VIDEO_ORIENTATION, Tag::VIDEO_ROTATION },
69     { AV_KEY_VIDEO_IS_HDR_VIVID, Tag::VIDEO_IS_HDR_VIVID },
70     { AV_KEY_LOCATION_LONGITUDE, Tag::MEDIA_LONGITUDE},
71     { AV_KEY_LOCATION_LATITUDE, Tag::MEDIA_LATITUDE},
72     { AV_KEY_CUSTOMINFO, "customInfo"},
73 };
74 
AVMetaDataCollector(std::shared_ptr<MediaDemuxer> & mediaDemuxer)75 AVMetaDataCollector::AVMetaDataCollector(std::shared_ptr<MediaDemuxer> &mediaDemuxer) : mediaDemuxer_(mediaDemuxer)
76 {
77     MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
78 }
79 
~AVMetaDataCollector()80 AVMetaDataCollector::~AVMetaDataCollector()
81 {
82     MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
83 }
84 
ExtractMetadata()85 std::unordered_map<int32_t, std::string> AVMetaDataCollector::ExtractMetadata()
86 {
87     if (collectedMeta_.size() != 0) {
88         return collectedMeta_;
89     }
90     const std::shared_ptr<Meta> globalInfo = mediaDemuxer_->GetGlobalMetaInfo();
91     const std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
92     collectedMeta_ = GetMetadata(globalInfo, trackInfos);
93     return collectedMeta_;
94 }
95 
GetVideoTrackId(uint32_t & trackId)96 Status AVMetaDataCollector::GetVideoTrackId(uint32_t &trackId)
97 {
98     if (hasVideo_) {
99         trackId = videoTrackId_;
100         return Status::OK;
101     }
102     const std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
103     size_t trackCount = trackInfos.size();
104     CHECK_AND_RETURN_RET_LOG(trackCount > 0, Status::ERROR_INVALID_DATA, "GetTargetTrackInfo trackCount is invalid");
105     for (size_t index = 0; index < trackCount; index++) {
106         std::string trackMime = "";
107         if (!(trackInfos[index]->GetData(Tag::MIME_TYPE, trackMime))) {
108             continue;
109         }
110         if (trackMime.find("video/") == 0) {
111             videoTrackId_ = index;
112             trackId = index;
113             hasVideo_ = true;
114             return Status::OK;
115         }
116     }
117     return Status::ERROR_INVALID_DATA;
118 }
119 
GetAVMetadata()120 std::shared_ptr<Meta> AVMetaDataCollector::GetAVMetadata()
121 {
122     if (collectedAVMetaData_ != nullptr) {
123         return collectedAVMetaData_;
124     }
125     collectedAVMetaData_ = std::make_shared<Meta>();
126     ExtractMetadata();
127     CHECK_AND_RETURN_RET_LOG(collectedMeta_.size() != 0, nullptr, "globalInfo or trackInfos are invalid.");
128     for (const auto &[avKey, value] : collectedMeta_) {
129         if (avKey == AV_KEY_LOCATION_LATITUDE || avKey == AV_KEY_LOCATION_LONGITUDE) {
130             continue;
131         }
132         if (avKey == AV_KEY_VIDEO_IS_HDR_VIVID) {
133             int32_t hdr;
134             if (value == "yes") {
135                 hdr = static_cast<int32_t>(HdrType::AV_HDR_TYPE_VIVID);
136             } else {
137                 hdr = static_cast<int32_t>(HdrType::AV_HDR_TYPE_NONE);
138             }
139             collectedAVMetaData_->SetData("hdrType", hdr);
140             continue;
141         }
142         auto iter = g_MetadataCodeMap.find(avKey);
143         if (iter != g_MetadataCodeMap.end()) {
144             collectedAVMetaData_->SetData(iter->second, value);
145         }
146     }
147 
148     customInfo_ = mediaDemuxer_->GetUserMeta();
149     if (customInfo_ == nullptr) {
150         MEDIA_LOGW("No valid user data");
151     } else {
152         if (AVMETA_KEY_TO_X_MAP.find(AV_KEY_CUSTOMINFO) != AVMETA_KEY_TO_X_MAP.end()) {
153             collectedAVMetaData_->SetData(AVMETA_KEY_TO_X_MAP.find(AV_KEY_CUSTOMINFO)->second, customInfo_);
154         }
155     }
156     return collectedAVMetaData_;
157 }
158 
ExtractMetadata(int32_t key)159 std::string AVMetaDataCollector::ExtractMetadata(int32_t key)
160 {
161     auto metadata = GetAVMetadata();
162     CHECK_AND_RETURN_RET_LOG(collectedMeta_.size() != 0, "", "Failed to call ExtractMetadata");
163 
164     auto it = collectedMeta_.find(key);
165     if (it == collectedMeta_.end() || it->second.empty()) {
166         MEDIA_LOGE("The specified metadata %{public}d cannot be obtained from the specified stream.", key);
167         return "";
168     }
169     return collectedMeta_[key];
170 }
171 
GetMetadata(const std::shared_ptr<Meta> & globalInfo,const std::vector<std::shared_ptr<Meta>> & trackInfos)172 std::unordered_map<int32_t, std::string> AVMetaDataCollector::GetMetadata(
173     const std::shared_ptr<Meta> &globalInfo, const std::vector<std::shared_ptr<Meta>> &trackInfos)
174 {
175     CHECK_AND_RETURN_RET_LOG(
176         globalInfo != nullptr && trackInfos.size() != 0, {}, "globalInfo or trackInfos are invalid.");
177 
178     Metadata metadata;
179     ConvertToAVMeta(globalInfo, metadata);
180 
181     int32_t imageTrackCount = 0;
182     size_t trackCount = trackInfos.size();
183     for (size_t index = 0; index < trackCount; index++) {
184         std::shared_ptr<Meta> meta = trackInfos[index];
185         CHECK_AND_RETURN_RET_LOG(meta != nullptr, metadata.tbl_, "meta is invalid, index: %zu", index);
186 
187         // skip the image track
188         std::string mime;
189         meta->Get<Tag::MIME_TYPE>(mime);
190         int32_t imageTypeLength = 5;
191         if (mime.substr(0, imageTypeLength).compare("image") == 0) {
192             MEDIA_LOGI("0x%{public}06" PRIXPTR " skip image track", FAKE_POINTER(this));
193             ++imageTrackCount;
194             continue;
195         }
196 
197         Plugins::MediaType mediaType;
198         CHECK_AND_CONTINUE(meta->GetData(Tag::MEDIA_TYPE, mediaType));
199         ConvertToAVMeta(meta, metadata);
200     }
201     FormatAVMeta(metadata, imageTrackCount, globalInfo);
202     auto it = metadata.tbl_.begin();
203     while (it != metadata.tbl_.end()) {
204         MEDIA_LOGD("metadata tbl, key: %{public}d, keyName: %{public}s, val: %{public}s", it->first,
205             AVMETA_KEY_TO_X_MAP.find(it->first)->second.c_str(), it->second.c_str());
206         it++;
207     }
208     return metadata.tbl_;
209 }
210 
GetArtPicture()211 std::shared_ptr<AVSharedMemory> AVMetaDataCollector::GetArtPicture()
212 {
213     MEDIA_LOGI("0x%{public}06" PRIXPTR " GetArtPicture In", FAKE_POINTER(this));
214 
215     if (collectedArtPicture_ != nullptr) {
216         return collectedArtPicture_;
217     }
218     const std::vector<std::shared_ptr<Meta>> trackInfos = mediaDemuxer_->GetStreamMetaInfo();
219     size_t trackCount = trackInfos.size();
220     for (size_t index = 0; index < trackCount; index++) {
221         std::shared_ptr<Meta> meta = trackInfos[index];
222         if (meta == nullptr) {
223             MEDIA_LOGW("meta is invalid, index: %zu", index);
224             continue;
225         }
226 
227         std::vector<uint8_t> coverAddr;
228         auto mapIt = meta->Find(Tag::MEDIA_COVER);
229         if (mapIt == meta->end()) {
230             continue;
231         }
232         if (Any::IsSameTypeWith<std::vector<uint8_t>>(mapIt->second)) {
233             coverAddr = AnyCast<std::vector<uint8_t>>(mapIt->second);
234         }
235         CHECK_AND_RETURN_RET_LOG(!(coverAddr.size() == 0 || static_cast<int>(coverAddr.size()) > PICTURE_MAX_SIZE),
236             nullptr, "InvalidArtPictureSize %d", coverAddr.size());
237         uint8_t *addr = coverAddr.data();
238         size_t size = coverAddr.size();
239         auto artPicMem =
240             AVSharedMemoryBase::CreateFromLocal(static_cast<int32_t>(size), AVSharedMemory::FLAGS_READ_ONLY, "artpic");
241         if (artPicMem == nullptr) {
242             MEDIA_LOGE("artPicMem is nullptr");
243             return nullptr;
244         }
245         errno_t rc = memcpy_s(artPicMem->GetBase(), static_cast<size_t>(artPicMem->GetSize()), addr, size);
246         if (rc != EOK) {
247             MEDIA_LOGE("memcpy_s failed, trackCount no %{public}d", index);
248             return nullptr;
249         }
250         collectedArtPicture_ = artPicMem;
251         return collectedArtPicture_;
252     }
253     MEDIA_LOGE("GetArtPicture Failed");
254     return nullptr;
255 }
256 
GetTimeByFrameIndex(uint32_t index,uint64_t & timeUs)257 int32_t AVMetaDataCollector::GetTimeByFrameIndex(uint32_t index, uint64_t &timeUs)
258 {
259     uint32_t trackId = 0;
260     CHECK_AND_RETURN_RET_LOG(GetVideoTrackId(trackId) == Status::OK, MSERR_UNSUPPORT_FILE, "No video track!");
261     CHECK_AND_RETURN_RET_LOG(mediaDemuxer_->GetRelativePresentationTimeUsByIndex(trackId, index, timeUs) == Status::OK,
262         MSERR_UNSUPPORT_FILE, "Get time by frame failed");
263     return MSERR_OK;
264 }
265 
GetFrameIndexByTime(uint64_t timeUs,uint32_t & index)266 int32_t AVMetaDataCollector::GetFrameIndexByTime(uint64_t timeUs, uint32_t &index)
267 {
268     uint32_t trackId = 0;
269     CHECK_AND_RETURN_RET_LOG(GetVideoTrackId(trackId) == Status::OK, MSERR_UNSUPPORT_FILE, "No video track!");
270     CHECK_AND_RETURN_RET_LOG(mediaDemuxer_->GetIndexByRelativePresentationTimeUs(trackId, timeUs, index) == Status::OK,
271         MSERR_UNSUPPORT_FILE, "Get frame by time failed");
272     return MSERR_OK;
273 }
274 
ConvertToAVMeta(const std::shared_ptr<Meta> & innerMeta,Metadata & avmeta) const275 void AVMetaDataCollector::ConvertToAVMeta(const std::shared_ptr<Meta> &innerMeta, Metadata &avmeta) const
276 {
277     for (const auto &[avKey, innerKey] : AVMETA_KEY_TO_X_MAP) {
278         if (innerKey.compare("") == 0) {
279             std::string strVal;
280             if (innerMeta->GetData(innerKey, strVal) && !strVal.empty()) {
281                 avmeta.SetMeta(avKey, TimeFormatUtils::ConvertTimestampToDatetime(strVal));
282             }
283         }
284         if (innerKey.compare("customInfo") == 0) {
285             continue;
286         }
287         if (!SetStringByValueType(innerMeta, avmeta, avKey, innerKey)) {
288             break;
289         }
290         SetEmptyStringIfNoData(avmeta, avKey);
291     }
292 }
293 
FormatAVMeta(Metadata & avmeta,int32_t imageTrackCount,const std::shared_ptr<Meta> & globalInfo)294 void AVMetaDataCollector::FormatAVMeta(
295     Metadata &avmeta, int32_t imageTrackCount, const std::shared_ptr<Meta> &globalInfo)
296 {
297     std::string str = avmeta.GetMeta(AV_KEY_NUM_TRACKS);
298     if (!str.empty()) {
299         avmeta.SetMeta(AV_KEY_NUM_TRACKS, std::to_string(std::stoi(str) - imageTrackCount));
300     }
301     FormatMimeType(avmeta, globalInfo);
302     FormatDateTime(avmeta, globalInfo);
303 }
304 
FormatMimeType(Metadata & avmeta,const std::shared_ptr<Meta> & globalInfo)305 void AVMetaDataCollector::FormatMimeType(Metadata &avmeta, const std::shared_ptr<Meta> &globalInfo)
306 {
307     Plugins::FileType fileType;
308     globalInfo->GetData(Tag::MEDIA_FILE_TYPE, fileType);
309     CHECK_AND_RETURN_LOG(fileType != Plugins::FileType::UNKNOW, "unknown file type");
310     if (fileTypeMap.find(fileType) == fileTypeMap.end()) {
311         return;
312     }
313     if (avmeta.GetMeta(AV_KEY_HAS_VIDEO).compare("yes") == 0) {
314         avmeta.SetMeta(AV_KEY_MIME_TYPE, "video/" + fileTypeMap.at(fileType));
315         return;
316     }
317     if (avmeta.GetMeta(AV_KEY_HAS_AUDIO).compare("yes") == 0) {
318         avmeta.SetMeta(AV_KEY_MIME_TYPE, "audio/" + fileTypeMap.at(fileType));
319     }
320 }
321 
FormatDateTime(Metadata & avmeta,const std::shared_ptr<Meta> & globalInfo)322 void AVMetaDataCollector::FormatDateTime(Metadata &avmeta, const std::shared_ptr<Meta> &globalInfo)
323 {
324     std::string date;
325     std::string creationTime;
326     globalInfo->GetData(Tag::MEDIA_DATE, date);
327     globalInfo->GetData(Tag::MEDIA_CREATION_TIME, creationTime);
328     std::string formattedDateTime;
329     if (!date.empty()) {
330         formattedDateTime = TimeFormatUtils::FormatDateTimeByTimeZone(date);
331     } else if (!creationTime.empty()) {
332         formattedDateTime = TimeFormatUtils::FormatDateTimeByTimeZone(creationTime);
333     }
334     avmeta.SetMeta(AV_KEY_DATE_TIME, formattedDateTime);
335     avmeta.SetMeta(AV_KEY_DATE_TIME_FORMAT,
336         formattedDateTime.compare(date) != 0 ? formattedDateTime : TimeFormatUtils::FormatDataTimeByString(date));
337 }
338 
SetEmptyStringIfNoData(Metadata & avmeta,int32_t avKey) const339 void AVMetaDataCollector::SetEmptyStringIfNoData(Metadata &avmeta, int32_t avKey) const
340 {
341     if (!avmeta.HasMeta(avKey)) {
342         avmeta.SetMeta(avKey, "");
343     }
344 }
345 
SetStringByValueType(const std::shared_ptr<Meta> & innerMeta,Metadata & avmeta,int32_t avKey,std::string innerKey) const346 bool AVMetaDataCollector::SetStringByValueType(const std::shared_ptr<Meta> &innerMeta,
347     Metadata &avmeta, int32_t avKey, std::string innerKey) const
348 {
349     Any type = OHOS::Media::GetDefaultAnyValue(innerKey);
350     if (Any::IsSameTypeWith<int32_t>(type)) {
351         int32_t intVal;
352         if (innerMeta->GetData(innerKey, intVal) && intVal != 0) {
353             avmeta.SetMeta(avKey, std::to_string(intVal));
354         }
355     } else if (Any::IsSameTypeWith<std::string>(type)) {
356         std::string strVal;
357         if (innerMeta->GetData(innerKey, strVal)) {
358             avmeta.SetMeta(avKey, strVal);
359         }
360     } else if (Any::IsSameTypeWith<Plugins::VideoRotation>(type)) {
361         Plugins::VideoRotation rotation;
362         if (innerMeta->GetData(innerKey, rotation)) {
363             avmeta.SetMeta(avKey, std::to_string(rotation));
364         }
365     } else if (Any::IsSameTypeWith<int64_t>(type)) {
366         int64_t duration;
367         if (innerMeta->GetData(innerKey, duration)) {
368             avmeta.SetMeta(avKey, std::to_string(duration / SECOND_DEVIDE_MS));
369         }
370     } else if (Any::IsSameTypeWith<bool>(type)) {
371         bool isTrue;
372         if (innerMeta->GetData(innerKey, isTrue)) {
373             avmeta.SetMeta(avKey, isTrue ? "yes" : "");
374         }
375     } else if (Any::IsSameTypeWith<float>(type)) {
376         float value;
377         if (innerMeta->GetData(innerKey, value) && collectedAVMetaData_ != nullptr) {
378             collectedAVMetaData_->SetData(innerKey, value);
379         }
380     } else {
381         MEDIA_LOGE("not found type matched with innerKey: %{public}s", innerKey.c_str());
382         return false;
383     }
384     return true;
385 }
386 
Reset()387 void AVMetaDataCollector::Reset()
388 {
389     mediaDemuxer_->Reset();
390     collectedMeta_.clear();
391     videoTrackId_ = 0;
392     hasVideo_ = false;
393     collectedArtPicture_ = nullptr;
394 }
395 
Destroy()396 void AVMetaDataCollector::Destroy()
397 {
398     mediaDemuxer_ = nullptr;
399 }
400 }  // namespace Media
401 }  // namespace OHOS