1 /*
2  * Copyright (C) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef AVMETADATAHELPER_H
17 #define AVMETADATAHELPER_H
18 
19 #include <memory>
20 #include <map>
21 #include <string>
22 #include <unordered_map>
23 #include <vector>
24 #include "buffer/avsharedmemory.h"
25 #include "meta/format.h"
26 #include "meta/meta.h"
27 #include "media_data_source.h"
28 #include "nocopyable.h"
29 #include "pixel_map.h"
30 
31 namespace OHOS {
32 namespace Media {
33 /**
34  * @brief Enumerates avmetadata helper callback error type.
35  */
36 enum HelperErrorType : int32_t {
37     /* State error, current operation is invalid. */
38     INVALID_OPERATION,
39     /* Result error, current result is invalid. */
40     INVALID_RESULT,
41 };
42 
43 /**
44  * @brief Enumerates avmetadata helper listener info type.
45  */
46 enum HelperOnInfoType : int32_t {
47     /* Current State changed, notify NAPI with onInfo callback. */
48     HELPER_INFO_TYPE_STATE_CHANGE,
49 };
50 
51 /**
52  * @brief Enumerates avmetadata helper states.
53  */
54 enum HelperStates : int32_t {
55     /* error states */
56     HELPER_STATE_ERROR = 0,
57     /* idle states */
58     HELPER_IDLE = 1,
59     /* prepared states */
60     HELPER_PREPARED = 2,
61     /* call done states */
62     HELPER_CALL_DONE = 3,
63     /* released states */
64     HELPER_RELEASED = 4,
65 };
66 
67 enum HelperState : int32_t {
68     /* error states */
69     HELPER_ERROR = -1,
70     /* idle states */
71     HELPER_STATE_IDLE,
72     /* RUNNABLE states */
73     HELPER_STATE_RUNNABLE,
74     /* released states */
75     HELPER_STATE_RELEASED,
76 };
77 
78 /**
79  * The meta data mappings from meta data enum keys to the string key.
80  */
81 static const std::map<int32_t, const char*> g_MetadataCodeMap = {
82     {0,     "album"},
83     {1,     "albumArtist"},
84     {2,     "artist"},
85     {3,     "author"},
86     {4,     "dateTime"},
87     {5,     "dateTimeFormat"},
88     {12,    "composer"},
89     {15,    "duration"},
90     {18,    "genre"},
91     {19,    "hasAudio"},
92     {21,    "hasVideo"},
93     {29,    "mimeType"},
94     {30,    "trackCount"},
95     {31,    "sampleRate"},
96     {33,    "title"},
97     {35,    "videoHeight"},
98     {37,    "videoWidth"},
99     {38,    "videoOrientation"},
100     {39,    "hdrType"},
101 };
102 
103 /**
104  * support metadata parameters
105 */
106 static const std::vector<std::string> g_Metadata = {
107     "album",
108     "albumArtist",
109     "artist",
110     "author",
111     "dateTime",
112     "dateTimeFormat",
113     "composer",
114     "duration",
115     "genre",
116     "hasAudio",
117     "hasVideo",
118     "mimeType",
119     "trackCount",
120     "sampleRate",
121     "title",
122     "videoHeight",
123     "videoWidth",
124     "videoOrientation",
125     "hdrType",
126     "latitude",
127     "longitude",
128     "customInfo",
129 };
130 
131 enum HdrType : int32_t {
132     /**
133      * This option is used to mark none HDR type.
134      */
135     AV_HDR_TYPE_NONE,
136     /**
137      * This option is used to mark HDR Vivid type.
138      */
139     AV_HDR_TYPE_VIVID,
140 };
141 
142 enum Scene : int32_t {
143     /**
144      * This option is used to mark normal scene.
145      */
146     AV_META_SCENE_NORMAL,
147     /**
148      * This option is used to mark clone scene.
149      */
150     AV_META_SCENE_CLONE,
151     /**
152      * This option is used to mark batch handle avmeta scene.
153      */
154     AV_META_SCENE_BATCH_HANDLE,
155 };
156 
157 /**
158  * @brief Enumerates avmetadata usage.
159  */
160 enum AVMetadataUsage : int32_t {
161     /**
162      * Indicates that the avmetadahelper's instance will only be used for resolving the
163      * metadata from the given media resource.
164      */
165     AV_META_USAGE_META_ONLY,
166     /**
167      * Indicates that the avmetadahelper's instance will be used for fetching the video frame
168      * and resolving metadata from the given media resource.
169      */
170     AV_META_USAGE_PIXEL_MAP,
171     /**
172      * Only for video frame index convert, if use this usage, can only GetTimeByFrameIndex / GetFrameIndexByTime
173      */
174     AV_META_USAGE_FRAME_INDEX_CONVERT,
175 };
176 
177 /**
178  * @brief Enumerates avmetadata's metadata key.
179  */
180 enum AVMetadataCode : int32_t {
181     /**
182      * The metadata key to retrieve the information about the album title
183      * of the media source.
184      */
185     AV_KEY_ALBUM = 0,
186     /**
187      * The metadata key to retrieve the information about the performers or
188      * artist associated with the media source.
189      */
190     AV_KEY_ALBUM_ARTIST = 1,
191     /**
192      * The metadata key to retrieve the information about the artist of
193      * the media source.
194      */
195     AV_KEY_ARTIST = 2,
196     /**
197      * The metadata key to retrieve the information about the author of
198      * the media source.
199      */
200     AV_KEY_AUTHOR = 3,
201     /**
202      * The metadata key to retrieve the information about the created time of
203      * the media source.
204      */
205     AV_KEY_DATE_TIME = 4,
206     /**
207      * The metadata key to retrieve the information about the created time of
208      * the media source. This keyword is provided for the media library.
209      */
210     AV_KEY_DATE_TIME_FORMAT = 5,
211     /**
212      * The metadata key to retrieve the information about the composer of
213      * the media source.
214      */
215     AV_KEY_COMPOSER = 12,
216     /**
217      * The metadata key to retrieve the playback duration of the media source.
218      */
219     AV_KEY_DURATION = 15,
220     /**
221      * The metadata key to retrieve the content type or genre of the data
222      * source.
223      */
224     AV_KEY_GENRE = 18,
225     /**
226      * If this key exists the media contains audio content.
227      */
228     AV_KEY_HAS_AUDIO = 19,
229     /**
230      * If this key exists the media contains video content.
231      */
232     AV_KEY_HAS_VIDEO = 21,
233     /**
234      * The metadata key to retrieve the mime type of the media source. Some
235      * example mime types include: "video/mp4", "audio/mp4", "audio/amr-wb",
236      * etc.
237      */
238     AV_KEY_MIME_TYPE = 29,
239     /**
240      * The metadata key to retrieve the number of tracks, such as audio, video,
241      * text, in the media source, such as a mp4 or 3gpp file.
242      */
243     AV_KEY_NUM_TRACKS = 30,
244     /**
245      * This key retrieves the sample rate, if available.
246      */
247     AV_KEY_SAMPLE_RATE = 31,
248     /**
249      * The metadata key to retrieve the media source title.
250      */
251     AV_KEY_TITLE = 33,
252     /**
253      * If the media contains video, this key retrieves its height.
254      */
255     AV_KEY_VIDEO_HEIGHT = 35,
256     /**
257      * If the media contains video, this key retrieves its width.
258      */
259     AV_KEY_VIDEO_WIDTH = 37,
260     /**
261      * The metadata key to retrieve the information about the video
262      * orientation.
263      */
264     AV_KEY_VIDEO_ORIENTATION = 38,
265     /**
266      * The metadata key to retrieve the information about the video
267      * is HDR or not.
268      */
269     AV_KEY_VIDEO_IS_HDR_VIVID = 39,
270 
271     /**
272      * The metadata key to retrieve the information about the location longitude
273     */
274     AV_KEY_LOCATION_LONGITUDE = 40,
275 
276     /**
277      * The metadata key to retrieve the information about the location latitude
278     */
279     AV_KEY_LOCATION_LATITUDE = 41,
280 
281     /**
282      * Custom parameter key-value map
283     */
284     AV_KEY_CUSTOMINFO = 42,
285 };
286 
287 /**
288  * @brief Enumerates avmetadata's query option.
289  */
290 enum AVMetadataQueryOption : int32_t {
291     /**
292      * This option is used to fetch a key frame from the given media
293      * resource that is located right after or at the given time.
294      */
295     AV_META_QUERY_NEXT_SYNC,
296     /**
297      * This option is used to fetch a key frame from the given media
298      * resource that is located right before or at the given time.
299      */
300     AV_META_QUERY_PREVIOUS_SYNC,
301     /**
302      * This option is used to fetch a key frame from the given media
303      * resource that is located closest to or at the given time.
304      */
305     AV_META_QUERY_CLOSEST_SYNC,
306     /**
307      * This option is used to fetch a frame (maybe not keyframe) from
308      * the given media resource that is located closest to or at the given time.
309      */
310     AV_META_QUERY_CLOSEST,
311 };
312 
313 /**
314  * @brief Provides the definition of the returned pixelmap's configuration
315  */
316 struct PixelMapParams {
317     /**
318      * Expected pixelmap's width, -1 means to keep consistent with the
319      * original dimensions of the given video resource.
320      */
321     int32_t dstWidth = -1;
322     /**
323      * Expected pixelmap's width, -1 means to keep consistent with the
324      * original dimensions of the given video resource.
325      */
326     int32_t dstHeight = -1;
327     /**
328      * Expected pixelmap's color format, see {@link PixelFormat}. Currently,
329      * RGB_565, RGB_888, RGBA_8888 are supported.
330      */
331     PixelFormat colorFormat = PixelFormat::RGB_565;
332 };
333 
334 /**
335  * @brief Provides the callback interfaces to notify client about errors or infos.
336  */
337 class HelperCallback {
338 public:
339     virtual ~HelperCallback() = default;
340     /**
341      * Called when a metadata helper message is notified.
342      *
343      * @param type Indicates the information type. For details, see {@link HelperOnInfoType}.
344      * @param extra Indicates other information, for example, the current state in metadata helper server.
345      * @param infoBody According to the info type, the information carrier passed. Is an optional parameter.
346      */
347     virtual void OnInfo(HelperOnInfoType type, int32_t extra, const Format &infoBody = {}) = 0;
348 
349     /**
350      * Called when an error occurred
351      *
352      * @param errorCode Error code.
353      * @param errorMsg Error message.
354      */
355     virtual void OnError(int32_t errorCode, const std::string &errorMsg) = 0;
356 };
357 
358 /**
359  * @brief Provides the interfaces to resolve metadata or fetch frame
360  * from a given media resource.
361  */
362 class AVMetadataHelper {
363 public:
364     virtual ~AVMetadataHelper() = default;
365 
366     /**
367      * Set the media source uri to resolve. Calling this method before the reset
368      * of the methods in this class. This method maybe time consuming.
369      * @param uri the URI of input media source.
370      * @param usage indicates which scene the avmedatahelper's instance will
371      * be used to, see {@link AVMetadataUsage}. If the usage need to be changed,
372      * this method must be called again.
373      * @return Returns {@link MSERR_OK} if the setting is successful; returns
374      * an error code otherwise.
375      */
376     virtual int32_t SetSource(const std::string &uri, int32_t usage = AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) = 0;
377 
378     /**
379      * @brief Sets the media file descriptor source to resolve. Calling this method
380      * before the reset of the methods in this class. This method maybe time consuming.
381      * @param fd Indicates the file descriptor of media source.
382      * @param offset Indicates the offset of media source in file descriptor.
383      * @param size Indicates the size of media source.
384      * @param usage Indicates which scene the avmedatahelper's instance will
385      * be used to, see {@link AVMetadataUsage}. If the usage need to be changed,
386      * this method must be called again.
387      * @return Returns {@link MSERR_OK} if the setting is successful; returns
388      * an error code otherwise.
389      */
390     virtual int32_t SetSource(int32_t fd, int64_t offset = 0, int64_t size = 0,
391         int32_t usage = AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) = 0;
392 
393     /**
394      * @brief Sets the playback media data source for the meta data helper.
395      *
396      * @param dataSrc Indicates the media data source. in {@link media_data_source.h}
397      * @return Returns {@link MSERR_OK} if the mediadatasource is set successfully; returns an error code defined
398      * in {@link media_errors.h} otherwise.
399      */
400     virtual int32_t SetSource(const std::shared_ptr<IMediaDataSource> &dataSrc) = 0;
401 
402     /**
403      * Retrieve the meta data associated with the specified key. This method must be
404      * called after the SetSource.
405      * @param key One of the constants listed above at the definition of {@link AVMetadataCode}.
406      * @return Returns the meta data value associate with the given key code on
407      * success; empty string on failure.
408      */
409     virtual std::string ResolveMetadata(int32_t key) = 0;
410 
411     /**
412      * Retrieve all meta data within the listed above at the definition of {@link AVMetadataCode}.
413      * This method must be called after the SetSource.
414      * @return Returns the meta data values on success; empty hash map on failure.
415      */
416     virtual std::unordered_map<int32_t, std::string> ResolveMetadata() = 0;
417 
418     /**
419      * Fetch the album art picture associated with the data source. If there are
420      * more than one pictures, the cover image will be returned preferably.
421      * @return Returns the a chunk of shared memory containing a picture, which can be
422      * null, if such a picture can not be fetched.
423      */
424     virtual std::shared_ptr<AVSharedMemory> FetchArtPicture() = 0;
425 
426     /**
427      * Fetch a representative video frame near a given timestamp by considering the given
428      * option if possible, and return a pixelmap with given parameters. This method must be
429      * called after the SetSource.
430      * @param timeUs The time position in microseconds where the frame will be fetched.
431      * When fetching the frame at the given time position, there is no guarantee that
432      * the video source has a frame located at the position. When this happens, a frame
433      * nearby will be returned. If timeUs is negative, time position and option will ignored,
434      * and any frame that the implementation considers as representative may be returned.
435      * @param option the hint about how to fetch a frame, see {@link AVMetadataQueryOption}
436      * @param param the desired configuration of returned pixelmap, see {@link PixelMapParams}.
437      * @return Returns a pixelmap containing a scaled video frame, which can be null, if such a
438      * frame cannot be fetched.
439      */
440     virtual std::shared_ptr<PixelMap> FetchFrameAtTime(int64_t timeUs, int32_t option, const PixelMapParams &param) = 0;
441 
442     /**
443      * Fetch a representative video frame near a given timestamp by considering the given
444      * option if possible, and return a pixelmap with given parameters. This method must be
445      * called after the SetSource.
446      * @param timeUs The time position in microseconds where the frame will be fetched.
447      * When fetching the frame at the given time position, there is no guarantee that
448      * the video source has a frame located at the position. When this happens, a frame
449      * nearby will be returned. If timeUs is negative, time position and option will ignored,
450      * and any frame that the implementation considers as representative may be returned.
451      * @param option the hint about how to fetch a frame, see {@link AVMetadataQueryOption}
452      * @param param the desired configuration of returned pixelmap, see {@link PixelMapParams}.
453      * @return Returns a pixelmap containing a scaled video frame, which can be null, if such a
454      * frame cannot be fetched.
455      */
456     virtual std::shared_ptr<PixelMap> FetchFrameYuv(int64_t timeUs, int32_t option, const PixelMapParams &param) = 0;
457 
458     /**
459      * all meta data.
460      * This method must be called after the SetSource.
461      * @return Returns the meta data values on success; nullptr on failure.
462      */
463     virtual std::shared_ptr<Meta> GetAVMetadata() = 0;
464 
465     /**
466      * Release the internel resource. After this method called, the avmetadatahelper instance
467      * can not be used again.
468      */
469     virtual void Release() = 0;
470 
471     /**
472      * @brief Method to set the meta data helper callback.
473      *
474      * @param callback object pointer.
475      * @return Returns {@link MSERR_OK} if the meta data helper callback is set; returns an error code defined
476      * in {@link media_errors.h} otherwise.
477      */
478     virtual int32_t SetHelperCallback(const std::shared_ptr<HelperCallback> &callback) = 0;
479 
480     /**
481      * Set scene, to identity the sepcial scene.
482      *
483      * @param scene indicates which scene the avmedatahelper's instance will
484      * be used to, see {@link Scene}.
485      */
486     virtual void SetScene(Scene scene) = 0;
487 
488     /**
489      * Get timestamp according to frame index.
490      * @param timeUs : Index of the frame.
491      * @returns returns time
492      */
493     virtual int32_t GetTimeByFrameIndex(uint32_t index, uint64_t &time) = 0;
494 
495     /**
496      * Get frame index according to the given timestamp.
497      * @param timeUs : Timestamp of the frame, in microseconds.
498      * @returns Returns frame
499      */
500     virtual int32_t GetFrameIndexByTime(uint64_t time, uint32_t &index) = 0;
501 };
502 
503 class __attribute__((visibility("default"))) AVMetadataHelperFactory {
504 public:
505 #ifdef UNSUPPORT_METADATA
CreateAVMetadataHelper()506     static std::shared_ptr<AVMetadataHelper> CreateAVMetadataHelper()
507     {
508         return nullptr;
509     }
510 #else
511     static std::shared_ptr<AVMetadataHelper> CreateAVMetadataHelper();
512 #endif
513 private:
514     AVMetadataHelperFactory() = default;
515     ~AVMetadataHelperFactory() = default;
516 };
517 } // namespace Media
518 } // namespace OHOS
519 #endif // AVMETADATAHELPER_H