1 /*
2  * Copyright (c) 2022 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "output/metadata_output.h"
17 
18 #include <algorithm>
19 #include <cinttypes>
20 #include <cstddef>
21 #include <cstdint>
22 
23 #include "camera_device_ability_items.h"
24 #include "camera_error_code.h"
25 #include "camera_log.h"
26 #include "camera_manager.h"
27 #include "camera_security_utils.h"
28 #include "camera_util.h"
29 #include "foundation/multimedia/camera_framework/interfaces/kits/native/include/camera/camera.h"
30 #include "input/camera_input.h"
31 #include "session/capture_session.h"
32 #include "camera_error_code.h"
33 
34 namespace OHOS {
35 namespace CameraStandard {
36 sptr<MetadataObjectFactory> MetadataObjectFactory::metaFactoryInstance_;
37 std::mutex MetadataObjectFactory::instanceMutex_;
38 
39 const std::unordered_map<uint32_t, MetadataObjectType> g_HALResultToFwCameraMetaDetect_ = {
40     {OHOS_STATISTICS_DETECT_HUMAN_FACE_INFOS, MetadataObjectType::FACE},
41     {OHOS_STATISTICS_DETECT_HUMAN_BODY_INFOS, MetadataObjectType::HUMAN_BODY},
42     {OHOS_STATISTICS_DETECT_CAT_FACE_INFOS, MetadataObjectType::CAT_FACE},
43     {OHOS_STATISTICS_DETECT_CAT_BODY_INFOS, MetadataObjectType::CAT_BODY},
44     {OHOS_STATISTICS_DETECT_DOG_FACE_INFOS, MetadataObjectType::DOG_FACE},
45     {OHOS_STATISTICS_DETECT_DOG_BODY_INFOS, MetadataObjectType::DOG_BODY},
46     {OHOS_STATISTICS_DETECT_SALIENT_INFOS, MetadataObjectType::SALIENT_DETECTION},
47     {OHOS_STATISTICS_DETECT_BAR_CODE_INFOS, MetadataObjectType::BAR_CODE_DETECTION},
48     {OHOS_STATISTICS_DETECT_BASE_FACE_INFO, MetadataObjectType::BASE_FACE_DETECTION},
49 };
50 
MetadataObject(const MetadataObjectType type,const int32_t timestamp,const Rect rect,const int32_t objectId,const int32_t confidence)51 MetadataObject::MetadataObject(const MetadataObjectType type, const int32_t timestamp, const Rect rect,
52                                const int32_t objectId, const int32_t confidence)
53     : type_(type),
54       timestamp_(timestamp),
55       box_(rect),
56       objectId_(objectId),
57       confidence_(confidence)
58 {}
59 
MetadataObject(const MetaObjectParms & parms)60 MetadataObject::MetadataObject(const MetaObjectParms &parms)
61 {
62     type_ = parms.type;
63     timestamp_ = parms.timestamp;
64     box_ = parms.box;
65     objectId_ = parms.objectId;
66     confidence_ = parms.confidence;
67 }
68 
MetadataFaceObject(const MetaObjectParms & parms,const Rect leftEyeBoundingBox,const Rect rightEyeBoundingBox,const Emotion emotion,const int32_t emotionConfidence,const int32_t pitchAngle,const int32_t yawAngle,const int32_t rollAngle)69 MetadataFaceObject::MetadataFaceObject(const MetaObjectParms &parms, const Rect leftEyeBoundingBox,
70                                        const Rect rightEyeBoundingBox, const Emotion emotion,
71                                        const int32_t emotionConfidence, const int32_t pitchAngle,
72                                        const int32_t yawAngle, const int32_t rollAngle)
73     : MetadataObject(parms),
74       leftEyeBoundingBox_(leftEyeBoundingBox),
75       rightEyeBoundingBox_(rightEyeBoundingBox),
76       emotion_(emotion),
77       emotionConfidence_(emotionConfidence),
78       pitchAngle_(pitchAngle),
79       yawAngle_(yawAngle),
80       rollAngle_(rollAngle)
81 {}
82 
MetadataHumanBodyObject(const MetaObjectParms & parms)83 MetadataHumanBodyObject::MetadataHumanBodyObject(const MetaObjectParms &parms) : MetadataObject(parms) {}
84 
MetadataCatFaceObject(const MetaObjectParms & parms,const Rect leftEyeBoundingBox,const Rect rightEyeBoundingBox)85 MetadataCatFaceObject::MetadataCatFaceObject(const MetaObjectParms &parms, const Rect leftEyeBoundingBox,
86                                              const Rect rightEyeBoundingBox)
87     : MetadataObject(parms),
88       leftEyeBoundingBox_(leftEyeBoundingBox),
89       rightEyeBoundingBox_(rightEyeBoundingBox)
90 {}
91 
MetadataCatBodyObject(const MetaObjectParms & parms)92 MetadataCatBodyObject::MetadataCatBodyObject(const MetaObjectParms &parms) : MetadataObject(parms) {}
93 
MetadataDogFaceObject(const MetaObjectParms & parms,const Rect leftEyeBoundingBox,const Rect rightEyeBoundingBox)94 MetadataDogFaceObject::MetadataDogFaceObject(const MetaObjectParms &parms, const Rect leftEyeBoundingBox,
95                                              const Rect rightEyeBoundingBox)
96     : MetadataObject(parms),
97       leftEyeBoundingBox_(leftEyeBoundingBox),
98       rightEyeBoundingBox_(rightEyeBoundingBox)
99 {}
100 
MetadataDogBodyObject(const MetaObjectParms & parms)101 MetadataDogBodyObject::MetadataDogBodyObject(const MetaObjectParms &parms) : MetadataObject(parms) {}
102 
MetadataSalientDetectionObject(const MetaObjectParms & parms)103 MetadataSalientDetectionObject::MetadataSalientDetectionObject(const MetaObjectParms &parms) : MetadataObject(parms) {}
104 
MetadataBarCodeDetectionObject(const MetaObjectParms & parms)105 MetadataBarCodeDetectionObject::MetadataBarCodeDetectionObject(const MetaObjectParms &parms) : MetadataObject(parms) {}
106 
MetadataOutput(sptr<IConsumerSurface> surface,sptr<IStreamMetadata> & streamMetadata)107 MetadataOutput::MetadataOutput(sptr<IConsumerSurface> surface, sptr<IStreamMetadata> &streamMetadata)
108     : CaptureOutput(CAPTURE_OUTPUT_TYPE_METADATA, StreamType::METADATA, surface->GetProducer(), nullptr),
109       surface_(surface)
110 {
111     MEDIA_DEBUG_LOG("MetadataOutput::MetadataOutput construct enter");
112 }
113 
~MetadataOutput()114 MetadataOutput::~MetadataOutput()
115 {
116     ReleaseSurface();
117 }
118 
GetAppObjectCallback()119 std::shared_ptr<MetadataObjectCallback> MetadataOutput::GetAppObjectCallback()
120 {
121     std::lock_guard<std::mutex> lock(outputCallbackMutex_);
122     return appObjectCallback_;
123 }
124 
GetAppStateCallback()125 std::shared_ptr<MetadataStateCallback> MetadataOutput::GetAppStateCallback()
126 {
127     MEDIA_DEBUG_LOG("CameraDeviceServiceCallback::GetAppStateCallback");
128     std::lock_guard<std::mutex> lock(outputCallbackMutex_);
129     return appStateCallback_;
130 }
131 
GetSupportedMetadataObjectTypes()132 std::vector<MetadataObjectType> MetadataOutput::GetSupportedMetadataObjectTypes()
133 {
134     auto session = GetSession();
135     CHECK_ERROR_RETURN_RET(session == nullptr, {});
136     auto inputDevice = session->GetInputDevice();
137     CHECK_ERROR_RETURN_RET(inputDevice == nullptr, {});
138     sptr<CameraDevice> cameraObj = inputDevice->GetCameraDeviceInfo();
139     CHECK_ERROR_RETURN_RET(cameraObj == nullptr, {});
140     std::shared_ptr<Camera::CameraMetadata> metadata = cameraObj->GetMetadata();
141     CHECK_ERROR_RETURN_RET(metadata == nullptr, {});
142     camera_metadata_item_t item;
143     int ret = Camera::FindCameraMetadataItem(metadata->get(), OHOS_STATISTICS_FACE_DETECT_MODE, &item);
144     CHECK_ERROR_RETURN_RET(ret, {});
145     std::vector<MetadataObjectType> objectTypes;
146     for (size_t index = 0; index < item.count; index++) {
147         if (item.data.u8[index] == OHOS_CAMERA_FACE_DETECT_MODE_SIMPLE) {
148             objectTypes.emplace_back(MetadataObjectType::FACE);
149         }
150     }
151     return objectTypes;
152 }
153 
SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)154 void MetadataOutput::SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)
155 {
156     auto session = GetSession();
157     CHECK_ERROR_RETURN((session == nullptr) || (session->GetInputDevice() == nullptr));
158     std::set<camera_face_detect_mode_t> objectTypes;
159     for (const auto& type : metadataObjectTypes) {
160         if (type == MetadataObjectType::FACE) {
161             objectTypes.insert(OHOS_CAMERA_FACE_DETECT_MODE_SIMPLE);
162         }
163     }
164     if (objectTypes.empty()) {
165         objectTypes.insert(OHOS_CAMERA_FACE_DETECT_MODE_OFF);
166     }
167 
168     session->SetCaptureMetadataObjectTypes(objectTypes);
169 }
170 
AddMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)171 int32_t MetadataOutput::AddMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)
172 {
173     const size_t maxSize4NonSystemApp  = 1;
174     if (!CameraSecurity::CheckSystemApp()) {
175         MEDIA_DEBUG_LOG("public calling for metadataOutput");
176         if (metadataObjectTypes.size() > maxSize4NonSystemApp ||
177             std::any_of(metadataObjectTypes.begin(), metadataObjectTypes.end(),
178                 [](MetadataObjectType type) { return type != MetadataObjectType::FACE; })) {
179             return CameraErrorCode::INVALID_ARGUMENT;
180         }
181     }
182     auto session = GetSession();
183     CHECK_ERROR_RETURN_RET_LOG(session == nullptr || !session->IsSessionCommited(), CameraErrorCode::SESSION_NOT_CONFIG,
184                                "MetadataOutput Failed to AddMetadataObjectTypes!, session not commited");
185 
186     auto inputDevice = session->GetInputDevice();
187     CHECK_ERROR_RETURN_RET_LOG(inputDevice == nullptr, CameraErrorCode::SESSION_NOT_CONFIG,
188                                "MetadataOutput Failed to AddMetadataObjectTypes!, inputDevice is null");
189 
190     sptr<CameraDevice> cameraObj = inputDevice->GetCameraDeviceInfo();
191     CHECK_ERROR_RETURN_RET_LOG(cameraObj == nullptr, CameraErrorCode::SESSION_NOT_CONFIG,
192                                "MetadataOutput Failed to AddMetadataObjectTypes!, cameraObj is null");
193 
194     auto outoputCapability = CameraManager::GetInstance()->GetSupportedOutputCapability(cameraObj, session->GetMode());
195     CHECK_ERROR_RETURN_RET_LOG(outoputCapability == nullptr, CameraErrorCode::SESSION_NOT_CONFIG,
196                                "MetadataOutput Failed to AddMetadataObjectTypes!, outoputCapability is null");
197 
198     std::vector<MetadataObjectType> supportMetadataType = outoputCapability->GetSupportedMetadataObjectType();
199     for (const auto &type : supportMetadataType) {
200         MEDIA_DEBUG_LOG("MetadataOutput::AddMetadataObjectTypes, support type: %{public}d", type);
201     }
202     CHECK_ERROR_RETURN_RET_LOG(!checkValidType(metadataObjectTypes, supportMetadataType),
203                                CameraErrorCode::INVALID_ARGUMENT,
204                                "MetadataOutput::AddMetadataObjectTypes, unsupported type!");
205 
206     auto stream = GetStream();
207     CHECK_ERROR_RETURN_RET_LOG(stream == nullptr, CameraErrorCode::SESSION_NOT_CONFIG,
208                                "MetadataOutput Failed to AddMetadataObjectTypes!, GetStream is nullptr");
209     std::vector<int32_t> numberOfTypes = convert(metadataObjectTypes);
210     int32_t errCode = static_cast<IStreamMetadata *>(stream.GetRefPtr())->EnableMetadataType(numberOfTypes);
211     CHECK_ERROR_RETURN_RET_LOG(
212         errCode != CAMERA_OK, CameraErrorCode::SERVICE_FATL_ERROR,
213         "MetadataOutput Failed to AddMetadataObjectTypes!, EnableMetadataType failed ret: %{public}d", errCode);
214     return CameraErrorCode::SUCCESS;
215 }
216 
RemoveMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)217 int32_t MetadataOutput::RemoveMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes)
218 {
219     const size_t maxSize4NonSystemApp  = 1;
220     if (!CameraSecurity::CheckSystemApp()) {
221         MEDIA_DEBUG_LOG("public calling for metadataOutput");
222         if (metadataObjectTypes.size() > maxSize4NonSystemApp ||
223             std::any_of(metadataObjectTypes.begin(), metadataObjectTypes.end(),
224                 [](MetadataObjectType type) { return type != MetadataObjectType::FACE; })) {
225             return CameraErrorCode::INVALID_ARGUMENT;
226         }
227     }
228     auto session = GetSession();
229     CHECK_ERROR_RETURN_RET_LOG(session == nullptr || !session->IsSessionCommited(), CameraErrorCode::SESSION_NOT_CONFIG,
230                                "MetadataOutput Failed to RemoveMetadataObjectTypes!, session not commited");
231 
232     auto stream = GetStream();
233     CHECK_ERROR_RETURN_RET_LOG(stream == nullptr, CameraErrorCode::SESSION_NOT_CONFIG,
234                                "MetadataOutput Failed to AddMetadataObjectTypes!, GetStream is nullptr");
235 
236     std::vector<int32_t> numberOfTypes = convert(metadataObjectTypes);
237     int32_t errCode = static_cast<IStreamMetadata *>(stream.GetRefPtr())->DisableMetadataType(numberOfTypes);
238     CHECK_ERROR_RETURN_RET_LOG(
239         errCode != CAMERA_OK, CameraErrorCode::SERVICE_FATL_ERROR,
240         "MetadataOutput Failed to AddMetadataObjectTypes!, EnableMetadataType failed ret: %{public}d", errCode);
241     return CameraErrorCode::SUCCESS;
242 }
243 
checkValidType(const std::vector<MetadataObjectType> & typeAdded,const std::vector<MetadataObjectType> & supportedType)244 bool MetadataOutput::checkValidType(const std::vector<MetadataObjectType> &typeAdded,
245                                     const std::vector<MetadataObjectType> &supportedType)
246 {
247     return std::all_of(typeAdded.begin(), typeAdded.end(), [&supportedType](MetadataObjectType type) {
248         return std::find(supportedType.begin(), supportedType.end(), type) != supportedType.end();
249     });
250 }
251 
convert(const std::vector<MetadataObjectType> & typesOfMetadata)252 std::vector<int32_t> MetadataOutput::convert(const std::vector<MetadataObjectType> &typesOfMetadata)
253 {
254     std::vector<int32_t> result(typesOfMetadata.size());
255     std::transform(typesOfMetadata.begin(), typesOfMetadata.end(), result.begin(),
256                    [](MetadataObjectType obj) { return static_cast<int32_t>(obj); });
257     return result;
258 }
259 
SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback)260 void MetadataOutput::SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback)
261 {
262     std::lock_guard<std::mutex> lock(outputCallbackMutex_);
263     appObjectCallback_ = metadataObjectCallback;
264     if (appObjectCallback_ != nullptr) {
265         if (cameraMetadataCallback_ == nullptr) {
266             cameraMetadataCallback_ = new HStreamMetadataCallbackImpl(this);
267         }
268     }
269     auto itemStream = static_cast<IStreamMetadata *>(GetStream().GetRefPtr());
270     int32_t errorCode = CAMERA_OK;
271     if (itemStream) {
272         errorCode = itemStream->SetCallback(cameraMetadataCallback_);
273     } else {
274         MEDIA_ERR_LOG("MetadataOutput::SetCallback() itemStream is nullptr");
275     }
276     if (errorCode != CAMERA_OK) {
277         MEDIA_ERR_LOG("MetadataOutput::SetCallback(): Failed to register callback, errorCode: %{public}d", errorCode);
278         cameraMetadataCallback_ = nullptr;
279         appObjectCallback_ = nullptr;
280     }
281 }
282 
SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback)283 void MetadataOutput::SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback)
284 {
285     std::lock_guard<std::mutex> lock(outputCallbackMutex_);
286     appStateCallback_ = metadataStateCallback;
287 }
288 
CreateStream()289 int32_t MetadataOutput::CreateStream()
290 {
291     return CameraErrorCode::SUCCESS;
292 }
293 
Start()294 int32_t MetadataOutput::Start()
295 {
296     MEDIA_DEBUG_LOG("MetadataOutput::Start is called");
297     auto session = GetSession();
298     CHECK_ERROR_RETURN_RET_LOG(session == nullptr || !session->IsSessionCommited(), CameraErrorCode::SUCCESS,
299                                "MetadataOutput Failed to Start!, session not commited");
300     auto stream = GetStream();
301     CHECK_ERROR_RETURN_RET_LOG(stream == nullptr, CameraErrorCode::SUCCESS,
302                                "MetadataOutput Failed to Start!, GetStream is nullptr");
303     int32_t errCode = static_cast<IStreamMetadata *>(stream.GetRefPtr())->Start();
304     CHECK_ERROR_PRINT_LOG(errCode != CAMERA_OK, "Failed to Start MetadataOutput!, errCode: %{public}d", errCode);
305     return CameraErrorCode::SUCCESS;
306 }
307 
CameraServerDied(pid_t pid)308 void MetadataOutput::CameraServerDied(pid_t pid)
309 {
310     MEDIA_ERR_LOG("camera server has died, pid:%{public}d!", pid);
311     std::lock_guard<std::mutex> lock(outputCallbackMutex_);
312     if (appStateCallback_ != nullptr) {
313         MEDIA_DEBUG_LOG("appCallback not nullptr");
314         int32_t serviceErrorType = ServiceToCameraError(CAMERA_INVALID_STATE);
315         appStateCallback_->OnError(serviceErrorType);
316     }
317 }
318 
Stop()319 int32_t MetadataOutput::Stop()
320 {
321     MEDIA_DEBUG_LOG("MetadataOutput::Stop");
322     auto stream = GetStream();
323     CHECK_ERROR_RETURN_RET_LOG(stream == nullptr, CameraErrorCode::SERVICE_FATL_ERROR,
324         "MetadataOutput Failed to Stop!, GetStream is nullptr");
325     int32_t errCode = static_cast<IStreamMetadata*>(stream.GetRefPtr())->Stop();
326     CHECK_ERROR_PRINT_LOG(errCode != CAMERA_OK, "Failed to Stop MetadataOutput!, errCode: %{public}d", errCode);
327     return ServiceToCameraError(errCode);
328 }
329 
Release()330 int32_t MetadataOutput::Release()
331 {
332     {
333         std::lock_guard<std::mutex> lock(outputCallbackMutex_);
334         appObjectCallback_ = nullptr;
335         appStateCallback_ = nullptr;
336         cameraMetadataCallback_ = nullptr;
337     }
338     auto stream = GetStream();
339     CHECK_ERROR_RETURN_RET_LOG(stream == nullptr, CameraErrorCode::SERVICE_FATL_ERROR,
340         "MetadataOutput Failed to Release!, GetStream is nullptr");
341     int32_t errCode = static_cast<IStreamMetadata *>(stream.GetRefPtr())->Release();
342     CHECK_ERROR_PRINT_LOG(errCode != CAMERA_OK, "Failed to Release MetadataOutput!, errCode: %{public}d", errCode);
343     ReleaseSurface();
344     CaptureOutput::Release();
345     return ServiceToCameraError(errCode);
346 }
347 
ReleaseSurface()348 void MetadataOutput::ReleaseSurface()
349 {
350     std::lock_guard<std::mutex> lock(surfaceMutex_);
351     if (surface_ != nullptr) {
352         SurfaceError ret = surface_->UnregisterConsumerListener();
353         CHECK_ERROR_PRINT_LOG(ret != SURFACE_ERROR_OK, "Failed to unregister surface consumer listener");
354         surface_ = nullptr;
355     }
356 }
357 
GetSurface()358 sptr<IConsumerSurface> MetadataOutput::GetSurface()
359 {
360     std::lock_guard<std::mutex> lock(surfaceMutex_);
361     return surface_;
362 }
363 
ProcessMetadata(const int32_t streamId,const std::shared_ptr<OHOS::Camera::CameraMetadata> & result,std::vector<sptr<MetadataObject>> & metaObjects,bool isNeedMirror,bool isNeedFlip)364 void MetadataOutput::ProcessMetadata(const int32_t streamId,
365                                      const std::shared_ptr<OHOS::Camera::CameraMetadata> &result,
366                                      std::vector<sptr<MetadataObject>> &metaObjects, bool isNeedMirror, bool isNeedFlip)
367 {
368     CHECK_ERROR_RETURN(result == nullptr);
369     // camera_metadata_item_t metadataItem;
370     common_metadata_header_t *metadata = result->get();
371     std::vector<camera_metadata_item_t> metadataResults;
372     std::vector<uint32_t> metadataTypes;
373     GetMetadataResults(metadata, metadataResults, metadataTypes);
374     if (metadataResults.size() == 0) {
375         reportLastFaceResults_ = false;
376         if (reportFaceResults_) {
377             reportLastFaceResults_ = true;
378             reportFaceResults_ = false;
379         }
380         metaObjects.clear();
381         MEDIA_ERR_LOG("Camera not ProcessFaceRectangles");
382         return;
383     }
384     int32_t ret = ProcessMetaObjects(streamId, metaObjects, metadataResults, metadataTypes, isNeedMirror, isNeedFlip);
385     reportFaceResults_ = true;
386     CHECK_ERROR_RETURN_LOG(ret != CameraErrorCode::SUCCESS, "MetadataOutput::ProcessFaceRectangles() is failed.");
387     return;
388 }
389 
ProcessMetaObjects(const int32_t streamId,std::vector<sptr<MetadataObject>> & metaObjects,const std::vector<camera_metadata_item_t> & metadataItem,const std::vector<uint32_t> & metadataTypes,bool isNeedMirror,bool isNeedFlip)390 int32_t MetadataOutput::ProcessMetaObjects(const int32_t streamId, std::vector<sptr<MetadataObject>>& metaObjects,
391                                            const std::vector<camera_metadata_item_t>& metadataItem,
392                                            const std::vector<uint32_t>& metadataTypes,
393                                            bool isNeedMirror, bool isNeedFlip)
394 {
395     for (size_t i = 0; i < metadataItem.size(); ++i) {
396         auto itr = g_HALResultToFwCameraMetaDetect_.find(metadataTypes[i]);
397         if (itr != g_HALResultToFwCameraMetaDetect_.end()) {
398             GenerateObjects(metadataItem[i], itr->second, metaObjects, isNeedMirror, isNeedFlip);
399         } else {
400             MEDIA_ERR_LOG("MetadataOutput::ProcessMetaObjects() unsupported type: %{public}d", metadataTypes[i]);
401         }
402     }
403     return CameraErrorCode::SUCCESS;
404 }
405 
GenerateObjects(const camera_metadata_item_t & metadataItem,MetadataObjectType type,std::vector<sptr<MetadataObject>> & metaObjects,bool isNeedMirror,bool isNeedFlip)406 void MetadataOutput::GenerateObjects(const camera_metadata_item_t &metadataItem, MetadataObjectType type,
407                                      std::vector<sptr<MetadataObject>> &metaObjects, bool isNeedMirror, bool isNeedFlip)
408 {
409     int32_t index = 0;
410     int32_t countOfObject = 0;
411     auto iteratorOfLengthMap = mapLengthOfType.find(type);
412     if (iteratorOfLengthMap != mapLengthOfType.end()) {
413         countOfObject = metadataItem.count / iteratorOfLengthMap->second;
414     }
415     for (int32_t itr = 0; itr < countOfObject; ++itr) {
416         sptr<MetadataObjectFactory> objectFactoryPtr = MetadataObjectFactory::GetInstance();
417         MetadataObjectType typeFromHal = static_cast<MetadataObjectType>(metadataItem.data.i32[index]);
418         index++;
419         ProcessBaseInfo(objectFactoryPtr, metadataItem, index, typeFromHal, isNeedMirror, isNeedFlip);
420         ProcessExternInfo(objectFactoryPtr, metadataItem, index, typeFromHal, isNeedMirror, isNeedFlip);
421         metaObjects.push_back(objectFactoryPtr->createMetadataObject(type));
422     }
423 }
424 
GetMetadataResults(const common_metadata_header_t * metadata,std::vector<camera_metadata_item_t> & metadataResults,std::vector<uint32_t> & metadataTypes)425 void MetadataOutput::GetMetadataResults(const common_metadata_header_t *metadata,
426     std::vector<camera_metadata_item_t>& metadataResults, std::vector<uint32_t>& metadataTypes)
427 {
428     for (auto itr : typesOfMetadata_) {
429         camera_metadata_item_t item;
430         int ret = Camera::FindCameraMetadataItem(metadata, itr, &item);
431         if (ret == CAM_META_SUCCESS) {
432             metadataResults.emplace_back(std::move(item));
433             metadataTypes.emplace_back(itr);
434         }
435     }
436 }
437 
ProcessRectBox(int32_t offsetTopLeftX,int32_t offsetTopLeftY,int32_t offsetBottomRightX,int32_t offsetBottomRightY,bool isNeedMirror,bool isNeedFlip)438 Rect MetadataOutput::ProcessRectBox(int32_t offsetTopLeftX, int32_t offsetTopLeftY,
439     int32_t offsetBottomRightX, int32_t offsetBottomRightY, bool isNeedMirror, bool isNeedFlip)
440 {
441     constexpr int32_t scale = 1000000;
442     double topLeftX = 0;
443     double topLeftY = 0;
444     double width = 0;
445     double height = 0;
446     if (isNeedMirror) {
447         topLeftX = scale - offsetBottomRightY;
448         topLeftY = scale - offsetBottomRightX;
449         width = offsetBottomRightY - offsetTopLeftY;
450         height = offsetBottomRightX - offsetTopLeftX;
451     } else if (isNeedFlip) {
452         topLeftX = offsetTopLeftY;
453         topLeftY = offsetTopLeftX;
454         width = offsetBottomRightY - offsetTopLeftY;
455         height = offsetBottomRightX - offsetTopLeftX;
456     } else {
457         topLeftX = scale - offsetBottomRightY;
458         topLeftY = offsetTopLeftX;
459         width = offsetBottomRightY - offsetTopLeftY;
460         height = offsetBottomRightX - offsetTopLeftX;
461     }
462     topLeftX = topLeftX < 0 ? 0 : topLeftX;
463     topLeftX = topLeftX > scale ? scale : topLeftX;
464     topLeftY = topLeftY < 0 ? 0 : topLeftY;
465     topLeftY = topLeftY > scale ? scale : topLeftY;
466 
467     return (Rect){ topLeftX / scale, topLeftY / scale, width / scale, height / scale};
468 }
469 
ProcessBaseInfo(sptr<MetadataObjectFactory> factoryPtr,const camera_metadata_item_t & metadataItem,int32_t & index,MetadataObjectType typeFromHal,bool isNeedMirror,bool isNeedFlip)470 void MetadataOutput::ProcessBaseInfo(sptr<MetadataObjectFactory> factoryPtr, const camera_metadata_item_t &metadataItem,
471                                      int32_t &index, MetadataObjectType typeFromHal, bool isNeedMirror, bool isNeedFlip)
472 {
473     const int32_t rectLength = 4;
474     const int32_t offsetOne = 1;
475     const int32_t offsetTwo = 2;
476     const int32_t offsetThree = 3;
477     factoryPtr->SetType(typeFromHal)
478         ->SetObjectId(metadataItem.data.i32[index]);
479     index++;
480     factoryPtr->SetTimestamp(metadataItem.data.i32[index]);
481     index++;
482     factoryPtr->SetBox(ProcessRectBox(metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
483                                       metadataItem.data.i32[index + offsetTwo],
484                                       metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
485     index += rectLength;
486     factoryPtr->SetConfidence(metadataItem.data.i32[index]);
487     index++;
488     int32_t externalLength = metadataItem.data.i32[index];
489     index++;
490     MEDIA_DEBUG_LOG("MetadataOutput::GenerateObjects, type: %{public}d, externalLength: %{public}d", typeFromHal,
491                     externalLength);
492 }
493 
ProcessExternInfo(sptr<MetadataObjectFactory> factoryPtr,const camera_metadata_item_t & metadataItem,int32_t & index,MetadataObjectType typeFromHal,bool isNeedMirror,bool isNeedFlip)494 void MetadataOutput::ProcessExternInfo(sptr<MetadataObjectFactory> factoryPtr,
495                                        const camera_metadata_item_t &metadataItem, int32_t &index,
496                                        MetadataObjectType typeFromHal, bool isNeedMirror, bool isNeedFlip)
497 {
498     switch (typeFromHal) {
499         case MetadataObjectType::FACE:
500             ProcessHumanFaceDetectInfo(factoryPtr, metadataItem, index, isNeedMirror, isNeedFlip);
501             break;
502         case MetadataObjectType::CAT_FACE:
503             ProcessCatFaceDetectInfo(factoryPtr, metadataItem, index, isNeedMirror, isNeedFlip);
504             break;
505         case MetadataObjectType::DOG_FACE:
506             ProcessDogFaceDetectInfo(factoryPtr, metadataItem, index, isNeedMirror, isNeedFlip);
507             break;
508         default:
509             break;
510     }
511 }
512 
ProcessHumanFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,const camera_metadata_item_t & metadataItem,int32_t & index,bool isNeedMirror,bool isNeedFlip)513 void MetadataOutput::ProcessHumanFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,
514                                                 const camera_metadata_item_t &metadataItem, int32_t &index,
515                                                 bool isNeedMirror, bool isNeedFlip)
516 {
517     int32_t version = metadataItem.data.i32[index++];
518     MEDIA_DEBUG_LOG("isNeedMirror: %{public}d, isNeedFlip: %{public}d, version: %{public}d",
519         isNeedMirror, isNeedFlip, version);
520     const int32_t rectLength = 4;
521     const int32_t offsetOne = 1;
522     const int32_t offsetTwo = 2;
523     const int32_t offsetThree = 3;
524     factoryPtr->SetLeftEyeBoundingBox(ProcessRectBox(
525         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
526         metadataItem.data.i32[index + offsetTwo],
527         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
528     index += rectLength;
529     factoryPtr->SetRightEyeBoundingBoxd(ProcessRectBox(
530         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
531         metadataItem.data.i32[index + offsetTwo],
532         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
533     index += rectLength;
534     factoryPtr->SetEmotion(static_cast<Emotion>(metadataItem.data.i32[index]));
535     index++;
536     factoryPtr->SetEmotionConfidence(static_cast<Emotion>(metadataItem.data.i32[index]));
537     index++;
538     factoryPtr->SetPitchAngle(metadataItem.data.i32[index]);
539     index++;
540     factoryPtr->SetYawAngle(metadataItem.data.i32[index]);
541     index++;
542     factoryPtr->SetRollAngle(metadataItem.data.i32[index]);
543     index++;
544 }
545 
ProcessCatFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,const camera_metadata_item_t & metadataItem,int32_t & index,bool isNeedMirror,bool isNeedFlip)546 void MetadataOutput::ProcessCatFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,
547                                               const camera_metadata_item_t &metadataItem, int32_t &index,
548                                               bool isNeedMirror, bool isNeedFlip)
549 {
550     int32_t version = metadataItem.data.i32[index++];
551     MEDIA_DEBUG_LOG("isNeedMirror: %{public}d, isNeedFlip: %{public}d, version: %{public}d",
552         isNeedMirror, isNeedFlip, version);
553     const int32_t rectLength = 4;
554     const int32_t offsetOne = 1;
555     const int32_t offsetTwo = 2;
556     const int32_t offsetThree = 3;
557     factoryPtr->SetLeftEyeBoundingBox(ProcessRectBox(
558         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
559         metadataItem.data.i32[index + offsetTwo],
560         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
561     index += rectLength;
562     factoryPtr->SetRightEyeBoundingBoxd(ProcessRectBox(
563         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
564         metadataItem.data.i32[index + offsetTwo],
565         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
566     index += rectLength;
567 }
568 
ProcessDogFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,const camera_metadata_item_t & metadataItem,int32_t & index,bool isNeedMirror,bool isNeedFlip)569 void MetadataOutput::ProcessDogFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr,
570                                               const camera_metadata_item_t &metadataItem, int32_t &index,
571                                               bool isNeedMirror, bool isNeedFlip)
572 {
573     int32_t version = metadataItem.data.i32[index++];
574     MEDIA_DEBUG_LOG("isNeedMirror: %{public}d, isNeedFlip: %{public}d, version: %{public}d",
575         isNeedMirror, isNeedFlip, version);
576     const int32_t rectLength = 4;
577     const int32_t offsetOne = 1;
578     const int32_t offsetTwo = 2;
579     const int32_t offsetThree = 3;
580     factoryPtr->SetLeftEyeBoundingBox(ProcessRectBox(
581         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
582         metadataItem.data.i32[index + offsetTwo],
583         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
584     index += rectLength;
585     factoryPtr->SetRightEyeBoundingBoxd(ProcessRectBox(
586         metadataItem.data.i32[index], metadataItem.data.i32[index + offsetOne],
587         metadataItem.data.i32[index + offsetTwo],
588         metadataItem.data.i32[index + offsetThree], isNeedMirror, isNeedFlip));
589     index += rectLength;
590 }
591 
MetadataObjectListener(sptr<MetadataOutput> metadata)592 MetadataObjectListener::MetadataObjectListener(sptr<MetadataOutput> metadata) : metadata_(metadata) {}
593 
ProcessMetadataBuffer(void * buffer,int64_t timestamp)594 int32_t MetadataObjectListener::ProcessMetadataBuffer(void *buffer, int64_t timestamp)
595 {
596     return CameraErrorCode::SUCCESS;
597 }
598 
OnBufferAvailable()599 void MetadataObjectListener::OnBufferAvailable()
600 {
601     MEDIA_INFO_LOG("MetadataOutput::OnBufferAvailable() is Called");
602     // metaoutput adapte later
603     bool adapterLater = true;
604     CHECK_ERROR_RETURN(adapterLater);
605     auto metadataOutput = metadata_.promote();
606     CHECK_ERROR_RETURN_LOG(metadataOutput == nullptr, "OnBufferAvailable metadataOutput is null");
607     auto surface = metadataOutput->GetSurface();
608     CHECK_ERROR_RETURN_LOG(surface == nullptr, "OnBufferAvailable Metadata surface is null");
609     int32_t fence = -1;
610     int64_t timestamp;
611     OHOS::Rect damage;
612     sptr<SurfaceBuffer> buffer = nullptr;
613     SurfaceError surfaceRet = surface->AcquireBuffer(buffer, fence, timestamp, damage);
614     CHECK_ERROR_RETURN_LOG(surfaceRet != SURFACE_ERROR_OK, "OnBufferAvailable Failed to acquire surface buffer");
615     int32_t ret = ProcessMetadataBuffer(buffer->GetVirAddr(), timestamp);
616     if (ret) {
617         std::shared_ptr<MetadataStateCallback> appStateCallback = metadataOutput->GetAppStateCallback();
618         if (appStateCallback) {
619             appStateCallback->OnError(ret);
620         }
621     }
622     surface->ReleaseBuffer(buffer, -1);
623 }
624 
OnMetadataResult(const int32_t streamId,const std::shared_ptr<OHOS::Camera::CameraMetadata> & result)625 int32_t HStreamMetadataCallbackImpl::OnMetadataResult(const int32_t streamId,
626                                                       const std::shared_ptr<OHOS::Camera::CameraMetadata> &result)
627 {
628     auto metadataOutput = GetMetadataOutput();
629     CHECK_ERROR_RETURN_RET_LOG(metadataOutput == nullptr, CAMERA_OK,
630                                "HStreamMetadataCallbackImpl::OnMetadataResult metadataOutput is nullptr");
631     auto session = metadataOutput->GetSession();
632     CHECK_ERROR_RETURN_RET_LOG(session == nullptr, SESSION_NOT_RUNNING,
633                                "HStreamMetadataCallbackImpl OnMetadataResult error!, session is nullptr");
634     auto inputDevice = session->GetInputDevice();
635     bool isNeedMirror = false;
636     bool isNeedFlip = false;
637     if (inputDevice) {
638         auto inputDeviceInfo = inputDevice->GetCameraDeviceInfo();
639         if (inputDeviceInfo) {
640             isNeedMirror = (inputDeviceInfo->GetPosition() == CAMERA_POSITION_FRONT ||
641                             inputDeviceInfo->GetPosition() == CAMERA_POSITION_FOLD_INNER);
642             isNeedFlip = inputDeviceInfo->GetUsedAsPosition() == CAMERA_POSITION_FRONT;
643         }
644     }
645     std::vector<sptr<MetadataObject>> metaObjects;
646     metadataOutput->ProcessMetadata(streamId, result, metaObjects, isNeedMirror, isNeedFlip);
647     auto objectCallback = metadataOutput->GetAppObjectCallback();
648     CHECK_ERROR_RETURN_RET(objectCallback == nullptr, INVALID_ARGUMENT);
649     if ((metadataOutput->reportFaceResults_ || metadataOutput->reportLastFaceResults_) && objectCallback) {
650         objectCallback->OnMetadataObjectsAvailable(metaObjects);
651     }
652     return SUCCESS;
653 }
654 
MetadataObjectFactory()655 MetadataObjectFactory::MetadataObjectFactory() {}
656 
GetInstance()657 sptr<MetadataObjectFactory> &MetadataObjectFactory::GetInstance()
658 {
659     if (metaFactoryInstance_ == nullptr) {
660         std::lock_guard<std::mutex> lock(instanceMutex_);
661         if (metaFactoryInstance_ == nullptr) {
662             MEDIA_INFO_LOG("Initializing MetadataObjectFactory instance");
663             metaFactoryInstance_ = new MetadataObjectFactory();
664         }
665     }
666     return metaFactoryInstance_;
667 }
668 
createMetadataObject(MetadataObjectType type)669 sptr<MetadataObject> MetadataObjectFactory::createMetadataObject(MetadataObjectType type)
670 {
671     MetaObjectParms baseMetaParms = { type_, timestamp_, box_, objectId_, confidence_ };
672     sptr<MetadataObject> metadataObject;
673     switch (type) {
674         case MetadataObjectType::FACE:
675             metadataObject = new MetadataFaceObject(baseMetaParms, leftEyeBoundingBox_, rightEyeBoundingBox_, emotion_,
676                                                     emotionConfidence_, pitchAngle_, yawAngle_, rollAngle_);
677             break;
678         case MetadataObjectType::HUMAN_BODY:
679             metadataObject = new MetadataHumanBodyObject(baseMetaParms);
680             break;
681         case MetadataObjectType::CAT_FACE:
682             metadataObject = new MetadataCatFaceObject(baseMetaParms, leftEyeBoundingBox_, rightEyeBoundingBox_);
683             break;
684         case MetadataObjectType::CAT_BODY:
685             metadataObject = new MetadataCatBodyObject(baseMetaParms);
686             break;
687         case MetadataObjectType::DOG_FACE:
688             metadataObject = new MetadataDogFaceObject(baseMetaParms, leftEyeBoundingBox_, rightEyeBoundingBox_);
689             break;
690         case MetadataObjectType::DOG_BODY:
691             metadataObject = new MetadataDogBodyObject(baseMetaParms);
692             break;
693         case MetadataObjectType::SALIENT_DETECTION:
694             metadataObject = new MetadataSalientDetectionObject(baseMetaParms);
695             break;
696         case MetadataObjectType::BAR_CODE_DETECTION:
697             metadataObject = new MetadataBarCodeDetectionObject(baseMetaParms);
698             break;
699         default:
700             metadataObject = new MetadataObject(baseMetaParms);
701     }
702     ResetParameters();
703     return metadataObject;
704 }
705 
ResetParameters()706 void MetadataObjectFactory::ResetParameters()
707 {
708     type_ = MetadataObjectType::INVALID;
709     timestamp_ = 0;
710     box_ = { 0, 0, 0, 0 };
711     objectId_ = 0;
712     confidence_ = 0.0f;
713     leftEyeBoundingBox_ = { 0, 0, 0, 0 };
714     rightEyeBoundingBox_ = { 0, 0, 0, 0 };
715     emotion_ = Emotion::NEUTRAL;
716     emotionConfidence_ = 0;
717     pitchAngle_ = 0;
718     yawAngle_ = 0;
719     rollAngle_ = 0;
720 }
721 }  // namespace CameraStandard
722 }  // namespace OHOS
723