1 /* 2 * Copyright (c) 2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef OHOS_CAMERA_METADATA_OUTPUT_H 17 #define OHOS_CAMERA_METADATA_OUTPUT_H 18 19 #include <cstdint> 20 #include <iostream> 21 #include <mutex> 22 #include <sys/types.h> 23 24 #include "camera_metadata_info.h" 25 #include "camera_metadata_operator.h" 26 #include "capture_output.h" 27 #include "hstream_metadata_callback_stub.h" 28 #include "iconsumer_surface.h" 29 #include "istream_metadata.h" 30 31 #include "nocopyable.h" 32 #include "singleton.h" 33 #include "surface.h" 34 35 namespace OHOS { 36 namespace CameraStandard { 37 static const std::map<MetadataObjectType, int32_t> mapLengthOfType = { 38 {MetadataObjectType::FACE, 23}, 39 {MetadataObjectType::HUMAN_BODY, 9}, 40 {MetadataObjectType::CAT_FACE, 18}, 41 {MetadataObjectType::CAT_BODY, 9}, 42 {MetadataObjectType::DOG_FACE, 18}, 43 {MetadataObjectType::DOG_BODY, 9}, 44 {MetadataObjectType::SALIENT_DETECTION, 9}, 45 {MetadataObjectType::BAR_CODE_DETECTION, 9}, 46 {MetadataObjectType::BASE_FACE_DETECTION, 9}, 47 }; 48 49 enum MetadataOutputErrorCode : int32_t { 50 ERROR_UNKNOWN = 1, 51 ERROR_INSUFFICIENT_RESOURCES, 52 }; 53 54 enum Emotion : int32_t { NEUTRAL = 0, SADNESS, SMILE, SUPRISE }; 55 56 struct Rect { 57 double topLeftX; 58 double topLeftY; 59 double width; 60 double height; 61 }; 62 63 struct MetaObjectParms { 64 MetadataObjectType type; 65 int32_t timestamp; 66 Rect box; 67 int32_t objectId; 68 int32_t confidence; 69 }; 70 71 72 class MetadataObject : public RefBase { 73 public: 74 MetadataObject(const MetadataObjectType type, const int32_t timestamp, const Rect rect, const int32_t objectId, 75 const int32_t confidence); 76 MetadataObject(const MetaObjectParms& parms); 77 virtual ~MetadataObject() = default; GetType()78 inline MetadataObjectType GetType() 79 { 80 return type_; 81 }; GetTimestamp()82 inline int32_t GetTimestamp() 83 { 84 return timestamp_; 85 }; GetBoundingBox()86 inline Rect GetBoundingBox() 87 { 88 return box_; 89 }; GetObjectId()90 inline int32_t GetObjectId() 91 { 92 return objectId_; 93 }; GetConfidence()94 inline int32_t GetConfidence() 95 { 96 return confidence_; 97 }; 98 99 private: 100 MetadataObjectType type_; 101 int32_t timestamp_; 102 Rect box_; 103 int32_t objectId_; 104 int32_t confidence_; 105 }; 106 107 class MetadataFaceObject : public MetadataObject { 108 public: 109 MetadataFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox, 110 const Emotion emotion, const int32_t emotionConfidence, const int32_t pitchAngle, 111 const int32_t yawAngle, const int32_t rollAngle); 112 ~MetadataFaceObject() = default; GetLeftEyeBoundingBox()113 inline Rect GetLeftEyeBoundingBox() 114 { 115 return leftEyeBoundingBox_; 116 }; GetRightEyeBoundingBox()117 inline Rect GetRightEyeBoundingBox() 118 { 119 return rightEyeBoundingBox_; 120 }; GetEmotion()121 inline Emotion GetEmotion() 122 { 123 return emotion_; 124 }; GetEmotionConfidence()125 inline int32_t GetEmotionConfidence() 126 { 127 return emotionConfidence_; 128 }; GetPitchAngle()129 inline int32_t GetPitchAngle() 130 { 131 return pitchAngle_; 132 }; GetYawAngle()133 inline int32_t GetYawAngle() 134 { 135 return yawAngle_; 136 }; GetRollAngle()137 inline int32_t GetRollAngle() 138 { 139 return rollAngle_; 140 }; 141 142 private: 143 Rect leftEyeBoundingBox_; 144 Rect rightEyeBoundingBox_; 145 Emotion emotion_; 146 int32_t emotionConfidence_; 147 int32_t pitchAngle_; 148 int32_t yawAngle_; 149 int32_t rollAngle_; 150 }; 151 152 class MetadataHumanBodyObject : public MetadataObject { 153 public: 154 MetadataHumanBodyObject(const MetaObjectParms& parms); 155 ~MetadataHumanBodyObject() = default; 156 }; 157 158 class MetadataCatFaceObject : public MetadataObject { 159 public: 160 MetadataCatFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox); 161 ~MetadataCatFaceObject() = default; GetLeftEyeBoundingBox()162 inline Rect GetLeftEyeBoundingBox() 163 { 164 return leftEyeBoundingBox_; 165 }; GetRightEyeBoundingBox()166 inline Rect GetRightEyeBoundingBox() 167 { 168 return rightEyeBoundingBox_; 169 }; 170 171 private: 172 Rect leftEyeBoundingBox_; 173 Rect rightEyeBoundingBox_; 174 }; 175 176 class MetadataCatBodyObject : public MetadataObject { 177 public: 178 MetadataCatBodyObject(const MetaObjectParms& parms); 179 ~MetadataCatBodyObject() = default; 180 }; 181 182 class MetadataDogFaceObject : public MetadataObject { 183 public: 184 MetadataDogFaceObject(const MetaObjectParms& parms, const Rect leftEyeBoundingBox, const Rect rightEyeBoundingBox); 185 ~MetadataDogFaceObject() = default; GetLeftEyeBoundingBox()186 inline Rect GetLeftEyeBoundingBox() 187 { 188 return leftEyeBoundingBox_; 189 }; GetRightEyeBoundingBox()190 inline Rect GetRightEyeBoundingBox() 191 { 192 return rightEyeBoundingBox_; 193 }; 194 195 private: 196 Rect leftEyeBoundingBox_; 197 Rect rightEyeBoundingBox_; 198 }; 199 200 class MetadataDogBodyObject : public MetadataObject { 201 public: 202 MetadataDogBodyObject(const MetaObjectParms& parms); 203 ~MetadataDogBodyObject() = default; 204 }; 205 206 class MetadataSalientDetectionObject : public MetadataObject { 207 public: 208 MetadataSalientDetectionObject(const MetaObjectParms& parms); 209 ~MetadataSalientDetectionObject() = default; 210 }; 211 212 class MetadataBarCodeDetectionObject : public MetadataObject { 213 public: 214 MetadataBarCodeDetectionObject(const MetaObjectParms& parms); 215 ~MetadataBarCodeDetectionObject() = default; 216 }; 217 218 class MetadataObjectFactory : public RefBase { 219 public: 220 virtual ~MetadataObjectFactory() = default; 221 222 static sptr<MetadataObjectFactory> &GetInstance(); SetType(MetadataObjectType type)223 inline sptr<MetadataObjectFactory> SetType(MetadataObjectType type) 224 { 225 type_ = type; 226 return this; 227 } SetTimestamp(int32_t timestamp)228 inline sptr<MetadataObjectFactory> SetTimestamp(int32_t timestamp) 229 { 230 timestamp_ = timestamp; 231 return this; 232 } SetBox(Rect box)233 inline sptr<MetadataObjectFactory> SetBox(Rect box) 234 { 235 box_ = box; 236 return this; 237 } SetObjectId(int32_t objectId)238 inline sptr<MetadataObjectFactory> SetObjectId(int32_t objectId) 239 { 240 objectId_ = objectId; 241 return this; 242 } SetConfidence(int32_t confidence)243 inline sptr<MetadataObjectFactory> SetConfidence(int32_t confidence) 244 { 245 confidence_ = confidence; 246 return this; 247 } SetLeftEyeBoundingBox(Rect leftEyeBoundingBox)248 inline sptr<MetadataObjectFactory> SetLeftEyeBoundingBox(Rect leftEyeBoundingBox) 249 { 250 leftEyeBoundingBox_ = leftEyeBoundingBox; 251 return this; 252 } SetRightEyeBoundingBoxd(Rect rightEyeBoundingBox)253 inline sptr<MetadataObjectFactory> SetRightEyeBoundingBoxd(Rect rightEyeBoundingBox) 254 { 255 rightEyeBoundingBox_ = rightEyeBoundingBox; 256 return this; 257 } SetEmotion(Emotion emotion)258 inline sptr<MetadataObjectFactory> SetEmotion(Emotion emotion) 259 { 260 emotion_ = emotion; 261 return this; 262 } SetEmotionConfidence(int32_t emotionConfidence)263 inline sptr<MetadataObjectFactory> SetEmotionConfidence(int32_t emotionConfidence) 264 { 265 emotionConfidence_ = emotionConfidence; 266 return this; 267 } SetPitchAngle(int32_t pitchAngle)268 inline sptr<MetadataObjectFactory> SetPitchAngle(int32_t pitchAngle) 269 { 270 pitchAngle_ = pitchAngle; 271 return this; 272 } SetYawAngle(int32_t yawAngle)273 inline sptr<MetadataObjectFactory> SetYawAngle(int32_t yawAngle) 274 { 275 yawAngle_ = yawAngle; 276 return this; 277 } SetRollAngle(int32_t rollAngle)278 inline sptr<MetadataObjectFactory> SetRollAngle(int32_t rollAngle) 279 { 280 rollAngle_ = rollAngle; 281 return this; 282 } 283 284 sptr<MetadataObject> createMetadataObject(MetadataObjectType type); 285 286 private: 287 MetadataObjectFactory(); 288 static sptr<MetadataObjectFactory> metaFactoryInstance_; 289 static std::mutex instanceMutex_; 290 // Parameters of metadataObject 291 MetadataObjectType type_ = MetadataObjectType::INVALID; 292 int32_t timestamp_ = 0; 293 Rect box_ = {0.0, 0.0, 0.0, 0.0}; 294 int32_t objectId_ = 0; 295 int32_t confidence_ = 0; 296 // Parameters of All face metadata 297 Rect leftEyeBoundingBox_ = {0.0, 0.0, 0.0, 0.0}; 298 Rect rightEyeBoundingBox_ = {0.0, 0.0, 0.0, 0.0}; 299 // Parameters of human face metadata 300 Emotion emotion_ = NEUTRAL; 301 int32_t emotionConfidence_ = 0; 302 int32_t pitchAngle_ = 0; 303 int32_t yawAngle_ = 0; 304 int32_t rollAngle_ = 0; 305 306 void ResetParameters(); 307 }; 308 309 class MetadataObjectCallback { 310 public: 311 MetadataObjectCallback() = default; 312 virtual ~MetadataObjectCallback() = default; 313 virtual void OnMetadataObjectsAvailable(std::vector<sptr<MetadataObject>> metaObjects) const = 0; 314 }; 315 316 class MetadataStateCallback { 317 public: 318 MetadataStateCallback() = default; 319 virtual ~MetadataStateCallback() = default; 320 virtual void OnError(int32_t errorCode) const = 0; 321 }; 322 323 class MetadataOutput : public CaptureOutput { 324 public: 325 MetadataOutput(sptr<IConsumerSurface> surface, sptr<IStreamMetadata> &streamMetadata); 326 ~MetadataOutput(); 327 328 /** 329 * @brief Get the supported metadata object types. 330 * 331 * @return Returns vector of MetadataObjectType. 332 */ 333 std::vector<MetadataObjectType> GetSupportedMetadataObjectTypes(); 334 335 /** 336 * @brief Set the metadata object types 337 * 338 * @param Vector of MetadataObjectType 339 */ 340 void SetCapturingMetadataObjectTypes(std::vector<MetadataObjectType> objectTypes); 341 342 /** 343 * @brief Add the metadata object types 344 * 345 * @param Vector of MetadataObjectType 346 */ 347 int32_t AddMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes); 348 349 /** 350 * @brief Remove the metadata object types 351 * 352 * @param Vector of MetadataObjectType 353 */ 354 int32_t RemoveMetadataObjectTypes(std::vector<MetadataObjectType> metadataObjectTypes); 355 356 /** 357 * @brief Set the metadata object callback for the metadata output. 358 * 359 * @param MetadataObjectCallback pointer to be triggered. 360 */ 361 void SetCallback(std::shared_ptr<MetadataObjectCallback> metadataObjectCallback); 362 363 /** 364 * @brief Set the metadata state callback for the metadata output. 365 * 366 * @param MetadataStateCallback pointer to be triggered. 367 */ 368 void SetCallback(std::shared_ptr<MetadataStateCallback> metadataStateCallback); 369 370 int32_t CreateStream() override; 371 372 /** 373 * @brief Start the metadata capture. 374 */ 375 int32_t Start(); 376 377 /** 378 * @brief Stop the metadata capture. 379 */ 380 int32_t Stop(); 381 382 /** 383 * @brief Releases a instance of the MetadataOutput. 384 */ 385 int32_t Release() override; 386 bool reportFaceResults_ = false; 387 bool reportLastFaceResults_ = false; 388 void ProcessMetadata(const int32_t streamId, const std::shared_ptr<OHOS::Camera::CameraMetadata>& result, 389 std::vector<sptr<MetadataObject>>& metaObjects, bool isNeedMirror, bool isNeedFlip); 390 int32_t ProcessMetaObjects(const int32_t streamId, std::vector<sptr<MetadataObject>> &metaObjects, 391 const std::vector<camera_metadata_item_t>& metadataItem, 392 const std::vector<uint32_t>& metadataTypes, bool isNeedMirror, bool isNeedFlip); 393 std::shared_ptr<MetadataObjectCallback> GetAppObjectCallback(); 394 std::shared_ptr<MetadataStateCallback> GetAppStateCallback(); 395 396 friend class MetadataObjectListener; 397 398 private: 399 void CameraServerDied(pid_t pid) override; 400 void ReleaseSurface(); 401 sptr<IConsumerSurface> GetSurface(); 402 bool checkValidType(const std::vector<MetadataObjectType>& typeAdded, 403 const std::vector<MetadataObjectType>& supportedType); 404 std::vector<int32_t> convert(const std::vector<MetadataObjectType>& typesOfMetadata); 405 void GetMetadataResults(const common_metadata_header_t *metadata, 406 std::vector<camera_metadata_item_t>& metadataResults, std::vector<uint32_t>& metadataTypes); 407 void GenerateObjects(const camera_metadata_item_t& metadataItem, MetadataObjectType type, 408 std::vector<sptr<MetadataObject>>& metaObjects, bool isNeedMirror, bool isNeedFlip); 409 Rect ProcessRectBox(int32_t offsetTopLeftX, int32_t offsetTopLeftY, 410 int32_t offsetBottomRightX, int32_t offsetBottomRightY, bool isNeedMirror, bool isNeedFlip); 411 void ProcessBaseInfo(sptr<MetadataObjectFactory> factoryPtr, const camera_metadata_item_t& metadataItem, 412 int32_t& index, MetadataObjectType typeFromHal, bool isNeedMirror, bool isNeedFlip); 413 void ProcessExternInfo(sptr<MetadataObjectFactory> factoryPtr, const camera_metadata_item_t& metadataItem, 414 int32_t& index, MetadataObjectType typeFromHal, bool isNeedMirror, bool isNeedFlip); 415 void ProcessHumanFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr, 416 const camera_metadata_item_t& metadataItem, int32_t& index, bool isNeedMirror, bool isNeedFlip); 417 void ProcessCatFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr, 418 const camera_metadata_item_t& metadataItem, int32_t& index, bool isNeedMirror, bool isNeedFlip); 419 void ProcessDogFaceDetectInfo(sptr<MetadataObjectFactory> factoryPtr, 420 const camera_metadata_item_t& metadataItem, int32_t& index, bool isNeedMirror, bool isNeedFlip); 421 422 std::mutex surfaceMutex_; 423 sptr<IConsumerSurface> surface_; 424 std::shared_ptr<MetadataObjectCallback> appObjectCallback_; 425 std::shared_ptr<MetadataStateCallback> appStateCallback_; 426 sptr<IStreamMetadataCallback> cameraMetadataCallback_; 427 std::vector<uint32_t> typesOfMetadata_ = { 428 OHOS_STATISTICS_DETECT_HUMAN_FACE_INFOS, 429 OHOS_STATISTICS_DETECT_HUMAN_BODY_INFOS, 430 OHOS_STATISTICS_DETECT_CAT_FACE_INFOS, 431 OHOS_STATISTICS_DETECT_CAT_BODY_INFOS, 432 OHOS_STATISTICS_DETECT_DOG_FACE_INFOS, 433 OHOS_STATISTICS_DETECT_DOG_BODY_INFOS, 434 OHOS_STATISTICS_DETECT_SALIENT_INFOS, 435 OHOS_STATISTICS_DETECT_BAR_CODE_INFOS, 436 OHOS_STATISTICS_DETECT_BASE_FACE_INFO}; 437 }; 438 439 class MetadataObjectListener : public IBufferConsumerListener { 440 public: 441 MetadataObjectListener(sptr<MetadataOutput> metadata); 442 void OnBufferAvailable() override; 443 444 private: 445 int32_t ProcessMetadataBuffer(void *buffer, int64_t timestamp); 446 wptr<MetadataOutput> metadata_; 447 }; 448 449 class HStreamMetadataCallbackImpl : public HStreamMetadataCallbackStub { 450 public: HStreamMetadataCallbackImpl(MetadataOutput * metaDataOutput)451 explicit HStreamMetadataCallbackImpl(MetadataOutput *metaDataOutput) : innerMetadataOutput(metaDataOutput) {} 452 453 ~HStreamMetadataCallbackImpl() = default; 454 455 int32_t OnMetadataResult(const int32_t streamId, 456 const std::shared_ptr<OHOS::Camera::CameraMetadata> &result) override; 457 GetMetadataOutput()458 inline sptr<MetadataOutput> GetMetadataOutput() 459 { 460 if (innerMetadataOutput == nullptr) { 461 return nullptr; 462 } 463 return innerMetadataOutput.promote(); 464 } 465 466 private: 467 wptr<MetadataOutput> innerMetadataOutput = nullptr; 468 }; 469 } // namespace CameraStandard 470 } // namespace OHOS 471 #endif // OHOS_CAMERA_METADATA_OUTPUT_H 472