1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef OHOS_CAMERA_H_CAPTURE_SESSION_H 17 #define OHOS_CAMERA_H_CAPTURE_SESSION_H 18 #define EXPORT_API __attribute__((visibility("default"))) 19 20 #include <atomic> 21 #include <cstdint> 22 #include <functional> 23 #include <iostream> 24 #include <list> 25 #include <memory> 26 #include <mutex> 27 #include <refbase.h> 28 #include <unordered_map> 29 #include <unordered_set> 30 #include "camera_util.h" 31 #include "camera_dynamic_loader.h" 32 #include "fixed_size_list.h" 33 #include "hcamera_device.h" 34 #include "hcapture_session_stub.h" 35 #include "hstream_capture.h" 36 #include "hstream_metadata.h" 37 #include "hstream_repeat.h" 38 #include "icapture_session.h" 39 #include "istream_common.h" 40 #include "camera_photo_proxy.h" 41 #include "moving_photo/moving_photo_surface_wrapper.h" 42 #include "surface.h" 43 #include "v1_0/istream_operator.h" 44 #include "v1_1/istream_operator.h" 45 #include "v1_2/istream_operator.h" 46 #include "v1_3/istream_operator_callback.h" 47 #include "hcamera_restore_param.h" 48 #include "iconsumer_surface.h" 49 #include "blocking_queue.h" 50 #include "audio_capturer.h" 51 #include "audio_info.h" 52 #include "avcodec_task_manager.h" 53 #include "moving_photo_video_cache.h" 54 #include "drain_manager.h" 55 #include "audio_capturer_session.h" 56 #include "safe_map.h" 57 #ifdef CAMERA_USE_SENSOR 58 #include "sensor_agent.h" 59 #include "sensor_agent_type.h" 60 #endif 61 namespace OHOS::Media { 62 class Picture; 63 } 64 namespace OHOS { 65 namespace CameraStandard { 66 using OHOS::HDI::Camera::V1_0::CaptureEndedInfo; 67 using OHOS::HDI::Camera::V1_0::CaptureErrorInfo; 68 using namespace AudioStandard; 69 using namespace std::chrono; 70 using namespace DeferredProcessing; 71 using namespace Media; 72 class PermissionStatusChangeCb; 73 class CameraUseStateChangeCb; 74 class DisplayRotationListener; 75 class CameraServerPhotoProxy; 76 77 static const int32_t STREAM_NOT_FOUNT = -1; 78 79 enum class CaptureSessionReleaseType : int32_t { 80 RELEASE_TYPE_CLIENT = 0, 81 RELEASE_TYPE_CLIENT_DIED, 82 RELEASE_TYPE_SECURE, 83 RELEASE_TYPE_OBJ_DIED, 84 }; 85 86 class StateMachine { 87 public: 88 explicit StateMachine(); 89 virtual ~StateMachine() = default; 90 bool CheckTransfer(CaptureSessionState targetState); 91 bool Transfer(CaptureSessionState targetState); 92 GetCurrentState()93 inline CaptureSessionState GetCurrentState() 94 { 95 std::lock_guard<std::recursive_mutex> lock(sessionStateLock_); 96 return currentState_; 97 } 98 StateGuard(const std::function<void (const CaptureSessionState)> & fun)99 inline void StateGuard(const std::function<void(const CaptureSessionState)>& fun) 100 { 101 std::lock_guard<std::recursive_mutex> lock(sessionStateLock_); 102 fun(currentState_); 103 } 104 IsStateNoLock(CaptureSessionState targetState)105 inline bool IsStateNoLock(CaptureSessionState targetState) 106 { 107 return currentState_ == targetState; 108 } 109 110 private: 111 std::vector<CaptureSessionState> stateTransferMap_[static_cast<uint32_t>(CaptureSessionState::SESSION_STATE_MAX)]; 112 std::recursive_mutex sessionStateLock_; 113 CaptureSessionState currentState_ = CaptureSessionState::SESSION_INIT; 114 }; 115 116 class StreamContainer { 117 public: StreamContainer()118 StreamContainer() {}; 119 virtual ~StreamContainer() = default; 120 121 bool AddStream(sptr<HStreamCommon> stream); 122 bool RemoveStream(sptr<HStreamCommon> stream); 123 sptr<HStreamCommon> GetStream(int32_t streamId); 124 sptr<HStreamCommon> GetHdiStream(int32_t streamId); 125 void Clear(); 126 size_t Size(); 127 128 std::list<sptr<HStreamCommon>> GetStreams(const StreamType streamType); 129 std::list<sptr<HStreamCommon>> GetAllStreams(); 130 131 private: 132 std::mutex streamsLock_; 133 std::map<const StreamType, std::list<sptr<HStreamCommon>>> streams_; 134 }; 135 136 class StreamOperatorCallback : public OHOS::HDI::Camera::V1_3::IStreamOperatorCallback { 137 public: 138 StreamOperatorCallback() = default; 139 virtual ~StreamOperatorCallback() = default; 140 141 int32_t OnCaptureStarted(int32_t captureId, const std::vector<int32_t>& streamIds) override; 142 int32_t OnCaptureStarted_V1_2( 143 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_2::CaptureStartedInfo>& infos) override; 144 int32_t OnCaptureEnded(int32_t captureId, const std::vector<CaptureEndedInfo>& infos) override; 145 int32_t OnCaptureEndedExt( 146 int32_t captureId, const std::vector<OHOS::HDI::Camera::V1_3::CaptureEndedInfoExt>& infos) override; 147 int32_t OnCaptureError(int32_t captureId, const std::vector<CaptureErrorInfo>& infos) override; 148 int32_t OnFrameShutter(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 149 int32_t OnFrameShutterEnd(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 150 int32_t OnCaptureReady(int32_t captureId, const std::vector<int32_t>& streamIds, uint64_t timestamp) override; 151 int32_t OnResult(int32_t streamId, const std::vector<uint8_t>& result) override; 152 153 virtual const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId) = 0; 154 virtual const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId) = 0; 155 virtual void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format, int32_t captureId) = 0; 156 157 private: 158 std::mutex cbMutex_; 159 }; 160 161 class SessionDrainImageCallback; 162 using MetaElementType = std::pair<int64_t, sptr<SurfaceBuffer>>; 163 class MovingPhotoListener : public MovingPhotoSurfaceWrapper::SurfaceBufferListener { 164 public: 165 explicit MovingPhotoListener(sptr<MovingPhotoSurfaceWrapper> surfaceWrapper, sptr<Surface> metaSurface, 166 shared_ptr<FixedSizeList<MetaElementType>> metaCache, uint32_t preCacheFrameCount, 167 uint32_t postCacheFrameCount); 168 ~MovingPhotoListener() override; 169 void OnBufferArrival(sptr<SurfaceBuffer> buffer, int64_t timestamp, GraphicTransformType transform) override; 170 void DrainOutImage(sptr<SessionDrainImageCallback> drainImageCallback); 171 void RemoveDrainImageManager(sptr<SessionDrainImageCallback> drainImageCallback); 172 void StopDrainOut(); 173 void ClearCache(uint64_t timestamp); 174 void SetClearFlag(); 175 176 private: 177 sptr<MovingPhotoSurfaceWrapper> movingPhotoSurfaceWrapper_; 178 sptr<Surface> metaSurface_; 179 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 180 BlockingQueue<sptr<FrameRecord>> recorderBufferQueue_; 181 SafeMap<sptr<SessionDrainImageCallback>, sptr<DrainImageManager>> callbackMap_; 182 std::atomic<bool> isNeededClear_ { false }; 183 std::atomic<bool> isNeededPop_ { false }; 184 int64_t shutterTime_; 185 uint64_t postCacheFrameCount_; 186 }; 187 188 class MovingPhotoMetaListener : public IBufferConsumerListener { 189 public: 190 explicit MovingPhotoMetaListener(sptr<Surface> surface, shared_ptr<FixedSizeList<MetaElementType>> metaCache); 191 ~MovingPhotoMetaListener(); 192 void OnBufferAvailable() override; 193 private: 194 sptr<Surface> surface_; 195 shared_ptr<FixedSizeList<MetaElementType>> metaCache_; 196 }; 197 198 class SessionDrainImageCallback : public DrainImageCallback { 199 public: 200 explicit SessionDrainImageCallback(std::vector<sptr<FrameRecord>>& frameCacheList, 201 wptr<MovingPhotoListener> listener, 202 wptr<MovingPhotoVideoCache> cache, 203 uint64_t timestamp, 204 int32_t rotation, 205 int32_t captureId); 206 ~SessionDrainImageCallback(); 207 void OnDrainImage(sptr<FrameRecord> frame) override; 208 void OnDrainImageFinish(bool isFinished) override; 209 210 private: 211 std::mutex mutex_; 212 std::vector<sptr<FrameRecord>> frameCacheList_; 213 wptr<MovingPhotoListener> listener_; 214 wptr<MovingPhotoVideoCache> videoCache_; 215 uint64_t timestamp_; 216 int32_t rotation_; 217 int32_t captureId_; 218 }; 219 220 class CameraInfoDumper; 221 222 class EXPORT_API HCaptureSession : public HCaptureSessionStub, public StreamOperatorCallback { 223 public: 224 static sptr<HCaptureSession> NewInstance(const uint32_t callerToken, int32_t opMode); 225 HCaptureSession(); 226 explicit HCaptureSession(const uint32_t callingTokenId, int32_t opMode); 227 virtual ~HCaptureSession(); 228 229 int32_t BeginConfig() override; 230 int32_t CommitConfig() override; 231 232 int32_t CanAddInput(sptr<ICameraDeviceService> cameraDevice, bool& result) override; 233 int32_t AddInput(sptr<ICameraDeviceService> cameraDevice) override; 234 int32_t AddOutput(StreamType streamType, sptr<IStreamCommon> stream) override; 235 236 int32_t RemoveInput(sptr<ICameraDeviceService> cameraDevice) override; 237 int32_t RemoveOutput(StreamType streamType, sptr<IStreamCommon> stream) override; 238 239 int32_t Start() override; 240 int32_t Stop() override; 241 int32_t Release() override; 242 int32_t Release(CaptureSessionReleaseType type); 243 244 static void DestroyStubObjectForPid(pid_t pid); 245 int32_t SetCallback(sptr<ICaptureSessionCallback>& callback) override; 246 247 int32_t GetSessionState(CaptureSessionState& sessionState) override; 248 int32_t GetActiveColorSpace(ColorSpace& colorSpace) override; 249 int32_t SetColorSpace(ColorSpace colorSpace, ColorSpace captureColorSpace, bool isNeedUpdate) override; 250 bool QueryFpsAndZoomRatio(float& currentFps, float& currentZoomRatio); 251 bool QueryZoomPerformance(std::vector<float>& crossZoomAndTime, int32_t operationMode); 252 int32_t SetSmoothZoom( 253 int32_t smoothZoomType, int32_t operationMode, float targetZoomRatio, float& duration) override; 254 int32_t EnableMovingPhoto(bool isEnable) override; 255 pid_t GetPid(); 256 int32_t GetCurrentStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 257 int32_t GetopMode(); 258 259 int32_t OperatePermissionCheck(uint32_t interfaceCode) override; 260 int32_t EnableMovingPhotoMirror(bool isMirror, bool isConfig) override; 261 int32_t CreateMediaLibrary(sptr<CameraPhotoProxy>& photoProxy, 262 std::string& uri, int32_t& cameraShotType, std::string& burstKey, int64_t timestamp) override; 263 int32_t CreateMediaLibrary(std::unique_ptr<Media::Picture> picture, sptr<CameraPhotoProxy> &photoProxy, 264 std::string &uri, int32_t &cameraShotType, std::string& burstKey, int64_t timestamp) override; 265 void SetCameraPhotoProxyInfo(sptr<CameraServerPhotoProxy> cameraPhotoProxy, int32_t &cameraShotType, 266 bool &isBursting, std::string &burstKey); 267 const sptr<HStreamCommon> GetStreamByStreamID(int32_t streamId) override; 268 int32_t SetFeatureMode(int32_t featureMode) override; 269 const sptr<HStreamCommon> GetHdiStreamByStreamID(int32_t streamId) override; 270 void StartMovingPhotoEncode(int32_t rotation, uint64_t timestamp, int32_t format, int32_t captureId) override; 271 void StartRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 272 void GetOutputStatus(int32_t &status); 273 int32_t SetPreviewRotation(std::string &deviceClass) override; 274 275 void DumpSessionInfo(CameraInfoDumper& infoDumper); 276 static void DumpSessions(CameraInfoDumper& infoDumper); 277 static void DumpCameraSessionSummary(CameraInfoDumper& infoDumper); 278 void ReleaseStreams(); 279 void StopMovingPhoto(); 280 281 static void OpenMediaLib(); 282 static void DelayCloseMediaLib(); 283 static std::optional<uint32_t> closeTimerId_; 284 static std::mutex g_mediaTaskLock_; 285 std::atomic<bool> isNeedMediaLib = false; 286 uint32_t preCacheFrameCount_ = CACHE_FRAME_COUNT; 287 uint32_t postCacheFrameCount_ = CACHE_FRAME_COUNT; 288 289 private: 290 int32_t Initialize(const uint32_t callerToken, int32_t opMode); 291 string lastDisplayName_ = ""; 292 string lastBurstPrefix_ = ""; 293 int32_t saveIndex = 0; 294 volatile bool isMovingPhotoMirror_ = false; 295 volatile bool isSetMotionPhoto_ = false; 296 std::mutex livePhotoStreamLock_; // Guard livePhotoStreamRepeat_ 297 sptr<HStreamRepeat> livePhotoStreamRepeat_; SetCameraDevice(sptr<HCameraDevice> device)298 inline void SetCameraDevice(sptr<HCameraDevice> device) 299 { 300 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 301 cameraDevice_ = device; 302 } 303 GetCameraDevice()304 inline const sptr<HCameraDevice> GetCameraDevice() 305 { 306 std::lock_guard<std::mutex> lock(cameraDeviceLock_); 307 return cameraDevice_; 308 } 309 string CreateDisplayName(const std::string& suffix); 310 string CreateBurstDisplayName(int32_t imageSeqId, int32_t seqId); 311 int32_t ValidateSessionInputs(); 312 int32_t ValidateSessionOutputs(); 313 int32_t ValidateSession(); 314 int32_t AddOutputStream(sptr<HStreamCommon> stream); 315 int32_t RemoveOutputStream(sptr<HStreamCommon> stream); 316 int32_t LinkInputAndOutputs(); 317 int32_t UnlinkInputAndOutputs(); 318 319 void ClearSketchRepeatStream(); 320 void ExpandSketchRepeatStream(); 321 void ExpandMovingPhotoRepeatStream(); 322 void ClearMovingPhotoRepeatStream(); 323 int32_t CreateMovingPhotoStreamRepeat(int32_t format, int32_t width, int32_t height, 324 sptr<OHOS::IBufferProducer> producer); 325 int32_t CheckIfColorSpaceMatchesFormat(ColorSpace colorSpace); 326 void CancelStreamsAndGetStreamInfos(std::vector<StreamInfo_V1_1>& streamInfos); 327 void RestartStreams(); 328 int32_t UpdateStreamInfos(); 329 void SetColorSpaceForStreams(); 330 331 void ProcessMetaZoomArray(std::vector<uint32_t>& zoomAndTimeArray, sptr<HCameraDevice>& cameraDevice); 332 void StartMovingPhotoStream(); 333 bool InitAudioCapture(); 334 bool StartAudioCapture(); 335 void ProcessAudioBuffer(); 336 void StartOnceRecord(uint64_t timestamp, int32_t rotation, int32_t captureId); 337 int32_t StartPreviewStream(const std::shared_ptr<OHOS::Camera::CameraMetadata>& settings, 338 camera_position_enum_t cameraPosition); 339 void UpdateMuteSetting(bool muteMode, std::shared_ptr<OHOS::Camera::CameraMetadata> &settings); 340 int32_t GetSensorOritation(); 341 int32_t GetMovingPhotoBufferDuration(); 342 void GetMovingPhotoStartAndEndTime(); 343 void StartMovingPhoto(sptr<HStreamRepeat>& curStreamRepeat); 344 345 std::string GetSessionState(); 346 void DynamicConfigStream(); 347 bool IsNeedDynamicConfig(); 348 void RegisterDisplayListener(sptr<HStreamRepeat> repeat); 349 void UnRegisterDisplayListener(sptr<HStreamRepeat> repeat); 350 StateMachine stateMachine_; 351 352 #ifdef CAMERA_USE_SENSOR 353 std::mutex sensorLock_; 354 bool isRegisterSensorSuccess_ = false; 355 void RegisterSensorCallback(); 356 void UnRegisterSensorCallback(); 357 static void GravityDataCallbackImpl(SensorEvent *event); 358 static int32_t CalcSensorRotation(int32_t sensorDegree); 359 static int32_t CalcRotationDegree(GravityData data); 360 #endif 361 // Make sure device thread safe,set device by {SetCameraDevice}, get device by {GetCameraDevice} 362 std::mutex cameraDeviceLock_; 363 sptr<HCameraDevice> cameraDevice_; 364 StreamContainer streamContainer_; 365 #ifdef CAMERA_USE_SENSOR 366 SensorUser user = {"", nullptr, nullptr}; 367 #endif 368 pid_t pid_; 369 uid_t uid_; 370 uint32_t callerToken_; 371 int32_t opMode_; 372 int32_t featureMode_; 373 ColorSpace currColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 374 ColorSpace currCaptureColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; 375 bool isSessionStarted_ = false; 376 bool enableStreamRotate_ = false; 377 bool isDynamicConfiged_ = false; 378 std::string deviceClass_ = "phone"; 379 std::mutex movingPhotoStatusLock_; // Guard movingPhotoStatus 380 sptr<MovingPhotoListener> livephotoListener_; 381 sptr<MovingPhotoMetaListener> livephotoMetaListener_; 382 sptr<AudioCapturerSession> audioCapturerSession_; 383 sptr<Surface> metaSurface_ = nullptr; 384 sptr<MovingPhotoVideoCache> videoCache_; 385 sptr<AvcodecTaskManager> taskManager_; 386 std::mutex displayListenerLock_; 387 sptr<DisplayRotationListener> displayListener_; 388 }; 389 } // namespace CameraStandard 390 } // namespace OHOS 391 #endif // OHOS_CAMERA_H_CAPTURE_SESSION_H 392