1 /*
2  * Copyright (c) 2024-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "photo_listener_impl.h"
17 
18 #include "securec.h"
19 #include "camera_log.h"
20 #include "video_key_info.h"
21 #include "output/photo_output.h"
22 #include "photo_native_impl.h"
23 #include "photo_output_impl.h"
24 #include "inner_api/native/camera/include/camera_photo_proxy.h"
25 #include "inner_api/native/camera/include/session/capture_session.h"
26 #include "media_photo_asset_proxy.h"
27 #include "userfile_manager_types.h"
28 #include "media_asset_helper.h"
29 
30 namespace OHOS {
31 namespace CameraStandard {
32 static std::mutex g_photoImageMutex;
33 
PhotoListener(Camera_PhotoOutput * photoOutput,sptr<Surface> surface)34 PhotoListener::PhotoListener(Camera_PhotoOutput* photoOutput, sptr<Surface> surface)
35     : photoOutput_(photoOutput), photoSurface_(surface)
36 {
37     if (bufferProcessor_ == nullptr && surface != nullptr) {
38         bufferProcessor_ = std::make_shared<PhotoBufferProcessor>(surface);
39     }
40 }
41 
~PhotoListener()42 PhotoListener::~PhotoListener()
43 {
44     photoSurface_ = nullptr;
45 }
46 
OnBufferAvailable()47 void PhotoListener::OnBufferAvailable()
48 {
49     MEDIA_INFO_LOG("PhotoListener::OnBufferAvailable");
50     std::lock_guard<std::mutex> lock(g_photoImageMutex);
51     CAMERA_SYNC_TRACE;
52     MEDIA_INFO_LOG("PhotoListener::OnBufferAvailable is called");
53     CHECK_AND_RETURN_LOG(photoSurface_ != nullptr, "photoSurface_ is null");
54 
55     int64_t timestamp;
56     OHOS::Rect damage;
57     sptr<SurfaceBuffer> surfaceBuffer = nullptr;
58     int32_t fence = -1;
59     SurfaceError surfaceRet = photoSurface_->AcquireBuffer(surfaceBuffer, fence, timestamp, damage);
60     CHECK_AND_RETURN_LOG(surfaceRet == SURFACE_ERROR_OK, "Failed to acquire surface buffer");
61 
62     CameraBufferExtraData extraData = GetCameraBufferExtraData(surfaceBuffer);
63 
64     if ((callbackFlag_ & CAPTURE_PHOTO_ASSET) != 0) {
65         MEDIA_DEBUG_LOG("PhotoListener on capture photo asset callback");
66         sptr<SurfaceBuffer> newSurfaceBuffer = SurfaceBuffer::Create();
67         DeepCopyBuffer(newSurfaceBuffer, surfaceBuffer);
68         photoSurface_->ReleaseBuffer(surfaceBuffer, -1);
69         CHECK_AND_RETURN_LOG(newSurfaceBuffer != nullptr, "deep copy buffer failed");
70 
71         ExecutePhotoAsset(newSurfaceBuffer, extraData, extraData.isDegradedImage == 0, timestamp);
72         MEDIA_DEBUG_LOG("PhotoListener on capture photo asset callback end");
73     } else if (extraData.isDegradedImage == 0 && (callbackFlag_ & CAPTURE_PHOTO) != 0) {
74         MEDIA_DEBUG_LOG("PhotoListener on capture photo callback");
75         ExecutePhoto(surfaceBuffer, timestamp);
76         photoSurface_->ReleaseBuffer(surfaceBuffer, -1);
77         MEDIA_DEBUG_LOG("PhotoListener on capture photo callback end");
78     } else {
79         MEDIA_INFO_LOG("PhotoListener on error callback");
80         photoSurface_->ReleaseBuffer(surfaceBuffer, -1);
81     }
82 }
83 
GetCameraBufferExtraData(const sptr<SurfaceBuffer> & surfaceBuffer)84 CameraBufferExtraData PhotoListener::GetCameraBufferExtraData(const sptr<SurfaceBuffer> &surfaceBuffer)
85 {
86     CameraBufferExtraData extraData;
87     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::captureId, extraData.captureId);
88     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::imageId, extraData.imageId);
89     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::deferredProcessingType, extraData.deferredProcessingType);
90     MEDIA_INFO_LOG("imageId:%{public}" PRId64
91                    ", deferredProcessingType:%{public}d",
92         extraData.imageId,
93         extraData.deferredProcessingType);
94 
95     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::dataWidth, extraData.photoWidth);
96     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::dataHeight, extraData.photoHeight);
97     uint64_t size = static_cast<uint64_t>(surfaceBuffer->GetSize());
98 
99     auto res = surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::dataSize, extraData.extraDataSize);
100     if (res != 0) {
101         MEDIA_INFO_LOG("ExtraGet dataSize error %{public}d", res);
102     } else if (extraData.extraDataSize <= 0) {
103         MEDIA_INFO_LOG("ExtraGet dataSize OK, but size <= 0");
104     } else if (static_cast<uint64_t>(extraData.extraDataSize) > size) {
105         MEDIA_INFO_LOG("ExtraGet dataSize OK, but dataSize %{public}d is bigger "
106                        "than bufferSize %{public}" PRIu64,
107             extraData.extraDataSize,
108             size);
109     } else {
110         MEDIA_INFO_LOG("ExtraGet dataSize %{public}d", extraData.extraDataSize);
111         size = static_cast<uint64_t>(extraData.extraDataSize);
112     }
113     extraData.size = size;
114 
115     res = surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::deferredImageFormat, extraData.deferredImageFormat);
116     MEDIA_INFO_LOG("deferredImageFormat:%{public}d, width:%{public}d, "
117                    "height:%{public}d, size:%{public}" PRId64,
118         extraData.deferredImageFormat,
119         extraData.photoWidth,
120         extraData.photoHeight,
121         size);
122 
123     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::isDegradedImage, extraData.isDegradedImage);
124     MEDIA_INFO_LOG("isDegradedImage:%{public}d", extraData.isDegradedImage);
125 
126     return extraData;
127 }
128 
SetCallbackFlag(uint8_t callbackFlag)129 void PhotoListener::SetCallbackFlag(uint8_t callbackFlag)
130 {
131     callbackFlag_ = callbackFlag;
132 }
133 
SetPhotoAvailableCallback(OH_PhotoOutput_PhotoAvailable callback)134 void PhotoListener::SetPhotoAvailableCallback(OH_PhotoOutput_PhotoAvailable callback)
135 {
136     MEDIA_DEBUG_LOG("PhotoListener::SetPhotoAvailableCallback");
137     std::lock_guard<std::mutex> lock(g_photoImageMutex);
138     MEDIA_INFO_LOG("PhotoListener::SetPhotoAvailableCallback is called");
139     if (callback != nullptr) {
140         photoCallback_ = callback;
141     }
142     return;
143 }
144 
UnregisterPhotoAvailableCallback(OH_PhotoOutput_PhotoAvailable callback)145 void PhotoListener::UnregisterPhotoAvailableCallback(OH_PhotoOutput_PhotoAvailable callback)
146 {
147     MEDIA_DEBUG_LOG("PhotoListener::UnregisterPhotoAvailableCallback");
148     std::lock_guard<std::mutex> lock(g_photoImageMutex);
149     MEDIA_INFO_LOG("PhotoListener::UnregisterPhotoAvailableCallback is called");
150     if (photoCallback_ != nullptr && callback != nullptr) {
151         photoCallback_ = nullptr;
152     }
153     return;
154 }
155 
SetPhotoAssetAvailableCallback(OH_PhotoOutput_PhotoAssetAvailable callback)156 void PhotoListener::SetPhotoAssetAvailableCallback(OH_PhotoOutput_PhotoAssetAvailable callback)
157 {
158     MEDIA_DEBUG_LOG("PhotoListener::SetPhotoAssetAvailableCallback");
159     std::lock_guard<std::mutex> lock(g_photoImageMutex);
160     MEDIA_INFO_LOG("PhotoListener::SetPhotoAssetAvailableCallback is called");
161     if (callback != nullptr) {
162         photoAssetCallback_ = callback;
163     }
164     return;
165 }
166 
UnregisterPhotoAssetAvailableCallback(OH_PhotoOutput_PhotoAssetAvailable callback)167 void PhotoListener::UnregisterPhotoAssetAvailableCallback(OH_PhotoOutput_PhotoAssetAvailable callback)
168 {
169     MEDIA_DEBUG_LOG("PhotoListener::UnregisterPhotoAssetAvailableCallback");
170     std::lock_guard<std::mutex> lock(g_photoImageMutex);
171     MEDIA_INFO_LOG("PhotoListener::UnregisterPhotoAssetAvailableCallback is called");
172     if (photoAssetCallback_ != nullptr && callback != nullptr) {
173         photoAssetCallback_ = nullptr;
174     }
175     return;
176 }
177 
ExecutePhoto(sptr<SurfaceBuffer> surfaceBuffer,int64_t timestamp)178 void PhotoListener::ExecutePhoto(sptr<SurfaceBuffer> surfaceBuffer, int64_t timestamp)
179 {
180     std::shared_ptr<Media::NativeImage> nativeImage = std::make_shared<Media::NativeImage>(surfaceBuffer,
181         bufferProcessor_, timestamp);
182     CHECK_AND_RETURN_LOG(nativeImage != nullptr, "Create native image failed");
183 
184     if (photoCallback_ != nullptr && photoOutput_ != nullptr) {
185         OH_PhotoNative *photoNative = photoOutput_->CreateCameraPhotoNative(nativeImage, true);
186         CHECK_AND_RETURN_LOG(photoNative != nullptr, "Create photo native failed");
187 
188         photoCallback_(photoOutput_, photoNative);
189     }
190     return;
191 }
192 
ExecutePhotoAsset(sptr<SurfaceBuffer> surfaceBuffer,CameraBufferExtraData extraData,bool isHighQuality,int64_t timestamp)193 void PhotoListener::ExecutePhotoAsset(sptr<SurfaceBuffer> surfaceBuffer, CameraBufferExtraData extraData,
194     bool isHighQuality, int64_t timestamp)
195 {
196     CAMERA_SYNC_TRACE;
197     BufferHandle* bufferHandle = surfaceBuffer->GetBufferHandle();
198     CHECK_AND_RETURN_LOG(bufferHandle != nullptr, "invalid bufferHandle");
199 
200     surfaceBuffer->Map();
201     std::string uri = "";
202     int32_t cameraShotType = 0;
203     std::string burstKey = "";
204     CreateMediaLibrary(surfaceBuffer, bufferHandle, extraData, isHighQuality,
205         uri, cameraShotType, burstKey, timestamp);
206     CHECK_AND_RETURN_LOG(!uri.empty(), "uri is empty");
207 
208     auto mediaAssetHelper = Media::MediaAssetHelperFactory::CreateMediaAssetHelper();
209     CHECK_AND_RETURN_LOG(mediaAssetHelper != nullptr, "create media asset helper failed");
210 
211     auto mediaAsset = mediaAssetHelper->GetMediaAsset(uri, cameraShotType, burstKey);
212     CHECK_AND_RETURN_LOG(mediaAsset != nullptr, "Create photo asset failed");
213 
214     if (photoAssetCallback_ != nullptr && photoOutput_ != nullptr) {
215         photoAssetCallback_(photoOutput_, mediaAsset);
216     }
217     return;
218 }
219 
DeepCopyBuffer(sptr<SurfaceBuffer> newSurfaceBuffer,sptr<SurfaceBuffer> surfaceBuffer)220 void PhotoListener::DeepCopyBuffer(sptr<SurfaceBuffer> newSurfaceBuffer, sptr<SurfaceBuffer> surfaceBuffer)
221 {
222     CAMERA_SYNC_TRACE;
223     BufferRequestConfig requestConfig = {
224         .width = surfaceBuffer->GetWidth(),
225         .height = surfaceBuffer->GetHeight(),
226         .strideAlignment = 0x8, // default stride is 8 Bytes.
227         .format = surfaceBuffer->GetFormat(),
228         .usage = surfaceBuffer->GetUsage(),
229         .timeout = 0,
230         .colorGamut = surfaceBuffer->GetSurfaceBufferColorGamut(),
231         .transform = surfaceBuffer->GetSurfaceBufferTransform(),
232     };
233     auto allocErrorCode = newSurfaceBuffer->Alloc(requestConfig);
234     MEDIA_INFO_LOG("SurfaceBuffer alloc ret: %d", allocErrorCode);
235     if (memcpy_s(newSurfaceBuffer->GetVirAddr(), newSurfaceBuffer->GetSize(),
236         surfaceBuffer->GetVirAddr(), surfaceBuffer->GetSize()) != EOK) {
237         MEDIA_ERR_LOG("PhotoListener memcpy_s failed");
238     }
239 }
240 
CreateMediaLibrary(sptr<SurfaceBuffer> surfaceBuffer,BufferHandle * bufferHandle,CameraBufferExtraData extraData,bool isHighQuality,std::string & uri,int32_t & cameraShotType,std::string & burstKey,int64_t timestamp)241 void PhotoListener::CreateMediaLibrary(sptr<SurfaceBuffer> surfaceBuffer, BufferHandle *bufferHandle,
242     CameraBufferExtraData extraData, bool isHighQuality, std::string &uri,
243     int32_t &cameraShotType, std::string &burstKey, int64_t timestamp)
244 {
245     CAMERA_SYNC_TRACE;
246     CHECK_AND_RETURN_LOG(bufferHandle != nullptr, "bufferHandle is nullptr");
247 
248     MEDIA_DEBUG_LOG("PhotoListener ExecutePhotoAsset captureId:%{public}d imageId:%{public}" PRId64
249         ", deferredProcessingType:%{public}d",
250         extraData.captureId, extraData.imageId, extraData.deferredProcessingType);
251     MEDIA_DEBUG_LOG("deferredImageFormat:%{public}d, width:%{public}d, height:%{public}d, size:%{public}" PRId64,
252         extraData.deferredImageFormat, extraData.photoWidth, extraData.photoHeight, extraData.size);
253 
254     int32_t format = bufferHandle->format;
255     sptr<CameraPhotoProxy> photoProxy;
256     std::string imageIdStr = std::to_string(extraData.imageId);
257     photoProxy = new(std::nothrow) CameraPhotoProxy(bufferHandle, format, extraData.photoWidth, extraData.photoHeight,
258         isHighQuality, extraData.captureId);
259     CHECK_AND_RETURN_LOG(photoProxy != nullptr, "Failed to new photoProxy");
260 
261     photoProxy->SetDeferredAttrs(imageIdStr, extraData.deferredProcessingType, extraData.size,
262         extraData.deferredImageFormat);
263     auto photoOutput = photoOutput_->GetInnerPhotoOutput();
264     if (photoOutput && photoOutput->GetSession()) {
265         auto settings = photoOutput->GetDefaultCaptureSetting();
266         if (settings) {
267             auto location = std::make_shared<Location>();
268             settings->GetLocation(location);
269             photoProxy->SetLocation(location->latitude, location->longitude);
270         }
271         photoOutput->GetSession()->CreateMediaLibrary(photoProxy, uri, cameraShotType, burstKey, timestamp);
272     }
273 }
274 
RawPhotoListener(Camera_PhotoOutput * photoOutput,const sptr<Surface> rawPhotoSurface)275 RawPhotoListener::RawPhotoListener(Camera_PhotoOutput* photoOutput, const sptr<Surface> rawPhotoSurface)
276     : photoOutput_(photoOutput), rawPhotoSurface_(rawPhotoSurface)
277 {
278     if (bufferProcessor_ == nullptr && rawPhotoSurface != nullptr) {
279         bufferProcessor_ = std::make_shared<PhotoBufferProcessor>(rawPhotoSurface);
280     }
281 }
282 
~RawPhotoListener()283 RawPhotoListener::~RawPhotoListener()
284 {
285     rawPhotoSurface_ = nullptr;
286 }
287 
OnBufferAvailable()288 void RawPhotoListener::OnBufferAvailable()
289 {
290     MEDIA_DEBUG_LOG("RawPhotoListener::OnBufferAvailable");
291     std::lock_guard<std::mutex> lock(g_photoImageMutex);
292     MEDIA_INFO_LOG("RawPhotoListener::OnBufferAvailable is called");
293     CHECK_AND_RETURN_LOG(rawPhotoSurface_ != nullptr, "rawPhotoSurface_ is null");
294 
295     int64_t timestamp;
296     OHOS::Rect damage;
297     sptr<SurfaceBuffer> surfaceBuffer = nullptr;
298     int32_t fence = -1;
299     SurfaceError surfaceRet = rawPhotoSurface_->AcquireBuffer(surfaceBuffer, fence, timestamp, damage);
300     CHECK_AND_RETURN_LOG(surfaceRet == SURFACE_ERROR_OK, "RawPhotoListener failed to acquire surface buffer");
301 
302     int32_t isDegradedImage;
303     surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::isDegradedImage, isDegradedImage);
304     if (isDegradedImage == 0) {
305         ExecuteRawPhoto(surfaceBuffer, timestamp);
306         rawPhotoSurface_->ReleaseBuffer(surfaceBuffer, -1);
307     }
308 }
309 
ExecuteRawPhoto(sptr<SurfaceBuffer> surfaceBuffer,int64_t timestamp)310 void RawPhotoListener::ExecuteRawPhoto(sptr<SurfaceBuffer> surfaceBuffer, int64_t timestamp)
311 {
312     std::shared_ptr<Media::NativeImage> nativeImage = std::make_shared<Media::NativeImage>(surfaceBuffer,
313         bufferProcessor_, timestamp);
314     CHECK_AND_RETURN_LOG(nativeImage != nullptr, "Create native image failed");
315 
316     if (callback_ != nullptr && photoOutput_ != nullptr) {
317         OH_PhotoNative *photoNative = photoOutput_->CreateCameraPhotoNative(nativeImage, false);
318         CHECK_AND_RETURN_LOG(photoNative != nullptr, "Create photo native failed");
319 
320         callback_(photoOutput_, photoNative);
321     }
322     return;
323 }
324 
SetCallback(OH_PhotoOutput_PhotoAvailable callback)325 void RawPhotoListener::SetCallback(OH_PhotoOutput_PhotoAvailable callback)
326 {
327     MEDIA_DEBUG_LOG("RawPhotoListener::SetCallback");
328     std::lock_guard<std::mutex> lock(g_photoImageMutex);
329     MEDIA_INFO_LOG("RawPhotoListener::SetCallback is called");
330     if (callback != nullptr) {
331         callback_ = callback;
332     }
333 }
334 
UnregisterCallback(OH_PhotoOutput_PhotoAvailable callback)335 void RawPhotoListener::UnregisterCallback(OH_PhotoOutput_PhotoAvailable callback)
336 {
337     MEDIA_DEBUG_LOG("RawPhotoListener::UnregisterCallback");
338     std::lock_guard<std::mutex> lock(g_photoImageMutex);
339     MEDIA_INFO_LOG("RawPhotoListener::UnregisterCallback is called");
340     if (callback != nullptr && callback_ != nullptr) {
341         callback_ = nullptr;
342     }
343 }
344 }
345 }