1 /*
2  * Copyright (c) 2023-2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include "photo_post_processor.h"
16 
17 #include <cstdint>
18 #include <string>
19 #include <sys/mman.h>
20 #include "foundation/multimedia/camera_framework/services/camera_service/include/camera_util.h"
21 #include "foundation/multimedia/media_library/interfaces/inner_api/media_library_helper/include/photo_proxy.h"
22 #include "image_format.h"
23 #include "image_mime_type.h"
24 #include "image_type.h"
25 #include "v1_3/iimage_process_service.h"
26 #include "iproxy_broker.h"
27 #include "v1_3/types.h"
28 #include "securec.h"
29 #include "shared_buffer.h"
30 #include "dp_utils.h"
31 #include "events_monitor.h"
32 #include "dps_event_report.h"
33 #include "picture.h"
34 #include "steady_clock.h"
35 #include "buffer_extra_data_impl.h"
36 
37 namespace OHOS {
38 namespace CameraStandard {
39 namespace DeferredProcessing {
40 constexpr uint32_t MAX_CONSECUTIVE_TIMEOUT_COUNT = 3;
41 constexpr uint32_t MAX_CONSECUTIVE_CRASH_COUNT = 3;
42 
MapHdiError(OHOS::HDI::Camera::V1_2::ErrorCode errorCode)43 DpsError MapHdiError(OHOS::HDI::Camera::V1_2::ErrorCode errorCode)
44 {
45     DpsError code = DpsError::DPS_ERROR_IMAGE_PROC_ABNORMAL;
46     switch (errorCode) {
47         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_INVALID_ID:
48             code = DpsError::DPS_ERROR_IMAGE_PROC_INVALID_PHOTO_ID;
49             break;
50         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_PROCESS:
51             code = DpsError::DPS_ERROR_IMAGE_PROC_FAILED;
52             break;
53         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_TIMEOUT:
54             code = DpsError::DPS_ERROR_IMAGE_PROC_TIMEOUT;
55             break;
56         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_HIGH_TEMPERATURE:
57             code = DpsError::DPS_ERROR_IMAGE_PROC_HIGH_TEMPERATURE;
58             break;
59         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_ABNORMAL:
60             code = DpsError::DPS_ERROR_IMAGE_PROC_ABNORMAL;
61             break;
62         case OHOS::HDI::Camera::V1_2::ErrorCode::ERROR_ABORT:
63             code = DpsError::DPS_ERROR_IMAGE_PROC_INTERRUPTED;
64             break;
65         default:
66             DP_ERR_LOG("unexpected error code: %{public}d.", errorCode);
67             break;
68     }
69     return code;
70 }
71 
MapHdiStatus(OHOS::HDI::Camera::V1_2::SessionStatus statusCode)72 HdiStatus MapHdiStatus(OHOS::HDI::Camera::V1_2::SessionStatus statusCode)
73 {
74     HdiStatus code = HdiStatus::HDI_DISCONNECTED;
75     switch (statusCode) {
76         case OHOS::HDI::Camera::V1_2::SessionStatus::SESSION_STATUS_READY:
77             code = HdiStatus::HDI_READY;
78             break;
79         case OHOS::HDI::Camera::V1_2::SessionStatus::SESSION_STATUS_READY_SPACE_LIMIT_REACHED:
80             code = HdiStatus::HDI_READY_SPACE_LIMIT_REACHED;
81             break;
82         case OHOS::HDI::Camera::V1_2::SessionStatus::SESSSON_STATUS_NOT_READY_TEMPORARILY:
83             code = HdiStatus::HDI_NOT_READY_TEMPORARILY;
84             break;
85         case OHOS::HDI::Camera::V1_2::SessionStatus::SESSION_STATUS_NOT_READY_OVERHEAT:
86             code = HdiStatus::HDI_NOT_READY_OVERHEAT;
87             break;
88         case OHOS::HDI::Camera::V1_2::SessionStatus::SESSION_STATUS_NOT_READY_PREEMPTED:
89             code = HdiStatus::HDI_NOT_READY_PREEMPTED;
90             break;
91         default:
92             DP_ERR_LOG("unexpected error code: %{public}d.", statusCode);
93             break;
94     }
95     return code;
96 }
97 
MapToHdiExecutionMode(ExecutionMode executionMode)98 OHOS::HDI::Camera::V1_2::ExecutionMode MapToHdiExecutionMode(ExecutionMode executionMode)
99 {
100     auto mode = OHOS::HDI::Camera::V1_2::ExecutionMode::LOW_POWER;
101     switch (executionMode) {
102         case ExecutionMode::HIGH_PERFORMANCE:
103             mode = OHOS::HDI::Camera::V1_2::ExecutionMode::HIGH_PREFORMANCE;
104             break;
105         case ExecutionMode::LOAD_BALANCE:
106             mode = OHOS::HDI::Camera::V1_2::ExecutionMode::BALANCED;
107             break;
108         case ExecutionMode::LOW_POWER:
109             mode = OHOS::HDI::Camera::V1_2::ExecutionMode::LOW_POWER;
110             break;
111         default:
112             DP_ERR_LOG("unexpected error code: %{public}d.", executionMode);
113             break;
114     }
115     return mode;
116 }
117 
118 class PhotoPostProcessor::PhotoProcessListener : public OHOS::HDI::Camera::V1_3::IImageProcessCallback {
119 public:
PhotoProcessListener(PhotoPostProcessor * photoPostProcessor)120     explicit PhotoProcessListener(PhotoPostProcessor* photoPostProcessor) : photoPostProcessor_(photoPostProcessor)
121     {
122     }
123 
~PhotoProcessListener()124     ~PhotoProcessListener()
125     {
126         photoPostProcessor_ = nullptr;
127     }
128 
129     int32_t OnProcessDone(const std::string& imageId, const OHOS::HDI::Camera::V1_2::ImageBufferInfo& buffer) override;
130 
131     int32_t OnProcessDoneExt(const std::string& imageId,
132         const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt& buffer) override;
133 
134     int32_t OnError(const std::string& imageId,  OHOS::HDI::Camera::V1_2::ErrorCode errorCode) override;
135     int32_t OnStatusChanged(OHOS::HDI::Camera::V1_2::SessionStatus status) override;
136 
137 private:
138     void ReportEvent(const std::string& imageId);
139     int32_t processBufferInfo(const std::string& imageId, const OHOS::HDI::Camera::V1_2::ImageBufferInfo& buffer);
140 
141     std::shared_ptr<Media::Picture> AssemblePicture(const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt& buffer);
142     PhotoPostProcessor* photoPostProcessor_;
143 };
144 
OnProcessDone(const std::string & imageId,const OHOS::HDI::Camera::V1_2::ImageBufferInfo & buffer)145 int32_t PhotoPostProcessor::PhotoProcessListener::OnProcessDone(const std::string& imageId,
146     const OHOS::HDI::Camera::V1_2::ImageBufferInfo& buffer)
147 {
148     DP_DEBUG_LOG("imageId: %{public}s", imageId.c_str());
149     auto ret = processBufferInfo(imageId, buffer);
150     if (ret != DP_OK) {
151         DP_ERR_LOG("process done failed imageId: %{public}s.", imageId.c_str());
152         photoPostProcessor_->OnError(imageId, DPS_ERROR_IMAGE_PROC_FAILED);
153     }
154     return DP_OK;
155 }
156 
processBufferInfo(const std::string & imageId,const OHOS::HDI::Camera::V1_2::ImageBufferInfo & buffer)157 int32_t PhotoPostProcessor::PhotoProcessListener::processBufferInfo(const std::string& imageId,
158     const OHOS::HDI::Camera::V1_2::ImageBufferInfo& buffer)
159 {
160     auto bufferHandle = buffer.imageHandle->GetBufferHandle();
161     DP_CHECK_ERROR_RETURN_RET_LOG(bufferHandle == nullptr, DPS_ERROR_IMAGE_PROC_FAILED, "bufferHandle is nullptr.");
162 
163     int32_t size = bufferHandle->size;
164     int32_t isDegradedImage = 0;
165     int32_t dataSize = size;
166     int32_t isCloudImageEnhanceSupported = 0;
167     if (buffer.metadata) {
168         int32_t retImageQuality = buffer.metadata->Get("isDegradedImage", isDegradedImage);
169         int32_t retDataSize = buffer.metadata->Get("dataSize", dataSize);
170         int32_t retCloudImageEnhanceSupported = buffer.metadata->Get("isCloudImageEnhanceSupported",
171             isCloudImageEnhanceSupported);
172         DP_INFO_LOG("retImageQuality: %{public}d, retDataSize: %{public}d, retCloudImageEnhanceSupported: %{public}d",
173             static_cast<int>(retImageQuality), static_cast<int>(retDataSize),
174             static_cast<int>(retCloudImageEnhanceSupported));
175     }
176     DP_INFO_LOG("bufferHandle param, size: %{public}d, dataSize: %{public}d, isDegradedImage: %{public}d",
177         size, static_cast<int>(dataSize), isDegradedImage);
178     auto bufferPtr = std::make_shared<SharedBuffer>(dataSize);
179     DP_CHECK_ERROR_RETURN_RET_LOG(bufferPtr->Initialize() != DP_OK, DPS_ERROR_IMAGE_PROC_FAILED,
180         "failed to initialize shared buffer.");
181 
182     auto addr = mmap(nullptr, dataSize, PROT_READ | PROT_WRITE, MAP_SHARED, bufferHandle->fd, 0);
183     DP_CHECK_ERROR_RETURN_RET_LOG(addr == MAP_FAILED, DPS_ERROR_IMAGE_PROC_FAILED, "failed to mmap shared buffer.");
184 
185     if (bufferPtr->CopyFrom(static_cast<uint8_t*>(addr), dataSize) == DP_OK) {
186         DP_INFO_LOG("bufferPtr fd: %{public}d, fd: %{public}d", bufferHandle->fd, bufferPtr->GetFd());
187         std::shared_ptr<BufferInfo> bufferInfo = std::make_shared<BufferInfo>(bufferPtr, dataSize,
188             isDegradedImage == 0, isCloudImageEnhanceSupported);
189         ReportEvent(imageId);
190         photoPostProcessor_->OnProcessDone(imageId, bufferInfo);
191     }
192     munmap(addr, dataSize);
193     return DP_OK;
194 }
195 
CloneBufferHandle(const BufferHandle * handle)196 BufferHandle *CloneBufferHandle(const BufferHandle *handle)
197 {
198     if (handle == nullptr) {
199         DP_ERR_LOG("%{public}s handle is nullptr", __func__);
200         return nullptr;
201     }
202 
203     BufferHandle *newHandle = AllocateBufferHandle(handle->reserveFds, handle->reserveInts);
204     if (newHandle == nullptr) {
205         DP_ERR_LOG("%{public}s AllocateBufferHandle failed, newHandle is nullptr", __func__);
206         return nullptr;
207     }
208 
209     if (handle->fd == -1) {
210         newHandle->fd = handle->fd;
211     } else {
212         newHandle->fd = dup(handle->fd);
213         if (newHandle->fd == -1) {
214             DP_ERR_LOG("CloneBufferHandle dup failed");
215             FreeBufferHandle(newHandle);
216             return nullptr;
217         }
218     }
219     newHandle->width = handle->width;
220     newHandle->stride = handle->stride;
221     newHandle->height = handle->height;
222     newHandle->size = handle->size;
223     newHandle->format = handle->format;
224     newHandle->usage = handle->usage;
225     newHandle->phyAddr = handle->phyAddr;
226 
227     for (uint32_t i = 0; i < newHandle->reserveFds; i++) {
228         newHandle->reserve[i] = dup(handle->reserve[i]);
229         if (newHandle->reserve[i] == -1) {
230             DP_ERR_LOG("CloneBufferHandle dup reserveFds failed");
231             FreeBufferHandle(newHandle);
232             return nullptr;
233         }
234     }
235 
236     if (handle->reserveInts == 0) {
237         DP_ERR_LOG("There is no reserved integer value in old handle, no need to copy");
238         return newHandle;
239     }
240 
241     if (memcpy_s(&newHandle->reserve[newHandle->reserveFds], sizeof(int32_t) * newHandle->reserveInts,
242         &handle->reserve[handle->reserveFds], sizeof(int32_t) * handle->reserveInts) != EOK) {
243         DP_ERR_LOG("CloneBufferHandle memcpy_s failed");
244         FreeBufferHandle(newHandle);
245         return nullptr;
246     }
247     return newHandle;
248 }
249 
TransBufferHandleToSurfaceBuffer(BufferHandle * bufferHandle)250 sptr<SurfaceBuffer> TransBufferHandleToSurfaceBuffer(BufferHandle *bufferHandle)
251 {
252     DP_INFO_LOG("entered");
253     if (bufferHandle == nullptr) {
254         DP_ERR_LOG("bufferHandle is null");
255         return nullptr;
256     }
257     BufferHandle *newBufferHandle = CloneBufferHandle(bufferHandle);
258     sptr<SurfaceBuffer> surfaceBuffer = SurfaceBuffer::Create();
259     surfaceBuffer->SetBufferHandle(newBufferHandle);
260     DP_INFO_LOG("TransBufferHandleToSurfaceBuffer w=%{public}d, h=%{public}d, f=%{public}d",
261         surfaceBuffer->GetWidth(), surfaceBuffer->GetHeight(), surfaceBuffer->GetFormat());
262     return surfaceBuffer;
263 }
264 
CreateAuxiliaryPicture(BufferHandle * bufferHandle,Media::AuxiliaryPictureType type)265 std::shared_ptr<Media::AuxiliaryPicture> CreateAuxiliaryPicture(BufferHandle *bufferHandle,
266     Media::AuxiliaryPictureType type)
267 {
268     DP_INFO_LOG("entered, AuxiliaryPictureType type = %{public}d", static_cast<int32_t>(type));
269     if (bufferHandle == nullptr) {
270         DP_ERR_LOG("bufferHandle is null");
271         return nullptr;
272     }
273     auto buffer = TransBufferHandleToSurfaceBuffer(bufferHandle);
274     auto uniquePtr = Media::AuxiliaryPicture::Create(buffer, type, {buffer->GetWidth(), buffer->GetHeight()});
275     auto auxiliaryPicture = std::shared_ptr<Media::AuxiliaryPicture>(uniquePtr.release());
276     return auxiliaryPicture;
277 }
278 
RotatePixelMap(std::shared_ptr<Media::PixelMap> pixelMap,const std::string & exifOrientation)279 inline void RotatePixelMap(std::shared_ptr<Media::PixelMap> pixelMap, const std::string& exifOrientation)
280 {
281     float degree = DeferredProcessing::TransExifOrientationToDegree(exifOrientation);
282     if (pixelMap) {
283         DP_INFO_LOG("RotatePicture degree is %{public}f", degree);
284         pixelMap->rotate(degree);
285     } else {
286         DP_ERR_LOG("RotatePicture Failed pixelMap is nullptr");
287     }
288 }
289 
GetAndSetExifOrientation(OHOS::Media::ImageMetadata * exifData)290 std::string GetAndSetExifOrientation(OHOS::Media::ImageMetadata* exifData)
291 {
292     std::string orientation = "";
293     if (exifData != nullptr) {
294         exifData->GetValue("Orientation", orientation);
295         std::string defalutExifOrientation = "1";
296         exifData->SetValue("Orientation", defalutExifOrientation);
297         DP_INFO_LOG("GetExifOrientation orientation:%{public}s", orientation.c_str());
298     } else {
299         DP_ERR_LOG("GetExifOrientation exifData is nullptr");
300     }
301     return orientation;
302 }
303 
RotatePicture(std::shared_ptr<Media::Picture> picture)304 void RotatePicture(std::shared_ptr<Media::Picture> picture)
305 {
306     std::string orientation = GetAndSetExifOrientation(
307         reinterpret_cast<OHOS::Media::ImageMetadata*>(picture->GetExifMetadata().get()));
308     RotatePixelMap(picture->GetMainPixel(), orientation);
309     auto gainMap = picture->GetAuxiliaryPicture(Media::AuxiliaryPictureType::GAINMAP);
310     if (gainMap) {
311         RotatePixelMap(gainMap->GetContentPixel(), orientation);
312     }
313     auto depthMap = picture->GetAuxiliaryPicture(Media::AuxiliaryPictureType::DEPTH_MAP);
314     if (depthMap) {
315         RotatePixelMap(depthMap->GetContentPixel(), orientation);
316     }
317     auto unrefocusMap = picture->GetAuxiliaryPicture(Media::AuxiliaryPictureType::UNREFOCUS_MAP);
318     if (unrefocusMap) {
319         RotatePixelMap(unrefocusMap->GetContentPixel(), orientation);
320     }
321     auto linearMap = picture->GetAuxiliaryPicture(Media::AuxiliaryPictureType::LINEAR_MAP);
322     if (linearMap) {
323         RotatePixelMap(linearMap->GetContentPixel(), orientation);
324     }
325 }
326 
AssemleAuxilaryPicture(const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt & buffer,std::shared_ptr<Media::Picture> & picture)327 void AssemleAuxilaryPicture(const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt& buffer,
328     std::shared_ptr<Media::Picture>& picture)
329 {
330     if (buffer.isGainMapValid) {
331         auto auxiliaryPicture = CreateAuxiliaryPicture(buffer.gainMapHandle->GetBufferHandle(),
332             Media::AuxiliaryPictureType::GAINMAP);
333         if (auxiliaryPicture) {
334             picture->SetAuxiliaryPicture(auxiliaryPicture);
335         }
336     }
337     if (buffer.isDepthMapValid) {
338         auto auxiliaryPicture = CreateAuxiliaryPicture(buffer.depthMapHandle->GetBufferHandle(),
339             Media::AuxiliaryPictureType::DEPTH_MAP);
340         if (auxiliaryPicture) {
341             picture->SetAuxiliaryPicture(auxiliaryPicture);
342         }
343     }
344     if (buffer.isUnrefocusImageValid) {
345         auto auxiliaryPicture = CreateAuxiliaryPicture(buffer.unrefocusImageHandle->GetBufferHandle(),
346             Media::AuxiliaryPictureType::UNREFOCUS_MAP);
347         if (auxiliaryPicture) {
348             picture->SetAuxiliaryPicture(auxiliaryPicture);
349         }
350     }
351     if (buffer.isHighBitDepthLinearImageValid) {
352         auto auxiliaryPicture = CreateAuxiliaryPicture(buffer.highBitDepthLinearImageHandle->GetBufferHandle(),
353             Media::AuxiliaryPictureType::LINEAR_MAP);
354         if (auxiliaryPicture) {
355             picture->SetAuxiliaryPicture(auxiliaryPicture);
356         }
357     }
358     if (buffer.isMakerInfoValid) {
359         auto makerInfoBuffer = TransBufferHandleToSurfaceBuffer(buffer.makerInfoHandle->GetBufferHandle());
360         picture->SetMaintenanceData(makerInfoBuffer);
361     }
362 }
363 
AssemblePicture(const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt & buffer)364 std::shared_ptr<Media::Picture> PhotoPostProcessor::PhotoProcessListener::AssemblePicture(
365     const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt& buffer)
366 {
367     DP_INFO_LOG("entered");
368     int32_t exifDataSize = 0;
369     if (buffer.metadata) {
370         int32_t retExifDataSize = buffer.metadata->Get("exifDataSize", exifDataSize);
371         DP_INFO_LOG("AssemblePicture retExifDataSize: %{public}d, exifDataSize: %{public}d",
372             static_cast<int>(retExifDataSize), static_cast<int>(exifDataSize));
373     }
374     auto imageBuffer = TransBufferHandleToSurfaceBuffer(buffer.imageHandle->GetBufferHandle());
375     DP_CHECK_ERROR_RETURN_RET_LOG(imageBuffer == nullptr, nullptr, "bufferHandle is nullptr.");
376     DP_INFO_LOG("AssemblePicture ImageBufferInfoExt valid: gainMap(%{public}d), depthMap(%{public}d), "
377         "unrefocusMap(%{public}d), linearMap(%{public}d), exif(%{public}d), makeInfo(%{public}d)",
378         buffer.isGainMapValid, buffer.isDepthMapValid, buffer.isUnrefocusImageValid,
379         buffer.isHighBitDepthLinearImageValid, buffer.isExifValid, buffer.isMakerInfoValid);
380     std::shared_ptr<Media::Picture> picture = Media::Picture::Create(imageBuffer);
381     DP_CHECK_ERROR_RETURN_RET_LOG(picture == nullptr, nullptr, "picture is nullptr.");
382     if (buffer.isExifValid) {
383         auto exifBuffer = TransBufferHandleToSurfaceBuffer(buffer.exifHandle->GetBufferHandle());
384         sptr<BufferExtraData> extraData = new BufferExtraDataImpl();
385         extraData->ExtraSet("exifDataSize", exifDataSize);
386         if (exifBuffer) {
387             exifBuffer->SetExtraData(extraData);
388         }
389         picture->SetExifMetadata(exifBuffer);
390     }
391     if (picture) {
392         AssemleAuxilaryPicture(buffer, picture);
393         RotatePicture(picture);
394     }
395     return picture;
396 }
397 
OnProcessDoneExt(const std::string & imageId,const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt & buffer)398 int32_t PhotoPostProcessor::PhotoProcessListener::OnProcessDoneExt(
399     const std::string& imageId, const OHOS::HDI::Camera::V1_3::ImageBufferInfoExt& buffer)
400 {
401     DP_INFO_LOG("entered");
402     auto imageBufferHandle = buffer.imageHandle->GetBufferHandle();
403     if (imageBufferHandle == nullptr) {
404         DP_ERR_LOG("bufferHandle is null");
405         return 0;
406     }
407     int size = imageBufferHandle->size;
408     int32_t isDegradedImage = 0;
409     int32_t dataSize = size;
410     int32_t deferredImageFormat = 0;
411     int32_t isCloudImageEnhanceSupported = 0;
412     if (buffer.metadata) {
413         int32_t retImageQuality = buffer.metadata->Get("isDegradedImage", isDegradedImage);
414         int32_t retDataSize = buffer.metadata->Get("dataSize", dataSize);
415         int32_t retFormat = buffer.metadata->Get("deferredImageFormat", deferredImageFormat);
416         int32_t retCloudImageEnhanceSupported = buffer.metadata->Get("isCloudImageEnhanceSupported",
417             isCloudImageEnhanceSupported);
418         DP_INFO_LOG("retImageQuality: %{public}d, retDataSize: %{public}d, retFormat: %{public}d,"
419             "retCloudImageEnhanceSupported %{public}d", retImageQuality, retDataSize, retFormat,
420             retCloudImageEnhanceSupported);
421     }
422     DP_INFO_LOG("bufferHandle param, bufferHandleSize: %{public}d, dataSize: %{public}d, isDegradedImage: %{public}d, "
423         "deferredImageFormat: %{public}d, isCloudImageEnhanceSupported: %{public}d",
424         size, dataSize, isDegradedImage, deferredImageFormat, isCloudImageEnhanceSupported);
425     if (deferredImageFormat == static_cast<int32_t>(Media::PhotoFormat::YUV)) {
426         std::shared_ptr<Media::Picture> picture = AssemblePicture(buffer);
427         std::shared_ptr<BufferInfoExt> bufferInfo = std::make_shared<BufferInfoExt>(picture, dataSize,
428             isDegradedImage == 0, isCloudImageEnhanceSupported);
429         photoPostProcessor_->OnProcessDoneExt(imageId, bufferInfo);
430     } else {
431         auto bufferPtr = std::make_shared<SharedBuffer>(dataSize);
432         DP_CHECK_ERROR_RETURN_RET_LOG(bufferPtr->Initialize() != DP_OK, DPS_ERROR_IMAGE_PROC_FAILED,
433             "failed to initialize shared buffer.");
434 
435         auto addr = mmap(nullptr, dataSize, PROT_READ | PROT_WRITE, MAP_SHARED, imageBufferHandle->fd, 0);
436         DP_CHECK_ERROR_RETURN_RET_LOG(
437             addr == MAP_FAILED, DPS_ERROR_IMAGE_PROC_FAILED, "failed to mmap shared buffer.");
438 
439         if (bufferPtr->CopyFrom(static_cast<uint8_t*>(addr), dataSize) == DP_OK) {
440             DP_INFO_LOG("bufferPtr fd: %{public}d, fd: %{public}d", imageBufferHandle->fd, bufferPtr->GetFd());
441             std::shared_ptr<BufferInfo> bufferInfo = std::make_shared<BufferInfo>(bufferPtr, dataSize,
442                 isDegradedImage == 0, isCloudImageEnhanceSupported);
443             photoPostProcessor_->OnProcessDone(imageId, bufferInfo);
444         }
445         munmap(addr, dataSize);
446     }
447     ReportEvent(imageId);
448     return 0;
449 }
450 
ReportEvent(const std::string & imageId)451 void PhotoPostProcessor::PhotoProcessListener::ReportEvent(const std::string& imageId)
452 {
453     DPSEventReport::GetInstance()
454         .UpdateProcessDoneTime(imageId, photoPostProcessor_->GetUserId());
455 }
456 
457 
OnError(const std::string & imageId,OHOS::HDI::Camera::V1_2::ErrorCode errorCode)458 int32_t PhotoPostProcessor::PhotoProcessListener::OnError(const std::string& imageId,
459     OHOS::HDI::Camera::V1_2::ErrorCode errorCode)
460 {
461     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
462     DpsError dpsErrorCode = MapHdiError(errorCode);
463     photoPostProcessor_->OnError(imageId, dpsErrorCode);
464     return DP_OK;
465 }
466 
OnStatusChanged(OHOS::HDI::Camera::V1_2::SessionStatus status)467 int32_t PhotoPostProcessor::PhotoProcessListener::OnStatusChanged(OHOS::HDI::Camera::V1_2::SessionStatus status)
468 {
469     DP_INFO_LOG("entered");
470     HdiStatus hdiStatus = MapHdiStatus(status);
471     photoPostProcessor_->OnStateChanged(hdiStatus);
472     return DP_OK;
473 }
474 
475 class PhotoPostProcessor::SessionDeathRecipient : public IRemoteObject::DeathRecipient {
476 public:
SessionDeathRecipient(PhotoPostProcessor * processor)477     explicit SessionDeathRecipient(PhotoPostProcessor* processor)
478         : photoPostProcessor_(processor)
479     {
480     }
~SessionDeathRecipient()481     ~SessionDeathRecipient()
482     {
483         photoPostProcessor_ = nullptr;
484     }
485 
OnRemoteDied(const wptr<IRemoteObject> & remote)486     void OnRemoteDied(const wptr<IRemoteObject> &remote) override
487     {
488         DP_ERR_LOG("Remote died.");
489         if (photoPostProcessor_ == nullptr) {
490             return;
491         }
492         photoPostProcessor_->OnSessionDied();
493     }
494 
495 private:
496     PhotoPostProcessor* photoPostProcessor_;
497 };
498 
499 
PhotoPostProcessor(const int32_t userId,TaskManager * taskManager,IImageProcessCallbacks * callbacks)500 PhotoPostProcessor::PhotoPostProcessor(const int32_t userId,
501     TaskManager* taskManager, IImageProcessCallbacks* callbacks)
502     : userId_(userId),
503       taskManager_(taskManager),
504       processCallacks_(callbacks),
505       listener_(nullptr),
506       session_(nullptr),
507       sessionDeathRecipient_(nullptr),
508       imageId2Handle_(),
509       imageId2CrashCount_(),
510       removeNeededList_(),
511       consecutiveTimeoutCount_(0)
512 {
513     DP_DEBUG_LOG("entered");
514 }
515 
~PhotoPostProcessor()516 PhotoPostProcessor::~PhotoPostProcessor()
517 {
518     DP_DEBUG_LOG("entered");
519     DisconnectServiceIfNecessary();
520     taskManager_ = nullptr;
521     processCallacks_ = nullptr;
522     session_ = nullptr;
523     sessionDeathRecipient_ = nullptr;
524     listener_ = nullptr;
525     imageId2Handle_.Clear();
526     consecutiveTimeoutCount_ = 0;
527 }
528 
Initialize()529 void PhotoPostProcessor::Initialize()
530 {
531     DP_DEBUG_LOG("entered");
532     sessionDeathRecipient_ = new SessionDeathRecipient(this); // sptr<SessionDeathRecipient>::MakeSptr(this);
533     listener_ = new PhotoProcessListener(this); // sptr<PhotoProcessListener>::MakeSptr(this);
534     ConnectServiceIfNecessary();
535 }
536 
GetUserId()537 int32_t PhotoPostProcessor::GetUserId()
538 {
539     return userId_;
540 }
541 
GetConcurrency(ExecutionMode mode)542 int PhotoPostProcessor::GetConcurrency(ExecutionMode mode)
543 {
544     std::lock_guard<std::mutex> lock(mutex_);
545     int count = 1;
546     if (session_) {
547         int32_t ret = session_->GetCoucurrency(OHOS::HDI::Camera::V1_2::ExecutionMode::BALANCED, count);
548         DP_INFO_LOG("getConcurrency, ret: %{public}d", ret);
549     }
550     DP_INFO_LOG("entered, count: %{public}d", count);
551     return count;
552 }
553 
GetPendingImages(std::vector<std::string> & pendingImages)554 bool PhotoPostProcessor::GetPendingImages(std::vector<std::string>& pendingImages)
555 {
556     std::lock_guard<std::mutex> lock(mutex_);
557     DP_INFO_LOG("entered");
558     if (session_) {
559         int32_t ret = session_->GetPendingImages(pendingImages);
560         DP_INFO_LOG("getPendingImages, ret: %{public}d", ret);
561         if (ret == 0) {
562         return true;
563         }
564     }
565     return false;
566 }
567 
SetExecutionMode(ExecutionMode executionMode)568 void PhotoPostProcessor::SetExecutionMode(ExecutionMode executionMode)
569 {
570     std::lock_guard<std::mutex> lock(mutex_);
571     DP_INFO_LOG("entered, executionMode: %{public}d", executionMode);
572     if (session_) {
573         int32_t ret = session_->SetExecutionMode(MapToHdiExecutionMode(executionMode));
574         DP_INFO_LOG("setExecutionMode, ret: %{public}d", ret);
575     }
576 }
577 
SetDefaultExecutionMode()578 void PhotoPostProcessor::SetDefaultExecutionMode()
579 {
580     // 采用直接新增方法，不适配1_2 和 1_3 模式的差异点
581     std::lock_guard<std::mutex> lock(mutex_);
582     DP_INFO_LOG("entered.");
583     if (session_) {
584         int32_t ret = session_->SetExecutionMode(
585             static_cast<OHOS::HDI::Camera::V1_2::ExecutionMode>(OHOS::HDI::Camera::V1_3::ExecutionMode::DEFAULT));
586         DP_INFO_LOG("setExecutionMode, ret: %{public}d", ret);
587     }
588 }
589 
ProcessImage(std::string imageId)590 void PhotoPostProcessor::ProcessImage(std::string imageId)
591 {
592     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
593     if (!ConnectServiceIfNecessary()) {
594         DP_INFO_LOG("failed to process image (%{public}s) due to connect service failed", imageId.c_str());
595         OnError(imageId, DpsError::DPS_ERROR_SESSION_NOT_READY_TEMPORARILY);
596         return;
597     }
598 
599     std::lock_guard<std::mutex> lock(mutex_);
600     DP_CHECK_ERROR_RETURN_LOG(session_ == nullptr, "PhotoPostProcessor::ProcessImage imageProcessSession is nullptr");
601     int32_t ret = session_->ProcessImage(imageId);
602     DP_INFO_LOG("processImage, ret: %{public}d", ret);
603     uint32_t callbackHandle;
604     constexpr uint32_t maxProcessingTimeMs = 11 * 1000;
605     GetGlobalWatchdog().StartMonitor(callbackHandle, maxProcessingTimeMs, [this, imageId](uint32_t handle) {
606         DP_INFO_LOG("PhotoPostProcessor-ProcessImage-Watchdog executed, userId: %{public}d, handle: %{public}d",
607             userId_, static_cast<int>(handle));
608         OnError(imageId, DpsError::DPS_ERROR_IMAGE_PROC_TIMEOUT);
609     });
610     DP_INFO_LOG("PhotoPostProcessor-ProcessImage-Watchdog registered, userId: %{public}d, handle: %{public}d",
611         userId_, static_cast<int>(callbackHandle));
612     imageId2Handle_.Insert(imageId, callbackHandle);
613 }
614 
RemoveImage(std::string imageId)615 void PhotoPostProcessor::RemoveImage(std::string imageId)
616 {
617     std::lock_guard<std::mutex> lock(mutex_);
618     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
619     if (session_) {
620         int32_t ret = session_->RemoveImage(imageId);
621         DP_INFO_LOG("removeImage, imageId: %{public}s, ret: %{public}d", imageId.c_str(), ret);
622         imageId2CrashCount_.erase(imageId);
623         DPSEventReport::GetInstance().UpdateRemoveTime(imageId, userId_);
624     } else {
625         removeNeededList_.emplace_back(imageId);
626     }
627 }
628 
Interrupt()629 void PhotoPostProcessor::Interrupt()
630 {
631     std::lock_guard<std::mutex> lock(mutex_);
632     DP_INFO_LOG("entered");
633     if (session_) {
634         int32_t ret = session_->Interrupt();
635         DP_INFO_LOG("interrupt, ret: %{public}d", ret);
636     }
637 }
638 
Reset()639 void PhotoPostProcessor::Reset()
640 {
641     std::lock_guard<std::mutex> lock(mutex_);
642     DP_INFO_LOG("entered");
643     if (session_) {
644         int32_t ret = session_->Reset();
645         DP_INFO_LOG("reset, ret: %{public}d", ret);
646     }
647 }
648 
OnProcessDone(const std::string & imageId,std::shared_ptr<BufferInfo> bufferInfo)649 void PhotoPostProcessor::OnProcessDone(const std::string& imageId, std::shared_ptr<BufferInfo> bufferInfo)
650 {
651     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
652     consecutiveTimeoutCount_ = 0;
653     StopTimer(imageId);
654     if (processCallacks_) {
655         taskManager_->SubmitTask([this, imageId, bufferInfo = std::move(bufferInfo)]() {
656             processCallacks_->OnProcessDone(userId_, imageId, std::move(bufferInfo));
657         });
658     }
659 }
660 
OnProcessDoneExt(const std::string & imageId,std::shared_ptr<BufferInfoExt> bufferInfo)661 void PhotoPostProcessor::OnProcessDoneExt(const std::string& imageId, std::shared_ptr<BufferInfoExt> bufferInfo)
662 {
663     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
664     consecutiveTimeoutCount_ = 0;
665     StopTimer(imageId);
666     if (processCallacks_) {
667         taskManager_->SubmitTask([this, imageId, bufferInfo = std::move(bufferInfo)]() {
668             processCallacks_->OnProcessDoneExt(userId_, imageId, std::move(bufferInfo));
669         });
670     }
671 }
672 
OnError(const std::string & imageId,DpsError errorCode)673 void PhotoPostProcessor::OnError(const std::string& imageId, DpsError errorCode)
674 {
675     DP_INFO_LOG("entered, imageId: %{public}s", imageId.c_str());
676     StopTimer(imageId);
677     if (errorCode == DpsError::DPS_ERROR_IMAGE_PROC_TIMEOUT) {
678         consecutiveTimeoutCount_++;
679         if (consecutiveTimeoutCount_ >= static_cast<int>(MAX_CONSECUTIVE_TIMEOUT_COUNT)) {
680             Reset();
681             consecutiveTimeoutCount_ = 0;
682         }
683     } else {
684         consecutiveTimeoutCount_ = 0;
685     }
686     if (processCallacks_) {
687         taskManager_->SubmitTask([this, imageId, errorCode]() {
688             processCallacks_->OnError(userId_, imageId, errorCode);
689         });
690     }
691 }
692 
OnStateChanged(HdiStatus hdiStatus)693 void PhotoPostProcessor::OnStateChanged(HdiStatus hdiStatus)
694 {
695     DP_INFO_LOG("entered, HdiStatus: %{public}d", hdiStatus);
696     EventsMonitor::GetInstance().NotifyImageEnhanceStatus(hdiStatus);
697 }
698 
OnSessionDied()699 void PhotoPostProcessor::OnSessionDied()
700 {
701     DP_INFO_LOG("entered, session died!");
702     std::lock_guard<std::mutex> lock(mutex_);
703 
704     session_ = nullptr;
705     consecutiveTimeoutCount_ = 0;
706     OnStateChanged(HdiStatus::HDI_DISCONNECTED);
707     std::vector<std::string> crashJobs;
708     imageId2Handle_.Iterate([&](const std::string& imageId, const uint32_t value) {
709         crashJobs.emplace_back(imageId);
710     });
711     for (const auto& id : crashJobs) {
712         DP_INFO_LOG("failed to process imageId(%{public}s) due to connect service failed", id.c_str());
713         if (imageId2CrashCount_.count(id) == 0) {
714             imageId2CrashCount_.emplace(id, 1);
715         } else {
716             imageId2CrashCount_[id] += 1;
717         }
718         if (imageId2CrashCount_[id] >= MAX_CONSECUTIVE_CRASH_COUNT) {
719             OnError(id, DpsError::DPS_ERROR_IMAGE_PROC_FAILED);
720         } else {
721             OnError(id, DpsError::DPS_ERROR_SESSION_NOT_READY_TEMPORARILY);
722         }
723     }
724     ScheduleConnectService();
725 }
726 
ConnectServiceIfNecessary()727 bool PhotoPostProcessor::ConnectServiceIfNecessary()
728 {
729     DP_INFO_LOG("entered.");
730     std::lock_guard<std::mutex> lock(mutex_);
731     if (session_ != nullptr) {
732         DP_INFO_LOG("connected");
733         return true;
734     }
735     sptr<OHOS::HDI::Camera::V1_2::IImageProcessSession> imageProcessSession;
736     sptr<OHOS::HDI::Camera::V1_2::IImageProcessService> imageProcessServiceProxyV1_2;
737     sptr<OHOS::HDI::Camera::V1_3::IImageProcessService> imageProcessServiceProxyV1_3;
738     imageProcessServiceProxyV1_2 = OHOS::HDI::Camera::V1_2::IImageProcessService::Get(
739         std::string("camera_image_process_service"));
740     if (imageProcessServiceProxyV1_2 == nullptr) {
741         DP_INFO_LOG("Failed to CreateImageProcessSession");
742         ScheduleConnectService();
743         return false;
744     }
745     uint32_t majorVer = 0;
746     uint32_t minorVer = 0;
747     imageProcessServiceProxyV1_2->GetVersion(majorVer, minorVer);
748     int32_t versionId = GetVersionId(majorVer, minorVer);
749     DP_INFO_LOG("CreateImageProcessSession version=%{public}d_%{public}d", majorVer, minorVer);
750     if (imageProcessServiceProxyV1_2 != nullptr && versionId >= GetVersionId(HDI_VERSION_1, HDI_VERSION_3)) {
751         imageProcessServiceProxyV1_3 =
752             OHOS::HDI::Camera::V1_3::IImageProcessService::CastFrom(imageProcessServiceProxyV1_2);
753         DP_INFO_LOG("CreateImageProcessSession CastFrom imageProcessServiceProxyV1_3 == nullptr %{public}d",
754             imageProcessServiceProxyV1_3 == nullptr);
755     }
756     if (imageProcessServiceProxyV1_3 != nullptr && versionId >= GetVersionId(HDI_VERSION_1, HDI_VERSION_3)) {
757         sptr<OHOS::HDI::Camera::V1_2::IImageProcessSession> imageProcessSessionV1_3;
758         imageProcessServiceProxyV1_3->CreateImageProcessSessionExt(userId_, listener_, imageProcessSessionV1_3);
759         DP_INFO_LOG("CreateImageProcessSessionExt");
760         imageProcessSession = imageProcessSessionV1_3;
761     } else if (imageProcessServiceProxyV1_2 != nullptr && versionId >= GetVersionId(HDI_VERSION_1, HDI_VERSION_2)) {
762         imageProcessServiceProxyV1_2->CreateImageProcessSession(userId_, listener_, imageProcessSession);
763     }
764     session_ = imageProcessSession;
765     DP_CHECK_ERROR_RETURN_RET_LOG(session_ == nullptr, false, "session_ is nullptr");
766     for (const auto& imageId : removeNeededList_) {
767         int32_t ret = session_->RemoveImage(imageId);
768         DP_INFO_LOG("removeImage, imageId: %{public}s, ret: %{public}d", imageId.c_str(), ret);
769     }
770     removeNeededList_.clear();
771     const sptr<IRemoteObject>& remote =
772         OHOS::HDI::hdi_objcast<OHOS::HDI::Camera::V1_2::IImageProcessSession>(session_);
773     DP_CHECK_ERROR_RETURN_RET_LOG(!remote->AddDeathRecipient(sessionDeathRecipient_),
774         false, "AddDeathRecipient for ImageProcessSession failed.");
775     OnStateChanged(HdiStatus::HDI_READY);
776     return true;
777 }
778 
DisconnectServiceIfNecessary()779 void PhotoPostProcessor::DisconnectServiceIfNecessary()
780 {
781     std::lock_guard<std::mutex> lock(mutex_);
782     DP_CHECK_ERROR_RETURN_LOG(session_ == nullptr, "imageProcessSession is nullptr");
783     const sptr<IRemoteObject> &remote =
784         OHOS::HDI::hdi_objcast<OHOS::HDI::Camera::V1_2::IImageProcessSession>(session_);
785     DP_CHECK_ERROR_PRINT_LOG(!remote->RemoveDeathRecipient(sessionDeathRecipient_),
786         "RemoveDeathRecipient for ImageProcessSession failed.");
787     session_ = nullptr;
788 }
789 
ScheduleConnectService()790 void PhotoPostProcessor::ScheduleConnectService()
791 {
792     DP_INFO_LOG("entered.");
793 
794     if (session_ == nullptr) {
795         constexpr uint32_t delayMilli = 10 * 1000;
796         uint32_t callbackHandle;
797         GetGlobalWatchdog().StartMonitor(callbackHandle, delayMilli, [this](uint32_t handle) {
798             DP_INFO_LOG("PhotoPostProcessor Watchdog executed, handle: %{public}d", static_cast<int>(handle));
799             ConnectServiceIfNecessary();
800         });
801         DP_INFO_LOG("PhotoPostProcessor Watchdog registered, handle: %{public}d", static_cast<int>(callbackHandle));
802     } else {
803         DP_INFO_LOG("already connected.");
804     }
805 }
806 
StopTimer(const std::string & imageId)807 void PhotoPostProcessor::StopTimer(const std::string& imageId)
808 {
809     uint32_t callbackHandle;
810     DP_CHECK_ERROR_RETURN_LOG(!imageId2Handle_.Find(imageId, callbackHandle),
811         "stoptimer failed not find imageId: %{public}s", imageId.c_str());
812     imageId2Handle_.Erase(imageId);
813     GetGlobalWatchdog().StopMonitor(callbackHandle);
814     DP_INFO_LOG("stoptimer success, imageId: %{public}s", imageId.c_str());
815 }
816 } // namespace DeferredProcessing
817 } // namespace CameraStandard
818 } // namespace OHOS