1 /*
2  * Copyright (C) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define MLOG_TAG "MultiStagesCaptureDeferredPhotoProcSessionCallback"
17 
18 #include "multistages_capture_deferred_photo_proc_session_callback.h"
19 
20 #include <sstream>
21 
22 #include "database_adapter.h"
23 #include "file_utils.h"
24 #include "media_file_utils.h"
25 #include "media_file_uri.h"
26 #include "media_log.h"
27 #include "medialibrary_asset_operations.h"
28 #include "medialibrary_errno.h"
29 #include "medialibrary_notify.h"
30 #include "medialibrary_object_utils.h"
31 #include "medialibrary_photo_operations.h"
32 #include "medialibrary_tracer.h"
33 #include "multistages_capture_manager.h"
34 #include "multistages_capture_dfx_result.h"
35 #include "multistages_capture_dfx_total_time.h"
36 #include "multistages_capture_request_task_manager.h"
37 #include "result_set_utils.h"
38 #include "media_change_effect.h"
39 
40 using namespace std;
41 using namespace OHOS::CameraStandard;
42 
43 namespace OHOS {
44 namespace Media {
MultiStagesCaptureDeferredPhotoProcSessionCallback()45 MultiStagesCaptureDeferredPhotoProcSessionCallback::MultiStagesCaptureDeferredPhotoProcSessionCallback()
46 {}
47 
~MultiStagesCaptureDeferredPhotoProcSessionCallback()48 MultiStagesCaptureDeferredPhotoProcSessionCallback::~MultiStagesCaptureDeferredPhotoProcSessionCallback()
49 {}
50 
NotifyIfTempFile(shared_ptr<NativeRdb::ResultSet> resultSet)51 void MultiStagesCaptureDeferredPhotoProcSessionCallback::NotifyIfTempFile(shared_ptr<NativeRdb::ResultSet> resultSet)
52 {
53     if (resultSet == nullptr) {
54         MEDIA_WARN_LOG("resultSet is nullptr");
55         return;
56     }
57     string displayName = GetStringVal(MediaColumn::MEDIA_NAME, resultSet);
58     string filePath = GetStringVal(MediaColumn::MEDIA_FILE_PATH, resultSet);
59     int32_t mediaType = GetInt32Val(MediaColumn::MEDIA_TYPE, resultSet);
60     int32_t fileId = GetInt32Val(MediaColumn::MEDIA_ID, resultSet);
61     resultSet->Close();
62 
63     auto watch = MediaLibraryNotify::GetInstance();
64     if (watch != nullptr) {
65         string extrUri = MediaFileUtils::GetExtraUri(displayName, filePath);
66         auto notifyUri = MediaFileUtils::GetUriByExtrConditions(ML_FILE_URI_PREFIX + MediaFileUri::GetMediaTypeUri(
67             static_cast<MediaType>(mediaType), MEDIA_API_VERSION_V10) + "/", to_string(fileId), extrUri);
68         MEDIA_DEBUG_LOG("notify %{public}s", MediaFileUtils::GetUriWithoutDisplayname(notifyUri).c_str());
69         watch->Notify(MediaFileUtils::GetUriWithoutDisplayname(notifyUri), NOTIFY_UPDATE);
70     }
71 }
72 
UpdatePhotoQuality(const string & photoId)73 int32_t MultiStagesCaptureDeferredPhotoProcSessionCallback::UpdatePhotoQuality(const string &photoId)
74 {
75     MediaLibraryTracer tracer;
76     tracer.Start("UpdatePhotoQuality " + photoId);
77     MediaLibraryCommand updateCmd(OperationObject::FILESYSTEM_PHOTO, OperationType::UPDATE);
78     NativeRdb::ValuesBucket updateValues;
79     updateValues.PutInt(PhotoColumn::PHOTO_QUALITY, static_cast<int32_t>(MultiStagesPhotoQuality::FULL));
80     updateCmd.SetValueBucket(updateValues);
81     updateCmd.GetAbsRdbPredicates()->EqualTo(PhotoColumn::PHOTO_ID, photoId);
82     int32_t updatePhotoQualityResult = DatabaseAdapter::Update(updateCmd);
83 
84     updateCmd.GetAbsRdbPredicates()->EqualTo(PhotoColumn::PHOTO_IS_TEMP, false);
85     updateCmd.GetAbsRdbPredicates()->NotEqualTo(PhotoColumn::PHOTO_SUBTYPE,
86         to_string(static_cast<int32_t>(PhotoSubType::MOVING_PHOTO)));
87     NativeRdb::ValuesBucket updateValuesDirty;
88     updateValuesDirty.PutInt(PhotoColumn::PHOTO_DIRTY, static_cast<int32_t>(DirtyType::TYPE_NEW));
89     updateCmd.SetValueBucket(updateValuesDirty);
90     auto isDirtyResult = DatabaseAdapter::Update(updateCmd);
91     if (isDirtyResult < 0) {
92         MEDIA_WARN_LOG("update dirty flag fail, photoId: %{public}s", photoId.c_str());
93     }
94 
95     return updatePhotoQualityResult;
96 }
97 
UpdateCEAvailable(const string & photoId)98 void MultiStagesCaptureDeferredPhotoProcSessionCallback::UpdateCEAvailable(const string& photoId)
99 {
100     MediaLibraryCommand updateCEAvailableCmd(OperationObject::FILESYSTEM_PHOTO, OperationType::UPDATE);
101     NativeRdb::ValuesBucket updateCEAvailable;
102     updateCEAvailableCmd.GetAbsRdbPredicates()->EqualTo(PhotoColumn::PHOTO_ID, photoId);
103     updateCEAvailable.PutInt(PhotoColumn::PHOTO_CE_AVAILABLE,
104         static_cast<int32_t>(CloudEnhancementAvailableType::SUPPORT));
105     updateCEAvailableCmd.SetValueBucket(updateCEAvailable);
106     auto ceAvailableResult = DatabaseAdapter::Update(updateCEAvailableCmd);
107     if (ceAvailableResult < 0) {
108         MEDIA_WARN_LOG("update CE available fail, photoId: %{public}s", photoId.c_str());
109         return;
110     }
111 }
112 
QuerySubType(const string & photoId)113 int32_t QuerySubType(const string &photoId)
114 {
115     MediaLibraryCommand cmd(OperationObject::FILESYSTEM_PHOTO, OperationType::QUERY);
116     string where = PhotoColumn::PHOTO_ID + " = ? ";
117     vector<string> whereArgs { photoId };
118     cmd.GetAbsRdbPredicates()->SetWhereClause(where);
119     cmd.GetAbsRdbPredicates()->SetWhereArgs(whereArgs);
120     vector<string> columns { PhotoColumn::PHOTO_SUBTYPE };
121     auto resultSet = DatabaseAdapter::Query(cmd, columns);
122     if (resultSet == nullptr || resultSet->GoToFirstRow() != E_OK) {
123         MEDIA_ERR_LOG("result set is empty, photoId: %{public}s", photoId.c_str());
124         return static_cast<int32_t>(PhotoSubType::CAMERA);
125     }
126     int32_t subType = GetInt32Val(PhotoColumn::PHOTO_SUBTYPE, resultSet);
127     resultSet->Close();
128     return subType == 0 ? static_cast<int32_t>(PhotoSubType::CAMERA) : subType;
129 }
130 
OnError(const string & imageId,const DpsErrorCode error)131 void MultiStagesCaptureDeferredPhotoProcSessionCallback::OnError(const string &imageId, const DpsErrorCode error)
132 {
133     switch (error) {
134         case ERROR_SESSION_SYNC_NEEDED:
135             MultiStagesPhotoCaptureManager::GetInstance().SyncWithDeferredProcSession();
136             break;
137         case ERROR_IMAGE_PROC_INVALID_PHOTO_ID:
138         case ERROR_IMAGE_PROC_FAILED: {
139             MultiStagesPhotoCaptureManager::GetInstance().RemoveImage(imageId, false);
140             UpdatePhotoQuality(imageId);
141             MEDIA_ERR_LOG("error %{public}d, photoid: %{public}s", static_cast<int32_t>(error), imageId.c_str());
142             break;
143         }
144         default:
145             break;
146     }
147 
148     if (error != ERROR_SESSION_SYNC_NEEDED) {
149         MultiStagesCaptureDfxResult::Report(imageId, static_cast<int32_t>(error),
150             static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
151     }
152 }
153 
QueryPhotoData(const std::string & imageId)154 std::shared_ptr<NativeRdb::ResultSet> QueryPhotoData(const std::string &imageId)
155 {
156     MediaLibraryCommand cmd(OperationObject::FILESYSTEM_PHOTO, OperationType::QUERY);
157     string where = PhotoColumn::PHOTO_ID + " = ? ";
158     vector<string> whereArgs { imageId };
159     cmd.GetAbsRdbPredicates()->SetWhereClause(where);
160     cmd.GetAbsRdbPredicates()->SetWhereArgs(whereArgs);
161     vector<string> columns { MediaColumn::MEDIA_ID, MediaColumn::MEDIA_FILE_PATH, PhotoColumn::PHOTO_EDIT_TIME,
162         MediaColumn::MEDIA_MIME_TYPE, PhotoColumn::PHOTO_SUBTYPE, PhotoColumn::MEDIA_TYPE, PhotoColumn::PHOTO_IS_TEMP};
163     return DatabaseAdapter::Query(cmd, columns);
164 }
165 
OnProcessImageDone(const std::string & imageId,std::shared_ptr<Media::Picture> picture,bool isCloudEnhancementAvailable)166 void MultiStagesCaptureDeferredPhotoProcSessionCallback::OnProcessImageDone(const std::string &imageId,
167     std::shared_ptr<Media::Picture> picture, bool isCloudEnhancementAvailable)
168 {
169     MediaLibraryTracer tracer;
170     tracer.Start("OnProcessImageDone " + imageId);
171     if (picture == nullptr || picture->GetMainPixel() == nullptr) {
172         tracer.Finish();
173         MEDIA_ERR_LOG("OnProcessImageDone picture is null");
174         return;
175     }
176     // 1. 分段式拍照已经处理完成，保存全质量图
177     MEDIA_INFO_LOG("yuv photoid: %{public}s, isCloudEnhancementAvailable: %{public}s enter", imageId.c_str(),
178         isCloudEnhancementAvailable?"true":"false");
179     tracer.Start("Query");
180     auto resultSet = QueryPhotoData(imageId);
181     if (resultSet == nullptr || resultSet->GoToFirstRow() != E_OK) {
182         tracer.Finish();
183         MEDIA_INFO_LOG("result set is empty.");
184         // 高质量图先上来，直接保存
185         MultiStagesPhotoCaptureManager::GetInstance().DealHighQualityPicture(imageId, std::move(picture), false);
186         return;
187     }
188     tracer.Finish();
189     int32_t isTemp = GetInt32Val(PhotoColumn::PHOTO_IS_TEMP, resultSet);
190     if (isTemp) {
191         MultiStagesPhotoCaptureManager::GetInstance().DealHighQualityPicture(imageId, std::move(picture), false);
192         UpdateHighQualityPictureInfo(imageId, isCloudEnhancementAvailable);
193         return;
194     }
195     string data = GetStringVal(MediaColumn::MEDIA_FILE_PATH, resultSet);
196     bool isEdited = (GetInt64Val(PhotoColumn::PHOTO_EDIT_TIME, resultSet) > 0);
197     int32_t fileId = GetInt32Val(MediaColumn::MEDIA_ID, resultSet);
198     string mime_type = GetStringVal(MediaColumn::MEDIA_MIME_TYPE, resultSet);
199     // 裸picture落盘处理
200     int ret = MediaLibraryPhotoOperations::ProcessMultistagesPhotoForPicture(isEdited,
201         data, picture, fileId, mime_type);
202     if (ret != E_OK) {
203         MEDIA_ERR_LOG("Save high quality image failed. ret: %{public}d, errno: %{public}d", ret, errno);
204         MultiStagesCaptureDfxResult::Report(imageId,
205             static_cast<int32_t>(MultiStagesCaptureResultErrCode::SAVE_IMAGE_FAIL),
206             static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
207         return;
208     }
209     MultiStagesPhotoCaptureManager::GetInstance().DealHighQualityPicture(imageId, std::move(picture), isEdited);
210 
211     UpdateHighQualityPictureInfo(imageId, isCloudEnhancementAvailable);
212     MediaLibraryObjectUtils::ScanFileAsync(data, to_string(fileId), MediaLibraryApi::API_10);
213     NotifyIfTempFile(resultSet);
214     MultiStagesCaptureDfxTotalTime::GetInstance().Report(imageId);
215     MultiStagesCaptureDfxResult::Report(imageId, static_cast<int32_t>(MultiStagesCaptureResultErrCode::SUCCESS),
216         static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
217 
218     // delete raw file
219     MultiStagesPhotoCaptureManager::GetInstance().RemoveImage(imageId, false);
220     MEDIA_INFO_LOG("yuv success photoid: %{public}s", imageId.c_str());
221 }
222 
GetCommandByImageId(const std::string & imageId,MediaLibraryCommand & cmd)223 void MultiStagesCaptureDeferredPhotoProcSessionCallback::GetCommandByImageId(const std::string &imageId,
224     MediaLibraryCommand &cmd)
225 {
226     size_t slashIndex = imageId.rfind("/");
227     string where = "";
228     vector<string> whereArgs;
229     if (slashIndex != string::npos) {
230         string fileId = MediaFileUtils::GetIdFromUri(imageId);
231         where = PhotoColumn::MEDIA_ID + " = ? ";
232         whereArgs = { fileId };
233     }
234     cmd.GetAbsRdbPredicates()->SetWhereClause(where);
235     cmd.GetAbsRdbPredicates()->SetWhereArgs(whereArgs);
236 }
237 
UpdateHighQualityPictureInfo(const std::string & imageId,bool isCloudEnhancementAvailable)238 void MultiStagesCaptureDeferredPhotoProcSessionCallback::UpdateHighQualityPictureInfo(const std::string &imageId,
239     bool isCloudEnhancementAvailable)
240 {
241     // 2. 更新数据库 photoQuality 到高质量
242     UpdatePhotoQuality(imageId);
243     // 3. update cloud enhancement avaiable
244     if (isCloudEnhancementAvailable) {
245         UpdateCEAvailable(imageId);
246     }
247 }
248 
OnDeliveryLowQualityImage(const std::string & imageId,std::shared_ptr<Media::Picture> picture)249 void MultiStagesCaptureDeferredPhotoProcSessionCallback::OnDeliveryLowQualityImage(const std::string &imageId,
250     std::shared_ptr<Media::Picture> picture)
251 {
252     MEDIA_INFO_LOG("photoid: %{public}s", imageId.c_str());
253     if (picture != nullptr && picture->GetMainPixel() != nullptr) {
254         MEDIA_INFO_LOG("OnDeliveryLowQualityImage picture is not null");
255     } else {
256         MEDIA_INFO_LOG("OnDeliveryLowQualityImage picture is null");
257         return;
258     }
259     MediaLibraryTracer tracer;
260     tracer.Start("OnDeliveryLowQualityImage " + imageId);
261     MediaLibraryCommand cmd(OperationObject::FILESYSTEM_PHOTO, OperationType::QUERY);
262     GetCommandByImageId(imageId, cmd);
263     vector<string> columns { MediaColumn::MEDIA_ID, MediaColumn::MEDIA_FILE_PATH, PhotoColumn::PHOTO_EDIT_TIME,
264         PhotoColumn::PHOTO_SUBTYPE, PhotoColumn::PHOTO_ID};
265     tracer.Start("Query");
266     auto resultSet = DatabaseAdapter::Query(cmd, columns);
267     if (resultSet == nullptr || resultSet->GoToFirstRow() != E_OK) {
268         tracer.Finish();
269         MEDIA_INFO_LOG("result set is empty");
270         return;
271     }
272     tracer.Finish();
273     string photoId = GetStringVal(PhotoColumn::PHOTO_ID, resultSet);
274     string data = GetStringVal(MediaColumn::MEDIA_FILE_PATH, resultSet);
275     bool isEdited = (GetInt64Val(PhotoColumn::PHOTO_EDIT_TIME, resultSet) > 0);
276     resultSet->Close();
277     MultiStagesPhotoCaptureManager::GetInstance().DealLowQualityPicture(photoId, std::move(picture), isEdited);
278     MEDIA_INFO_LOG("save low quality image end");
279 }
280 
OnProcessImageDone(const string & imageId,const uint8_t * addr,const long bytes,bool isCloudEnhancementAvailable)281 void MultiStagesCaptureDeferredPhotoProcSessionCallback::OnProcessImageDone(const string &imageId, const uint8_t *addr,
282     const long bytes, bool isCloudEnhancementAvailable)
283 {
284     CHECK_AND_RETURN_LOG((addr != nullptr) && (bytes != 0), "addr is nullptr or bytes(%{public}ld) is 0", bytes);
285     CHECK_AND_RETURN_LOG(MultiStagesCaptureRequestTaskManager::IsPhotoInProcess(imageId),
286         "this photo was delete or err photoId: %{public}s", imageId.c_str());
287     MediaLibraryTracer tracer;
288     tracer.Start("OnProcessImageDone " + imageId);
289     // 1. 分段式拍照已经处理完成，保存全质量图
290     MEDIA_INFO_LOG("photoid: %{public}s, bytes: %{public}ld, isCloudEnhancementAvailable: %{public}s enter",
291         imageId.c_str(), bytes, isCloudEnhancementAvailable?"true":"false");
292     MediaLibraryCommand cmd(OperationObject::FILESYSTEM_PHOTO, OperationType::QUERY);
293     string where = PhotoColumn::PHOTO_ID + " = ? ";
294     vector<string> whereArgs { imageId };
295     cmd.GetAbsRdbPredicates()->SetWhereClause(where);
296     cmd.GetAbsRdbPredicates()->SetWhereArgs(whereArgs);
297     vector<string> columns { MediaColumn::MEDIA_ID, MediaColumn::MEDIA_FILE_PATH, PhotoColumn::PHOTO_EDIT_TIME,
298         PhotoColumn::MEDIA_NAME, PhotoColumn::MEDIA_TYPE, PhotoColumn::PHOTO_IS_TEMP };
299     tracer.Start("Query");
300     auto resultSet = DatabaseAdapter::Query(cmd, columns);
301     if (resultSet == nullptr || resultSet->GoToFirstRow() != E_OK) {
302         tracer.Finish();
303         MEDIA_INFO_LOG("result set is empty");
304         MultiStagesCaptureDfxTotalTime::GetInstance().RemoveStartTime(imageId);
305         MultiStagesCaptureDfxResult::Report(imageId, static_cast<int32_t>(MultiStagesCaptureResultErrCode::SQL_ERR),
306             static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
307         return;
308     }
309     tracer.Finish();
310     string data = GetStringVal(MediaColumn::MEDIA_FILE_PATH, resultSet);
311     bool isEdited = (GetInt64Val(PhotoColumn::PHOTO_EDIT_TIME, resultSet) > 0);
312     int32_t fileId = GetInt32Val(MediaColumn::MEDIA_ID, resultSet);
313     int ret = MediaLibraryPhotoOperations::ProcessMultistagesPhoto(isEdited, data, addr, bytes, fileId);
314     if (ret != E_OK) {
315         MEDIA_ERR_LOG("Save high quality image failed. ret: %{public}d, errno: %{public}d", ret, errno);
316         MultiStagesCaptureDfxResult::Report(imageId,
317             static_cast<int32_t>(MultiStagesCaptureResultErrCode::SAVE_IMAGE_FAIL),
318             static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
319         return;
320     }
321     // 2. 更新数据库 photoQuality 到高质量
322     UpdatePhotoQuality(imageId);
323     // 3. update cloud enhancement avaiable
324     if (isCloudEnhancementAvailable) {
325         UpdateCEAvailable(imageId);
326     }
327 
328     MediaLibraryObjectUtils::ScanFileAsync(data, to_string(fileId), MediaLibraryApi::API_10);
329     NotifyIfTempFile(resultSet);
330 
331     MultiStagesCaptureDfxTotalTime::GetInstance().Report(imageId);
332     MultiStagesCaptureDfxResult::Report(imageId, static_cast<int32_t>(MultiStagesCaptureResultErrCode::SUCCESS),
333         static_cast<int32_t>(MultiStagesCaptureMediaType::Photo));
334 
335     // delete raw file
336     MultiStagesPhotoCaptureManager::GetInstance().RemoveImage(imageId, false);
337     MEDIA_INFO_LOG("success photoid: %{public}s", imageId.c_str());
338 }
339 
OnStateChanged(const DpsStatusCode state)340 void MultiStagesCaptureDeferredPhotoProcSessionCallback::OnStateChanged(const DpsStatusCode state)
341 {
342     MEDIA_INFO_LOG("status: %{public}d", state);
343 }
344 } // namespace Media
345 } // namespace OHOS