1 /*
2  * Copyright (C) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "core/common/ai/image_analyzer_manager.h"
17 
18 #include "interfaces/inner_api/ace/ai/image_analyzer.h"
19 #include "js_native_api_types.h"
20 
21 #include "base/geometry/offset.h"
22 #include "base/image/pixel_map.h"
23 #include "base/utils/utils.h"
24 #include "core/common/ai/image_analyzer_adapter.h"
25 #include "core/common/ai/image_analyzer_mgr.h"
26 #include "core/components_ng/base/view_stack_processor.h"
27 #include "core/components_ng/pattern/image/image_pattern.h"
28 #include "core/components_ng/pattern/image/image_render_property.h"
29 #include "core/components_ng/pattern/video/video_layout_property.h"
30 #include "core/components_ng/property/measure_property.h"
31 
32 namespace OHOS::Ace {
33 
ImageAnalyzerManager(const RefPtr<NG::FrameNode> & frameNode,ImageAnalyzerHolder holder)34 ImageAnalyzerManager::ImageAnalyzerManager(const RefPtr<NG::FrameNode>& frameNode, ImageAnalyzerHolder holder)
35     : frameNode_(frameNode), holder_(holder)
36 {
37     imageAnalyzerAdapter_ = std::shared_ptr<ImageAnalyzerAdapter>(CreateImageAnalyzerAdapter());
38 }
39 
CreateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,const NG::OffsetF & offset)40 void ImageAnalyzerManager::CreateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap,
41     const NG::OffsetF& offset)
42 {
43     CHECK_NULL_VOID(imageAnalyzerAdapter_);
44     void* pixelmapNapiVal = nullptr;
45     if (pixelMap) {
46         pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
47     }
48 
49     analyzerUIConfig_.holder = holder_;
50     if (holder_ != ImageAnalyzerHolder::IMAGE && holder_ != ImageAnalyzerHolder::WEB) {
51         analyzerUIConfig_.contentWidth = pixelMap->GetWidth();
52         analyzerUIConfig_.contentHeight = pixelMap->GetHeight();
53     }
54 
55     if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
56         analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
57         analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
58         analyzerUIConfig_.overlayOffset = offset;
59     }
60 
61     RefPtr<NG::UINode> customNode;
62     {
63         NG::ScopedViewStackProcessor builderViewStackProcessor;
64         auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
65         ImageAnalyzerMgr::GetInstance().BuildNodeFunc(
66             pixelmapNapiVal, analyzerConfig, &analyzerUIConfig_, &overlayData_);
67         customNode = NG::ViewStackProcessor::GetInstance()->Finish();
68     }
69     auto overlayNode = AceType::DynamicCast<NG::FrameNode>(customNode);
70     CHECK_NULL_VOID(overlayNode);
71     auto node = frameNode_.Upgrade();
72     CHECK_NULL_VOID(node);
73     node->SetOverlayNode(overlayNode);
74     overlayNode->SetParent(AceType::WeakClaim(AceType::RawPtr(node)));
75     overlayNode->SetActive(true);
76     UpdateAnalyzerOverlayLayout();
77 
78     auto renderContext = overlayNode->GetRenderContext();
79     CHECK_NULL_VOID(renderContext);
80     renderContext->UpdateZIndex(INT32_MAX);
81     auto focusHub = overlayNode->GetOrCreateFocusHub();
82     CHECK_NULL_VOID(focusHub);
83     focusHub->SetFocusable(false);
84     overlayNode->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE_SELF);
85 
86     isAnalyzerOverlayBuild_ = true;
87     CHECK_NULL_VOID(analyzerUIConfig_.onAnalyzed);
88     (analyzerUIConfig_.onAnalyzed.value())(ImageAnalyzerState::FINISHED);
89     analyzerUIConfig_.onAnalyzed = std::nullopt;
90 }
91 
UpdateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,const NG::OffsetF & offset)92 void ImageAnalyzerManager::UpdateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap,
93     const NG::OffsetF& offset)
94 {
95     if (!isAnalyzerOverlayBuild_) {
96         return;
97     }
98 
99     auto node = frameNode_.Upgrade();
100     CHECK_NULL_VOID(node);
101     if (holder_ == ImageAnalyzerHolder::IMAGE) {
102         auto imagePattern = AceType::DynamicCast<NG::ImagePattern>(node->GetPattern());
103         CHECK_NULL_VOID(imagePattern);
104         if (!imagePattern->hasSceneChanged()) {
105             return;
106         }
107     }
108 
109     CHECK_NULL_VOID(pixelMap);
110     if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
111         analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
112         analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
113         analyzerUIConfig_.overlayOffset = offset;
114     }
115 
116     if (holder_ != ImageAnalyzerHolder::IMAGE) {
117         analyzerUIConfig_.contentWidth = pixelMap->GetWidth();
118         analyzerUIConfig_.contentHeight = pixelMap->GetHeight();
119     }
120 
121     CHECK_NULL_VOID(imageAnalyzerAdapter_);
122     auto pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
123     auto overlayNode = node->GetOverlayNode();
124     CHECK_NULL_VOID(overlayNode);
125     auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
126     ImageAnalyzerMgr::GetInstance().UpdateImage(&overlayData_, pixelmapNapiVal, analyzerConfig, &analyzerUIConfig_);
127     overlayNode->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE_SELF);
128 }
129 
DestroyAnalyzerOverlay()130 void ImageAnalyzerManager::DestroyAnalyzerOverlay()
131 {
132     ReleaseImageAnalyzer();
133 
134     if (!isAnalyzerOverlayBuild_) {
135         return;
136     }
137     auto node = frameNode_.Upgrade();
138     CHECK_NULL_VOID(node);
139     auto overlayNode = node->GetOverlayNode();
140     CHECK_NULL_VOID(overlayNode);
141     node->SetOverlayNode(RefPtr<NG::FrameNode>());
142 
143     isAnalyzerOverlayBuild_ = false;
144     CHECK_NULL_VOID(analyzerUIConfig_.onAnalyzed);
145     (analyzerUIConfig_.onAnalyzed.value())(ImageAnalyzerState::STOPPED);
146     analyzerUIConfig_.onAnalyzed = std::nullopt;
147 
148     napi_value nullValue = nullptr;
149     CHECK_NULL_VOID(imageAnalyzerAdapter_);
150     imageAnalyzerAdapter_->SetImageAnalyzerConfig(nullValue);
151 }
152 
IsSupportImageAnalyzerFeature()153 bool ImageAnalyzerManager::IsSupportImageAnalyzerFeature()
154 {
155     auto node = frameNode_.Upgrade();
156     CHECK_NULL_RETURN(node, false);
157     auto eventHub = node->GetEventHub<NG::EventHub>();
158     CHECK_NULL_RETURN(eventHub, false);
159     if (!eventHub->IsEnabled()) {
160         return false;
161     }
162 
163     bool hasObscured = false;
164     if (node->GetRenderContext()->GetObscured().has_value()) {
165         auto obscuredReasons = node->GetRenderContext()->GetObscured().value();
166         hasObscured = std::any_of(obscuredReasons.begin(), obscuredReasons.end(),
167             [](const auto& reason) { return reason == ObscuredReasons::PLACEHOLDER; });
168         if (hasObscured) {
169             return false;
170         }
171     }
172 
173     if (holder_ == ImageAnalyzerHolder::IMAGE) {
174         auto imageRenderProperty = node->GetPaintProperty<NG::ImageRenderProperty>();
175         CHECK_NULL_RETURN(imageRenderProperty, false);
176         ImageRepeat repeat = imageRenderProperty->GetImageRepeat().value_or(ImageRepeat::NO_REPEAT);
177         if (repeat != ImageRepeat::NO_REPEAT) {
178             return false;
179         }
180     }
181 
182     return ImageAnalyzerMgr::GetInstance().IsImageAnalyzerSupported();
183 }
184 
IsOverlayCreated()185 bool ImageAnalyzerManager::IsOverlayCreated()
186 {
187     return isAnalyzerOverlayBuild_;
188 }
189 
UpdateAnalyzerOverlayLayout()190 void ImageAnalyzerManager::UpdateAnalyzerOverlayLayout()
191 {
192     auto node = frameNode_.Upgrade();
193     CHECK_NULL_VOID(node);
194     auto layoutProperty = node->GetLayoutProperty();
195     CHECK_NULL_VOID(layoutProperty);
196     auto padding = layoutProperty->CreatePaddingAndBorder();
197     auto overlayNode = node->GetOverlayNode();
198     CHECK_NULL_VOID(overlayNode);
199     auto overlayLayoutProperty = overlayNode->GetLayoutProperty();
200     CHECK_NULL_VOID(overlayLayoutProperty);
201     overlayLayoutProperty->UpdateMeasureType(NG::MeasureType::MATCH_PARENT);
202     overlayLayoutProperty->UpdateAlignment(Alignment::TOP_LEFT);
203     if (holder_ == ImageAnalyzerHolder::IMAGE || holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
204         overlayLayoutProperty->SetOverlayOffset(Dimension(padding.Offset().GetX()),
205                                                 Dimension(padding.Offset().GetY()));
206         if (holder_ == ImageAnalyzerHolder::IMAGE) {
207             auto renderContext = overlayNode->GetRenderContext();
208             CHECK_NULL_VOID(renderContext);
209             renderContext->SetRenderFrameOffset({ -padding.Offset().GetX(), -padding.Offset().GetY() });
210         }
211     }
212 }
213 
UpdateAnalyzerUIConfig(const RefPtr<NG::GeometryNode> & geometryNode,const PixelMapInfo & info)214 void ImageAnalyzerManager::UpdateAnalyzerUIConfig(const RefPtr<NG::GeometryNode>& geometryNode,
215     const PixelMapInfo& info)
216 {
217     CHECK_NULL_VOID(geometryNode);
218     auto node = frameNode_.Upgrade();
219     CHECK_NULL_VOID(node);
220     bool isUIConfigUpdate = false;
221 
222     auto layoutProps = node->GetLayoutProperty();
223     CHECK_NULL_VOID(layoutProps);
224     if (holder_ == ImageAnalyzerHolder::IMAGE) {
225         auto props = DynamicCast<NG::ImageLayoutProperty>(layoutProps);
226         CHECK_NULL_VOID(props);
227         if (analyzerUIConfig_.imageFit != props->GetImageFit().value_or(ImageFit::COVER)) {
228             analyzerUIConfig_.imageFit = props->GetImageFit().value_or(ImageFit::COVER);
229             isUIConfigUpdate = true;
230         }
231     }
232 
233     if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
234         isUIConfigUpdate = UpdateVideoConfig(info);
235     } else {
236         auto padding = layoutProps->CreatePaddingAndBorder();
237         float paddingWidth = holder_ == ImageAnalyzerHolder::IMAGE ? padding.left.value_or(0) +
238                                                                      padding.right.value_or(0) : 0.0f;
239         float paddingHeight = holder_ == ImageAnalyzerHolder::IMAGE ? padding.top.value_or(0) +
240                                                                       padding.bottom.value_or(0) : 0.0f;
241         NG::SizeF frameSize = geometryNode->GetFrameSize();
242         bool shouldUpdateSize = analyzerUIConfig_.contentWidth != frameSize.Width() - paddingWidth ||
243                                 analyzerUIConfig_.contentHeight != frameSize.Height() - paddingHeight;
244         if (shouldUpdateSize) {
245             analyzerUIConfig_.contentWidth = frameSize.Width() - paddingWidth;
246             analyzerUIConfig_.contentHeight = frameSize.Height()- paddingHeight;
247             isUIConfigUpdate = true;
248         }
249     }
250 
251     auto renderContext = node->GetRenderContext();
252     CHECK_NULL_VOID(renderContext);
253     auto transformMat = renderContext->GetTransformMatrixValue(Matrix4::CreateIdentity());
254     if (!(analyzerUIConfig_.transformMat == transformMat)) {
255         analyzerUIConfig_.transformMat = transformMat;
256         isUIConfigUpdate = true;
257     }
258 
259     if (isUIConfigUpdate) {
260         ImageAnalyzerMgr::GetInstance().UpdateInnerConfig(&overlayData_, &analyzerUIConfig_);
261     }
262 }
263 
UpdateVideoConfig(const PixelMapInfo & info)264 bool ImageAnalyzerManager::UpdateVideoConfig(const PixelMapInfo& info)
265 {
266     bool shouldUpdateFit = false;
267     auto node = frameNode_.Upgrade();
268     CHECK_NULL_RETURN(node, false);
269     auto layoutProps = node->GetLayoutProperty();
270     CHECK_NULL_RETURN(layoutProps, false);
271     auto videoProps = DynamicCast<NG::VideoLayoutProperty>(layoutProps);
272     if (analyzerUIConfig_.imageFit != videoProps->GetObjectFitValue(ImageFit::COVER)) {
273         analyzerUIConfig_.imageFit = videoProps->GetObjectFitValue(ImageFit::COVER);
274         shouldUpdateFit = true;
275     }
276 
277     bool shouldUpdateSize = analyzerUIConfig_.contentWidth != info.width ||
278                             analyzerUIConfig_.contentHeight != info.height ||
279                             analyzerUIConfig_.overlayOffset != info.overlayOffset;
280     if (shouldUpdateSize) {
281         analyzerUIConfig_.UpdateFromInfo(info);
282     }
283     return shouldUpdateFit || shouldUpdateSize;
284 }
285 
SetImageAnalyzerConfig(void * config)286 void ImageAnalyzerManager::SetImageAnalyzerConfig(void* config)
287 {
288     CHECK_NULL_VOID(imageAnalyzerAdapter_);
289     bool hasConfig = imageAnalyzerAdapter_->HasImageAnalyzerConfig();
290     if (hasConfig) {
291         return;
292     }
293     imageAnalyzerAdapter_->SetImageAnalyzerConfig(config);
294     auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
295     if (isAnalyzerOverlayBuild_) {
296         ImageAnalyzerMgr::GetInstance().UpdateConfig(&overlayData_, analyzerConfig);
297     }
298 }
299 
SetImageAIOptions(void * options)300 void ImageAnalyzerManager::SetImageAIOptions(void* options)
301 {
302     CHECK_NULL_VOID(imageAnalyzerAdapter_);
303     imageAnalyzerAdapter_->SetImageAnalyzerConfig(options, true);
304     auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
305     if (isAnalyzerOverlayBuild_) {
306         ImageAnalyzerMgr::GetInstance().UpdateConfig(&overlayData_, analyzerConfig);
307     }
308 }
309 
SetImageAnalyzerCallback(OnAnalyzedCallback & callback)310 void ImageAnalyzerManager::SetImageAnalyzerCallback(OnAnalyzedCallback& callback)
311 {
312     analyzerUIConfig_.onAnalyzed = callback;
313 }
314 
ReleaseImageAnalyzer()315 void ImageAnalyzerManager::ReleaseImageAnalyzer()
316 {
317     if (isAnalyzerOverlayBuild_) {
318         ImageAnalyzerMgr::GetInstance().Release(&overlayData_);
319     }
320 }
321 
UpdatePressOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,int offsetX,int offsetY,int rectWidth,int rectHeight,int pointX,int pointY,OnTextSelectedCallback callback)322 void ImageAnalyzerManager::UpdatePressOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap, int offsetX, int offsetY,
323     int rectWidth, int rectHeight, int pointX, int pointY, OnTextSelectedCallback callback)
324 {
325     analyzerUIConfig_.overlayOffset.SetX(offsetX);
326     analyzerUIConfig_.overlayOffset.SetY(offsetY);
327     if (rectWidth > 0 && rectHeight > 0) {
328         analyzerUIConfig_.touchInfo.touchPoint.x = 1.0 * pointX / rectWidth * pixelMap->GetWidth();
329         analyzerUIConfig_.touchInfo.touchPoint.y = 1.0 * pointY / rectHeight * pixelMap->GetHeight();
330     }
331     analyzerUIConfig_.touchInfo.touchType = TouchType::DOWN;
332     analyzerUIConfig_.selectedStatus = Status::SELECTED;
333     analyzerUIConfig_.menuStatus = Status::MENU_SHOW;
334     if (!analyzerUIConfig_.onTextSelected) {
335         analyzerUIConfig_.onTextSelected = std::move(callback);
336     }
337     if (pixelMap && imageAnalyzerAdapter_) {
338         analyzerUIConfig_.contentWidth = rectWidth;
339         analyzerUIConfig_.contentHeight = rectHeight;
340         analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
341         analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
342         analyzerUIConfig_.pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
343     }
344     ImageAnalyzerMgr::GetInstance().UpdatePressOverlay(&overlayData_, &analyzerUIConfig_);
345     analyzerUIConfig_.pixelmapNapiVal = nullptr;
346 }
347 
UpdateOverlayTouchInfo(int touchPointX,int touchPointY,TouchType touchType)348 void ImageAnalyzerManager::UpdateOverlayTouchInfo(int touchPointX, int touchPointY, TouchType touchType)
349 {
350     analyzerUIConfig_.touchInfo.touchPoint.x = touchPointX - analyzerUIConfig_.overlayOffset.GetX();
351     analyzerUIConfig_.touchInfo.touchPoint.y = touchPointY - analyzerUIConfig_.overlayOffset.GetY();
352     analyzerUIConfig_.touchInfo.touchType = touchType;
353     ImageAnalyzerMgr::GetInstance().UpdatePressOverlay(&overlayData_, &analyzerUIConfig_);
354 }
355 
UpdateOverlayStatus(bool status,int offsetX,int offsetY,int rectWidth,int rectHeight)356 void ImageAnalyzerManager::UpdateOverlayStatus(bool status, int offsetX, int offsetY, int rectWidth, int rectHeight)
357 {
358     if (status) {
359         analyzerUIConfig_.overlayOffset.SetX(offsetX);
360         analyzerUIConfig_.overlayOffset.SetY(offsetY);
361         analyzerUIConfig_.contentWidth = rectWidth;
362         analyzerUIConfig_.contentHeight = rectHeight;
363         analyzerUIConfig_.selectedStatus = Status::SELECTED;
364         analyzerUIConfig_.menuStatus = Status::MENU_SHOW;
365     } else {
366         analyzerUIConfig_.selectedStatus = Status::UNSELECTED;
367         analyzerUIConfig_.menuStatus = Status::MENU_HIDE;
368     }
369     ImageAnalyzerMgr::GetInstance().UpdateOverlayStatus(&overlayData_, &analyzerUIConfig_);
370 }
371 
UpdateAIButtonConfig(AIButtonConfig config)372 void ImageAnalyzerManager::UpdateAIButtonConfig(AIButtonConfig config)
373 {
374     CHECK_NULL_VOID(isAnalyzerOverlayBuild_);
375     ImageAnalyzerMgr::GetInstance().UpdateAIButtonConfig(&overlayData_, &config);
376 }
377 
UpdateOverlayActiveStatus(bool status)378 void ImageAnalyzerManager::UpdateOverlayActiveStatus(bool status)
379 {
380     CHECK_NULL_VOID(isAnalyzerOverlayBuild_);
381     ImageAnalyzerMgr::GetInstance().UpdateOverlayActiveStatus(&overlayData_, status);
382 }
383 
SetNotifySelectedCallback(OnNotifySelectedStatusCallback && callback)384 void ImageAnalyzerManager::SetNotifySelectedCallback(
385     OnNotifySelectedStatusCallback&& callback)
386 {
387     analyzerUIConfig_.onNotifySelectedStatus = std::move(callback);
388 }
389 } // namespace OHOS::Ace