1  /*
2   * Copyright (C) 2024 Huawei Device Co., Ltd.
3   * Licensed under the Apache License, Version 2.0 (the "License");
4   * you may not use this file except in compliance with the License.
5   * You may obtain a copy of the License at
6   *
7   *     http://www.apache.org/licenses/LICENSE-2.0
8   *
9   * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  
16  #include "core/common/ai/image_analyzer_manager.h"
17  
18  #include "interfaces/inner_api/ace/ai/image_analyzer.h"
19  #include "js_native_api_types.h"
20  
21  #include "base/geometry/offset.h"
22  #include "base/image/pixel_map.h"
23  #include "base/utils/utils.h"
24  #include "core/common/ai/image_analyzer_adapter.h"
25  #include "core/common/ai/image_analyzer_mgr.h"
26  #include "core/components_ng/base/view_stack_processor.h"
27  #include "core/components_ng/pattern/image/image_pattern.h"
28  #include "core/components_ng/pattern/image/image_render_property.h"
29  #include "core/components_ng/pattern/video/video_layout_property.h"
30  #include "core/components_ng/property/measure_property.h"
31  
32  namespace OHOS::Ace {
33  
ImageAnalyzerManager(const RefPtr<NG::FrameNode> & frameNode,ImageAnalyzerHolder holder)34  ImageAnalyzerManager::ImageAnalyzerManager(const RefPtr<NG::FrameNode>& frameNode, ImageAnalyzerHolder holder)
35      : frameNode_(frameNode), holder_(holder)
36  {
37      imageAnalyzerAdapter_ = std::shared_ptr<ImageAnalyzerAdapter>(CreateImageAnalyzerAdapter());
38  }
39  
CreateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,const NG::OffsetF & offset)40  void ImageAnalyzerManager::CreateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap,
41      const NG::OffsetF& offset)
42  {
43      CHECK_NULL_VOID(imageAnalyzerAdapter_);
44      void* pixelmapNapiVal = nullptr;
45      if (pixelMap) {
46          pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
47      }
48  
49      analyzerUIConfig_.holder = holder_;
50      if (holder_ != ImageAnalyzerHolder::IMAGE && holder_ != ImageAnalyzerHolder::WEB) {
51          analyzerUIConfig_.contentWidth = pixelMap->GetWidth();
52          analyzerUIConfig_.contentHeight = pixelMap->GetHeight();
53      }
54  
55      if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
56          analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
57          analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
58          analyzerUIConfig_.overlayOffset = offset;
59      }
60  
61      RefPtr<NG::UINode> customNode;
62      {
63          NG::ScopedViewStackProcessor builderViewStackProcessor;
64          auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
65          ImageAnalyzerMgr::GetInstance().BuildNodeFunc(
66              pixelmapNapiVal, analyzerConfig, &analyzerUIConfig_, &overlayData_);
67          customNode = NG::ViewStackProcessor::GetInstance()->Finish();
68      }
69      auto overlayNode = AceType::DynamicCast<NG::FrameNode>(customNode);
70      CHECK_NULL_VOID(overlayNode);
71      auto node = frameNode_.Upgrade();
72      CHECK_NULL_VOID(node);
73      node->SetOverlayNode(overlayNode);
74      overlayNode->SetParent(AceType::WeakClaim(AceType::RawPtr(node)));
75      overlayNode->SetActive(true);
76      UpdateAnalyzerOverlayLayout();
77  
78      auto renderContext = overlayNode->GetRenderContext();
79      CHECK_NULL_VOID(renderContext);
80      renderContext->UpdateZIndex(INT32_MAX);
81      auto focusHub = overlayNode->GetOrCreateFocusHub();
82      CHECK_NULL_VOID(focusHub);
83      focusHub->SetFocusable(false);
84      overlayNode->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE_SELF);
85  
86      isAnalyzerOverlayBuild_ = true;
87      CHECK_NULL_VOID(analyzerUIConfig_.onAnalyzed);
88      (analyzerUIConfig_.onAnalyzed.value())(ImageAnalyzerState::FINISHED);
89      analyzerUIConfig_.onAnalyzed = std::nullopt;
90  }
91  
UpdateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,const NG::OffsetF & offset)92  void ImageAnalyzerManager::UpdateAnalyzerOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap,
93      const NG::OffsetF& offset)
94  {
95      if (!isAnalyzerOverlayBuild_) {
96          return;
97      }
98  
99      auto node = frameNode_.Upgrade();
100      CHECK_NULL_VOID(node);
101      if (holder_ == ImageAnalyzerHolder::IMAGE) {
102          auto imagePattern = AceType::DynamicCast<NG::ImagePattern>(node->GetPattern());
103          CHECK_NULL_VOID(imagePattern);
104          if (!imagePattern->hasSceneChanged()) {
105              return;
106          }
107      }
108  
109      CHECK_NULL_VOID(pixelMap);
110      if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
111          analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
112          analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
113          analyzerUIConfig_.overlayOffset = offset;
114      }
115  
116      if (holder_ != ImageAnalyzerHolder::IMAGE) {
117          analyzerUIConfig_.contentWidth = pixelMap->GetWidth();
118          analyzerUIConfig_.contentHeight = pixelMap->GetHeight();
119      }
120  
121      CHECK_NULL_VOID(imageAnalyzerAdapter_);
122      auto pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
123      auto overlayNode = node->GetOverlayNode();
124      CHECK_NULL_VOID(overlayNode);
125      auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
126      ImageAnalyzerMgr::GetInstance().UpdateImage(&overlayData_, pixelmapNapiVal, analyzerConfig, &analyzerUIConfig_);
127      overlayNode->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE_SELF);
128  }
129  
DestroyAnalyzerOverlay()130  void ImageAnalyzerManager::DestroyAnalyzerOverlay()
131  {
132      ReleaseImageAnalyzer();
133  
134      if (!isAnalyzerOverlayBuild_) {
135          return;
136      }
137      auto node = frameNode_.Upgrade();
138      CHECK_NULL_VOID(node);
139      auto overlayNode = node->GetOverlayNode();
140      CHECK_NULL_VOID(overlayNode);
141      node->SetOverlayNode(RefPtr<NG::FrameNode>());
142  
143      isAnalyzerOverlayBuild_ = false;
144      CHECK_NULL_VOID(analyzerUIConfig_.onAnalyzed);
145      (analyzerUIConfig_.onAnalyzed.value())(ImageAnalyzerState::STOPPED);
146      analyzerUIConfig_.onAnalyzed = std::nullopt;
147  
148      napi_value nullValue = nullptr;
149      CHECK_NULL_VOID(imageAnalyzerAdapter_);
150      imageAnalyzerAdapter_->SetImageAnalyzerConfig(nullValue);
151  }
152  
IsSupportImageAnalyzerFeature()153  bool ImageAnalyzerManager::IsSupportImageAnalyzerFeature()
154  {
155      auto node = frameNode_.Upgrade();
156      CHECK_NULL_RETURN(node, false);
157      auto eventHub = node->GetEventHub<NG::EventHub>();
158      CHECK_NULL_RETURN(eventHub, false);
159      if (!eventHub->IsEnabled()) {
160          return false;
161      }
162  
163      bool hasObscured = false;
164      if (node->GetRenderContext()->GetObscured().has_value()) {
165          auto obscuredReasons = node->GetRenderContext()->GetObscured().value();
166          hasObscured = std::any_of(obscuredReasons.begin(), obscuredReasons.end(),
167              [](const auto& reason) { return reason == ObscuredReasons::PLACEHOLDER; });
168          if (hasObscured) {
169              return false;
170          }
171      }
172  
173      if (holder_ == ImageAnalyzerHolder::IMAGE) {
174          auto imageRenderProperty = node->GetPaintProperty<NG::ImageRenderProperty>();
175          CHECK_NULL_RETURN(imageRenderProperty, false);
176          ImageRepeat repeat = imageRenderProperty->GetImageRepeat().value_or(ImageRepeat::NO_REPEAT);
177          if (repeat != ImageRepeat::NO_REPEAT) {
178              return false;
179          }
180      }
181  
182      return ImageAnalyzerMgr::GetInstance().IsImageAnalyzerSupported();
183  }
184  
IsOverlayCreated()185  bool ImageAnalyzerManager::IsOverlayCreated()
186  {
187      return isAnalyzerOverlayBuild_;
188  }
189  
UpdateAnalyzerOverlayLayout()190  void ImageAnalyzerManager::UpdateAnalyzerOverlayLayout()
191  {
192      auto node = frameNode_.Upgrade();
193      CHECK_NULL_VOID(node);
194      auto layoutProperty = node->GetLayoutProperty();
195      CHECK_NULL_VOID(layoutProperty);
196      auto padding = layoutProperty->CreatePaddingAndBorder();
197      auto overlayNode = node->GetOverlayNode();
198      CHECK_NULL_VOID(overlayNode);
199      auto overlayLayoutProperty = overlayNode->GetLayoutProperty();
200      CHECK_NULL_VOID(overlayLayoutProperty);
201      overlayLayoutProperty->UpdateMeasureType(NG::MeasureType::MATCH_PARENT);
202      overlayLayoutProperty->UpdateAlignment(Alignment::TOP_LEFT);
203      if (holder_ == ImageAnalyzerHolder::IMAGE || holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
204          overlayLayoutProperty->SetOverlayOffset(Dimension(padding.Offset().GetX()),
205                                                  Dimension(padding.Offset().GetY()));
206          if (holder_ == ImageAnalyzerHolder::IMAGE) {
207              auto renderContext = overlayNode->GetRenderContext();
208              CHECK_NULL_VOID(renderContext);
209              renderContext->SetRenderFrameOffset({ -padding.Offset().GetX(), -padding.Offset().GetY() });
210          }
211      }
212  }
213  
UpdateAnalyzerUIConfig(const RefPtr<NG::GeometryNode> & geometryNode,const PixelMapInfo & info)214  void ImageAnalyzerManager::UpdateAnalyzerUIConfig(const RefPtr<NG::GeometryNode>& geometryNode,
215      const PixelMapInfo& info)
216  {
217      CHECK_NULL_VOID(geometryNode);
218      auto node = frameNode_.Upgrade();
219      CHECK_NULL_VOID(node);
220      bool isUIConfigUpdate = false;
221  
222      auto layoutProps = node->GetLayoutProperty();
223      CHECK_NULL_VOID(layoutProps);
224      if (holder_ == ImageAnalyzerHolder::IMAGE) {
225          auto props = DynamicCast<NG::ImageLayoutProperty>(layoutProps);
226          CHECK_NULL_VOID(props);
227          if (analyzerUIConfig_.imageFit != props->GetImageFit().value_or(ImageFit::COVER)) {
228              analyzerUIConfig_.imageFit = props->GetImageFit().value_or(ImageFit::COVER);
229              isUIConfigUpdate = true;
230          }
231      }
232  
233      if (holder_ == ImageAnalyzerHolder::VIDEO_CUSTOM) {
234          isUIConfigUpdate = UpdateVideoConfig(info);
235      } else {
236          auto padding = layoutProps->CreatePaddingAndBorder();
237          float paddingWidth = holder_ == ImageAnalyzerHolder::IMAGE ? padding.left.value_or(0) +
238                                                                       padding.right.value_or(0) : 0.0f;
239          float paddingHeight = holder_ == ImageAnalyzerHolder::IMAGE ? padding.top.value_or(0) +
240                                                                        padding.bottom.value_or(0) : 0.0f;
241          NG::SizeF frameSize = geometryNode->GetFrameSize();
242          bool shouldUpdateSize = analyzerUIConfig_.contentWidth != frameSize.Width() - paddingWidth ||
243                                  analyzerUIConfig_.contentHeight != frameSize.Height() - paddingHeight;
244          if (shouldUpdateSize) {
245              analyzerUIConfig_.contentWidth = frameSize.Width() - paddingWidth;
246              analyzerUIConfig_.contentHeight = frameSize.Height()- paddingHeight;
247              isUIConfigUpdate = true;
248          }
249      }
250  
251      auto renderContext = node->GetRenderContext();
252      CHECK_NULL_VOID(renderContext);
253      auto transformMat = renderContext->GetTransformMatrixValue(Matrix4::CreateIdentity());
254      if (!(analyzerUIConfig_.transformMat == transformMat)) {
255          analyzerUIConfig_.transformMat = transformMat;
256          isUIConfigUpdate = true;
257      }
258  
259      if (isUIConfigUpdate) {
260          ImageAnalyzerMgr::GetInstance().UpdateInnerConfig(&overlayData_, &analyzerUIConfig_);
261      }
262  }
263  
UpdateVideoConfig(const PixelMapInfo & info)264  bool ImageAnalyzerManager::UpdateVideoConfig(const PixelMapInfo& info)
265  {
266      bool shouldUpdateFit = false;
267      auto node = frameNode_.Upgrade();
268      CHECK_NULL_RETURN(node, false);
269      auto layoutProps = node->GetLayoutProperty();
270      CHECK_NULL_RETURN(layoutProps, false);
271      auto videoProps = DynamicCast<NG::VideoLayoutProperty>(layoutProps);
272      if (analyzerUIConfig_.imageFit != videoProps->GetObjectFitValue(ImageFit::COVER)) {
273          analyzerUIConfig_.imageFit = videoProps->GetObjectFitValue(ImageFit::COVER);
274          shouldUpdateFit = true;
275      }
276  
277      bool shouldUpdateSize = analyzerUIConfig_.contentWidth != info.width ||
278                              analyzerUIConfig_.contentHeight != info.height ||
279                              analyzerUIConfig_.overlayOffset != info.overlayOffset;
280      if (shouldUpdateSize) {
281          analyzerUIConfig_.UpdateFromInfo(info);
282      }
283      return shouldUpdateFit || shouldUpdateSize;
284  }
285  
SetImageAnalyzerConfig(void * config)286  void ImageAnalyzerManager::SetImageAnalyzerConfig(void* config)
287  {
288      CHECK_NULL_VOID(imageAnalyzerAdapter_);
289      bool hasConfig = imageAnalyzerAdapter_->HasImageAnalyzerConfig();
290      if (hasConfig) {
291          return;
292      }
293      imageAnalyzerAdapter_->SetImageAnalyzerConfig(config);
294      auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
295      if (isAnalyzerOverlayBuild_) {
296          ImageAnalyzerMgr::GetInstance().UpdateConfig(&overlayData_, analyzerConfig);
297      }
298  }
299  
SetImageAIOptions(void * options)300  void ImageAnalyzerManager::SetImageAIOptions(void* options)
301  {
302      CHECK_NULL_VOID(imageAnalyzerAdapter_);
303      imageAnalyzerAdapter_->SetImageAnalyzerConfig(options, true);
304      auto analyzerConfig = imageAnalyzerAdapter_->GetImageAnalyzerConfig();
305      if (isAnalyzerOverlayBuild_) {
306          ImageAnalyzerMgr::GetInstance().UpdateConfig(&overlayData_, analyzerConfig);
307      }
308  }
309  
SetImageAnalyzerCallback(OnAnalyzedCallback & callback)310  void ImageAnalyzerManager::SetImageAnalyzerCallback(OnAnalyzedCallback& callback)
311  {
312      analyzerUIConfig_.onAnalyzed = callback;
313  }
314  
ReleaseImageAnalyzer()315  void ImageAnalyzerManager::ReleaseImageAnalyzer()
316  {
317      if (isAnalyzerOverlayBuild_) {
318          ImageAnalyzerMgr::GetInstance().Release(&overlayData_);
319      }
320  }
321  
UpdatePressOverlay(const RefPtr<OHOS::Ace::PixelMap> & pixelMap,int offsetX,int offsetY,int rectWidth,int rectHeight,int pointX,int pointY,OnTextSelectedCallback callback)322  void ImageAnalyzerManager::UpdatePressOverlay(const RefPtr<OHOS::Ace::PixelMap>& pixelMap, int offsetX, int offsetY,
323      int rectWidth, int rectHeight, int pointX, int pointY, OnTextSelectedCallback callback)
324  {
325      analyzerUIConfig_.overlayOffset.SetX(offsetX);
326      analyzerUIConfig_.overlayOffset.SetY(offsetY);
327      if (rectWidth > 0 && rectHeight > 0) {
328          analyzerUIConfig_.touchInfo.touchPoint.x = 1.0 * pointX / rectWidth * pixelMap->GetWidth();
329          analyzerUIConfig_.touchInfo.touchPoint.y = 1.0 * pointY / rectHeight * pixelMap->GetHeight();
330      }
331      analyzerUIConfig_.touchInfo.touchType = TouchType::DOWN;
332      analyzerUIConfig_.selectedStatus = Status::SELECTED;
333      analyzerUIConfig_.menuStatus = Status::MENU_SHOW;
334      if (!analyzerUIConfig_.onTextSelected) {
335          analyzerUIConfig_.onTextSelected = std::move(callback);
336      }
337      if (pixelMap && imageAnalyzerAdapter_) {
338          analyzerUIConfig_.contentWidth = rectWidth;
339          analyzerUIConfig_.contentHeight = rectHeight;
340          analyzerUIConfig_.pixelMapWidth = pixelMap->GetWidth();
341          analyzerUIConfig_.pixelMapHeight = pixelMap->GetHeight();
342          analyzerUIConfig_.pixelmapNapiVal = imageAnalyzerAdapter_->ConvertPixmapNapi(pixelMap);
343      }
344      ImageAnalyzerMgr::GetInstance().UpdatePressOverlay(&overlayData_, &analyzerUIConfig_);
345      analyzerUIConfig_.pixelmapNapiVal = nullptr;
346  }
347  
UpdateOverlayTouchInfo(int touchPointX,int touchPointY,TouchType touchType)348  void ImageAnalyzerManager::UpdateOverlayTouchInfo(int touchPointX, int touchPointY, TouchType touchType)
349  {
350      analyzerUIConfig_.touchInfo.touchPoint.x = touchPointX - analyzerUIConfig_.overlayOffset.GetX();
351      analyzerUIConfig_.touchInfo.touchPoint.y = touchPointY - analyzerUIConfig_.overlayOffset.GetY();
352      analyzerUIConfig_.touchInfo.touchType = touchType;
353      ImageAnalyzerMgr::GetInstance().UpdatePressOverlay(&overlayData_, &analyzerUIConfig_);
354  }
355  
UpdateOverlayStatus(bool status,int offsetX,int offsetY,int rectWidth,int rectHeight)356  void ImageAnalyzerManager::UpdateOverlayStatus(bool status, int offsetX, int offsetY, int rectWidth, int rectHeight)
357  {
358      if (status) {
359          analyzerUIConfig_.overlayOffset.SetX(offsetX);
360          analyzerUIConfig_.overlayOffset.SetY(offsetY);
361          analyzerUIConfig_.contentWidth = rectWidth;
362          analyzerUIConfig_.contentHeight = rectHeight;
363          analyzerUIConfig_.selectedStatus = Status::SELECTED;
364          analyzerUIConfig_.menuStatus = Status::MENU_SHOW;
365      } else {
366          analyzerUIConfig_.selectedStatus = Status::UNSELECTED;
367          analyzerUIConfig_.menuStatus = Status::MENU_HIDE;
368      }
369      ImageAnalyzerMgr::GetInstance().UpdateOverlayStatus(&overlayData_, &analyzerUIConfig_);
370  }
371  
UpdateAIButtonConfig(AIButtonConfig config)372  void ImageAnalyzerManager::UpdateAIButtonConfig(AIButtonConfig config)
373  {
374      CHECK_NULL_VOID(isAnalyzerOverlayBuild_);
375      ImageAnalyzerMgr::GetInstance().UpdateAIButtonConfig(&overlayData_, &config);
376  }
377  
UpdateOverlayActiveStatus(bool status)378  void ImageAnalyzerManager::UpdateOverlayActiveStatus(bool status)
379  {
380      CHECK_NULL_VOID(isAnalyzerOverlayBuild_);
381      ImageAnalyzerMgr::GetInstance().UpdateOverlayActiveStatus(&overlayData_, status);
382  }
383  
SetNotifySelectedCallback(OnNotifySelectedStatusCallback && callback)384  void ImageAnalyzerManager::SetNotifySelectedCallback(
385      OnNotifySelectedStatusCallback&& callback)
386  {
387      analyzerUIConfig_.onNotifySelectedStatus = std::move(callback);
388  }
389  } // namespace OHOS::Ace