1 /*
2  * Copyright (C) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "core/common/ai/data_detector_adapter.h"
17 
18 #include "iremote_object.h"
19 
20 #include "adapter/ohos/entrance/ace_container.h"
21 #include "base/log/log_wrapper.h"
22 #include "bridge/common/utils/engine_helper.h"
23 #include "core/common/ai/data_detector_mgr.h"
24 #include "core/pipeline_ng/pipeline_context.h"
25 
26 namespace OHOS::Ace {
27 
28 constexpr int32_t AI_TEXT_MAX_LENGTH = 500;
29 constexpr int32_t AI_TEXT_GAP = 100;
30 constexpr int32_t AI_DELAY_TIME = 100;
31 constexpr uint32_t SECONDS_TO_MILLISECONDS = 1000;
32 
33 const std::unordered_map<TextDataDetectType, std::string> TEXT_DETECT_MAP = {
34     { TextDataDetectType::PHONE_NUMBER, "phoneNum" }, { TextDataDetectType::URL, "url" },
35     { TextDataDetectType::EMAIL, "email" }, { TextDataDetectType::ADDRESS, "location" },
36     { TextDataDetectType::DATE_TIME, "datetime" }
37 };
38 const std::unordered_map<std::string, TextDataDetectType> TEXT_DETECT_MAP_REVERSE = {
39     { "phoneNum", TextDataDetectType::PHONE_NUMBER }, { "url", TextDataDetectType::URL },
40     { "email", TextDataDetectType::EMAIL }, { "location", TextDataDetectType::ADDRESS },
41     { "datetime", TextDataDetectType::DATE_TIME }
42 };
43 
GetAIEntityMenu()44 void DataDetectorAdapter::GetAIEntityMenu()
45 {
46     auto context = PipelineContext::GetCurrentContextSafely();
47     CHECK_NULL_VOID(context);
48     auto uiTaskExecutor = SingleTaskExecutor::Make(context->GetTaskExecutor(), TaskExecutor::TaskType::UI);
49     uiTaskExecutor.PostTask(
50         [weak = AceType::WeakClaim(this), instanceId = context->GetInstanceId()] {
51             ContainerScope scope(instanceId);
52             auto dataDetectorAdapter = weak.Upgrade();
53             CHECK_NULL_VOID(dataDetectorAdapter);
54             TAG_LOGI(AceLogTag::ACE_TEXT, "Get AI entity menu from ai_engine");
55             DataDetectorMgr::GetInstance().GetAIEntityMenu(dataDetectorAdapter->textDetectResult_);
56         },
57         "ArkUITextInitDataDetect");
58 }
59 
ShowAIEntityMenu(const AISpan & aiSpan,const NG::RectF & aiRect,const RefPtr<NG::FrameNode> & targetNode,bool isShowCopy,bool isShowSelectText)60 bool DataDetectorAdapter::ShowAIEntityMenu(const AISpan& aiSpan, const NG::RectF& aiRect,
61     const RefPtr<NG::FrameNode>& targetNode, bool isShowCopy, bool isShowSelectText)
62 {
63     if (textDetectResult_.menuOptionAndAction.empty()) {
64         TAG_LOGW(AceLogTag::ACE_TEXT, "menu option is empty, please try again");
65         GetAIEntityMenu();
66         return false;
67     }
68 
69     mainContainerId_ = Container::CurrentId();
70     std::vector<std::pair<std::string, std::function<void()>>> menuOptions;
71     auto menuOptionAndAction = textDetectResult_.menuOptionAndAction[TEXT_DETECT_MAP.at(aiSpan.type)];
72     if (menuOptionAndAction.empty()) {
73         return false;
74     }
75     if (!isShowSelectText) {
76         // delete the last option: selectText.
77         menuOptionAndAction.pop_back();
78         if (!isShowCopy) {
79         // delete the last option: copy.
80             menuOptionAndAction.pop_back();
81         }
82     }
83 
84     for (auto menuOption : menuOptionAndAction) {
85         std::function<void()> onClickEvent = [aiSpan, menuOption, weak = AceType::WeakClaim(this),
86                                                  targetNodeWeak = AceType::WeakClaim(AceType::RawPtr(targetNode))]() {
87             auto dataDetectorAdapter = weak.Upgrade();
88             CHECK_NULL_VOID(dataDetectorAdapter);
89             auto targetNode = targetNodeWeak.Upgrade();
90             CHECK_NULL_VOID(targetNode);
91             dataDetectorAdapter->OnClickAIMenuOption(aiSpan, menuOption, targetNode);
92         };
93         menuOptions.push_back(std::make_pair(menuOption.first, onClickEvent));
94     }
95     auto pipeline = NG::PipelineContext::GetCurrentContextSafely();
96     CHECK_NULL_RETURN(pipeline, false);
97     auto overlayManager = pipeline->GetOverlayManager();
98     CHECK_NULL_RETURN(overlayManager, false);
99     return overlayManager->ShowAIEntityMenu(menuOptions, aiRect, targetNode);
100 }
101 
OnClickAIMenuOption(const AISpan & aiSpan,const std::pair<std::string,FuncVariant> & menuOption,const RefPtr<NG::FrameNode> & targetNode)102 void DataDetectorAdapter::OnClickAIMenuOption(const AISpan& aiSpan,
103     const std::pair<std::string, FuncVariant>& menuOption, const RefPtr<NG::FrameNode>& targetNode)
104 {
105     TAG_LOGI(AceLogTag::ACE_TEXT, "Click AI menu option: %{public}s", menuOption.first.c_str());
106     auto pipeline = NG::PipelineContext::GetCurrentContextSafely();
107     CHECK_NULL_VOID(pipeline);
108     auto overlayManager = pipeline->GetOverlayManager();
109     CHECK_NULL_VOID(overlayManager);
110     if (targetNode) {
111         overlayManager->CloseAIEntityMenu(targetNode->GetId());
112     }
113     Container::UpdateCurrent(mainContainerId_);
114 
115     auto runtimeContext = Platform::AceContainer::GetRuntimeContext(pipeline->GetInstanceId());
116     CHECK_NULL_VOID(runtimeContext);
117     auto token = runtimeContext->GetToken();
118     auto bundleName = runtimeContext->GetBundleName();
119 
120     hasClickedMenuOption_ = true;
121     if (onClickMenu_ && std::holds_alternative<std::function<std::string()>>(menuOption.second)) {
122         onClickMenu_(std::get<std::function<std::string()>>(menuOption.second)());
123     } else if (std::holds_alternative<std::function<void(sptr<IRemoteObject>, std::string)>>(menuOption.second)) {
124         std::get<std::function<void(sptr<IRemoteObject>, std::string)>>(menuOption.second)(token, aiSpan.content);
125     } else if (std::holds_alternative<std::function<void(int32_t, std::string)>>(menuOption.second)) {
126         std::get<std::function<void(int32_t, std::string)>>(menuOption.second)(mainContainerId_, aiSpan.content);
127     } else if (std::holds_alternative<std::function<void(int32_t, std::string, std::string, int32_t, std::string)>>(
128                    menuOption.second)) {
129         std::get<std::function<void(int32_t, std::string, std::string, int32_t, std::string)>>(menuOption.second)(
130             mainContainerId_, textForAI_, bundleName, aiSpan.start, aiSpan.content);
131     } else {
132         TAG_LOGW(AceLogTag::ACE_TEXT, "No matching menu option");
133     }
134     hasClickedMenuOption_ = false;
135 }
136 
ResponseBestMatchItem(const AISpan & aiSpan)137 void DataDetectorAdapter::ResponseBestMatchItem(const AISpan& aiSpan)
138 {
139     if (textDetectResult_.menuOptionAndAction.empty()) {
140         TAG_LOGW(AceLogTag::ACE_TEXT, "menu option is empty, please try again");
141         GetAIEntityMenu();
142         return;
143     }
144     auto menuOptions = textDetectResult_.menuOptionAndAction[TEXT_DETECT_MAP.at(aiSpan.type)];
145     if (menuOptions.empty()) {
146         TAG_LOGW(AceLogTag::ACE_TEXT, "menu option is empty");
147         return;
148     }
149     OnClickAIMenuOption(aiSpan, menuOptions[0]);
150 }
151 
SetTextDetectTypes(const std::string & types)152 void DataDetectorAdapter::SetTextDetectTypes(const std::string& types)
153 {
154     textDetectTypes_ = types;
155 
156     std::set<std::string> newTypesSet;
157     std::istringstream iss(types);
158     std::string type;
159     while (std::getline(iss, type, ',')) {
160         newTypesSet.insert(type);
161     }
162     if (newTypesSet != textDetectTypesSet_) {
163         textDetectTypesSet_ = newTypesSet;
164         typeChanged_ = true;
165         aiDetectInitialized_ = false;
166         auto host = GetHost();
167         CHECK_NULL_VOID(host);
168         host->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE);
169     }
170 }
171 
ParseOriText(const std::unique_ptr<JsonValue> & entityJson,std::string & text)172 bool DataDetectorAdapter::ParseOriText(const std::unique_ptr<JsonValue>& entityJson, std::string& text)
173 {
174     TAG_LOGI(AceLogTag::ACE_TEXT, "Parse origin text entry");
175     auto runtimeContext = Platform::AceContainer::GetRuntimeContext(Container::CurrentId());
176     CHECK_NULL_RETURN(runtimeContext, false);
177     if (runtimeContext->GetBundleName() != entityJson->GetString("bundleName")) {
178         TAG_LOGW(AceLogTag::ACE_TEXT,
179             "Wrong bundleName, the context bundleName is: %{public}s, but your bundleName is: %{public}s",
180             runtimeContext->GetBundleName().c_str(), entityJson->GetString("bundleName").c_str());
181         return false;
182     }
183     auto aiSpanArray = entityJson->GetValue("entity");
184     if (aiSpanArray->IsNull() || !aiSpanArray->IsArray()) {
185         TAG_LOGW(AceLogTag::ACE_TEXT, "Wrong AI entity");
186         return false;
187     }
188 
189     aiSpanMap_.clear();
190     detectTexts_.clear();
191     AISpan aiSpan;
192     for (int32_t i = 0; i < aiSpanArray->GetArraySize(); ++i) {
193         auto item = aiSpanArray->GetArrayItem(i);
194         aiSpan.content = item->GetString("entityContent");
195         aiSpan.type = TEXT_DETECT_MAP_REVERSE.at(item->GetString("entityType"));
196         aiSpan.start = item->GetInt("start");
197         aiSpan.end = item->GetInt("end");
198         aiSpanMap_[aiSpan.start] = aiSpan;
199     }
200     aiDetectInitialized_ = true;
201     text = entityJson->GetString("content");
202     textForAI_ = text;
203     lastTextForAI_ = textForAI_;
204     if (textDetectResult_.menuOptionAndAction.empty()) {
205         GetAIEntityMenu();
206     }
207 
208     TAG_LOGI(AceLogTag::ACE_TEXT, "Parse origin text successful");
209     return true;
210 }
211 
InitTextDetect(int32_t startPos,std::string detectText)212 void DataDetectorAdapter::InitTextDetect(int32_t startPos, std::string detectText)
213 {
214     TextDataDetectInfo info;
215     info.text = detectText;
216     info.module = textDetectTypes_;
217 
218     auto context = PipelineContext::GetCurrentContextSafely();
219     CHECK_NULL_VOID(context);
220     int32_t instanceID = context->GetInstanceId();
221     auto textFunc = [weak = WeakClaim(this), instanceID, startPos, info](const TextDataDetectResult result) {
222         ContainerScope scope(instanceID);
223         auto context = PipelineContext::GetCurrentContextSafely();
224         CHECK_NULL_VOID(context);
225         auto uiTaskExecutor = SingleTaskExecutor::Make(context->GetTaskExecutor(), TaskExecutor::TaskType::UI);
226         uiTaskExecutor.PostTask(
227             [result, weak, instanceID, startPos, info] {
228                 ContainerScope scope(instanceID);
229                 auto dataDetectorAdapter = weak.Upgrade();
230                 CHECK_NULL_VOID(dataDetectorAdapter);
231                 if (info.module != dataDetectorAdapter->textDetectTypes_) {
232                     return;
233                 }
234                 dataDetectorAdapter->ParseAIResult(result, startPos);
235                 auto host = dataDetectorAdapter->GetHost();
236                 CHECK_NULL_VOID(host);
237                 host->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE);
238             },
239             "ArkUITextParseAIResult");
240     };
241 
242     auto uiTaskExecutor = SingleTaskExecutor::Make(context->GetTaskExecutor(), TaskExecutor::TaskType::BACKGROUND);
243     uiTaskExecutor.PostTask(
244         [info, textFunc] {
245             TAG_LOGI(AceLogTag::ACE_TEXT, "Start entity detect using AI");
246             DataDetectorMgr::GetInstance().DataDetect(info, textFunc);
247         },
248         "ArkUITextInitDataDetect");
249 }
250 
ParseAIResult(const TextDataDetectResult & result,int32_t startPos)251 void DataDetectorAdapter::ParseAIResult(const TextDataDetectResult& result, int32_t startPos)
252 {
253     auto entityJson = JsonUtil::ParseJsonString(result.entity);
254     CHECK_NULL_VOID(entityJson);
255     for (const auto& type : TEXT_DETECT_MAP) {
256         auto jsonValue = entityJson->GetValue(type.second);
257         ParseAIJson(jsonValue, type.first, startPos);
258     }
259 
260     if (startPos + AI_TEXT_MAX_LENGTH >= static_cast<int32_t>(StringUtils::ToWstring(textForAI_).length())) {
261         aiDetectInitialized_ = true;
262         auto entityJsonArray = JsonUtil::CreateArray(true);
263         // process with overlapping entities, leaving only the earlier ones
264         int32_t preEnd = 0;
265         auto aiSpanIterator = aiSpanMap_.begin();
266         while (aiSpanIterator != aiSpanMap_.end()) {
267             auto aiSpan = aiSpanIterator->second;
268             if (aiSpan.start < preEnd) {
269                 aiSpanIterator = aiSpanMap_.erase(aiSpanIterator);
270             } else {
271                 preEnd = aiSpan.end;
272                 ++aiSpanIterator;
273                 auto aiSpanJson = JsonUtil::Create(true);
274                 aiSpanJson->Put("start", aiSpan.start);
275                 aiSpanJson->Put("end", aiSpan.end);
276                 aiSpanJson->Put("entityContent", aiSpan.content.c_str());
277                 aiSpanJson->Put("entityType", TEXT_DETECT_MAP.at(aiSpan.type).c_str());
278                 entityJsonArray->Put(aiSpanJson);
279             }
280         }
281         auto resultJson = JsonUtil::Create(true);
282         resultJson->Put("entity", entityJsonArray);
283         resultJson->Put("code", result.code);
284         SetTextDetectResult(result);
285         FireOnResult(resultJson->ToString());
286     }
287 }
288 
ParseAIJson(const std::unique_ptr<JsonValue> & jsonValue,TextDataDetectType type,int32_t startPos)289 void DataDetectorAdapter::ParseAIJson(
290     const std::unique_ptr<JsonValue>& jsonValue, TextDataDetectType type, int32_t startPos)
291 {
292     if (!jsonValue || !jsonValue->IsArray()) {
293         TAG_LOGW(AceLogTag::ACE_TEXT, "Wrong AI result");
294         return;
295     }
296 
297     for (int32_t i = 0; i < jsonValue->GetArraySize(); ++i) {
298         auto item = jsonValue->GetArrayItem(i);
299         auto charOffset = item->GetInt("charOffset");
300         auto oriText = item->GetString("oriText");
301         auto wTextForAI = StringUtils::ToWstring(textForAI_);
302         auto wOriText = StringUtils::ToWstring(oriText);
303         int32_t end = startPos + charOffset + static_cast<int32_t>(wOriText.length());
304         if (charOffset < 0 || startPos + charOffset >= static_cast<int32_t>(wTextForAI.length()) ||
305             end >= startPos + AI_TEXT_MAX_LENGTH || oriText.empty()) {
306             TAG_LOGW(AceLogTag::ACE_TEXT, "The result of AI is wrong");
307             continue;
308         }
309         if (oriText !=
310             StringUtils::ToString(wTextForAI.substr(startPos + charOffset, static_cast<int32_t>(wOriText.length())))) {
311             TAG_LOGW(AceLogTag::ACE_TEXT, "The charOffset is wrong");
312             continue;
313         }
314         int32_t start = startPos + charOffset;
315         auto iter = aiSpanMap_.find(start);
316         if (iter != aiSpanMap_.end() && iter->second.content.length() >= oriText.length()) {
317             // both entities start at the same position, leaving the longer one
318             continue;
319         }
320 
321         TimeStamp currentDetectorTimeStamp = std::chrono::high_resolution_clock::now();
322         std::chrono::duration<float, std::ratio<1, SECONDS_TO_MILLISECONDS>> costTime =
323             currentDetectorTimeStamp - startDetectorTimeStamp_;
324         item->Put("costTime", costTime.count());
325         item->Put("resultCode", textDetectResult_.code);
326         entityJson_[start] = item->ToString();
327         TAG_LOGI(AceLogTag::ACE_TEXT, "The json of the entity is: %{private}s", entityJson_[start].c_str());
328 
329         AISpan aiSpan;
330         aiSpan.start = start;
331         aiSpan.end = end;
332         aiSpan.content = oriText;
333         aiSpan.type = type;
334         aiSpanMap_[aiSpan.start] = aiSpan;
335     }
336 }
337 
GetDetectDelayTask(const std::map<int32_t,AISpan> & aiSpanMap)338 std::function<void()> DataDetectorAdapter::GetDetectDelayTask(const std::map<int32_t, AISpan>& aiSpanMap)
339 {
340     return [aiSpanMap, weak = WeakClaim(this)]() {
341         auto dataDetectorAdapter = weak.Upgrade();
342         CHECK_NULL_VOID(dataDetectorAdapter);
343         if (dataDetectorAdapter->textForAI_.empty()) {
344             return;
345         }
346         dataDetectorAdapter->lastTextForAI_ = dataDetectorAdapter->textForAI_;
347         size_t detectTextIdx = 0;
348         auto aiSpanMapIt = aiSpanMap.begin();
349         int32_t startPos = 0;
350         bool hasSame = false;
351         auto wTextForAI = StringUtils::ToWstring(dataDetectorAdapter->textForAI_);
352         auto wTextForAILength = static_cast<int32_t>(wTextForAI.length());
353         do {
354             std::string detectText = StringUtils::ToString(
355                 wTextForAI.substr(startPos, std::min(AI_TEXT_MAX_LENGTH, wTextForAILength - startPos)));
356             bool isSameDetectText = detectTextIdx < dataDetectorAdapter->detectTexts_.size() &&
357                                     detectText == dataDetectorAdapter->detectTexts_[detectTextIdx];
358             while (!aiSpanMap.empty() && aiSpanMapIt != aiSpanMap.end() && aiSpanMapIt->first >= 0 &&
359                    aiSpanMapIt->first < std::min(wTextForAILength, startPos + AI_TEXT_MAX_LENGTH - AI_TEXT_GAP)) {
360                 auto aiContent = aiSpanMapIt->second.content;
361                 auto wAIContent = StringUtils::ToWstring(aiContent);
362                 if (isSameDetectText || aiContent == StringUtils::ToString(wTextForAI.substr(aiSpanMapIt->first,
363                     std::min(static_cast<int32_t>(wAIContent.length()), wTextForAILength - aiSpanMapIt->first)))) {
364                     dataDetectorAdapter->aiSpanMap_[aiSpanMapIt->first] = aiSpanMapIt->second;
365                     hasSame = true;
366                 }
367                 ++aiSpanMapIt;
368             }
369             if (!isSameDetectText) {
370                 dataDetectorAdapter->InitTextDetect(startPos, detectText);
371                 if (detectTextIdx < dataDetectorAdapter->detectTexts_.size()) {
372                     dataDetectorAdapter->detectTexts_[detectTextIdx] = detectText;
373                 } else {
374                     dataDetectorAdapter->detectTexts_.emplace_back(detectText);
375                 }
376             }
377             ++detectTextIdx;
378             startPos += AI_TEXT_MAX_LENGTH - AI_TEXT_GAP;
379         } while (startPos + AI_TEXT_GAP < wTextForAILength);
380         if (hasSame) {
381             auto host = dataDetectorAdapter->GetHost();
382             CHECK_NULL_VOID(host);
383             host->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE);
384         }
385     };
386 }
387 
StartAITask()388 void DataDetectorAdapter::StartAITask()
389 {
390     if (textForAI_.empty() || (!typeChanged_ && lastTextForAI_ == textForAI_)) {
391         auto host = GetHost();
392         CHECK_NULL_VOID(host);
393         host->MarkDirtyNode(NG::PROPERTY_UPDATE_MEASURE);
394         return;
395     }
396     std::map<int32_t, AISpan> aiSpanMapCopy;
397     if (!typeChanged_) {
398         aiSpanMapCopy = aiSpanMap_;
399     } else {
400         detectTexts_.clear();
401     }
402     aiSpanMap_.clear();
403     typeChanged_ = false;
404     startDetectorTimeStamp_ = std::chrono::high_resolution_clock::now();
405     auto context = PipelineContext::GetCurrentContextSafely();
406     CHECK_NULL_VOID(context);
407     auto taskExecutor = context->GetTaskExecutor();
408     CHECK_NULL_VOID(taskExecutor);
409     aiDetectDelayTask_.Cancel();
410     aiDetectDelayTask_.Reset(GetDetectDelayTask(aiSpanMapCopy));
411     taskExecutor->PostDelayedTask(
412         aiDetectDelayTask_, TaskExecutor::TaskType::UI, AI_DELAY_TIME, "ArkUITextStartAIDetect");
413 }
414 } // namespace OHOS::Ace
415