1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "js_third_provider_interaction_operation.h"
17
18 #include <algorithm>
19
20 #include "accessibility_constants.h"
21 #include "accessibility_event_info.h"
22 #include "accessibility_system_ability_client.h"
23
24 #include "adapter/ohos/entrance/ace_application_info.h"
25 #include "adapter/ohos/entrance/ace_container.h"
26 #include "base/log/ace_trace.h"
27 #include "base/log/dump_log.h"
28 #include "base/log/event_report.h"
29 #include "base/log/log.h"
30 #include "core/pipeline/pipeline_context.h"
31 #include "core/pipeline_ng/pipeline_context.h"
32 #include "js_third_provider_interaction_operation_utils.h"
33
34 namespace OHOS::Ace::Framework {
35 namespace {
LogAccessibilityElementInfo(const std::string & tag,const Accessibility::AccessibilityElementInfo & info)36 void LogAccessibilityElementInfo(
37 const std::string& tag, const Accessibility::AccessibilityElementInfo& info)
38 {
39 std::string printStr;
40 printStr.append(tag).append(", info: [");
41 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
42 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
43 printStr.append(", childCount: ").append(std::to_string(info.GetChildCount()));
44 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
45 printStr.append(", parentNodeId: ").append(std::to_string(info.GetParentNodeId()));
46 printStr.append(", checkable: ").append(info.IsCheckable() ? "true" : "false");
47 printStr.append(", checked: ").append(info.IsChecked() ? "true" : "false");
48 printStr.append(", focusable: ").append(info.IsFocusable() ? "true" : "false");
49 printStr.append(", focused: ").append(info.IsFocused() ? "true" : "false");
50 printStr.append(", visible: ").append(info.IsVisible() ? "true" : "false");
51 printStr.append(", hasAccessibilityFocus: ")
52 .append(info.HasAccessibilityFocus() ? "true" : "false");
53 printStr.append(", selected: ").append(info.IsSelected() ? "true" : "false");
54 printStr.append(", clickable: ").append(info.IsClickable() ? "true" : "false");
55 printStr.append(", longClickable: ").append(info.IsLongClickable() ? "true" : "false");
56 printStr.append(", enabled: ").append(info.IsEnabled() ? "true" : "false");
57 printStr.append(", componentType: ").append(info.GetComponentType());
58 printStr.append(", content: ").append(info.GetContent());
59 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
60 printStr.append(", triggerAction: ")
61 .append(std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
62 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
63 printStr.append(", childTreeId: ").append(std::to_string(info.GetChildTreeId()));
64 printStr.append(", belongTreeId: ").append(std::to_string(info.GetBelongTreeId()));
65 printStr.append(", parentWindowId: ").append(std::to_string(info.GetParentWindowId()));
66 printStr.append(", accessibilityGroup: ").append(info.GetAccessibilityGroup() ? "true" : "false");
67 std::string actionStr;
68 actionStr.append("{");
69 for (const auto& action : info.GetActionList()) {
70 actionStr.append(std::to_string(
71 static_cast<int32_t>(action.GetActionType()))).append(" ");
72 }
73 actionStr.append("}");
74 printStr.append(", action: ").append(actionStr);
75 printStr.append("]");
76 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
77 }
78
LogAccessibilityEventInfo(const std::string & tag,const Accessibility::AccessibilityEventInfo & info)79 void LogAccessibilityEventInfo(
80 const std::string& tag, const Accessibility::AccessibilityEventInfo& info)
81 {
82 std::string printStr;
83 printStr.append(tag).append(", info: [");
84 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
85 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
86 printStr.append(", viewId: ").append(std::to_string(info.GetViewId()));
87 printStr.append(", componentType: ").append(info.GetComponentType());
88 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
89 printStr.append(", eventType: ").append(
90 std::to_string(static_cast<int32_t>(info.GetEventType())));
91 printStr.append(", triggerAction: ").append(
92 std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
93 printStr.append(", requestFocusElementId: ").append(
94 std::to_string(info.GetRequestFocusElementId()));
95 printStr.append("]");
96 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
97 }
FillNodeConfig(const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)98 void FillNodeConfig(
99 const NodeConfig& config, Accessibility::AccessibilityElementInfo& info)
100 {
101 OHOS::Accessibility::Rect oldRect = info.GetRectInScreen();
102 OHOS::Accessibility::Rect newRect = OHOS::Accessibility::Rect(
103 oldRect.GetLeftTopXScreenPostion() + static_cast<int32_t>(config.offset.GetX()),
104 oldRect.GetLeftTopYScreenPostion() + static_cast<int32_t>(config.offset.GetY()),
105 oldRect.GetRightBottomXScreenPostion() + static_cast<int32_t>(config.offset.GetX()),
106 oldRect.GetRightBottomYScreenPostion() + static_cast<int32_t>(config.offset.GetY()));
107 info.SetRectInScreen(newRect);
108 info.SetPageId(config.pageId);
109 info.SetWindowId(config.windowId);
110 info.SetBelongTreeId(config.belongTreeId);
111 info.SetChildTreeIdAndWinId(config.childTreeId, config.childWindowId);
112 info.SetParentWindowId(config.parentWindowId);
113 info.SetBundleName(config.bundleName);
114 info.SetInspectorKey(config.inspectorKey);
115 int64_t splitElementId = info.GetAccessibilityId();
116 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(config.belongTreeId, splitElementId);
117 info.SetAccessibilityId(splitElementId);
118 }
119
CopyNativeInfoToAccessibilityElementInfo(const ArkUI_AccessibilityElementInfo & elementInfo,const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)120 void CopyNativeInfoToAccessibilityElementInfo(
121 const ArkUI_AccessibilityElementInfo& elementInfo,
122 const NodeConfig& config,
123 Accessibility::AccessibilityElementInfo& info)
124 {
125 TransformAccessbilityElementInfo(elementInfo, info);
126 FillNodeConfig(config, info);
127 }
128
CopyNativeInfosToAccessibilityElementInfos(const std::vector<ArkUI_AccessibilityElementInfo> & nativeInfos,const NodeConfig & config,std::list<Accessibility::AccessibilityElementInfo> & infos)129 void CopyNativeInfosToAccessibilityElementInfos(
130 const std::vector<ArkUI_AccessibilityElementInfo>& nativeInfos,
131 const NodeConfig& config,
132 std::list<Accessibility::AccessibilityElementInfo>& infos)
133 {
134 for (const auto& nativeInfo : nativeInfos) {
135 Accessibility::AccessibilityElementInfo info;
136 TransformAccessbilityElementInfo(nativeInfo, info);
137 FillNodeConfig(config, info);
138 infos.push_back(info);
139 }
140 }
141 }
142
JsThirdProviderInteractionOperation(const WeakPtr<AccessibilityProvider> & accessibilityProvider,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager,WeakPtr<NG::FrameNode> host)143 JsThirdProviderInteractionOperation::JsThirdProviderInteractionOperation(
144 const WeakPtr<AccessibilityProvider>& accessibilityProvider,
145 const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager,
146 WeakPtr<NG::FrameNode> host)
147 : accessibilityProvider_(accessibilityProvider),
148 jsAccessibilityManager_(jsAccessibilityManager),
149 host_(host)
150 {
151 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Create");
152 }
153
~JsThirdProviderInteractionOperation()154 JsThirdProviderInteractionOperation::~JsThirdProviderInteractionOperation()
155 {
156 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Destory");
157 }
158
Initialize()159 void JsThirdProviderInteractionOperation::Initialize()
160 {
161 auto provider = accessibilityProvider_.Upgrade();
162 CHECK_NULL_VOID(provider);
163 }
164
SearchElementInfoByAccessibilityId(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t mode)165 void JsThirdProviderInteractionOperation::SearchElementInfoByAccessibilityId(
166 const int64_t elementId, const int32_t requestId,
167 Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode)
168 {
169 // 1. Get real elementId
170 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
171 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
172 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
173 elementId, splitElementId, splitTreeId);
174
175 // 2. FindAccessibilityNodeInfosById by provider
176 std::list<Accessibility::AccessibilityElementInfo> infos;
177 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
178 splitElementId, mode, requestId, infos);
179 if (!ret) {
180 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
181 "SearchElementInfoByAccessibilityId failed.");
182 infos.clear();
183 }
184
185 // 3. Return result
186 SetSearchElementInfoByAccessibilityIdResult(callback, std::move(infos), requestId);
187 }
188
FindAccessibilityNodeInfosByIdFromProvider(const int64_t splitElementId,const int32_t mode,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos)189 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByIdFromProvider(
190 const int64_t splitElementId, const int32_t mode, const int32_t requestId,
191 std::list<Accessibility::AccessibilityElementInfo>& infos)
192 {
193 auto provider = accessibilityProvider_.Upgrade();
194 CHECK_NULL_RETURN(provider, false);
195 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
196 int32_t code = provider->FindAccessibilityNodeInfosById(
197 splitElementId, mode, requestId, nativeInfos);
198 if (code != 0) {
199 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
200 "FindAccessibilityNodeInfosByIdFromProvider failed: %{public}d", code);
201 return false;
202 }
203
204 NodeConfig config;
205 GetNodeConfig(config);
206 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
207 return !infos.empty();
208 }
209
SetSearchElementInfoByAccessibilityIdResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)210 void JsThirdProviderInteractionOperation::SetSearchElementInfoByAccessibilityIdResult(
211 AccessibilityElementOperatorCallback& callback,
212 std::list<AccessibilityElementInfo>&& infos,
213 const int32_t requestId)
214 {
215 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
216 CHECK_NULL_VOID(jsAccessibilityManager);
217 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
218 CHECK_NULL_VOID(context);
219 CHECK_NULL_VOID(context->GetTaskExecutor());
220 context->GetTaskExecutor()->PostTask(
221 [jsMgr = jsAccessibilityManager_,
222 infos = std::move(infos), &callback, requestId] () mutable {
223 auto jsAccessibilityManager = jsMgr.Upgrade();
224 CHECK_NULL_VOID(jsAccessibilityManager);
225 if (!jsAccessibilityManager->IsRegister()) {
226 return;
227 }
228 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
229 callback.SetSearchElementInfoByAccessibilityIdResult(infos, requestId);
230 }, TaskExecutor::TaskType::BACKGROUND, "SearchElementInfoByAccessibilityId");
231 }
232
SearchElementInfosByText(const int64_t elementId,const std::string & text,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)233 void JsThirdProviderInteractionOperation::SearchElementInfosByText(
234 const int64_t elementId, const std::string& text, const int32_t requestId,
235 Accessibility::AccessibilityElementOperatorCallback& callback)
236 {
237 // 1. Get real elementId
238 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
239 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
240 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
241 elementId, splitElementId, splitTreeId);
242
243 // 2. FindAccessibilityNodeInfosByText from provider
244 std::list<Accessibility::AccessibilityElementInfo> infos;
245 bool ret = FindAccessibilityNodeInfosByTextFromProvider(
246 splitElementId, text, requestId, infos);
247 if (!ret) {
248 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
249 "SearchElementInfosByText failed.");
250 infos.clear();
251 }
252
253 // 3. Return result
254 SetSearchElementInfoByTextResult(callback, std::move(infos), requestId);
255 }
256
FindAccessibilityNodeInfosByTextFromProvider(const int64_t splitElementId,const std::string & text,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos)257 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByTextFromProvider(
258 const int64_t splitElementId, const std::string& text, const int32_t requestId,
259 std::list<Accessibility::AccessibilityElementInfo>& infos)
260 {
261 auto provider = accessibilityProvider_.Upgrade();
262 CHECK_NULL_RETURN(provider, false);
263 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
264 int32_t code = provider->FindAccessibilityNodeInfosByText(
265 splitElementId, text, requestId, nativeInfos);
266 if (code != 0) {
267 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
268 "FindAccessibilityNodeInfosByTextFromProvider failed: %{public}d", code);
269 return false;
270 }
271
272 NodeConfig config;
273 GetNodeConfig(config);
274 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
275 return true;
276 }
277
SetSearchElementInfoByTextResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)278 void JsThirdProviderInteractionOperation::SetSearchElementInfoByTextResult(
279 AccessibilityElementOperatorCallback& callback,
280 std::list<AccessibilityElementInfo>&& infos,
281 const int32_t requestId)
282 {
283 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
284 CHECK_NULL_VOID(jsAccessibilityManager);
285 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
286 CHECK_NULL_VOID(context);
287 CHECK_NULL_VOID(context->GetTaskExecutor());
288 context->GetTaskExecutor()->PostTask(
289 [jsMgr = jsAccessibilityManager_,
290 infos = std::move(infos), &callback, requestId] () mutable {
291 auto jsAccessibilityManager = jsMgr.Upgrade();
292 CHECK_NULL_VOID(jsAccessibilityManager);
293 if (!jsAccessibilityManager->IsRegister()) {
294 return;
295 }
296 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
297 callback.SetSearchElementInfoByTextResult(infos, requestId);
298 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
299 }
300
FindFocusedElementInfo(const int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)301 void JsThirdProviderInteractionOperation::FindFocusedElementInfo(
302 const int64_t elementId, const int32_t focusType, const int32_t requestId,
303 Accessibility::AccessibilityElementOperatorCallback& callback)
304 {
305 // 1. Get real elementId
306 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
307 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
308 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
309 elementId, splitElementId, splitTreeId);
310
311 // 2. FindFocusedAccessibilityNode from provider
312 Accessibility::AccessibilityElementInfo info;
313 bool ret = FindFocusedElementInfoFromProvider(
314 splitElementId, focusType, requestId, info);
315 if (!ret) {
316 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
317 "SearchElementInfosByText failed.");
318 info.SetValidElement(false);
319 }
320
321 // 3. Return result
322 SetFindFocusedElementInfoResult(callback, info, requestId);
323 }
324
FindFocusedElementInfoFromProvider(int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)325 bool JsThirdProviderInteractionOperation::FindFocusedElementInfoFromProvider(
326 int64_t elementId, const int32_t focusType, const int32_t requestId,
327 Accessibility::AccessibilityElementInfo& info)
328 {
329 auto provider = accessibilityProvider_.Upgrade();
330 CHECK_NULL_RETURN(provider, false);
331 ArkUI_AccessibilityElementInfo nativeInfo;
332 int32_t code = provider->FindFocusedAccessibilityNode(
333 elementId, focusType, requestId, nativeInfo);
334 if (code != 0) {
335 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
336 "FindFocusedElementInfoFromProvider failed: %{public}d", code);
337 return false;
338 }
339
340 NodeConfig config;
341 GetNodeConfig(config);
342 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
343 return true;
344 }
345
SetFindFocusedElementInfoResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)346 void JsThirdProviderInteractionOperation::SetFindFocusedElementInfoResult(
347 AccessibilityElementOperatorCallback& callback,
348 AccessibilityElementInfo& info,
349 const int32_t requestId)
350 {
351 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
352 CHECK_NULL_VOID(jsAccessibilityManager);
353 jsAccessibilityManager->UpdateElementInfoTreeId(info);
354 callback.SetFindFocusedElementInfoResult(info, requestId);
355 }
356
FocusMoveSearch(const int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)357 void JsThirdProviderInteractionOperation::FocusMoveSearch(
358 const int64_t elementId, const int32_t direction, const int32_t requestId,
359 Accessibility::AccessibilityElementOperatorCallback& callback)
360 {
361 // 1. Get real elementId
362 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
363 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
364 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
365 elementId, splitElementId, splitTreeId);
366
367 // 2. FindNextFocusAccessibilityNode from provider
368 Accessibility::AccessibilityElementInfo info;
369 bool ret = FocusMoveSearchProvider(
370 splitElementId, direction, requestId, info);
371 if (!ret) {
372 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
373 "FocusMoveSearch failed.");
374 info.SetValidElement(false);
375 }
376
377 // 3. Return result
378 SetFocusMoveSearchResult(callback, info, requestId);
379 }
380
FocusMoveSearchProvider(int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)381 bool JsThirdProviderInteractionOperation::FocusMoveSearchProvider(
382 int64_t elementId, const int32_t direction, const int32_t requestId,
383 Accessibility::AccessibilityElementInfo& info)
384 {
385 auto provider = accessibilityProvider_.Upgrade();
386 CHECK_NULL_RETURN(provider, false);
387 ArkUI_AccessibilityElementInfo nativeInfo;
388 int32_t code = provider->FindNextFocusAccessibilityNode(
389 elementId, direction, requestId, nativeInfo);
390 if (code != 0) {
391 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
392 "FocusMoveSearchProvider failed: %{public}d", code);
393 return false;
394 }
395
396 NodeConfig config;
397 GetNodeConfig(config);
398 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
399 return true;
400 }
401
SetFocusMoveSearchResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)402 void JsThirdProviderInteractionOperation::SetFocusMoveSearchResult(
403 AccessibilityElementOperatorCallback& callback,
404 AccessibilityElementInfo& info,
405 const int32_t requestId)
406 {
407 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
408 CHECK_NULL_VOID(jsAccessibilityManager);
409 jsAccessibilityManager->UpdateElementInfoTreeId(info);
410 callback.SetFocusMoveSearchResult(info, requestId);
411 }
412
ExecuteAction(const int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)413 void JsThirdProviderInteractionOperation::ExecuteAction(
414 const int64_t elementId, const int32_t action,
415 const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
416 Accessibility::AccessibilityElementOperatorCallback& callback)
417 {
418 // 1. Get real elementId
419 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
420 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
421 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
422 elementId, splitElementId, splitTreeId);
423
424 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAction elementId: %{public}" PRId64 ","
425 " action: %{public}d, requestId: %{public}d, splitElementId: %{public}" PRId64 ","
426 " splitTreeId: %{public}d",
427 elementId, action, requestId, splitElementId, splitTreeId);
428 // 2. FindNextFocusAccessibilityNode from provider
429 std::list<Accessibility::AccessibilityElementInfo> infos;
430 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
431 splitElementId, 0, requestId, infos);
432 if (!ret) {
433 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "Find info failed when ExecuteAction.");
434 SetExecuteActionResult(callback, false, requestId);
435 return;
436 }
437
438 // 3. DrawBound
439 ExecuteActionForThird(splitElementId, infos.front(), action);
440
441 //4. ExecuteAccessibilityAction To provider
442 ret = ExecuteActionFromProvider(splitElementId, action, actionArguments, requestId);
443 if (!ret) {
444 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAccessibilityAction failed.");
445 }
446
447 // 5. Return result
448 SetExecuteActionResult(callback, ret, requestId);
449 }
450
ExecuteActionFromProvider(int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId)451 bool JsThirdProviderInteractionOperation::ExecuteActionFromProvider(
452 int64_t elementId, const int32_t action,
453 const std::map<std::string, std::string>& actionArguments, const int32_t requestId)
454 {
455 auto provider = accessibilityProvider_.Upgrade();
456 CHECK_NULL_RETURN(provider, false);
457 int32_t code = provider->ExecuteAccessibilityAction(
458 elementId, action, requestId, actionArguments);
459 if (code != 0) {
460 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
461 "ExecuteActionFromProvider failed: %{public}d", code);
462 return false;
463 }
464
465 return true;
466 }
467
ExecuteActionForThird(int64_t elementId,const AccessibilityElementInfo & nodeInfo,const int32_t action)468 bool JsThirdProviderInteractionOperation::ExecuteActionForThird(
469 int64_t elementId, const AccessibilityElementInfo& nodeInfo, const int32_t action)
470 {
471 auto hostNode = GetHost();
472 CHECK_NULL_RETURN(hostNode, false);
473 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
474 CHECK_NULL_RETURN(jsAccessibilityManager, false);
475 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
476 CHECK_NULL_RETURN(context, false);
477 auto ngPipeline = AceType::DynamicCast<NG::PipelineContext>(context);
478 CHECK_NULL_RETURN(ngPipeline, false);
479 if (action == static_cast<int32_t>(
480 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS)) {
481 jsAccessibilityManager->ActThirdAccessibilityFocus(
482 elementId, nodeInfo, hostNode, ngPipeline, false);
483 } else if (action == static_cast<int32_t>(
484 Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
485 jsAccessibilityManager->ActThirdAccessibilityFocus(
486 elementId, nodeInfo, hostNode, ngPipeline, true);
487 }
488 return true;
489 }
490
SetExecuteActionResult(AccessibilityElementOperatorCallback & callback,const bool succeeded,const int32_t requestId)491 void JsThirdProviderInteractionOperation::SetExecuteActionResult(
492 AccessibilityElementOperatorCallback& callback,
493 const bool succeeded, const int32_t requestId)
494 {
495 callback.SetExecuteActionResult(succeeded, requestId);
496 }
497
ClearFocus()498 void JsThirdProviderInteractionOperation::ClearFocus()
499 {
500 // 1. Clear focus from provider
501 ClearFocusFromProvider();
502 // 2. Clear DrawBound
503 ClearDrawBound();
504 }
505
ClearFocusFromProvider()506 bool JsThirdProviderInteractionOperation::ClearFocusFromProvider()
507 {
508 auto provider = accessibilityProvider_.Upgrade();
509 CHECK_NULL_RETURN(provider, false);
510 int32_t code = provider->ClearFocusedAccessibilityNode();
511 if (code != 0) {
512 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
513 "ExecuteActionFromProvider failed: %{public}d", code);
514 return false;
515 }
516 return true;
517 }
518
ClearDrawBound()519 bool JsThirdProviderInteractionOperation::ClearDrawBound()
520 {
521 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
522 CHECK_NULL_RETURN(jsAccessibilityManager, false);
523 auto hostNode = GetHost();
524 CHECK_NULL_RETURN(hostNode, false);
525 return jsAccessibilityManager->ClearThirdAccessibilityFocus(hostNode);
526 }
527
OutsideTouch()528 void JsThirdProviderInteractionOperation::OutsideTouch()
529 {}
530
GetCursorPosition(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)531 void JsThirdProviderInteractionOperation::GetCursorPosition(
532 const int64_t elementId, const int32_t requestId,
533 Accessibility::AccessibilityElementOperatorCallback &callback)
534 {
535 auto provider = accessibilityProvider_.Upgrade();
536 CHECK_NULL_VOID(provider);
537 int32_t cursorPosition = -1;
538 int32_t code = provider->GetAccessibilityNodeCursorPosition(
539 elementId, requestId, cursorPosition);
540 if (code != 0) {
541 callback.SetCursorPositionResult(-1, requestId);
542 return;
543 }
544
545 callback.SetCursorPositionResult(cursorPosition, requestId);
546 }
547
GetNodeConfig(NodeConfig & config)548 void JsThirdProviderInteractionOperation::GetNodeConfig(NodeConfig& config)
549 {
550 auto host = host_.Upgrade();
551 CHECK_NULL_VOID(host);
552 auto jsAccessibilityManager = GetHandler().Upgrade();
553 CHECK_NULL_VOID(jsAccessibilityManager);
554 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
555 CHECK_NULL_VOID(context);
556 auto [displayOffset, err] = host->GetPaintRectGlobalOffsetWithTranslate();
557 config.offset = displayOffset;
558 config.pageId = host->GetPageId();
559 config.windowId = static_cast<int32_t>(context->GetRealHostWindowId());
560 config.belongTreeId = belongTreeId_;
561 config.parentWindowId = static_cast<int32_t>(context->GetRealHostWindowId());
562 config.bundleName = AceApplicationInfo::GetInstance().GetPackageName();
563 }
564
SetChildTreeIdAndWinId(const int64_t nodeId,const int32_t treeId,const int32_t childWindowId)565 void JsThirdProviderInteractionOperation::SetChildTreeIdAndWinId(
566 const int64_t nodeId, const int32_t treeId, const int32_t childWindowId)
567 {
568 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetChildTreeIdAndWinId, node: %{public}" PRId64 ","
569 "treeId: %{public}d, childWindowId: %{public}d", nodeId, treeId, childWindowId);
570 }
571
SetBelongTreeId(const int32_t treeId)572 void JsThirdProviderInteractionOperation::SetBelongTreeId(const int32_t treeId)
573 {
574 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetBelongTreeId treeId: %{public}d", treeId);
575 belongTreeId_ = treeId;
576 }
577
SendAccessibilityAsyncEventForThird(int64_t thirdElementId,Accessibility::EventType eventType)578 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEventForThird(
579 int64_t thirdElementId,
580 Accessibility::EventType eventType)
581 {
582 // 1. generate event
583 Accessibility::AccessibilityEventInfo event;
584 event.SetTimeStamp(GetMicroTickCount());
585 event.SetWindowChangeTypes(
586 Accessibility::WindowUpdateType::WINDOW_UPDATE_INVALID);
587 event.SetWindowContentChangeTypes(
588 Accessibility::WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_INVALID);
589 event.SetSource(thirdElementId);
590 event.SetEventType(eventType);
591 event.SetBundleName(AceApplicationInfo::GetInstance().GetPackageName());
592
593 // 2. get element from third
594 // cut tree id
595 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
596 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
597 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
598 thirdElementId, splitElementId, splitTreeId);
599 std::list<Accessibility::AccessibilityElementInfo> infos;
600 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
601 splitElementId, 0, 0, infos);
602 if ((!ret) || (infos.size() == 0)) {
603 return -1;
604 }
605
606 auto jsAccessibilityManager = GetHandler().Upgrade();
607 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
608 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
609 event.SetElementInfo(infos.front());
610
611 // 3. change event info by host info
612 GetAccessibilityEventInfoFromNativeEvent(event);
613
614 // 4. SendEvent
615 auto host = host_.Upgrade();
616 CHECK_NULL_RETURN(host, -1);
617 SendAccessibilitySyncEventToService(event, nullptr);
618 return 0;
619 }
620
HandleNativeFocusUpdate(int64_t elementId,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)621 bool JsThirdProviderInteractionOperation::HandleNativeFocusUpdate(
622 int64_t elementId,
623 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
624 {
625 ExecuteActionForThird(
626 elementId,
627 accessibilityEventInfo.GetElementInfo(),
628 static_cast<int32_t>(
629 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS));
630 return false;
631 }
632
HandleEventByFramework(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)633 bool JsThirdProviderInteractionOperation::HandleEventByFramework(
634 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
635 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
636 {
637 auto eventType = nativeAccessibilityEvent.GetEventType();
638 bool needSendEvent = true;
639 switch (eventType) {
640 case ArkUI_AccessibilityEventType::
641 ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
642 if (nativeAccessibilityEvent.GetElementInfo()) {
643 needSendEvent = HandleNativeFocusUpdate(
644 nativeAccessibilityEvent.GetElementInfo()->GetElementId(),
645 accessibilityEventInfo);
646 }
647 break;
648 default:
649 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "Invalid eventType");
650 }
651 return needSendEvent;
652 }
653
SendAccessibilityAsyncEvent(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,void (* callback)(int32_t errorCode))654 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEvent(
655 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
656 void (*callback)(int32_t errorCode))
657 {
658 // 1. Get OHOS::Accessibility::AccessibilityEventInfo
659 OHOS::Accessibility::AccessibilityEventInfo accessibilityEventInfo;
660 GetAccessibilityEventInfoFromNativeEvent(
661 nativeAccessibilityEvent, accessibilityEventInfo);
662
663 // 2. handleEvent by frame work
664 bool needSendEvent = HandleEventByFramework(
665 nativeAccessibilityEvent,
666 accessibilityEventInfo);
667 // 3. SendEvent
668 if (needSendEvent) {
669 auto jsAccessibilityManager = GetHandler().Upgrade();
670 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
671 auto host = host_.Upgrade();
672 CHECK_NULL_RETURN(host, -1);
673 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Inner SendAccessibilityAsyncEvent");
674 SendAccessibilitySyncEventToService(accessibilityEventInfo, callback);
675 }
676 callback(0);
677 return 0;
678 }
679
GetAccessibilityEventInfoFromNativeEvent(OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo)680 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
681 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
682 {
683 // 1. Fill node config
684 NodeConfig config;
685 GetNodeConfig(config);
686
687 // 1.1. Fill elementInfo config
688 auto elementInfo = accessibilityEventInfo.GetElementInfo();
689 FillNodeConfig(config, elementInfo);
690 int64_t elementId = elementInfo.GetAccessibilityId();
691 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(belongTreeId_, elementId);
692 elementInfo.SetAccessibilityId(elementId);
693
694 // 1.2. Fill eventInfo config
695 accessibilityEventInfo.SetPageId(config.pageId);
696 accessibilityEventInfo.SetWindowId(config.windowId);
697 accessibilityEventInfo.SetSource(elementId);
698 accessibilityEventInfo.SetComponentType(elementInfo.GetComponentType());
699 accessibilityEventInfo.AddContent(elementInfo.GetContent());
700 accessibilityEventInfo.SetElementInfo(elementInfo);
701 LogAccessibilityElementInfo("sendEvent", elementInfo);
702 LogAccessibilityEventInfo("sendEvent", accessibilityEventInfo);
703 }
704
GetAccessibilityEventInfoFromNativeEvent(const ArkUI_AccessibilityEventInfo & nativeEventInfo,OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo)705 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
706 const ArkUI_AccessibilityEventInfo& nativeEventInfo,
707 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
708 {
709 // 1. Transform native event to OHOS::Accessibility::AccessibilityEventInfo
710 TransformAccessbilityEventInfo(
711 nativeEventInfo, accessibilityEventInfo);
712
713 // 2. Transform Accessibility::AccessibilityEventInfo with host info
714 GetAccessibilityEventInfoFromNativeEvent(accessibilityEventInfo);
715 }
716
SendAccessibilitySyncEventToService(const OHOS::Accessibility::AccessibilityEventInfo & eventInfo,void (* callback)(int32_t errorCode))717 bool JsThirdProviderInteractionOperation::SendAccessibilitySyncEventToService(
718 const OHOS::Accessibility::AccessibilityEventInfo& eventInfo,
719 void (*callback)(int32_t errorCode))
720 {
721 auto jsAccessibilityManager = GetHandler().Upgrade();
722 CHECK_NULL_RETURN(jsAccessibilityManager, false);
723 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
724 CHECK_NULL_RETURN(context, false);
725 CHECK_NULL_RETURN(context->GetTaskExecutor(), false);
726 context->GetTaskExecutor()->PostTask(
727 [jsMgr = jsAccessibilityManager_, eventInfo, callback] () mutable {
728 auto jsAccessibilityManager = jsMgr.Upgrade();
729 if (jsAccessibilityManager == nullptr) {
730 return;
731 }
732
733 if (!jsAccessibilityManager->IsRegister()) {
734 return;
735 }
736
737 auto client = AccessibilitySystemAbilityClient::GetInstance();
738 CHECK_NULL_VOID(client);
739 bool isEnabled = false;
740 client->IsEnabled(isEnabled);
741 if (!isEnabled) {
742 return;
743 }
744
745 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY,
746 "send accessibility componentType:%{public}s event:%{public}d accessibilityId:%{public}" PRId64,
747 eventInfo.GetComponentType().c_str(), eventInfo.GetEventType(), eventInfo.GetAccessibilityId());
748 client->SendEvent(eventInfo);
749 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
750 return true;
751 }
752 } // namespace OHOS::Ace::Framework
753