/* * Copyright (c) 2022-2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef FOUNDATION_ACE_FRAMEWORKS_BRIDGE_COMMON_ACCESSIBILITY_JS_ACCESSIBILITY_MANAGER_H #define FOUNDATION_ACE_FRAMEWORKS_BRIDGE_COMMON_ACCESSIBILITY_JS_ACCESSIBILITY_MANAGER_H #include #include #include #include "accessibility_config.h" #include "accessibility_element_operator.h" #include "accessibility_event_info.h" #include "accessibility_state_event.h" #include "core/accessibility/accessibility_manager.h" #include "core/accessibility/accessibility_utils.h" #include "frameworks/bridge/common/accessibility/accessibility_node_manager.h" #include "js_third_accessibility_hover_ng.h" namespace OHOS::NWeb { class NWebAccessibilityNodeInfo; } // namespace OHOS::NWeb::NWebAccessibilityNodeInfo namespace OHOS::Ace::Framework { struct SearchParameter { int64_t nodeId; std::string text; int32_t mode; int64_t uiExtensionOffset; }; struct CommonProperty { int32_t windowId = 0; int32_t windowLeft = 0; int32_t windowTop = 0; int32_t pageId = 0; std::string pagePath; }; struct ActionTable { AceAction aceAction; ActionType action; }; struct ActionStrTable { ActionType action; std::string actionStr; }; struct FillEventInfoParam { int64_t elementId = 0; int64_t stackNodeId = 0; uint32_t windowId = 0; }; struct AccessibilityActionParam { RefPtr accessibilityProperty; std::string setTextArgument = ""; int32_t setSelectionStart = -1; int32_t setSelectionEnd = -1; bool setSelectionDir = false; int32_t setCursorIndex = -1; TextMoveUnit moveUnit = TextMoveUnit::STEP_CHARACTER; AccessibilityScrollType scrollType = AccessibilityScrollType::SCROLL_DEFAULT; int32_t spanId = -1; }; struct ActionParam { Accessibility::ActionType action; std::map actionArguments; }; class JsAccessibilityManager : public AccessibilityNodeManager, public AccessibilityHoverManagerForThirdNG { DECLARE_ACE_TYPE(JsAccessibilityManager, AccessibilityNodeManager); public: JsAccessibilityManager() = default; ~JsAccessibilityManager() override; // JsAccessibilityManager overrides functions. void InitializeCallback() override; void SendAccessibilityAsyncEvent(const AccessibilityEvent& accessibilityEvent) override; void UpdateVirtualNodeFocus() override; void SetCardViewParams(const std::string& key, bool focus) override; void HandleComponentPostBinding() override; void RegisterSubWindowInteractionOperation(int windowId) override; void SetPipelineContext(const RefPtr& context) override; void UpdateElementInfosTreeId(std::list& infos); void UpdateElementInfoTreeId(Accessibility::AccessibilityElementInfo& info); void UpdateViewScale(); float GetScaleX() const { return scaleX_; } float GetScaleY() const { return scaleY_; } uint32_t GetWindowId() const { return windowId_; } void SetWindowId(uint32_t windowId) { windowId_ = windowId; } void SaveLast(const int64_t elementId, const RefPtr& node) { lastElementId_ = elementId; lastFrameNode_ = node; } void SaveCurrentFocusNodeSize(const RefPtr& currentFocusNode) { if (currentFocusNode->IsAccessibilityVirtualNode()) { auto oldGeometryNode = currentFocusNode->GetGeometryNode(); CHECK_NULL_VOID(oldGeometryNode); oldGeometrySize_ = oldGeometryNode->GetFrameSize(); } } bool SubscribeToastObserver(); bool UnsubscribeToastObserver(); bool SubscribeStateObserver(int eventType); bool UnsubscribeStateObserver(int eventType); int RegisterInteractionOperation(int windowId); void DeregisterInteractionOperation(); bool SendAccessibilitySyncEvent( const AccessibilityEvent& accessibilityEvent, Accessibility::AccessibilityEventInfo eventInfo); bool TransferAccessibilityAsyncEvent( const Accessibility::AccessibilityEventInfo& eventInfo, int64_t uiExtensionOffset) override; void SendExtensionAccessibilityEvent( const Accessibility::AccessibilityEventInfo& eventInfo, int64_t uiExtensionOffset) override; void SearchElementInfoByAccessibilityId(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode, const int32_t windowId); void SearchElementInfosByText(const int64_t elementId, const std::string& text, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId); void FindFocusedElementInfo(const int64_t elementId, const int32_t focusType, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId); void FocusMoveSearch(const int64_t elementId, const int32_t direction, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId); void ExecuteAction(const int64_t accessibilityId, const ActionParam& param, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId); bool ClearCurrentFocus(); void UpdateNodeChildIds(const RefPtr& node); void SendActionEvent(const Accessibility::ActionType& action, int64_t nodeId); void SearchElementInfoByAccessibilityIdNG(int64_t elementId, int32_t mode, std::list& infos, const RefPtr& context, const int64_t uiExtensionOffset = 0) override; void SearchElementInfosByTextNG(int64_t elementId, const std::string& text, std::list& infos, const RefPtr& context, const int64_t uiExtensionOffset = 0) override; void FindFocusedElementInfoNG(int64_t elementId, int32_t focusType, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const int64_t uiExtensionOffset = 0) override; void FocusMoveSearchNG(int64_t elementId, int32_t direction, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const int64_t uiExtensionOffset = 0) override; bool ExecuteExtensionActionNG(int64_t elementId, const std::map& actionArguments, int32_t action, const RefPtr& context, int64_t uiExtensionOffset) override; #ifdef WEB_SUPPORTED void SendWebAccessibilityAsyncEvent(const AccessibilityEvent& accessibilityEvent, const RefPtr& webPattern) override; void SearchWebElementInfoByAccessibilityId(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode, const int32_t windowId, const RefPtr& webPattern); void SearchWebElementInfoByAccessibilityIdNG(int64_t elementId, int32_t mode, std::list& infos, const RefPtr& context, const RefPtr& webPattern); void FindWebFocusedElementInfo(const int64_t elementId, const int32_t focusType, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId, const RefPtr& webPattern); void FindWebFocusedElementInfoNG(int64_t elementId, int32_t focusType, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const RefPtr& webPattern); void WebFocusMoveSearch(const int64_t elementId, const int32_t direction, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId, const RefPtr& webPattern); void WebFocusMoveSearchNG(int64_t elementId, int32_t direction, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const RefPtr& webPattern); void ExecuteWebAction(const int64_t elementId, const ActionParam& param, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t windowId, const RefPtr& webPattern); bool ExecuteWebActionNG(int64_t elementId, Accessibility::ActionType action, const std::map& actionArguments, const RefPtr& webPattern); bool DeregisterWebInteractionOperationAsChildTree(int32_t treeID) override; bool RegisterWebInteractionOperationAsChildTree(int64_t accessibilityId, const WeakPtr& webPattern) override; void GetWebCursorPosition(const int64_t elementId, const int32_t requestId, AccessibilityElementOperatorCallback& callback, const RefPtr& webPattern); #endif //WEB_SUPPORTED void GetResultOfFocusMoveSearchNG( int64_t elementId, int32_t direction, Accessibility::AccessibilityElementInfo& info); std::string GetPagePath(); void RegisterAccessibilityChildTreeCallback( int64_t elementId, const std::shared_ptr &callback) override; void DeregisterAccessibilityChildTreeCallback(int64_t elementId) override; void RegisterAccessibilitySAObserverCallback( int64_t elementId, const std::shared_ptr &callback) override; void DeregisterAccessibilitySAObserverCallback(int64_t elementId) override; void RegisterInteractionOperationAsChildTree(uint32_t parentWindowId, int32_t parentTreeId, int64_t parentElementId) override; void SetAccessibilityGetParentRectHandler(std::function &&callback) override; void DeregisterInteractionOperationAsChildTree() override; void SendEventToAccessibilityWithNode(const AccessibilityEvent& accessibilityEvent, const RefPtr& node, const RefPtr& context) override; bool RegisterInteractionOperationAsChildTree(const Registration& registration) override; bool DeregisterInteractionOperationAsChildTree(uint32_t windowId, int32_t treeId) override; void TransferThirdProviderHoverEvent( const WeakPtr& hostNode, const NG::PointF& point, SourceType source, NG::AccessibilityHoverEventType eventType, TimeStamp time) override; void DumpAccessibilityPropertyNG(const AccessibilityElementInfo& nodeInfo); void DumpCommonPropertyNG(const AccessibilityElementInfo& nodeInfo, int32_t treeId); bool OnDumpChildInfoForThird( int64_t hostElementId, const std::vector& params, std::vector& info) override; void FireAccessibilityEventCallback(uint32_t eventId, int64_t parameter) override; protected: void OnDumpInfoNG(const std::vector& params, uint32_t windowId) override; void DumpHandleEvent(const std::vector& params) override; void DumpProperty(const std::vector& params) override; void DumpTree(int32_t depth, int64_t nodeID, bool isDumpSimplify = false) override; private: static constexpr int32_t INVALID_PARENT_ID = -2100000; class JsInteractionOperation : public Accessibility::AccessibilityElementOperator { public: explicit JsInteractionOperation(int32_t windowId) : windowId_(windowId) {} virtual ~JsInteractionOperation() = default; // Accessibility override. void SearchElementInfoByAccessibilityId(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode) override; void SearchElementInfosByText(const int64_t elementId, const std::string& text, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void FindFocusedElementInfo(const int64_t elementId, const int32_t focusType, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void FocusMoveSearch(const int64_t elementId, const int32_t direction, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void ExecuteAction(const int64_t elementId, const int32_t action, const std::map& actionArguments, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void ClearFocus() override; void OutsideTouch() override; void GetCursorPosition(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback &callback) override; void SetChildTreeIdAndWinId(const int64_t nodeId, const int32_t treeId, const int32_t childWindowId) override; void SetBelongTreeId(const int32_t treeId) override; void SetHandler(const WeakPtr& js) { js_ = js; } const WeakPtr& GetHandler() const { return js_; } private: WeakPtr js_; uint32_t windowId_ = 0; }; #ifdef WEB_SUPPORTED class WebInteractionOperation : public Accessibility::AccessibilityElementOperator { public: explicit WebInteractionOperation(int32_t windowId) : windowId_(windowId) {} virtual ~WebInteractionOperation() = default; // Accessibility override. void SearchElementInfoByAccessibilityId(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode) override; void SearchElementInfosByText(const int64_t elementId, const std::string& text, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void FindFocusedElementInfo(const int64_t elementId, const int32_t focusType, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void FocusMoveSearch(const int64_t elementId, const int32_t direction, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void ExecuteAction(const int64_t elementId, const int32_t action, const std::map& actionArguments, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback& callback) override; void ClearFocus() override; void OutsideTouch() override; void GetCursorPosition(const int64_t elementId, const int32_t requestId, Accessibility::AccessibilityElementOperatorCallback &callback) override; void SetChildTreeIdAndWinId(const int64_t nodeId, const int32_t treeId, const int32_t childWindowId) override; void SetBelongTreeId(const int32_t treeId) override; void SetHandler(const WeakPtr& js) { js_ = js; } const WeakPtr& GetHandler() const { return js_; } void SetWebPattern(const WeakPtr& webPattern) { webPattern_ = webPattern; } const WeakPtr& GetWebPattern() const { return webPattern_; } private: WeakPtr js_; uint32_t windowId_ = 0; WeakPtr webPattern_; }; #endif // WEB_SUPPORTED class ToastAccessibilityConfigObserver : public AccessibilityConfig::AccessibilityConfigObserver { public: ToastAccessibilityConfigObserver() = default; void OnConfigChanged( const AccessibilityConfig::CONFIG_ID id, const AccessibilityConfig::ConfigValue& value) override; }; class JsAccessibilityStateObserver : public Accessibility::AccessibilityStateObserver { public: void OnStateChanged(const bool state) override; void SetAccessibilityManager(const WeakPtr& accessibilityManager) { accessibilityManager_ = accessibilityManager; } void SetPipeline(const WeakPtr& pipeline) { pipeline_ = pipeline; } private: // Do not upgrade accessibilityManager_ on async thread, destructor might cause freeze WeakPtr accessibilityManager_; WeakPtr pipeline_; }; bool AccessibilityActionEvent(const Accessibility::ActionType& action, const std::map& actionArguments, const RefPtr& node, const RefPtr& context); bool RequestAccessibilityFocus(const RefPtr& node); bool ClearAccessibilityFocus(const RefPtr& node); void AddFocusableNode(std::list>& nodeList, const RefPtr& node); bool CanAccessibilityFocused(const RefPtr& node); RefPtr FindNodeInRelativeDirection( const std::list>& nodeList, RefPtr& node, const int direction); RefPtr FindNodeInAbsoluteDirection( const std::list>& nodeList, RefPtr& node, const int direction); RefPtr GetNextFocusableNode( const std::list>& nodeList, RefPtr& node); RefPtr GetPreviousFocusableNode( const std::list>& nodeList, RefPtr& node); bool ExecuteActionNG(int64_t elementId, const std::map& actionArguments, Accessibility::ActionType action, const RefPtr& context, int64_t uiExtensionOffset); bool ConvertActionTypeToBoolen(Accessibility::ActionType action, RefPtr& frameNode, int64_t elementId, RefPtr& context); void SetSearchElementInfoByAccessibilityIdResult(Accessibility::AccessibilityElementOperatorCallback& callback, std::list&& infos, const int32_t requestId); void SetSearchElementInfoByTextResult(Accessibility::AccessibilityElementOperatorCallback& callback, std::list&& infos, const int32_t requestId); void SetFindFocusedElementInfoResult(Accessibility::AccessibilityElementOperatorCallback& callback, Accessibility::AccessibilityElementInfo& info, const int32_t requestId); void SetFocusMoveSearchResult(Accessibility::AccessibilityElementOperatorCallback& callback, Accessibility::AccessibilityElementInfo& info, const int32_t requestId); void SetExecuteActionResult( Accessibility::AccessibilityElementOperatorCallback& callback, const bool succeeded, const int32_t requestId); void SearchExtensionElementInfoByAccessibilityIdNG(const SearchParameter& searchParam, const RefPtr& node, std::list& infos, const RefPtr& context, const RefPtr& ngPipeline); void SearchElementInfosByTextNG(const SearchParameter& searchParam, const RefPtr& node, std::list& infos, const RefPtr& context, const RefPtr& ngPipeline); std::list SearchElementInfosByTextNG( int64_t elementId, const std::string& text, const RefPtr& node, int64_t offset); void FindFocusedExtensionElementInfoNG(const SearchParameter& searchParam, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const RefPtr& root); void FocusExtensionElementMoveSearchNG(const SearchParameter& searchParam, Accessibility::AccessibilityElementInfo& info, const RefPtr& context, const RefPtr& root, RefPtr& outputExtensionNode); void TransferExecuteAction(int64_t elementId, RefPtr& node, const std::map& actionArguments, Accessibility::ActionType& action, int64_t uiExtensionOffset); RefPtr FindNodeFromRootByExtensionId(const RefPtr& root, const int64_t uiExtensionId); bool RegisterThirdProviderInteractionOperationAsChildTree(const Registration& registration); void DumpProperty(const RefPtr& node); void DumpPropertyNG(int64_t nodeID); void DumpHoverTestNG(uint32_t windowId, int64_t nodeID, int32_t x, int32_t y, bool verbose); RefPtr FindPipelineByElementId(const int64_t elementId, RefPtr& node); RefPtr FindNodeFromPipeline(const WeakPtr& context, const int64_t elementId); RefPtr GetPipelineByWindowId(const int32_t windowId); void ProcessParameters(Accessibility::ActionType op, const std::vector& params, std::map& paramsMap); RefPtr GetPipelineByWindowId(uint32_t windowId); void DumpTreeNG(bool useWindowId, uint32_t windowId, int64_t rootId, bool isDumpSimplify = false); void DumpTreeNG(const RefPtr& parent, int32_t depth, int64_t nodeID, const CommonProperty& commonProperty, bool isDumpSimplify = false); void DumpTreeNodeSafeAreaInfoNg(const RefPtr& node); void DumpPadding(const std::unique_ptr& padding, std::string label); void DumpBorder(const std::unique_ptr& border, std::string label); void DumpTreeNodeCommonInfoNg(const RefPtr& node, const CommonProperty& commonProperty); void DumpTreeNodeSimplifyInfoNG( const RefPtr& node, int32_t depth, const CommonProperty& commonProperty, int32_t childSize); void DumpTreeAccessibilityNodeNG(const RefPtr& uiNodeParent, int32_t depth, int64_t nodeID, const CommonProperty& commonProperty); bool CheckDumpInfoParams(const std::vector ¶ms); void GenerateCommonProperty(const RefPtr& context, CommonProperty& output, const RefPtr& mainContext); void FindText(const RefPtr& node, std::list& infos, const RefPtr& context, const CommonProperty& commonProperty, const SearchParameter& searchParam); void FindTextByTextHint(const RefPtr& node, std::list& infos, const RefPtr& context, const CommonProperty& commonProperty, const SearchParameter& searchParam); void UpdateAccessibilityElementInfo( const RefPtr& node, Accessibility::AccessibilityElementInfo& nodeInfo); void UpdateAccessibilityVisible( const RefPtr& node, Accessibility::AccessibilityElementInfo& nodeInfo); void UpdateVirtualNodeInfo(std::list& infos, Accessibility::AccessibilityElementInfo& nodeInfo, const RefPtr& uiVirtualNode, const CommonProperty& commonProperty, const RefPtr& ngPipeline); void UpdateVirtualNodeChildAccessibilityElementInfo( const RefPtr& node, const CommonProperty& commonProperty, Accessibility::AccessibilityElementInfo& nodeParentInfo, Accessibility::AccessibilityElementInfo& nodeInfo, const RefPtr& ngPipeline); void UpdateVirtualNodeAccessibilityElementInfo( const RefPtr& parent, const RefPtr& node, const CommonProperty& commonProperty, Accessibility::AccessibilityElementInfo& nodeInfo, const RefPtr& ngPipeline); void UpdateAccessibilityElementInfo( const RefPtr& node, const CommonProperty& commonProperty, Accessibility::AccessibilityElementInfo& nodeInfo, const RefPtr& ngPipeline); void UpdateCacheInfoNG(std::list& infos, const RefPtr& node, const CommonProperty& commonProperty, const RefPtr& ngPipeline, const SearchParameter& searchParam); #ifdef WEB_SUPPORTED void UpdateWebAccessibilityElementInfo(const std::shared_ptr& node, Accessibility::AccessibilityElementInfo& nodeInfo, int32_t treeId); void UpdateWebAccessibilityElementInfo(const std::shared_ptr& node, const CommonProperty& commonProperty, Accessibility::AccessibilityElementInfo& nodeInfo, const RefPtr& webPattern); void UpdateWebCacheInfo(std::list& infos, int64_t nodeId, const CommonProperty& commonProperty, const RefPtr& ngPipeline, const SearchParameter& searchParam, const RefPtr& webPattern); #endif //WEB_SUPPORTED void NotifyChildTreeOnRegister(int32_t treeId); void NotifyChildTreeOnDeregister(); void SendUecOnTreeEvent(int64_t splitElementId); void NotifySetChildTreeIdAndWinId(int64_t elementId, const int32_t treeId, const int32_t childWindowId); bool CheckIsChildElement( int64_t &elementId, const std::vector& params, std::vector& info); bool NeedRegisterChildTree(uint32_t parentWindowId, int32_t parentTreeId, int64_t parentElementId); void FillEventInfoWithNode( const RefPtr& node, Accessibility::AccessibilityEventInfo& eventInfo, const RefPtr& context, int64_t elementId); void NotifyAccessibilitySAStateChange(bool state); void SendEventToAccessibilityWithNodeInner(const AccessibilityEvent& accessibilityEvent, const RefPtr& node, const RefPtr& context); void SendAccessibilityAsyncEventInner(const AccessibilityEvent& accessibilityEvent); int64_t GetDelayTimeBeforeSendEvent(const AccessibilityEvent& accessibilityEvent, const RefPtr& node); std::string callbackKey_; uint32_t windowId_ = 0; std::shared_ptr stateObserver_ = nullptr; std::shared_ptr toastObserver_ = nullptr; float scaleX_ = 1.0f; float scaleY_ = 1.0f; int64_t currentFocusNodeId_ = -1; int64_t lastElementId_ = -1; WeakPtr lastFrameNode_; NG::SizeF oldGeometrySize_; mutable std::mutex childTreeCallbackMapMutex_; std::unordered_map> childTreeCallbackMap_; mutable std::mutex componentSACallbackMutex_; std::unordered_map> componentSACallbackMap_; int64_t parentElementId_ = INVALID_PARENT_ID; uint32_t parentWindowId_ = 0; int32_t parentTreeId_ = 0; uint32_t parentWebWindowId_ = 0; std::function getParentRectHandler_; }; } // namespace OHOS::Ace::Framework #endif // FOUNDATION_ACE_FRAMEWORKS_BRIDGE_COMMON_ACCESSIBILITY_JS_ACCESSIBILITY_MANAGER_H