1 /* 2 * Copyright (C) 2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef ACCESSIBILITY_TOUCH_GUIDER_H 17 #define ACCESSIBILITY_TOUCH_GUIDER_H 18 19 #include <string> 20 #include "accessibility_element_info.h" 21 #include "accessibility_element_operator_callback_stub.h" 22 #include "accessibility_event_transmission.h" 23 #include "accessibility_gesture_recognizer.h" 24 #include "accessibility_multifinger_multitap.h" 25 #include "accessible_ability_manager_service.h" 26 27 namespace OHOS { 28 namespace Accessibility { 29 class TouchGuider; 30 31 const int64_t EXIT_GESTURE_REC_TIMEOUT = 400; // millisecond 32 const double MAX_DRAG_GESTURE_COSINE = 0.525321989; 33 const int32_t MINI_POINTER_DISTANCE_DIP = 200; 34 const int32_t INDEX_0 = 0; 35 const int32_t INDEX_1 = 1; 36 const int32_t INIT_POINT_ID = -1; 37 const float INIT_MMIPOINT = 0.0f; 38 const int32_t INIT_POINT_DISPLAY = 0; 39 #define DIVIDE_2(num) ((num) / 2) 40 #define EPSINON 0.01 41 42 /** 43 * @brief touch Guider state define 44 */ 45 enum class TouchGuideState : int32_t { 46 TOUCH_GUIDING, 47 DRAGGING, 48 TRANSMITTING, 49 GESTURE_RECOGNIZING, 50 PASSING_THROUGH 51 }; 52 53 /** 54 * @brief Click location define 55 */ 56 enum ClickLocation : int32_t { 57 CLICK_NONE, 58 CLICK_ACCESSIBILITY_FOCUS, 59 CLICK_LAST_TOUCH_GUIDE 60 }; 61 62 /** 63 * @brief struct to record injected pointers. 64 */ 65 struct InjectedEventRecorder { 66 std::set<int32_t> downPointers {}; 67 int32_t downPointerNum; 68 int64_t lastDownTime; 69 std::shared_ptr<MMI::PointerEvent> lastHoverEvent; 70 }; 71 72 /** 73 * @brief struct to record received pointers. 74 */ 75 struct ReceivedEventRecorder { 76 std::map<int32_t, int32_t> pointerDownX; 77 std::map<int32_t, int32_t> pointerDownY; 78 std::map<int32_t, int64_t> pointerActionTime; 79 std::shared_ptr<MMI::PointerEvent> lastEvent; 80 }; 81 82 enum ChangeAction : int32_t { 83 NO_CHANGE, 84 HOVER_MOVE, 85 POINTER_DOWN, 86 POINTER_UP, 87 POINTER_MOVE, 88 HOVER_ENTER, 89 HOVER_EXIT, 90 }; 91 92 class TGEventHandler : public AppExecFwk::EventHandler { 93 public: 94 TGEventHandler(const std::shared_ptr<AppExecFwk::EventRunner> &runner, 95 TouchGuider &tgServer); 96 virtual ~TGEventHandler() = default; 97 /** 98 * @brief Process the event of install system bundles. 99 * @param event Indicates the event to be processed. 100 */ 101 virtual void ProcessEvent(const AppExecFwk::InnerEvent::Pointer &event) override; 102 103 private: 104 /** 105 * @brief Send HoverEnter and HoverMove to Multimodal. 106 */ 107 void HoverEnterAndMoveRunner(); 108 109 /** 110 * @brief Send HoverExit to Multimodal. 111 */ 112 void HoverExitRunner(); 113 TouchGuider &tgServer_; 114 }; 115 116 class TouchGuider : public EventTransmission { 117 public: 118 static constexpr uint32_t EXIT_GESTURE_REC_MSG = 0; 119 static constexpr uint32_t SEND_HOVER_ENTER_MOVE_MSG = 1; 120 static constexpr uint32_t SEND_HOVER_EXIT_MSG = 2; 121 static constexpr uint32_t SEND_TOUCH_INTERACTION_END_MSG = 3; 122 static constexpr uint32_t SEND_TOUCH_GUIDE_END_MSG = 4; 123 124 /** 125 * @brief A constructor used to create a touchGuide instance. 126 */ 127 TouchGuider(); 128 129 /** 130 * @brief A destructor used to delete the touchGuide instance. 131 */ ~TouchGuider()132 ~TouchGuider() {} 133 134 /** 135 * @brief TouchGuide start up. 136 */ 137 void StartUp(); 138 139 /** 140 * @brief Handle pointer events from previous event stream node. 141 * 142 * @param event the pointer event to be handled. 143 * @return true: the event has been processed and does not need to be passed to the next node; 144 * false: the event is not processed. 145 */ 146 bool OnPointerEvent(MMI::PointerEvent &event) override; 147 148 /** 149 * @brief Handle pointer events from previous event stream node. 150 * 151 * @param event the pointer event to be handled. 152 */ 153 void HandlePointerEvent(MMI::PointerEvent &event); 154 155 /** 156 * @brief Destroy event state. 157 */ 158 void DestroyEvents() override; 159 160 /** 161 * @brief Send pointer down event to multimodal input. 162 * @param event event the touch event from Multimodal, set the down point to the event and send. 163 * @param action point action send to multimode. 164 */ 165 void SendPointerDownEventToMultimodal(MMI::PointerEvent event, int32_t action); 166 167 /** 168 * @brief Send event to multimodal input. 169 * @param event the event prepared to send to Multimodal 170 * @param action the action of the event 171 */ 172 void SendEventToMultimodal(MMI::PointerEvent &event, int32_t action); 173 174 /** 175 * @brief Send accessibility event to specific AccessibleAbility. 176 * @param eventType the type of the event 177 */ 178 void SendAccessibilityEventToAA(EventType eventType); 179 180 /** 181 * @brief Send gesture event to specific AccessibleAbility. 182 * @param gestureId the gesture id of the event 183 */ 184 void SendGestureEventToAA(GestureType gestureId); 185 186 /** 187 * @brief Get hover enter and move event. 188 * @return Returns pointerEvents_ list. 189 */ 190 std::list<MMI::PointerEvent> getHoverEnterAndMoveEvent(); 191 192 /** 193 * @brief Clear hover enter and move event. 194 */ 195 void ClearHoverEnterAndMoveEvent(); 196 197 /** 198 * @brief Get last received event. 199 * @return Returns last event ptr. 200 */ 201 std::shared_ptr<MMI::PointerEvent> getLastReceivedEvent(); 202 203 /* For TouchGuide */ OnTouchInteractionStart()204 inline void OnTouchInteractionStart() 205 { 206 isTouchStart_ = true; 207 } 208 OnTouchInteractionEnd()209 inline void OnTouchInteractionEnd() 210 { 211 isTouchStart_ = false; 212 } 213 214 /** 215 * @brief whether touch guide end. 216 * @return true if touch guide end, else false. 217 */ IsTouchInteractionEnd()218 inline bool IsTouchInteractionEnd() 219 { 220 return isTouchStart_ == false; 221 } 222 223 /** 224 * @brief Perform action on Accessibility Focus. 225 * @param action the action of Accessibility node. 226 * @return Returns true if the action perform successfully; returns false code otherwise. 227 */ 228 bool ExecuteActionOnAccessibilityFocused(const ActionType &action); 229 230 static int64_t lastDoubleTapTime; 231 private: 232 class TouchGuideListener : public AccessibilityGestureRecognizeListener { 233 public: 234 /** 235 * @brief A constructor used to create a TouchGuideListener instance. 236 */ TouchGuideListener(TouchGuider & server)237 explicit TouchGuideListener(TouchGuider &server) : server_(server) {}; 238 239 /** 240 * @brief Prepare to send the event corresponding to the single tap to the Multimodal. 241 * @param event the touch event from Multimodal 242 */ 243 bool OnDoubleTap(MMI::PointerEvent &event) override; 244 245 /** 246 * @brief Send GESTURE_BEGIN to AccessibleAbility. 247 */ 248 bool OnStarted() override; 249 250 /** 251 * @brief Send GESTURE_BEGIN to AccessibleAbility when multi finger gesture start. 252 * @param isTwoFingerGesture whether the gesture is triggered by two finger. 253 */ 254 void MultiFingerGestureOnStarted(bool isTwoFingerGesture) override; 255 256 /** 257 * @brief Send GESTURE_END and TOUCH_END to AccessibleAbility. 258 * @param gestureId the id of gesture 259 */ 260 bool OnCompleted(GestureType gestureId) override; 261 262 /** 263 * @brief Send GESTURE_END and TOUCH_END to AccessibleAbility when multi finger gesture complete. 264 * @param gestureId the id of gesture. 265 */ 266 void MultiFingerGestureOnCompleted(GestureType gestureId) override; 267 268 /** 269 * @brief The gesture has been cancelled. 270 * @param event the touch event from Multimodal 271 */ 272 bool OnCancelled(MMI::PointerEvent &event) override; 273 274 /** 275 * @brief The gesture has been cancelled. 276 * @param isNoDelayFlag whether the gesture recognize process is immediately canceled. 277 */ 278 void MultiFingerGestureOnCancelled(const bool isNoDelayFlag) override; 279 private: 280 TouchGuider &server_; 281 }; 282 283 class ElementOperatorCallbackImpl : public AccessibilityElementOperatorCallbackStub { 284 public: 285 ElementOperatorCallbackImpl() = default; 286 ~ElementOperatorCallbackImpl() = default; 287 288 virtual void SetSearchElementInfoByAccessibilityIdResult(const std::vector<AccessibilityElementInfo> &infos, 289 const int32_t requestId) override; 290 virtual void SetSearchElementInfoByTextResult(const std::vector<AccessibilityElementInfo> &infos, 291 const int32_t requestId) override; 292 virtual void SetFindFocusedElementInfoResult(const AccessibilityElementInfo &info, 293 const int32_t requestId) override; 294 virtual void SetFocusMoveSearchResult(const AccessibilityElementInfo &info, const int32_t requestId) override; 295 virtual void SetExecuteActionResult(const bool succeeded, const int32_t requestId) override; 296 297 private: 298 ffrt::promise<void> promise_; 299 bool executeActionResult_ = false; 300 AccessibilityElementInfo accessibilityInfoResult_ = {}; 301 std::vector<AccessibilityElementInfo> elementInfosResult_; 302 303 friend class TouchGuider; 304 }; 305 306 /** 307 * @brief Determine whether to clear the touchguide. 308 */ 309 void Clear(); 310 311 /** 312 * @brief clear the touchguide. 313 * @param event the last event from Multimodal 314 */ 315 void Clear(MMI::PointerEvent &event); 316 317 /** 318 * @brief Handle touch events on touchExploring state. 319 * @param event the touch event from Multimodal 320 */ 321 void HandleTouchGuidingState(MMI::PointerEvent &event); 322 323 /** 324 * @brief Handle touch events on dragging state. 325 * @param event the touch event from Multimodal 326 */ 327 void HandleDraggingState(MMI::PointerEvent &event); 328 329 /** 330 * @brief Handle touch events on transmitting state. 331 * @param event the touch event from Multimodal 332 */ 333 void HandleTransmitingState(MMI::PointerEvent &event); 334 335 /** 336 * @brief Handle touch events on passing through state. 337 * @param event the touch event from Multimodal 338 */ 339 void HandlePassingThroughState(MMI::PointerEvent &event); 340 341 /** 342 * @brief Determine whether it is a drag gesture. 343 * @param event the touch event from Multimodal 344 * @return whether the dragGesture is accepted. 345 */ 346 bool IsDragGestureAccept(MMI::PointerEvent &event); 347 348 /** 349 * @brief Get the offset of current points and touch down points. 350 * @param event the current touch event from Multimodal. 351 * @param firstPointOffset the first finger offset result, xAxis offset and yAxis offset. 352 * @param firstPointOffset the second finger offset result, xAxis offset and yAxis offset. 353 */ 354 void GetPointOffset(MMI::PointerEvent &event, std::vector<float> &firstPointOffset, 355 std::vector<float> &secondPointOffset) const; 356 357 /** 358 * @brief Determine whether it is a move gesture. 359 * @param event the touch event from Multimodal. 360 * @return whether this is a scolling. 361 */ 362 bool IsRealMoveState(MMI::PointerEvent &event) const; 363 364 /** 365 * @brief Get Angle Cos value. 366 * @param offsetX the X value 367 * @param offsetY the Y value 368 * @param isGetX whether is the Angle corresponding to the X axis 369 * @return Angle Cos value. 370 */ 371 float GetAngleCos(float offsetX, float offsetY, bool isGetX); 372 373 /** 374 * @brief Get the info of injected event. 375 * @param event the event prepared to send to Multimodal 376 */ 377 void RecordInjectedEvent(MMI::PointerEvent &event); 378 379 /** 380 * @brief Get the info of Received event. 381 * @param event event the touch event from Multimodal 382 */ 383 void RecordReceivedEvent(MMI::PointerEvent &event); 384 385 /** 386 * @brief Send touch event to specific AccessibleAbility. 387 * @param event the touch event from Multimodal 388 */ 389 void SendTouchEventToAA(MMI::PointerEvent &event); 390 391 /** 392 * @brief Clear received recorder info. 393 */ 394 void ClearReceivedEventRecorder(); 395 396 /** 397 * @brief Clear Injected recorder info. 398 */ 399 void ClearInjectedEventRecorder(); 400 401 /** 402 * @brief Send exit event to multimodal. 403 */ 404 void SendExitEvents(); 405 406 /** 407 * @brief Send all down events to multimodal. 408 * @param event the event prepared to send to Multimodal 409 */ 410 void SendAllDownEvents(MMI::PointerEvent &event); 411 412 /** 413 * @brief Send all up events to multimodal. 414 * @param event the event prepared to send to Multimodal 415 */ 416 void SendAllUpEvents(MMI::PointerEvent &event); 417 418 /** 419 * @brief Send all up events to multimodal. 420 * @param event the event prepared to send to Multimodal 421 */ 422 void SendUpForAllInjectedEvent(MMI::PointerEvent &event); 423 424 /** 425 * @brief Send exit message. 426 */ 427 void PostGestureRecognizeExit(); 428 429 /** 430 * @brief Send enter and move message. 431 * @param event event the touch event from Multimodal 432 */ 433 void PostHoverEnterAndMove(MMI::PointerEvent &event); 434 435 /** 436 * @brief Send exit message. 437 */ 438 void PostHoverExit(); 439 440 /** 441 * @brief Send accessibility event message. 442 * @param innerEventID the id of inner event 443 */ 444 void PostAccessibilityEvent(uint32_t innerEventID); 445 446 /** 447 * @brief Cancel message. 448 * @param innerEventID the id of inner event 449 */ 450 void CancelPostEvent(uint32_t innerEventID); 451 452 /** 453 * @brief Cancel message if it has been sent. 454 * @param innerEventID the id of inner event 455 */ 456 void CancelPostEventIfNeed(uint32_t innerEventID); 457 458 /** 459 * @brief Check whether it has been sending. 460 * @param innerEventID the id of inner event 461 */ 462 bool HasEventPending(uint32_t innerEventID); 463 464 /** 465 * @brief Force send and remove event. 466 * @param innerEventID the id of inner event 467 * @param event event the touch event from Multimodal 468 */ 469 void ForceSendAndRemoveEvent(uint32_t innerEventID, MMI::PointerEvent &event); 470 471 /** 472 * @brief Handle down events on touchExploring state. 473 * @param event event the touch event from Multimodal 474 */ 475 void HandleTouchGuidingStateInnerDown(MMI::PointerEvent &event); 476 477 /** 478 * @brief Handle move events on touchExploring state. 479 * @param event event the touch event from Multimodal 480 */ 481 void HandleTouchGuidingStateInnerMove(MMI::PointerEvent &event); 482 483 /** 484 * @brief Handle move events on dragging state. 485 * @param event event the touch event from Multimodal 486 */ 487 void HandleDraggingStateInnerMove(MMI::PointerEvent &event); 488 489 /** 490 * @brief Ignore repeat execute action. 491 */ 492 bool IgnoreRepeatExecuteAction(); 493 494 /** 495 * @brief Calculate Offset. 496 * @param event event the touch event from Multimodal 497 */ 498 void OffsetEvent(MMI::PointerEvent &event); 499 500 /** 501 * @brief Find Focused Element. 502 * @param elementInfo the focused element. 503 * @return Returns true if find focused flement successfully; returns false code otherwise. 504 */ 505 bool FindFocusedElement(AccessibilityElementInfo &elementInfo); 506 507 int32_t currentState_ = -1; 508 int32_t longPressPointId_ = INIT_POINT_ID; 509 float longPressOffsetX_ = INIT_MMIPOINT; 510 float longPressOffsetY_ = INIT_MMIPOINT; 511 bool isTouchStart_ = false; 512 bool isTouchGuiding_ = false; 513 ReceivedEventRecorder receivedRecorder_ = {}; 514 InjectedEventRecorder injectedRecorder_ = {}; 515 std::list<MMI::PointerEvent> pointerEvents_ {}; 516 AccessibilityGestureRecognizer gestureRecognizer_; 517 AccessibilityMultiTapGestureRecognizer multiFingerGestureRecognizer_; 518 std::unique_ptr<TouchGuideListener> touchGuideListener_ = nullptr; 519 std::shared_ptr<TGEventHandler> handler_ = nullptr; 520 std::shared_ptr<AppExecFwk::EventRunner> runner_ = nullptr; 521 std::shared_ptr<MMI::PointerEvent> doubleTapLongPressDownEvent_ = nullptr; 522 bool focusedElementExist_ = false; 523 int32_t leftTopX_ = INIT_POINT_DISPLAY; 524 int32_t leftTopY_ = INIT_POINT_DISPLAY; 525 int32_t rightBottomX_ = INIT_POINT_DISPLAY; 526 int32_t rightBottomY_ = INIT_POINT_DISPLAY; 527 int32_t currentPid_ = -1; 528 std::list<MMI::PointerEvent> cachedPointerEvents_ {}; 529 }; 530 } // namespace Accessibility 531 } // namespace OHOS 532 #endif // ACCESSIBILITY_TOUCH_GUIDER_H