1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.wm; 18 19 import static com.android.internal.util.DumpUtils.KeyDumper; 20 import static com.android.internal.util.DumpUtils.ValueDumper; 21 import static com.android.internal.util.DumpUtils.dumpSparseArray; 22 import static com.android.server.wm.utils.RegionUtils.forEachRect; 23 24 import android.annotation.NonNull; 25 import android.graphics.Matrix; 26 import android.graphics.Rect; 27 import android.graphics.RectF; 28 import android.graphics.Region; 29 import android.os.Handler; 30 import android.os.IBinder; 31 import android.os.InputConfig; 32 import android.os.Looper; 33 import android.os.Message; 34 import android.util.Pair; 35 import android.util.Slog; 36 import android.util.SparseArray; 37 import android.view.IWindow; 38 import android.view.InputWindowHandle; 39 import android.view.MagnificationSpec; 40 import android.view.WindowInfo; 41 import android.view.WindowManager; 42 import android.window.WindowInfosListener; 43 44 import com.android.internal.annotations.GuardedBy; 45 46 import java.io.PrintWriter; 47 import java.util.ArrayList; 48 import java.util.HashMap; 49 import java.util.List; 50 import java.util.Map; 51 52 /** 53 * This class is the accessibility windows population adapter. 54 */ 55 public final class AccessibilityWindowsPopulator extends WindowInfosListener { 56 57 private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); 58 // If the surface flinger callback is not coming within in 2 frames time, i.e. about 59 // 35ms, then assuming the windows become stable. 60 private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35; 61 // To avoid the surface flinger callbacks always comes within in 2 frames, then no windows 62 // are reported to the A11y framework, and the animation duration time is 500ms, so setting 63 // this value as the max timeout value to force computing changed windows. However, since 64 // UiAutomator waits 500ms to determine that things are idle. Since we aren't actually idle, 65 // we need to reduce the timeout here a little so that we can deliver an updated state before 66 // UiAutomator reports idle based-on stale information. 67 private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 450; 68 69 private static final float[] sTempFloats = new float[9]; 70 71 private final WindowManagerService mService; 72 private final AccessibilityController mAccessibilityController; 73 @GuardedBy("mLock") 74 private final SparseArray<List<InputWindowHandle>> mInputWindowHandlesOnDisplays = 75 new SparseArray<>(); 76 @GuardedBy("mLock") 77 private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>(); 78 @GuardedBy("mLock") 79 private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>(); 80 private final SparseArray<MagnificationSpec> mCurrentMagnificationSpec = new SparseArray<>(); 81 @GuardedBy("mLock") 82 private final SparseArray<MagnificationSpec> mPreviousMagnificationSpec = new SparseArray<>(); 83 @GuardedBy("mLock") 84 private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>(); 85 @GuardedBy("mLock") 86 private boolean mWindowsNotificationEnabled = false; 87 @GuardedBy("mLock") 88 private final Map<IBinder, Matrix> mWindowsTransformMatrixMap = new HashMap<>(); 89 private final Object mLock = new Object(); 90 private final Handler mHandler; 91 92 private final Matrix mTempMatrix1 = new Matrix(); 93 private final Matrix mTempMatrix2 = new Matrix(); 94 private final float[] mTempFloat1 = new float[9]; 95 private final float[] mTempFloat2 = new float[9]; 96 private final float[] mTempFloat3 = new float[9]; 97 AccessibilityWindowsPopulator(WindowManagerService service, AccessibilityController accessibilityController)98 AccessibilityWindowsPopulator(WindowManagerService service, 99 AccessibilityController accessibilityController) { 100 mService = service; 101 mAccessibilityController = accessibilityController; 102 mHandler = new MyHandler(mService.mH.getLooper()); 103 } 104 105 /** 106 * Gets the visible windows list with the window layer on the specified display. 107 * 108 * @param displayId The display. 109 * @param outWindows The visible windows list. The z-order of each window in the list 110 * is from the top to bottom. 111 */ populateVisibleWindowsOnScreenLocked(int displayId, List<AccessibilityWindow> outWindows)112 public void populateVisibleWindowsOnScreenLocked(int displayId, 113 List<AccessibilityWindow> outWindows) { 114 List<InputWindowHandle> inputWindowHandles; 115 final Matrix inverseMatrix = new Matrix(); 116 final Matrix displayMatrix = new Matrix(); 117 118 synchronized (mLock) { 119 inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); 120 if (inputWindowHandles == null) { 121 outWindows.clear(); 122 123 return; 124 } 125 inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); 126 127 final DisplayInfo displayInfo = mDisplayInfos.get(displayId); 128 if (displayInfo != null) { 129 displayMatrix.set(displayInfo.mTransform); 130 } else { 131 Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called " 132 + "back from the surface fligner is null"); 133 } 134 } 135 136 final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); 137 final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); 138 final IBinder pipMenuIBinder = 139 shellroot != null ? shellroot.getAccessibilityWindowToken() : null; 140 141 for (final InputWindowHandle windowHandle : inputWindowHandles) { 142 final AccessibilityWindow accessibilityWindow = 143 AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, 144 pipMenuIBinder, displayMatrix); 145 146 outWindows.add(accessibilityWindow); 147 } 148 } 149 150 @Override onWindowInfosChanged(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)151 public void onWindowInfosChanged(InputWindowHandle[] windowHandles, 152 DisplayInfo[] displayInfos) { 153 mHandler.post(() -> onWindowInfosChangedInternal(windowHandles, displayInfos)); 154 } 155 onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos)156 private void onWindowInfosChangedInternal(InputWindowHandle[] windowHandles, 157 DisplayInfo[] displayInfos) { 158 final List<InputWindowHandle> tempVisibleWindows = new ArrayList<>(); 159 160 for (InputWindowHandle window : windowHandles) { 161 final boolean visible = (window.inputConfig & InputConfig.NOT_VISIBLE) == 0; 162 final boolean isNotClone = (window.inputConfig & InputConfig.CLONE) == 0; 163 final boolean hasTouchableRegion = !window.touchableRegion.isEmpty(); 164 final boolean hasNonEmptyFrame = 165 (window.frameBottom != window.frameTop) && (window.frameLeft 166 != window.frameRight); 167 if (visible && isNotClone && hasTouchableRegion && hasNonEmptyFrame) { 168 tempVisibleWindows.add(window); 169 } 170 } 171 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = 172 getWindowsTransformMatrix(tempVisibleWindows); 173 174 synchronized (mLock) { 175 mWindowsTransformMatrixMap.clear(); 176 mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap); 177 178 mVisibleWindows.clear(); 179 mVisibleWindows.addAll(tempVisibleWindows); 180 181 mDisplayInfos.clear(); 182 for (final DisplayInfo displayInfo : displayInfos) { 183 mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); 184 } 185 186 if (mWindowsNotificationEnabled) { 187 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { 188 mHandler.sendEmptyMessageDelayed( 189 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, 190 WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); 191 } 192 populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); 193 } 194 } 195 } 196 getWindowsTransformMatrix(List<InputWindowHandle> windows)197 private HashMap<IBinder, Matrix> getWindowsTransformMatrix(List<InputWindowHandle> windows) { 198 synchronized (mService.mGlobalLock) { 199 final HashMap<IBinder, Matrix> windowsTransformMatrixMap = new HashMap<>(); 200 201 for (InputWindowHandle inputWindowHandle : windows) { 202 final IWindow iWindow = inputWindowHandle.getWindow(); 203 final WindowState windowState = iWindow != null ? mService.mWindowMap.get( 204 iWindow.asBinder()) : null; 205 206 if (windowState != null && windowState.shouldMagnify()) { 207 final Matrix transformMatrix = new Matrix(); 208 windowState.getTransformationMatrix(sTempFloats, transformMatrix); 209 windowsTransformMatrixMap.put(iWindow.asBinder(), transformMatrix); 210 } 211 } 212 213 return windowsTransformMatrixMap; 214 } 215 } 216 217 /** 218 * Sets to notify the accessibilityController to compute changed windows on 219 * the display after populating the visible windows if the windows reported 220 * from the surface flinger changes. 221 * 222 * @param register {@code true} means starting windows population. 223 */ setWindowsNotification(boolean register)224 public void setWindowsNotification(boolean register) { 225 synchronized (mLock) { 226 if (mWindowsNotificationEnabled == register) { 227 return; 228 } 229 mWindowsNotificationEnabled = register; 230 if (mWindowsNotificationEnabled) { 231 Pair<InputWindowHandle[], DisplayInfo[]> info = register(); 232 onWindowInfosChangedInternal(info.first, info.second); 233 } else { 234 unregister(); 235 releaseResources(); 236 } 237 } 238 } 239 240 /** 241 * Sets the magnification spec for calculating the window bounds of all windows 242 * reported from the surface flinger in the magnifying. 243 * 244 * @param displayId The display Id. 245 * @param spec THe magnification spec. 246 */ setMagnificationSpec(int displayId, MagnificationSpec spec)247 public void setMagnificationSpec(int displayId, MagnificationSpec spec) { 248 synchronized (mLock) { 249 MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId); 250 if (currentMagnificationSpec == null) { 251 currentMagnificationSpec = new MagnificationSpec(); 252 currentMagnificationSpec.setTo(spec); 253 mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec); 254 255 return; 256 } 257 258 MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId); 259 if (previousMagnificationSpec == null) { 260 previousMagnificationSpec = new MagnificationSpec(); 261 mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec); 262 } 263 previousMagnificationSpec.setTo(currentMagnificationSpec); 264 currentMagnificationSpec.setTo(spec); 265 } 266 } 267 268 @GuardedBy("mLock") populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded()269 private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() { 270 final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>(); 271 272 for (final InputWindowHandle windowHandle : mVisibleWindows) { 273 List<InputWindowHandle> inputWindowHandles = tempWindowHandleList.get( 274 windowHandle.displayId); 275 276 if (inputWindowHandles == null) { 277 inputWindowHandles = new ArrayList<>(); 278 tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); 279 } 280 inputWindowHandles.add(windowHandle); 281 } 282 findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList); 283 284 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 285 getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList, 286 mInputWindowHandlesOnDisplays); 287 288 // Clones all windows from the callback of the surface flinger. 289 mInputWindowHandlesOnDisplays.clear(); 290 for (int i = 0; i < tempWindowHandleList.size(); i++) { 291 final int displayId = tempWindowHandleList.keyAt(i); 292 mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); 293 } 294 295 if (!displayIdsForWindowsChanged.isEmpty()) { 296 if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { 297 mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, 298 displayIdsForWindowsChanged).sendToTarget(); 299 } 300 301 return; 302 } 303 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 304 mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE, 305 SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); 306 } 307 308 @GuardedBy("mLock") getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, SparseArray<List<InputWindowHandle>> newWindowsList, SparseArray<List<InputWindowHandle>> oldWindowsList)309 private static void getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged, 310 SparseArray<List<InputWindowHandle>> newWindowsList, 311 SparseArray<List<InputWindowHandle>> oldWindowsList) { 312 for (int i = 0; i < newWindowsList.size(); i++) { 313 final int displayId = newWindowsList.keyAt(i); 314 final List<InputWindowHandle> newWindows = newWindowsList.get(displayId); 315 final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId); 316 317 if (hasWindowsChanged(newWindows, oldWindows)) { 318 outDisplayIdsForWindowsChanged.add(displayId); 319 } 320 } 321 } 322 323 @GuardedBy("mLock") hasWindowsChanged(List<InputWindowHandle> newWindows, List<InputWindowHandle> oldWindows)324 private static boolean hasWindowsChanged(List<InputWindowHandle> newWindows, 325 List<InputWindowHandle> oldWindows) { 326 if (oldWindows == null || oldWindows.size() != newWindows.size()) { 327 return true; 328 } 329 330 final int windowsCount = newWindows.size(); 331 // Since we always traverse windows from high to low layer, 332 // the old and new windows at the same index should be the 333 // same, otherwise something changed. 334 for (int i = 0; i < windowsCount; i++) { 335 final IWindow newWindowToken = newWindows.get(i).getWindow(); 336 final IWindow oldWindowToken = oldWindows.get(i).getWindow(); 337 final boolean hasNewWindowToken = newWindowToken != null; 338 final boolean hasOldWindowToken = oldWindowToken != null; 339 340 // If window token presence has changed then the windows have changed. 341 if (hasNewWindowToken != hasOldWindowToken) { 342 return true; 343 } 344 345 // If both old and new windows had window tokens, but those tokens differ, 346 // then the windows have changed. 347 if (hasNewWindowToken && hasOldWindowToken 348 && !newWindowToken.asBinder().equals(oldWindowToken.asBinder())) { 349 return true; 350 } 351 } 352 353 return false; 354 } 355 356 @GuardedBy("mLock") findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> windowHandleList)357 private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>> 358 windowHandleList) { 359 MagnificationSpec currentMagnificationSpec; 360 MagnificationSpec previousMagnificationSpec; 361 for (int i = 0; i < windowHandleList.size(); i++) { 362 final int displayId = windowHandleList.keyAt(i); 363 List<InputWindowHandle> inputWindowHandles = windowHandleList.get(displayId); 364 365 final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId); 366 if (currentSpec == null) { 367 continue; 368 } 369 currentMagnificationSpec = new MagnificationSpec(); 370 currentMagnificationSpec.setTo(currentSpec); 371 372 final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId); 373 374 if (previousSpec == null) { 375 final Matrix inverseMatrixForCurrentSpec = new Matrix(); 376 generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec); 377 mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec); 378 continue; 379 } 380 previousMagnificationSpec = new MagnificationSpec(); 381 previousMagnificationSpec.setTo(previousSpec); 382 383 generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles, 384 currentMagnificationSpec, previousMagnificationSpec); 385 } 386 } 387 388 @GuardedBy("mLock") generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, MagnificationSpec previousMagnificationSpec)389 private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( 390 List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec, 391 MagnificationSpec previousMagnificationSpec) { 392 // To decrease the counts of holding the WindowManagerService#mGlogalLock in 393 // the method, getWindowTransformMatrix(), this for loop begins from the bottom 394 // to top of the z-order windows. 395 for (int index = inputWindowHandles.size() - 1; index >= 0; index--) { 396 final Matrix windowTransformMatrix = mTempMatrix2; 397 final InputWindowHandle windowHandle = inputWindowHandles.get(index); 398 final IBinder iBinder = 399 windowHandle.getWindow() != null ? windowHandle.getWindow().asBinder() : null; 400 401 if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) { 402 generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec, 403 previousMagnificationSpec, windowTransformMatrix); 404 405 break; 406 } 407 } 408 } 409 410 @GuardedBy("mLock") getWindowTransformMatrix(IBinder iBinder, Matrix outTransform)411 private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) { 412 final Matrix windowMatrix = iBinder != null 413 ? mWindowsTransformMatrixMap.get(iBinder) : null; 414 415 if (windowMatrix == null) { 416 return false; 417 } 418 outTransform.set(windowMatrix); 419 420 return true; 421 } 422 423 /** 424 * Generates the inverse matrix based on the proper magnification spec. 425 * The magnification spec associated with the InputWindowHandle might not the current 426 * spec set by WM, which might be the previous one. To find the appropriate spec, 427 * we store two consecutive magnification specs, and found out which one is the proper 428 * one closing the identity matrix for generating the inverse matrix. 429 * 430 * @param inputWindowHandle The window from the surface flinger. 431 * @param currentMagnificationSpec The current magnification spec. 432 * @param previousMagnificationSpec The previous magnification spec. 433 * @param transformMatrix The transform matrix of the window doesn't consider the 434 * magnifying effect. 435 */ 436 @GuardedBy("mLock") generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec currentMagnificationSpec, @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix)437 private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, 438 @NonNull MagnificationSpec currentMagnificationSpec, 439 @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) { 440 441 final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1; 442 computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec, 443 transformMatrix, identityMatrixFloatsForCurrentSpec); 444 final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2; 445 computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec, 446 transformMatrix, identityMatrixFloatsForPreviousSpec); 447 448 Matrix inverseMatrixForMagnificationSpec = new Matrix(); 449 if (selectProperMagnificationSpecByComparingIdentityDegree( 450 identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) { 451 generateInverseMatrix(currentMagnificationSpec, 452 inverseMatrixForMagnificationSpec); 453 454 // Choosing the current spec means the previous spec is out of date, 455 // so removing it. And if the current spec is no magnifying, meaning 456 // the magnifying is done so removing the inverse matrix of this display. 457 mPreviousMagnificationSpec.remove(inputWindowHandle.displayId); 458 if (currentMagnificationSpec.isNop()) { 459 mCurrentMagnificationSpec.remove(inputWindowHandle.displayId); 460 mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId); 461 return; 462 } 463 } else { 464 generateInverseMatrix(previousMagnificationSpec, 465 inverseMatrixForMagnificationSpec); 466 } 467 468 mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId, 469 inverseMatrixForMagnificationSpec); 470 } 471 472 /** 473 * Computes the identity matrix for generating the 474 * inverse matrix based on below formula under window is at the stable state: 475 * inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform 476 * = IdentityMatrix 477 */ 478 @GuardedBy("mLock") computeIdentityMatrix(InputWindowHandle inputWindowHandle, @NonNull MagnificationSpec magnificationSpec, Matrix transformMatrix, float[] magnifyMatrixFloats)479 private void computeIdentityMatrix(InputWindowHandle inputWindowHandle, 480 @NonNull MagnificationSpec magnificationSpec, 481 Matrix transformMatrix, float[] magnifyMatrixFloats) { 482 final Matrix specMatrix = mTempMatrix1; 483 transformMagnificationSpecToMatrix(magnificationSpec, specMatrix); 484 485 final Matrix resultMatrix = new Matrix(inputWindowHandle.transform); 486 resultMatrix.preConcat(specMatrix); 487 resultMatrix.preConcat(transformMatrix); 488 489 resultMatrix.getValues(magnifyMatrixFloats); 490 } 491 492 /** 493 * @return true if selecting the magnification spec one, otherwise selecting the 494 * magnification spec two. 495 */ 496 @GuardedBy("mLock") selectProperMagnificationSpecByComparingIdentityDegree( float[] magnifyMatrixFloatsForSpecOne, float[] magnifyMatrixFloatsForSpecTwo)497 private boolean selectProperMagnificationSpecByComparingIdentityDegree( 498 float[] magnifyMatrixFloatsForSpecOne, 499 float[] magnifyMatrixFloatsForSpecTwo) { 500 final float[] IdentityMatrixValues = mTempFloat3; 501 Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues); 502 503 final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 504 - magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]); 505 final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] 506 - magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]); 507 final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 508 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]); 509 final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] 510 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]); 511 final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 512 - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]); 513 final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] 514 - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]); 515 final float offsetDiffForSpecOne = offsetXDiffForSpecOne 516 + offsetYDiffForSpecOne; 517 final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo 518 + offsetYDiffForSpecTwo; 519 520 return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0 521 || (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0 522 && Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0); 523 } 524 525 @GuardedBy("mLock") generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix)526 private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) { 527 outMatrix.reset(); 528 529 final Matrix tempMatrix = new Matrix(); 530 transformMagnificationSpecToMatrix(spec, tempMatrix); 531 532 final boolean result = tempMatrix.invert(outMatrix); 533 if (!result) { 534 Slog.e(TAG, "Can't inverse the magnification spec matrix with the " 535 + "magnification spec = " + spec); 536 outMatrix.reset(); 537 } 538 } 539 540 @GuardedBy("mLock") transformMagnificationSpecToMatrix(MagnificationSpec spec, Matrix outMatrix)541 private static void transformMagnificationSpecToMatrix(MagnificationSpec spec, 542 Matrix outMatrix) { 543 outMatrix.reset(); 544 outMatrix.postScale(spec.scale, spec.scale); 545 outMatrix.postTranslate(spec.offsetX, spec.offsetY); 546 } 547 notifyWindowsChanged(@onNull List<Integer> displayIdsForWindowsChanged)548 private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) { 549 mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT); 550 551 for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { 552 mAccessibilityController.performComputeChangedWindowsNot( 553 displayIdsForWindowsChanged.get(i), false); 554 } 555 } 556 forceUpdateWindows()557 private void forceUpdateWindows() { 558 final List<Integer> displayIdsForWindowsChanged = new ArrayList<>(); 559 560 synchronized (mLock) { 561 for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) { 562 final int displayId = mInputWindowHandlesOnDisplays.keyAt(i); 563 displayIdsForWindowsChanged.add(displayId); 564 } 565 } 566 notifyWindowsChanged(displayIdsForWindowsChanged); 567 } 568 dump(PrintWriter pw, String prefix)569 void dump(PrintWriter pw, String prefix) { 570 pw.print(prefix); pw.println("AccessibilityWindowsPopulator"); 571 String prefix2 = prefix + " "; 572 573 pw.print(prefix2); pw.print("mWindowsNotificationEnabled: "); 574 pw.println(mWindowsNotificationEnabled); 575 576 if (mVisibleWindows.isEmpty()) { 577 pw.print(prefix2); pw.println("No visible windows"); 578 } else { 579 pw.print(prefix2); pw.print(mVisibleWindows.size()); 580 pw.print(" visible windows: "); pw.println(mVisibleWindows); 581 } 582 KeyDumper noKeyDumper = (i, k) -> {}; // display id is already shown on value; 583 KeyDumper displayDumper = (i, d) -> pw.printf("%sDisplay #%d: ", prefix, d); 584 // Ideally magnificationSpecDumper should use spec.dump(pw), but there is no such method 585 ValueDumper<MagnificationSpec> magnificationSpecDumper = spec -> pw.print(spec); 586 587 dumpSparseArray(pw, prefix2, mDisplayInfos, "display info", noKeyDumper, d -> pw.print(d)); 588 dumpSparseArray(pw, prefix2, mInputWindowHandlesOnDisplays, "window handles on display", 589 displayDumper, list -> pw.print(list)); 590 dumpSparseArray(pw, prefix2, mMagnificationSpecInverseMatrix, "magnification spec matrix", 591 noKeyDumper, matrix -> matrix.dump(pw)); 592 dumpSparseArray(pw, prefix2, mCurrentMagnificationSpec, "current magnification spec", 593 noKeyDumper, magnificationSpecDumper); 594 dumpSparseArray(pw, prefix2, mPreviousMagnificationSpec, "previous magnification spec", 595 noKeyDumper, magnificationSpecDumper); 596 } 597 598 @GuardedBy("mLock") releaseResources()599 private void releaseResources() { 600 mInputWindowHandlesOnDisplays.clear(); 601 mMagnificationSpecInverseMatrix.clear(); 602 mVisibleWindows.clear(); 603 mDisplayInfos.clear(); 604 mCurrentMagnificationSpec.clear(); 605 mPreviousMagnificationSpec.clear(); 606 mWindowsTransformMatrixMap.clear(); 607 mWindowsNotificationEnabled = false; 608 mHandler.removeCallbacksAndMessages(null); 609 } 610 611 private class MyHandler extends Handler { 612 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; 613 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2; 614 public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3; 615 MyHandler(Looper looper)616 MyHandler(Looper looper) { 617 super(looper, null, false); 618 } 619 620 @Override handleMessage(Message message)621 public void handleMessage(Message message) { 622 switch (message.what) { 623 case MESSAGE_NOTIFY_WINDOWS_CHANGED: { 624 final List<Integer> displayIdsForWindowsChanged = (List<Integer>) message.obj; 625 notifyWindowsChanged(displayIdsForWindowsChanged); 626 } break; 627 628 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: { 629 forceUpdateWindows(); 630 } break; 631 632 case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: { 633 Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms " 634 + "and notify windows changed immediately"); 635 mHandler.removeMessages( 636 MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); 637 638 forceUpdateWindows(); 639 } break; 640 } 641 } 642 } 643 644 /** 645 * This class represents information about a window from the 646 * surface flinger to the accessibility framework. 647 */ 648 public static class AccessibilityWindow { 649 // Data 650 private IWindow mWindow; 651 private int mDisplayId; 652 @WindowManager.LayoutParams.WindowType 653 private int mType; 654 @InputWindowHandle.InputConfigFlags 655 private int mInputConfig; 656 private int mPrivateFlags; 657 private boolean mIsPIPMenu; 658 private boolean mIsFocused; 659 private boolean mShouldMagnify; 660 private boolean mIgnoreDuetoRecentsAnimation; 661 private final Region mTouchableRegionInScreen = new Region(); 662 private final Region mTouchableRegionInWindow = new Region(); 663 private WindowInfo mWindowInfo; 664 665 666 /** 667 * Returns the instance after initializing the internal data. 668 * @param service The window manager service. 669 * @param inputWindowHandle The window from the surface flinger. 670 * @param magnificationInverseMatrix The magnification spec inverse matrix. 671 */ initializeData(WindowManagerService service, InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, IBinder pipIBinder, Matrix displayMatrix)672 public static AccessibilityWindow initializeData(WindowManagerService service, 673 InputWindowHandle inputWindowHandle, Matrix magnificationInverseMatrix, 674 IBinder pipIBinder, Matrix displayMatrix) { 675 final IWindow window = inputWindowHandle.getWindow(); 676 final WindowState windowState = window != null ? service.mWindowMap.get( 677 window.asBinder()) : null; 678 679 final AccessibilityWindow instance = new AccessibilityWindow(); 680 681 instance.mWindow = window; 682 instance.mDisplayId = inputWindowHandle.displayId; 683 instance.mInputConfig = inputWindowHandle.inputConfig; 684 instance.mType = inputWindowHandle.layoutParamsType; 685 instance.mIsPIPMenu = window != null && window.asBinder().equals(pipIBinder); 686 687 // TODO (b/199357848): gets the private flag of the window from other way. 688 instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; 689 // TODO (b/199358208) : using new way to implement the focused window. 690 instance.mIsFocused = windowState != null && windowState.isFocused(); 691 instance.mShouldMagnify = windowState == null || windowState.shouldMagnify(); 692 693 final RecentsAnimationController controller = service.getRecentsAnimationController(); 694 instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null 695 && controller.shouldIgnoreForAccessibility(windowState); 696 697 final Rect windowFrame = new Rect(inputWindowHandle.frameLeft, 698 inputWindowHandle.frameTop, inputWindowHandle.frameRight, 699 inputWindowHandle.frameBottom); 700 getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, 701 instance.mTouchableRegionInWindow, windowFrame, magnificationInverseMatrix, 702 displayMatrix); 703 getUnMagnifiedTouchableRegion(instance.mShouldMagnify, 704 inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, 705 magnificationInverseMatrix, displayMatrix); 706 instance.mWindowInfo = windowState != null 707 ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); 708 709 // Compute the transform matrix that will transform bounds from the window 710 // coordinates to screen coordinates. 711 final Matrix inverseTransform = new Matrix(); 712 inputWindowHandle.transform.invert(inverseTransform); 713 inverseTransform.postConcat(displayMatrix); 714 inverseTransform.getValues(instance.mWindowInfo.mTransformMatrix); 715 716 // Compute the magnification spec matrix. 717 final Matrix magnificationSpecMatrix = new Matrix(); 718 if (instance.shouldMagnify() && magnificationInverseMatrix != null 719 && !magnificationInverseMatrix.isIdentity()) { 720 if (magnificationInverseMatrix.invert(magnificationSpecMatrix)) { 721 magnificationSpecMatrix.getValues(sTempFloats); 722 final MagnificationSpec spec = instance.mWindowInfo.mMagnificationSpec; 723 spec.scale = sTempFloats[Matrix.MSCALE_X]; 724 spec.offsetX = sTempFloats[Matrix.MTRANS_X]; 725 spec.offsetY = sTempFloats[Matrix.MTRANS_Y]; 726 } else { 727 Slog.w(TAG, "can't find spec"); 728 } 729 } 730 return instance; 731 } 732 733 /** 734 * Returns the touchable region in the screen. 735 * @param outRegion The touchable region. 736 */ getTouchableRegionInScreen(Region outRegion)737 public void getTouchableRegionInScreen(Region outRegion) { 738 outRegion.set(mTouchableRegionInScreen); 739 } 740 741 /** 742 * Returns the touchable region in the window. 743 * @param outRegion The touchable region. 744 */ getTouchableRegionInWindow(Region outRegion)745 public void getTouchableRegionInWindow(Region outRegion) { 746 outRegion.set(mTouchableRegionInWindow); 747 } 748 749 /** 750 * @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}. 751 */ getType()752 public int getType() { 753 return mType; 754 } 755 756 /** 757 * @return the layout parameter private flag 758 * {@link android.view.WindowManager.LayoutParams#privateFlags}. 759 */ getPrivateFlag()760 public int getPrivateFlag() { 761 return mPrivateFlags; 762 } 763 764 /** 765 * @return the windowInfo {@link WindowInfo}. 766 */ getWindowInfo()767 public WindowInfo getWindowInfo() { 768 return mWindowInfo; 769 } 770 771 /** 772 * @return true if this window should be magnified. 773 */ shouldMagnify()774 public boolean shouldMagnify() { 775 return mShouldMagnify; 776 } 777 778 /** 779 * @return true if this window is focused. 780 */ isFocused()781 public boolean isFocused() { 782 return mIsFocused; 783 } 784 785 /** 786 * @return true if it's running the recent animation but not the target app. 787 */ ignoreRecentsAnimationForAccessibility()788 public boolean ignoreRecentsAnimationForAccessibility() { 789 return mIgnoreDuetoRecentsAnimation; 790 } 791 792 /** 793 * @return true if this window is the trusted overlay. 794 */ isTrustedOverlay()795 public boolean isTrustedOverlay() { 796 return (mInputConfig & InputConfig.TRUSTED_OVERLAY) != 0; 797 } 798 799 /** 800 * @return true if this window is touchable. 801 */ isTouchable()802 public boolean isTouchable() { 803 return (mInputConfig & InputConfig.NOT_TOUCHABLE) == 0; 804 } 805 806 /** 807 * @return true if this window is the navigation bar with the gesture mode. 808 */ isUntouchableNavigationBar()809 public boolean isUntouchableNavigationBar() { 810 if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) { 811 return false; 812 } 813 814 return mTouchableRegionInScreen.isEmpty(); 815 } 816 817 /** 818 * @return true if this window is PIP menu. 819 */ isPIPMenu()820 public boolean isPIPMenu() { 821 return mIsPIPMenu; 822 } 823 getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix)824 private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, 825 Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) { 826 // Some modal windows, like the activity with Theme.dialog, has the full screen 827 // as its touchable region, but its window frame is smaller than the touchable 828 // region. The region we report should be the touchable area in the window frame 829 // for the consistency and match developers expectation. 830 // So we need to make the intersection between the frame and touchable region to 831 // obtain the real touch region in the screen. 832 Region touchRegion = new Region(); 833 touchRegion.set(inRegion); 834 touchRegion.op(frame, Region.Op.INTERSECT); 835 836 getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix, 837 displayMatrix); 838 } 839 840 /** 841 * Gets the un-magnified touchable region. If this window can be magnified and magnifying, 842 * we will transform the input touchable region by applying the inverse matrix of the 843 * magnification spec to get the un-magnified touchable region. 844 * @param shouldMagnify The window can be magnified. 845 * @param inRegion The touchable region of this window. 846 * @param outRegion The un-magnified touchable region of this window. 847 * @param inverseMatrix The inverse matrix of the magnification spec. 848 * @param displayMatrix The display transform matrix which takes display coordinates to 849 * logical display coordinates. 850 */ getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, Region outRegion, Matrix inverseMatrix, Matrix displayMatrix)851 private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, 852 Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) { 853 if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) { 854 outRegion.set(inRegion); 855 return; 856 } 857 858 forEachRect(inRegion, rect -> { 859 // Move to origin as all transforms are captured by the matrix. 860 RectF windowFrame = new RectF(rect); 861 862 displayMatrix.mapRect(windowFrame); 863 inverseMatrix.mapRect(windowFrame); 864 // Union all rects. 865 outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, 866 (int) windowFrame.right, (int) windowFrame.bottom)); 867 }); 868 } 869 getWindowInfoForWindowlessWindows(AccessibilityWindow window)870 private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) { 871 WindowInfo windowInfo = WindowInfo.obtain(); 872 windowInfo.displayId = window.mDisplayId; 873 windowInfo.type = window.mType; 874 windowInfo.token = window.mWindow != null ? window.mWindow.asBinder() : null; 875 windowInfo.hasFlagWatchOutsideTouch = (window.mInputConfig 876 & InputConfig.WATCH_OUTSIDE_TOUCH) != 0; 877 // Set it to true to be consistent with the legacy implementation. 878 windowInfo.inPictureInPicture = window.mIsPIPMenu; 879 return windowInfo; 880 } 881 882 @Override toString()883 public String toString() { 884 String windowToken = 885 mWindow != null ? mWindow.asBinder().toString() : "(no window token)"; 886 return "A11yWindow=[" + windowToken 887 + ", displayId=" + mDisplayId 888 + ", inputConfig=0x" + Integer.toHexString(mInputConfig) 889 + ", type=" + mType 890 + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) 891 + ", focused=" + mIsFocused 892 + ", shouldMagnify=" + mShouldMagnify 893 + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation 894 + ", isTrustedOverlay=" + isTrustedOverlay() 895 + ", regionInScreen=" + mTouchableRegionInScreen 896 + ", touchableRegion=" + mTouchableRegionInWindow 897 + ", isPIPMenu=" + mIsPIPMenu 898 + ", windowInfo=" + mWindowInfo 899 + "]"; 900 } 901 } 902 } 903