1 package com.android.systemui.assist; 2 3 import static com.android.systemui.DejankUtils.whitelistIpcs; 4 import static com.android.systemui.shared.system.QuickStepContract.SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED; 5 6 import android.annotation.NonNull; 7 import android.annotation.Nullable; 8 import android.app.ActivityManager; 9 import android.app.ActivityOptions; 10 import android.app.SearchManager; 11 import android.content.ActivityNotFoundException; 12 import android.content.ComponentName; 13 import android.content.Context; 14 import android.content.Intent; 15 import android.metrics.LogMaker; 16 import android.os.AsyncTask; 17 import android.os.Bundle; 18 import android.os.Handler; 19 import android.os.RemoteException; 20 import android.os.SystemClock; 21 import android.os.UserHandle; 22 import android.provider.Settings; 23 import android.service.voice.VoiceInteractionSession; 24 import android.util.Log; 25 26 import com.android.internal.app.AssistUtils; 27 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 28 import com.android.internal.app.IVisualQueryRecognitionStatusListener; 29 import com.android.internal.app.IVoiceInteractionSessionListener; 30 import com.android.internal.logging.MetricsLogger; 31 import com.android.internal.logging.nano.MetricsProto.MetricsEvent; 32 import com.android.keyguard.KeyguardUpdateMonitor; 33 import com.android.systemui.R; 34 import com.android.systemui.assist.ui.DefaultUiController; 35 import com.android.systemui.dagger.SysUISingleton; 36 import com.android.systemui.dagger.qualifiers.Main; 37 import com.android.systemui.model.SysUiState; 38 import com.android.systemui.recents.OverviewProxyService; 39 import com.android.systemui.settings.DisplayTracker; 40 import com.android.systemui.settings.UserTracker; 41 import com.android.systemui.statusbar.CommandQueue; 42 import com.android.systemui.statusbar.policy.DeviceProvisionedController; 43 import com.android.systemui.util.settings.SecureSettings; 44 45 import dagger.Lazy; 46 47 import java.util.ArrayList; 48 import java.util.Arrays; 49 import java.util.List; 50 51 import javax.inject.Inject; 52 53 /** 54 * Class to manage everything related to assist in SystemUI. 55 */ 56 @SysUISingleton 57 public class AssistManager { 58 59 /** 60 * Controls the UI for showing Assistant invocation progress. 61 */ 62 public interface UiController { 63 /** 64 * Updates the invocation progress. 65 * 66 * @param type one of INVOCATION_TYPE_GESTURE, INVOCATION_TYPE_ACTIVE_EDGE, 67 * INVOCATION_TYPE_VOICE, INVOCATION_TYPE_QUICK_SEARCH_BAR, 68 * INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS 69 * @param progress a float between 0 and 1 inclusive. 0 represents the beginning of the 70 * gesture; 1 represents the end. 71 */ onInvocationProgress(int type, float progress)72 void onInvocationProgress(int type, float progress); 73 74 /** 75 * Called when an invocation gesture completes. 76 * 77 * @param velocity the speed of the invocation gesture, in pixels per millisecond. For 78 * drags, this is 0. 79 */ onGestureCompletion(float velocity)80 void onGestureCompletion(float velocity); 81 82 /** 83 * Hides any SysUI for the assistant, but _does not_ close the assistant itself. 84 */ hide()85 void hide(); 86 } 87 88 /** 89 * An interface for a listener that receives notification that visual query attention has 90 * either been gained or lost. 91 */ 92 public interface VisualQueryAttentionListener { 93 /** Called when visual query attention has been gained. */ onAttentionGained()94 void onAttentionGained(); 95 96 /** Called when visual query attention has been lost. */ onAttentionLost()97 void onAttentionLost(); 98 } 99 100 private static final String TAG = "AssistManager"; 101 102 // Note that VERBOSE logging may leak PII (e.g. transcription contents). 103 private static final boolean VERBOSE = false; 104 105 private static final String INVOCATION_TIME_MS_KEY = "invocation_time_ms"; 106 private static final String INVOCATION_PHONE_STATE_KEY = "invocation_phone_state"; 107 protected static final String ACTION_KEY = "action"; 108 protected static final String SET_ASSIST_GESTURE_CONSTRAINED_ACTION = 109 "set_assist_gesture_constrained"; 110 protected static final String CONSTRAINED_KEY = "should_constrain"; 111 112 public static final String INVOCATION_TYPE_KEY = "invocation_type"; 113 public static final int INVOCATION_TYPE_UNKNOWN = 114 AssistUtils.INVOCATION_TYPE_UNKNOWN; 115 public static final int INVOCATION_TYPE_GESTURE = 116 AssistUtils.INVOCATION_TYPE_GESTURE; 117 public static final int INVOCATION_TYPE_OTHER = 118 AssistUtils.INVOCATION_TYPE_PHYSICAL_GESTURE; 119 public static final int INVOCATION_TYPE_VOICE = 120 AssistUtils.INVOCATION_TYPE_VOICE; 121 public static final int INVOCATION_TYPE_QUICK_SEARCH_BAR = 122 AssistUtils.INVOCATION_TYPE_QUICK_SEARCH_BAR; 123 public static final int INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS = 124 AssistUtils.INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS; 125 public static final int INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS = 126 AssistUtils.INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS; 127 public static final int INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS = 128 AssistUtils.INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS; 129 130 public static final int DISMISS_REASON_INVOCATION_CANCELLED = 1; 131 public static final int DISMISS_REASON_TAP = 2; 132 public static final int DISMISS_REASON_BACK = 3; 133 public static final int DISMISS_REASON_TIMEOUT = 4; 134 135 private static final long TIMEOUT_SERVICE = 2500; 136 private static final long TIMEOUT_ACTIVITY = 1000; 137 138 protected final Context mContext; 139 private final AssistDisclosure mAssistDisclosure; 140 private final PhoneStateMonitor mPhoneStateMonitor; 141 private final OverviewProxyService mOverviewProxyService; 142 private final UiController mUiController; 143 protected final Lazy<SysUiState> mSysUiState; 144 protected final AssistLogger mAssistLogger; 145 private final UserTracker mUserTracker; 146 private final DisplayTracker mDisplayTracker; 147 private final SecureSettings mSecureSettings; 148 private final ActivityManager mActivityManager; 149 150 private final DeviceProvisionedController mDeviceProvisionedController; 151 152 private final List<VisualQueryAttentionListener> mVisualQueryAttentionListeners = 153 new ArrayList<>(); 154 155 private final IVisualQueryDetectionAttentionListener mVisualQueryDetectionAttentionListener = 156 new IVisualQueryDetectionAttentionListener.Stub() { 157 @Override 158 public void onAttentionGained() { 159 handleVisualAttentionChanged(true); 160 } 161 162 @Override 163 public void onAttentionLost() { 164 handleVisualAttentionChanged(false); 165 } 166 }; 167 168 private final CommandQueue mCommandQueue; 169 protected final AssistUtils mAssistUtils; 170 171 // Invocation types that should be sent over OverviewProxy instead of handled here. 172 private int[] mAssistOverrideInvocationTypes; 173 174 @Inject AssistManager( DeviceProvisionedController controller, Context context, AssistUtils assistUtils, CommandQueue commandQueue, PhoneStateMonitor phoneStateMonitor, OverviewProxyService overviewProxyService, Lazy<SysUiState> sysUiState, DefaultUiController defaultUiController, AssistLogger assistLogger, @Main Handler uiHandler, UserTracker userTracker, DisplayTracker displayTracker, SecureSettings secureSettings, ActivityManager activityManager)175 public AssistManager( 176 DeviceProvisionedController controller, 177 Context context, 178 AssistUtils assistUtils, 179 CommandQueue commandQueue, 180 PhoneStateMonitor phoneStateMonitor, 181 OverviewProxyService overviewProxyService, 182 Lazy<SysUiState> sysUiState, 183 DefaultUiController defaultUiController, 184 AssistLogger assistLogger, 185 @Main Handler uiHandler, 186 UserTracker userTracker, 187 DisplayTracker displayTracker, 188 SecureSettings secureSettings, 189 ActivityManager activityManager) { 190 mContext = context; 191 mDeviceProvisionedController = controller; 192 mCommandQueue = commandQueue; 193 mAssistUtils = assistUtils; 194 mAssistDisclosure = new AssistDisclosure(context, uiHandler); 195 mOverviewProxyService = overviewProxyService; 196 mPhoneStateMonitor = phoneStateMonitor; 197 mAssistLogger = assistLogger; 198 mUserTracker = userTracker; 199 mDisplayTracker = displayTracker; 200 mSecureSettings = secureSettings; 201 mActivityManager = activityManager; 202 203 registerVoiceInteractionSessionListener(); 204 registerVisualQueryRecognitionStatusListener(); 205 206 mUiController = defaultUiController; 207 208 mSysUiState = sysUiState; 209 210 mOverviewProxyService.addCallback(new OverviewProxyService.OverviewProxyListener() { 211 @Override 212 public void onAssistantProgress(float progress) { 213 // Progress goes from 0 to 1 to indicate how close the assist gesture is to 214 // completion. 215 onInvocationProgress(INVOCATION_TYPE_GESTURE, progress); 216 } 217 218 @Override 219 public void onAssistantGestureCompletion(float velocity) { 220 onGestureCompletion(velocity); 221 } 222 }); 223 } 224 registerVoiceInteractionSessionListener()225 protected void registerVoiceInteractionSessionListener() { 226 mAssistUtils.registerVoiceInteractionSessionListener( 227 new IVoiceInteractionSessionListener.Stub() { 228 @Override 229 public void onVoiceSessionShown() throws RemoteException { 230 if (VERBOSE) { 231 Log.v(TAG, "Voice open"); 232 } 233 mAssistLogger.reportAssistantSessionEvent( 234 AssistantSessionEvent.ASSISTANT_SESSION_UPDATE); 235 } 236 237 @Override 238 public void onVoiceSessionHidden() throws RemoteException { 239 if (VERBOSE) { 240 Log.v(TAG, "Voice closed"); 241 } 242 mAssistLogger.reportAssistantSessionEvent( 243 AssistantSessionEvent.ASSISTANT_SESSION_CLOSE); 244 } 245 246 @Override 247 public void onVoiceSessionWindowVisibilityChanged(boolean visible) 248 throws RemoteException { 249 if (VERBOSE) { 250 Log.v(TAG, "Window visibility changed: " + visible); 251 } 252 } 253 254 @Override 255 public void onSetUiHints(Bundle hints) { 256 if (VERBOSE) { 257 Log.v(TAG, "UI hints received"); 258 } 259 260 String action = hints.getString(ACTION_KEY); 261 if (SET_ASSIST_GESTURE_CONSTRAINED_ACTION.equals(action)) { 262 mSysUiState.get() 263 .setFlag( 264 SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED, 265 hints.getBoolean(CONSTRAINED_KEY, false)) 266 .commitUpdate(mDisplayTracker.getDefaultDisplayId()); 267 } 268 } 269 }); 270 } 271 startAssist(Bundle args)272 public void startAssist(Bundle args) { 273 if (mActivityManager.getLockTaskModeState() == ActivityManager.LOCK_TASK_MODE_LOCKED) { 274 return; 275 } 276 if (shouldOverrideAssist(args)) { 277 try { 278 if (mOverviewProxyService.getProxy() == null) { 279 Log.w(TAG, "No OverviewProxyService to invoke assistant override"); 280 return; 281 } 282 mOverviewProxyService.getProxy().onAssistantOverrideInvoked( 283 args.getInt(INVOCATION_TYPE_KEY)); 284 } catch (RemoteException e) { 285 Log.w(TAG, "Unable to invoke assistant via OverviewProxyService override", e); 286 } 287 return; 288 } 289 290 final ComponentName assistComponent = getAssistInfo(); 291 if (assistComponent == null) { 292 return; 293 } 294 295 final boolean isService = assistComponent.equals(getVoiceInteractorComponentName()); 296 297 if (args == null) { 298 args = new Bundle(); 299 } 300 int legacyInvocationType = args.getInt(INVOCATION_TYPE_KEY, 0); 301 int legacyDeviceState = mPhoneStateMonitor.getPhoneState(); 302 args.putInt(INVOCATION_PHONE_STATE_KEY, legacyDeviceState); 303 args.putLong(INVOCATION_TIME_MS_KEY, SystemClock.elapsedRealtime()); 304 mAssistLogger.reportAssistantInvocationEventFromLegacy( 305 legacyInvocationType, 306 /* isInvocationComplete = */ true, 307 assistComponent, 308 legacyDeviceState); 309 logStartAssistLegacy(legacyInvocationType, legacyDeviceState); 310 startAssistInternal(args, assistComponent, isService); 311 } 312 shouldOverrideAssist(Bundle args)313 private boolean shouldOverrideAssist(Bundle args) { 314 if (args == null || !args.containsKey(INVOCATION_TYPE_KEY)) { 315 return false; 316 } 317 318 int invocationType = args.getInt(INVOCATION_TYPE_KEY); 319 return shouldOverrideAssist(invocationType); 320 } 321 322 /** @return true if the invocation type should be handled by OverviewProxy instead of SysUI. */ shouldOverrideAssist(int invocationType)323 public boolean shouldOverrideAssist(int invocationType) { 324 return mAssistOverrideInvocationTypes != null 325 && Arrays.stream(mAssistOverrideInvocationTypes).anyMatch( 326 override -> override == invocationType); 327 } 328 329 /** 330 * @param invocationTypes The invocation types that will henceforth be handled via 331 * OverviewProxy (Launcher); other invocation types should be handled by this class. 332 */ setAssistantOverridesRequested(int[] invocationTypes)333 public void setAssistantOverridesRequested(int[] invocationTypes) { 334 mAssistOverrideInvocationTypes = invocationTypes; 335 } 336 337 /** Called when the user is performing an assistant invocation action (e.g. Active Edge) */ onInvocationProgress(int type, float progress)338 public void onInvocationProgress(int type, float progress) { 339 mUiController.onInvocationProgress(type, progress); 340 } 341 342 /** 343 * Called when the user has invoked the assistant with the incoming velocity, in pixels per 344 * millisecond. For invocations without a velocity (e.g. slow drag), the velocity is set to 345 * zero. 346 */ onGestureCompletion(float velocity)347 public void onGestureCompletion(float velocity) { 348 mUiController.onGestureCompletion(velocity); 349 } 350 hideAssist()351 public void hideAssist() { 352 mAssistUtils.hideCurrentSession(); 353 } 354 355 /** 356 * Add the given {@link VisualQueryAttentionListener} to the list of listeners awaiting 357 * notification of gaining/losing visual query attention. 358 */ addVisualQueryAttentionListener(VisualQueryAttentionListener listener)359 public void addVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 360 if (!mVisualQueryAttentionListeners.contains(listener)) { 361 mVisualQueryAttentionListeners.add(listener); 362 } 363 } 364 365 /** 366 * Remove the given {@link VisualQueryAttentionListener} from the list of listeners awaiting 367 * notification of gaining/losing visual query attention. 368 */ removeVisualQueryAttentionListener(VisualQueryAttentionListener listener)369 public void removeVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 370 mVisualQueryAttentionListeners.remove(listener); 371 } 372 startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, boolean isService)373 private void startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, 374 boolean isService) { 375 if (isService) { 376 startVoiceInteractor(args); 377 } else { 378 startAssistActivity(args, assistComponent); 379 } 380 } 381 startAssistActivity(Bundle args, @NonNull ComponentName assistComponent)382 private void startAssistActivity(Bundle args, @NonNull ComponentName assistComponent) { 383 if (!mDeviceProvisionedController.isDeviceProvisioned()) { 384 return; 385 } 386 387 // Close Recent Apps if needed 388 mCommandQueue.animateCollapsePanels( 389 CommandQueue.FLAG_EXCLUDE_SEARCH_PANEL | CommandQueue.FLAG_EXCLUDE_RECENTS_PANEL, 390 false /* force */); 391 392 boolean structureEnabled = mSecureSettings.getIntForUser( 393 Settings.Secure.ASSIST_STRUCTURE_ENABLED, 1, UserHandle.USER_CURRENT) != 0; 394 395 final SearchManager searchManager = 396 (SearchManager) mContext.getSystemService(Context.SEARCH_SERVICE); 397 if (searchManager == null) { 398 return; 399 } 400 final Intent intent = searchManager.getAssistIntent(structureEnabled); 401 if (intent == null) { 402 return; 403 } 404 intent.setComponent(assistComponent); 405 intent.putExtras(args); 406 407 if (structureEnabled && AssistUtils.isDisclosureEnabled(mContext)) { 408 showDisclosure(); 409 } 410 411 try { 412 final ActivityOptions opts = ActivityOptions.makeCustomAnimation(mContext, 413 R.anim.search_launch_enter, R.anim.search_launch_exit); 414 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 415 AsyncTask.execute(new Runnable() { 416 @Override 417 public void run() { 418 mContext.startActivityAsUser(intent, opts.toBundle(), 419 mUserTracker.getUserHandle()); 420 } 421 }); 422 } catch (ActivityNotFoundException e) { 423 Log.w(TAG, "Activity not found for " + intent.getAction()); 424 } 425 } 426 startVoiceInteractor(Bundle args)427 private void startVoiceInteractor(Bundle args) { 428 mAssistUtils.showSessionForActiveService(args, 429 VoiceInteractionSession.SHOW_SOURCE_ASSIST_GESTURE, mContext.getAttributionTag(), 430 null, null); 431 } 432 registerVisualQueryRecognitionStatusListener()433 private void registerVisualQueryRecognitionStatusListener() { 434 if (!mContext.getResources() 435 .getBoolean(R.bool.config_enableVisualQueryAttentionDetection)) { 436 return; 437 } 438 439 mAssistUtils.subscribeVisualQueryRecognitionStatus( 440 new IVisualQueryRecognitionStatusListener.Stub() { 441 @Override 442 public void onStartPerceiving() { 443 mAssistUtils.enableVisualQueryDetection( 444 mVisualQueryDetectionAttentionListener); 445 } 446 447 @Override 448 public void onStopPerceiving() { 449 // Treat this as a signal that attention has been lost (and inform listeners 450 // accordingly). 451 handleVisualAttentionChanged(false); 452 mAssistUtils.disableVisualQueryDetection(); 453 } 454 }); 455 } 456 handleVisualAttentionChanged(boolean attentionGained)457 private void handleVisualAttentionChanged(boolean attentionGained) { 458 mVisualQueryAttentionListeners.forEach( 459 attentionGained 460 ? VisualQueryAttentionListener::onAttentionGained 461 : VisualQueryAttentionListener::onAttentionLost); 462 } 463 launchVoiceAssistFromKeyguard()464 public void launchVoiceAssistFromKeyguard() { 465 mAssistUtils.launchVoiceAssistFromKeyguard(); 466 } 467 canVoiceAssistBeLaunchedFromKeyguard()468 public boolean canVoiceAssistBeLaunchedFromKeyguard() { 469 // TODO(b/140051519) 470 return whitelistIpcs(() -> mAssistUtils.activeServiceSupportsLaunchFromKeyguard()); 471 } 472 getVoiceInteractorComponentName()473 public ComponentName getVoiceInteractorComponentName() { 474 return mAssistUtils.getActiveServiceComponentName(); 475 } 476 isVoiceSessionRunning()477 private boolean isVoiceSessionRunning() { 478 return mAssistUtils.isSessionRunning(); 479 } 480 481 @Nullable getAssistInfoForUser(int userId)482 public ComponentName getAssistInfoForUser(int userId) { 483 return mAssistUtils.getAssistComponentForUser(userId); 484 } 485 486 @Nullable getAssistInfo()487 private ComponentName getAssistInfo() { 488 return getAssistInfoForUser(KeyguardUpdateMonitor.getCurrentUser()); 489 } 490 showDisclosure()491 public void showDisclosure() { 492 mAssistDisclosure.postShow(); 493 } 494 onLockscreenShown()495 public void onLockscreenShown() { 496 AsyncTask.execute(new Runnable() { 497 @Override 498 public void run() { 499 mAssistUtils.onLockscreenShown(); 500 } 501 }); 502 } 503 504 /** Returns the logging flags for the given Assistant invocation type. */ toLoggingSubType(int invocationType)505 public int toLoggingSubType(int invocationType) { 506 return toLoggingSubType(invocationType, mPhoneStateMonitor.getPhoneState()); 507 } 508 logStartAssistLegacy(int invocationType, int phoneState)509 protected void logStartAssistLegacy(int invocationType, int phoneState) { 510 MetricsLogger.action( 511 new LogMaker(MetricsEvent.ASSISTANT) 512 .setType(MetricsEvent.TYPE_OPEN) 513 .setSubtype(toLoggingSubType(invocationType, phoneState))); 514 } 515 toLoggingSubType(int invocationType, int phoneState)516 protected final int toLoggingSubType(int invocationType, int phoneState) { 517 // Note that this logic will break if the number of Assistant invocation types exceeds 7. 518 // There are currently 5 invocation types, but we will be migrating to the new logging 519 // framework in the next update. 520 int subType = 0; 521 subType |= invocationType << 1; 522 subType |= phoneState << 4; 523 return subType; 524 } 525 } 526