1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.statusbar.policy 18 19 import android.app.ActivityOptions 20 import android.app.Notification 21 import android.app.Notification.Action.SEMANTIC_ACTION_MARK_CONVERSATION_AS_PRIORITY 22 import android.app.PendingIntent 23 import android.app.RemoteInput 24 import android.content.Context 25 import android.content.Intent 26 import android.graphics.Bitmap 27 import android.graphics.ImageDecoder 28 import android.graphics.drawable.AdaptiveIconDrawable 29 import android.graphics.drawable.BitmapDrawable 30 import android.graphics.drawable.Drawable 31 import android.graphics.drawable.GradientDrawable 32 import android.graphics.drawable.Icon 33 import android.os.Build 34 import android.os.Bundle 35 import android.os.SystemClock 36 import android.util.Log 37 import android.view.ContextThemeWrapper 38 import android.view.LayoutInflater 39 import android.view.View 40 import android.view.ViewGroup 41 import android.view.accessibility.AccessibilityNodeInfo 42 import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction 43 import android.widget.Button 44 import com.android.systemui.R 45 import com.android.systemui.plugins.ActivityStarter 46 import com.android.systemui.shared.system.ActivityManagerWrapper 47 import com.android.systemui.shared.system.DevicePolicyManagerWrapper 48 import com.android.systemui.shared.system.PackageManagerWrapper 49 import com.android.systemui.statusbar.NotificationRemoteInputManager 50 import com.android.systemui.statusbar.NotificationUiAdjustment 51 import com.android.systemui.statusbar.SmartReplyController 52 import com.android.systemui.statusbar.notification.collection.NotificationEntry 53 import com.android.systemui.statusbar.notification.logging.NotificationLogger 54 import com.android.systemui.statusbar.phone.KeyguardDismissUtil 55 import com.android.systemui.statusbar.policy.InflatedSmartReplyState.SuppressedActions 56 import com.android.systemui.statusbar.policy.SmartReplyView.SmartActions 57 import com.android.systemui.statusbar.policy.SmartReplyView.SmartButtonType 58 import com.android.systemui.statusbar.policy.SmartReplyView.SmartReplies 59 import java.util.concurrent.FutureTask 60 import java.util.concurrent.SynchronousQueue 61 import java.util.concurrent.ThreadPoolExecutor 62 import java.util.concurrent.TimeUnit 63 import javax.inject.Inject 64 import kotlin.system.measureTimeMillis 65 66 67 /** Returns whether we should show the smart reply view and its smart suggestions. */ 68 fun shouldShowSmartReplyView( 69 entry: NotificationEntry, 70 smartReplyState: InflatedSmartReplyState 71 ): Boolean { 72 if (smartReplyState.smartReplies == null && 73 smartReplyState.smartActions == null) { 74 // There are no smart replies and no smart actions. 75 return false 76 } 77 // If we are showing the spinner we don't want to add the buttons. 78 val showingSpinner = entry.sbn.notification.extras 79 .getBoolean(Notification.EXTRA_SHOW_REMOTE_INPUT_SPINNER, false) 80 if (showingSpinner) { 81 return false 82 } 83 // If we are keeping the notification around while sending we don't want to add the buttons. 84 return !entry.sbn.notification.extras 85 .getBoolean(Notification.EXTRA_HIDE_SMART_REPLIES, false) 86 } 87 88 /** Determines if two [InflatedSmartReplyState] are visually similar. */ 89 fun areSuggestionsSimilar( 90 left: InflatedSmartReplyState?, 91 right: InflatedSmartReplyState? 92 ): Boolean = when { 93 left === right -> true 94 left == null || right == null -> false 95 left.hasPhishingAction != right.hasPhishingAction -> false 96 left.smartRepliesList != right.smartRepliesList -> false 97 left.suppressedActionIndices != right.suppressedActionIndices -> false 98 else -> !NotificationUiAdjustment.areDifferent(left.smartActionsList, right.smartActionsList) 99 } 100 101 interface SmartReplyStateInflater { 102 fun inflateSmartReplyState(entry: NotificationEntry): InflatedSmartReplyState 103 104 fun inflateSmartReplyViewHolder( 105 sysuiContext: Context, 106 notifPackageContext: Context, 107 entry: NotificationEntry, 108 existingSmartReplyState: InflatedSmartReplyState?, 109 newSmartReplyState: InflatedSmartReplyState 110 ): InflatedSmartReplyViewHolder 111 } 112 113 /*internal*/ class SmartReplyStateInflaterImpl @Inject constructor( 114 private val constants: SmartReplyConstants, 115 private val activityManagerWrapper: ActivityManagerWrapper, 116 private val packageManagerWrapper: PackageManagerWrapper, 117 private val devicePolicyManagerWrapper: DevicePolicyManagerWrapper, 118 private val smartRepliesInflater: SmartReplyInflater, 119 private val smartActionsInflater: SmartActionInflater 120 ) : SmartReplyStateInflater { 121 122 override fun inflateSmartReplyState(entry: NotificationEntry): InflatedSmartReplyState = 123 chooseSmartRepliesAndActions(entry) 124 125 override fun inflateSmartReplyViewHolder( 126 sysuiContext: Context, 127 notifPackageContext: Context, 128 entry: NotificationEntry, 129 existingSmartReplyState: InflatedSmartReplyState?, 130 newSmartReplyState: InflatedSmartReplyState 131 ): InflatedSmartReplyViewHolder { 132 if (!shouldShowSmartReplyView(entry, newSmartReplyState)) { 133 return InflatedSmartReplyViewHolder( 134 null /* smartReplyView */, 135 null /* smartSuggestionButtons */) 136 } 137 138 // Only block clicks if the smart buttons are different from the previous set - to avoid 139 // scenarios where a user incorrectly cannot click smart buttons because the 140 // notification is updated. 141 val delayOnClickListener = 142 !areSuggestionsSimilar(existingSmartReplyState, newSmartReplyState) 143 144 val smartReplyView = SmartReplyView.inflate(sysuiContext, constants) 145 146 val smartReplies = newSmartReplyState.smartReplies 147 smartReplyView.setSmartRepliesGeneratedByAssistant(smartReplies?.fromAssistant ?: false) 148 val smartReplyButtons = smartReplies?.let { 149 smartReplies.choices.asSequence().mapIndexed { index, choice -> 150 smartRepliesInflater.inflateReplyButton( 151 smartReplyView, 152 entry, 153 smartReplies, 154 index, 155 choice, 156 delayOnClickListener) 157 } 158 } ?: emptySequence() 159 160 val smartActionButtons = newSmartReplyState.smartActions?.let { smartActions -> 161 val themedPackageContext = 162 ContextThemeWrapper(notifPackageContext, sysuiContext.theme) 163 smartActions.actions.asSequence() 164 .filter { it.actionIntent != null } 165 .mapIndexed { index, action -> 166 smartActionsInflater.inflateActionButton( 167 smartReplyView, 168 entry, 169 smartActions, 170 index, 171 action, 172 delayOnClickListener, 173 themedPackageContext) 174 } 175 } ?: emptySequence() 176 177 return InflatedSmartReplyViewHolder( 178 smartReplyView, 179 (smartReplyButtons + smartActionButtons).toList()) 180 } 181 182 /** 183 * Chose what smart replies and smart actions to display. App generated suggestions take 184 * precedence. So if the app provides any smart replies, we don't show any 185 * replies or actions generated by the NotificationAssistantService (NAS), and if the app 186 * provides any smart actions we also don't show any NAS-generated replies or actions. 187 */ 188 fun chooseSmartRepliesAndActions(entry: NotificationEntry): InflatedSmartReplyState { 189 val notification = entry.sbn.notification 190 val remoteInputActionPair = notification.findRemoteInputActionPair(false /* freeform */) 191 val freeformRemoteInputActionPair = 192 notification.findRemoteInputActionPair(true /* freeform */) 193 if (!constants.isEnabled) { 194 if (DEBUG) { 195 Log.d(TAG, "Smart suggestions not enabled, not adding suggestions for " + 196 entry.sbn.key) 197 } 198 return InflatedSmartReplyState(null, null, null, false) 199 } 200 // Only use smart replies from the app if they target P or above. We have this check because 201 // the smart reply API has been used for other things (Wearables) in the past. The API to 202 // add smart actions is new in Q so it doesn't require a target-sdk check. 203 val enableAppGeneratedSmartReplies = (!constants.requiresTargetingP() || 204 entry.targetSdk >= Build.VERSION_CODES.P) 205 val appGeneratedSmartActions = notification.contextualActions 206 207 var smartReplies: SmartReplies? = when { 208 enableAppGeneratedSmartReplies -> remoteInputActionPair?.let { pair -> 209 pair.second.actionIntent?.let { actionIntent -> 210 if (pair.first.choices?.isNotEmpty() == true) 211 SmartReplies( 212 pair.first.choices.asList(), 213 pair.first, 214 actionIntent, 215 false /* fromAssistant */) 216 else null 217 } 218 } 219 else -> null 220 } 221 var smartActions: SmartActions? = when { 222 appGeneratedSmartActions.isNotEmpty() -> 223 SmartActions(appGeneratedSmartActions, false /* fromAssistant */) 224 else -> null 225 } 226 // Apps didn't provide any smart replies / actions, use those from NAS (if any). 227 if (smartReplies == null && smartActions == null) { 228 val entryReplies = entry.smartReplies 229 val entryActions = entry.smartActions 230 if (entryReplies.isNotEmpty() && 231 freeformRemoteInputActionPair != null && 232 freeformRemoteInputActionPair.second.allowGeneratedReplies && 233 freeformRemoteInputActionPair.second.actionIntent != null) { 234 smartReplies = SmartReplies( 235 entryReplies, 236 freeformRemoteInputActionPair.first, 237 freeformRemoteInputActionPair.second.actionIntent, 238 true /* fromAssistant */) 239 } 240 if (entryActions.isNotEmpty() && 241 notification.allowSystemGeneratedContextualActions) { 242 val systemGeneratedActions: List<Notification.Action> = when { 243 activityManagerWrapper.isLockTaskKioskModeActive -> 244 // Filter actions if we're in kiosk-mode - we don't care about screen 245 // pinning mode, since notifications aren't shown there anyway. 246 filterAllowlistedLockTaskApps(entryActions) 247 else -> entryActions 248 } 249 smartActions = SmartActions(systemGeneratedActions, true /* fromAssistant */) 250 } 251 } 252 val hasPhishingAction = smartActions?.actions?.any { 253 it.isContextual && it.semanticAction == 254 Notification.Action.SEMANTIC_ACTION_CONVERSATION_IS_PHISHING 255 } ?: false 256 var suppressedActions: SuppressedActions? = null 257 if (hasPhishingAction) { 258 // If there is a phishing action, calculate the indices of the actions with RemoteInput 259 // as those need to be hidden from the view. 260 val suppressedActionIndices = notification.actions.mapIndexedNotNull { index, action -> 261 if (action.remoteInputs?.isNotEmpty() == true) index else null 262 } 263 suppressedActions = SuppressedActions(suppressedActionIndices) 264 } 265 return InflatedSmartReplyState(smartReplies, smartActions, suppressedActions, 266 hasPhishingAction) 267 } 268 269 /** 270 * Filter actions so that only actions pointing to allowlisted apps are permitted. 271 * This filtering is only meaningful when in lock-task mode. 272 */ 273 private fun filterAllowlistedLockTaskApps( 274 actions: List<Notification.Action> 275 ): List<Notification.Action> = actions.filter { action -> 276 // Only allow actions that are explicit (implicit intents are not handled in lock-task 277 // mode), and link to allowlisted apps. 278 action.actionIntent?.intent?.let { intent -> 279 packageManagerWrapper.resolveActivity(intent, 0 /* flags */) 280 }?.let { resolveInfo -> 281 devicePolicyManagerWrapper.isLockTaskPermitted(resolveInfo.activityInfo.packageName) 282 } ?: false 283 } 284 } 285 286 interface SmartActionInflater { 287 fun inflateActionButton( 288 parent: ViewGroup, 289 entry: NotificationEntry, 290 smartActions: SmartActions, 291 actionIndex: Int, 292 action: Notification.Action, 293 delayOnClickListener: Boolean, 294 packageContext: Context 295 ): Button 296 } 297 298 private const val ICON_TASK_TIMEOUT_MS = 500L 299 private val iconTaskThreadPool = ThreadPoolExecutor(0, 25, 1, TimeUnit.MINUTES, SynchronousQueue()) 300 301 private fun loadIconDrawableWithTimeout( 302 icon: Icon, 303 packageContext: Context, 304 targetSize: Int, 305 ): Drawable? { 306 if (icon.type != Icon.TYPE_URI && icon.type != Icon.TYPE_URI_ADAPTIVE_BITMAP) { 307 return icon.loadDrawable(packageContext) 308 } 309 val bitmapTask = FutureTask { 310 val bitmap: Bitmap? 311 val durationMillis = measureTimeMillis { 312 val source = ImageDecoder.createSource(packageContext.contentResolver, icon.uri) 313 bitmap = ImageDecoder.decodeBitmap(source) { decoder, _, _ -> 314 decoder.setTargetSize(targetSize, targetSize) 315 decoder.allocator = ImageDecoder.ALLOCATOR_DEFAULT 316 } 317 } 318 if (durationMillis > ICON_TASK_TIMEOUT_MS) { 319 Log.w(TAG, "Loading $icon took ${durationMillis / 1000f} sec") 320 } 321 checkNotNull(bitmap) { "ImageDecoder.decodeBitmap() returned null" } 322 } 323 val bitmap = runCatching { 324 iconTaskThreadPool.execute(bitmapTask) 325 bitmapTask.get(ICON_TASK_TIMEOUT_MS, TimeUnit.MILLISECONDS) 326 }.getOrElse { ex -> 327 Log.e(TAG, "Failed to load $icon: $ex") 328 bitmapTask.cancel(true) 329 return null 330 } 331 // TODO(b/288561520): rewrite Icon so that we don't need to duplicate this logic 332 val bitmapDrawable = BitmapDrawable(packageContext.resources, bitmap) 333 val result = if (icon.type == Icon.TYPE_URI_ADAPTIVE_BITMAP) 334 AdaptiveIconDrawable(null, bitmapDrawable) else bitmapDrawable 335 if (icon.hasTint()) { 336 result.mutate() 337 result.setTintList(icon.tintList) 338 result.setTintBlendMode(icon.tintBlendMode) 339 } 340 return result 341 } 342 343 /* internal */ class SmartActionInflaterImpl @Inject constructor( 344 private val constants: SmartReplyConstants, 345 private val activityStarter: ActivityStarter, 346 private val smartReplyController: SmartReplyController, 347 private val headsUpManager: HeadsUpManager 348 ) : SmartActionInflater { 349 350 override fun inflateActionButton( 351 parent: ViewGroup, 352 entry: NotificationEntry, 353 smartActions: SmartActions, 354 actionIndex: Int, 355 action: Notification.Action, 356 delayOnClickListener: Boolean, 357 packageContext: Context 358 ): Button = 359 (LayoutInflater.from(parent.context) 360 .inflate(R.layout.smart_action_button, parent, false) as Button 361 ).apply { 362 text = action.title 363 364 // We received the Icon from the application - so use the Context of the application to 365 // reference icon resources. 366 val newIconSize = context.resources 367 .getDimensionPixelSize(R.dimen.smart_action_button_icon_size) 368 val iconDrawable = 369 loadIconDrawableWithTimeout(action.getIcon(), packageContext, newIconSize) 370 ?: GradientDrawable() 371 iconDrawable.setBounds(0, 0, newIconSize, newIconSize) 372 // Add the action icon to the Smart Action button. 373 setCompoundDrawablesRelative(iconDrawable, null, null, null) 374 375 val onClickListener = View.OnClickListener { 376 onSmartActionClick(entry, smartActions, actionIndex, action) 377 } 378 setOnClickListener( 379 if (delayOnClickListener) 380 DelayedOnClickListener(onClickListener, constants.onClickInitDelay) 381 else onClickListener) 382 383 // Mark this as an Action button 384 (layoutParams as SmartReplyView.LayoutParams).mButtonType = SmartButtonType.ACTION 385 } 386 387 private fun onSmartActionClick( 388 entry: NotificationEntry, 389 smartActions: SmartActions, 390 actionIndex: Int, 391 action: Notification.Action 392 ) = 393 if (smartActions.fromAssistant && 394 SEMANTIC_ACTION_MARK_CONVERSATION_AS_PRIORITY == action.semanticAction) { 395 entry.row.doSmartActionClick(entry.row.x.toInt() / 2, 396 entry.row.y.toInt() / 2, SEMANTIC_ACTION_MARK_CONVERSATION_AS_PRIORITY) 397 smartReplyController 398 .smartActionClicked(entry, actionIndex, action, smartActions.fromAssistant) 399 } else { 400 activityStarter.startPendingIntentDismissingKeyguard(action.actionIntent, entry.row) { 401 smartReplyController 402 .smartActionClicked(entry, actionIndex, action, smartActions.fromAssistant) 403 } 404 } 405 } 406 407 interface SmartReplyInflater { 408 fun inflateReplyButton( 409 parent: SmartReplyView, 410 entry: NotificationEntry, 411 smartReplies: SmartReplies, 412 replyIndex: Int, 413 choice: CharSequence, 414 delayOnClickListener: Boolean 415 ): Button 416 } 417 418 class SmartReplyInflaterImpl @Inject constructor( 419 private val constants: SmartReplyConstants, 420 private val keyguardDismissUtil: KeyguardDismissUtil, 421 private val remoteInputManager: NotificationRemoteInputManager, 422 private val smartReplyController: SmartReplyController, 423 private val context: Context 424 ) : SmartReplyInflater { 425 426 override fun inflateReplyButton( 427 parent: SmartReplyView, 428 entry: NotificationEntry, 429 smartReplies: SmartReplies, 430 replyIndex: Int, 431 choice: CharSequence, 432 delayOnClickListener: Boolean 433 ): Button = 434 (LayoutInflater.from(parent.context) 435 .inflate(R.layout.smart_reply_button, parent, false) as Button 436 ).apply { 437 text = choice 438 val onClickListener = View.OnClickListener { 439 onSmartReplyClick( 440 entry, 441 smartReplies, 442 replyIndex, 443 parent, 444 this, 445 choice) 446 } 447 setOnClickListener( 448 if (delayOnClickListener) 449 DelayedOnClickListener(onClickListener, constants.onClickInitDelay) 450 else onClickListener) 451 accessibilityDelegate = object : View.AccessibilityDelegate() { 452 override fun onInitializeAccessibilityNodeInfo( 453 host: View, 454 info: AccessibilityNodeInfo 455 ) { 456 super.onInitializeAccessibilityNodeInfo(host, info) 457 val label = parent.resources 458 .getString(R.string.accessibility_send_smart_reply) 459 val action = AccessibilityAction(AccessibilityNodeInfo.ACTION_CLICK, label) 460 info.addAction(action) 461 } 462 } 463 // TODO: probably shouldn't do this here, bad API 464 // Mark this as a Reply button 465 (layoutParams as SmartReplyView.LayoutParams).mButtonType = SmartButtonType.REPLY 466 } 467 468 private fun onSmartReplyClick( 469 entry: NotificationEntry, 470 smartReplies: SmartReplies, 471 replyIndex: Int, 472 smartReplyView: SmartReplyView, 473 button: Button, 474 choice: CharSequence 475 ) = keyguardDismissUtil.executeWhenUnlocked(!entry.isRowPinned) { 476 val canEditBeforeSend = constants.getEffectiveEditChoicesBeforeSending( 477 smartReplies.remoteInput.editChoicesBeforeSending) 478 if (canEditBeforeSend) { 479 remoteInputManager.activateRemoteInput( 480 button, 481 arrayOf(smartReplies.remoteInput), 482 smartReplies.remoteInput, 483 smartReplies.pendingIntent, 484 NotificationEntry.EditedSuggestionInfo(choice, replyIndex)) 485 } else { 486 smartReplyController.smartReplySent( 487 entry, 488 replyIndex, 489 button.text, 490 NotificationLogger.getNotificationLocation(entry).toMetricsEventEnum(), 491 false /* modifiedBeforeSending */) 492 entry.setHasSentReply() 493 try { 494 val intent = createRemoteInputIntent(smartReplies, choice) 495 val opts = ActivityOptions.makeBasic() 496 opts.setPendingIntentBackgroundActivityStartMode( 497 ActivityOptions.MODE_BACKGROUND_ACTIVITY_START_ALLOWED) 498 smartReplies.pendingIntent.send(context, 0, intent, /* onFinished */null, 499 /* handler */ null, /* requiredPermission */ null, opts.toBundle()) 500 } catch (e: PendingIntent.CanceledException) { 501 Log.w(TAG, "Unable to send smart reply", e) 502 } 503 smartReplyView.hideSmartSuggestions() 504 } 505 false // do not defer 506 } 507 508 private fun createRemoteInputIntent(smartReplies: SmartReplies, choice: CharSequence): Intent { 509 val results = Bundle() 510 results.putString(smartReplies.remoteInput.resultKey, choice.toString()) 511 val intent = Intent().addFlags(Intent.FLAG_RECEIVER_FOREGROUND) 512 RemoteInput.addResultsToIntent(arrayOf(smartReplies.remoteInput), intent, results) 513 RemoteInput.setResultsSource(intent, RemoteInput.SOURCE_CHOICE) 514 return intent 515 } 516 } 517 518 /** 519 * An OnClickListener wrapper that blocks the underlying OnClickListener for a given amount of 520 * time. 521 */ 522 private class DelayedOnClickListener( 523 private val mActualListener: View.OnClickListener, 524 private val mInitDelayMs: Long 525 ) : View.OnClickListener { 526 527 private val mInitTimeMs = SystemClock.elapsedRealtime() 528 529 override fun onClick(v: View) { 530 if (hasFinishedInitialization()) { 531 mActualListener.onClick(v) 532 } else { 533 Log.i(TAG, "Accidental Smart Suggestion click registered, delay: $mInitDelayMs") 534 } 535 } 536 537 private fun hasFinishedInitialization(): Boolean = 538 SystemClock.elapsedRealtime() >= mInitTimeMs + mInitDelayMs 539 } 540 541 private const val TAG = "SmartReplyViewInflater" 542 private val DEBUG = Log.isLoggable(TAG, Log.DEBUG) 543 544 // convenience function that swaps parameter order so that lambda can be placed at the end 545 private fun KeyguardDismissUtil.executeWhenUnlocked( 546 requiresShadeOpen: Boolean, 547 onDismissAction: () -> Boolean 548 ) = executeWhenUnlocked(onDismissAction, requiresShadeOpen, false) 549 550 // convenience function that swaps parameter order so that lambda can be placed at the end 551 private fun ActivityStarter.startPendingIntentDismissingKeyguard( 552 intent: PendingIntent, 553 associatedView: View?, 554 runnable: () -> Unit 555 ) = startPendingIntentDismissingKeyguard(intent, runnable::invoke, associatedView)