1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.service.voice;
18 
19 import android.annotation.DurationMillisLong;
20 import android.annotation.NonNull;
21 import android.annotation.Nullable;
22 import android.annotation.SdkConstant;
23 import android.annotation.SuppressLint;
24 import android.annotation.SystemApi;
25 import android.app.Service;
26 import android.content.ContentCaptureOptions;
27 import android.content.Context;
28 import android.content.Intent;
29 import android.hardware.soundtrigger.SoundTrigger;
30 import android.media.AudioFormat;
31 import android.media.AudioSystem;
32 import android.os.IBinder;
33 import android.os.IRemoteCallback;
34 import android.os.ParcelFileDescriptor;
35 import android.os.PersistableBundle;
36 import android.os.RemoteException;
37 import android.os.SharedMemory;
38 import android.speech.IRecognitionServiceManager;
39 import android.util.Log;
40 import android.view.contentcapture.ContentCaptureManager;
41 import android.view.contentcapture.IContentCaptureManager;
42 
43 import com.android.internal.infra.AndroidFuture;
44 
45 import java.io.FileInputStream;
46 import java.io.FileNotFoundException;
47 import java.util.Objects;
48 import java.util.concurrent.ExecutionException;
49 import java.util.function.IntConsumer;
50 
51 /**
52  * Implemented by an application that wants to offer query detection with visual signals.
53  *
54  * This service leverages visual signals such as camera frames to detect and stream queries from the
55  * device microphone to the {@link VoiceInteractionService}, without the support of hotword. The
56  * system will bind an application's {@link VoiceInteractionService} first. When
57  * {@link VoiceInteractionService#createVisualQueryDetector(PersistableBundle, SharedMemory,
58  * Executor, VisualQueryDetector.Callback)} is called, the system will bind the application's
59  * {@link VisualQueryDetectionService}. When requested from {@link VoiceInteractionService}, the
60  * system calls into the {@link VisualQueryDetectionService#onStartDetection()} to enable
61  * detection. This method MUST be implemented to support visual query detection service.
62  *
63  * Note: Methods in this class may be called concurrently.
64  *
65  * @hide
66  */
67 @SystemApi
68 public abstract class VisualQueryDetectionService extends Service
69         implements SandboxedDetectionInitializer {
70 
71     private static final String TAG = VisualQueryDetectionService.class.getSimpleName();
72 
73     private static final long UPDATE_TIMEOUT_MILLIS = 20000;
74 
75     /**
76      * The {@link Intent} that must be declared as handled by the service.
77      * To be supported, the service must also require the
78      * {@link android.Manifest.permission#BIND_VISUAL_QUERY_DETECTION_SERVICE} permission
79      * so that other applications can not abuse it.
80      */
81     @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION)
82     public static final String SERVICE_INTERFACE =
83             "android.service.voice.VisualQueryDetectionService";
84 
85 
86     /** @hide */
87     public static final String KEY_INITIALIZATION_STATUS = "initialization_status";
88 
89     private IDetectorSessionVisualQueryDetectionCallback mRemoteCallback = null;
90     @Nullable
91     private ContentCaptureManager mContentCaptureManager;
92     @Nullable
93     private IRecognitionServiceManager mIRecognitionServiceManager;
94     @Nullable
95     private IDetectorSessionStorageService mDetectorSessionStorageService;
96 
97 
98     private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {
99 
100         @Override
101         public void detectWithVisualSignals(
102                 IDetectorSessionVisualQueryDetectionCallback callback) {
103             Log.v(TAG, "#detectWithVisualSignals");
104             mRemoteCallback = callback;
105             VisualQueryDetectionService.this.onStartDetection();
106         }
107 
108         @Override
109         public void stopDetection() {
110             Log.v(TAG, "#stopDetection");
111             VisualQueryDetectionService.this.onStopDetection();
112         }
113 
114         @Override
115         public void updateState(PersistableBundle options, SharedMemory sharedMemory,
116                 IRemoteCallback callback) throws RemoteException {
117             Log.v(TAG, "#updateState" + (callback != null ? " with callback" : ""));
118             VisualQueryDetectionService.this.onUpdateStateInternal(
119                     options,
120                     sharedMemory,
121                     callback);
122         }
123 
124         @Override
125         public void ping(IRemoteCallback callback) throws RemoteException {
126             callback.sendResult(null);
127         }
128 
129         @Override
130         public void detectFromDspSource(
131                 SoundTrigger.KeyphraseRecognitionEvent event,
132                 AudioFormat audioFormat,
133                 long timeoutMillis,
134                 IDspHotwordDetectionCallback callback) {
135             throw new UnsupportedOperationException("Not supported by VisualQueryDetectionService");
136         }
137 
138         @Override
139         public void detectFromMicrophoneSource(
140                 ParcelFileDescriptor audioStream,
141                 @HotwordDetectionService.AudioSource int audioSource,
142                 AudioFormat audioFormat,
143                 PersistableBundle options,
144                 IDspHotwordDetectionCallback callback) {
145             throw new UnsupportedOperationException("Not supported by VisualQueryDetectionService");
146         }
147 
148         @Override
149         public void updateAudioFlinger(IBinder audioFlinger) {
150             AudioSystem.setAudioFlingerBinder(audioFlinger);
151         }
152 
153         @Override
154         public void updateContentCaptureManager(IContentCaptureManager manager,
155                 ContentCaptureOptions options) {
156             mContentCaptureManager = new ContentCaptureManager(
157                     VisualQueryDetectionService.this, manager, options);
158         }
159 
160         @Override
161         public void updateRecognitionServiceManager(IRecognitionServiceManager manager) {
162             mIRecognitionServiceManager = manager;
163         }
164 
165         @Override
166         public void registerRemoteStorageService(IDetectorSessionStorageService
167                 detectorSessionStorageService) {
168             mDetectorSessionStorageService = detectorSessionStorageService;
169         }
170     };
171 
172     @Override
173     @SuppressLint("OnNameExpected")
getSystemService(@erviceName @onNull String name)174     public @Nullable Object getSystemService(@ServiceName @NonNull String name) {
175         if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) {
176             return mContentCaptureManager;
177         } else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name)
178                 && mIRecognitionServiceManager != null) {
179             return mIRecognitionServiceManager.asBinder();
180         } else {
181             return super.getSystemService(name);
182         }
183     }
184 
185     /**
186      * {@inheritDoc}
187      * @hide
188      */
189     @Override
190     @SystemApi
onUpdateState( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @DurationMillisLong long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)191     public void onUpdateState(
192             @Nullable PersistableBundle options,
193             @Nullable SharedMemory sharedMemory,
194             @DurationMillisLong long callbackTimeoutMillis,
195             @Nullable IntConsumer statusCallback) {
196     }
197 
198     @Override
199     @Nullable
onBind(@onNull Intent intent)200     public IBinder onBind(@NonNull Intent intent) {
201         if (SERVICE_INTERFACE.equals(intent.getAction())) {
202             return mInterface.asBinder();
203         }
204         Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": "
205                 + intent);
206         return null;
207     }
208 
onUpdateStateInternal(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, IRemoteCallback callback)209     private void onUpdateStateInternal(@Nullable PersistableBundle options,
210             @Nullable SharedMemory sharedMemory, IRemoteCallback callback) {
211         IntConsumer intConsumer =
212                 SandboxedDetectionInitializer.createInitializationStatusConsumer(callback);
213         onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer);
214     }
215 
216     /**
217      * This is called after the service is set up and the client should open the camera and the
218      * microphone to start recognition. When the {@link VoiceInteractionService} requests that this
219      * service {@link HotwordDetector#startRecognition()} start recognition on audio coming directly
220      * from the device microphone.
221      * <p>
222      * Signal senders that return attention and query results are also expected to be called in this
223      * method according to the detection outcomes.
224      * <p>
225      * On successful user attention, developers should call
226      * {@link VisualQueryDetectionService#gainedAttention()} to enable the streaming of the query.
227      * <p>
228      * On user attention is lost, developers should call
229      * {@link VisualQueryDetectionService#lostAttention()} to disable the streaming of the query.
230      * <p>
231      * On query is detected and ready to stream, developers should call
232      * {@link VisualQueryDetectionService#streamQuery(String)} to return detected query to the
233      * {@link VisualQueryDetector}.
234      * <p>
235      * On streamed query should be rejected, clients should call
236      * {@link VisualQueryDetectionService#rejectQuery()} to abandon query streamed to the
237      * {@link VisualQueryDetector}.
238      * <p>
239      * On streamed query is finished, clients should call
240      * {@link VisualQueryDetectionService#finishQuery()} to complete query streamed to
241      * {@link VisualQueryDetector}.
242      * <p>
243      * Before a call for {@link VisualQueryDetectionService#streamQuery(String)} is triggered,
244      * {@link VisualQueryDetectionService#gainedAttention()} MUST be called to enable the streaming
245      * of query. A query streaming is also expected to be finished by calling either
246      * {@link VisualQueryDetectionService#finishQuery()} or
247      * {@link VisualQueryDetectionService#rejectQuery()} before a new query should start streaming.
248      * When the service enters the state where query streaming should be disabled,
249      * {@link VisualQueryDetectionService#lostAttention()} MUST be called to block unnecessary
250      * streaming.
251      */
onStartDetection()252     public void onStartDetection() {
253         throw new UnsupportedOperationException();
254     }
255 
256     /**
257      * Called when the {@link VoiceInteractionService}
258      * {@link HotwordDetector#stopRecognition()} requests that recognition be stopped.
259      */
onStopDetection()260     public void onStopDetection() {
261     }
262 
263     /**
264      * Informs the system that the user attention is gained so queries can be streamed.
265      */
gainedAttention()266     public final void gainedAttention() {
267         try {
268             mRemoteCallback.onAttentionGained();
269         } catch (RemoteException e) {
270             throw e.rethrowFromSystemServer();
271         }
272     }
273 
274     /**
275      * Informs the system that the user attention is lost to stop streaming.
276      */
lostAttention()277     public final void lostAttention() {
278         try {
279             mRemoteCallback.onAttentionLost();
280         } catch (RemoteException e) {
281             throw e.rethrowFromSystemServer();
282         }
283     }
284 
285     /**
286      * Informs the {@link VisualQueryDetector} with the text content being captured about the
287      * query from the audio source. {@code partialQuery} is provided to the
288      * {@link VisualQueryDetector}. This method is expected to be only triggered if
289      * {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into the
290      * attention gained state.
291      *
292      * @param partialQuery Partially detected query in string.
293      * @throws IllegalStateException if method called without attention gained.
294      */
streamQuery(@onNull String partialQuery)295     public final void streamQuery(@NonNull String partialQuery) throws IllegalStateException {
296         Objects.requireNonNull(partialQuery);
297         try {
298             mRemoteCallback.onQueryDetected(partialQuery);
299         } catch (RemoteException e) {
300             throw new IllegalStateException("#streamQuery must be only be triggered after "
301                     + "calling #gainedAttention to be in the attention gained state.");
302         }
303     }
304 
305     /**
306      * Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has
307      * been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if
308      * {@link VisualQueryDetectionService#streamQuery(String)} is called to put the service into
309      * the query streaming state.
310      *
311      * @throws IllegalStateException if method called without query streamed.
312      */
rejectQuery()313     public final void rejectQuery() throws IllegalStateException {
314         try {
315             mRemoteCallback.onQueryRejected();
316         } catch (RemoteException e) {
317             throw new IllegalStateException("#rejectQuery must be only be triggered after "
318                     + "calling #streamQuery to be in the query streaming state.");
319         }
320     }
321 
322     /**
323      * Informs {@link VisualQueryDetector} with the metadata to complete the streamed partial
324      * query that has been sent to {@link VisualQueryDetector}. This method is expected to be
325      * only triggered if {@link VisualQueryDetectionService#streamQuery(String)} is called to put
326      * the service into the query streaming state.
327      *
328      * @throws IllegalStateException if method called without query streamed.
329      */
finishQuery()330     public final void finishQuery() throws IllegalStateException {
331         try {
332             mRemoteCallback.onQueryFinished();
333         } catch (RemoteException e) {
334             throw new IllegalStateException("#finishQuery must be only be triggered after "
335                     + "calling #streamQuery to be in the query streaming state.");
336         }
337     }
338 
339     /**
340      * Overrides {@link Context#openFileInput} to read files with the given file names under the
341      * internal app storage of the {@link VoiceInteractionService}, i.e., only files stored in
342      * {@link Context#getFilesDir()} can be opened.
343      */
344     @Override
openFileInput(@onNull String filename)345     public @Nullable FileInputStream openFileInput(@NonNull String filename) throws
346             FileNotFoundException {
347         try {
348             AndroidFuture<ParcelFileDescriptor> future = new AndroidFuture<>();
349             mDetectorSessionStorageService.openFile(filename, future);
350             ParcelFileDescriptor pfd = future.get();
351             return new FileInputStream(pfd.getFileDescriptor());
352         } catch (RemoteException | ExecutionException | InterruptedException e) {
353             Log.w(TAG, "Cannot open file due to remote service failure");
354             throw new FileNotFoundException(e.getMessage());
355         }
356     }
357 
358 }
359