1 /**
2  * Copyright (c) 2020, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.cameraextensions;
17 
18 import android.app.Service;
19 import android.content.Context;
20 import android.content.Intent;
21 import android.content.pm.PackageManager;
22 import android.graphics.GraphicBuffer;
23 import android.graphics.Rect;
24 import android.hardware.HardwareBuffer;
25 import android.hardware.camera2.CameraAccessException;
26 import android.hardware.camera2.CameraCharacteristics;
27 import android.hardware.camera2.CameraExtensionCharacteristics;
28 import android.hardware.camera2.CameraManager;
29 import android.hardware.camera2.CaptureRequest;
30 import android.hardware.camera2.CaptureResult;
31 import android.hardware.camera2.TotalCaptureResult;
32 import android.hardware.camera2.extension.CameraOutputConfig;
33 import android.hardware.camera2.extension.CameraSessionConfig;
34 import android.hardware.camera2.extension.CaptureBundle;
35 import android.hardware.camera2.extension.CaptureFailure;
36 import android.hardware.camera2.extension.CaptureStageImpl;
37 import android.hardware.camera2.extension.IAdvancedExtenderImpl;
38 import android.hardware.camera2.extension.ICameraExtensionsProxyService;
39 import android.hardware.camera2.extension.ICaptureCallback;
40 import android.hardware.camera2.extension.ICaptureProcessorImpl;
41 import android.hardware.camera2.extension.IImageCaptureExtenderImpl;
42 import android.hardware.camera2.extension.IImageProcessorImpl;
43 import android.hardware.camera2.extension.IInitializeSessionCallback;
44 import android.hardware.camera2.extension.IPreviewExtenderImpl;
45 import android.hardware.camera2.extension.IPreviewImageProcessorImpl;
46 import android.hardware.camera2.extension.IProcessResultImpl;
47 import android.hardware.camera2.extension.IRequestCallback;
48 import android.hardware.camera2.extension.IRequestProcessorImpl;
49 import android.hardware.camera2.extension.IRequestUpdateProcessorImpl;
50 import android.hardware.camera2.extension.ISessionProcessorImpl;
51 import android.hardware.camera2.extension.LatencyPair;
52 import android.hardware.camera2.extension.LatencyRange;
53 import android.hardware.camera2.extension.OutputConfigId;
54 import android.hardware.camera2.extension.OutputSurface;
55 import android.hardware.camera2.extension.ParcelCaptureResult;
56 import android.hardware.camera2.extension.ParcelImage;
57 import android.hardware.camera2.extension.ParcelTotalCaptureResult;
58 import android.hardware.camera2.extension.Request;
59 import android.hardware.camera2.extension.SizeList;
60 import android.hardware.camera2.impl.CameraMetadataNative;
61 import android.hardware.camera2.impl.PhysicalCaptureResultInfo;
62 import android.media.Image;
63 import android.media.ImageReader;
64 import android.os.Binder;
65 import android.os.ConditionVariable;
66 import android.os.Handler;
67 import android.os.HandlerExecutor;
68 import android.os.HandlerThread;
69 import android.os.IBinder;
70 import android.os.RemoteException;
71 import android.util.ArraySet;
72 import android.util.Log;
73 import android.util.Pair;
74 import android.util.Range;
75 import android.util.Size;
76 import android.view.Surface;
77 
78 import androidx.annotation.NonNull;
79 import androidx.camera.extensions.impl.AutoImageCaptureExtenderImpl;
80 import androidx.camera.extensions.impl.AutoPreviewExtenderImpl;
81 import androidx.camera.extensions.impl.BeautyImageCaptureExtenderImpl;
82 import androidx.camera.extensions.impl.BeautyPreviewExtenderImpl;
83 import androidx.camera.extensions.impl.BokehImageCaptureExtenderImpl;
84 import androidx.camera.extensions.impl.BokehPreviewExtenderImpl;
85 import androidx.camera.extensions.impl.CaptureProcessorImpl;
86 import androidx.camera.extensions.impl.ExtensionVersionImpl;
87 import androidx.camera.extensions.impl.HdrImageCaptureExtenderImpl;
88 import androidx.camera.extensions.impl.HdrPreviewExtenderImpl;
89 import androidx.camera.extensions.impl.ImageCaptureExtenderImpl;
90 import androidx.camera.extensions.impl.InitializerImpl;
91 import androidx.camera.extensions.impl.NightImageCaptureExtenderImpl;
92 import androidx.camera.extensions.impl.NightPreviewExtenderImpl;
93 import androidx.camera.extensions.impl.PreviewExtenderImpl;
94 import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType;
95 import androidx.camera.extensions.impl.PreviewImageProcessorImpl;
96 import androidx.camera.extensions.impl.ProcessResultImpl;
97 import androidx.camera.extensions.impl.RequestUpdateProcessorImpl;
98 import androidx.camera.extensions.impl.advanced.AdvancedExtenderImpl;
99 import androidx.camera.extensions.impl.advanced.AutoAdvancedExtenderImpl;
100 import androidx.camera.extensions.impl.advanced.BeautyAdvancedExtenderImpl;
101 import androidx.camera.extensions.impl.advanced.BokehAdvancedExtenderImpl;
102 import androidx.camera.extensions.impl.advanced.Camera2OutputConfigImpl;
103 import androidx.camera.extensions.impl.advanced.Camera2SessionConfigImpl;
104 import androidx.camera.extensions.impl.advanced.HdrAdvancedExtenderImpl;
105 import androidx.camera.extensions.impl.advanced.ImageProcessorImpl;
106 import androidx.camera.extensions.impl.advanced.ImageReaderOutputConfigImpl;
107 import androidx.camera.extensions.impl.advanced.MultiResolutionImageReaderOutputConfigImpl;
108 import androidx.camera.extensions.impl.advanced.NightAdvancedExtenderImpl;
109 import androidx.camera.extensions.impl.advanced.OutputSurfaceConfigurationImpl;
110 import androidx.camera.extensions.impl.advanced.OutputSurfaceImpl;
111 import androidx.camera.extensions.impl.advanced.RequestProcessorImpl;
112 import androidx.camera.extensions.impl.advanced.SessionProcessorImpl;
113 import androidx.camera.extensions.impl.advanced.SurfaceOutputConfigImpl;
114 
115 import java.io.IOException;
116 import java.util.ArrayList;
117 import java.util.HashMap;
118 import java.util.List;
119 import java.util.Map;
120 import java.util.concurrent.Future;
121 import java.util.concurrent.TimeUnit;
122 import java.util.concurrent.TimeoutException;
123 
124 public class CameraExtensionsProxyService extends Service {
125     private static final String TAG = "CameraExtensionsProxyService";
126 
127     private static final String CAMERA_EXTENSION_VERSION_NAME =
128             "androidx.camera.extensions.impl.ExtensionVersionImpl";
129     private static final String LATEST_VERSION = "1.4.0";
130     // No support for the init sequence
131     private static final String NON_INIT_VERSION_PREFIX = "1.0";
132     // Support advanced API and latency queries
133     private static final String ADVANCED_VERSION_PREFIX = "1.2";
134     // Support for the capture request & result APIs
135     private static final String RESULTS_VERSION_PREFIX = "1.3";
136     // Support for various latency improvements
137     private static final String LATENCY_VERSION_PREFIX = "1.4";
138     private static final String[] ADVANCED_VERSION_PREFIXES = {LATENCY_VERSION_PREFIX,
139             ADVANCED_VERSION_PREFIX, RESULTS_VERSION_PREFIX };
140     private static final String[] SUPPORTED_VERSION_PREFIXES = {LATENCY_VERSION_PREFIX,
141             RESULTS_VERSION_PREFIX, ADVANCED_VERSION_PREFIX, "1.1", NON_INIT_VERSION_PREFIX};
142     private static final boolean EXTENSIONS_PRESENT = checkForExtensions();
143     private static final String EXTENSIONS_VERSION = EXTENSIONS_PRESENT ?
144             (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION) : null;
145     private static final boolean ESTIMATED_LATENCY_API_SUPPORTED = checkForLatencyAPI();
146     private static final boolean LATENCY_IMPROVEMENTS_SUPPORTED = EXTENSIONS_PRESENT &&
147             (EXTENSIONS_VERSION.startsWith(LATENCY_VERSION_PREFIX));
148     private static final boolean ADVANCED_API_SUPPORTED = checkForAdvancedAPI();
149     private static final boolean INIT_API_SUPPORTED = EXTENSIONS_PRESENT &&
150             (!EXTENSIONS_VERSION.startsWith(NON_INIT_VERSION_PREFIX));
151     private static final boolean RESULT_API_SUPPORTED = EXTENSIONS_PRESENT &&
152             (EXTENSIONS_VERSION.startsWith(RESULTS_VERSION_PREFIX) ||
153             EXTENSIONS_VERSION.startsWith(LATENCY_VERSION_PREFIX));
154 
155     private HashMap<String, Long> mMetadataVendorIdMap = new HashMap<>();
156     private CameraManager mCameraManager;
157 
checkForLatencyAPI()158     private static boolean checkForLatencyAPI() {
159         if (!EXTENSIONS_PRESENT) {
160             return false;
161         }
162 
163         for (String advancedVersions : ADVANCED_VERSION_PREFIXES) {
164             if (EXTENSIONS_VERSION.startsWith(advancedVersions)) {
165                 return true;
166             }
167         }
168 
169         return false;
170     }
171 
checkForAdvancedAPI()172     private static boolean checkForAdvancedAPI() {
173         if (!checkForLatencyAPI()) {
174             return false;
175         }
176 
177         try {
178             return (new ExtensionVersionImpl()).isAdvancedExtenderImplemented();
179         } catch (NoSuchMethodError e) {
180             // This could happen in case device specific extension implementations are using
181             // an older extension API but incorrectly set the extension version.
182         }
183 
184         return false;
185     }
186 
checkForExtensions()187     private static boolean checkForExtensions() {
188         try {
189             Class.forName(CAMERA_EXTENSION_VERSION_NAME);
190         } catch (ClassNotFoundException e) {
191             return false;
192         }
193 
194         String extensionVersion = (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION);
195         for (String supportedVersion : SUPPORTED_VERSION_PREFIXES) {
196             if (extensionVersion.startsWith(supportedVersion)) {
197                 return true;
198             }
199         }
200 
201         return false;
202     }
203 
204     /**
205      * A per-process global camera extension manager instance, to track and
206      * initialize/release extensions depending on client activity.
207      */
208     private static final class CameraExtensionManagerGlobal implements IBinder.DeathRecipient {
209         private static final String TAG = "CameraExtensionManagerGlobal";
210         private final int EXTENSION_DELAY_MS = 1000;
211 
212         private final Handler mHandler;
213         private final HandlerThread mHandlerThread;
214         private final Object mLock = new Object();
215 
216         private ArraySet<IBinder> mActiveClients = new ArraySet<>();
217         private HashMap<IBinder, ArraySet<IBinder.DeathRecipient>> mClientDeathRecipient =
218                 new HashMap<>();
219         private IInitializeSessionCallback mInitializeCb = null;
220 
221         // Singleton instance
222         private static final CameraExtensionManagerGlobal GLOBAL_CAMERA_MANAGER =
223                 new CameraExtensionManagerGlobal();
224 
225         // Singleton, don't allow construction
CameraExtensionManagerGlobal()226         private CameraExtensionManagerGlobal() {
227             mHandlerThread = new HandlerThread(TAG);
228             mHandlerThread.start();
229             mHandler = new Handler(mHandlerThread.getLooper());
230         }
231 
232         private final static class InitializeHandler
233                 implements InitializerImpl.OnExtensionsInitializedCallback {
234             private final InitializerFuture mStatusFuture;
235 
InitializeHandler(InitializerFuture statusFuture)236             public InitializeHandler(InitializerFuture statusFuture) {
237                 mStatusFuture = statusFuture;
238             }
239 
240             @Override
onSuccess()241             public void onSuccess() {
242                 mStatusFuture.setStatus(true);
243             }
244 
245             @Override
onFailure(int error)246             public void onFailure(int error) {
247                 mStatusFuture.setStatus(false);
248             }
249         }
250 
251         private final static class ReleaseHandler
252                 implements InitializerImpl.OnExtensionsDeinitializedCallback {
253             private final InitializerFuture mStatusFuture;
254 
ReleaseHandler(InitializerFuture statusFuture)255             public ReleaseHandler(InitializerFuture statusFuture) {
256                 mStatusFuture = statusFuture;
257             }
258 
onSuccess()259             @Override public void onSuccess() {
260                 mStatusFuture.setStatus(true);
261             }
262 
263             @Override
onFailure(int i)264             public void onFailure(int i) {
265                 mStatusFuture.setStatus(false);
266             }
267         }
268 
269         private static class InitializerFuture implements Future<Boolean> {
270             private volatile Boolean mStatus;
271             ConditionVariable mCondVar = new ConditionVariable(/*opened*/false);
272 
setStatus(boolean status)273             public void setStatus(boolean status) {
274                 mStatus = status;
275                 mCondVar.open();
276             }
277 
278             @Override
cancel(boolean mayInterruptIfRunning)279             public boolean cancel(boolean mayInterruptIfRunning) {
280                 return false; // don't allow canceling this task
281             }
282 
283             @Override
isCancelled()284             public boolean isCancelled() {
285                 return false; // can never cancel this task
286             }
287 
288             @Override
isDone()289             public boolean isDone() {
290                 return mStatus != null;
291             }
292 
293             @Override
get()294             public Boolean get() {
295                 mCondVar.block();
296                 return mStatus;
297             }
298 
299             @Override
get(long timeout, TimeUnit unit)300             public Boolean get(long timeout, TimeUnit unit) throws TimeoutException {
301                 long timeoutMs = unit.convert(timeout, TimeUnit.MILLISECONDS);
302                 if (!mCondVar.block(timeoutMs)) {
303                     throw new TimeoutException(
304                             "Failed to receive status after " + timeout + " " + unit);
305                 }
306 
307                 if (mStatus == null) {
308                     throw new AssertionError();
309                 }
310                 return mStatus;
311             }
312 
313         }
314 
get()315         public static CameraExtensionManagerGlobal get() {
316             return GLOBAL_CAMERA_MANAGER;
317         }
318 
registerClient(Context ctx, IBinder token)319         public boolean registerClient(Context ctx, IBinder token) {
320             synchronized (mLock) {
321                 if (mActiveClients.contains(token)) {
322                     Log.e(TAG, "Failed to register existing client!");
323                     return false;
324                 }
325 
326                 try {
327                     token.linkToDeath(this, 0);
328                 } catch (RemoteException e) {
329                     Log.e(TAG, "Failed to link to binder token!");
330                     return false;
331                 }
332 
333                 if (INIT_API_SUPPORTED) {
334                     if (mActiveClients.isEmpty()) {
335                         InitializerFuture status = new InitializerFuture();
336                         InitializerImpl.init(LATEST_VERSION, ctx, new InitializeHandler(status),
337                                 new HandlerExecutor(mHandler));
338                         boolean initSuccess;
339                         try {
340                             initSuccess = status.get(EXTENSION_DELAY_MS,
341                                     TimeUnit.MILLISECONDS);
342                         } catch (TimeoutException e) {
343                             Log.e(TAG, "Timed out while initializing camera extensions!");
344                             return false;
345                         }
346                         if (!initSuccess) {
347                             Log.e(TAG, "Failed while initializing camera extensions!");
348                             return false;
349                         }
350                     }
351                 }
352 
353                 mActiveClients.add(token);
354                 mClientDeathRecipient.put(token, new ArraySet<>());
355 
356                 return true;
357             }
358         }
359 
unregisterClient(IBinder token)360         public void unregisterClient(IBinder token) {
361             synchronized (mLock) {
362                 if (mActiveClients.remove(token)) {
363                     token.unlinkToDeath(this, 0);
364                     mClientDeathRecipient.remove(token);
365                     if (mActiveClients.isEmpty() && INIT_API_SUPPORTED) {
366                         InitializerFuture status = new InitializerFuture();
367                         InitializerImpl.deinit(new ReleaseHandler(status),
368                                 new HandlerExecutor(mHandler));
369                         boolean releaseSuccess;
370                         try {
371                             releaseSuccess = status.get(EXTENSION_DELAY_MS, TimeUnit.MILLISECONDS);
372                         } catch (TimeoutException e) {
373                             Log.e(TAG, "Timed out while releasing camera extensions!");
374                             return;
375                         }
376                         if (!releaseSuccess) {
377                             Log.e(TAG, "Failed while releasing camera extensions!");
378                         }
379                     }
380                 }
381             }
382         }
383 
384         @Override
binderDied()385         public void binderDied() {
386             // Do nothing, handled below
387         }
388 
389         @Override
binderDied(@onNull IBinder who)390         public void binderDied(@NonNull IBinder who) {
391             synchronized (mLock) {
392                 if (mClientDeathRecipient.containsKey(who)) {
393                     mClientDeathRecipient.get(who).stream().forEach(
394                             recipient -> recipient.binderDied(who));
395                 }
396                 unregisterClient(who);
397             }
398         }
399 
registerDeathRecipient(IBinder token, IBinder.DeathRecipient recipient)400         public void registerDeathRecipient(IBinder token, IBinder.DeathRecipient recipient) {
401             synchronized (mLock) {
402                 if (mClientDeathRecipient.containsKey(token)) {
403                     ArraySet<IBinder.DeathRecipient> recipients = mClientDeathRecipient.get(token);
404                     recipients.add(recipient);
405                 }
406             }
407         }
408 
unregisterDeathRecipient(IBinder token, IBinder.DeathRecipient recipient)409         public void unregisterDeathRecipient(IBinder token, IBinder.DeathRecipient recipient) {
410             synchronized (mLock) {
411                 if (mClientDeathRecipient.containsKey(token)) {
412                     ArraySet<IBinder.DeathRecipient> recipients = mClientDeathRecipient.get(token);
413                     recipients.remove(recipient);
414                 }
415             }
416         }
417 
418         private IBinder.DeathRecipient mDeathRecipient = new IBinder.DeathRecipient() {
419             @Override
420             public void binderDied() {
421                 synchronized (mLock) {
422                     mInitializeCb = null;
423                 }
424             }
425         };
426 
initializeSession(IInitializeSessionCallback cb)427         public boolean initializeSession(IInitializeSessionCallback cb) {
428             synchronized (mLock) {
429                 if (mInitializeCb == null) {
430                     mInitializeCb = cb;
431                     try {
432                         mInitializeCb.asBinder().linkToDeath(mDeathRecipient, 0);
433                     } catch (RemoteException e) {
434                         e.printStackTrace();
435                     }
436                 } else {
437                     return false;
438                 }
439             }
440             return true;
441         }
442 
releaseSession()443         public void releaseSession() {
444             synchronized (mLock) {
445                 if (mInitializeCb != null) {
446                     mInitializeCb.asBinder().unlinkToDeath(mDeathRecipient, 0);
447                     mInitializeCb = null;
448                 }
449             }
450         }
451     }
452 
453     /**
454      * @hide
455      */
registerClient(Context ctx, IBinder token)456     private static boolean registerClient(Context ctx, IBinder token) {
457         if (!EXTENSIONS_PRESENT) {
458             return false;
459         }
460         return CameraExtensionManagerGlobal.get().registerClient(ctx, token);
461     }
462 
463     /**
464      * @hide
465      */
unregisterClient(IBinder token)466     public static void unregisterClient(IBinder token) {
467         if (!EXTENSIONS_PRESENT) {
468             return;
469         }
470         CameraExtensionManagerGlobal.get().unregisterClient(token);
471     }
472 
473     /**
474      * @hide
475      */
registerDeathRecipient(IBinder token, IBinder.DeathRecipient recipient)476     private static void registerDeathRecipient(IBinder token, IBinder.DeathRecipient recipient) {
477         CameraExtensionManagerGlobal.get().registerDeathRecipient(token, recipient);
478     }
479 
480     /**
481      * @hide
482      */
unregisterDeathRecipient(IBinder token, IBinder.DeathRecipient recipient)483     private static void unregisterDeathRecipient(IBinder token, IBinder.DeathRecipient recipient) {
484         CameraExtensionManagerGlobal.get().unregisterDeathRecipient(token, recipient);
485     }
486 
487     /**
488      * @hide
489      */
initializeSession(IInitializeSessionCallback cb)490     public static boolean initializeSession(IInitializeSessionCallback cb) {
491         if (!EXTENSIONS_PRESENT) {
492             return false;
493         }
494         return CameraExtensionManagerGlobal.get().initializeSession(cb);
495     }
496 
497     /**
498      * @hide
499      */
releaseSession()500     public static void releaseSession() {
501         if (!EXTENSIONS_PRESENT) {
502             return;
503         }
504         CameraExtensionManagerGlobal.get().releaseSession();
505     }
506 
507     /**
508      * @hide
509      */
initializeExtension( int extensionType)510     public static Pair<PreviewExtenderImpl, ImageCaptureExtenderImpl> initializeExtension(
511             int extensionType) {
512         switch (extensionType) {
513             case CameraExtensionCharacteristics.EXTENSION_AUTOMATIC:
514                 return new Pair<>(new AutoPreviewExtenderImpl(),
515                         new AutoImageCaptureExtenderImpl());
516             case CameraExtensionCharacteristics.EXTENSION_FACE_RETOUCH:
517                 return new Pair<>(new BeautyPreviewExtenderImpl(),
518                         new BeautyImageCaptureExtenderImpl());
519             case CameraExtensionCharacteristics.EXTENSION_BOKEH:
520                 return new Pair<>(new BokehPreviewExtenderImpl(),
521                         new BokehImageCaptureExtenderImpl());
522             case CameraExtensionCharacteristics.EXTENSION_HDR:
523                 return new Pair<>(new HdrPreviewExtenderImpl(), new HdrImageCaptureExtenderImpl());
524             case CameraExtensionCharacteristics.EXTENSION_NIGHT:
525                 return new Pair<>(new NightPreviewExtenderImpl(),
526                         new NightImageCaptureExtenderImpl());
527             default:
528                 throw new IllegalArgumentException("Unknown extension: " + extensionType);
529         }
530     }
531 
532     /**
533      * @hide
534      */
initializeAdvancedExtensionImpl(int extensionType)535     public static AdvancedExtenderImpl initializeAdvancedExtensionImpl(int extensionType) {
536         switch (extensionType) {
537             case CameraExtensionCharacteristics.EXTENSION_AUTOMATIC:
538                 return new AutoAdvancedExtenderImpl();
539             case CameraExtensionCharacteristics.EXTENSION_FACE_RETOUCH:
540                 return new BeautyAdvancedExtenderImpl();
541             case CameraExtensionCharacteristics.EXTENSION_BOKEH:
542                 return new BokehAdvancedExtenderImpl();
543             case CameraExtensionCharacteristics.EXTENSION_HDR:
544                 return new HdrAdvancedExtenderImpl();
545             case CameraExtensionCharacteristics.EXTENSION_NIGHT:
546                 return new NightAdvancedExtenderImpl();
547             default:
548                 throw new IllegalArgumentException("Unknown extension: " + extensionType);
549         }
550     }
551 
552     @Override
onCreate()553     public void onCreate() {
554         super.onCreate();
555         // This will setup the camera vendor tag descriptor in the service process
556         // along with all camera characteristics.
557         try {
558             mCameraManager = getSystemService(CameraManager.class);
559 
560             String [] cameraIds = mCameraManager.getCameraIdListNoLazy();
561             if (cameraIds != null) {
562                 for (String cameraId : cameraIds) {
563                     CameraCharacteristics chars = mCameraManager.getCameraCharacteristics(cameraId);
564                     Object thisClass = CameraCharacteristics.Key.class;
565                     Class<CameraCharacteristics.Key<?>> keyClass =
566                             (Class<CameraCharacteristics.Key<?>>)thisClass;
567                     ArrayList<CameraCharacteristics.Key<?>> vendorKeys =
568                             chars.getNativeMetadata().getAllVendorKeys(keyClass);
569                     if ((vendorKeys != null) && !vendorKeys.isEmpty()) {
570                         mMetadataVendorIdMap.put(cameraId, vendorKeys.get(0).getVendorId());
571                     }
572                 }
573             }
574         } catch (CameraAccessException e) {
575             Log.e(TAG, "Failed to query camera characteristics!");
576         }
577     }
578 
579     @Override
onDestroy()580     public void onDestroy() {
581         super.onDestroy();
582     }
583 
584 
585     @Override
onBind(Intent intent)586     public IBinder onBind(Intent intent) {
587         return new CameraExtensionsProxyServiceStub();
588     }
589 
initializeParcelable( List<Pair<Integer, android.util.Size[]>> sizes)590     private static List<SizeList> initializeParcelable(
591             List<Pair<Integer, android.util.Size[]>> sizes) {
592         if (sizes == null) {
593             return null;
594         }
595         ArrayList<SizeList> ret = new ArrayList<>(sizes.size());
596         for (Pair<Integer, Size[]> entry : sizes) {
597             SizeList sizeList = new SizeList();
598             sizeList.format = entry.first;
599             sizeList.sizes = new ArrayList<>();
600             for (android.util.Size size : entry.second) {
601                 android.hardware.camera2.extension.Size sz =
602                         new android.hardware.camera2.extension.Size();
603                 sz.width = size.getWidth();
604                 sz.height = size.getHeight();
605                 sizeList.sizes.add(sz);
606             }
607 
608             if (!sizeList.sizes.isEmpty()) {
609                 ret.add(sizeList);
610             }
611         }
612 
613         return ret;
614     }
615 
getCharacteristicsMap( Map<String, CameraMetadataNative> charsMap)616     private static Map<String, CameraCharacteristics> getCharacteristicsMap(
617             Map<String, CameraMetadataNative> charsMap) {
618         HashMap<String, CameraCharacteristics> ret = new HashMap<>();
619         for (Map.Entry<String, CameraMetadataNative> entry : charsMap.entrySet()) {
620             ret.put(entry.getKey(), new CameraCharacteristics(entry.getValue()));
621         }
622         return ret;
623     }
624 
initializeParcelable( Map<Integer, List<android.util.Size>> sizes)625     private static List<SizeList> initializeParcelable(
626             Map<Integer, List<android.util.Size>> sizes) {
627         if (sizes == null) {
628             return null;
629         }
630         ArrayList<SizeList> ret = new ArrayList<>(sizes.size());
631         for (Map.Entry<Integer, List<android.util.Size>> entry : sizes.entrySet()) {
632             SizeList sizeList = new SizeList();
633             sizeList.format = entry.getKey();
634             sizeList.sizes = new ArrayList<>();
635             for (android.util.Size size : entry.getValue()) {
636                 android.hardware.camera2.extension.Size sz =
637                         new android.hardware.camera2.extension.Size();
638                 sz.width = size.getWidth();
639                 sz.height = size.getHeight();
640                 sizeList.sizes.add(sz);
641             }
642             ret.add(sizeList);
643         }
644 
645         return ret;
646     }
647 
initializeParcelableMetadata( List<Pair<CaptureRequest.Key, Object>> paramList, String cameraId)648     private CameraMetadataNative initializeParcelableMetadata(
649             List<Pair<CaptureRequest.Key, Object>> paramList, String cameraId) {
650         if (paramList == null) {
651             return null;
652         }
653 
654         CameraMetadataNative ret = new CameraMetadataNative();
655         if (mMetadataVendorIdMap.containsKey(cameraId)) {
656             ret.setVendorId(mMetadataVendorIdMap.get(cameraId));
657         }
658         for (Pair<CaptureRequest.Key, Object> param : paramList) {
659             ret.set(param.first, param.second);
660         }
661 
662         return ret;
663     }
664 
initializeParcelableMetadata( Map<CaptureRequest.Key<?>, Object> paramMap, String cameraId)665     private CameraMetadataNative initializeParcelableMetadata(
666             Map<CaptureRequest.Key<?>, Object> paramMap, String cameraId) {
667         if (paramMap == null) {
668             return null;
669         }
670 
671         CameraMetadataNative ret = new CameraMetadataNative();
672         if (mMetadataVendorIdMap.containsKey(cameraId)) {
673             ret.setVendorId(mMetadataVendorIdMap.get(cameraId));
674         }
675         for (Map.Entry<CaptureRequest.Key<?>, Object> param : paramMap.entrySet()) {
676             ret.set(((CaptureRequest.Key) param.getKey()), param.getValue());
677         }
678 
679         return ret;
680     }
681 
initializeParcelable( androidx.camera.extensions.impl.CaptureStageImpl captureStage, String cameraId)682     private android.hardware.camera2.extension.CaptureStageImpl initializeParcelable(
683             androidx.camera.extensions.impl.CaptureStageImpl captureStage, String cameraId) {
684         if (captureStage == null) {
685             return null;
686         }
687 
688         android.hardware.camera2.extension.CaptureStageImpl ret =
689                 new android.hardware.camera2.extension.CaptureStageImpl();
690         ret.id = captureStage.getId();
691         ret.parameters = initializeParcelableMetadata(captureStage.getParameters(), cameraId);
692 
693         return ret;
694     }
695 
initializeParcelable(RequestProcessorImpl.Request request, int requestId, String cameraId)696     private Request initializeParcelable(RequestProcessorImpl.Request request, int requestId,
697             String cameraId) {
698         Request ret = new Request();
699         ret.targetOutputConfigIds = new ArrayList<>();
700         for (int id : request.getTargetOutputConfigIds()) {
701             OutputConfigId configId = new OutputConfigId();
702             configId.id = id;
703             ret.targetOutputConfigIds.add(configId);
704         }
705         ret.templateId = request.getTemplateId();
706         ret.parameters = initializeParcelableMetadata(request.getParameters(), cameraId);
707         ret.requestId = requestId;
708         return ret;
709     }
710 
711     private class CameraExtensionsProxyServiceStub extends ICameraExtensionsProxyService.Stub {
712         @Override
registerClient(IBinder token)713         public boolean registerClient(IBinder token) {
714             return CameraExtensionsProxyService.registerClient(CameraExtensionsProxyService.this,
715                     token);
716         }
717 
718         @Override
unregisterClient(IBinder token)719         public void unregisterClient(IBinder token) {
720             CameraExtensionsProxyService.unregisterClient(token);
721         }
722 
checkCameraPermission()723         private boolean checkCameraPermission() {
724             int allowed = CameraExtensionsProxyService.this.checkPermission(
725                     android.Manifest.permission.CAMERA, Binder.getCallingPid(),
726                     Binder.getCallingUid());
727             return (PackageManager.PERMISSION_GRANTED == allowed);
728         }
729 
730         @Override
initializeSession(IInitializeSessionCallback cb)731         public void initializeSession(IInitializeSessionCallback cb) {
732             try {
733                 if (!checkCameraPermission()) {
734                     Log.i(TAG, "Camera permission required for initializing capture session");
735                     cb.onFailure();
736                     return;
737                 }
738 
739                 if (CameraExtensionsProxyService.initializeSession(cb)) {
740                     cb.onSuccess();
741                 } else {
742                     cb.onFailure();
743                 }
744             } catch (RemoteException e) {
745                 Log.e(TAG, "Client doesn't respond!");
746             }
747         }
748 
749         @Override
releaseSession()750         public void releaseSession() {
751             if (checkCameraPermission()) {
752                 CameraExtensionsProxyService.releaseSession();
753             }
754         }
755 
756         @Override
advancedExtensionsSupported()757         public boolean advancedExtensionsSupported() {
758             return ADVANCED_API_SUPPORTED;
759         }
760 
761         @Override
initializeAdvancedExtension(int extensionType)762         public IAdvancedExtenderImpl initializeAdvancedExtension(int extensionType) {
763             AdvancedExtenderImpl extension;
764             try {
765                 extension = initializeAdvancedExtensionImpl(extensionType);
766             } catch (IllegalArgumentException e) {
767                 return null;
768             }
769 
770             return new AdvancedExtenderImplStub(extension);
771         }
772 
773         @Override
initializePreviewExtension(int extensionType)774         public IPreviewExtenderImpl initializePreviewExtension(int extensionType) {
775             Pair<PreviewExtenderImpl, ImageCaptureExtenderImpl> extension;
776             try {
777                 extension = initializeExtension(extensionType);
778             } catch (IllegalArgumentException e) {
779                 return null;
780             }
781 
782             return new PreviewExtenderImplStub(extension.first);
783         }
784 
785         @Override
initializeImageExtension(int extensionType)786         public IImageCaptureExtenderImpl initializeImageExtension(int extensionType) {
787             Pair<PreviewExtenderImpl, ImageCaptureExtenderImpl> extension;
788             try {
789                 extension = initializeExtension(extensionType);
790             } catch (IllegalArgumentException e) {
791                 return null;
792             }
793 
794             return new ImageCaptureExtenderImplStub(extension.second);
795         }
796     }
797 
798     private class AdvancedExtenderImplStub extends IAdvancedExtenderImpl.Stub {
799         private final AdvancedExtenderImpl mAdvancedExtender;
800 
AdvancedExtenderImplStub(AdvancedExtenderImpl advancedExtender)801         public AdvancedExtenderImplStub(AdvancedExtenderImpl advancedExtender) {
802             mAdvancedExtender = advancedExtender;
803         }
804 
805         @Override
isExtensionAvailable(String cameraId, Map<String, CameraMetadataNative> charsMapNative)806         public boolean isExtensionAvailable(String cameraId,
807                 Map<String, CameraMetadataNative> charsMapNative) {
808             return mAdvancedExtender.isExtensionAvailable(cameraId,
809                     getCharacteristicsMap(charsMapNative));
810         }
811 
812         @Override
init(String cameraId, Map<String, CameraMetadataNative> charsMapNative)813         public void init(String cameraId, Map<String, CameraMetadataNative> charsMapNative) {
814             mAdvancedExtender.init(cameraId, getCharacteristicsMap(charsMapNative));
815         }
816 
817         @Override
getSupportedPostviewResolutions( android.hardware.camera2.extension.Size captureSize)818         public List<SizeList> getSupportedPostviewResolutions(
819                 android.hardware.camera2.extension.Size captureSize) {
820             Size sz = new Size(captureSize.width, captureSize.height);
821             Map<Integer, List<Size>> supportedSizesMap =
822                     mAdvancedExtender.getSupportedPostviewResolutions(sz);
823             if (supportedSizesMap != null) {
824                 return initializeParcelable(supportedSizesMap);
825             }
826 
827             return null;
828         }
829 
830         @Override
getSupportedPreviewOutputResolutions(String cameraId)831         public List<SizeList> getSupportedPreviewOutputResolutions(String cameraId) {
832             Map<Integer, List<Size>> supportedSizesMap =
833                     mAdvancedExtender.getSupportedPreviewOutputResolutions(cameraId);
834             if (supportedSizesMap != null) {
835                 return initializeParcelable(supportedSizesMap);
836             }
837 
838             return null;
839         }
840 
841         @Override
getSupportedCaptureOutputResolutions(String cameraId)842         public List<SizeList> getSupportedCaptureOutputResolutions(String cameraId) {
843             Map<Integer, List<Size>> supportedSizesMap =
844                     mAdvancedExtender.getSupportedCaptureOutputResolutions(cameraId);
845             if (supportedSizesMap != null) {
846                 return initializeParcelable(supportedSizesMap);
847             }
848 
849             return null;
850         }
851 
852         @Override
getEstimatedCaptureLatencyRange(String cameraId, android.hardware.camera2.extension.Size outputSize, int format)853         public LatencyRange getEstimatedCaptureLatencyRange(String cameraId,
854                 android.hardware.camera2.extension.Size outputSize, int format) {
855             Size sz = new Size(outputSize.width, outputSize.height);
856             Range<Long> latencyRange = mAdvancedExtender.getEstimatedCaptureLatencyRange(cameraId,
857                     sz, format);
858             if (latencyRange != null) {
859                 LatencyRange ret = new LatencyRange();
860                 ret.min = latencyRange.getLower();
861                 ret.max = latencyRange.getUpper();
862                 return ret;
863             }
864 
865             return null;
866         }
867 
868         @Override
getSessionProcessor()869         public ISessionProcessorImpl getSessionProcessor() {
870             return new SessionProcessorImplStub(mAdvancedExtender.createSessionProcessor());
871         }
872 
873         @Override
getAvailableCaptureRequestKeys(String cameraId)874         public CameraMetadataNative getAvailableCaptureRequestKeys(String cameraId) {
875             if (RESULT_API_SUPPORTED) {
876                 List<CaptureRequest.Key> supportedCaptureKeys =
877                         mAdvancedExtender.getAvailableCaptureRequestKeys();
878 
879                 if ((supportedCaptureKeys != null) && !supportedCaptureKeys.isEmpty()) {
880                     CameraMetadataNative ret = new CameraMetadataNative();
881                     long vendorId = mMetadataVendorIdMap.containsKey(cameraId) ?
882                             mMetadataVendorIdMap.get(cameraId) : Long.MAX_VALUE;
883                     ret.setVendorId(vendorId);
884                     int requestKeyTags [] = new int[supportedCaptureKeys.size()];
885                     int i = 0;
886                     for (CaptureRequest.Key key : supportedCaptureKeys) {
887                         requestKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId);
888                     }
889                     ret.set(CameraCharacteristics.REQUEST_AVAILABLE_REQUEST_KEYS, requestKeyTags);
890 
891                     return ret;
892                 }
893             }
894 
895             return null;
896         }
897 
898         @Override
getAvailableCaptureResultKeys(String cameraId)899         public CameraMetadataNative getAvailableCaptureResultKeys(String cameraId) {
900             if (RESULT_API_SUPPORTED) {
901                 List<CaptureResult.Key> supportedResultKeys =
902                         mAdvancedExtender.getAvailableCaptureResultKeys();
903 
904                 if ((supportedResultKeys != null) && !supportedResultKeys.isEmpty()) {
905                     CameraMetadataNative ret = new CameraMetadataNative();
906                     long vendorId = mMetadataVendorIdMap.containsKey(cameraId) ?
907                             mMetadataVendorIdMap.get(cameraId) : Long.MAX_VALUE;
908                     ret.setVendorId(vendorId);
909                     int resultKeyTags [] = new int[supportedResultKeys.size()];
910                     int i = 0;
911                     for (CaptureResult.Key key : supportedResultKeys) {
912                         resultKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId);
913                     }
914                     ret.set(CameraCharacteristics.REQUEST_AVAILABLE_RESULT_KEYS, resultKeyTags);
915 
916                     return ret;
917                 }
918             }
919 
920             return null;
921         }
922 
923         @Override
isCaptureProcessProgressAvailable()924         public boolean isCaptureProcessProgressAvailable() {
925             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
926                 return mAdvancedExtender.isCaptureProcessProgressAvailable();
927             }
928 
929             return false;
930         }
931 
932         @Override
isPostviewAvailable()933         public boolean isPostviewAvailable() {
934             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
935                 return mAdvancedExtender.isPostviewAvailable();
936             }
937 
938             return false;
939         }
940     }
941 
942     private class CaptureCallbackStub implements SessionProcessorImpl.CaptureCallback {
943         private final ICaptureCallback mCaptureCallback;
944         private final String mCameraId;
945 
CaptureCallbackStub(ICaptureCallback captureCallback, String cameraId)946         private CaptureCallbackStub(ICaptureCallback captureCallback, String cameraId) {
947             mCaptureCallback = captureCallback;
948             mCameraId = cameraId;
949         }
950 
951         @Override
onCaptureStarted(int captureSequenceId, long timestamp)952         public void onCaptureStarted(int captureSequenceId, long timestamp) {
953             if (mCaptureCallback != null) {
954                 try {
955                     mCaptureCallback.onCaptureStarted(captureSequenceId, timestamp);
956                 } catch (RemoteException e) {
957                     Log.e(TAG, "Failed to notify capture start due to remote " +
958                             "exception!");
959                 }
960             }
961         }
962 
963         @Override
onCaptureProcessStarted(int captureSequenceId)964         public void onCaptureProcessStarted(int captureSequenceId) {
965             if (mCaptureCallback != null) {
966                 try {
967                     mCaptureCallback.onCaptureProcessStarted(captureSequenceId);
968                 } catch (RemoteException e) {
969                     Log.e(TAG, "Failed to notify capture process start due to remote " +
970                             "exception!");
971                 }
972             }
973         }
974 
975         @Override
onCaptureFailed(int captureSequenceId)976         public void onCaptureFailed(int captureSequenceId) {
977             if (mCaptureCallback != null) {
978                 try {
979                     mCaptureCallback.onCaptureFailed(captureSequenceId);
980                 } catch (RemoteException e) {
981                     Log.e(TAG, "Failed to notify capture failure due to remote " +
982                             "exception!");
983                 }
984             }
985         }
986 
987         @Override
onCaptureSequenceCompleted(int captureSequenceId)988         public void onCaptureSequenceCompleted(int captureSequenceId) {
989             if (mCaptureCallback != null) {
990                 try {
991                     mCaptureCallback.onCaptureSequenceCompleted(captureSequenceId);
992                 } catch (RemoteException e) {
993                     Log.e(TAG, "Failed to notify capture sequence end due to remote " +
994                             "exception!");
995                 }
996             }
997         }
998 
999         @Override
onCaptureSequenceAborted(int captureSequenceId)1000         public void onCaptureSequenceAborted(int captureSequenceId) {
1001             if (mCaptureCallback != null) {
1002                 try {
1003                     mCaptureCallback.onCaptureSequenceAborted(captureSequenceId);
1004                 } catch (RemoteException e) {
1005                     Log.e(TAG, "Failed to notify capture sequence abort due to remote " +
1006                             "exception!");
1007                 }
1008             }
1009         }
1010 
1011         @Override
onCaptureCompleted(long timestamp, int requestId, Map<CaptureResult.Key, Object> result)1012         public void onCaptureCompleted(long timestamp, int requestId,
1013                 Map<CaptureResult.Key, Object> result) {
1014 
1015             if (result == null) {
1016                 Log.e(TAG, "Invalid capture result received!");
1017             }
1018 
1019             CameraMetadataNative captureResults = new CameraMetadataNative();
1020             if (mMetadataVendorIdMap.containsKey(mCameraId)) {
1021                 captureResults.setVendorId(mMetadataVendorIdMap.get(mCameraId));
1022             }
1023             for (Map.Entry<CaptureResult.Key, Object> entry : result.entrySet()) {
1024                 captureResults.set(entry.getKey(), entry.getValue());
1025             }
1026 
1027             try {
1028                 mCaptureCallback.onCaptureCompleted(timestamp, requestId, captureResults);
1029             } catch (RemoteException e) {
1030                 Log.e(TAG, "Failed to notify capture complete due to remote exception!");
1031             }
1032         }
1033 
1034         @Override
onCaptureProcessProgressed(int progress)1035         public void onCaptureProcessProgressed(int progress) {
1036             try {
1037                 mCaptureCallback.onCaptureProcessProgressed(progress);
1038             } catch (RemoteException e) {
1039                 Log.e(TAG, "Remote client doesn't respond to capture progress callbacks!");
1040             }
1041         }
1042     }
1043 
1044     private class RequestCallbackStub extends IRequestCallback.Stub {
1045         private final List<RequestProcessorImpl.Request> mRequests;
1046         private final RequestProcessorImpl.Callback mCallback;
1047 
RequestCallbackStub(List<RequestProcessorImpl.Request> requests, RequestProcessorImpl.Callback callback)1048         public RequestCallbackStub(List<RequestProcessorImpl.Request> requests,
1049                 RequestProcessorImpl.Callback callback) {
1050             mCallback = callback;
1051             if (mCallback != null) {
1052                 mRequests = requests;
1053             } else {
1054                 Log.w(TAG, "No valid request callbacks!");
1055                 mRequests = new ArrayList<>();
1056             }
1057         }
1058 
1059         @Override
onCaptureStarted(int requestId, long frameNumber, long timestamp)1060         public void onCaptureStarted(int requestId, long frameNumber, long timestamp) {
1061             if (mCallback != null) {
1062                 if (mRequests.get(requestId) != null) {
1063                     mCallback.onCaptureStarted(mRequests.get(requestId), frameNumber, timestamp);
1064                 } else {
1065                     Log.e(TAG,"Request id: " + requestId + " not found!");
1066                 }
1067             }
1068         }
1069 
1070         @Override
onCaptureProgressed(int requestId, ParcelCaptureResult partialResult)1071         public void onCaptureProgressed(int requestId, ParcelCaptureResult partialResult) {
1072             if (mCallback != null) {
1073                 if (mRequests.get(requestId) != null) {
1074                     CaptureResult result = new CaptureResult(partialResult.cameraId,
1075                             partialResult.results, partialResult.parent, partialResult.sequenceId,
1076                             partialResult.frameNumber);
1077                     mCallback.onCaptureProgressed(mRequests.get(requestId), result);
1078                 } else {
1079                     Log.e(TAG,"Request id: " + requestId + " not found!");
1080                 }
1081             }
1082         }
1083 
1084         @Override
onCaptureCompleted(int requestId, ParcelTotalCaptureResult totalCaptureResult)1085         public void onCaptureCompleted(int requestId, ParcelTotalCaptureResult totalCaptureResult) {
1086             if (mCallback != null) {
1087                 if (mRequests.get(requestId) != null) {
1088                     PhysicalCaptureResultInfo[] physicalResults = new PhysicalCaptureResultInfo[0];
1089                     if ((totalCaptureResult.physicalResult != null) &&
1090                             (!totalCaptureResult.physicalResult.isEmpty())) {
1091                         int count = totalCaptureResult.physicalResult.size();
1092                         physicalResults = new PhysicalCaptureResultInfo[count];
1093                         physicalResults = totalCaptureResult.physicalResult.toArray(
1094                                 physicalResults);
1095                     }
1096                     ArrayList<CaptureResult> partials = new ArrayList<>(
1097                             totalCaptureResult.partials.size());
1098                     for (ParcelCaptureResult parcelResult : totalCaptureResult.partials) {
1099                         partials.add(new CaptureResult(parcelResult.cameraId, parcelResult.results,
1100                                 parcelResult.parent, parcelResult.sequenceId,
1101                                 parcelResult.frameNumber));
1102                     }
1103                     TotalCaptureResult result = new TotalCaptureResult(
1104                             totalCaptureResult.logicalCameraId, totalCaptureResult.results,
1105                             totalCaptureResult.parent, totalCaptureResult.sequenceId,
1106                             totalCaptureResult.frameNumber, partials, totalCaptureResult.sessionId,
1107                             physicalResults);
1108                     mCallback.onCaptureCompleted(mRequests.get(requestId), result);
1109                 } else {
1110                     Log.e(TAG,"Request id: " + requestId + " not found!");
1111                 }
1112             }
1113         }
1114 
1115         @Override
onCaptureFailed(int requestId, CaptureFailure captureFailure)1116         public void onCaptureFailed(int requestId, CaptureFailure captureFailure) {
1117             if (mCallback != null) {
1118                 if (mRequests.get(requestId) != null) {
1119                     android.hardware.camera2.CaptureFailure failure =
1120                             new android.hardware.camera2.CaptureFailure(captureFailure.request,
1121                                     captureFailure.reason, captureFailure.dropped,
1122                                     captureFailure.sequenceId, captureFailure.frameNumber,
1123                                     captureFailure.errorPhysicalCameraId);
1124                     mCallback.onCaptureFailed(mRequests.get(requestId), failure);
1125                 } else {
1126                     Log.e(TAG,"Request id: " + requestId + " not found!");
1127                 }
1128             }
1129         }
1130 
1131         @Override
onCaptureBufferLost(int requestId, long frameNumber, int outputStreamId)1132         public void onCaptureBufferLost(int requestId, long frameNumber, int outputStreamId) {
1133             if (mCallback != null) {
1134                 if (mRequests.get(requestId) != null) {
1135                     mCallback.onCaptureBufferLost(mRequests.get(requestId), frameNumber,
1136                             outputStreamId);
1137                 } else {
1138                     Log.e(TAG,"Request id: " + requestId + " not found!");
1139                 }
1140             }
1141         }
1142 
1143         @Override
onCaptureSequenceCompleted(int sequenceId, long frameNumber)1144         public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) {
1145             if (mCallback != null) {
1146                 mCallback.onCaptureSequenceCompleted(sequenceId, frameNumber);
1147             }
1148         }
1149 
1150         @Override
onCaptureSequenceAborted(int sequenceId)1151         public void onCaptureSequenceAborted(int sequenceId) {
1152             if (mCallback != null) {
1153                 mCallback.onCaptureSequenceAborted(sequenceId);
1154             }
1155         }
1156     }
1157 
1158     private class ImageProcessorImplStub extends IImageProcessorImpl.Stub {
1159         private final ImageProcessorImpl mImageProcessor;
1160 
ImageProcessorImplStub(ImageProcessorImpl imageProcessor)1161         public ImageProcessorImplStub(ImageProcessorImpl imageProcessor) {
1162             mImageProcessor = imageProcessor;
1163         }
1164 
1165         @Override
onNextImageAvailable(OutputConfigId outputConfigId, ParcelImage img, String physicalCameraId)1166         public void onNextImageAvailable(OutputConfigId outputConfigId, ParcelImage img,
1167                 String physicalCameraId) {
1168             if (mImageProcessor != null) {
1169                 mImageProcessor.onNextImageAvailable(outputConfigId.id, img.timestamp,
1170                         new ImageReferenceImpl(img), physicalCameraId);
1171             }
1172         }
1173     }
1174 
1175     private class RequestProcessorStub implements RequestProcessorImpl {
1176         private final IRequestProcessorImpl mRequestProcessor;
1177         private final String mCameraId;
1178 
RequestProcessorStub(IRequestProcessorImpl requestProcessor, String cameraId)1179         public RequestProcessorStub(IRequestProcessorImpl requestProcessor, String cameraId) {
1180             mRequestProcessor = requestProcessor;
1181             mCameraId = cameraId;
1182         }
1183 
1184         @Override
setImageProcessor(int outputConfigId, ImageProcessorImpl imageProcessor)1185         public void setImageProcessor(int outputConfigId,
1186                 ImageProcessorImpl imageProcessor) {
1187             OutputConfigId  configId = new OutputConfigId();
1188             configId.id = outputConfigId;
1189             try {
1190                 mRequestProcessor.setImageProcessor(configId,
1191                         new ImageProcessorImplStub(imageProcessor));
1192             } catch (RemoteException e) {
1193                 Log.e(TAG, "Failed to set image processor due to remote exception!");
1194             }
1195         }
1196 
1197         @Override
submit(Request request, Callback callback)1198         public int submit(Request request, Callback callback) {
1199             ArrayList<Request> requests = new ArrayList<>();
1200             requests.add(request);
1201             return submit(requests, callback);
1202         }
1203 
1204         @Override
submit(List<Request> requests, Callback callback)1205         public int submit(List<Request> requests, Callback callback) {
1206             ArrayList<android.hardware.camera2.extension.Request> captureRequests =
1207                     new ArrayList<>();
1208             int requestId = 0;
1209             for (Request request : requests) {
1210                 captureRequests.add(initializeParcelable(request, requestId, mCameraId));
1211                 requestId++;
1212             }
1213 
1214             try {
1215                 return mRequestProcessor.submitBurst(captureRequests,
1216                         new RequestCallbackStub(requests, callback));
1217             } catch (RemoteException e) {
1218                 Log.e(TAG, "Failed to submit request due to remote exception!");
1219             }
1220             return -1;
1221         }
1222 
1223         @Override
setRepeating(Request request, Callback callback)1224         public int setRepeating(Request request, Callback callback) {
1225             try {
1226                 ArrayList<Request> requests = new ArrayList<>();
1227                 requests.add(request);
1228                 return mRequestProcessor.setRepeating(
1229                         initializeParcelable(request, 0, mCameraId),
1230                         new RequestCallbackStub(requests, callback));
1231             } catch (RemoteException e) {
1232                 Log.e(TAG, "Failed to submit repeating request due to remote exception!");
1233             }
1234 
1235             return -1;
1236         }
1237 
1238         @Override
abortCaptures()1239         public void abortCaptures() {
1240             try {
1241                 mRequestProcessor.abortCaptures();
1242             } catch (RemoteException e) {
1243                 Log.e(TAG, "Failed to abort requests due to remote exception!");
1244             }
1245         }
1246 
1247         @Override
stopRepeating()1248         public void stopRepeating() {
1249             try {
1250                 mRequestProcessor.stopRepeating();
1251             } catch (RemoteException e) {
1252                 Log.e(TAG, "Failed to stop repeating request due to remote exception!");
1253             }
1254         }
1255     }
1256 
1257     private class SessionProcessorImplStub extends ISessionProcessorImpl.Stub implements
1258             IBinder.DeathRecipient {
1259         private final SessionProcessorImpl mSessionProcessor;
1260         private String mCameraId = null;
1261         private IBinder mToken;
1262 
SessionProcessorImplStub(SessionProcessorImpl sessionProcessor)1263         public SessionProcessorImplStub(SessionProcessorImpl sessionProcessor) {
1264             mSessionProcessor = sessionProcessor;
1265         }
1266 
1267         @Override
initSession(IBinder token, String cameraId, Map<String, CameraMetadataNative> charsMapNative, OutputSurface previewSurface, OutputSurface imageCaptureSurface, OutputSurface postviewSurface)1268         public CameraSessionConfig initSession(IBinder token, String cameraId,
1269                 Map<String, CameraMetadataNative> charsMapNative, OutputSurface previewSurface,
1270                 OutputSurface imageCaptureSurface, OutputSurface postviewSurface) {
1271             OutputSurfaceImplStub outputPreviewSurfaceImpl =
1272                     new OutputSurfaceImplStub(previewSurface);
1273             OutputSurfaceImplStub outputImageCaptureSurfaceImpl =
1274                     new OutputSurfaceImplStub(imageCaptureSurface);
1275             OutputSurfaceImplStub outputPostviewSurfaceImpl =
1276                     new OutputSurfaceImplStub(postviewSurface);
1277 
1278             Camera2SessionConfigImpl sessionConfig;
1279 
1280             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1281                 OutputSurfaceConfigurationImplStub outputSurfaceConfigs =
1282                         new OutputSurfaceConfigurationImplStub(outputPreviewSurfaceImpl,
1283                         // Image Analysis Output is currently only supported in CameraX
1284                         outputImageCaptureSurfaceImpl, null /*imageAnalysisSurfaceConfig*/,
1285                         outputPostviewSurfaceImpl);
1286 
1287                 sessionConfig = mSessionProcessor.initSession(cameraId,
1288                         getCharacteristicsMap(charsMapNative),
1289                         getApplicationContext(), outputSurfaceConfigs);
1290             } else {
1291                 sessionConfig = mSessionProcessor.initSession(cameraId,
1292                         getCharacteristicsMap(charsMapNative),
1293                         getApplicationContext(), outputPreviewSurfaceImpl,
1294                         outputImageCaptureSurfaceImpl, null /*imageAnalysisSurfaceConfig*/);
1295             }
1296 
1297             List<Camera2OutputConfigImpl> outputConfigs = sessionConfig.getOutputConfigs();
1298             CameraSessionConfig ret = new CameraSessionConfig();
1299             ret.outputConfigs = new ArrayList<>();
1300             for (Camera2OutputConfigImpl output : outputConfigs) {
1301                 CameraOutputConfig entry = getCameraOutputConfig(output);
1302                 List<Camera2OutputConfigImpl> sharedOutputs =
1303                         output.getSurfaceSharingOutputConfigs();
1304                 if ((sharedOutputs != null) && (!sharedOutputs.isEmpty())) {
1305                     entry.sharedSurfaceConfigs = new ArrayList<>();
1306                     for (Camera2OutputConfigImpl sharedOutput : sharedOutputs) {
1307                         entry.sharedSurfaceConfigs.add(getCameraOutputConfig(sharedOutput));
1308                     }
1309                 }
1310                 ret.outputConfigs.add(entry);
1311             }
1312             ret.sessionTemplateId = sessionConfig.getSessionTemplateId();
1313             ret.sessionType = -1;
1314             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1315                 ret.sessionType = sessionConfig.getSessionType();
1316             }
1317             ret.sessionParameter = initializeParcelableMetadata(
1318                     sessionConfig.getSessionParameters(), cameraId);
1319             mCameraId = cameraId;
1320             mToken = token;
1321             CameraExtensionsProxyService.registerDeathRecipient(mToken, this);
1322             return ret;
1323         }
1324 
1325         @Override
deInitSession(IBinder token)1326         public void deInitSession(IBinder token) {
1327             CameraExtensionsProxyService.unregisterDeathRecipient(mToken, this);
1328             mSessionProcessor.deInitSession();
1329         }
1330 
1331         @Override
onCaptureSessionStart(IRequestProcessorImpl requestProcessor)1332         public void onCaptureSessionStart(IRequestProcessorImpl requestProcessor) {
1333             mSessionProcessor.onCaptureSessionStart(
1334                     new RequestProcessorStub(requestProcessor, mCameraId));
1335         }
1336 
1337         @Override
onCaptureSessionEnd()1338         public void onCaptureSessionEnd() {
1339             mSessionProcessor.onCaptureSessionEnd();
1340         }
1341 
1342         @Override
startRepeating(ICaptureCallback callback)1343         public int startRepeating(ICaptureCallback callback) {
1344             return mSessionProcessor.startRepeating(new CaptureCallbackStub(callback, mCameraId));
1345         }
1346 
1347         @Override
stopRepeating()1348         public void stopRepeating() {
1349             mSessionProcessor.stopRepeating();
1350         }
1351 
1352         @Override
setParameters(CaptureRequest captureRequest)1353         public void setParameters(CaptureRequest captureRequest) {
1354             HashMap<CaptureRequest.Key<?>, Object> paramMap = new HashMap<>();
1355             for (CaptureRequest.Key captureRequestKey : captureRequest.getKeys()) {
1356                 paramMap.put(captureRequestKey, captureRequest.get(captureRequestKey));
1357             }
1358 
1359             mSessionProcessor.setParameters(paramMap);
1360         }
1361 
1362         @Override
startTrigger(CaptureRequest captureRequest, ICaptureCallback callback)1363         public int startTrigger(CaptureRequest captureRequest, ICaptureCallback callback) {
1364             HashMap<CaptureRequest.Key<?>, Object> triggerMap = new HashMap<>();
1365             for (CaptureRequest.Key captureRequestKey : captureRequest.getKeys()) {
1366                 triggerMap.put(captureRequestKey, captureRequest.get(captureRequestKey));
1367             }
1368 
1369             return mSessionProcessor.startTrigger(triggerMap,
1370                     new CaptureCallbackStub(callback, mCameraId));
1371         }
1372 
1373         @Override
startCapture(ICaptureCallback callback, boolean isPostviewRequested)1374         public int startCapture(ICaptureCallback callback, boolean isPostviewRequested) {
1375             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1376                 return isPostviewRequested ? mSessionProcessor.startCaptureWithPostview(
1377                         new CaptureCallbackStub(callback, mCameraId)) :
1378                         mSessionProcessor.startCapture(new CaptureCallbackStub(callback,
1379                         mCameraId));
1380             }
1381 
1382             return mSessionProcessor.startCapture(new CaptureCallbackStub(callback, mCameraId));
1383         }
1384 
1385         @Override
getRealtimeCaptureLatency()1386         public LatencyPair getRealtimeCaptureLatency() {
1387             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1388                 Pair<Long, Long> latency = mSessionProcessor.getRealtimeCaptureLatency();
1389                 if (latency != null) {
1390                     LatencyPair ret = new LatencyPair();
1391                     ret.first = latency.first;
1392                     ret.second = latency.second;
1393                     return ret;
1394                 }
1395             }
1396 
1397             return null;
1398         }
1399 
1400         @Override
binderDied()1401         public void binderDied() {
1402             mSessionProcessor.deInitSession();
1403         }
1404     }
1405 
1406     private class OutputSurfaceConfigurationImplStub implements OutputSurfaceConfigurationImpl {
1407         private OutputSurfaceImpl mOutputPreviewSurfaceImpl;
1408         private OutputSurfaceImpl mOutputImageCaptureSurfaceImpl;
1409         private OutputSurfaceImpl mOutputImageAnalysisSurfaceImpl;
1410         private OutputSurfaceImpl mOutputPostviewSurfaceImpl;
1411 
OutputSurfaceConfigurationImplStub(OutputSurfaceImpl previewOutput, OutputSurfaceImpl imageCaptureOutput, OutputSurfaceImpl imageAnalysisOutput, OutputSurfaceImpl postviewOutput)1412         public OutputSurfaceConfigurationImplStub(OutputSurfaceImpl previewOutput,
1413                 OutputSurfaceImpl imageCaptureOutput, OutputSurfaceImpl imageAnalysisOutput,
1414                 OutputSurfaceImpl postviewOutput) {
1415             mOutputPreviewSurfaceImpl = previewOutput;
1416             mOutputImageCaptureSurfaceImpl = imageCaptureOutput;
1417             mOutputImageAnalysisSurfaceImpl = imageAnalysisOutput;
1418             mOutputPostviewSurfaceImpl = postviewOutput;
1419         }
1420 
1421         @Override
getPreviewOutputSurface()1422         public OutputSurfaceImpl getPreviewOutputSurface() {
1423             return mOutputPreviewSurfaceImpl;
1424         }
1425 
1426         @Override
getImageCaptureOutputSurface()1427         public OutputSurfaceImpl getImageCaptureOutputSurface() {
1428             return mOutputImageCaptureSurfaceImpl;
1429         }
1430 
1431         @Override
getImageAnalysisOutputSurface()1432         public OutputSurfaceImpl getImageAnalysisOutputSurface() {
1433             return mOutputImageAnalysisSurfaceImpl;
1434         }
1435 
1436         @Override
getPostviewOutputSurface()1437         public OutputSurfaceImpl getPostviewOutputSurface() {
1438             return mOutputPostviewSurfaceImpl;
1439         }
1440     }
1441 
1442     private class OutputSurfaceImplStub implements OutputSurfaceImpl {
1443         private final Surface mSurface;
1444         private final Size mSize;
1445         private final int mImageFormat;
1446 
OutputSurfaceImplStub(OutputSurface outputSurface)1447         public OutputSurfaceImplStub(OutputSurface outputSurface) {
1448             mSurface = outputSurface.surface;
1449             mSize = new Size(outputSurface.size.width, outputSurface.size.height);
1450             mImageFormat = outputSurface.imageFormat;
1451         }
1452 
1453         @Override
getSurface()1454         public Surface getSurface() {
1455             return mSurface;
1456         }
1457 
1458         @Override
getSize()1459         public Size getSize() {
1460             return mSize;
1461         }
1462 
1463         @Override
getImageFormat()1464         public int getImageFormat() {
1465             return mImageFormat;
1466         }
1467     }
1468 
1469     private class PreviewExtenderImplStub extends IPreviewExtenderImpl.Stub implements
1470             IBinder.DeathRecipient {
1471         private final PreviewExtenderImpl mPreviewExtender;
1472         private String mCameraId = null;
1473         private boolean mSessionEnabled;
1474         private IBinder mToken;
1475 
PreviewExtenderImplStub(PreviewExtenderImpl previewExtender)1476         public PreviewExtenderImplStub(PreviewExtenderImpl previewExtender) {
1477             mPreviewExtender = previewExtender;
1478         }
1479 
1480         @Override
onInit(IBinder token, String cameraId, CameraMetadataNative cameraCharacteristics)1481         public void onInit(IBinder token, String cameraId,
1482                 CameraMetadataNative cameraCharacteristics) {
1483             mCameraId = cameraId;
1484             CameraCharacteristics chars = new CameraCharacteristics(cameraCharacteristics);
1485             mCameraManager.registerDeviceStateListener(chars);
1486             mPreviewExtender.onInit(cameraId, chars, CameraExtensionsProxyService.this);
1487             mToken = token;
1488             CameraExtensionsProxyService.registerDeathRecipient(mToken, this);
1489         }
1490 
1491         @Override
onDeInit(IBinder token)1492         public void onDeInit(IBinder token) {
1493             CameraExtensionsProxyService.unregisterDeathRecipient(mToken, this);
1494             mPreviewExtender.onDeInit();
1495         }
1496 
1497         @Override
onPresetSession()1498         public CaptureStageImpl onPresetSession() {
1499             return initializeParcelable(mPreviewExtender.onPresetSession(), mCameraId);
1500         }
1501 
1502         @Override
onEnableSession()1503         public CaptureStageImpl onEnableSession() {
1504             mSessionEnabled = true;
1505             return initializeParcelable(mPreviewExtender.onEnableSession(), mCameraId);
1506         }
1507 
1508         @Override
onDisableSession()1509         public CaptureStageImpl onDisableSession() {
1510             mSessionEnabled = false;
1511             return initializeParcelable(mPreviewExtender.onDisableSession(), mCameraId);
1512         }
1513 
1514         @Override
init(String cameraId, CameraMetadataNative chars)1515         public void init(String cameraId, CameraMetadataNative chars) {
1516             CameraCharacteristics c = new CameraCharacteristics(chars);
1517             mCameraManager.registerDeviceStateListener(c);
1518             mPreviewExtender.init(cameraId, c);
1519         }
1520 
1521         @Override
isExtensionAvailable(String cameraId, CameraMetadataNative chars)1522         public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) {
1523             CameraCharacteristics c = new CameraCharacteristics(chars);
1524             mCameraManager.registerDeviceStateListener(c);
1525             return mPreviewExtender.isExtensionAvailable(cameraId, c);
1526         }
1527 
1528         @Override
getCaptureStage()1529         public CaptureStageImpl getCaptureStage() {
1530             return initializeParcelable(mPreviewExtender.getCaptureStage(), mCameraId);
1531         }
1532 
1533         @Override
getSessionType()1534         public int getSessionType() {
1535             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1536                 return mPreviewExtender.onSessionType();
1537             }
1538 
1539             return -1;
1540         }
1541 
1542         @Override
getProcessorType()1543         public int getProcessorType() {
1544             ProcessorType processorType = mPreviewExtender.getProcessorType();
1545             if (processorType == ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
1546                 return IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY;
1547             } else if (processorType == ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
1548                 return IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR;
1549             } else {
1550                 return IPreviewExtenderImpl.PROCESSOR_TYPE_NONE;
1551             }
1552         }
1553 
1554         @Override
getPreviewImageProcessor()1555         public IPreviewImageProcessorImpl getPreviewImageProcessor() {
1556             PreviewImageProcessorImpl processor;
1557             try {
1558                 processor = (PreviewImageProcessorImpl) mPreviewExtender.getProcessor();
1559             } catch (ClassCastException e) {
1560                 Log.e(TAG, "Failed casting preview processor!");
1561                 return null;
1562             }
1563 
1564             if (processor != null) {
1565                 return new PreviewImageProcessorImplStub(processor, mCameraId);
1566             }
1567 
1568             return null;
1569         }
1570 
1571         @Override
getRequestUpdateProcessor()1572         public IRequestUpdateProcessorImpl getRequestUpdateProcessor() {
1573             RequestUpdateProcessorImpl processor;
1574             try {
1575                 processor = (RequestUpdateProcessorImpl) mPreviewExtender.getProcessor();
1576             } catch (ClassCastException e) {
1577                 Log.e(TAG, "Failed casting preview processor!");
1578                 return null;
1579             }
1580 
1581             if (processor != null) {
1582                 return new RequestUpdateProcessorImplStub(processor, mCameraId);
1583             }
1584 
1585             return null;
1586         }
1587 
1588         @Override
getSupportedResolutions()1589         public List<SizeList> getSupportedResolutions() {
1590             if (INIT_API_SUPPORTED) {
1591                 List<Pair<Integer, android.util.Size[]>> sizes =
1592                         mPreviewExtender.getSupportedResolutions();
1593                 if ((sizes != null) && !sizes.isEmpty()) {
1594                     return initializeParcelable(sizes);
1595                 }
1596             }
1597             return null;
1598         }
1599 
1600         @Override
binderDied()1601         public void binderDied() {
1602             if (mSessionEnabled) {
1603                 mPreviewExtender.onDisableSession();
1604             }
1605             mPreviewExtender.onDeInit();
1606         }
1607     }
1608 
1609     private class ImageCaptureExtenderImplStub extends IImageCaptureExtenderImpl.Stub implements
1610             IBinder.DeathRecipient {
1611         private final ImageCaptureExtenderImpl mImageExtender;
1612         private String mCameraId = null;
1613         private boolean mSessionEnabled;
1614         private IBinder mToken;
1615 
ImageCaptureExtenderImplStub(ImageCaptureExtenderImpl imageExtender)1616         public ImageCaptureExtenderImplStub(ImageCaptureExtenderImpl imageExtender) {
1617             mImageExtender = imageExtender;
1618         }
1619 
1620         @Override
onInit(IBinder token, String cameraId, CameraMetadataNative cameraCharacteristics)1621         public void onInit(IBinder token, String cameraId,
1622                 CameraMetadataNative cameraCharacteristics) {
1623             CameraCharacteristics chars = new CameraCharacteristics(cameraCharacteristics);
1624             mCameraManager.registerDeviceStateListener(chars);
1625             mImageExtender.onInit(cameraId, chars, CameraExtensionsProxyService.this);
1626             mCameraId = cameraId;
1627             mToken = token;
1628             CameraExtensionsProxyService.registerDeathRecipient(mToken, this);
1629         }
1630 
1631         @Override
onDeInit(IBinder token)1632         public void onDeInit(IBinder token) {
1633             CameraExtensionsProxyService.unregisterDeathRecipient(mToken, this);
1634             mImageExtender.onDeInit();
1635         }
1636 
1637         @Override
onPresetSession()1638         public CaptureStageImpl onPresetSession() {
1639             return initializeParcelable(mImageExtender.onPresetSession(), mCameraId);
1640         }
1641 
1642         @Override
isCaptureProcessProgressAvailable()1643         public boolean isCaptureProcessProgressAvailable() {
1644             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1645                 return mImageExtender.isCaptureProcessProgressAvailable();
1646             }
1647 
1648             return false;
1649         }
1650 
1651         @Override
isPostviewAvailable()1652         public boolean isPostviewAvailable() {
1653             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1654                 return mImageExtender.isPostviewAvailable();
1655             }
1656 
1657             return false;
1658         }
1659 
1660         @Override
onEnableSession()1661         public CaptureStageImpl onEnableSession() {
1662             mSessionEnabled = true;
1663             return initializeParcelable(mImageExtender.onEnableSession(), mCameraId);
1664         }
1665 
1666         @Override
onDisableSession()1667         public CaptureStageImpl onDisableSession() {
1668             mSessionEnabled = false;
1669             return initializeParcelable(mImageExtender.onDisableSession(), mCameraId);
1670         }
1671 
1672         @Override
getSessionType()1673         public int getSessionType() {
1674             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1675                 return mImageExtender.onSessionType();
1676             }
1677 
1678             return -1;
1679         }
1680 
1681         @Override
init(String cameraId, CameraMetadataNative chars)1682         public void init(String cameraId, CameraMetadataNative chars) {
1683             CameraCharacteristics c = new CameraCharacteristics(chars);
1684             mCameraManager.registerDeviceStateListener(c);
1685             mImageExtender.init(cameraId, c);
1686         }
1687 
1688         @Override
isExtensionAvailable(String cameraId, CameraMetadataNative chars)1689         public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) {
1690             CameraCharacteristics c = new CameraCharacteristics(chars);
1691             mCameraManager.registerDeviceStateListener(c);
1692             return mImageExtender.isExtensionAvailable(cameraId, c);
1693         }
1694 
1695         @Override
getCaptureProcessor()1696         public ICaptureProcessorImpl getCaptureProcessor() {
1697             CaptureProcessorImpl captureProcessor = mImageExtender.getCaptureProcessor();
1698             if (captureProcessor != null) {
1699                 return new CaptureProcessorImplStub(captureProcessor, mCameraId);
1700             }
1701 
1702             return null;
1703         }
1704 
1705 
1706         @Override
getCaptureStages()1707         public List<CaptureStageImpl> getCaptureStages() {
1708             List<androidx.camera.extensions.impl.CaptureStageImpl> captureStages =
1709                 mImageExtender.getCaptureStages();
1710             if (captureStages != null) {
1711                 ArrayList<android.hardware.camera2.extension.CaptureStageImpl> ret =
1712                         new ArrayList<>();
1713                 for (androidx.camera.extensions.impl.CaptureStageImpl stage : captureStages) {
1714                     ret.add(initializeParcelable(stage, mCameraId));
1715                 }
1716 
1717                 return ret;
1718             }
1719 
1720             return null;
1721         }
1722 
1723         @Override
getMaxCaptureStage()1724         public int getMaxCaptureStage() {
1725             return mImageExtender.getMaxCaptureStage();
1726         }
1727 
1728         @Override
getSupportedResolutions()1729         public List<SizeList> getSupportedResolutions() {
1730             if (INIT_API_SUPPORTED) {
1731                 List<Pair<Integer, android.util.Size[]>> sizes =
1732                         mImageExtender.getSupportedResolutions();
1733                 if ((sizes != null) && !sizes.isEmpty()) {
1734                     return initializeParcelable(sizes);
1735                 }
1736             }
1737 
1738             return null;
1739         }
1740 
1741         @Override
getSupportedPostviewResolutions( android.hardware.camera2.extension.Size captureSize)1742         public List<SizeList> getSupportedPostviewResolutions(
1743                 android.hardware.camera2.extension.Size captureSize) {
1744             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1745                 Size sz = new Size(captureSize.width, captureSize.height);
1746                 List<Pair<Integer, android.util.Size[]>> sizes =
1747                         mImageExtender.getSupportedPostviewResolutions(sz);
1748                 if ((sizes != null) && !sizes.isEmpty()) {
1749                     return initializeParcelable(sizes);
1750                 }
1751             }
1752 
1753             return null;
1754         }
1755 
1756         @Override
getEstimatedCaptureLatencyRange( android.hardware.camera2.extension.Size outputSize)1757         public LatencyRange getEstimatedCaptureLatencyRange(
1758                 android.hardware.camera2.extension.Size outputSize) {
1759             if (ESTIMATED_LATENCY_API_SUPPORTED) {
1760                 Size sz = new Size(outputSize.width, outputSize.height);
1761                 Range<Long> latencyRange = mImageExtender.getEstimatedCaptureLatencyRange(sz);
1762                 if (latencyRange != null) {
1763                     LatencyRange ret = new LatencyRange();
1764                     ret.min = latencyRange.getLower();
1765                     ret.max = latencyRange.getUpper();
1766                     return ret;
1767                 }
1768             }
1769 
1770             return null;
1771         }
1772 
1773         @Override
getRealtimeCaptureLatency()1774         public LatencyPair getRealtimeCaptureLatency() {
1775             if (LATENCY_IMPROVEMENTS_SUPPORTED) {
1776                 Pair<Long, Long> latency = mImageExtender.getRealtimeCaptureLatency();
1777                 if (latency != null) {
1778                     LatencyPair ret = new LatencyPair();
1779                     ret.first = latency.first;
1780                     ret.second = latency.second;
1781                     return ret;
1782                 }
1783             }
1784 
1785             return null;
1786         }
1787 
1788         @Override
getAvailableCaptureRequestKeys()1789         public CameraMetadataNative getAvailableCaptureRequestKeys() {
1790             if (RESULT_API_SUPPORTED) {
1791                 List<CaptureRequest.Key> supportedCaptureKeys =
1792                         mImageExtender.getAvailableCaptureRequestKeys();
1793 
1794                 if ((supportedCaptureKeys != null) && !supportedCaptureKeys.isEmpty()) {
1795                     CameraMetadataNative ret = new CameraMetadataNative();
1796                     long vendorId = mMetadataVendorIdMap.containsKey(mCameraId) ?
1797                             mMetadataVendorIdMap.get(mCameraId) : Long.MAX_VALUE;
1798                     ret.setVendorId(vendorId);
1799                     int requestKeyTags [] = new int[supportedCaptureKeys.size()];
1800                     int i = 0;
1801                     for (CaptureRequest.Key key : supportedCaptureKeys) {
1802                         requestKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId);
1803                     }
1804                     ret.set(CameraCharacteristics.REQUEST_AVAILABLE_REQUEST_KEYS, requestKeyTags);
1805 
1806                     return ret;
1807                 }
1808             }
1809 
1810             return null;
1811         }
1812 
1813         @Override
getAvailableCaptureResultKeys()1814         public CameraMetadataNative getAvailableCaptureResultKeys() {
1815             if (RESULT_API_SUPPORTED) {
1816                 List<CaptureResult.Key> supportedResultKeys =
1817                         mImageExtender.getAvailableCaptureResultKeys();
1818 
1819                 if ((supportedResultKeys != null) && !supportedResultKeys.isEmpty()) {
1820                     CameraMetadataNative ret = new CameraMetadataNative();
1821                     long vendorId = mMetadataVendorIdMap.containsKey(mCameraId) ?
1822                             mMetadataVendorIdMap.get(mCameraId) : Long.MAX_VALUE;
1823                     ret.setVendorId(vendorId);
1824                     int resultKeyTags [] = new int[supportedResultKeys.size()];
1825                     int i = 0;
1826                     for (CaptureResult.Key key : supportedResultKeys) {
1827                         resultKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId);
1828                     }
1829                     ret.set(CameraCharacteristics.REQUEST_AVAILABLE_RESULT_KEYS, resultKeyTags);
1830 
1831                     return ret;
1832                 }
1833             }
1834 
1835             return null;
1836         }
1837 
1838         @Override
binderDied()1839         public void binderDied() {
1840             if (mSessionEnabled) {
1841                 mImageExtender.onDisableSession();
1842             }
1843             mImageExtender.onDeInit();
1844         }
1845     }
1846 
1847     private class ProcessResultCallback implements ProcessResultImpl {
1848         private final IProcessResultImpl mProcessResult;
1849         private final String mCameraId;
1850 
ProcessResultCallback(IProcessResultImpl processResult, String cameraId)1851         private ProcessResultCallback(IProcessResultImpl processResult, String cameraId) {
1852             mProcessResult = processResult;
1853             mCameraId = cameraId;
1854         }
1855 
1856         @Override
onCaptureProcessProgressed(int progress)1857         public void onCaptureProcessProgressed(int progress) {
1858             try {
1859                 mProcessResult.onCaptureProcessProgressed(progress);
1860             } catch (RemoteException e) {
1861                 Log.e(TAG, "Remote client doesn't respond to capture progress callbacks!");
1862             }
1863         }
1864 
1865         @Override
onCaptureCompleted(long shutterTimestamp, List<Pair<CaptureResult.Key, Object>> result)1866         public void onCaptureCompleted(long shutterTimestamp,
1867                 List<Pair<CaptureResult.Key, Object>> result) {
1868             if (result == null) {
1869                 Log.e(TAG, "Invalid capture result received!");
1870             }
1871 
1872             CameraMetadataNative captureResults = new CameraMetadataNative();
1873             if (mMetadataVendorIdMap.containsKey(mCameraId)) {
1874                 captureResults.setVendorId(mMetadataVendorIdMap.get(mCameraId));
1875             }
1876             for (Pair<CaptureResult.Key, Object> pair : result) {
1877                 captureResults.set(pair.first, pair.second);
1878             }
1879 
1880             try {
1881                 mProcessResult.onCaptureCompleted(shutterTimestamp, captureResults);
1882             } catch (RemoteException e) {
1883                 Log.e(TAG, "Remote client doesn't respond to capture results!");
1884             }
1885         }
1886     }
1887 
1888     private class CaptureProcessorImplStub extends ICaptureProcessorImpl.Stub {
1889         private final CaptureProcessorImpl mCaptureProcessor;
1890         private final String mCameraId;
1891 
CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor, String cameraId)1892         public CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor, String cameraId) {
1893             mCaptureProcessor = captureProcessor;
1894             mCameraId = cameraId;
1895         }
1896 
1897         @Override
onOutputSurface(Surface surface, int imageFormat)1898         public void onOutputSurface(Surface surface, int imageFormat) {
1899             mCaptureProcessor.onOutputSurface(surface, imageFormat);
1900         }
1901 
1902         @Override
onPostviewOutputSurface(Surface surface)1903         public void onPostviewOutputSurface(Surface surface) {
1904             mCaptureProcessor.onPostviewOutputSurface(surface);
1905         }
1906 
1907         @Override
onResolutionUpdate(android.hardware.camera2.extension.Size size, android.hardware.camera2.extension.Size postviewSize)1908         public void onResolutionUpdate(android.hardware.camera2.extension.Size size,
1909                 android.hardware.camera2.extension.Size postviewSize) {
1910             if (postviewSize != null) {
1911                 mCaptureProcessor.onResolutionUpdate(
1912                         new android.util.Size(size.width, size.height),
1913                         new android.util.Size(postviewSize.width, postviewSize.height));
1914             } else {
1915                 mCaptureProcessor.onResolutionUpdate(
1916                         new android.util.Size(size.width, size.height));
1917             }
1918         }
1919 
1920         @Override
onImageFormatUpdate(int imageFormat)1921         public void onImageFormatUpdate(int imageFormat) {
1922             mCaptureProcessor.onImageFormatUpdate(imageFormat);
1923         }
1924 
1925         @Override
process(List<CaptureBundle> captureList, IProcessResultImpl resultCallback, boolean isPostviewRequested)1926         public void process(List<CaptureBundle> captureList, IProcessResultImpl resultCallback,
1927                 boolean isPostviewRequested) {
1928             HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap = new HashMap<>();
1929             for (CaptureBundle captureBundle : captureList) {
1930                 captureMap.put(captureBundle.stage, new Pair<> (
1931                         new ExtensionImage(captureBundle.captureImage),
1932                         new TotalCaptureResult(captureBundle.captureResult,
1933                                 captureBundle.sequenceId)));
1934             }
1935             if (!captureMap.isEmpty()) {
1936                 if ((LATENCY_IMPROVEMENTS_SUPPORTED) && (isPostviewRequested)) {
1937                     ProcessResultCallback processResultCallback = (resultCallback != null)
1938                             ? new ProcessResultCallback(resultCallback, mCameraId) : null;
1939                     mCaptureProcessor.processWithPostview(captureMap, processResultCallback,
1940                             null /*executor*/);
1941                 } else if ((resultCallback != null) && (RESULT_API_SUPPORTED)) {
1942                     mCaptureProcessor.process(captureMap, new ProcessResultCallback(resultCallback,
1943                                     mCameraId), null /*executor*/);
1944                 } else if (resultCallback == null) {
1945                     mCaptureProcessor.process(captureMap);
1946                 } else {
1947                     Log.e(TAG, "Process requests with capture results are not supported!");
1948                 }
1949             } else {
1950                 Log.e(TAG, "Process request with absent capture stages!");
1951             }
1952         }
1953     }
1954 
1955     private class PreviewImageProcessorImplStub extends IPreviewImageProcessorImpl.Stub {
1956         private final PreviewImageProcessorImpl mProcessor;
1957         private final String mCameraId;
1958 
PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor, String cameraId)1959         public PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor, String cameraId) {
1960             mProcessor = processor;
1961             mCameraId = cameraId;
1962         }
1963 
1964         @Override
onOutputSurface(Surface surface, int imageFormat)1965         public void onOutputSurface(Surface surface, int imageFormat) {
1966             mProcessor.onOutputSurface(surface, imageFormat);
1967         }
1968 
1969         @Override
onResolutionUpdate(android.hardware.camera2.extension.Size size)1970         public void onResolutionUpdate(android.hardware.camera2.extension.Size size) {
1971             mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height));
1972         }
1973 
1974         @Override
onImageFormatUpdate(int imageFormat)1975         public void onImageFormatUpdate(int imageFormat) {
1976             mProcessor.onImageFormatUpdate(imageFormat);
1977         }
1978 
1979         @Override
process(android.hardware.camera2.extension.ParcelImage image, CameraMetadataNative result, int sequenceId, IProcessResultImpl resultCallback)1980         public void process(android.hardware.camera2.extension.ParcelImage image,
1981                 CameraMetadataNative result, int sequenceId, IProcessResultImpl resultCallback) {
1982             if ((resultCallback != null) && RESULT_API_SUPPORTED) {
1983                 mProcessor.process(new ExtensionImage(image),
1984                         new TotalCaptureResult(result, sequenceId),
1985                         new ProcessResultCallback(resultCallback, mCameraId), null /*executor*/);
1986             } else if (resultCallback == null) {
1987                 mProcessor.process(new ExtensionImage(image),
1988                         new TotalCaptureResult(result, sequenceId));
1989             } else {
1990 
1991             }
1992         }
1993     }
1994 
1995     private class RequestUpdateProcessorImplStub extends IRequestUpdateProcessorImpl.Stub {
1996         private final RequestUpdateProcessorImpl mProcessor;
1997         private final String mCameraId;
1998 
RequestUpdateProcessorImplStub(RequestUpdateProcessorImpl processor, String cameraId)1999         public RequestUpdateProcessorImplStub(RequestUpdateProcessorImpl processor,
2000                 String cameraId) {
2001             mProcessor = processor;
2002             mCameraId = cameraId;
2003         }
2004 
2005         @Override
onOutputSurface(Surface surface, int imageFormat)2006         public void onOutputSurface(Surface surface, int imageFormat) {
2007             mProcessor.onOutputSurface(surface, imageFormat);
2008         }
2009 
2010         @Override
onResolutionUpdate(android.hardware.camera2.extension.Size size)2011         public void onResolutionUpdate(android.hardware.camera2.extension.Size size) {
2012             mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height));
2013         }
2014 
2015         @Override
onImageFormatUpdate(int imageFormat)2016         public void onImageFormatUpdate(int imageFormat) {
2017             mProcessor.onImageFormatUpdate(imageFormat);
2018         }
2019 
2020         @Override
process(CameraMetadataNative result, int sequenceId)2021         public CaptureStageImpl process(CameraMetadataNative result, int sequenceId) {
2022             return initializeParcelable(
2023                     mProcessor.process(new TotalCaptureResult(result, sequenceId)), mCameraId);
2024         }
2025     }
2026 
2027     private class ImageReferenceImpl extends ExtensionImage
2028             implements androidx.camera.extensions.impl.advanced.ImageReferenceImpl {
2029 
2030         private final Object mImageLock = new Object();
2031         private int mReferenceCount;
2032 
ImageReferenceImpl(ParcelImage parcelImage)2033         private ImageReferenceImpl(ParcelImage parcelImage) {
2034             super(parcelImage);
2035             mReferenceCount = 1;
2036         }
2037 
2038         @Override
increment()2039         public boolean increment() {
2040             synchronized (mImageLock) {
2041                 if (mReferenceCount <= 0) {
2042                     return false;
2043                 }
2044                 mReferenceCount++;
2045             }
2046 
2047             return true;
2048         }
2049 
2050         @Override
decrement()2051         public boolean decrement() {
2052             synchronized (mImageLock) {
2053                 if (mReferenceCount <= 0) {
2054                     return false;
2055                 }
2056                 mReferenceCount--;
2057 
2058                 if (mReferenceCount <= 0) {
2059                     close();
2060                 }
2061             }
2062 
2063             return true;
2064         }
2065 
2066         @Override
get()2067         public Image get() {
2068             return this;
2069         }
2070     }
2071 
2072     private class ExtensionImage extends android.media.Image {
2073         private final android.hardware.camera2.extension.ParcelImage mParcelImage;
2074         private GraphicBuffer mGraphicBuffer;
2075         private ImageReader.ImagePlane[] mPlanes;
2076 
ExtensionImage(android.hardware.camera2.extension.ParcelImage parcelImage)2077         private ExtensionImage(android.hardware.camera2.extension.ParcelImage parcelImage) {
2078             mParcelImage = parcelImage;
2079             mIsImageValid = true;
2080         }
2081 
2082         @Override
getFormat()2083         public int getFormat() {
2084             throwISEIfImageIsInvalid();
2085             return mParcelImage.format;
2086         }
2087 
2088         @Override
getWidth()2089         public int getWidth() {
2090             throwISEIfImageIsInvalid();
2091             return mParcelImage.width;
2092         }
2093 
2094         @Override
getHardwareBuffer()2095         public HardwareBuffer getHardwareBuffer() {
2096             throwISEIfImageIsInvalid();
2097             return mParcelImage.buffer;
2098         }
2099 
2100         @Override
getHeight()2101         public int getHeight() {
2102             throwISEIfImageIsInvalid();
2103             return mParcelImage.height;
2104         }
2105 
2106         @Override
getTimestamp()2107         public long getTimestamp() {
2108             throwISEIfImageIsInvalid();
2109             return mParcelImage.timestamp;
2110         }
2111 
2112         @Override
getTransform()2113         public int getTransform() {
2114             throwISEIfImageIsInvalid();
2115             return mParcelImage.transform;
2116         }
2117 
2118         @Override
getScalingMode()2119         public int getScalingMode() {
2120             throwISEIfImageIsInvalid();
2121             return mParcelImage.scalingMode;
2122         }
2123 
2124         @Override
getPlanes()2125         public Plane[] getPlanes() {
2126             throwISEIfImageIsInvalid();
2127             if (mPlanes == null) {
2128                 int fenceFd = mParcelImage.fence != null ? mParcelImage.fence.getFd() : -1;
2129                 mGraphicBuffer = GraphicBuffer.createFromHardwareBuffer(mParcelImage.buffer);
2130                 mPlanes = ImageReader.initializeImagePlanes(mParcelImage.planeCount, mGraphicBuffer,
2131                         fenceFd, mParcelImage.format, mParcelImage.timestamp,
2132                         mParcelImage.transform, mParcelImage.scalingMode, mParcelImage.crop);
2133             }
2134             // Shallow copy is fine.
2135             return mPlanes.clone();
2136         }
2137 
2138         @Override
finalize()2139         protected final void finalize() throws Throwable {
2140             try {
2141                 close();
2142             } finally {
2143                 super.finalize();
2144             }
2145         }
2146 
2147         @Override
isAttachable()2148         public boolean isAttachable() {
2149             throwISEIfImageIsInvalid();
2150             // Clients must always detach parcelable images
2151             return true;
2152         }
2153 
2154         @Override
getCropRect()2155         public Rect getCropRect() {
2156             throwISEIfImageIsInvalid();
2157             return mParcelImage.crop;
2158         }
2159 
2160         @Override
close()2161         public void close() {
2162             mIsImageValid = false;
2163 
2164             if (mGraphicBuffer != null) {
2165                 try {
2166                     ImageReader.unlockGraphicBuffer(mGraphicBuffer);
2167                 } catch (RuntimeException e) {
2168                     e.printStackTrace();
2169                 }
2170                 mGraphicBuffer.destroy();
2171                 mGraphicBuffer = null;
2172             }
2173 
2174             if (mPlanes != null) {
2175                 mPlanes = null;
2176             }
2177 
2178             if (mParcelImage.buffer != null) {
2179                 mParcelImage.buffer.close();
2180                 mParcelImage.buffer = null;
2181             }
2182 
2183             if (mParcelImage.fence != null) {
2184                 try {
2185                     mParcelImage.fence.close();
2186                 } catch (IOException e) {
2187                     e.printStackTrace();
2188                 }
2189                 mParcelImage.fence = null;
2190             }
2191         }
2192     }
2193 
getCameraOutputConfig(Camera2OutputConfigImpl output)2194     private static CameraOutputConfig getCameraOutputConfig(Camera2OutputConfigImpl output) {
2195         CameraOutputConfig ret = new CameraOutputConfig();
2196         ret.outputId = new OutputConfigId();
2197         ret.outputId.id = output.getId();
2198         ret.physicalCameraId = output.getPhysicalCameraId();
2199         ret.surfaceGroupId = output.getSurfaceGroupId();
2200         ret.isMultiResolutionOutput = false;
2201         if (output instanceof SurfaceOutputConfigImpl) {
2202             SurfaceOutputConfigImpl surfaceConfig = (SurfaceOutputConfigImpl) output;
2203             ret.type = CameraOutputConfig.TYPE_SURFACE;
2204             ret.surface = surfaceConfig.getSurface();
2205         } else if (output instanceof ImageReaderOutputConfigImpl) {
2206             ImageReaderOutputConfigImpl imageReaderOutputConfig =
2207                     (ImageReaderOutputConfigImpl) output;
2208             ret.type = CameraOutputConfig.TYPE_IMAGEREADER;
2209             ret.size = new android.hardware.camera2.extension.Size();
2210             ret.size.width = imageReaderOutputConfig.getSize().getWidth();
2211             ret.size.height = imageReaderOutputConfig.getSize().getHeight();
2212             ret.imageFormat = imageReaderOutputConfig.getImageFormat();
2213             ret.capacity = imageReaderOutputConfig.getMaxImages();
2214         } else if (output instanceof MultiResolutionImageReaderOutputConfigImpl) {
2215             MultiResolutionImageReaderOutputConfigImpl multiResReaderConfig =
2216                     (MultiResolutionImageReaderOutputConfigImpl) output;
2217             ret.type = CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER;
2218             ret.imageFormat = multiResReaderConfig.getImageFormat();
2219             ret.capacity = multiResReaderConfig.getMaxImages();
2220             ret.isMultiResolutionOutput = true;
2221         } else {
2222             throw new IllegalStateException("Unknown output config type!");
2223         }
2224 
2225         return ret;
2226     }
2227 }
2228