1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.hardware.camera2.extension.IProcessResultImpl; 28 import android.media.Image; 29 import android.media.Image.Plane; 30 import android.media.ImageReader; 31 import android.media.ImageWriter; 32 import android.os.Handler; 33 import android.os.HandlerThread; 34 import android.os.IBinder; 35 import android.os.RemoteException; 36 import android.util.Log; 37 import android.view.Surface; 38 39 import java.nio.ByteBuffer; 40 import java.util.HashSet; 41 import java.util.Iterator; 42 import java.util.List; 43 import java.util.concurrent.ConcurrentLinkedQueue; 44 45 // Jpeg compress input YUV and queue back in the client target surface. 46 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 47 public final static String TAG = "CameraExtensionJpeg"; 48 private final static int JPEG_QUEUE_SIZE = 1; 49 private final static int JPEG_APP_SEGMENT_SIZE = 64 * 1024; 50 51 private final Handler mHandler; 52 private final HandlerThread mHandlerThread; 53 private final ICaptureProcessorImpl mProcessor; 54 55 private ImageReader mYuvReader = null; 56 private ImageReader mPostviewYuvReader = null; 57 private android.hardware.camera2.extension.Size mResolution = null; 58 private android.hardware.camera2.extension.Size mPostviewResolution = null; 59 private int mFormat = -1; 60 private Surface mOutputSurface = null; 61 private ImageWriter mOutputWriter = null; 62 private Surface mPostviewOutputSurface = null; 63 private ImageWriter mPostviewOutputWriter = null; 64 65 private static final class JpegParameters { 66 public HashSet<Long> mTimeStamps = new HashSet<>(); 67 public int mRotation = JPEG_DEFAULT_ROTATION; // CW multiple of 90 degrees 68 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 69 } 70 71 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 72 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)73 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 74 mProcessor = processor; 75 mHandlerThread = new HandlerThread(TAG); 76 mHandlerThread.start(); 77 mHandler = new Handler(mHandlerThread.getLooper()); 78 } 79 close()80 public void close() { 81 mHandlerThread.quitSafely(); 82 83 if (mOutputWriter != null) { 84 mOutputWriter.close(); 85 mOutputWriter = null; 86 } 87 88 if (mYuvReader != null) { 89 mYuvReader.close(); 90 mYuvReader = null; 91 } 92 } 93 getJpegParameters(List<CaptureBundle> captureBundles)94 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 95 JpegParameters ret = new JpegParameters(); 96 if (!captureBundles.isEmpty()) { 97 // The quality and orientation settings must be equal for requests in a burst 98 99 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 100 if (jpegQuality != null) { 101 ret.mQuality = jpegQuality; 102 } else { 103 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 104 } 105 106 Integer orientation = captureBundles.get(0).captureResult.get( 107 CaptureResult.JPEG_ORIENTATION); 108 if (orientation != null) { 109 // The jpeg encoder expects CCW rotation, convert from CW 110 ret.mRotation = (360 - (orientation % 360)) / 90; 111 } else { 112 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 113 } 114 115 for (CaptureBundle bundle : captureBundles) { 116 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 117 if (timeStamp != null) { 118 ret.mTimeStamps.add(timeStamp); 119 } else { 120 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 121 } 122 } 123 } 124 125 return ret; 126 } 127 128 /** 129 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 130 * <p> 131 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 132 * each channel of Y, Cb, Cr.<br> 133 * The Y plane is assumed to have the same width and height of the entire 134 * image.<br> 135 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 136 * have dimensions (floor(width / 2), floor(height / 2)).<br> 137 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 138 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 139 * from byteBuffer[x * pixel_stride + y * row_stride]. 140 * <p> 141 * The pre-compression transformation is applied as follows: 142 * <ol> 143 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 144 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 145 * (width, height) is a no-op.</li> 146 * <li>The rotation is applied counter-clockwise relative to the coordinate 147 * space of the image, so a CCW rotation will appear CW when the image is 148 * rendered in scanline order. Only rotations which are multiples of 149 * 90-degrees are suppored, so the parameter 'rot90' specifies which 150 * multiple of 90 to rotate the image.</li> 151 * </ol> 152 * 153 * @param width the width of the image to compress 154 * @param height the height of the image to compress 155 * @param yBuf the buffer containing the Y component of the image 156 * @param yPStride the stride between adjacent pixels in the same row in 157 * yBuf 158 * @param yRStride the stride between adjacent rows in yBuf 159 * @param cbBuf the buffer containing the Cb component of the image 160 * @param cbPStride the stride between adjacent pixels in the same row in 161 * cbBuf 162 * @param cbRStride the stride between adjacent rows in cbBuf 163 * @param crBuf the buffer containing the Cr component of the image 164 * @param crPStride the stride between adjacent pixels in the same row in 165 * crBuf 166 * @param crRStride the stride between adjacent rows in crBuf 167 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 168 * This must have enough capacity to store the result, or an 169 * error code will be returned. 170 * @param outBufCapacity the capacity of outBuf 171 * @param quality the jpeg-quality (1-100) to use 172 * @param cropLeft left-edge of the bounds of the image to crop to before 173 * rotation 174 * @param cropTop top-edge of the bounds of the image to crop to before 175 * rotation 176 * @param cropRight right-edge of the bounds of the image to crop to before 177 * rotation 178 * @param cropBottom bottom-edge of the bounds of the image to crop to 179 * before rotation 180 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 181 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)182 private static native int compressJpegFromYUV420pNative( 183 int width, int height, 184 ByteBuffer yBuf, int yPStride, int yRStride, 185 ByteBuffer cbBuf, int cbPStride, int cbRStride, 186 ByteBuffer crBuf, int crPStride, int crRStride, 187 ByteBuffer outBuf, int outBufCapacity, 188 int quality, 189 int cropLeft, int cropTop, int cropRight, int cropBottom, 190 int rot90); 191 192 @Override process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, boolean isPostviewRequested)193 public void process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, 194 boolean isPostviewRequested) 195 throws RemoteException { 196 JpegParameters jpegParams = getJpegParameters(captureBundle); 197 try { 198 mJpegParameters.add(jpegParams); 199 mProcessor.process(captureBundle, captureCallback, isPostviewRequested); 200 } catch (Exception e) { 201 mJpegParameters.remove(jpegParams); 202 throw e; 203 } 204 } 205 onOutputSurface(Surface surface, int format)206 public void onOutputSurface(Surface surface, int format) throws RemoteException { 207 if (format != ImageFormat.JPEG) { 208 Log.e(TAG, "Unsupported output format: " + format); 209 return; 210 } 211 mOutputSurface = surface; 212 initializePipeline(); 213 } 214 onPostviewOutputSurface(Surface surface)215 public void onPostviewOutputSurface(Surface surface) throws RemoteException { 216 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo = 217 CameraExtensionUtils.querySurface(surface); 218 if (postviewSurfaceInfo.mFormat != ImageFormat.JPEG) { 219 Log.e(TAG, "Unsupported output format: " + postviewSurfaceInfo.mFormat); 220 return; 221 } 222 mPostviewOutputSurface = surface; 223 initializePostviewPipeline(); 224 } 225 226 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size, android.hardware.camera2.extension.Size postviewSize)227 public void onResolutionUpdate(android.hardware.camera2.extension.Size size, 228 android.hardware.camera2.extension.Size postviewSize) 229 throws RemoteException { 230 mResolution = size; 231 mPostviewResolution = postviewSize; 232 initializePipeline(); 233 } 234 onImageFormatUpdate(int format)235 public void onImageFormatUpdate(int format) throws RemoteException { 236 if (format != ImageFormat.YUV_420_888) { 237 Log.e(TAG, "Unsupported input format: " + format); 238 return; 239 } 240 mFormat = format; 241 initializePipeline(); 242 } 243 initializePipeline()244 private void initializePipeline() throws RemoteException { 245 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 246 (mYuvReader == null)) { 247 // Jpeg/blobs are expected to be configured with (w*h)x1.5 + 64k Jpeg APP1 segment 248 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 249 ImageFormat.JPEG, 250 (mResolution.width * mResolution.height * 3)/2 + JPEG_APP_SEGMENT_SIZE, 1); 251 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, mFormat, 252 JPEG_QUEUE_SIZE); 253 mYuvReader.setOnImageAvailableListener( 254 new YuvCallback(mYuvReader, mOutputWriter), mHandler); 255 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 256 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 257 mProcessor.onImageFormatUpdate(mFormat); 258 } 259 } 260 initializePostviewPipeline()261 private void initializePostviewPipeline() throws RemoteException { 262 if ((mFormat != -1) && (mPostviewOutputSurface != null) && (mPostviewResolution != null) 263 && (mPostviewYuvReader == null)) { 264 // Jpeg/blobs are expected to be configured with (w*h)x1 265 mPostviewOutputWriter = ImageWriter.newInstance(mPostviewOutputSurface, 1/*maxImages*/, 266 ImageFormat.JPEG, mPostviewResolution.width * mPostviewResolution.height, 1); 267 mPostviewYuvReader = ImageReader.newInstance(mPostviewResolution.width, 268 mPostviewResolution.height, mFormat, JPEG_QUEUE_SIZE); 269 mPostviewYuvReader.setOnImageAvailableListener( 270 new YuvCallback(mPostviewYuvReader, mPostviewOutputWriter), mHandler); 271 mProcessor.onPostviewOutputSurface(mPostviewYuvReader.getSurface()); 272 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 273 mProcessor.onImageFormatUpdate(mFormat); 274 } 275 } 276 277 @Override asBinder()278 public IBinder asBinder() { 279 throw new UnsupportedOperationException("Binder IPC not supported!"); 280 } 281 282 private class YuvCallback implements ImageReader.OnImageAvailableListener { 283 private ImageReader mImageReader; 284 private ImageWriter mImageWriter; 285 YuvCallback(ImageReader imageReader, ImageWriter imageWriter)286 public YuvCallback(ImageReader imageReader, ImageWriter imageWriter) { 287 mImageReader = imageReader; 288 mImageWriter = imageWriter; 289 } 290 291 @Override onImageAvailable(ImageReader reader)292 public void onImageAvailable(ImageReader reader) { 293 Image yuvImage = null; 294 Image jpegImage = null; 295 try { 296 yuvImage = mImageReader.acquireNextImage(); 297 jpegImage = mImageWriter.dequeueInputImage(); 298 } catch (IllegalStateException e) { 299 if (yuvImage != null) { 300 yuvImage.close(); 301 } 302 if (jpegImage != null) { 303 jpegImage.close(); 304 } 305 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 306 return; 307 } 308 309 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 310 jpegBuffer.clear(); 311 // Jpeg/blobs are expected to be configured with (w*h)x1 312 int jpegCapacity = jpegImage.getWidth(); 313 314 Plane lumaPlane = yuvImage.getPlanes()[0]; 315 Plane crPlane = yuvImage.getPlanes()[1]; 316 Plane cbPlane = yuvImage.getPlanes()[2]; 317 318 ConcurrentLinkedQueue<JpegParameters> jpegParameters = 319 new ConcurrentLinkedQueue(mJpegParameters); 320 Iterator<JpegParameters> jpegIter = jpegParameters.iterator(); 321 JpegParameters jpegParams = null; 322 while(jpegIter.hasNext()) { 323 JpegParameters currentParams = jpegIter.next(); 324 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 325 jpegParams = currentParams; 326 jpegIter.remove(); 327 break; 328 } 329 } 330 if (jpegParams == null) { 331 if (jpegParameters.isEmpty()) { 332 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 333 + " and quality!"); 334 jpegParams = new JpegParameters(); 335 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 336 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 337 } else { 338 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 339 + " processed input!"); 340 Log.w(TAG, "Using values from the top of the queue!"); 341 jpegParams = jpegParameters.poll(); 342 } 343 } 344 345 compressJpegFromYUV420pNative( 346 yuvImage.getWidth(), yuvImage.getHeight(), 347 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 348 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 349 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 350 jpegBuffer, jpegCapacity, jpegParams.mQuality, 351 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 352 jpegParams.mRotation); 353 jpegImage.setTimestamp(yuvImage.getTimestamp()); 354 yuvImage.close(); 355 356 try { 357 mImageWriter.queueInputImage(jpegImage); 358 } catch (IllegalStateException e) { 359 Log.e(TAG, "Failed to queue encoded result!"); 360 } finally { 361 jpegImage.close(); 362 } 363 } 364 } 365 } 366