1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.screenrecord; 18 19 import static android.content.Context.MEDIA_PROJECTION_SERVICE; 20 21 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.INTERNAL; 22 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC; 23 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC_AND_INTERNAL; 24 25 import android.annotation.Nullable; 26 import android.content.ContentResolver; 27 import android.content.ContentValues; 28 import android.content.Context; 29 import android.graphics.Bitmap; 30 import android.hardware.display.DisplayManager; 31 import android.hardware.display.VirtualDisplay; 32 import android.media.MediaCodec; 33 import android.media.MediaCodecInfo; 34 import android.media.MediaFormat; 35 import android.media.MediaMuxer; 36 import android.media.MediaRecorder; 37 import android.media.ThumbnailUtils; 38 import android.media.projection.IMediaProjection; 39 import android.media.projection.IMediaProjectionManager; 40 import android.media.projection.MediaProjection; 41 import android.media.projection.MediaProjectionManager; 42 import android.net.Uri; 43 import android.os.Handler; 44 import android.os.IBinder; 45 import android.os.RemoteException; 46 import android.os.ServiceManager; 47 import android.provider.MediaStore; 48 import android.util.DisplayMetrics; 49 import android.util.Log; 50 import android.util.Size; 51 import android.view.Surface; 52 import android.view.WindowManager; 53 54 import com.android.systemui.media.MediaProjectionCaptureTarget; 55 56 import java.io.Closeable; 57 import java.io.File; 58 import java.io.IOException; 59 import java.io.OutputStream; 60 import java.nio.file.Files; 61 import java.text.SimpleDateFormat; 62 import java.util.ArrayList; 63 import java.util.Date; 64 import java.util.List; 65 66 /** 67 * Recording screen and mic/internal audio 68 */ 69 public class ScreenMediaRecorder extends MediaProjection.Callback { 70 private static final int TOTAL_NUM_TRACKS = 1; 71 private static final int VIDEO_FRAME_RATE = 30; 72 private static final int VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO = 6; 73 private static final int AUDIO_BIT_RATE = 196000; 74 private static final int AUDIO_SAMPLE_RATE = 44100; 75 private static final int MAX_DURATION_MS = 60 * 60 * 1000; 76 private static final long MAX_FILESIZE_BYTES = 5000000000L; 77 private static final String TAG = "ScreenMediaRecorder"; 78 79 80 private File mTempVideoFile; 81 private File mTempAudioFile; 82 private MediaProjection mMediaProjection; 83 private Surface mInputSurface; 84 private VirtualDisplay mVirtualDisplay; 85 private MediaRecorder mMediaRecorder; 86 private int mUser; 87 private ScreenRecordingMuxer mMuxer; 88 private ScreenInternalAudioRecorder mAudio; 89 private ScreenRecordingAudioSource mAudioSource; 90 private final MediaProjectionCaptureTarget mCaptureRegion; 91 private final Handler mHandler; 92 93 private Context mContext; 94 ScreenMediaRecorderListener mListener; 95 ScreenMediaRecorder(Context context, Handler handler, int user, ScreenRecordingAudioSource audioSource, MediaProjectionCaptureTarget captureRegion, ScreenMediaRecorderListener listener)96 public ScreenMediaRecorder(Context context, Handler handler, 97 int user, ScreenRecordingAudioSource audioSource, 98 MediaProjectionCaptureTarget captureRegion, 99 ScreenMediaRecorderListener listener) { 100 mContext = context; 101 mHandler = handler; 102 mUser = user; 103 mCaptureRegion = captureRegion; 104 mListener = listener; 105 mAudioSource = audioSource; 106 } 107 prepare()108 private void prepare() throws IOException, RemoteException, RuntimeException { 109 //Setup media projection 110 IBinder b = ServiceManager.getService(MEDIA_PROJECTION_SERVICE); 111 IMediaProjectionManager mediaService = 112 IMediaProjectionManager.Stub.asInterface(b); 113 IMediaProjection proj = null; 114 proj = mediaService.createProjection(mUser, mContext.getPackageName(), 115 MediaProjectionManager.TYPE_SCREEN_CAPTURE, false); 116 IMediaProjection projection = IMediaProjection.Stub.asInterface(proj.asBinder()); 117 if (mCaptureRegion != null) { 118 projection.setLaunchCookie(mCaptureRegion.getLaunchCookie()); 119 } 120 mMediaProjection = new MediaProjection(mContext, projection); 121 mMediaProjection.registerCallback(this, mHandler); 122 123 File cacheDir = mContext.getCacheDir(); 124 cacheDir.mkdirs(); 125 mTempVideoFile = File.createTempFile("temp", ".mp4", cacheDir); 126 127 // Set up media recorder 128 mMediaRecorder = new MediaRecorder(); 129 130 // Set up audio source 131 if (mAudioSource == MIC) { 132 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT); 133 } 134 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 135 136 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); 137 138 139 // Set up video 140 DisplayMetrics metrics = new DisplayMetrics(); 141 WindowManager wm = (WindowManager) mContext.getSystemService(Context.WINDOW_SERVICE); 142 wm.getDefaultDisplay().getRealMetrics(metrics); 143 int refreshRate = (int) wm.getDefaultDisplay().getRefreshRate(); 144 int[] dimens = getSupportedSize(metrics.widthPixels, metrics.heightPixels, refreshRate); 145 int width = dimens[0]; 146 int height = dimens[1]; 147 refreshRate = dimens[2]; 148 int vidBitRate = width * height * refreshRate / VIDEO_FRAME_RATE 149 * VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO; 150 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 151 mMediaRecorder.setVideoEncodingProfileLevel( 152 MediaCodecInfo.CodecProfileLevel.AVCProfileHigh, 153 MediaCodecInfo.CodecProfileLevel.AVCLevel3); 154 mMediaRecorder.setVideoSize(width, height); 155 mMediaRecorder.setVideoFrameRate(refreshRate); 156 mMediaRecorder.setVideoEncodingBitRate(vidBitRate); 157 mMediaRecorder.setMaxDuration(MAX_DURATION_MS); 158 mMediaRecorder.setMaxFileSize(MAX_FILESIZE_BYTES); 159 160 // Set up audio 161 if (mAudioSource == MIC) { 162 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.HE_AAC); 163 mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS); 164 mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE); 165 mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE); 166 } 167 168 mMediaRecorder.setOutputFile(mTempVideoFile); 169 mMediaRecorder.prepare(); 170 // Create surface 171 mInputSurface = mMediaRecorder.getSurface(); 172 mVirtualDisplay = mMediaProjection.createVirtualDisplay( 173 "Recording Display", 174 width, 175 height, 176 metrics.densityDpi, 177 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 178 mInputSurface, 179 new VirtualDisplay.Callback() { 180 @Override 181 public void onStopped() { 182 onStop(); 183 } 184 }, 185 mHandler); 186 187 mMediaRecorder.setOnInfoListener((mr, what, extra) -> mListener.onInfo(mr, what, extra)); 188 if (mAudioSource == INTERNAL || 189 mAudioSource == MIC_AND_INTERNAL) { 190 mTempAudioFile = File.createTempFile("temp", ".aac", 191 mContext.getCacheDir()); 192 mAudio = new ScreenInternalAudioRecorder(mTempAudioFile.getAbsolutePath(), 193 mMediaProjection, mAudioSource == MIC_AND_INTERNAL); 194 } 195 196 } 197 198 /** 199 * Find the highest supported screen resolution and refresh rate for the given dimensions on 200 * this device, up to actual size and given rate. 201 * If possible this will return the same values as given, but values may be smaller on some 202 * devices. 203 * 204 * @param screenWidth Actual pixel width of screen 205 * @param screenHeight Actual pixel height of screen 206 * @param refreshRate Desired refresh rate 207 * @return array with supported width, height, and refresh rate 208 */ getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate)209 private int[] getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate) 210 throws IOException { 211 String videoType = MediaFormat.MIMETYPE_VIDEO_AVC; 212 213 // Get max size from the decoder, to ensure recordings will be playable on device 214 MediaCodec decoder = MediaCodec.createDecoderByType(videoType); 215 MediaCodecInfo.VideoCapabilities vc = decoder.getCodecInfo() 216 .getCapabilitiesForType(videoType).getVideoCapabilities(); 217 decoder.release(); 218 219 // Check if we can support screen size as-is 220 int width = vc.getSupportedWidths().getUpper(); 221 int height = vc.getSupportedHeights().getUpper(); 222 223 int screenWidthAligned = screenWidth; 224 if (screenWidthAligned % vc.getWidthAlignment() != 0) { 225 screenWidthAligned -= (screenWidthAligned % vc.getWidthAlignment()); 226 } 227 int screenHeightAligned = screenHeight; 228 if (screenHeightAligned % vc.getHeightAlignment() != 0) { 229 screenHeightAligned -= (screenHeightAligned % vc.getHeightAlignment()); 230 } 231 232 if (width >= screenWidthAligned && height >= screenHeightAligned 233 && vc.isSizeSupported(screenWidthAligned, screenHeightAligned)) { 234 // Desired size is supported, now get the rate 235 int maxRate = vc.getSupportedFrameRatesFor(screenWidthAligned, 236 screenHeightAligned).getUpper().intValue(); 237 238 if (maxRate < refreshRate) { 239 refreshRate = maxRate; 240 } 241 Log.d(TAG, "Screen size supported at rate " + refreshRate); 242 return new int[]{screenWidthAligned, screenHeightAligned, refreshRate}; 243 } 244 245 // Otherwise, resize for max supported size 246 double scale = Math.min(((double) width / screenWidth), 247 ((double) height / screenHeight)); 248 249 int scaledWidth = (int) (screenWidth * scale); 250 int scaledHeight = (int) (screenHeight * scale); 251 if (scaledWidth % vc.getWidthAlignment() != 0) { 252 scaledWidth -= (scaledWidth % vc.getWidthAlignment()); 253 } 254 if (scaledHeight % vc.getHeightAlignment() != 0) { 255 scaledHeight -= (scaledHeight % vc.getHeightAlignment()); 256 } 257 258 // Find max supported rate for size 259 int maxRate = vc.getSupportedFrameRatesFor(scaledWidth, scaledHeight) 260 .getUpper().intValue(); 261 if (maxRate < refreshRate) { 262 refreshRate = maxRate; 263 } 264 265 Log.d(TAG, "Resized by " + scale + ": " + scaledWidth + ", " + scaledHeight 266 + ", " + refreshRate); 267 return new int[]{scaledWidth, scaledHeight, refreshRate}; 268 } 269 270 /** 271 * Start screen recording 272 */ start()273 void start() throws IOException, RemoteException, RuntimeException { 274 Log.d(TAG, "start recording"); 275 prepare(); 276 mMediaRecorder.start(); 277 recordInternalAudio(); 278 } 279 280 /** 281 * End screen recording, throws an exception if stopping recording failed 282 */ end()283 void end() throws IOException { 284 Closer closer = new Closer(); 285 286 // MediaRecorder might throw RuntimeException if stopped immediately after starting 287 // We should remove the recording in this case as it will be invalid 288 closer.register(mMediaRecorder::stop); 289 closer.register(mMediaRecorder::release); 290 closer.register(mInputSurface::release); 291 closer.register(mVirtualDisplay::release); 292 closer.register(mMediaProjection::stop); 293 closer.register(this::stopInternalAudioRecording); 294 295 closer.close(); 296 297 mMediaRecorder = null; 298 mMediaProjection = null; 299 300 Log.d(TAG, "end recording"); 301 } 302 303 @Override onStop()304 public void onStop() { 305 Log.d(TAG, "The system notified about stopping the projection"); 306 mListener.onStopped(); 307 } 308 stopInternalAudioRecording()309 private void stopInternalAudioRecording() { 310 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 311 mAudio.end(); 312 mAudio = null; 313 } 314 } 315 recordInternalAudio()316 private void recordInternalAudio() throws IllegalStateException { 317 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 318 mAudio.start(); 319 } 320 } 321 322 /** 323 * Store recorded video 324 */ save()325 protected SavedRecording save() throws IOException, IllegalStateException { 326 String fileName = new SimpleDateFormat("'screen-'yyyyMMdd-HHmmss'.mp4'") 327 .format(new Date()); 328 329 ContentValues values = new ContentValues(); 330 values.put(MediaStore.Video.Media.DISPLAY_NAME, fileName); 331 values.put(MediaStore.Video.Media.MIME_TYPE, "video/mp4"); 332 values.put(MediaStore.Video.Media.DATE_ADDED, System.currentTimeMillis()); 333 values.put(MediaStore.Video.Media.DATE_TAKEN, System.currentTimeMillis()); 334 335 ContentResolver resolver = mContext.getContentResolver(); 336 Uri collectionUri = MediaStore.Video.Media.getContentUri( 337 MediaStore.VOLUME_EXTERNAL_PRIMARY); 338 Uri itemUri = resolver.insert(collectionUri, values); 339 340 Log.d(TAG, itemUri.toString()); 341 if (mAudioSource == MIC_AND_INTERNAL || mAudioSource == INTERNAL) { 342 try { 343 Log.d(TAG, "muxing recording"); 344 File file = File.createTempFile("temp", ".mp4", 345 mContext.getCacheDir()); 346 mMuxer = new ScreenRecordingMuxer(MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4, 347 file.getAbsolutePath(), 348 mTempVideoFile.getAbsolutePath(), 349 mTempAudioFile.getAbsolutePath()); 350 mMuxer.mux(); 351 mTempVideoFile.delete(); 352 mTempVideoFile = file; 353 } catch (IOException e) { 354 Log.e(TAG, "muxing recording " + e.getMessage()); 355 e.printStackTrace(); 356 } 357 } 358 359 // Add to the mediastore 360 OutputStream os = resolver.openOutputStream(itemUri, "w"); 361 Files.copy(mTempVideoFile.toPath(), os); 362 os.close(); 363 if (mTempAudioFile != null) mTempAudioFile.delete(); 364 DisplayMetrics metrics = mContext.getResources().getDisplayMetrics(); 365 Size size = new Size(metrics.widthPixels, metrics.heightPixels); 366 SavedRecording recording = new SavedRecording(itemUri, mTempVideoFile, size); 367 mTempVideoFile.delete(); 368 return recording; 369 } 370 371 /** 372 * Release the resources without saving the data 373 */ release()374 protected void release() { 375 if (mTempVideoFile != null) { 376 mTempVideoFile.delete(); 377 } 378 if (mTempAudioFile != null) { 379 mTempAudioFile.delete(); 380 } 381 } 382 383 /** 384 * Object representing the recording 385 */ 386 public class SavedRecording { 387 388 private Uri mUri; 389 private Bitmap mThumbnailBitmap; 390 SavedRecording(Uri uri, File file, Size thumbnailSize)391 protected SavedRecording(Uri uri, File file, Size thumbnailSize) { 392 mUri = uri; 393 try { 394 mThumbnailBitmap = ThumbnailUtils.createVideoThumbnail( 395 file, thumbnailSize, null); 396 } catch (IOException e) { 397 Log.e(TAG, "Error creating thumbnail", e); 398 } 399 } 400 getUri()401 public Uri getUri() { 402 return mUri; 403 } 404 getThumbnail()405 public @Nullable Bitmap getThumbnail() { 406 return mThumbnailBitmap; 407 } 408 } 409 410 interface ScreenMediaRecorderListener { 411 /** 412 * Called to indicate an info or a warning during recording. 413 * See {@link MediaRecorder.OnInfoListener} for the full description. 414 */ onInfo(MediaRecorder mr, int what, int extra)415 void onInfo(MediaRecorder mr, int what, int extra); 416 417 /** 418 * Called when the recording stopped by the system. 419 * For example, this might happen when doing partial screen sharing of an app 420 * and the app that is being captured is closed. 421 */ onStopped()422 void onStopped(); 423 } 424 425 /** 426 * Allows to register multiple {@link Closeable} objects and close them all by calling 427 * {@link Closer#close}. If there is an exception thrown during closing of one 428 * of the registered closeables it will continue trying closing the rest closeables. 429 * If there are one or more exceptions thrown they will be re-thrown at the end. 430 * In case of multiple exceptions only the first one will be thrown and all the rest 431 * will be printed. 432 */ 433 private static class Closer implements Closeable { 434 private final List<Closeable> mCloseables = new ArrayList<>(); 435 register(Closeable closeable)436 void register(Closeable closeable) { 437 mCloseables.add(closeable); 438 } 439 440 @Override close()441 public void close() throws IOException { 442 Throwable throwable = null; 443 444 for (int i = 0; i < mCloseables.size(); i++) { 445 Closeable closeable = mCloseables.get(i); 446 447 try { 448 closeable.close(); 449 } catch (Throwable e) { 450 if (throwable == null) { 451 throwable = e; 452 } else { 453 e.printStackTrace(); 454 } 455 } 456 } 457 458 if (throwable != null) { 459 if (throwable instanceof IOException) { 460 throw (IOException) throwable; 461 } 462 463 if (throwable instanceof RuntimeException) { 464 throw (RuntimeException) throwable; 465 } 466 467 throw (Error) throwable; 468 } 469 } 470 } 471 } 472