/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.camera.one.v2; import android.annotation.TargetApi; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.CaptureResult.Key; import android.hardware.camera2.TotalCaptureResult; import android.media.Image; import android.media.ImageReader; import android.os.Build; import android.os.Handler; import android.os.SystemClock; import android.util.Pair; import com.android.camera.debug.Log; import com.android.camera.debug.Log.Tag; import com.android.camera.util.ConcurrentSharedRingBuffer; import com.android.camera.util.ConcurrentSharedRingBuffer.PinStateListener; import com.android.camera.util.ConcurrentSharedRingBuffer.Selector; import com.android.camera.util.ConcurrentSharedRingBuffer.SwapTask; import com.android.camera.util.Task; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; /** * Implements {@link android.media.ImageReader.OnImageAvailableListener} and * {@link android.hardware.camera2.CameraCaptureSession.CaptureCallback} to * store the results of capture requests (both {@link Image}s and * {@link TotalCaptureResult}s in a ring-buffer from which they may be saved. *
* This also manages the lifecycle of {@link Image}s within the application as * they are passed in from the lower-level camera2 API. */ @TargetApi(Build.VERSION_CODES.LOLLIPOP) public class ImageCaptureManager extends CameraCaptureSession.CaptureCallback implements ImageReader.OnImageAvailableListener { /** * Callback to listen for changes to the ability to capture an existing * image from the internal ring-buffer. */ public interface CaptureReadyListener { /** * Called whenever the ability to capture an existing image from the * ring-buffer changes. Calls to {@link #tryCaptureExistingImage} are * more likely to succeed or fail depending on the value passed in to * this function. * * @param capturePossible true if capture is more-likely to be possible, * false if capture is less-likely to be possible. */ public void onReadyStateChange(boolean capturePossible); } /** * Callback for listening to changes to individual metadata values. */ public static interface MetadataChangeListener { /** * This will be called whenever a metadata value changes. * Implementations should not take too much time to execute since this * will be called faster than the camera's frame rate. * * @param key the {@link CaptureResult} key this listener listens for. * @param second the previous value, or null if no such value existed. * The type will be that associated with the * {@link android.hardware.camera2.CaptureResult.Key} this * listener is bound to. * @param newValue the new value. The type will be that associated with * the {@link android.hardware.camera2.CaptureResult.Key} * this listener is bound to. * @param result the CaptureResult containing the new value */ public void onImageMetadataChange(Key key, Object second, Object newValue, CaptureResult result); } /** * Callback for saving an image. */ public interface ImageCaptureListener { /** * Called with the {@link Image} and associated * {@link TotalCaptureResult}. A typical implementation would save this * to disk. *

* Note: Implementations must be thread-safe and must not close the * image. *

*/ public void onImageCaptured(Image image, TotalCaptureResult captureResult); } /** * Callback for placing constraints on which images to capture. See * {@link #tryCaptureExistingImage} and {@link #captureNextImage}. */ public static interface CapturedImageConstraint { /** * Implementations should return true if the provided * TotalCaptureResults satisfies constraints necessary for the intended * image capture. For example, a constraint may return false if * {@captureResult} indicates that the lens was moving during image * capture. * * @param captureResult The metadata associated with the image. * @return true if this image satisfies the constraint and can be * captured, false otherwise. */ boolean satisfiesConstraint(TotalCaptureResult captureResult); } /** * Holds an {@link Image} and {@link TotalCaptureResult} pair which may be * added asynchronously. */ private class CapturedImage { /** * The Image and TotalCaptureResult may be received at different times * (via the onImageAvailableListener and onCaptureProgressed callbacks, * respectively). */ private Image mImage = null; private TotalCaptureResult mMetadata = null; /** * Resets the object, closing and removing any existing image and * metadata. */ public void reset() { if (mImage != null) { mImage.close(); int numOpenImages = mNumOpenImages.decrementAndGet(); if (DEBUG_PRINT_OPEN_IMAGE_COUNT) { Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages); } } mImage = null; mMetadata = null; } /** * @return true if both the image and metadata are present, false * otherwise. */ public boolean isComplete() { return mImage != null && mMetadata != null; } /** * Adds the image. Note that this can only be called once before a * {@link #reset()} is necessary. * * @param image the {@Link Image} to add. */ public void addImage(Image image) { if (mImage != null) { throw new IllegalArgumentException( "Unable to add an Image when one already exists."); } mImage = image; } /** * Retrieves the {@link Image} if it has been added, returns null if it * is not available yet. */ public Image tryGetImage() { return mImage; } /** * Adds the metadata. Note that this can only be called once before a * {@link #reset()} is necessary. * * @param metadata the {@Link TotalCaptureResult} to add. */ public void addMetadata(TotalCaptureResult metadata) { if (mMetadata != null) { throw new IllegalArgumentException( "Unable to add a TotalCaptureResult when one already exists."); } mMetadata = metadata; } /** * Retrieves the {@link TotalCaptureResult} if it has been added, * returns null if it is not available yet. */ public TotalCaptureResult tryGetMetadata() { return mMetadata; } } private static final Tag TAG = new Tag("ZSLImageListener"); /** * If true, the number of open images will be printed to LogCat every time * an image is opened or closed. */ private static final boolean DEBUG_PRINT_OPEN_IMAGE_COUNT = false; /** * The maximum duration for an onImageAvailable() callback before debugging * output is printed. This is a little under 1/30th of a second to enable * detecting jank in the preview stream caused by {@link #onImageAvailable} * taking too long to return. */ private static final long DEBUG_MAX_IMAGE_CALLBACK_DUR = 25; /** * If spacing between onCaptureCompleted() callbacks is lower than this * value, camera operations at the Java level have stalled, and are now * catching up. In milliseconds. */ private static final long DEBUG_INTERFRAME_STALL_WARNING = 5; /** * Last called to onCaptureCompleted() in SystemClock.uptimeMillis(). */ private long mDebugLastOnCaptureCompletedMillis = 0; /** * Number of frames in a row exceeding DEBUG_INTERFRAME_STALL_WARNING. */ private long mDebugStalledFrameCount = 0; /** * Stores the ring-buffer of captured images.
* Note that this takes care of thread-safe reference counting of images to * ensure that they are never leaked by the app. */ private final ConcurrentSharedRingBuffer mCapturedImageBuffer; /** Track the number of open images for debugging purposes. */ private final AtomicInteger mNumOpenImages = new AtomicInteger(0); /** * The handler used to invoke light-weight listeners: * {@link CaptureReadyListener} and {@link MetadataChangeListener}. */ private final Handler mListenerHandler; /** * The executor used to invoke {@link ImageCaptureListener}. Note that this * is different from mListenerHandler because a typical ImageCaptureListener * will compress the image to jpeg, and we may wish to execute these tasks * on multiple threads. */ private final Executor mImageCaptureListenerExecutor; /** * The set of constraints which must be satisfied for a newly acquired image * to be captured and sent to {@link #mPendingImageCaptureListener}. null if * there is no pending capture request. */ private List mPendingImageCaptureConstraints; /** * The callback to be invoked upon successfully capturing a newly-acquired * image which satisfies {@link #mPendingImageCaptureConstraints}. null if * there is no pending capture request. */ private ImageCaptureManager.ImageCaptureListener mPendingImageCaptureListener; /** * Map from CaptureResult key to the frame number of the capture result * containing the most recent value for this key and the most recent value * of the key. */ private final Map, Pair> mMetadata = new ConcurrentHashMap, Pair>(); /** * The set of callbacks to be invoked when an entry in {@link #mMetadata} is * changed. */ private final Map, Set> mMetadataChangeListeners = new ConcurrentHashMap, Set>(); /** * @param maxImages the maximum number of images provided by the * {@link ImageReader}. This must be greater than 2. * @param listenerHandler the handler on which to invoke listeners. Note * that this should probably be on a different thread than the * one used for camera operations, such as capture requests and * OnImageAvailable listeners, to avoid stalling the preview. * @param imageCaptureCallbackExecutor the executor on which to invoke image * capture listeners, {@link ImageCaptureListener}. */ ImageCaptureManager(int maxImages, Handler listenerHandler, Executor imageCaptureCallbackExecutor) { // Ensure that there are always 2 images available for the framework to // continue processing frames. // TODO Could we make this tighter? mCapturedImageBuffer = new ConcurrentSharedRingBuffer( maxImages - 2); mListenerHandler = listenerHandler; mImageCaptureListenerExecutor = imageCaptureCallbackExecutor; } /** * See {@link CaptureReadyListener}. */ public void setCaptureReadyListener(final CaptureReadyListener listener) { mCapturedImageBuffer.setListener(mListenerHandler, new PinStateListener() { @Override public void onPinStateChange(boolean pinsAvailable) { listener.onReadyStateChange(pinsAvailable); } }); } /** * Adds a metadata stream listener associated with the given key. * * @param key the key of the metadata to track. * @param listener the listener to be invoked when the value associated with * key changes. */ public void addMetadataChangeListener(Key key, MetadataChangeListener listener) { if (!mMetadataChangeListeners.containsKey(key)) { // Listeners may be added to this set from a different thread than // that which must iterate over this set to invoke the listeners. // Therefore, we need a thread save hash set. mMetadataChangeListeners.put(key, Collections.newSetFromMap(new ConcurrentHashMap< ImageCaptureManager.MetadataChangeListener, Boolean>())); } mMetadataChangeListeners.get(key).add(listener); } /** * Removes the metadata stream listener associated with the given key. * * @param key the key associated with the metadata to track. * @param listener the listener to be invoked when the value associated with * key changes. * @return true if the listener was removed, false if no such listener had * been added. */ public boolean removeMetadataChangeListener(Key key, MetadataChangeListener listener) { if (!mMetadataChangeListeners.containsKey(key)) { return false; } else { return mMetadataChangeListeners.get(key).remove(listener); } } @Override public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, final CaptureResult partialResult) { long frameNumber = partialResult.getFrameNumber(); // Update mMetadata for whichever keys are present, if this frame is // supplying newer values. for (final Key key : partialResult.getKeys()) { Pair oldEntry = mMetadata.get(key); final Object oldValue = (oldEntry != null) ? oldEntry.second : null; boolean newerValueAlreadyExists = oldEntry != null && frameNumber < oldEntry.first; if (newerValueAlreadyExists) { continue; } final Object newValue = partialResult.get(key); mMetadata.put(key, new Pair(frameNumber, newValue)); // If the value has changed, call the appropriate listeners, if // any exist. if (oldValue == newValue || !mMetadataChangeListeners.containsKey(key)) { continue; } for (final MetadataChangeListener listener : mMetadataChangeListeners.get(key)) { Log.v(TAG, "Dispatching to metadata change listener for key: " + key.toString()); mListenerHandler.post(new Runnable() { @Override public void run() { listener.onImageMetadataChange(key, oldValue, newValue, partialResult); } }); } } } @Override public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, final TotalCaptureResult result) { final long timestamp = result.get(TotalCaptureResult.SENSOR_TIMESTAMP); // Detect camera thread stall. long now = SystemClock.uptimeMillis(); if (now - mDebugLastOnCaptureCompletedMillis < DEBUG_INTERFRAME_STALL_WARNING) { Log.e(TAG, "Camera thread has stalled for " + ++mDebugStalledFrameCount + " frames at # " + result.getFrameNumber() + "."); } else { mDebugStalledFrameCount = 0; } mDebugLastOnCaptureCompletedMillis = now; // Find the CapturedImage in the ring-buffer and attach the // TotalCaptureResult to it. // See documentation for swapLeast() for details. boolean swapSuccess = mCapturedImageBuffer.swapLeast(timestamp, new SwapTask() { @Override public CapturedImage create() { CapturedImage image = new CapturedImage(); image.addMetadata(result); return image; } @Override public CapturedImage swap(CapturedImage oldElement) { oldElement.reset(); oldElement.addMetadata(result); return oldElement; } @Override public void update(CapturedImage existingElement) { existingElement.addMetadata(result); } }); if (!swapSuccess) { // Do nothing on failure to swap in. Log.v(TAG, "Unable to add new image metadata to ring-buffer."); } tryExecutePendingCaptureRequest(timestamp); } @Override public void onImageAvailable(ImageReader reader) { long startTime = SystemClock.currentThreadTimeMillis(); final Image img = reader.acquireLatestImage(); if (img != null) { int numOpenImages = mNumOpenImages.incrementAndGet(); if (DEBUG_PRINT_OPEN_IMAGE_COUNT) { Log.v(TAG, "Acquired an image. Number of open images = " + numOpenImages); } // Try to place the newly-acquired image into the ring buffer. boolean swapSuccess = mCapturedImageBuffer.swapLeast( img.getTimestamp(), new SwapTask() { @Override public CapturedImage create() { CapturedImage image = new CapturedImage(); image.addImage(img); return image; } @Override public CapturedImage swap(CapturedImage oldElement) { oldElement.reset(); oldElement.addImage(img); return oldElement; } @Override public void update(CapturedImage existingElement) { existingElement.addImage(img); } }); if (!swapSuccess) { // If we were unable to save the image to the ring buffer, we // must close it now. // We should only get here if the ring buffer is closed. img.close(); numOpenImages = mNumOpenImages.decrementAndGet(); if (DEBUG_PRINT_OPEN_IMAGE_COUNT) { Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages); } } tryExecutePendingCaptureRequest(img.getTimestamp()); long endTime = SystemClock.currentThreadTimeMillis(); long totTime = endTime - startTime; if (totTime > DEBUG_MAX_IMAGE_CALLBACK_DUR) { // If it takes too long to swap elements, we will start skipping // preview frames, resulting in visible jank. Log.v(TAG, "onImageAvailable() took " + totTime + "ms"); } } } /** * Closes the listener, eventually freeing all currently-held {@link Image} * s. */ public void close() { try { mCapturedImageBuffer.close(new Task() { @Override public void run(CapturedImage e) { e.reset(); } }); } catch (InterruptedException e) { e.printStackTrace(); } } /** * Sets the pending image capture request, overriding any previous calls to * {@link #captureNextImage} which have not yet been resolved. When the next * available image which satisfies the given constraints can be captured, * onImageCaptured will be invoked. * * @param onImageCaptured the callback which will be invoked with the * captured image. * @param constraints the set of constraints which must be satisfied in * order for the image to be captured. */ public void captureNextImage(final ImageCaptureListener onImageCaptured, final List constraints) { mPendingImageCaptureListener = onImageCaptured; mPendingImageCaptureConstraints = constraints; } /** * Tries to resolve any pending image capture requests. * * @param newImageTimestamp the timestamp of a newly-acquired image which * should be captured if appropriate and possible. */ private void tryExecutePendingCaptureRequest(long newImageTimestamp) { if (mPendingImageCaptureListener != null) { final Pair pinnedImage = mCapturedImageBuffer.tryPin( newImageTimestamp); if (pinnedImage != null) { CapturedImage image = pinnedImage.second; if (!image.isComplete()) { mCapturedImageBuffer.release(pinnedImage.first); return; } // Check to see if the image satisfies all constraints. TotalCaptureResult captureResult = image.tryGetMetadata(); if (mPendingImageCaptureConstraints != null) { for (CapturedImageConstraint constraint : mPendingImageCaptureConstraints) { if (!constraint.satisfiesConstraint(captureResult)) { mCapturedImageBuffer.release(pinnedImage.first); return; } } } // If we get here, the image satisfies all the necessary // constraints. if (tryExecuteCaptureOrRelease(pinnedImage, mPendingImageCaptureListener)) { // If we successfully handed the image off to the callback, // remove the pending // capture request. mPendingImageCaptureListener = null; mPendingImageCaptureConstraints = null; } } } } /** * Tries to capture an existing image from the ring-buffer, if one exists * that satisfies the given constraint and can be pinned. * * @return true if the image could be captured, false otherwise. */ public boolean tryCaptureExistingImage(final ImageCaptureListener onImageCaptured, final List constraints) { // The selector to use in choosing the image to capture. Selector selector; if (constraints == null || constraints.isEmpty()) { // If there are no constraints, use a trivial Selector. selector = new Selector() { @Override public boolean select(CapturedImage image) { return true; } }; } else { // If there are constraints, create a Selector which will return // true if all constraints // are satisfied. selector = new Selector() { @Override public boolean select(CapturedImage e) { // If this image already has metadata associated with it, // then use it. // Otherwise, we can't block until it's available, so assume // it doesn't // satisfy the required constraints. TotalCaptureResult captureResult = e.tryGetMetadata(); if (captureResult == null || e.tryGetImage() == null) { return false; } for (CapturedImageConstraint constraint : constraints) { if (!constraint.satisfiesConstraint(captureResult)) { return false; } } return true; } }; } // Acquire a lock (pin) on the most recent (greatest-timestamp) image in // the ring buffer which satisfies our constraints. // Note that this must be released as soon as we are done with it. final Pair toCapture = mCapturedImageBuffer.tryPinGreatestSelected( selector); return tryExecuteCaptureOrRelease(toCapture, onImageCaptured); } /** * Tries to execute the image capture callback with the pinned CapturedImage * provided. * * @param toCapture The pinned CapturedImage to pass to the callback, or * release on failure. * @param callback The callback to execute. * @return true upon success, false upon failure and the release of the * pinned image. */ private boolean tryExecuteCaptureOrRelease(final Pair toCapture, final ImageCaptureListener callback) { if (toCapture == null) { return false; } else { try { mImageCaptureListenerExecutor.execute(new Runnable() { @Override public void run() { try { CapturedImage img = toCapture.second; callback.onImageCaptured(img.tryGetImage(), img.tryGetMetadata()); } finally { mCapturedImageBuffer.release(toCapture.first); } } }); } catch (RejectedExecutionException e) { // We may get here if the thread pool has been closed. mCapturedImageBuffer.release(toCapture.first); return false; } return true; } } }