camera2: add reprocess support

Add support to create input stream, submit reprocess capture requests,
and receive reprocess capture results.

Change-Id: Iee2d4313f3d52616a484eaea7a28f5ef9d8a674b

8 files changed, 147 insertions, 19 deletions

diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp
index 8236788a2d..dc752a6c75 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -62,6 +62,7 @@ Camera3Device::Camera3Device(int id):
         mUsePartialResult(false),
         mNumPartialResults(1),
         mNextResultFrameNumber(0),
+        mNextReprocessResultFrameNumber(0),
         mNextShutterFrameNumber(0),
         mListener(NULL)
 {
@@ -201,6 +202,17 @@ status_t Camera3Device::initialize(CameraModule *module)
         }
     }
 
+    camera_metadata_entry configs =
+            mDeviceInfo.find(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS);
+    for (uint32_t i = 0; i < configs.count; i += 4) {
+        if (configs.data.i32[i] == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED &&
+                configs.data.i32[i + 3] ==
+                ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_INPUT) {
+            mSupportedOpaqueInputSizes.add(Size(configs.data.i32[i + 1],
+                    configs.data.i32[i + 2]));
+        }
+    }
+
     return OK;
 }
 
@@ -1019,6 +1031,20 @@ status_t Camera3Device::configureStreams() {
     return configureStreamsLocked();
 }
 
+status_t Camera3Device::getInputBufferProducer(
+        sp<IGraphicBufferProducer> *producer) {
+    Mutex::Autolock il(mInterfaceLock);
+    Mutex::Autolock l(mLock);
+
+    if (producer == NULL) {
+        return BAD_VALUE;
+    } else if (mInputStream == NULL) {
+        return INVALID_OPERATION;
+    }
+
+    return mInputStream->getInputBufferProducer(producer);
+}
+
 status_t Camera3Device::createDefaultRequest(int templateId,
         CameraMetadata *request) {
     ATRACE_CALL();
@@ -1423,6 +1449,17 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest(
     return newRequest;
 }
 
+bool Camera3Device::isOpaqueInputSizeSupported(uint32_t width, uint32_t height) {
+    for (uint32_t i = 0; i < mSupportedOpaqueInputSizes.size(); i++) {
+        Size size = mSupportedOpaqueInputSizes[i];
+        if (size.width == width && size.height == height) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 status_t Camera3Device::configureStreamsLocked() {
     ATRACE_CALL();
     status_t res;
@@ -1947,20 +1984,31 @@ void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) {
 void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata,
         CaptureResultExtras &resultExtras,
         CameraMetadata &collectedPartialResult,
-        uint32_t frameNumber) {
+        uint32_t frameNumber,
+        bool reprocess) {
     if (pendingMetadata.isEmpty())
         return;
 
     Mutex::Autolock l(mOutputLock);
 
     // TODO: need to track errors for tighter bounds on expected frame number
-    if (frameNumber < mNextResultFrameNumber) {
-        SET_ERR("Out-of-order capture result metadata submitted! "
+    if (reprocess) {
+        if (frameNumber < mNextReprocessResultFrameNumber) {
+            SET_ERR("Out-of-order reprocess capture result metadata submitted! "
                 "(got frame number %d, expecting %d)",
-                frameNumber, mNextResultFrameNumber);
-        return;
+                frameNumber, mNextReprocessResultFrameNumber);
+            return;
+        }
+        mNextReprocessResultFrameNumber = frameNumber + 1;
+    } else {
+        if (frameNumber < mNextResultFrameNumber) {
+            SET_ERR("Out-of-order capture result metadata submitted! "
+                    "(got frame number %d, expecting %d)",
+                    frameNumber, mNextResultFrameNumber);
+            return;
+        }
+        mNextResultFrameNumber = frameNumber + 1;
     }
-    mNextResultFrameNumber = frameNumber + 1;
 
     CaptureResult captureResult;
     captureResult.mResultExtras = resultExtras;
@@ -2170,7 +2218,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
                 CameraMetadata metadata;
                 metadata = result->result;
                 sendCaptureResult(metadata, request.resultExtras,
-                    collectedPartialResult, frameNumber);
+                    collectedPartialResult, frameNumber, hasInputBufferInRequest);
             }
         }
 
@@ -2332,7 +2380,8 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg,
 
             // send pending result and buffers
             sendCaptureResult(r.pendingMetadata, r.resultExtras,
-                r.partialResult.collectedResult, msg.frame_number);
+                r.partialResult.collectedResult, msg.frame_number,
+                r.hasInputBuffer);
             returnOutputBuffers(r.pendingOutputBuffers.array(),
                 r.pendingOutputBuffers.size(), r.shutterTimestamp);
             r.pendingOutputBuffers.clear();
@@ -2669,7 +2718,6 @@ bool Camera3Device::RequestThread::threadLoop() {
     // Fill in buffers
 
     if (nextRequest->mInputStream != NULL) {
-        request.input_buffer = &inputBuffer;
         res = nextRequest->mInputStream->getInputBuffer(&inputBuffer);
         if (res != OK) {
             // Can't get input buffer from gralloc queue - this could be due to
@@ -2686,6 +2734,7 @@ bool Camera3Device::RequestThread::threadLoop() {
             cleanUpFailedRequest(request, nextRequest, outputBuffers);
             return true;
         }
+        request.input_buffer = &inputBuffer;
         totalNumBuffers += 1;
     } else {
         request.input_buffer = NULL;
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h
index a77548d35e..b08ba8152f 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.h
+++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -116,6 +116,8 @@ class Camera3Device :
     virtual status_t deleteReprocessStream(int id);
 
     virtual status_t configureStreams();
+    virtual status_t getInputBufferProducer(
+            sp<IGraphicBufferProducer> *producer);
 
     virtual status_t createDefaultRequest(int templateId, CameraMetadata *request);
 
@@ -179,6 +181,14 @@ class Camera3Device :
 
     uint32_t                   mDeviceVersion;
 
+    struct Size {
+        uint32_t width;
+        uint32_t height;
+        Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){}
+    };
+    // Map from format to size.
+    Vector<Size>               mSupportedOpaqueInputSizes;
+
     enum Status {
         STATUS_ERROR,
         STATUS_UNINITIALIZED,
@@ -324,11 +334,11 @@ class Camera3Device :
      */
     bool               tryLockSpinRightRound(Mutex& lock);
 
-    struct Size {
-        int width;
-        int height;
-        Size(int w, int h) : width(w), height(h){}
-    };
+    /**
+     * Helper function to determine if an input size for implementation defined
+     * format is supported.
+     */
+    bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height);
 
     /**
      * Helper function to get the largest Jpeg resolution (in area)
@@ -639,8 +649,10 @@ class Camera3Device :
     Mutex                  mOutputLock;
 
     /**** Scope for mOutputLock ****/
-
+    // the minimal frame number of the next non-reprocess result
     uint32_t               mNextResultFrameNumber;
+    // the minimal frame number of the next reprocess result
+    uint32_t               mNextReprocessResultFrameNumber;
     uint32_t               mNextShutterFrameNumber;
     List<CaptureResult>   mResultQueue;
     Condition              mResultSignal;
@@ -669,7 +681,8 @@ class Camera3Device :
     // partial results, and the frame number to the result queue.
     void sendCaptureResult(CameraMetadata &pendingMetadata,
             CaptureResultExtras &resultExtras,
-            CameraMetadata &collectedPartialResult, uint32_t frameNumber);
+            CameraMetadata &collectedPartialResult, uint32_t frameNumber,
+            bool reprocess);
 
     /**** Scope for mInFlightLock ****/
 
diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.cpp b/services/camera/libcameraservice/device3/Camera3InputStream.cpp
index 6bf671e2ab..fa97e57821 100644
--- a/services/camera/libcameraservice/device3/Camera3InputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3InputStream.cpp
@@ -65,8 +65,8 @@ status_t Camera3InputStream::getInputBufferLocked(
     assert(mConsumer != 0);
 
     BufferItem bufferItem;
-    res = mConsumer->acquireBuffer(&bufferItem, /*waitForFence*/false);
 
+    res = mConsumer->acquireBuffer(&bufferItem, /*waitForFence*/false);
     if (res != OK) {
         ALOGE("%s: Stream %d: Can't acquire next output buffer: %s (%d)",
                 __FUNCTION__, mId, strerror(-res), res);
@@ -162,6 +162,21 @@ status_t Camera3InputStream::returnInputBufferLocked(
     return returnAnyBufferLocked(buffer, /*timestamp*/0, /*output*/false);
 }
 
+status_t Camera3InputStream::getInputBufferProducerLocked(
+            sp<IGraphicBufferProducer> *producer) {
+    ATRACE_CALL();
+
+    if (producer == NULL) {
+        return BAD_VALUE;
+    } else if (mProducer == NULL) {
+        ALOGE("%s: No input stream is configured");
+        return INVALID_OPERATION;
+    }
+
+    *producer = mProducer;
+    return OK;
+}
+
 status_t Camera3InputStream::disconnectLocked() {
 
     status_t res;
@@ -212,10 +227,17 @@ status_t Camera3InputStream::configureQueueLocked() {
         res = producer->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
         if (res != OK || minUndequeuedBuffers < 0) {
             ALOGE("%s: Stream %d: Could not query min undequeued buffers (error %d, bufCount %d)",
-                  __FUNCTION__, mId, res, minUndequeuedBuffers);
+                    __FUNCTION__, mId, res, minUndequeuedBuffers);
             return res;
         }
         size_t minBufs = static_cast<size_t>(minUndequeuedBuffers);
+
+        if (camera3_stream::max_buffers == 0) {
+            ALOGE("%s: %d: HAL sets max_buffer to 0. Must be at least 1.",
+                    __FUNCTION__, __LINE__);
+            return INVALID_OPERATION;
+        }
+
         /*
          * We promise never to 'acquire' more than camera3_stream::max_buffers
          * at any one time.
@@ -232,6 +254,8 @@ status_t Camera3InputStream::configureQueueLocked() {
         mConsumer = new BufferItemConsumer(consumer, camera3_stream::usage,
                                            mTotalBufferCount);
         mConsumer->setName(String8::format("Camera3-InputStream-%d", mId));
+
+        mProducer = producer;
     }
 
     res = mConsumer->setDefaultBufferSize(camera3_stream::width,
diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.h b/services/camera/libcameraservice/device3/Camera3InputStream.h
index fd17f4f7af..7ba36c9f87 100644
--- a/services/camera/libcameraservice/device3/Camera3InputStream.h
+++ b/services/camera/libcameraservice/device3/Camera3InputStream.h
@@ -49,6 +49,7 @@ class Camera3InputStream : public Camera3IOStreamBase {
   private:
 
     sp<BufferItemConsumer> mConsumer;
+    sp<IGraphicBufferProducer> mProducer;
     Vector<BufferItem> mBuffersInFlight;
 
     /**
@@ -68,6 +69,8 @@ class Camera3InputStream : public Camera3IOStreamBase {
     virtual status_t getInputBufferLocked(camera3_stream_buffer *buffer);
     virtual status_t returnInputBufferLocked(
             const camera3_stream_buffer &buffer);
+    virtual status_t getInputBufferProducerLocked(
+            sp<IGraphicBufferProducer> *producer);
     virtual status_t disconnectLocked();
 
     virtual status_t configureQueueLocked();
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
index 0c739e9e24..8d9b360e8c 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -396,8 +396,22 @@ status_t Camera3OutputStream::getEndpointUsage(uint32_t *usage) {
     int32_t u = 0;
     res = mConsumer->query(mConsumer.get(),
             NATIVE_WINDOW_CONSUMER_USAGE_BITS, &u);
-    *usage = u;
 
+    // If an opaque output stream's endpoint is ImageReader, add
+    // GRALLOC_USAGE_HW_CAMERA_ZSL to the usage so HAL knows it will be used
+    // for the ZSL use case.
+    // Assume it's for ImageReader if the consumer usage doesn't have any of these bits set:
+    //     1. GRALLOC_USAGE_HW_TEXTURE
+    //     2. GRALLOC_USAGE_HW_RENDER
+    //     3. GRALLOC_USAGE_HW_COMPOSER
+    //     4. GRALLOC_USAGE_HW_VIDEO_ENCODER
+    if (camera3_stream::format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED &&
+            (u & (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_COMPOSER |
+            GRALLOC_USAGE_HW_VIDEO_ENCODER)) == 0) {
+        u |= GRALLOC_USAGE_HW_CAMERA_ZSL;
+    }
+
+    *usage = u;
     return res;
 }
 
diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp
index 4acbce39d9..d3c5cc3f3d 100644
--- a/services/camera/libcameraservice/device3/Camera3Stream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp
@@ -346,6 +346,13 @@ status_t Camera3Stream::returnInputBuffer(const camera3_stream_buffer &buffer) {
     return res;
 }
 
+status_t Camera3Stream::getInputBufferProducer(sp<IGraphicBufferProducer> *producer) {
+    ATRACE_CALL();
+    Mutex::Autolock l(mLock);
+
+    return getInputBufferProducerLocked(producer);
+}
+
 void Camera3Stream::fireBufferListenersLocked(
         const camera3_stream_buffer& /*buffer*/, bool acquired, bool output) {
     List<wp<Camera3StreamBufferListener> >::iterator it, end;
@@ -505,6 +512,10 @@ status_t Camera3Stream::returnInputBufferLocked(
     ALOGE("%s: This type of stream does not support input", __FUNCTION__);
     return INVALID_OPERATION;
 }
+status_t Camera3Stream::getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer) {
+    ALOGE("%s: This type of stream does not support input", __FUNCTION__);
+    return INVALID_OPERATION;
+}
 
 void Camera3Stream::addBufferListener(
         wp<Camera3StreamBufferListener> listener) {
diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h
index aba27fece2..e89361e286 100644
--- a/services/camera/libcameraservice/device3/Camera3Stream.h
+++ b/services/camera/libcameraservice/device3/Camera3Stream.h
@@ -205,6 +205,10 @@ class Camera3Stream :
      */
     status_t         returnInputBuffer(const camera3_stream_buffer &buffer);
 
+    // get the buffer producer of the input buffer queue.
+    // only apply to input streams.
+    status_t         getInputBufferProducer(sp<IGraphicBufferProducer> *producer);
+
     /**
      * Whether any of the stream's buffers are currently in use by the HAL,
      * including buffers that have been returned but not yet had their
@@ -285,6 +289,9 @@ class Camera3Stream :
     virtual status_t returnInputBufferLocked(
             const camera3_stream_buffer &buffer);
     virtual bool     hasOutstandingBuffersLocked() const = 0;
+    // Get the buffer producer of the input buffer queue. Only apply to input streams.
+    virtual status_t getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer);
+
     // Can return -ENOTCONN when we are already disconnected (not an error)
     virtual status_t disconnectLocked() = 0;
 
diff --git a/services/camera/libcameraservice/device3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h
index da989cd242..ea90dd9dbb 100644
--- a/services/camera/libcameraservice/device3/Camera3StreamInterface.h
+++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h
@@ -128,6 +128,13 @@ class Camera3StreamInterface : public virtual RefBase {
     virtual status_t returnInputBuffer(const camera3_stream_buffer &buffer) = 0;
 
     /**
+     * Get the buffer producer of the input buffer queue.
+     *
+     * This method only applies to input streams.
+     */
+    virtual status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer) = 0;
+
+    /**
      * Whether any of the stream's buffers are currently in use by the HAL,
      * including buffers that have been returned but not yet had their
      * release fence signaled.