From 5e049a353ba2675149241240baa0808cf3a8b9d5 Mon Sep 17 00:00:00 2001 From: Jack Yoo Date: Mon, 1 Aug 2016 14:10:02 -0700 Subject: SnapdragonCamera: ZSL for camera2 Camera2 Zero Shutter lag. Change-Id: I09c65002e5f703e776b4c32efe78d3e95e76acda CRs-Fixed: 1060586 --- src/com/android/camera/CaptureModule.java | 117 ++++--- .../camera/imageprocessor/PostProcessor.java | 245 +++++++++++++-- .../android/camera/imageprocessor/ZSLQueue.java | 348 +++++++++++++++++++++ 3 files changed, 641 insertions(+), 69 deletions(-) create mode 100644 src/com/android/camera/imageprocessor/ZSLQueue.java diff --git a/src/com/android/camera/CaptureModule.java b/src/com/android/camera/CaptureModule.java index e2b3b4759..9d48689e1 100644 --- a/src/com/android/camera/CaptureModule.java +++ b/src/com/android/camera/CaptureModule.java @@ -44,6 +44,7 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.Face; +import android.hardware.camera2.params.InputConfiguration; import android.hardware.camera2.params.MeteringRectangle; import android.hardware.camera2.params.StreamConfigurationMap; import android.location.Location; @@ -472,6 +473,7 @@ public class CaptureModule implements CameraModule, PhotoController, Face[] faces = result.get(CaptureResult.STATISTICS_FACES); updateFaceView(faces); processCaptureResult(result); + mPostProcessor.onMetaAvailable(result); } }; @@ -781,6 +783,16 @@ public class CaptureModule implements CameraModule, PhotoController, } } + private CaptureRequest.Builder getRequestBuilder(int id) throws CameraAccessException { + CaptureRequest.Builder builder; + if(mPostProcessor.isZSLEnabled() && id == getMainCameraId()) { + builder = mCameraDevice[id].createCaptureRequest(CameraDevice.TEMPLATE_ZERO_SHUTTER_LAG); + } else { + builder = mCameraDevice[id].createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); + } + return builder; + } + private void createSession(final int id) { if (mPaused || !mCameraOpened[id] || !mSurfaceReady) return; Log.d(TAG, "createSession " + id); @@ -788,8 +800,7 @@ public class CaptureModule implements CameraModule, PhotoController, try { Surface surface = getPreviewSurfaceForSession(id); // We set up a CaptureRequest.Builder with the output Surface. - mPreviewRequestBuilder[id] = mCameraDevice[id].createCaptureRequest(CameraDevice - .TEMPLATE_PREVIEW); + mPreviewRequestBuilder[id] = getRequestBuilder(id); mPreviewRequestBuilder[id].setTag(id); CameraCaptureSession.StateCallback captureSessionCallback = @@ -825,6 +836,8 @@ public class CaptureModule implements CameraModule, PhotoController, } if (isClearSightOn()) { ClearSightImageProcessor.getInstance().onCaptureSessionConfigured(id == BAYER_ID, cameraCaptureSession); + } else if(id == getMainCameraId() && mPostProcessor.isZSLEnabled()) { + mPostProcessor.onSessionConfiguredForZSL(mCaptureSession[id]); } } catch (CameraAccessException e) { e.printStackTrace(); @@ -878,7 +891,14 @@ public class CaptureModule implements CameraModule, PhotoController, list.add(surs); } list.add(mImageReader[id].getSurface()); - mCameraDevice[id].createCaptureSession(list, captureSessionCallback, null); + if(mPostProcessor.isZSLEnabled()) { + mPreviewRequestBuilder[id].addTarget(mImageReader[id].getSurface()); + mCameraDevice[id].createReprocessableCaptureSession(new InputConfiguration(mImageReader[id].getWidth(), + mImageReader[id].getHeight(), mImageReader[id].getImageFormat()), + list, captureSessionCallback, null); + } else { + mCameraDevice[id].createCaptureSession(list, captureSessionCallback, null); + } } else { mPreviewRequestBuilder[id].addTarget(surface); list.add(surface); @@ -980,6 +1000,9 @@ public class CaptureModule implements CameraModule, PhotoController, lockFocus(MONO_ID); break; case BAYER_MODE: + if(takeZSLPicture(BAYER_ID)) { + return; + } lockFocus(BAYER_ID); break; case MONO_MODE: @@ -987,10 +1010,22 @@ public class CaptureModule implements CameraModule, PhotoController, break; } } else { + if(takeZSLPicture(FRONT_ID)) { + return; + } lockFocus(FRONT_ID); } } + private boolean takeZSLPicture(int cameraId) { + if(mPostProcessor.isZSLEnabled() && mPostProcessor.takeZSLPicture(mCameraDevice[cameraId], mCaptureSession[cameraId], mImageReader[cameraId])) { + checkAndPlayShutterSound(cameraId); + mUI.enableShutter(true); + return true; + } + return false; + } + /** * Lock the focus as the first step for a still image capture. */ @@ -1020,8 +1055,7 @@ public class CaptureModule implements CameraModule, PhotoController, } try { - CaptureRequest.Builder builder = mCameraDevice[id].createCaptureRequest(CameraDevice - .TEMPLATE_PREVIEW); + CaptureRequest.Builder builder = getRequestBuilder(id); builder.setTag(id); addPreviewSurface(builder, null, id); @@ -1042,8 +1076,7 @@ public class CaptureModule implements CameraModule, PhotoController, return; } try { - CaptureRequest.Builder builder = mCameraDevice[id].createCaptureRequest(CameraDevice - .TEMPLATE_PREVIEW); + CaptureRequest.Builder builder = getRequestBuilder(id); builder.setTag(id); addPreviewSurface(builder, null, id); @@ -1080,6 +1113,11 @@ public class CaptureModule implements CameraModule, PhotoController, mPreviewRequestBuilder[id].set(BayerMonoLinkEnableKey, (byte) 0); } } + + public PostProcessor getPostProcessor() { + return mPostProcessor; + } + private void captureStillPicture(final int id) { Log.d(TAG, "captureStillPicture " + id); mIsRefocus = false; @@ -1196,30 +1234,7 @@ public class CaptureModule implements CameraModule, PhotoController, mMpoSaveHandler.obtainMessage(MpoSaveHandler.MSG_CONFIGURE, Long.valueOf(mCaptureStartTime)).sendToTarget(); } - mCaptureSession[id].capture(captureBuilder.build(), - new CameraCaptureSession.CaptureCallback() { - - @Override - public void onCaptureCompleted(CameraCaptureSession session, - CaptureRequest request, - TotalCaptureResult result) { - Log.d(TAG, "captureStillPicture onCaptureCompleted: " + id); - } - - @Override - public void onCaptureFailed(CameraCaptureSession session, - CaptureRequest request, - CaptureFailure result) { - Log.d(TAG, "captureStillPicture onCaptureFailed: " + id); - } - - @Override - public void onCaptureSequenceCompleted(CameraCaptureSession session, int - sequenceId, long frameNumber) { - Log.d(TAG, "captureStillPicture onCaptureSequenceCompleted: " + id); - unlockFocus(id); - } - }, mCaptureCallbackHandler); + mCaptureSession[id].capture(captureBuilder.build(), captureCallback, mCaptureCallbackHandler); } } } catch (CameraAccessException e) { @@ -1282,8 +1297,7 @@ public class CaptureModule implements CameraModule, PhotoController, private void runPrecaptureSequence(int id) { Log.d(TAG, "runPrecaptureSequence: " + id); try { - CaptureRequest.Builder builder = mCameraDevice[id].createCaptureRequest(CameraDevice - .TEMPLATE_PREVIEW); + CaptureRequest.Builder builder = getRequestBuilder(id); builder.setTag(id); addPreviewSurface(builder, null, id); applySettingsForPrecapture(builder, id); @@ -1339,9 +1353,9 @@ public class CaptureModule implements CameraModule, PhotoController, // No Clearsight mImageReader[i] = ImageReader.newInstance(mPictureSize.getWidth(), mPictureSize.getHeight(), imageFormat, PostProcessor.MAX_REQUIRED_IMAGE_NUM); - if((mPostProcessor.isFilterOn() || getFrameFilters().size() != 0) + if ((mPostProcessor.isFilterOn() || getFrameFilters().size() != 0 || mPostProcessor.isZSLEnabled()) && i == getMainCameraId()) { - mImageReader[i].setOnImageAvailableListener(mPostProcessor, mImageAvailableHandler); + mImageReader[i].setOnImageAvailableListener(mPostProcessor.getImageHandler(), mImageAvailableHandler); } else { mImageReader[i].setOnImageAvailableListener(new ImageAvailableListener(i) { @Override @@ -1429,8 +1443,7 @@ public class CaptureModule implements CameraModule, PhotoController, private void unlockFocus(int id) { Log.d(TAG, "unlockFocus " + id); try { - CaptureRequest.Builder builder = mCameraDevice[id].createCaptureRequest(CameraDevice - .TEMPLATE_PREVIEW); + CaptureRequest.Builder builder = getRequestBuilder(id); builder.setTag(id); addPreviewSurface(builder, null, id); @@ -1773,9 +1786,10 @@ public class CaptureModule implements CameraModule, PhotoController, return PostProcessor.FILTER_SHARPSHOOTER; } else if (mode == SettingsManager.SCENE_MODE_UBIFOCUS_INT) { return PostProcessor.FILTER_UBIFOCUS; - } else if (mode == SettingsManager.SCENE_MODE_AUTO_INT && StillmoreFilter.isSupportedStatic()) { - return PostProcessor.FILTER_STILLMORE; - } else if (mode == SettingsManager.SCENE_MODE_BESTPICTURE_INT) { + }// else if (mode == SettingsManager.SCENE_MODE_AUTO_INT && StillmoreFilter.isSupportedStatic()) { + // return PostProcessor.FILTER_STILLMORE; + //TODO: Need to put this back + else if (mode == SettingsManager.SCENE_MODE_BESTPICTURE_INT) { return PostProcessor.FILTER_BESTPICTURE; } return PostProcessor.FILTER_NONE; @@ -1841,20 +1855,24 @@ public class CaptureModule implements CameraModule, PhotoController, } if(mPostProcessor != null) { + String longshot = mSettingsManager.getValue(SettingsManager.KEY_LONGSHOT); + String flashMode = mSettingsManager.getValue(SettingsManager.KEY_FLASH_MODE); String scene = mSettingsManager.getValue(SettingsManager.KEY_SCENE_MODE); if (scene != null) { int mode = Integer.parseInt(scene); Log.d(TAG, "Chosen postproc filter id : " + getPostProcFilterId(mode)); - mPostProcessor.onOpen(getPostProcFilterId(mode)); + mPostProcessor.onOpen(getPostProcFilterId(mode), longshot != null && longshot.equals("on"), + flashMode != null && flashMode.equals(CaptureRequest.CONTROL_AE_MODE_ON_ALWAYS_FLASH+"")); } else { - mPostProcessor.onOpen(PostProcessor.FILTER_NONE); + mPostProcessor.onOpen(PostProcessor.FILTER_NONE, longshot != null && longshot.equals("on"), + flashMode != null && flashMode.equals(CaptureRequest.CONTROL_AE_MODE_ON_ALWAYS_FLASH+"")); } } if(mFrameProcessor != null) { mFrameProcessor.onOpen(getFrameProcFilterId()); } - if(mPostProcessor.isFilterOn() || getFrameFilters().size() != 0) { + if(mPostProcessor.isFilterOn() || getFrameFilters().size() != 0 || mPostProcessor.isZSLEnabled()) { setUpCameraOutputs(ImageFormat.YUV_420_888); } else { setUpCameraOutputs(ImageFormat.JPEG); @@ -3290,11 +3308,14 @@ public class CaptureModule implements CameraModule, PhotoController, if (count == 0) restart(); return; case SettingsManager.KEY_SCENE_MODE: - if (count == 0 && checkNeedToRestart(value)) { - restart(); - return; - } - break; + if (count == 0) restart(); + return; + case SettingsManager.KEY_LONGSHOT: + if (count == 0) restart(); + return; + case SettingsManager.KEY_FLASH_MODE: + if (count == 0) restart(); //Restart is due to ZSL mode change + return; } if (isBackCamera()) { diff --git a/src/com/android/camera/imageprocessor/PostProcessor.java b/src/com/android/camera/imageprocessor/PostProcessor.java index 18aa497dd..74372d70a 100644 --- a/src/com/android/camera/imageprocessor/PostProcessor.java +++ b/src/com/android/camera/imageprocessor/PostProcessor.java @@ -35,9 +35,15 @@ import android.graphics.Rect; import android.graphics.YuvImage; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraMetadata; +import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; import android.media.Image; import android.media.ImageReader; +import android.media.ImageWriter; import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; @@ -65,11 +71,12 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.TimeZone; +import java.util.concurrent.Semaphore; import com.android.camera.imageprocessor.filter.ImageFilter; import com.android.camera.util.CameraUtil; -public class PostProcessor implements ImageReader.OnImageAvailableListener{ +public class PostProcessor{ private CaptureModule mController; @@ -82,8 +89,8 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ public static final int FILTER_BESTPICTURE = 5; public static final int FILTER_MAX = 6; - //Max image is now Bestpicture filter with 10 - public static final int MAX_REQUIRED_IMAGE_NUM = 10; + //BestPicture requires 10 which is the biggest among filters + public static final int MAX_REQUIRED_IMAGE_NUM = 11; private int mCurrentNumImage = 0; private ImageFilter mFilter; private int mFilterIndex; @@ -99,26 +106,192 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ private PhotoModule.NamedImages mNamedImages; private WatchdogThread mWatchdog; private int mOrientation = 0; + private ImageWriter mZSLImageWriter; //This is for the debug feature. private static boolean DEBUG_FILTER = false; + private static boolean DEBUG_ZSL = false; private ImageFilter.ResultImage mDebugResultImage; + private ZSLQueue mZSLQueue; + private CameraDevice mCameraDevice; + private CameraCaptureSession mCaptureSession; + private ImageReader mImageReader; + private boolean mUseZSL = true; + private Handler mZSLHandler; + private HandlerThread mZSLHandlerThread; + private ImageHandlerTask mImageHandlerTask; + + public boolean isZSLEnabled() { + return mUseZSL; + } + + public ImageHandlerTask getImageHandler() { + return mImageHandlerTask; + } + + private class ImageWrapper { + Image mImage; + boolean mIsTaken; + + public ImageWrapper(Image image) { + mImage = image; + mIsTaken = false; + } + + public boolean isTaken() { + return mIsTaken; + } + + public Image getImage() { + mIsTaken = true; + return mImage; + } + } + + class ImageHandlerTask implements Runnable, ImageReader.OnImageAvailableListener { + private ImageWrapper mImageWrapper = null; + Semaphore mMutureLock = new Semaphore(1); + + @Override + public void onImageAvailable(ImageReader reader) { + try { + Image image = reader.acquireLatestImage(); + if(image == null) { + return; + } + if (!mMutureLock.tryAcquire()) { + image.close(); + return; + } + if(mImageWrapper == null || mImageWrapper.isTaken()) { + mImageWrapper = new ImageWrapper(image); + mMutureLock.release(); + } else { + image.close(); + mMutureLock.release(); + return; + } + if(mZSLHandler != null) { + mZSLHandler.post(this); + } + } catch (IllegalStateException e) { + Log.e(TAG, "Max images has been already acquired. "); + } + } + + @Override + public void run() { + try { + mMutureLock.acquire(); + Image image = mImageWrapper.getImage(); + if (mUseZSL) { + if (mZSLQueue != null) { + mZSLQueue.add(image); + } + } else { + onImageToProcess(image); + } + mMutureLock.release(); + } catch (InterruptedException e) { + } + } + } + + public void onMetaAvailable(TotalCaptureResult metadata) { + if(mUseZSL && mZSLQueue != null) { + mZSLQueue.add(metadata); + } + } + + public void onSessionConfiguredForZSL(CameraCaptureSession captureSession) { + mZSLImageWriter = ImageWriter.newInstance(captureSession.getInputSurface(), 2); + } - @Override - public void onImageAvailable(ImageReader reader) { + public boolean takeZSLPicture(CameraDevice cameraDevice, CameraCaptureSession captureSession, ImageReader imageReader) { + if(mCameraDevice == null || mCaptureSession == null || mImageReader == null) { + mCameraDevice = cameraDevice; + mCaptureSession = captureSession; + mImageReader = imageReader; + } + ZSLQueue.ImageItem imageItem = mZSLQueue.tryToGetMatchingItem(); + if(mController.getPreviewCaptureResult().get(CaptureResult.CONTROL_AE_STATE) == CameraMetadata.CONTROL_AE_STATE_FLASH_REQUIRED) { + if(DEBUG_ZSL) Log.d(TAG, "Flash required image"); + imageItem = null; + } + if (imageItem != null) { + if(DEBUG_ZSL) Log.d(TAG,"Got the item from the queue"); + reprocessImage(imageItem); + return true; + } else { + if(DEBUG_ZSL) Log.d(TAG, "No good item in queue, reigster the request for the future"); + mZSLQueue.addPictureRequest(); + return false; + } + } + + public void onMatchingZSLPictureAvailable(ZSLQueue.ImageItem imageItem) { + reprocessImage(imageItem); + } + + private void reprocessImage(ZSLQueue.ImageItem imageItem) { + if(mCameraDevice == null || mCaptureSession == null || mImageReader == null) { + Log.e(TAG, "Reprocess request is called even before taking picture"); + new Throwable().printStackTrace(); + return; + } + CaptureRequest.Builder builder = null; try { - Image image = reader.acquireNextImage(); - addImage(image); - if (isReadyToProcess()) { - long captureStartTime = System.currentTimeMillis(); - mNamedImages.nameNewImage(captureStartTime); - PhotoModule.NamedImages.NamedEntity name = mNamedImages.getNextNameEntity(); - String title = (name == null) ? null : name.title; - long date = (name == null) ? -1 : name.date; - processImage(title, date, mController.getMediaSavedListener(), mActivity.getContentResolver()); + builder = mCameraDevice.createReprocessCaptureRequest(imageItem.getMetadata()); + builder.addTarget(mImageReader.getSurface()); + builder.set(CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE, + CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_HIGH_QUALITY); + builder.set(CaptureRequest.EDGE_MODE, + CaptureRequest.EDGE_MODE_HIGH_QUALITY); + builder.set(CaptureRequest.NOISE_REDUCTION_MODE, + CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY); + try { + mZSLImageWriter.queueInputImage(imageItem.getImage()); + } catch(IllegalStateException e) { + Log.e(TAG, "Queueing more than it can have"); } - } catch (IllegalStateException e) { - Log.e(TAG, "Max images has been already acquired. "); + mCaptureSession.capture(builder.build(), + new CameraCaptureSession.CaptureCallback() { + @Override + public void onCaptureCompleted( + CameraCaptureSession session, + CaptureRequest request, + TotalCaptureResult result) { + super.onCaptureCompleted(session, request, result); + Log.d(TAG, "Success to reprocess zsl image"); + try { + onImageToProcess(mZSLImageWriter.dequeueInputImage()); + } catch(IllegalStateException e) { + Log.e(TAG, "Dequeueing more than what it has"); + } + } + + @Override + public void onCaptureFailed( + CameraCaptureSession session, + CaptureRequest request, + CaptureFailure failure) { + super.onCaptureFailed(session, request, failure); + Log.e(TAG, "failed to reprocess"); + } + }, null); + } catch (CameraAccessException e) { + } + } + + private void onImageToProcess(Image image) { + addImage(image); + if (isReadyToProcess()) { + long captureStartTime = System.currentTimeMillis(); + mNamedImages.nameNewImage(captureStartTime); + PhotoModule.NamedImages.NamedEntity name = mNamedImages.getNextNameEntity(); + String title = (name == null) ? null : name.title; + long date = (name == null) ? -1 : name.date; + processImage(title, date, mController.getMediaSavedListener(), mActivity.getContentResolver()); } } @@ -133,7 +306,6 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ mController = module; mActivity = activity; mNamedImages = new PhotoModule.NamedImages(); - } public boolean isItBusy() { @@ -168,10 +340,18 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ return false; } - public void onOpen(int postFilterId) { - setFilter(postFilterId); - startBackgroundThread(); + public void onOpen(int postFilterId, boolean isLongShotOn, boolean isFlashModeOn) { + mImageHandlerTask = new ImageHandlerTask(); + if(setFilter(postFilterId) || isLongShotOn || isFlashModeOn) { + mUseZSL = false; + } else { + mUseZSL = true; + } + startBackgroundThread(); + if(mUseZSL) { + mZSLQueue = new ZSLQueue(mController); + } } public int getFilterIndex() { @@ -186,6 +366,17 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ stopBackgroundThread(); } setFilter(FILTER_NONE); + if(mZSLQueue != null) { + mZSLQueue.onClose(); + mZSLQueue = null; + } + if (mZSLImageWriter != null) { + mZSLImageWriter.close(); + mZSLImageWriter = null; + } + mCameraDevice = null; + mCaptureSession = null; + mImageReader = null; } private void startBackgroundThread() { @@ -193,6 +384,10 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ mHandlerThread.start(); mHandler = new ProcessorHandler(mHandlerThread.getLooper()); + mZSLHandlerThread = new HandlerThread("ZSLHandlerThread"); + mZSLHandlerThread.start(); + mZSLHandler = new ProcessorHandler(mZSLHandlerThread.getLooper()); + mWatchdog = new WatchdogThread(); mWatchdog.start(); } @@ -215,7 +410,6 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ } } } - } public void startMonitor() { @@ -261,6 +455,15 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ mHandlerThread = null; mHandler = null; } + if (mZSLHandlerThread != null) { + mZSLHandlerThread.quitSafely(); + try { + mZSLHandlerThread.join(); + } catch (InterruptedException e) { + } + mZSLHandlerThread = null; + mZSLHandler = null; + } if(mWatchdog != null) { mWatchdog.kill(); mWatchdog = null; diff --git a/src/com/android/camera/imageprocessor/ZSLQueue.java b/src/com/android/camera/imageprocessor/ZSLQueue.java new file mode 100644 index 000000000..ce2da1c6b --- /dev/null +++ b/src/com/android/camera/imageprocessor/ZSLQueue.java @@ -0,0 +1,348 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.android.camera.imageprocessor; + +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; +import android.media.Image; +import android.util.Log; + +import com.android.camera.CaptureModule; +import android.os.SystemProperties; + +import java.util.LinkedList; + +public class ZSLQueue { + private static final String CIRCULAR_BUFFER_SIZE_PERSIST = "persist.camera.zsl.buffer.size"; + private static final int CIRCULAR_BUFFER_SIZE_DEFAULT = 5; + private int mCircularBufferSize = CIRCULAR_BUFFER_SIZE_DEFAULT; + private ImageItem[] mBuffer; + private int mImageHead; + private int mMetaHead; + private Object mLock = new Object(); + private LinkedList mPendingRequestList = new LinkedList(); + private CaptureModule mModule; + private static final boolean DEBUG = false; + private static final boolean DEBUG_QUEUE = false; + private static final String TAG = "ZSLQueue"; + private static final int REQUEST_LIFESPAN = 5; //This is in frame count. + + class PendingRequest { + private int mLifeTimeInFrame; + + public PendingRequest(){ + mLifeTimeInFrame = 0; + } + + public int getLifeTime() { + return mLifeTimeInFrame; + } + + public void incLifeTime() { + mLifeTimeInFrame++; + } + } + + public ZSLQueue(CaptureModule module) { + mCircularBufferSize = SystemProperties.getInt(CIRCULAR_BUFFER_SIZE_PERSIST, CIRCULAR_BUFFER_SIZE_DEFAULT); + synchronized (mLock) { + mBuffer = new ImageItem[mCircularBufferSize]; + mImageHead = 0; + mMetaHead = 0; + mPendingRequestList.clear(); + mModule = module; + } + } + + private int findMeta(long timestamp, int index) { + int startIndex = index; + do { + if(mBuffer[index] != null && mBuffer[index].getMetadata() != null && + mBuffer[index].getMetadata().get(CaptureResult.SENSOR_TIMESTAMP).longValue() == timestamp) { + return index; + } + index = (index + 1) % mBuffer.length; + } while(index != startIndex); + return -1; + } + + private int findImage(long timestamp, int index) { + int startIndex = index; + do { + if(mBuffer[index] != null && mBuffer[index].getImage() != null && + mBuffer[index].getImage().getTimestamp() == timestamp) { + return index; + } + index = (index + 1) % mBuffer.length; + } while(index != startIndex); + return -1; + } + + public void add(Image image) { + int lastIndex = -1; + synchronized (mLock) { + if(mBuffer == null) + return; + if(mBuffer[mImageHead] != null) { + mBuffer[mImageHead].closeImage(); + } else { + mBuffer[mImageHead] = new ImageItem(); + } + if(mBuffer[mImageHead].getMetadata() != null) { + if((mBuffer[mImageHead].getMetadata().get(CaptureResult.SENSOR_TIMESTAMP)).longValue() == image.getTimestamp()) { + mBuffer[mImageHead].setImage(image); + lastIndex = mImageHead; + mImageHead = (mImageHead + 1) % mBuffer.length; + } else if((mBuffer[mImageHead].getMetadata().get(CaptureResult.SENSOR_TIMESTAMP)).longValue() > image.getTimestamp()) { + image.close(); + } else { + int i = findMeta(image.getTimestamp(), mImageHead); + if(i == -1) { + mBuffer[mImageHead].setImage(image); + mBuffer[mImageHead].setMetadata(null); + mImageHead = (mImageHead + 1) % mBuffer.length; + } else { + lastIndex = mImageHead = i; + mBuffer[mImageHead].setImage(image); + mImageHead = (mImageHead + 1) % mBuffer.length; + } + } + } else { + mBuffer[mImageHead].setImage(image); + lastIndex = mImageHead; + mImageHead = (mImageHead + 1) % mBuffer.length; + } + } + + if(DEBUG_QUEUE) Log.d(TAG, "imageIndex: " + lastIndex + " " + image.getTimestamp()); + + if(mPendingRequestList.size() != 0) { + if(lastIndex != -1) { + processPendingRequest(lastIndex); + } + for(int i=0; i < mPendingRequestList.size(); i++) { + mPendingRequestList.get(i).incLifeTime(); + } + } + } + + private void processPendingRequest(int index) { + ImageItem item; + synchronized (mLock) { + if(mBuffer == null) + return; + item = mBuffer[index]; + if (item != null && item.isValid() && checkImageRequirement(item.getMetadata())) { + if(DEBUG && (mBuffer[index].getMetadata().get(CaptureResult.SENSOR_TIMESTAMP)).longValue() != + mBuffer[index].getImage().getTimestamp()) { + Log.e(TAG,"Not matching image coming through"); + } + mBuffer[index] = null; + mPendingRequestList.removeFirst(); + for(PendingRequest request : mPendingRequestList) { + if(request.getLifeTime() >= REQUEST_LIFESPAN) { + mPendingRequestList.remove(request); + } + } + } else { + return; + } + } + mModule.getPostProcessor().onMatchingZSLPictureAvailable(item); + } + + public void add(TotalCaptureResult metadata) { + int lastIndex = -1; + synchronized (mLock) { + if(mBuffer == null) + return; + long timestamp = -1; + try { + timestamp = metadata.get(CaptureResult.SENSOR_TIMESTAMP).longValue(); + } catch(IllegalStateException e) { + //This happens when corresponding image to this metadata is closed and discarded. + return; + } + if(timestamp == -1) { + return; + } + if(mBuffer[mMetaHead] == null) { + mBuffer[mMetaHead] = new ImageItem(); + } else { + mBuffer[mMetaHead].closeMeta(); + } + if(mBuffer[mMetaHead].getImage() != null) { + if(mBuffer[mMetaHead].getImage().getTimestamp() == timestamp) { + mBuffer[mMetaHead].setMetadata(metadata); + lastIndex = mMetaHead; + mMetaHead = (mMetaHead + 1) % mBuffer.length; + } else if(mBuffer[mMetaHead].getImage().getTimestamp() > timestamp) { + //Disard + } else { + int i = findImage(timestamp, mMetaHead); + if(i == -1) { + mBuffer[mMetaHead].setImage(null); + mBuffer[mMetaHead].setMetadata(metadata); + mMetaHead = (mMetaHead + 1) % mBuffer.length; + } else { + lastIndex = mMetaHead = i; + mBuffer[mMetaHead].setMetadata(metadata); + mMetaHead = (mMetaHead + 1) % mBuffer.length; + } + } + } else { + mBuffer[mMetaHead].setMetadata(metadata); + lastIndex = mImageHead; + mMetaHead = (mMetaHead + 1) % mBuffer.length; + } + } + + if(DEBUG_QUEUE) Log.d(TAG, "Meta: " + lastIndex + " " + metadata.get(CaptureResult.SENSOR_TIMESTAMP)); + + if(mPendingRequestList.size() != 0) { + if(lastIndex != -1) { + processPendingRequest(lastIndex); + } + for(int i=0; i < mPendingRequestList.size(); i++) { + mPendingRequestList.get(i).incLifeTime(); + } + } + } + + public ImageItem tryToGetMatchingItem() { + synchronized (mLock) { + int index = mImageHead; + ImageItem item; + do { + item = mBuffer[index]; + if (item != null && item.isValid() && checkImageRequirement(item.getMetadata())) { + mBuffer[index] = null; + return item; + } + index--; + if (index < 0) index = mBuffer.length - 1; + } while (index != mImageHead); + } + return null; + } + + public void addPictureRequest() { + if(DEBUG) Log.d(TAG, "RequsetPendingCount: "+mPendingRequestList.size()); + synchronized (mLock) { + mPendingRequestList.addLast(new PendingRequest()); + } + } + + public void onClose() { + synchronized (mLock) { + for (int i = 0; i < mBuffer.length; i++) { + if (mBuffer[i] != null) { + mBuffer[i].closeImage(); + mBuffer[i].closeMeta(); + mBuffer[i] = null; + } + } + mBuffer = null; + mImageHead = 0; + mMetaHead = 0; + mPendingRequestList.clear(); + } + } + + private boolean checkImageRequirement(TotalCaptureResult captureResult) { + if( (captureResult.get(CaptureResult.LENS_STATE) != null && + captureResult.get(CaptureResult.LENS_STATE).intValue() == CaptureResult.LENS_STATE_MOVING) + || + (captureResult.get(CaptureResult.CONTROL_AE_STATE) != null && + (captureResult.get(CaptureResult.CONTROL_AE_STATE).intValue() == CaptureResult.CONTROL_AE_STATE_SEARCHING || + captureResult.get(CaptureResult.CONTROL_AE_STATE).intValue() == CaptureResult.CONTROL_AE_STATE_PRECAPTURE)) + || + (captureResult.get(CaptureResult.CONTROL_AF_STATE) != null) && + (captureResult.get(CaptureResult.CONTROL_AF_STATE) == CaptureResult.CONTROL_AF_STATE_ACTIVE_SCAN || + captureResult.get(CaptureResult.CONTROL_AF_STATE) == CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN)) { + return false; + } + + if( captureResult.get(CaptureResult.CONTROL_AE_STATE) != null && + captureResult.get(CaptureResult.CONTROL_AF_STATE) != null && + captureResult.get(CaptureResult.CONTROL_AE_STATE).intValue() == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED && + captureResult.get(CaptureResult.CONTROL_AF_STATE).intValue() != CaptureResult.FLASH_MODE_OFF) { + return true; + } + + if( captureResult.get(CaptureResult.CONTROL_AWB_STATE) != null && + captureResult.get(CaptureResult.CONTROL_AWB_STATE).intValue() == CaptureResult.CONTROL_AWB_STATE_SEARCHING ) { + return false; + } + + return true; + } + + class ImageItem { + private Image mImage = null; + private TotalCaptureResult mMetadata = null; + + public Image getImage() { + return mImage; + } + + public void setImage(Image image) { + if(mImage != null) { + mImage.close(); + } + mImage = image; + } + + public TotalCaptureResult getMetadata() { + return mMetadata; + } + + public void setMetadata(TotalCaptureResult metadata) { + mMetadata = metadata; + } + + public void closeImage() { + if(mImage != null) { + mImage.close(); + } + mImage = null; + } + + public void closeMeta() { + mMetadata = null; + } + + public boolean isValid() { + if(mImage != null && mMetadata != null) { + return true; + } + return false; + } + } +} -- cgit v1.2.3