From 48f72ee9ef618d31e9a08bb7ad66b3ca8da67b9f Mon Sep 17 00:00:00 2001 From: Jack Yoo Date: Wed, 23 Mar 2016 14:17:49 -0700 Subject: SnapdragonCamera: PostProcessor and Optizoom Introducing PostProcessor with Optizoom filter Change-Id: Ib9ac6d4a9526be3a5163d02e298ed783daad48c1 CRs-Fixed: 1023183 --- res/values/camera2arrays.xml | 2 +- src/com/android/camera/CaptureModule.java | 157 ++++++-- src/com/android/camera/SettingsManager.java | 9 +- .../camera/imageprocessor/PostProcessor.java | 438 +++++++++++++++++++++ .../camera/imageprocessor/filter/ImageFilter.java | 74 ++++ .../imageprocessor/filter/OptizoomFilter.java | 148 +++++++ 6 files changed, 800 insertions(+), 28 deletions(-) create mode 100644 src/com/android/camera/imageprocessor/PostProcessor.java create mode 100644 src/com/android/camera/imageprocessor/filter/ImageFilter.java create mode 100644 src/com/android/camera/imageprocessor/filter/OptizoomFilter.java diff --git a/res/values/camera2arrays.xml b/res/values/camera2arrays.xml index 9b7db30f6..59e06f260 100644 --- a/res/values/camera2arrays.xml +++ b/res/values/camera2arrays.xml @@ -90,7 +90,7 @@ 100 18 -1 - -1 + 101 3 4 13 diff --git a/src/com/android/camera/CaptureModule.java b/src/com/android/camera/CaptureModule.java index a4658daef..8810c6df4 100644 --- a/src/com/android/camera/CaptureModule.java +++ b/src/com/android/camera/CaptureModule.java @@ -60,6 +60,7 @@ import android.view.SurfaceHolder; import android.view.View; import android.widget.Toast; +import com.android.camera.imageprocessor.PostProcessor; import com.android.camera.PhotoModule.NamedImages; import com.android.camera.PhotoModule.NamedImages.NamedEntity; import com.android.camera.ui.CountDownView; @@ -77,6 +78,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -138,6 +140,7 @@ public class CaptureModule implements CameraModule, PhotoController, ORIENTATIONS.append(Surface.ROTATION_180, 270); ORIENTATIONS.append(Surface.ROTATION_270, 180); } + private static final int MAX_IMAGE_NUM = 8; MeteringRectangle[][] mAFRegions = new MeteringRectangle[MAX_NUM_CAM][]; CaptureRequest.Key BayerMonoLinkEnableKey = @@ -193,6 +196,7 @@ public class CaptureModule implements CameraModule, PhotoController, /** * A {@link Handler} for running tasks in the background. */ + private PostProcessor mPostProcessor; private Handler mCameraHandler; private Handler mImageAvailableHandler; private Handler mCaptureCallbackHandler; @@ -239,6 +243,10 @@ public class CaptureModule implements CameraModule, PhotoController, } } + public void updateThumbnailJpegData(byte[] jpegData) { + mLastJpegData = jpegData; + } + private MediaSaveNotifyThread mediaSaveNotifyThread; private MediaSaveService.OnMediaSavedListener mOnMediaSavedListener = new MediaSaveService.OnMediaSavedListener() { @@ -259,6 +267,10 @@ public class CaptureModule implements CameraModule, PhotoController, } }; + public MediaSaveService.OnMediaSavedListener getMediaSavedListener() { + return mOnMediaSavedListener; + } + static abstract class ImageAvailableListener implements ImageReader.OnImageAvailableListener { int mCamId; @@ -645,6 +657,9 @@ public class CaptureModule implements CameraModule, PhotoController, for (int i = 0; i < MAX_NUM_CAM; i++) { mState[i] = STATE_PREVIEW; } + + mPostProcessor = new PostProcessor(mActivity, this); + setCurrentMode(); mContentResolver = mActivity.getContentResolver(); mUI = new CaptureUI(activity, this, parent); @@ -684,7 +699,12 @@ public class CaptureModule implements CameraModule, PhotoController, * Lock the focus as the first step for a still image capture. */ private void lockFocus(int id) { + if (mActivity == null || mCameraDevice[id] == null) { + warningToast("Camera is not ready yet to take a picture."); + return; + } Log.d(TAG, "lockFocus " + id); + mTakingPicture[id] = true; if (mState[id] == STATE_WAITING_TOUCH_FOCUS) { mCameraHandler.removeMessages(CANCEL_TOUCH_FOCUS, id); @@ -710,6 +730,10 @@ public class CaptureModule implements CameraModule, PhotoController, private void autoFocusTrigger(int id) { Log.d(TAG, "autoFocusTrigger " + id); + if (null == mActivity || null == mCameraDevice[id]) { + warningToast("Camera is not ready yet to take a picture."); + return; + } try { CaptureRequest.Builder builder = mCameraDevice[id].createCaptureRequest(CameraDevice .TEMPLATE_PREVIEW); @@ -759,6 +783,7 @@ public class CaptureModule implements CameraModule, PhotoController, Log.d(TAG, "captureStillPicture " + id); try { if (null == mActivity || null == mCameraDevice[id]) { + warningToast("Camera is not ready yet to take a picture."); return; } @@ -784,6 +809,32 @@ public class CaptureModule implements CameraModule, PhotoController, if(csEnabled) { ClearSightImageProcessor.getInstance().capture( id==BAYER_ID, mCaptureSession[id], captureBuilder, mCaptureCallbackHandler); + } else if(id == BAYER_ID && mPostProcessor.isFilterOn()) { + captureBuilder.addTarget(mImageReader[id].getSurface()); + List captureList = mPostProcessor.setRequiredImages(captureBuilder); + mCaptureSession[id].captureBurst(captureList, new CameraCaptureSession.CaptureCallback() { + + @Override + public void onCaptureCompleted(CameraCaptureSession session, + CaptureRequest request, + TotalCaptureResult result) { + Log.d(TAG, "captureStillPicture onCaptureCompleted: " + id); + } + + @Override + public void onCaptureFailed(CameraCaptureSession session, + CaptureRequest request, + CaptureFailure result) { + Log.d(TAG, "captureStillPicture onCaptureFailed: " + id); + } + + @Override + public void onCaptureSequenceCompleted(CameraCaptureSession session, int + sequenceId, long frameNumber) { + Log.d(TAG, "captureStillPicture onCaptureSequenceCompleted: " + id); + unlockFocus(id); + } + }, mCaptureCallbackHandler); } else { captureBuilder.addTarget(mImageReader[id].getSurface()); mCaptureSession[id].stopRepeating(); @@ -895,7 +946,7 @@ public class CaptureModule implements CameraModule, PhotoController, * @param width The width of available size for camera preview * @param height The height of available size for camera preview */ - private void setUpCameraOutputs() { + private void setUpCameraOutputs(int imageFomat) { Log.d(TAG, "setUpCameraOutputs"); CameraManager manager = (CameraManager) mActivity.getSystemService(Context.CAMERA_SERVICE); try { @@ -920,7 +971,7 @@ public class CaptureModule implements CameraModule, PhotoController, if (i == getMainCameraId()) { Point screenSize = new Point(); mActivity.getWindowManager().getDefaultDisplay().getSize(screenSize); - Size[] prevSizes = map.getOutputSizes(SurfaceHolder.class); + Size[] prevSizes = map.getOutputSizes(imageFomat); Size prevSize = getOptimalPreviewSize(size, prevSizes, screenSize.x, screenSize.y); mUI.setPreviewSize(prevSize.getWidth(), prevSize.getHeight()); @@ -932,29 +983,34 @@ public class CaptureModule implements CameraModule, PhotoController, } else { // No Clearsight mImageReader[i] = ImageReader.newInstance(size.getWidth(), size.getHeight(), - ImageFormat.JPEG, 3); - mImageReader[i].setOnImageAvailableListener(new ImageAvailableListener(i) { - @Override - public void onImageAvailable(ImageReader reader) { - Log.d(TAG, "image available for cam: " + mCamId); - Image image = reader.acquireNextImage(); - mCaptureStartTime = System.currentTimeMillis(); - mNamedImages.nameNewImage(mCaptureStartTime); - NamedEntity name = mNamedImages.getNextNameEntity(); - String title = (name == null) ? null : name.title; - long date = (name == null) ? -1 : name.date; - - ByteBuffer buffer = image.getPlanes()[0].getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - mLastJpegData = bytes; - buffer.get(bytes); - - mActivity.getMediaSaveService().addImage(bytes, title, date, - null, image.getWidth(), image.getHeight(), 0, null, - mOnMediaSavedListener, mContentResolver, "jpeg"); - image.close(); - } - }, mImageAvailableHandler); + imageFomat, MAX_IMAGE_NUM); + + if(mPostProcessor.isFilterOn() && i == BAYER_ID) { + mImageReader[i].setOnImageAvailableListener(mPostProcessor, mImageAvailableHandler); + } else { + mImageReader[i].setOnImageAvailableListener(new ImageAvailableListener(i) { + @Override + public void onImageAvailable(ImageReader reader) { + Log.d(TAG, "image available for cam: " + mCamId); + Image image = reader.acquireNextImage(); + mCaptureStartTime = System.currentTimeMillis(); + mNamedImages.nameNewImage(mCaptureStartTime); + NamedEntity name = mNamedImages.getNextNameEntity(); + String title = (name == null) ? null : name.title; + long date = (name == null) ? -1 : name.date; + + ByteBuffer buffer = image.getPlanes()[0].getBuffer(); + byte[] bytes = new byte[buffer.remaining()]; + mLastJpegData = bytes; + buffer.get(bytes); + + mActivity.getMediaSaveService().addImage(bytes, title, date, + null, image.getWidth(), image.getHeight(), 0, null, + mOnMediaSavedListener, mContentResolver, "jpeg"); + image.close(); + } + }, mImageAvailableHandler); + } } } @@ -1010,6 +1066,9 @@ public class CaptureModule implements CameraModule, PhotoController, Log.d(TAG, "closeCamera"); try { mCameraOpenCloseLock.acquire(); + if(mPostProcessor != null) { + mPostProcessor.onClose(); + } for (int i = 0; i < MAX_NUM_CAM; i++) { if (null != mCaptureSession[i]) { if (mIsLinked) { @@ -1195,13 +1254,38 @@ public class CaptureModule implements CameraModule, PhotoController, mCurrentMode = isBackCamera() ? getCameraMode() : FRONT_MODE; } + private int getPostProcFilterId() { + String scene = mSettingsManager.getValue(SettingsManager.KEY_SCENE_MODE); + if (scene != null) { + int mode = Integer.parseInt(scene); + if (mode == SettingsManager.SCENE_MODE_OPTIZOOM_INT) + return PostProcessor.FILTER_OPTIZOOM; + } + return PostProcessor.FILTER_NONE; + } + + private boolean isPostProcFilter(String value) { + if(value.equalsIgnoreCase(SettingsManager.SCENE_MODE_OPTIZOOM_INT+"")) { + return true; + } + return false; + } + @Override public void onResumeAfterSuper() { Log.d(TAG, "onResume " + getCameraMode()); mUI.showSurfaceView(); mUI.setSwitcherIndex(); mCameraIdList = new ArrayList<>(); - setUpCameraOutputs(); + if(mPostProcessor != null) { + Log.d(TAG, "Chosen postproc filter id : "+getPostProcFilterId()); + mPostProcessor.onOpen(getPostProcFilterId()); + } + if(mPostProcessor.isFilterOn()) { + setUpCameraOutputs(ImageFormat.YUV_420_888); + } else { + setUpCameraOutputs(ImageFormat.JPEG); + } startBackgroundThread(); Message msg = Message.obtain(); msg.what = OPEN_CAMERA; @@ -1556,10 +1640,24 @@ public class CaptureModule implements CameraModule, PhotoController, if (seconds > 0) { mUI.startCountDown(seconds, true); } else { + if(mPostProcessor.isFilterOn() && mPostProcessor.isItBusy()) { + warningToast("It's still busy processing previous scene mode request."); + return; + } takePicture(); } } + private void warningToast(final String msg) { + mActivity.runOnUiThread(new Runnable() { + public void run() { + RotateTextToast.makeText(mActivity, msg, + Toast.LENGTH_SHORT).show(); + } + }); + + } + @Override public void onShutterButtonLongClick() { if (isBackCamera() && getCameraMode() == DUAL_MODE) return; @@ -1668,6 +1766,10 @@ public class CaptureModule implements CameraModule, PhotoController, applyCommonSettings(mPreviewRequestBuilder[id], id); } + public float getZoomValue() { + return mZoomValue; + } + public Rect cropRegionForZoom(int id) { Log.d(TAG, "cropRegionForZoom " + id); Rect activeRegion = mSettingsManager.getSensorActiveArraySize(id); @@ -1971,6 +2073,9 @@ public class CaptureModule implements CameraModule, PhotoController, private int mCurrentMode; private boolean checkNeedToRestart(String value) { + mPostProcessor.setFilter(PostProcessor.FILTER_NONE); + if (isPostProcFilter(value)) + return true; if (value.equals(SettingsManager.SCENE_MODE_DUAL_STRING) && mCurrentMode != DUAL_MODE) return true; if (!value.equals(SettingsManager.SCENE_MODE_DUAL_STRING) && mCurrentMode == DUAL_MODE) diff --git a/src/com/android/camera/SettingsManager.java b/src/com/android/camera/SettingsManager.java index a01d2543b..d19cab2a6 100644 --- a/src/com/android/camera/SettingsManager.java +++ b/src/com/android/camera/SettingsManager.java @@ -41,6 +41,7 @@ import android.util.Range; import android.util.Rational; import android.util.Size; +import com.android.camera.imageprocessor.filter.OptizoomFilter; import com.android.camera.ui.ListMenu; import org.codeaurora.snapcam.R; @@ -59,6 +60,7 @@ public class SettingsManager implements ListMenu.SettingsListener { public static final int RESOURCE_TYPE_LARGEICON = 1; // Custom-Scenemodes start from 100 public static final int SCENE_MODE_DUAL_INT = 100; + public static final int SCENE_MODE_OPTIZOOM_INT = 101; public static final String SCENE_MODE_DUAL_STRING = "100"; public static final String KEY_CAMERA_SAVEPATH = "pref_camera2_savepath_key"; public static final String KEY_RECORD_LOCATION = "pref_camera2_recordlocation_key"; @@ -108,7 +110,11 @@ public class SettingsManager implements ListMenu.SettingsListener { String cameraId = cameraIdList[i]; CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); - Byte monoOnly = characteristics.get(CaptureModule.MetaDataMonoOnlyKey); + Byte monoOnly = 0; + try { + monoOnly = characteristics.get(CaptureModule.MetaDataMonoOnlyKey); + }catch(Exception e) { + } if (monoOnly == 1) { CaptureModule.MONO_ID = i; mIsMonoCameraPresent = true; @@ -680,6 +686,7 @@ public class SettingsManager implements ListMenu.SettingsListener { List modes = new ArrayList<>(); modes.add("0"); // need special case handle for auto scene mode if (mIsMonoCameraPresent) modes.add(SCENE_MODE_DUAL_STRING); // need special case handle for dual mode + if (OptizoomFilter.isSupportedStatic()) modes.add(SCENE_MODE_OPTIZOOM_INT + ""); // need special case handle for dual mode for (int mode : sceneModes) { modes.add("" + mode); } diff --git a/src/com/android/camera/imageprocessor/PostProcessor.java b/src/com/android/camera/imageprocessor/PostProcessor.java new file mode 100644 index 000000000..7f0e63990 --- /dev/null +++ b/src/com/android/camera/imageprocessor/PostProcessor.java @@ -0,0 +1,438 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.android.camera.imageprocessor; + +import android.content.ContentResolver; +import android.graphics.ImageFormat; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.hardware.camera2.CaptureRequest; +import android.media.Image; +import android.media.ImageReader; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.util.Log; +import android.widget.Toast; + +import com.android.camera.CameraActivity; +import com.android.camera.CaptureModule; +import com.android.camera.MediaSaveService; +import com.android.camera.PhotoModule; +import com.android.camera.SettingsManager; +import com.android.camera.imageprocessor.filter.OptizoomFilter; +import com.android.camera.ui.RotateTextToast; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import com.android.camera.imageprocessor.filter.ImageFilter; + +public class PostProcessor implements ImageReader.OnImageAvailableListener{ + + private CaptureModule mController; + + private static final String TAG = "PostProcessor"; + public static final int FILTER_NONE = 0; + public static final int FILTER_OPTIZOOM = 1; + public static final int FILTER_MAX = 2; + + private int mCurrentNumImage = 0; + private ImageFilter mFilter; + private int mFilterIndex; + private HandlerThread mHandlerThread; + private ProcessorHandler mHandler; + private CameraActivity mActivity; + private int mWidth; + private int mHeight; + private int mStride; + private Object lock = new Object(); + private ImageFilter.ResultImage mDefaultResultImage; //This is used only no filter is chosen. + private Image[] mImages; + private PhotoModule.NamedImages mNamedImages; + private WatchdogThread mWatchdog; + + //This is for the debug feature. + private static boolean DEBUG_FILTER = true; //TODO: This has to be false before releasing. + private ImageFilter.ResultImage mDebugResultImage; + + @Override + public void onImageAvailable(ImageReader reader) { + try { + Image image = reader.acquireNextImage(); + addImage(image); + if (isReadyToProcess()) { + long captureStartTime = System.currentTimeMillis(); + mNamedImages.nameNewImage(captureStartTime); + PhotoModule.NamedImages.NamedEntity name = mNamedImages.getNextNameEntity(); + String title = (name == null) ? null : name.title; + long date = (name == null) ? -1 : name.date; + processImage(title, date, mController.getMediaSavedListener(), mActivity.getContentResolver()); + } + } catch (IllegalStateException e) { + Log.e(TAG, "Max images has been already acquired. "); + } + } + + enum STATUS { + DEINIT, + INIT, + BUSY + } + private STATUS mStatus = STATUS.DEINIT; + + public PostProcessor(CameraActivity activity, CaptureModule module) { + mController = module; + mActivity = activity; + mNamedImages = new PhotoModule.NamedImages(); + + } + + public boolean isItBusy() { + if(mStatus == STATUS.BUSY) + return true; + return false; + } + + public List setRequiredImages(CaptureRequest.Builder builder) { + if(mFilter == null) { + List list = new ArrayList(); + list.add(builder.build()); + return list; + } else { + return mFilter.setRequiredImages(builder); + } + } + + public boolean isFilterOn() { + if(mFilter == null) + return false; + return true; + } + + public void onOpen(int postFilterId) { + setFilter(postFilterId); + startBackgroundThread(); + + } + + public int getFilterIndex() { + return mFilterIndex; + } + + public void onClose() { + synchronized (lock) { + if(mHandler != null) { + mHandler.setInActive(); + } + stopBackgroundThread(); + } + setFilter(FILTER_NONE); + } + + private void startBackgroundThread() { + mHandlerThread = new HandlerThread("PostProcessorThread"); + mHandlerThread.start(); + mHandler = new ProcessorHandler(mHandlerThread.getLooper()); + + mWatchdog = new WatchdogThread(); + mWatchdog.start(); + } + + class WatchdogThread extends Thread { + private boolean isAlive = true; + private boolean isMonitor = false; + private int counter = 0; + public void run() { + while(isAlive) { + try { + Thread.sleep(200); + }catch(InterruptedException e) { + } + if(isMonitor) { + counter++; + if(counter >= 40) { //This is 4 seconds. + bark(); + break; + } + } + } + + } + + public void startMonitor() { + isMonitor = true; + } + + public void stopMonitor() { + isMonitor = false; + counter = 0; + } + + public void kill() { + isAlive = false; + } + private void bark() { + Log.e(TAG, "It takes too long to get the images and process the filter!"); + int index = getFilterIndex(); + setFilter(FILTER_NONE); + setFilter(index); + } + } + + class ProcessorHandler extends Handler { + boolean isRunning; + + public ProcessorHandler(Looper looper) { + super(looper); + isRunning = true; + } + + public void setInActive() { + isRunning = false; + } + } + + private void stopBackgroundThread() { + if (mHandlerThread != null) { + mHandlerThread.quitSafely(); + try { + mHandlerThread.join(); + } catch (InterruptedException e) { + } + mHandlerThread = null; + mHandler = null; + } + if(mWatchdog != null) { + mWatchdog.kill(); + mWatchdog = null; + } + clear(); + } + + public boolean setFilter(int index) { + if(index < 0 || index >= FILTER_MAX) { + Log.e(TAG, "Invalid scene filter ID"); + return false; + } + synchronized (lock) { + if (mFilter != null) { + mFilter.deinit(); + } + mStatus = STATUS.DEINIT; + switch (index) { + case FILTER_NONE: + mFilter = null; + break; + case FILTER_OPTIZOOM: + mFilter = new OptizoomFilter(mController); + break; + } + } + + if(mFilter != null && !mFilter.isSupported()) { + final String filterName = mFilter.getStringName(); + mFilter = null; + mActivity.runOnUiThread(new Runnable() { + public void run() { + RotateTextToast.makeText(mActivity, filterName+" is not supported. ", Toast.LENGTH_SHORT).show(); + } + }); + } + + if(mFilter == null) { + mFilterIndex = FILTER_NONE; + return false; + } + mFilterIndex = index; + mImages = new Image[mFilter.getNumRequiredImage()]; + return true; + } + + private boolean isReadyToProcess() { + synchronized (lock) { + if (mFilter == null) { + return true; + } + if (mCurrentNumImage >= mFilter.getNumRequiredImage()) { + return true; + } + } + return false; + } + + private void addImage(final Image image) { + if(mHandler == null || !mHandler.isRunning) { + return; + } + final ProcessorHandler handler = mHandler; + if (mStatus == STATUS.DEINIT) { + mWidth = image.getWidth(); + mHeight = image.getHeight(); + mStride = image.getPlanes()[0].getRowStride(); + mStatus = STATUS.INIT; + mHandler.post(new Runnable() { + public void run() { + synchronized (lock) { + if(!handler.isRunning) { + return; + } + if(mFilter == null) { + //Nothing here we have to do if filter is not chosen. + } else { + mFilter.init(mWidth, mHeight, mStride, mStride); + } + } + } + }); + } + if(mCurrentNumImage == 0) { + mStatus = STATUS.BUSY; + if(mWatchdog != null) { + mWatchdog.startMonitor(); + } + } + if(mFilter != null && mCurrentNumImage >= mFilter.getNumRequiredImage()) { + return; + } + final int numImage = mCurrentNumImage; + mCurrentNumImage++; + if(mHandler == null) { + return; + } + mHandler.post(new Runnable() { + public void run() { + synchronized (lock) { + if(!handler.isRunning) { + return; + } + ByteBuffer yBuf = image.getPlanes()[0].getBuffer(); + ByteBuffer vuBuf = image.getPlanes()[2].getBuffer(); + if(mFilter != null && DEBUG_FILTER && numImage == 0) { + mDebugResultImage = new ImageFilter.ResultImage(ByteBuffer.allocateDirect(mStride * mHeight*3/2), + new Rect(0, 0, mWidth, mHeight), mWidth, mHeight, mStride); + yBuf.get(mDebugResultImage.outBuffer.array(), 0, yBuf.remaining()); + vuBuf.get(mDebugResultImage.outBuffer.array(), mStride * mHeight, vuBuf.remaining()); + yBuf.rewind(); + vuBuf.rewind(); + } + if(mFilter == null) { + mDefaultResultImage = new ImageFilter.ResultImage(ByteBuffer.allocateDirect(mStride * mHeight*3/2), + new Rect(0, 0, mWidth, mHeight), mWidth, mHeight, mStride); + yBuf.get(mDefaultResultImage.outBuffer.array(), 0, yBuf.remaining()); + vuBuf.get(mDefaultResultImage.outBuffer.array(), mStride*mHeight, vuBuf.remaining()); + image.close(); + } else { + mFilter.addImage(image.getPlanes()[0].getBuffer(), + image.getPlanes()[2].getBuffer(), numImage, null); + mImages[numImage] = image; + } + } + } + }); + } + + private void clear() { + mCurrentNumImage = 0; + } + + private void processImage(final String title, final long date, + final MediaSaveService.OnMediaSavedListener mediaSavedListener, + final ContentResolver contentResolver) { + if(mHandler == null || !mHandler.isRunning) { + return; + } + final ProcessorHandler handler = mHandler; + mHandler.post(new Runnable() { + public void run() { + byte[] bytes; + ImageFilter.ResultImage resultImage = null; + synchronized (lock) { + if (!handler.isRunning) { + return; + } + if (mFilter == null) { //In case no post filter is chosen + resultImage = mDefaultResultImage; + } else { + resultImage = mFilter.processImage(); + for (int i = 0; i < mImages.length; i++) { + if(mImages[i] != null) { + mImages[i].close(); + mImages[i] = null; + } + } + } + clear(); + mStatus = STATUS.INIT; + if(mWatchdog != null) { + mWatchdog.stopMonitor(); + } + if((resultImage.outRoi.left + resultImage.outRoi.width() > resultImage.width) || + (resultImage.outRoi.top + resultImage.outRoi.height() > resultImage.height) + ) { + Log.e(TAG, "Processed outRoi is not within picture range"); + } else { + if(mFilter != null && DEBUG_FILTER) { + bytes = nv21ToJpeg(mDebugResultImage); + mActivity.getMediaSaveService().addImage( + bytes, title + "_beforeApplyingFilter", date, null, mDebugResultImage.outRoi.width(), mDebugResultImage.outRoi.height(), + 0, null, mediaSavedListener, contentResolver, "jpeg"); + } + bytes = nv21ToJpeg(resultImage); + mController.updateThumbnailJpegData(bytes); + mActivity.getMediaSaveService().addImage( + bytes, title, date, null, resultImage.outRoi.width(), resultImage.outRoi.height(), + 0, null, mediaSavedListener, contentResolver, "jpeg"); + } + } + } + }); + } + + private byte[] nv21ToJpeg(ImageFilter.ResultImage resultImage) { + BitmapOutputStream bos = new BitmapOutputStream(1024); + YuvImage im = new YuvImage(resultImage.outBuffer.array(), ImageFormat.NV21, + resultImage.width, resultImage.height, new int[]{resultImage.stride, resultImage.stride}); + im.compressToJpeg(resultImage.outRoi, 50, bos); + return bos.getArray(); + } + + private class BitmapOutputStream extends ByteArrayOutputStream { + public BitmapOutputStream(int size) { + super(size); + } + + public byte[] getArray() { + return buf; + } + } + + +} diff --git a/src/com/android/camera/imageprocessor/filter/ImageFilter.java b/src/com/android/camera/imageprocessor/filter/ImageFilter.java new file mode 100644 index 000000000..e62d9b30a --- /dev/null +++ b/src/com/android/camera/imageprocessor/filter/ImageFilter.java @@ -0,0 +1,74 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.android.camera.imageprocessor.filter; + +import android.graphics.Rect; +import android.hardware.camera2.CaptureRequest; + +import java.nio.ByteBuffer; +import java.util.List; + +public interface ImageFilter { + + /* Return the number of required images to process*/ + List setRequiredImages(CaptureRequest.Builder builder); + + String getStringName(); + + int getNumRequiredImage(); + + void init(int width, int height, int strideY, int strideVU); + + /* Free all buffer */ + void deinit(); + + /* Adding the image to process */ + void addImage(ByteBuffer bY, ByteBuffer bVU, int imageNum, Object param); + + /* Processing all the added images and return roi*/ + ResultImage processImage(); + + boolean isSupported(); + + class ResultImage { + public ByteBuffer outBuffer; + public Rect outRoi; + public int width; + public int height; + public int stride; + + public ResultImage(ByteBuffer buf, Rect roi, int width, int height, int stride) { + outBuffer = buf; + outRoi = roi; + this.width = width; + this.height = height; + this.stride = stride; + } + } +} diff --git a/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java b/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java new file mode 100644 index 000000000..4773418de --- /dev/null +++ b/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java @@ -0,0 +1,148 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.android.camera.imageprocessor.filter; + +import android.graphics.Rect; +import android.hardware.camera2.CaptureRequest; +import android.util.Log; + +import com.android.camera.CaptureModule; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class OptizoomFilter implements ImageFilter{ + public static final int NUM_REQUIRED_IMAGE = 8; + private int mWidth; + private int mHeight; + private int mStrideY; + private int mStrideVU; + private static String TAG = "OptizoomFilter"; + private static final boolean DEBUG = true; //TODO: Have to be false before releasing. + private int temp; + private static boolean mIsSupported = true; + private ByteBuffer mOutBuf; + private CaptureModule mModule; + + private static void Log(String msg) { + if(DEBUG) { + Log.d(TAG, msg); + } + } + + public OptizoomFilter(CaptureModule module) { + mModule = module; + } + + @Override + public List setRequiredImages(CaptureRequest.Builder builder) { + List list = new ArrayList(); + for(int i=0; i < NUM_REQUIRED_IMAGE; i++) { + list.add(builder.build()); + } + return list; + } + + @Override + public String getStringName() { + return "OptizoomFilter"; + } + + @Override + public int getNumRequiredImage() { + return NUM_REQUIRED_IMAGE; + } + + @Override + public void init(int width, int height, int strideY, int strideVU) { + Log("init"); + mWidth = width/2*2; + mHeight = height/2*2; + mStrideY = strideY/2*2; + mStrideVU = strideVU/2*2; + mOutBuf = ByteBuffer.allocate(mStrideY*mHeight*6); // YUV Buffer to hold (mWidth*2) X (mHeight*2) + Log("width: "+mWidth+" height: "+mHeight+" strideY: "+mStrideY+" strideVU: "+mStrideVU); + nativeInit(mWidth, mHeight, mStrideY, mStrideVU, + 0, 0, mWidth, mHeight, NUM_REQUIRED_IMAGE); + } + + @Override + public void deinit() { + Log("deinit"); + mOutBuf = null; + nativeDeinit(); + } + + @Override + public void addImage(ByteBuffer bY, ByteBuffer bVU, int imageNum, Object param) { + Log("addImage"); + int yActualSize = bY.remaining(); + int vuActualSize = bVU.remaining(); + nativeAddImage(bY, bVU, yActualSize, vuActualSize, imageNum); + } + + @Override + public ResultImage processImage() { + Log("processImage " + mModule.getZoomValue()); + int[] roi = new int[4]; + int status = nativeProcessImage(mOutBuf.array(), mModule.getZoomValue(), roi); + Log("processImage done"); + if(status < 0) { //In failure case, library will return the first image as it is. + Log.w(TAG, "Fail to process the optizoom. It only processes when zoomValue >= 1.5f"); + return new ResultImage(mOutBuf, new Rect(roi[0], roi[1], roi[0]+roi[2], roi[1] + roi[3]), mWidth, mHeight, mStrideY); + } else { //In success case, it will return twice bigger width and height. + return new ResultImage(mOutBuf, new Rect(roi[0], roi[1], roi[0]+roi[2], roi[1] + roi[3]), mWidth*2, mHeight*2, mStrideY*2); + } + } + + @Override + public boolean isSupported() { + return mIsSupported; + } + + public static boolean isSupportedStatic() { + return mIsSupported; + } + + private native int nativeInit(int width, int height, int yStride, int vuStride, + int roiX, int roiY, int roiW, int roiH, int numImages); + private native int nativeDeinit(); + private native int nativeAddImage(ByteBuffer yB, ByteBuffer vuB, int ySize, int vuSize, int imageNum); + private native int nativeProcessImage(byte[] buffer, float zoomLvl, int[] roi); + + static { + try { + System.loadLibrary("jni_optizoom"); + mIsSupported = true; + }catch(UnsatisfiedLinkError e) { + mIsSupported = false; + } + } +} -- cgit v1.2.3