From f3db9356da2a5cdb8372e186c597c83c52d421e6 Mon Sep 17 00:00:00 2001 From: Jack Yoo Date: Wed, 6 Apr 2016 16:07:22 -0700 Subject: SnapdragonCamera: FrameProcessor Introducing Frameprocessor with beautifiation. Change-Id: Ie6d8f4157a7d0c1a21e6f347457e84685e397286 CRs-Fixed: 1023183 --- src/com/android/camera/CaptureModule.java | 176 +++++++++-- src/com/android/camera/CaptureUI.java | 3 +- src/com/android/camera/SettingsManager.java | 1 + .../camera/imageprocessor/FrameProcessor.java | 335 +++++++++++++++++++++ .../camera/imageprocessor/PostProcessor.java | 16 +- .../filter/BeautificationFilter.java | 134 +++++++++ 6 files changed, 638 insertions(+), 27 deletions(-) create mode 100644 src/com/android/camera/imageprocessor/FrameProcessor.java create mode 100644 src/com/android/camera/imageprocessor/filter/BeautificationFilter.java (limited to 'src') diff --git a/src/com/android/camera/CaptureModule.java b/src/com/android/camera/CaptureModule.java index 8810c6df4..0f65f21c3 100644 --- a/src/com/android/camera/CaptureModule.java +++ b/src/com/android/camera/CaptureModule.java @@ -26,6 +26,7 @@ import android.content.Context; import android.content.DialogInterface; import android.content.Intent; import android.content.res.Configuration; +import android.graphics.Camera; import android.graphics.ImageFormat; import android.graphics.Point; import android.graphics.Rect; @@ -39,6 +40,7 @@ import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; +import android.hardware.camera2.params.Face; import android.hardware.camera2.params.MeteringRectangle; import android.hardware.camera2.params.StreamConfigurationMap; import android.media.CameraProfile; @@ -53,6 +55,7 @@ import android.os.Message; import android.util.Log; import android.util.Size; import android.util.SparseIntArray; +import android.view.Display; import android.view.KeyEvent; import android.view.OrientationEventListener; import android.view.Surface; @@ -60,7 +63,9 @@ import android.view.SurfaceHolder; import android.view.View; import android.widget.Toast; +import com.android.camera.imageprocessor.filter.ImageFilter; import com.android.camera.imageprocessor.PostProcessor; +import com.android.camera.imageprocessor.FrameProcessor; import com.android.camera.PhotoModule.NamedImages; import com.android.camera.PhotoModule.NamedImages.NamedEntity; import com.android.camera.ui.CountDownView; @@ -75,6 +80,7 @@ import org.codeaurora.snapcam.filter.ClearSightImageProcessor; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.LinkedList; import java.util.List; @@ -197,6 +203,11 @@ public class CaptureModule implements CameraModule, PhotoController, * A {@link Handler} for running tasks in the background. */ private PostProcessor mPostProcessor; + private FrameProcessor mFrameProcessor; + private Size mFrameProcPreviewOutputSize; + private Face[] mPreviewFaces = null; + private Face[] mStickyFaces = null; + private Rect mBayerCameraRegion; private Handler mCameraHandler; private Handler mImageAvailableHandler; private Handler mCaptureCallbackHandler; @@ -302,6 +313,20 @@ public class CaptureModule implements CameraModule, PhotoController, * camera. */ private Semaphore mCameraOpenCloseLock = new Semaphore(1); + + + public Face[] getPreviewFaces() { + return mPreviewFaces; + } + + public Face[] getStickyFaces() { + return mStickyFaces; + } + + public Rect getCameraRegion() { + return mBayerCameraRegion; + } + /** * A {@link CameraCaptureSession.CaptureCallback} that handles events related to JPEG capture. */ @@ -310,6 +335,7 @@ public class CaptureModule implements CameraModule, PhotoController, private void process(CaptureResult result) { int id = (int) result.getRequest().getTag(); + if (!mFirstPreviewLoaded) { mActivity.runOnUiThread(new Runnable() { @Override @@ -319,6 +345,13 @@ public class CaptureModule implements CameraModule, PhotoController, }); mFirstPreviewLoaded = true; } + + Face[] faces = result.get(CaptureResult.STATISTICS_FACES); + mPreviewFaces = faces; + if(faces != null && faces.length != 0) { + mStickyFaces = faces; + } + switch (mState[id]) { case STATE_PREVIEW: { break; @@ -540,17 +573,31 @@ public class CaptureModule implements CameraModule, PhotoController, mNamedImages = new NamedImages(); } + public ArrayList getFrameFilters() { + if(mFrameProcessor == null) { + return new ArrayList(); + } else { + return mFrameProcessor.getFrameFilters(); + } + } + + private void applyFaceDetect(CaptureRequest.Builder builder, int id) { + if(id == getMainCameraId()) { + builder.set(CaptureRequest.STATISTICS_FACE_DETECT_MODE, + CameraMetadata.STATISTICS_FACE_DETECT_MODE_SIMPLE); + } + } + private void createSession(final int id) { if (mPaused || !mCameraOpened[id] || !mSurfaceReady) return; Log.d(TAG, "createSession " + id); List list = new LinkedList(); try { - Surface surface = getPreviewSurface(id); + Surface surface = getPreviewSurfaceForSession(id); // We set up a CaptureRequest.Builder with the output Surface. mPreviewRequestBuilder[id] = mCameraDevice[id].createCaptureRequest(CameraDevice .TEMPLATE_PREVIEW); mPreviewRequestBuilder[id].setTag(id); - mPreviewRequestBuilder[id].addTarget(surface); CameraCaptureSession.StateCallback captureSessionCallback = new CameraCaptureSession.StateCallback() { @@ -571,7 +618,6 @@ public class CaptureModule implements CameraModule, PhotoController, // Finally, we start displaying the camera preview. mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id] .build(), mCaptureCallback, mCameraHandler); - if (isClearSightOn()) { ClearSightImageProcessor.getInstance().onCaptureSessionConfigured(id == BAYER_ID, cameraCaptureSession); } @@ -603,12 +649,28 @@ public class CaptureModule implements CameraModule, PhotoController, } }; - list.add(surface); - if(isClearSightOn()) { + mPreviewRequestBuilder[id].addTarget(surface); + list.add(surface); ClearSightImageProcessor.getInstance().createCaptureSession( - id==BAYER_ID, mCameraDevice[id], list, captureSessionCallback); + id == BAYER_ID, mCameraDevice[id], list, captureSessionCallback); + } else if (id == getMainCameraId()) { + if(mFrameProcessor.isFrameFilterEnabled()) { + mFrameProcessor.init(mFrameProcPreviewOutputSize); + mActivity.runOnUiThread(new Runnable() { + public void run() { + mUI.getSurfaceHolder().setFixedSize(mFrameProcPreviewOutputSize.getHeight(), mFrameProcPreviewOutputSize.getWidth()); + } + }); + } + mFrameProcessor.setOutputSurface(surface); + mPreviewRequestBuilder[id].addTarget(mFrameProcessor.getInputSurface()); + list.add(mFrameProcessor.getInputSurface()); + list.add(mImageReader[id].getSurface()); + mCameraDevice[id].createCaptureSession(list, captureSessionCallback, null); } else { + mPreviewRequestBuilder[id].addTarget(surface); + list.add(surface); list.add(mImageReader[id].getSurface()); // Here, we create a CameraCaptureSession for camera preview. mCameraDevice[id].createCaptureSession(list, captureSessionCallback, null); @@ -659,6 +721,7 @@ public class CaptureModule implements CameraModule, PhotoController, } mPostProcessor = new PostProcessor(mActivity, this); + mFrameProcessor = new FrameProcessor(mActivity, this); setCurrentMode(); mContentResolver = mActivity.getContentResolver(); @@ -742,7 +805,6 @@ public class CaptureModule implements CameraModule, PhotoController, mControlAFMode = CaptureRequest.CONTROL_AF_MODE_AUTO; applySettingsForAutoFocus(builder, id); - mState[id] = STATE_WAITING_TOUCH_FOCUS; mCaptureSession[id].capture(builder.build(), mCaptureCallback, mCameraHandler); setAFModeToPreview(id, mControlAFMode); @@ -775,10 +837,6 @@ public class CaptureModule implements CameraModule, PhotoController, } } - /** - * Capture a still picture. This method should be called when we get a response in - * {@link #mCaptureCallback} from both {@link #lockFocus()}. - */ private void captureStillPicture(final int id) { Log.d(TAG, "captureStillPicture " + id); try { @@ -809,7 +867,7 @@ public class CaptureModule implements CameraModule, PhotoController, if(csEnabled) { ClearSightImageProcessor.getInstance().capture( id==BAYER_ID, mCaptureSession[id], captureBuilder, mCaptureCallbackHandler); - } else if(id == BAYER_ID && mPostProcessor.isFilterOn()) { + } else if(id == getMainCameraId() && mPostProcessor.isFilterOn()) { captureBuilder.addTarget(mImageReader[id].getSurface()); List captureList = mPostProcessor.setRequiredImages(captureBuilder); mCaptureSession[id].captureBurst(captureList, new CameraCaptureSession.CaptureCallback() { @@ -940,22 +998,56 @@ public class CaptureModule implements CameraModule, PhotoController, } } + private void determineFrameProcPreviewOutputSize(List sizeList, float targetRatio) { + Display display = mActivity.getWindowManager().getDefaultDisplay(); + Point ds = new Point(); + display.getSize(ds); + int i=0, j=0, width, height; + float ratio; + for(; i < sizeList.size(); i++) { + width = sizeList.get(i).getHeight(); + height = sizeList.get(i).getWidth(); + ratio = (float)height/width; + if(ds.x >= width || ds.y >= height) { + if(j == 0) { + j = i; + } + if(ratio < targetRatio + 0.2f && ratio > targetRatio - 0.2f) { + break; + } + } + } + if(i == sizeList.size()) { + if(j != 0) { + mFrameProcPreviewOutputSize = sizeList.get(j); + } else { + mFrameProcPreviewOutputSize = sizeList.get(sizeList.size()-1); + } + } else { + mFrameProcPreviewOutputSize = sizeList.get(i); + } + } /** * Sets up member variables related to camera. * * @param width The width of available size for camera preview * @param height The height of available size for camera preview */ - private void setUpCameraOutputs(int imageFomat) { + private void setUpCameraOutputs(int imageFormat) { Log.d(TAG, "setUpCameraOutputs"); CameraManager manager = (CameraManager) mActivity.getSystemService(Context.CAMERA_SERVICE); try { String[] cameraIdList = manager.getCameraIdList(); for (int i = 0; i < cameraIdList.length; i++) { String cameraId = cameraIdList[i]; + CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); if (isInMode(i)) mCameraIdList.add(i); + if(i == getMainCameraId()) { + mBayerCameraRegion = characteristics.get(CameraCharacteristics + .SENSOR_INFO_ACTIVE_ARRAY_SIZE); + } StreamConfigurationMap map = characteristics.get( CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); if (map == null) { @@ -971,10 +1063,10 @@ public class CaptureModule implements CameraModule, PhotoController, if (i == getMainCameraId()) { Point screenSize = new Point(); mActivity.getWindowManager().getDefaultDisplay().getSize(screenSize); - Size[] prevSizes = map.getOutputSizes(imageFomat); - Size prevSize = getOptimalPreviewSize(size, prevSizes, screenSize.x, + Size[] prevSizes = map.getOutputSizes(imageFormat); + mFrameProcPreviewOutputSize = getOptimalPreviewSize(size, prevSizes, screenSize.x, screenSize.y); - mUI.setPreviewSize(prevSize.getWidth(), prevSize.getHeight()); + mUI.setPreviewSize(mFrameProcPreviewOutputSize.getWidth(), mFrameProcPreviewOutputSize.getHeight()); } if (isClearSightOn()) { ClearSightImageProcessor.getInstance().init(size.getWidth(), size.getHeight(), @@ -982,11 +1074,13 @@ public class CaptureModule implements CameraModule, PhotoController, ClearSightImageProcessor.getInstance().setCallback(this); } else { // No Clearsight - mImageReader[i] = ImageReader.newInstance(size.getWidth(), size.getHeight(), - imageFomat, MAX_IMAGE_NUM); - - if(mPostProcessor.isFilterOn() && i == BAYER_ID) { + mImageReader[i] = ImageReader.newInstance(size.getWidth(), size.getHeight(), imageFormat, MAX_IMAGE_NUM); + if(mPostProcessor.isFilterOn() && i == getMainCameraId()) { mImageReader[i].setOnImageAvailableListener(mPostProcessor, mImageAvailableHandler); +// if(mFrameProcessor.isFrameFilterEnabled()) { +// determineFrameProcPreviewOutputSize(Arrays.asList(map.getOutputSizes(imageFormat)), +// (float) size.getWidth() / (float) size.getHeight()); +// } } else { mImageReader[i].setOnImageAvailableListener(new ImageAvailableListener(i) { @Override @@ -1033,7 +1127,6 @@ public class CaptureModule implements CameraModule, PhotoController, builder.addTarget(getPreviewSurface(id)); applySettingsForUnlockFocus(builder, id); - mCaptureSession[id].capture(builder.build(), mCaptureCallback, mCameraHandler); mState[id] = STATE_PREVIEW; mControlAFMode = CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE; @@ -1069,6 +1162,9 @@ public class CaptureModule implements CameraModule, PhotoController, if(mPostProcessor != null) { mPostProcessor.onClose(); } + if(mFrameProcessor != null) { + mFrameProcessor.onClose(); + } for (int i = 0; i < MAX_NUM_CAM; i++) { if (null != mCaptureSession[i]) { if (mIsLinked) { @@ -1113,6 +1209,7 @@ public class CaptureModule implements CameraModule, PhotoController, builder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START); applyAFRegions(builder, id); applyCommonSettings(builder, id); + applyFaceDetect(builder, id); } private void applySettingsForCapture(CaptureRequest.Builder builder, int id) { @@ -1125,12 +1222,14 @@ public class CaptureModule implements CameraModule, PhotoController, builder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); applyCommonSettings(builder, id); + applyFaceDetect(builder, id); } private void applySettingsForUnlockFocus(CaptureRequest.Builder builder, int id) { builder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL); applyCommonSettings(builder, id); + applyFaceDetect(builder, id); } private void applySettingsForAutoFocus(CaptureRequest.Builder builder, int id) { @@ -1138,6 +1237,7 @@ public class CaptureModule implements CameraModule, PhotoController, .CONTROL_AF_TRIGGER_START); applyAFRegions(builder, id); applyCommonSettings(builder, id); + applyFaceDetect(builder, id); } private void applyCommonSettings(CaptureRequest.Builder builder, int id) { @@ -1254,6 +1354,16 @@ public class CaptureModule implements CameraModule, PhotoController, mCurrentMode = isBackCamera() ? getCameraMode() : FRONT_MODE; } + private ArrayList getFrameProcFilterId() { + String scene = mSettingsManager.getValue(SettingsManager.KEY_MAKEUP); + ArrayList filters = new ArrayList(); + if(scene != null && scene.equalsIgnoreCase("on")) { + filters.add(FrameProcessor.FILTER_MAKEUP); + } + + return filters; + } + private int getPostProcFilterId() { String scene = mSettingsManager.getValue(SettingsManager.KEY_SCENE_MODE); if (scene != null) { @@ -1281,6 +1391,9 @@ public class CaptureModule implements CameraModule, PhotoController, Log.d(TAG, "Chosen postproc filter id : "+getPostProcFilterId()); mPostProcessor.onOpen(getPostProcFilterId()); } + if(mFrameProcessor != null) { + mFrameProcessor.onOpen(getFrameProcFilterId()); + } if(mPostProcessor.isFilterOn()) { setUpCameraOutputs(ImageFormat.YUV_420_888); } else { @@ -1551,7 +1664,6 @@ public class CaptureModule implements CameraModule, PhotoController, @Override public void onPreviewUIDestroyed() { - } @Override @@ -1764,6 +1876,7 @@ public class CaptureModule implements CameraModule, PhotoController, mPreviewRequestBuilder[id].set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest .CONTROL_AF_TRIGGER_IDLE); applyCommonSettings(mPreviewRequestBuilder[id], id); + applyFaceDetect(mPreviewRequestBuilder[id], id); } public float getZoomValue() { @@ -1816,6 +1929,7 @@ public class CaptureModule implements CameraModule, PhotoController, applyIso(mPreviewRequestBuilder[cameraId]); break; } + applyFaceDetect(mPreviewRequestBuilder[cameraId], cameraId); return updatePreview; } @@ -1915,6 +2029,18 @@ public class CaptureModule implements CameraModule, PhotoController, } private Surface getPreviewSurface(int id) { + if (isBackCamera()) { + if (getCameraMode() == DUAL_MODE && id == MONO_ID) { + return mUI.getSurfaceHolder2().getSurface(); + } else { + return mFrameProcessor.getInputSurface(); + } + } else { + return mFrameProcessor.getInputSurface(); + } + } + + private Surface getPreviewSurfaceForSession(int id) { if (isBackCamera()) { if (getCameraMode() == DUAL_MODE && id == MONO_ID) { return mUI.getSurfaceHolder2().getSurface(); @@ -2018,6 +2144,9 @@ public class CaptureModule implements CameraModule, PhotoController, case SettingsManager.KEY_MONO_PREVIEW: if (count == 0) restart(); return; + case SettingsManager.KEY_MAKEUP: + restart(); + return; case SettingsManager.KEY_SCENE_MODE: if (count == 0 && checkNeedToRestart(value)) { restart(); @@ -2107,7 +2236,8 @@ public class CaptureModule implements CameraModule, PhotoController, float prevRatio = (float) prevSize.getWidth() / prevSize.getHeight(); if (Math.abs(prevRatio - ratio) < 0.01) { // flip w and h - if (prevSize.getWidth() <= screenH && prevSize.getHeight() <= screenW) { + if (prevSize.getWidth() <= screenH && prevSize.getHeight() <= screenW && + prevSize.getWidth() <= pictureSize.getWidth() && prevSize.getHeight() <= pictureSize.getHeight()) { return prevSize; } else { optimal = prevSize; diff --git a/src/com/android/camera/CaptureUI.java b/src/com/android/camera/CaptureUI.java index 50104570d..5b8ecad6e 100644 --- a/src/com/android/camera/CaptureUI.java +++ b/src/com/android/camera/CaptureUI.java @@ -92,7 +92,8 @@ public class CaptureUI implements FocusOverlayManager.FocusUI, SettingsManager.KEY_LONGSHOT, SettingsManager.KEY_EXPOSURE, SettingsManager.KEY_WHITE_BALANCE, - SettingsManager.KEY_CAMERA2 + SettingsManager.KEY_CAMERA2, + SettingsManager.KEY_MAKEUP }; String[] mDeveloperKeys = new String[]{ SettingsManager.KEY_MONO_ONLY, diff --git a/src/com/android/camera/SettingsManager.java b/src/com/android/camera/SettingsManager.java index d19cab2a6..215ab8436 100644 --- a/src/com/android/camera/SettingsManager.java +++ b/src/com/android/camera/SettingsManager.java @@ -68,6 +68,7 @@ public class SettingsManager implements ListMenu.SettingsListener { public static final String KEY_FOCUS_MODE = "pref_camera2_focusmode_key"; public static final String KEY_FLASH_MODE = "pref_camera2_flashmode_key"; public static final String KEY_WHITE_BALANCE = "pref_camera2_whitebalance_key"; + public static final String KEY_MAKEUP = "pref_camera2_makeup_key"; public static final String KEY_CAMERA2 = "pref_camera2_camera2_key"; public static final String KEY_MONO_ONLY = "pref_camera2_mono_only_key"; public static final String KEY_MONO_PREVIEW = "pref_camera2_mono_preview_key"; diff --git a/src/com/android/camera/imageprocessor/FrameProcessor.java b/src/com/android/camera/imageprocessor/FrameProcessor.java new file mode 100644 index 000000000..951479de9 --- /dev/null +++ b/src/com/android/camera/imageprocessor/FrameProcessor.java @@ -0,0 +1,335 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.android.camera.imageprocessor; + +import android.app.Activity; +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image; +import android.media.ImageReader; +import android.os.Handler; +import android.os.HandlerThread; +import android.renderscript.Allocation; +import android.renderscript.Element; +import android.renderscript.RenderScript; +import android.renderscript.ScriptIntrinsicYuvToRGB; +import android.renderscript.Type; +import android.util.Log; +import android.util.Size; +import android.view.Surface; + +import com.android.camera.CaptureModule; +import com.android.camera.PhotoModule; +import com.android.camera.imageprocessor.filter.BeautificationFilter; +import com.android.camera.imageprocessor.filter.ImageFilter; +import com.android.camera.util.CameraUtil; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; + +public class FrameProcessor { + + private ImageReader mInputImageReader; + private Allocation mInputAllocation; + private Allocation mProcessAllocation; + private Allocation mOutputAllocation; + + private HandlerThread mProcessingThread; + private Handler mProcessingHandler; + private HandlerThread mOutingThread; + private Handler mOutingHandler; + + public ProcessingTask mTask; + private RenderScript mRs; + private Activity mActivity; + ScriptC_YuvToRgb mRsYuvToRGB; + ScriptC_rotator mRsRotator; + private Size mSize; + private Object mAllocationLock = new Object(); + private boolean mIsAllocationEverUsed; + private ArrayList mPreviewFilters; + private ArrayList mFinalFilters; + private Surface mSurfaceAsItIs; + private boolean mIsActive = false; + public static final int FILTER_NONE = 0; + public static final int FILTER_MAKEUP = 1; + private CaptureModule mModule; + + public FrameProcessor(Activity activity, CaptureModule module) { + mActivity = activity; + mModule = module; + mPreviewFilters = new ArrayList(); + mFinalFilters = new ArrayList(); + } + + public void init(Size previewDim) { + mSize = previewDim; + synchronized (mAllocationLock) { + mRs = RenderScript.create(mActivity); + mRsYuvToRGB = new ScriptC_YuvToRgb(mRs); + mRsRotator = new ScriptC_rotator(mRs); + mInputImageReader = ImageReader.newInstance(mSize.getWidth(), mSize.getHeight(), ImageFormat.YUV_420_888, 8); + + Type.Builder rgbTypeBuilder = new Type.Builder(mRs, Element.RGBA_8888(mRs)); + rgbTypeBuilder.setX(mSize.getHeight()); + rgbTypeBuilder.setY(mSize.getWidth()); + mOutputAllocation = Allocation.createTyped(mRs, rgbTypeBuilder.create(), + Allocation.USAGE_SCRIPT | Allocation.USAGE_IO_OUTPUT); + + if (mProcessingThread == null) { + mProcessingThread = new HandlerThread("FrameProcessor"); + mProcessingThread.start(); + mProcessingHandler = new Handler(mProcessingThread.getLooper()); + } + + if (mOutingThread == null) { + mOutingThread = new HandlerThread("FrameOutingThread"); + mOutingThread.start(); + mOutingHandler = new Handler(mOutingThread.getLooper()); + } + + mTask = new ProcessingTask(); + mInputImageReader.setOnImageAvailableListener(mTask, mProcessingHandler); + mIsAllocationEverUsed = false; + } + } + + private void createAllocation(int width, int height) { + Type.Builder yuvTypeBuilder = new Type.Builder(mRs, Element.YUV(mRs)); + yuvTypeBuilder.setX(width); + yuvTypeBuilder.setY(height); + yuvTypeBuilder.setYuvFormat(ImageFormat.NV21); + mInputAllocation = Allocation.createTyped(mRs, yuvTypeBuilder.create(), Allocation.USAGE_SCRIPT); + Type.Builder nv21TypeBuilder = new Type.Builder(mRs, Element.U8(mRs)); + nv21TypeBuilder.setX(width * height * 3 / 2); + mProcessAllocation = Allocation.createTyped(mRs, nv21TypeBuilder.create(), Allocation.USAGE_SCRIPT); + mRsRotator.set_gIn(mInputAllocation); + mRsRotator.set_gOut(mProcessAllocation); + mRsRotator.set_width(width); + mRsRotator.set_height(height); + mRsYuvToRGB.set_gIn(mProcessAllocation); + mRsYuvToRGB.set_width(height); + mRsYuvToRGB.set_height(width); + } + + public ArrayList getFrameFilters() { + return mFinalFilters; + } + + private void cleanFilterSet() { + if(mPreviewFilters != null) { + for (ImageFilter filter : mPreviewFilters) { + filter.deinit(); + } + } + if(mFinalFilters != null) { + for (ImageFilter filter : mFinalFilters) { + filter.deinit(); + } + } + mPreviewFilters = new ArrayList(); + mFinalFilters = new ArrayList(); + } + + public void onOpen(ArrayList filterIds) { + mIsActive = true; + synchronized (mAllocationLock) { + cleanFilterSet(); + if (filterIds != null) { + for (Integer i : filterIds) { + addFilter(i.intValue()); + } + } + } + } + + private void addFilter(int filterId) { + if(filterId == FILTER_MAKEUP) { + ImageFilter filter = new BeautificationFilter(mModule); + if(filter.isSupported()) { + mPreviewFilters.add(filter); + mFinalFilters.add(filter); + } + } + } + + public void onClose() { + mIsActive = false; + synchronized (mAllocationLock) { + if (mIsAllocationEverUsed) { + if (mInputAllocation != null) { + mInputAllocation.destroy(); + } + if (mOutputAllocation != null) { + mOutputAllocation.destroy(); + } + if (mProcessAllocation != null) { + mProcessAllocation.destroy(); + } + } + if (mRs != null) { + mRs.destroy(); + } + mRs = null; + mProcessAllocation = null; + mOutputAllocation = null; + mInputAllocation = null; + } + if (mProcessingThread != null) { + mProcessingThread.quitSafely(); + try { + mProcessingThread.join(); + mProcessingThread = null; + mProcessingHandler = null; + } catch (InterruptedException e) { + } + } + if (mOutingThread != null) { + mOutingThread.quitSafely(); + try { + mOutingThread.join(); + mOutingThread = null; + mOutingHandler = null; + } catch (InterruptedException e) { + } + } + for(ImageFilter filter : mPreviewFilters) { + filter.deinit(); + } + for(ImageFilter filter : mFinalFilters) { + filter.deinit(); + } + } + + public Surface getInputSurface() { + if(mPreviewFilters.size() == 0) { + return mSurfaceAsItIs; + } + synchronized (mAllocationLock) { + if (mInputImageReader == null) + return null; + return mInputImageReader.getSurface(); + } + } + + public boolean isFrameFilterEnabled() { + if(mPreviewFilters.size() == 0) { + return false; + } + return true; + } + + public void setOutputSurface(Surface surface) { + if(mPreviewFilters.size() == 0) { + mSurfaceAsItIs = surface; + } else { + mOutputAllocation.setSurface(surface); + } + } + + class ProcessingTask implements Runnable, ImageReader.OnImageAvailableListener { + byte[] yvuBytes = null; + int ySize; + int stride; + int height; + + public ProcessingTask() { + } + + @Override + public void onImageAvailable(ImageReader reader) { + synchronized (mAllocationLock) { + if(mOutputAllocation == null) + return; + try { + Image image = reader.acquireLatestImage(); + if(image == null) + return; + if(!mIsActive) { + image.close(); + return; + } + mIsAllocationEverUsed = true; + ByteBuffer bY = image.getPlanes()[0].getBuffer(); + ByteBuffer bVU = image.getPlanes()[2].getBuffer(); + if(yvuBytes == null) { + stride = image.getPlanes()[0].getRowStride(); + height = mSize.getHeight(); + ySize = stride * mSize.getHeight(); + yvuBytes = new byte[ySize*3/2]; + } + //Start processing yvu buf + for (ImageFilter filter : mPreviewFilters) { + filter.init(mSize.getWidth(), mSize.getHeight(), stride, stride); + filter.addImage(bY, bVU, 0, new Boolean(true)); + } + //End processing yvu buf + bY.get(yvuBytes, 0, bY.remaining()); + bVU.get(yvuBytes, ySize, bVU.remaining()); + image.close(); + mOutingHandler.post(this); + } catch (IllegalStateException e) { + } + } + } + + @Override + public void run() { + synchronized (mAllocationLock) { + if(!mIsActive) { + return; + } + if(mInputAllocation == null) { + createAllocation(stride, height); + } + mInputAllocation.copyFrom(yvuBytes); + mRsRotator.forEach_rotate90andMerge(mInputAllocation); + mRsYuvToRGB.forEach_nv21ToRgb(mOutputAllocation); + mOutputAllocation.ioSend(); + } + } + } + + private native int nativeRotateNV21(ByteBuffer inBuf, int imageWidth, int imageHeight, int degree, ByteBuffer outBuf); + + private native int nativeNV21toRgb(ByteBuffer yvuBuf, ByteBuffer rgbBuf, int width, int height); + + static { + System.loadLibrary("jni_imageutil"); + } +} + diff --git a/src/com/android/camera/imageprocessor/PostProcessor.java b/src/com/android/camera/imageprocessor/PostProcessor.java index 7f0e63990..a126e8817 100644 --- a/src/com/android/camera/imageprocessor/PostProcessor.java +++ b/src/com/android/camera/imageprocessor/PostProcessor.java @@ -132,9 +132,13 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ } public boolean isFilterOn() { - if(mFilter == null) - return false; - return true; + if(mFilter != null) { + return true; + } + if(mController.getFrameFilters().size() != 0) { + return true; + } + return false; } public void onOpen(int postFilterId) { @@ -389,6 +393,12 @@ public class PostProcessor implements ImageReader.OnImageAvailableListener{ } } } + //Start processing FrameProcessor filter as well + for (ImageFilter filter : mController.getFrameFilters()) { + filter.init(resultImage.width, resultImage.height, resultImage.stride, resultImage.stride); + filter.addImage(resultImage.outBuffer, null, 0, new Boolean(false)); + } + //End processing FrameProessor filter clear(); mStatus = STATUS.INIT; if(mWatchdog != null) { diff --git a/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java b/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java new file mode 100644 index 000000000..6ec9376d0 --- /dev/null +++ b/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java @@ -0,0 +1,134 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.android.camera.imageprocessor.filter; + +import android.graphics.Rect; +import android.hardware.Camera; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.params.Face; +import android.util.Log; +import android.util.Size; + +import com.android.camera.CaptureModule; +import com.android.camera.ui.FilmstripBottomControls; + +import java.nio.ByteBuffer; +import java.util.List; + +public class BeautificationFilter implements ImageFilter { + + int mWidth; + int mHeight; + int mStrideY; + int mStrideVU; + private CaptureModule mModule; + private static boolean DEBUG = false; + private static String TAG = "BeautificationFilter"; + private static boolean mIsSupported = false; + + public BeautificationFilter(CaptureModule module) { + mModule = module; + } + + @Override + public List setRequiredImages(CaptureRequest.Builder builder) { + return null; + } + + @Override + public String getStringName() { + return "BeautificationFilter"; + } + + @Override + public int getNumRequiredImage() { + return 0; + } + + @Override + public void init(int width, int height, int strideY, int strideVU) { + mWidth = width; + mHeight = height; + mStrideY = strideY; + mStrideVU = strideVU; + } + + @Override + public void deinit() { + + } + + @Override + public void addImage(ByteBuffer bY, ByteBuffer bVU, int imageNum, Object isPreview) { + Rect back = mModule.getCameraRegion(); + Face[] faces; + if(((Boolean)isPreview).booleanValue()) { + faces = mModule.getPreviewFaces(); + } else { + faces = mModule.getStickyFaces(); + } + float widthRatio = (float)mWidth/back.width(); + float heightRatio = (float)mHeight/back.height(); + if(faces == null || faces.length == 0) + return; + Rect rect = faces[0].getBounds(); + int value = nativeBeautificationProcess(bY, bVU, mWidth, mHeight, mStrideY, + (int)(rect.left*widthRatio), (int)(rect.top*heightRatio), + (int)(rect.right*widthRatio), (int)(rect.bottom*heightRatio)); + if(DEBUG && value < 0) { + if(value == -1) { + Log.d(TAG, "library initialization is failed."); + } else if(value == -2) { + Log.d(TAG, "No face is recognized"); + } + } + } + + @Override + public ResultImage processImage() { + return null; + } + + @Override + public boolean isSupported() { + return mIsSupported; + } + + private native int nativeBeautificationProcess(ByteBuffer yB, ByteBuffer vuB, + int width, int height, int stride, int fleft, int ftop, int fright, int fbottom); + + static { + try { + System.loadLibrary("jni_makeup"); + mIsSupported = true; + }catch(UnsatisfiedLinkError e) { + mIsSupported = false; + } + } +} -- cgit v1.2.3