diff options
Diffstat (limited to 'src/com/android/camera/imageprocessor')
6 files changed, 477 insertions, 44 deletions
diff --git a/src/com/android/camera/imageprocessor/FrameProcessor.java b/src/com/android/camera/imageprocessor/FrameProcessor.java index 6a2091158..74db08b31 100644 --- a/src/com/android/camera/imageprocessor/FrameProcessor.java +++ b/src/com/android/camera/imageprocessor/FrameProcessor.java @@ -30,14 +30,7 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.android.camera.imageprocessor; import android.app.Activity; -import android.content.Context; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.graphics.Canvas; import android.graphics.ImageFormat; -import android.graphics.Matrix; -import android.graphics.Rect; -import android.graphics.YuvImage; import android.media.Image; import android.media.ImageReader; import android.os.Handler; @@ -45,21 +38,23 @@ import android.os.HandlerThread; import android.renderscript.Allocation; import android.renderscript.Element; import android.renderscript.RenderScript; -import android.renderscript.ScriptIntrinsicYuvToRGB; import android.renderscript.Type; -import android.util.Log; import android.util.Size; import android.view.Surface; +import android.widget.Toast; import com.android.camera.CaptureModule; -import com.android.camera.PhotoModule; +import com.android.camera.SettingsManager; import com.android.camera.imageprocessor.filter.BeautificationFilter; import com.android.camera.imageprocessor.filter.ImageFilter; -import com.android.camera.util.CameraUtil; +import com.android.camera.imageprocessor.filter.TrackingFocusFrameListener; +import com.android.camera.ui.RotateTextToast; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Semaphore; +import org.codeaurora.snapcam.R; public class FrameProcessor { @@ -67,13 +62,17 @@ public class FrameProcessor { private Allocation mInputAllocation; private Allocation mProcessAllocation; private Allocation mOutputAllocation; + private Allocation mVideoOutputAllocation; private HandlerThread mProcessingThread; private Handler mProcessingHandler; private HandlerThread mOutingThread; private Handler mOutingHandler; + private HandlerThread mListeningThread; + private Handler mListeningHandler; - public ProcessingTask mTask; + private ProcessingTask mTask; + private ListeningTask mListeningTask; private RenderScript mRs; private Activity mActivity; ScriptC_YuvToRgb mRsYuvToRGB; @@ -84,10 +83,13 @@ public class FrameProcessor { private ArrayList<ImageFilter> mPreviewFilters; private ArrayList<ImageFilter> mFinalFilters; private Surface mSurfaceAsItIs; + private Surface mVideoSurfaceAsItIs; private boolean mIsActive = false; public static final int FILTER_NONE = 0; public static final int FILTER_MAKEUP = 1; + public static final int LISTENER_TRACKING_FOCUS = 2; private CaptureModule mModule; + private boolean mIsVideoOn = false; public FrameProcessor(Activity activity, CaptureModule module) { mActivity = activity; @@ -97,6 +99,7 @@ public class FrameProcessor { } public void init(Size previewDim) { + mIsActive = true; mSize = previewDim; synchronized (mAllocationLock) { mRs = RenderScript.create(mActivity); @@ -122,6 +125,13 @@ public class FrameProcessor { mOutingHandler = new Handler(mOutingThread.getLooper()); } + if(mListeningThread == null) { + mListeningThread = new HandlerThread("FrameListeningThread"); + mListeningThread.start(); + mListeningHandler = new Handler(mListeningThread.getLooper()); + } + + mListeningTask = new ListeningTask(); mTask = new ProcessingTask(); mInputImageReader.setOnImageAvailableListener(mTask, mProcessingHandler); mIsAllocationEverUsed = false; @@ -153,12 +163,12 @@ public class FrameProcessor { } private void cleanFilterSet() { - if(mPreviewFilters != null) { + if (mPreviewFilters != null) { for (ImageFilter filter : mPreviewFilters) { filter.deinit(); } } - if(mFinalFilters != null) { + if (mFinalFilters != null) { for (ImageFilter filter : mFinalFilters) { filter.deinit(); } @@ -168,25 +178,29 @@ public class FrameProcessor { } public void onOpen(ArrayList<Integer> filterIds) { - mIsActive = true; - synchronized (mAllocationLock) { - cleanFilterSet(); - if (filterIds != null) { - for (Integer i : filterIds) { - addFilter(i.intValue()); - } + cleanFilterSet(); + if (filterIds != null) { + for (Integer i : filterIds) { + addFilter(i.intValue()); } } } private void addFilter(int filterId) { - if(filterId == FILTER_MAKEUP) { - ImageFilter filter = new BeautificationFilter(mModule); - if(filter.isSupported()) { - mPreviewFilters.add(filter); + ImageFilter filter = null; + if (filterId == FILTER_MAKEUP) { + filter = new BeautificationFilter(mModule); + } else if (filterId == LISTENER_TRACKING_FOCUS) { + filter = new TrackingFocusFrameListener(mModule); + } + + if (filter != null && filter.isSupported()) { + mPreviewFilters.add(filter); + if (!filter.isFrameListener()) { mFinalFilters.add(filter); } } + } public void onClose() { @@ -202,6 +216,9 @@ public class FrameProcessor { if (mProcessAllocation != null) { mProcessAllocation.destroy(); } + if (mVideoOutputAllocation != null) { + mVideoOutputAllocation.destroy(); + } } if (mRs != null) { mRs.destroy(); @@ -210,6 +227,7 @@ public class FrameProcessor { mProcessAllocation = null; mOutputAllocation = null; mInputAllocation = null; + mVideoOutputAllocation = null; } if (mProcessingThread != null) { mProcessingThread.quitSafely(); @@ -229,6 +247,15 @@ public class FrameProcessor { } catch (InterruptedException e) { } } + if (mListeningThread != null) { + mListeningThread.quitSafely(); + try { + mListeningThread.join(); + mListeningThread = null; + mListeningHandler = null; + } catch (InterruptedException e) { + } + } for(ImageFilter filter : mPreviewFilters) { filter.deinit(); } @@ -237,32 +264,75 @@ public class FrameProcessor { } } - public Surface getInputSurface() { - if(mPreviewFilters.size() == 0) { - return mSurfaceAsItIs; - } + private Surface getReaderSurface() { synchronized (mAllocationLock) { - if (mInputImageReader == null) + if (mInputImageReader == null) { return null; + } return mInputImageReader.getSurface(); } } + public List<Surface> getInputSurfaces() { + List<Surface> surfaces = new ArrayList<Surface>(); + if(mPreviewFilters.size() == 0 && mFinalFilters.size() == 0) { + surfaces.add(mSurfaceAsItIs); + if(mIsVideoOn) { + surfaces.add(mVideoSurfaceAsItIs); + } + } else if (mFinalFilters.size() == 0) { + surfaces.add(mSurfaceAsItIs); + if(mIsVideoOn) { + surfaces.add(mVideoSurfaceAsItIs); + } + surfaces.add(getReaderSurface()); + } else { + surfaces.add(getReaderSurface()); + } + return surfaces; + } + public boolean isFrameFilterEnabled() { - if(mPreviewFilters.size() == 0) { + if(mFinalFilters.size() == 0) { return false; } return true; } public void setOutputSurface(Surface surface) { - if(mPreviewFilters.size() == 0) { - mSurfaceAsItIs = surface; - } else { + mSurfaceAsItIs = surface; + if(mFinalFilters.size() != 0) { mOutputAllocation.setSurface(surface); } } + public void setVideoOutputSurface(Surface surface) { + if(surface == null) { + synchronized (mAllocationLock) { + if (mVideoOutputAllocation != null) { + mVideoOutputAllocation.destroy(); + } + mVideoOutputAllocation = null; + } + mIsVideoOn = false; + return; + } + mVideoSurfaceAsItIs = surface; + mIsVideoOn = true; + if(mFinalFilters.size() != 0) { + synchronized (mAllocationLock) { + if (mVideoOutputAllocation == null) { + Type.Builder rgbTypeBuilder = new Type.Builder(mRs, Element.RGBA_8888(mRs)); + rgbTypeBuilder.setX(mSize.getHeight()); + rgbTypeBuilder.setY(mSize.getWidth()); + mVideoOutputAllocation = Allocation.createTyped(mRs, rgbTypeBuilder.create(), + Allocation.USAGE_SCRIPT | Allocation.USAGE_IO_OUTPUT); + } + mVideoOutputAllocation.setSurface(surface); + } + } + } + class ProcessingTask implements Runnable, ImageReader.OnImageAvailableListener { byte[] yvuBytes = null; int ySize; @@ -276,12 +346,14 @@ public class FrameProcessor { @Override public void onImageAvailable(ImageReader reader) { synchronized (mAllocationLock) { - if(mOutputAllocation == null) + if(mOutputAllocation == null) { return; + } try { Image image = reader.acquireLatestImage(); - if(image == null) + if(image == null) { return; + } if(!mIsActive) { image.close(); return; @@ -289,23 +361,35 @@ public class FrameProcessor { mIsAllocationEverUsed = true; ByteBuffer bY = image.getPlanes()[0].getBuffer(); ByteBuffer bVU = image.getPlanes()[2].getBuffer(); - if(yvuBytes == null) { + if(yvuBytes == null || width != mSize.getWidth() || height != mSize.getHeight()) { stride = image.getPlanes()[0].getRowStride(); width = mSize.getWidth(); height = mSize.getHeight(); ySize = stride * mSize.getHeight(); yvuBytes = new byte[ySize*3/2]; } + boolean needToFeedSurface = false; //Start processing yvu buf for (ImageFilter filter : mPreviewFilters) { - filter.init(mSize.getWidth(), mSize.getHeight(), stride, stride); - filter.addImage(bY, bVU, 0, new Boolean(true)); + if(filter.isFrameListener()) { + if (mListeningTask.setParam(filter, bY, bVU, mSize.getWidth(), mSize.getHeight(), stride)) { + mListeningHandler.post(mListeningTask); + } + } else { + filter.init(mSize.getWidth(), mSize.getHeight(), stride, stride); + filter.addImage(bY, bVU, 0, new Boolean(true)); + needToFeedSurface = true; + } + bY.rewind(); + bVU.rewind(); } //End processing yvu buf - bY.get(yvuBytes, 0, bY.remaining()); - bVU.get(yvuBytes, ySize, bVU.remaining()); + if(needToFeedSurface) { + bY.get(yvuBytes, 0, bY.remaining()); + bVU.get(yvuBytes, ySize, bVU.remaining()); + mOutingHandler.post(this); + } image.close(); - mOutingHandler.post(this); } catch (IllegalStateException e) { } } @@ -318,12 +402,68 @@ public class FrameProcessor { return; } if(mInputAllocation == null) { - createAllocation(stride, height, stride-width); + createAllocation(stride, height, stride - width); } mInputAllocation.copyFrom(yvuBytes); mRsRotator.forEach_rotate90andMerge(mInputAllocation); mRsYuvToRGB.forEach_nv21ToRgb(mOutputAllocation); mOutputAllocation.ioSend(); + if(mVideoOutputAllocation != null) { + mVideoOutputAllocation.copyFrom(mOutputAllocation); + mVideoOutputAllocation.ioSend(); + } + } + } + } + + class ListeningTask implements Runnable { + + ImageFilter mFilter; + ByteBuffer mBY = null, mBVU = null; + int mWidth, mHeight, mStride; + int bYSize, bVUSize; + Semaphore mMutureLock = new Semaphore(1); + + public boolean setParam(ImageFilter filter, ByteBuffer bY, ByteBuffer bVU, int width, int height, int stride) { + if(!mIsActive) { + return false; + } + if (!mMutureLock.tryAcquire()) { + return false; + } + mFilter = filter; + if (mBY == null || bYSize != bY.remaining()) { + bYSize = bY.remaining(); + mBY = ByteBuffer.allocateDirect(bYSize); + } + if (mBVU == null || bVUSize != bVU.remaining()) { + bVUSize = bVU.remaining(); + mBVU = ByteBuffer.allocateDirect(bVUSize); + } + mBY.rewind(); + mBVU.rewind(); + mBY.put(bY); + mBVU.put(bVU); + mWidth = width; + mHeight = height; + mStride = stride; + mMutureLock.release(); + return true; + } + + @Override + public void run() { + try { + if (!mIsActive) { + return; + } + mMutureLock.acquire(); + mBY.rewind(); + mBVU.rewind(); + mFilter.init(mWidth, mHeight, mStride, mStride); + mFilter.addImage(mBY, mBVU, 0, new Boolean(true)); + mMutureLock.release(); + } catch (InterruptedException e) { } } } diff --git a/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java b/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java index 6ec9376d0..7a13895f4 100644 --- a/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java +++ b/src/com/android/camera/imageprocessor/filter/BeautificationFilter.java @@ -120,6 +120,11 @@ public class BeautificationFilter implements ImageFilter { return mIsSupported; } + @Override + public boolean isFrameListener() { + return false; + } + private native int nativeBeautificationProcess(ByteBuffer yB, ByteBuffer vuB, int width, int height, int stride, int fleft, int ftop, int fright, int fbottom); diff --git a/src/com/android/camera/imageprocessor/filter/ImageFilter.java b/src/com/android/camera/imageprocessor/filter/ImageFilter.java index e62d9b30a..b778dee03 100644 --- a/src/com/android/camera/imageprocessor/filter/ImageFilter.java +++ b/src/com/android/camera/imageprocessor/filter/ImageFilter.java @@ -71,4 +71,6 @@ public interface ImageFilter { this.stride = stride; } } + + boolean isFrameListener(); } diff --git a/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java b/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java index 9b5af29dc..16b8ae163 100644 --- a/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java +++ b/src/com/android/camera/imageprocessor/filter/OptizoomFilter.java @@ -127,6 +127,11 @@ public class OptizoomFilter implements ImageFilter{ return mIsSupported; } + @Override + public boolean isFrameListener() { + return false; + } + public static boolean isSupportedStatic() { return mIsSupported; } diff --git a/src/com/android/camera/imageprocessor/filter/SharpshooterFilter.java b/src/com/android/camera/imageprocessor/filter/SharpshooterFilter.java index 74469afc3..e123f3439 100644 --- a/src/com/android/camera/imageprocessor/filter/SharpshooterFilter.java +++ b/src/com/android/camera/imageprocessor/filter/SharpshooterFilter.java @@ -150,6 +150,11 @@ public class SharpshooterFilter implements ImageFilter{ return mIsSupported; } + @Override + public boolean isFrameListener() { + return false; + } + public static boolean isSupportedStatic() { return mIsSupported; } diff --git a/src/com/android/camera/imageprocessor/filter/TrackingFocusFrameListener.java b/src/com/android/camera/imageprocessor/filter/TrackingFocusFrameListener.java new file mode 100644 index 000000000..a12eef91a --- /dev/null +++ b/src/com/android/camera/imageprocessor/filter/TrackingFocusFrameListener.java @@ -0,0 +1,276 @@ +/* +Copyright (c) 2016, The Linux Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of The Linux Foundation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.android.camera.imageprocessor.filter; + +import android.graphics.Point; +import android.graphics.Rect; +import android.hardware.Camera; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.params.Face; +import android.util.Log; +import android.util.Size; + +import com.android.camera.CaptureModule; +import com.android.camera.ui.FilmstripBottomControls; +import com.android.camera.ui.TrackingFocusRenderer; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.List; + +public class TrackingFocusFrameListener implements ImageFilter { + + int mWidth; + int mHeight; + int mStrideY; + int mStrideVU; + private CaptureModule mModule; + private static boolean DEBUG = false; + private static String TAG = "TrackingFocusFrameListener"; + private static boolean mIsSupported = false; + private Rect imageRect; + public static final int PENDING_REGISTRATION = -1; + public static final int MAX_NUM_TRACKED_OBJECTS = 3; + private int mTrackedId = PENDING_REGISTRATION; + private boolean mIsInitialzed = false; + private TrackingFocusRenderer mTrackingFocusRender; + byte[] yvuBytes = null; + private int[] mInputCords = null; + private boolean mIsFirstTime = true; + + public enum OperationMode { + DEFAULT, + PERFORMANCE, + CPU_OFFLOAD, + LOW_POWER + } + + public enum Precision { + HIGH, + LOW + } + + public TrackingFocusFrameListener(CaptureModule module) { + mModule = module; + } + + @Override + public List<CaptureRequest> setRequiredImages(CaptureRequest.Builder builder) { + return null; + } + + @Override + public String getStringName() { + return "TrackingFocusFrameListener"; + } + + @Override + public int getNumRequiredImage() { + return 1; + } + + @Override + public void init(int width, int height, int strideY, int strideVU) { + mWidth = width; + mHeight = height; + mStrideY = strideY; + mStrideVU = strideVU; + if(!mIsInitialzed) { + if (nInit(OperationMode.PERFORMANCE.ordinal(), Precision.HIGH.ordinal(), mWidth, mHeight, mStrideY) < 0) { + Log.e(TAG, "Initialization failed."); + } + imageRect = new Rect(0, 0, width, height); + mTrackingFocusRender = mModule.getTrackingForcusRenderer(); + yvuBytes = new byte[mStrideY*mHeight*3/2]; + mIsInitialzed = true; + } + } + + @Override + public void deinit() { + if (mIsInitialzed) { + nRelease(); + mIsInitialzed = false; + } + } + + @Override + public void addImage(ByteBuffer bY, ByteBuffer bVU, int imageNum, Object isPreview) { + bY.get(yvuBytes, 0, bY.remaining()); + bVU.get(yvuBytes, mStrideY * mHeight, bVU.remaining()); + int[] cords = mTrackingFocusRender.getInputCords(mWidth, mHeight); + if(cords != null) { + if(mTrackedId != PENDING_REGISTRATION) { + unregisterObject(mTrackedId); + mTrackedId = PENDING_REGISTRATION; + } + mIsFirstTime = true; + mInputCords = cords; + } + if(mInputCords != null) { + if (mTrackedId == PENDING_REGISTRATION) { + try { + mTrackedId = registerObject(yvuBytes, new Point(mInputCords[0], mInputCords[1]), mIsFirstTime); + mIsFirstTime = false; + }catch(IllegalArgumentException e) { + mTrackedId = PENDING_REGISTRATION; + Log.e(TAG, e.toString()); + } + } + if(mTrackedId != PENDING_REGISTRATION) { + mTrackingFocusRender.putRegisteredCords(trackObjects(yvuBytes), mWidth, mHeight); + } + } + } + + public static class Result { + public final int id; + public final int confidence; + public Rect pos; + + private Result(int id, int confidence, int left, int top, int right, int bottom) { + this.id = id; + this.confidence = confidence; + this.pos = new Rect(left, top, right, bottom); + } + + public static Result Copy(Result old) { + Result result = new Result(old.id, old.confidence, old.pos.left, old.pos.top, old.pos.right, old.pos.bottom); + return result; + } + } + + public int getMinRoiDimension() { + if (!mIsInitialzed) { + throw new IllegalArgumentException("already released"); + } + + return nGetMinRoiDimension(); + } + + public int getMaxRoiDimension() { + if (!mIsInitialzed) { + throw new IllegalArgumentException("already released"); + } + + return nGetMaxRoiDimension(); + } + + public int registerObject(byte[] imageDataNV21, Rect rect) + { + if (imageDataNV21 == null || imageDataNV21.length < getMinFrameSize()) { + throw new IllegalArgumentException("imageDataNV21 null or too small to encode frame"); + } else if (rect == null || rect.isEmpty() || !imageRect.contains(rect)) { + throw new IllegalArgumentException("rect must be non-empty and be entirely inside " + + "the frame"); + } else if (!mIsInitialzed) { + throw new IllegalArgumentException("already released"); + } + int id = nRegisterObjectByRect(imageDataNV21, rect.left, rect.top, rect.right, rect.bottom); + if(id == 0) { + id = PENDING_REGISTRATION; + } + mTrackedId = id; + return mTrackedId; + } + + public int registerObject(byte[] imageDataNV21, Point point, boolean firstTime) + { + if (imageDataNV21 == null || imageDataNV21.length < getMinFrameSize()) { + throw new IllegalArgumentException("imageDataNV21 null or too small to encode frame" + + imageDataNV21.length+ " "+getMinFrameSize()); + } else if (point == null || !imageRect.contains(point.x, point.y)) { + throw new IllegalArgumentException("point is outside the image frame: "+imageRect.toString()); + } else if (!mIsInitialzed) { + throw new IllegalArgumentException("already released"); + } + int id = nRegisterObjectByPoint(imageDataNV21, point.x, point.y, firstTime); + if(id == 0) { + id = PENDING_REGISTRATION; + } + mTrackedId = id; + return mTrackedId; + } + + public void unregisterObject(int id) + { + if (id == PENDING_REGISTRATION) { + Log.e(TAG, "There's a pending object"); + } else if (!mIsInitialzed) { + Log.e(TAG, "already released"); + } + nUnregisterObject(id); + } + + public Result trackObjects(byte[] imageDataNV21) + { + if (imageDataNV21 == null || imageDataNV21.length < getMinFrameSize()) { + Log.e(TAG, "imageDataNV21 null or too small to encode frame " + + imageDataNV21.length+ " "+getMinFrameSize()); + } else if (!mIsInitialzed) { + Log.e(TAG, "It's released"); + } + + int[] nResults = nTrackObjects(imageDataNV21); + return new Result(nResults[0], nResults[1], nResults[2], nResults[3], nResults[4], nResults[5]); + } + + private int getMinFrameSize() { + return ((mStrideY * imageRect.bottom * 3) / 2); + } + + @Override + public ResultImage processImage() { + return null; + } + + @Override + public boolean isSupported() { + return mIsSupported; + } + + @Override + public boolean isFrameListener() { + return true; + } + + private native int nInit(int operationMode, int precision, int width, int height, int stride); + private native void nRelease(); + private native int nGetMinRoiDimension(); + private native int nGetMaxRoiDimension(); + private native int nRegisterObjectByRect(byte[] imageDataNV21, int left, int top, int right, int bottom); + private native int nRegisterObjectByPoint(byte[] imageDataNV21, int x, int y, boolean firstTime); + private native void nUnregisterObject(int id); + private native int[] nTrackObjects(byte[] imageDataNV21); + + static { + mIsSupported = false; + } +} |