From 12d17d80e1227e9164e7e0aa7524cd39d23bf124 Mon Sep 17 00:00:00 2001 From: Jack Yoo Date: Tue, 7 Jun 2016 09:57:52 -0700 Subject: SnapdragonCamera: FrameListener and Video frame processor Adding FrameListener with TrackingFocus Adding VideoFrameProcessor with beautification Change-Id: Id51a0417af43140ed27ea5859ff939adc2e78c43 CRs-Fixed: 1035245 --- .../camera/imageprocessor/FrameProcessor.java | 228 +++++++++++++++++---- 1 file changed, 184 insertions(+), 44 deletions(-) (limited to 'src/com/android/camera/imageprocessor/FrameProcessor.java') diff --git a/src/com/android/camera/imageprocessor/FrameProcessor.java b/src/com/android/camera/imageprocessor/FrameProcessor.java index 6a2091158..74db08b31 100644 --- a/src/com/android/camera/imageprocessor/FrameProcessor.java +++ b/src/com/android/camera/imageprocessor/FrameProcessor.java @@ -30,14 +30,7 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.android.camera.imageprocessor; import android.app.Activity; -import android.content.Context; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.graphics.Canvas; import android.graphics.ImageFormat; -import android.graphics.Matrix; -import android.graphics.Rect; -import android.graphics.YuvImage; import android.media.Image; import android.media.ImageReader; import android.os.Handler; @@ -45,21 +38,23 @@ import android.os.HandlerThread; import android.renderscript.Allocation; import android.renderscript.Element; import android.renderscript.RenderScript; -import android.renderscript.ScriptIntrinsicYuvToRGB; import android.renderscript.Type; -import android.util.Log; import android.util.Size; import android.view.Surface; +import android.widget.Toast; import com.android.camera.CaptureModule; -import com.android.camera.PhotoModule; +import com.android.camera.SettingsManager; import com.android.camera.imageprocessor.filter.BeautificationFilter; import com.android.camera.imageprocessor.filter.ImageFilter; -import com.android.camera.util.CameraUtil; +import com.android.camera.imageprocessor.filter.TrackingFocusFrameListener; +import com.android.camera.ui.RotateTextToast; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Semaphore; +import org.codeaurora.snapcam.R; public class FrameProcessor { @@ -67,13 +62,17 @@ public class FrameProcessor { private Allocation mInputAllocation; private Allocation mProcessAllocation; private Allocation mOutputAllocation; + private Allocation mVideoOutputAllocation; private HandlerThread mProcessingThread; private Handler mProcessingHandler; private HandlerThread mOutingThread; private Handler mOutingHandler; + private HandlerThread mListeningThread; + private Handler mListeningHandler; - public ProcessingTask mTask; + private ProcessingTask mTask; + private ListeningTask mListeningTask; private RenderScript mRs; private Activity mActivity; ScriptC_YuvToRgb mRsYuvToRGB; @@ -84,10 +83,13 @@ public class FrameProcessor { private ArrayList mPreviewFilters; private ArrayList mFinalFilters; private Surface mSurfaceAsItIs; + private Surface mVideoSurfaceAsItIs; private boolean mIsActive = false; public static final int FILTER_NONE = 0; public static final int FILTER_MAKEUP = 1; + public static final int LISTENER_TRACKING_FOCUS = 2; private CaptureModule mModule; + private boolean mIsVideoOn = false; public FrameProcessor(Activity activity, CaptureModule module) { mActivity = activity; @@ -97,6 +99,7 @@ public class FrameProcessor { } public void init(Size previewDim) { + mIsActive = true; mSize = previewDim; synchronized (mAllocationLock) { mRs = RenderScript.create(mActivity); @@ -122,6 +125,13 @@ public class FrameProcessor { mOutingHandler = new Handler(mOutingThread.getLooper()); } + if(mListeningThread == null) { + mListeningThread = new HandlerThread("FrameListeningThread"); + mListeningThread.start(); + mListeningHandler = new Handler(mListeningThread.getLooper()); + } + + mListeningTask = new ListeningTask(); mTask = new ProcessingTask(); mInputImageReader.setOnImageAvailableListener(mTask, mProcessingHandler); mIsAllocationEverUsed = false; @@ -153,12 +163,12 @@ public class FrameProcessor { } private void cleanFilterSet() { - if(mPreviewFilters != null) { + if (mPreviewFilters != null) { for (ImageFilter filter : mPreviewFilters) { filter.deinit(); } } - if(mFinalFilters != null) { + if (mFinalFilters != null) { for (ImageFilter filter : mFinalFilters) { filter.deinit(); } @@ -168,25 +178,29 @@ public class FrameProcessor { } public void onOpen(ArrayList filterIds) { - mIsActive = true; - synchronized (mAllocationLock) { - cleanFilterSet(); - if (filterIds != null) { - for (Integer i : filterIds) { - addFilter(i.intValue()); - } + cleanFilterSet(); + if (filterIds != null) { + for (Integer i : filterIds) { + addFilter(i.intValue()); } } } private void addFilter(int filterId) { - if(filterId == FILTER_MAKEUP) { - ImageFilter filter = new BeautificationFilter(mModule); - if(filter.isSupported()) { - mPreviewFilters.add(filter); + ImageFilter filter = null; + if (filterId == FILTER_MAKEUP) { + filter = new BeautificationFilter(mModule); + } else if (filterId == LISTENER_TRACKING_FOCUS) { + filter = new TrackingFocusFrameListener(mModule); + } + + if (filter != null && filter.isSupported()) { + mPreviewFilters.add(filter); + if (!filter.isFrameListener()) { mFinalFilters.add(filter); } } + } public void onClose() { @@ -202,6 +216,9 @@ public class FrameProcessor { if (mProcessAllocation != null) { mProcessAllocation.destroy(); } + if (mVideoOutputAllocation != null) { + mVideoOutputAllocation.destroy(); + } } if (mRs != null) { mRs.destroy(); @@ -210,6 +227,7 @@ public class FrameProcessor { mProcessAllocation = null; mOutputAllocation = null; mInputAllocation = null; + mVideoOutputAllocation = null; } if (mProcessingThread != null) { mProcessingThread.quitSafely(); @@ -229,6 +247,15 @@ public class FrameProcessor { } catch (InterruptedException e) { } } + if (mListeningThread != null) { + mListeningThread.quitSafely(); + try { + mListeningThread.join(); + mListeningThread = null; + mListeningHandler = null; + } catch (InterruptedException e) { + } + } for(ImageFilter filter : mPreviewFilters) { filter.deinit(); } @@ -237,32 +264,75 @@ public class FrameProcessor { } } - public Surface getInputSurface() { - if(mPreviewFilters.size() == 0) { - return mSurfaceAsItIs; - } + private Surface getReaderSurface() { synchronized (mAllocationLock) { - if (mInputImageReader == null) + if (mInputImageReader == null) { return null; + } return mInputImageReader.getSurface(); } } + public List getInputSurfaces() { + List surfaces = new ArrayList(); + if(mPreviewFilters.size() == 0 && mFinalFilters.size() == 0) { + surfaces.add(mSurfaceAsItIs); + if(mIsVideoOn) { + surfaces.add(mVideoSurfaceAsItIs); + } + } else if (mFinalFilters.size() == 0) { + surfaces.add(mSurfaceAsItIs); + if(mIsVideoOn) { + surfaces.add(mVideoSurfaceAsItIs); + } + surfaces.add(getReaderSurface()); + } else { + surfaces.add(getReaderSurface()); + } + return surfaces; + } + public boolean isFrameFilterEnabled() { - if(mPreviewFilters.size() == 0) { + if(mFinalFilters.size() == 0) { return false; } return true; } public void setOutputSurface(Surface surface) { - if(mPreviewFilters.size() == 0) { - mSurfaceAsItIs = surface; - } else { + mSurfaceAsItIs = surface; + if(mFinalFilters.size() != 0) { mOutputAllocation.setSurface(surface); } } + public void setVideoOutputSurface(Surface surface) { + if(surface == null) { + synchronized (mAllocationLock) { + if (mVideoOutputAllocation != null) { + mVideoOutputAllocation.destroy(); + } + mVideoOutputAllocation = null; + } + mIsVideoOn = false; + return; + } + mVideoSurfaceAsItIs = surface; + mIsVideoOn = true; + if(mFinalFilters.size() != 0) { + synchronized (mAllocationLock) { + if (mVideoOutputAllocation == null) { + Type.Builder rgbTypeBuilder = new Type.Builder(mRs, Element.RGBA_8888(mRs)); + rgbTypeBuilder.setX(mSize.getHeight()); + rgbTypeBuilder.setY(mSize.getWidth()); + mVideoOutputAllocation = Allocation.createTyped(mRs, rgbTypeBuilder.create(), + Allocation.USAGE_SCRIPT | Allocation.USAGE_IO_OUTPUT); + } + mVideoOutputAllocation.setSurface(surface); + } + } + } + class ProcessingTask implements Runnable, ImageReader.OnImageAvailableListener { byte[] yvuBytes = null; int ySize; @@ -276,12 +346,14 @@ public class FrameProcessor { @Override public void onImageAvailable(ImageReader reader) { synchronized (mAllocationLock) { - if(mOutputAllocation == null) + if(mOutputAllocation == null) { return; + } try { Image image = reader.acquireLatestImage(); - if(image == null) + if(image == null) { return; + } if(!mIsActive) { image.close(); return; @@ -289,23 +361,35 @@ public class FrameProcessor { mIsAllocationEverUsed = true; ByteBuffer bY = image.getPlanes()[0].getBuffer(); ByteBuffer bVU = image.getPlanes()[2].getBuffer(); - if(yvuBytes == null) { + if(yvuBytes == null || width != mSize.getWidth() || height != mSize.getHeight()) { stride = image.getPlanes()[0].getRowStride(); width = mSize.getWidth(); height = mSize.getHeight(); ySize = stride * mSize.getHeight(); yvuBytes = new byte[ySize*3/2]; } + boolean needToFeedSurface = false; //Start processing yvu buf for (ImageFilter filter : mPreviewFilters) { - filter.init(mSize.getWidth(), mSize.getHeight(), stride, stride); - filter.addImage(bY, bVU, 0, new Boolean(true)); + if(filter.isFrameListener()) { + if (mListeningTask.setParam(filter, bY, bVU, mSize.getWidth(), mSize.getHeight(), stride)) { + mListeningHandler.post(mListeningTask); + } + } else { + filter.init(mSize.getWidth(), mSize.getHeight(), stride, stride); + filter.addImage(bY, bVU, 0, new Boolean(true)); + needToFeedSurface = true; + } + bY.rewind(); + bVU.rewind(); } //End processing yvu buf - bY.get(yvuBytes, 0, bY.remaining()); - bVU.get(yvuBytes, ySize, bVU.remaining()); + if(needToFeedSurface) { + bY.get(yvuBytes, 0, bY.remaining()); + bVU.get(yvuBytes, ySize, bVU.remaining()); + mOutingHandler.post(this); + } image.close(); - mOutingHandler.post(this); } catch (IllegalStateException e) { } } @@ -318,12 +402,68 @@ public class FrameProcessor { return; } if(mInputAllocation == null) { - createAllocation(stride, height, stride-width); + createAllocation(stride, height, stride - width); } mInputAllocation.copyFrom(yvuBytes); mRsRotator.forEach_rotate90andMerge(mInputAllocation); mRsYuvToRGB.forEach_nv21ToRgb(mOutputAllocation); mOutputAllocation.ioSend(); + if(mVideoOutputAllocation != null) { + mVideoOutputAllocation.copyFrom(mOutputAllocation); + mVideoOutputAllocation.ioSend(); + } + } + } + } + + class ListeningTask implements Runnable { + + ImageFilter mFilter; + ByteBuffer mBY = null, mBVU = null; + int mWidth, mHeight, mStride; + int bYSize, bVUSize; + Semaphore mMutureLock = new Semaphore(1); + + public boolean setParam(ImageFilter filter, ByteBuffer bY, ByteBuffer bVU, int width, int height, int stride) { + if(!mIsActive) { + return false; + } + if (!mMutureLock.tryAcquire()) { + return false; + } + mFilter = filter; + if (mBY == null || bYSize != bY.remaining()) { + bYSize = bY.remaining(); + mBY = ByteBuffer.allocateDirect(bYSize); + } + if (mBVU == null || bVUSize != bVU.remaining()) { + bVUSize = bVU.remaining(); + mBVU = ByteBuffer.allocateDirect(bVUSize); + } + mBY.rewind(); + mBVU.rewind(); + mBY.put(bY); + mBVU.put(bVU); + mWidth = width; + mHeight = height; + mStride = stride; + mMutureLock.release(); + return true; + } + + @Override + public void run() { + try { + if (!mIsActive) { + return; + } + mMutureLock.acquire(); + mBY.rewind(); + mBVU.rewind(); + mFilter.init(mWidth, mHeight, mStride, mStride); + mFilter.addImage(mBY, mBVU, 0, new Boolean(true)); + mMutureLock.release(); + } catch (InterruptedException e) { } } } -- cgit v1.2.3