From 2f15393db4fb2d22281578ecfe10e7e27f64a47f Mon Sep 17 00:00:00 2001 From: Jay Wang Date: Fri, 29 Apr 2016 17:24:03 -0700 Subject: SnapdragonCamera: add OTP calib data support and fix AE state Add support to receive, parse, and send OTP calibration data from HAL to ClearSight library and add check for AE convergence state. CRs-Fixed: 993611 Change-Id: I52d6b71dc7bde4e08342afee7aa9d41bb9a4dde6 --- .../snapcam/filter/ClearSightImageProcessor.java | 69 ++++-- .../snapcam/filter/ClearSightNativeEngine.java | 235 ++++++++++++++++++--- 2 files changed, 251 insertions(+), 53 deletions(-) (limited to 'src/org/codeaurora') diff --git a/src/org/codeaurora/snapcam/filter/ClearSightImageProcessor.java b/src/org/codeaurora/snapcam/filter/ClearSightImageProcessor.java index 45856c752..141d8e412 100644 --- a/src/org/codeaurora/snapcam/filter/ClearSightImageProcessor.java +++ b/src/org/codeaurora/snapcam/filter/ClearSightImageProcessor.java @@ -35,16 +35,21 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; +import org.codeaurora.snapcam.filter.ClearSightNativeEngine.CamSystemCalibrationData; +import org.codeaurora.snapcam.filter.ClearSightNativeEngine.ClearsightImage; + +import android.content.Context; import android.graphics.ImageFormat; import android.graphics.Rect; import android.graphics.YuvImage; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCaptureSession.CaptureCallback; +import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; -import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.InputConfiguration; import android.media.Image; @@ -63,7 +68,6 @@ import android.view.Surface; import com.android.camera.MediaSaveService; import com.android.camera.PhotoModule.NamedImages; import com.android.camera.PhotoModule.NamedImages.NamedEntity; -import org.codeaurora.snapcam.filter.ClearSightNativeEngine.ClearsightImage; public class ClearSightImageProcessor { private static final String TAG = "ClearSightImageProcessor"; @@ -82,10 +86,10 @@ public class ClearSightImageProcessor { private static final int CAM_TYPE_MONO = 1; private static final int NUM_CAM = 2; - private static CaptureResult.Key OTP_CALIB_BLOB = - new CaptureResult.Key<>( + private static CameraCharacteristics.Key OTP_CALIB_BLOB = + new CameraCharacteristics.Key<>( "org.codeaurora.qcamera3.dualcam_calib_meta_data.dualcam_calib_meta_data_blob", - Byte.class); + byte[].class); private NamedImages mNamedImages; private ImageReader[] mImageReader = new ImageReader[NUM_CAM]; @@ -131,7 +135,7 @@ public class ClearSightImageProcessor { return mInstance; } - public void init(int width, int height) { + public void init(int width, int height, Context context) { mImageProcessThread = new HandlerThread("CameraImageProcess"); mImageProcessThread.start(); mImageReprocessThread = new HandlerThread("CameraImageReprocess"); @@ -144,6 +148,15 @@ public class ClearSightImageProcessor { mImageReader[CAM_TYPE_MONO] = createImageReader(CAM_TYPE_MONO, width, height); mReprocessImageReader[CAM_TYPE_BAYER] = createReprocImageReader(CAM_TYPE_BAYER, width, height); mReprocessImageReader[CAM_TYPE_MONO] = createReprocImageReader(CAM_TYPE_MONO, width, height); + + CameraManager cm = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); + try { + CameraCharacteristics cc = cm.getCameraCharacteristics("0"); + byte[] blob = cc.get(OTP_CALIB_BLOB); + ClearSightNativeEngine.setOtpCalibData(CamSystemCalibrationData.createFromBytes(blob)); + } catch (CameraAccessException e) { + e.printStackTrace(); + } } public void close() { @@ -419,12 +432,17 @@ public class ClearSightImageProcessor { // if timestamps are within threshold, keep frames if (Math.abs(bayer.mImage.getTimestamp() - mono.mImage.getTimestamp()) > mTimestampThresholdNs) { - Log.d(TAG, "checkForValidFramePair - toss pair"); - // no match, toss - bayer = mBayerFrames.poll(); - mono = mMonoFrames.poll(); - bayer.mImage.close(); - mono.mImage.close(); + if(bayer.mImage.getTimestamp() > mono.mImage.getTimestamp()) { + Log.d(TAG, "checkForValidFramePair - toss mono"); + // no match, toss + mono = mMonoFrames.poll(); + mono.mImage.close(); + } else { + Log.d(TAG, "checkForValidFramePair - toss bayer"); + // no match, toss + bayer = mBayerFrames.poll(); + bayer.mImage.close(); + } } } } @@ -447,16 +465,17 @@ public class ClearSightImageProcessor { private void processReprocess() { if(mCallback != null) { - if (mBayerFrames.size() != mMonoFrames.size() + if (mMonoFrames.isEmpty() || mBayerFrames.isEmpty()) { - Log.d(TAG, "processReprocess - frame size mismatch or empty"); + Log.d(TAG, "processReprocess - frames are empty"); releaseBayerFrames(); releaseMonoFrames(); mCallback.onClearSightFailure(null, null); return; } else { - sendReprocessRequests(CAM_TYPE_BAYER); - sendReprocessRequests(CAM_TYPE_MONO); + int frameCount = Math.min(mMonoFrames.size(), mBayerFrames.size()); + sendReprocessRequests(CAM_TYPE_BAYER, frameCount); + sendReprocessRequests(CAM_TYPE_MONO, frameCount); } } else { releaseBayerFrames(); @@ -464,7 +483,7 @@ public class ClearSightImageProcessor { } } - private void sendReprocessRequests(final int camId) { + private void sendReprocessRequests(final int camId, int frameCount) { CameraCaptureSession session = mCallback.onReprocess(camId == CAM_TYPE_BAYER); CameraDevice device = session.getDevice(); @@ -476,11 +495,12 @@ public class ClearSightImageProcessor { frameQueue = mMonoFrames; } Log.d(TAG, "sendReprocessRequests - start cam: " + camId - + " num frames: " + frameQueue.size()); + + " num frames: " + frameQueue.size() + + " frameCount: " + frameCount); ArrayList reprocRequests = new ArrayList( frameQueue.size()); - while (!frameQueue.isEmpty()) { + while (reprocRequests.size() < frameCount) { ReprocessableImage reprocImg = frameQueue.poll(); CaptureRequest.Builder reprocRequest = device @@ -492,6 +512,15 @@ public class ClearSightImageProcessor { mImageWriter[camId].queueInputImage(reprocImg.mImage); } + if(!frameQueue.isEmpty()) { + // clear remaining frames + if (camId == CAM_TYPE_BAYER) { + releaseBayerFrames(); + } else { + releaseMonoFrames(); + } + } + mImageReprocessHandler.obtainMessage(MSG_START_CAPTURE, camId, reprocRequests.size()).sendToTarget(); session.captureBurst(reprocRequests, @@ -504,8 +533,6 @@ public class ClearSightImageProcessor { super.onCaptureCompleted(session, request, result); Log.d(TAG, "reprocess - onCaptureCompleted: " + camId); - // TODO: parse OTP Calib data to be used in final CS - // result.get(OTP_CALIB_BLOB); } @Override diff --git a/src/org/codeaurora/snapcam/filter/ClearSightNativeEngine.java b/src/org/codeaurora/snapcam/filter/ClearSightNativeEngine.java index d6f0899ad..c79d15a84 100644 --- a/src/org/codeaurora/snapcam/filter/ClearSightNativeEngine.java +++ b/src/org/codeaurora/snapcam/filter/ClearSightNativeEngine.java @@ -31,6 +31,7 @@ package org.codeaurora.snapcam.filter; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; @@ -42,6 +43,7 @@ import android.media.Image.Plane; import android.util.Log; public class ClearSightNativeEngine { + private static final boolean DEBUG = false; private static final String TAG = "ClearSightNativeEngine"; static { try { @@ -59,32 +61,8 @@ public class ClearSightNativeEngine { private static final int Y_PLANE = 0; private static final int VU_PLANE = 2; - // dummy OTP calib data - private static final String otp_calib = "Calibration OTP format version = 10301\n" - + "Main Native Sensor Resolution width = 4224px\n" - + "Main Native Sensor Resolution height = 3136px\n" - + "Main Calibration Resolution width = 1280px\n" - + "Main Calibration Resolution height = 950px\n" - + "Main Focal length ratio = 1.004896\n" - + "Aux Native Sensor Resolution width = 1600px\n" - + "Aux Native Sensor Resolution height = 1200px\n" - + "Aux Calibration Resolution width = 1280px\n" - + "Aux Calibration Resolution height = 960px\n" - + "Aux Focal length ratio = 1.000000\n" - + "Relative Rotation matrix [0] through [8] = 1.000000,-0.003008,0.000251,0.003073,1.000189,0.003329,0.019673,-0.003329,1.000284\n" - + "Relative Geometric surface parameters [0] through [31] = -0.307164,-0.879074,4.636152,0.297486,-0.157539,-6.889396,0.109467,-2.797022,-0.066306,-0.120142,0.196464,0.021974,2.905827,0.241197,0.048328,-5.116615,0.496533,-5.263813,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000\n" - + "Relative Principal point X axis offset (ox) = 0.000000px\n" - + "Relative Principal point Y axis offset (oy) = 0.000000px\n" - + "Relative position flag = 1\n" - + "Baseline distance = 20.000000mm\n" - + "Main sensor mirror and flip setting = 3\n" - + "Aux sensor mirror and flip setting = 3\n" - + "Module orientation during calibration = 0\n" - + "Rotation flag = 0\n" - + "Main Normalized Focal length = 1000.0px\n" - + "Aux Normalized Focal length = 1000.0px"; - private static boolean mLibLoaded; + private static CamSystemCalibrationData mOtpCalibData; private static ClearSightNativeEngine mInstance; private Image mRefColorImage; @@ -106,6 +84,10 @@ public class ClearSightNativeEngine { return mInstance; } + public static void setOtpCalibData(CamSystemCalibrationData calibData) { + mOtpCalibData = calibData; + } + public boolean isLibLoaded() { return mLibLoaded; } @@ -133,7 +115,7 @@ public class ClearSightNativeEngine { mRefColorImage = reference; if (mRefColorImage != null) { - Log.e(TAG, "setRefColorImage"); + Log.d(TAG, "setRefColorImage"); mSrcColor.add(new SourceImage(mRefColorImage.getPlanes()[Y_PLANE] .getBuffer(), mRefColorImage.getPlanes()[VU_PLANE] .getBuffer(), new int[] { 0, 0, 0, 0, 0 })); @@ -149,7 +131,7 @@ public class ClearSightNativeEngine { mRefMonoImage = reference; if (mRefMonoImage != null) { - Log.e(TAG, "setRefMonoImage"); + Log.d(TAG, "setRefMonoImage"); mSrcMono.add(new SourceImage(mRefMonoImage.getPlanes()[Y_PLANE] .getBuffer(), null, new int[] { 0, 0, 0, 0, 0 })); } @@ -206,7 +188,7 @@ public class ClearSightNativeEngine { // check data validity if (mSrcColor.size() != mSrcMono.size()) { // mis-match in num images - Log.e(TAG, "processImage - numImages mismatch - bayer: " + Log.d(TAG, "processImage - numImages mismatch - bayer: " + mSrcColor.size() + ", mono: " + mSrcMono.size()); return null; } @@ -218,7 +200,7 @@ public class ClearSightNativeEngine { ByteBuffer[] srcMonoY = new ByteBuffer[numImages]; int[][] metadataMono = new int[numImages][]; - Log.e(TAG, "processImage - numImages: " + numImages); + Log.d(TAG, "processImage - numImages: " + numImages); for (int i = 0; i < numImages; i++) { SourceImage color = mSrcColor.get(i); @@ -240,20 +222,23 @@ public class ClearSightNativeEngine { .getBuffer().capacity()); int[] roiRect = new int[4]; - Log.e(TAG, "processImage - refImage size - y: " + Log.d(TAG, "processImage - refImage size - y: " + colorPlanes[Y_PLANE].getBuffer().capacity() + " vu: " + colorPlanes[VU_PLANE].getBuffer().capacity()); - Log.e(TAG, "processImage - dst size - y: " + Log.d(TAG, "processImage - dst size - y: " + dstY.capacity() + " vu: " + dstVU.capacity()); + String otp_string = mOtpCalibData.toString(); + Log.d(TAG, "processImage - otp_calib: " + otp_string); + boolean result = nativeClearSightProcess(numImages, srcColorY, srcColorVU, metadataColor, mRefColorImage.getWidth(), mRefColorImage.getHeight(), colorPlanes[Y_PLANE].getRowStride(), colorPlanes[VU_PLANE].getRowStride(), srcMonoY, metadataMono, mRefMonoImage.getWidth(), mRefMonoImage.getHeight(), - monoPlanes[Y_PLANE].getRowStride(), otp_calib.getBytes(), dstY, dstVU, + monoPlanes[Y_PLANE].getRowStride(), otp_string.getBytes(), dstY, dstVU, colorPlanes[Y_PLANE].getRowStride(), colorPlanes[VU_PLANE].getRowStride(), roiRect); @@ -333,4 +318,190 @@ public class ClearSightNativeEngine { return baos.toByteArray(); } } + + public static class CamSensorCalibrationData { + /* Focal length in pixels @ calibration resolution.*/ + float normalized_focal_length; + /* Native sensor resolution W that was used to capture calibration image */ + short native_sensor_resolution_width; + /* Native sensor resolution H that was used to capture calibration image */ + short native_sensor_resolution_height; + /* Image size W used internally by calibration tool */ + short calibration_sensor_resolution_width; + /* Image size H used internally by calibration tool */ + short calibration_sensor_resolution_height; + /* Focal length ratio @ Calibration */ + float focal_length_ratio; + + private CamSensorCalibrationData() {} + + public static CamSensorCalibrationData createFromBytes(byte[] bytes) { + final ByteBuffer buf = ByteBuffer.wrap(bytes); + return createFromByteBuff(buf); + } + + public static CamSensorCalibrationData createFromByteBuff(ByteBuffer buf) { + final CamSensorCalibrationData data = new CamSensorCalibrationData(); + + data.normalized_focal_length = buf.getFloat(); + data.native_sensor_resolution_width = buf.getShort(); + data.native_sensor_resolution_height = buf.getShort(); + data.calibration_sensor_resolution_width = buf.getShort(); + data.calibration_sensor_resolution_height = buf.getShort(); + data.focal_length_ratio = buf.getFloat(); + + return data; + } + } + + public static class CamSystemCalibrationData { + private static final String[] CALIB_FMT_STRINGS = { + "Calibration OTP format version = %x\n", + "Main Native Sensor Resolution width = %dpx\n", + "Main Native Sensor Resolution height = %dpx\n", + "Main Calibration Resolution width = %dpx\n", + "Main Calibration Resolution height = %dpx\n", + "Main Focal length ratio = %f\n", + "Aux Native Sensor Resolution width = %dpx\n", + "Aux Native Sensor Resolution height = %dpx\n", + "Aux Calibration Resolution width = %dpx\n", + "Aux Calibration Resolution height = %dpx\n", + "Aux Focal length ratio = %f\n", + "Relative Rotation matrix [0] through [8] = %s\n", + "Relative Geometric surface parameters [0] through [31] = %s\n", + "Relative Principal point X axis offset (ox) = %fpx\n", + "Relative Principal point Y axis offset (oy) = %fpx\n", + "Relative position flag = %d\n", + "Baseline distance = %fmm\n", + "Main sensor mirror and flip setting = %d\n", + "Aux sensor mirror and flip setting = %d\n", + "Module orientation during calibration = %d\n", + "Rotation flag = %d\n", + "Main Normalized Focal length = %fpx\n", + "Aux Normalized Focal length = %fpx" + }; + + /* Version information */ + int calibration_format_version; + + /* Main Camera Sensor specific calibration */ + CamSensorCalibrationData main_cam_specific_calibration; + /* Aux Camera Sensor specific calibration */ + CamSensorCalibrationData aux_cam_specific_calibration; + + /* Relative viewpoint matching matrix w.r.t Main */ + float[] relative_rotation_matrix = new float[9]; + /* Relative geometric surface description parameters */ + float[] relative_geometric_surface_parameters = new float[32]; + + /* Relative offset of sensor center from optical axis along horizontal dimension */ + float relative_principle_point_x_offset; + /* Relative offset of sensor center from optical axis along vertical dimension */ + float relative_principle_point_y_offset; + + /* 0=Main Camera is on the left of Aux; 1=Main Camera is on the right of Aux */ + short relative_position_flag; + /* Camera separation in mm */ + float relative_baseline_distance; + + /* calibration orientation fields */ + short main_sensor_mirror_and_flip_setting; + short aux_sensor_mirror_and_flip_setting; + short module_orientation_during_calibration; + short rotation_flag; + + private CamSystemCalibrationData() {} + + public static CamSystemCalibrationData createFromBytes(byte[] bytes) { + if(bytes == null) + return null; + + final ByteBuffer buf = ByteBuffer.wrap(bytes); + buf.order(ByteOrder.LITTLE_ENDIAN); + CamSystemCalibrationData data = createFromByteBuff(buf); + + if(DEBUG) { + StringBuilder sb = new StringBuilder(); + sb.append("OTP Calib Data:"); + for(int i=0; i