/* * Copyright (c) 2016, The Linux Foundation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials provided * with the distribution. * * Neither the name of The Linux Foundation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.codeaurora.snapcam.filter; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import android.graphics.Rect; import android.hardware.camera2.TotalCaptureResult; import android.media.Image; import android.media.Image.Plane; import android.util.Log; public class ClearSightNativeEngine { private static final boolean DEBUG = false; private static final String TAG = "ClearSightNativeEngine"; static { try { System.loadLibrary("jni_clearsight"); mLibLoaded = true; Log.v(TAG, "successfully loaded clearsight lib"); } catch (UnsatisfiedLinkError e) { Log.e(TAG, "failed to load clearsight lib"); e.printStackTrace(); mLibLoaded = false; } } private static final int METADATA_SIZE = 5; private static final int Y_PLANE = 0; private static final int VU_PLANE = 2; private static boolean mLibLoaded; private static byte[] mOtpCalibData; private static ClearSightNativeEngine mInstance; private Image mRefColorImage; private Image mRefMonoImage; private TotalCaptureResult mRefColorResult; private TotalCaptureResult mRefMonoResult; private ArrayList mSrcColor = new ArrayList(); private ArrayList mSrcMono = new ArrayList(); private ClearSightNativeEngine() { } public static void createInstance() { if (mInstance == null) { mInstance = new ClearSightNativeEngine(); } } public static ClearSightNativeEngine getInstance() { createInstance(); return mInstance; } public static void setOtpCalibData(CamSystemCalibrationData calibData) { String calibStr = calibData.toString(); Log.d(TAG, "OTP calibration data: \n" + calibStr); mOtpCalibData = calibStr.getBytes(); } public boolean isLibLoaded() { return mLibLoaded; } public void reset() { mSrcColor.clear(); mSrcMono.clear(); setReferenceColorImage(null); setReferenceMonoImage(null); setReferenceColorResult(null); setReferenceMonoResult(null); } public void setReferenceResult(boolean color, TotalCaptureResult result) { if (color) setReferenceColorResult(result); else setReferenceMonoResult(result); } private void setReferenceColorResult(TotalCaptureResult result) { mRefColorResult = result; } private void setReferenceMonoResult(TotalCaptureResult result) { mRefMonoResult = result; } public void setReferenceImage(boolean color, Image image) { if (color) setReferenceColorImage(image); else setReferenceMonoImage(image); } private void setReferenceColorImage(Image reference) { if (mRefColorImage != null) { mRefColorImage.close(); mRefColorImage = null; } mRefColorImage = reference; if (mRefColorImage != null) { Log.d(TAG, "setRefColorImage"); mSrcColor.add(new SourceImage(mRefColorImage.getPlanes()[Y_PLANE] .getBuffer(), mRefColorImage.getPlanes()[VU_PLANE] .getBuffer(), new int[] { 0, 0, 0, 0, 0 })); } } private void setReferenceMonoImage(Image reference) { if (mRefMonoImage != null) { mRefMonoImage.close(); mRefMonoImage = null; } mRefMonoImage = reference; if (mRefMonoImage != null) { Log.d(TAG, "setRefMonoImage"); mSrcMono.add(new SourceImage(mRefMonoImage.getPlanes()[Y_PLANE] .getBuffer(), null, new int[] { 0, 0, 0, 0, 0 })); } } public boolean hasReferenceImage(boolean color) { return (getImageCount(color) > 0); } public int getImageCount(boolean color) { return color ? mSrcColor.size() : mSrcMono.size(); } public Image getReferenceImage(boolean color) { return color ? mRefColorImage : mRefMonoImage; } public TotalCaptureResult getReferenceResult(boolean color) { return color ? mRefColorResult : mRefMonoResult; } public boolean registerImage(boolean color, Image image) { List sourceImages = color?mSrcColor:mSrcMono; if (sourceImages.isEmpty()) { Log.w(TAG, "reference image not yet set"); return false; } Image referenceImage = color?mRefColorImage:mRefMonoImage; Plane[] planes = image.getPlanes(); ByteBuffer yBuf = planes[Y_PLANE].getBuffer(); ByteBuffer refY = referenceImage.getPlanes()[Y_PLANE].getBuffer(); ByteBuffer regY = ByteBuffer.allocateDirect(refY.capacity()); int yRowStride = planes[Y_PLANE].getRowStride(); ByteBuffer vuBuf = null; ByteBuffer refVU = null; ByteBuffer regVU = null; int vuRowStride = 0; if(color) { vuBuf = planes[VU_PLANE].getBuffer(); refVU = referenceImage.getPlanes()[VU_PLANE].getBuffer(); regVU = ByteBuffer.allocateDirect(refVU.capacity()); vuRowStride = planes[VU_PLANE].getRowStride(); } int[] metadata = new int[METADATA_SIZE]; boolean result = nativeClearSightRegisterImage(refY, yBuf, vuBuf, image.getWidth(), image.getHeight(), yRowStride, vuRowStride, regY, regVU, metadata); if (result) { sourceImages.add(new SourceImage(regY, regVU, metadata)); } image.close(); return result; } public ClearsightImage processImage() { // check data validity if (mSrcColor.size() != mSrcMono.size()) { // mis-match in num images Log.d(TAG, "processImage - numImages mismatch - bayer: " + mSrcColor.size() + ", mono: " + mSrcMono.size()); return null; } int numImages = mSrcColor.size(); ByteBuffer[] srcColorY = new ByteBuffer[numImages]; ByteBuffer[] srcColorVU = new ByteBuffer[numImages]; int[][] metadataColor = new int[numImages][]; ByteBuffer[] srcMonoY = new ByteBuffer[numImages]; int[][] metadataMono = new int[numImages][]; Log.d(TAG, "processImage - numImages: " + numImages); for (int i = 0; i < numImages; i++) { SourceImage color = mSrcColor.get(i); SourceImage mono = mSrcMono.get(i); srcColorY[i] = color.mY; srcColorVU[i] = color.mVU; metadataColor[i] = color.mMetadata; srcMonoY[i] = mono.mY; metadataMono[i] = mono.mMetadata; } Plane[] colorPlanes = mRefColorImage.getPlanes(); Plane[] monoPlanes = mRefMonoImage.getPlanes(); ByteBuffer dstY = ByteBuffer.allocateDirect(colorPlanes[Y_PLANE] .getBuffer().capacity()); ByteBuffer dstVU = ByteBuffer.allocateDirect(colorPlanes[VU_PLANE] .getBuffer().capacity()); int[] roiRect = new int[4]; Log.d(TAG, "processImage - refImage size - y: " + colorPlanes[Y_PLANE].getBuffer().capacity() + " vu: " + colorPlanes[VU_PLANE].getBuffer().capacity()); Log.d(TAG, "processImage - dst size - y: " + dstY.capacity() + " vu: " + dstVU.capacity()); boolean result = nativeClearSightProcess(numImages, srcColorY, srcColorVU, metadataColor, mRefColorImage.getWidth(), mRefColorImage.getHeight(), colorPlanes[Y_PLANE].getRowStride(), colorPlanes[VU_PLANE].getRowStride(), srcMonoY, metadataMono, mRefMonoImage.getWidth(), mRefMonoImage.getHeight(), monoPlanes[Y_PLANE].getRowStride(), mOtpCalibData, dstY, dstVU, colorPlanes[Y_PLANE].getRowStride(), colorPlanes[VU_PLANE].getRowStride(), roiRect); if (result) { dstY.rewind(); dstVU.rewind(); return new ClearsightImage(dstY, dstVU, roiRect); } else { return null; } } private native final boolean nativeConfigureClearSight(float brIntensity, float sharpenIntensity); private native final boolean nativeClearSightRegisterImage(ByteBuffer refY, ByteBuffer srcY, ByteBuffer srcVU, int width, int height, int strideY, int strideVU, ByteBuffer dstY, ByteBuffer dstVU, int[] metadata); private native final boolean nativeClearSightProcess(int numImagePairs, ByteBuffer[] srcColorY, ByteBuffer[] srcColorVU, int[][] metadataColor, int srcColorWidth, int srcColorHeight, int srcColorStrideY, int srcColorStrideVU, ByteBuffer[] srcMonoY, int[][] metadataMono, int srcMonoWidth, int srcMonoHeight, int srcMonoStrideY, byte[] otp, ByteBuffer dstY, ByteBuffer dstVU, int dstStrideY, int dstStrideVU, int[] roiRect); private class SourceImage { ByteBuffer mY; ByteBuffer mVU; int[] mMetadata; SourceImage(ByteBuffer y, ByteBuffer vu, int[] metadata) { mY = y; mVU = vu; mMetadata = metadata; } } public static class ClearsightImage { public final ByteBuffer mYplane; public final ByteBuffer mVUplane; private Rect mRoiRect; ClearsightImage(ByteBuffer yPlane, ByteBuffer vuPlane, int[] rect) { mYplane = yPlane; mVUplane = vuPlane; mRoiRect = new Rect(rect[0], rect[1], rect[0] + rect[2], rect[1] + rect[3]); } public Rect getRoiRect() { return mRoiRect; } public int getWidth() { return (mRoiRect.right - mRoiRect.left); } public int getHeight() { return (mRoiRect.bottom - mRoiRect.top); } } public static class CamSensorCalibrationData { /* Focal length in pixels @ calibration resolution.*/ float normalized_focal_length; /* Native sensor resolution W that was used to capture calibration image */ short native_sensor_resolution_width; /* Native sensor resolution H that was used to capture calibration image */ short native_sensor_resolution_height; /* Image size W used internally by calibration tool */ short calibration_sensor_resolution_width; /* Image size H used internally by calibration tool */ short calibration_sensor_resolution_height; /* Focal length ratio @ Calibration */ float focal_length_ratio; private CamSensorCalibrationData() {} public static CamSensorCalibrationData createFromBytes(byte[] bytes) { final ByteBuffer buf = ByteBuffer.wrap(bytes); return createFromByteBuff(buf); } public static CamSensorCalibrationData createFromByteBuff(ByteBuffer buf) { final CamSensorCalibrationData data = new CamSensorCalibrationData(); data.normalized_focal_length = buf.getFloat(); data.native_sensor_resolution_width = buf.getShort(); data.native_sensor_resolution_height = buf.getShort(); data.calibration_sensor_resolution_width = buf.getShort(); data.calibration_sensor_resolution_height = buf.getShort(); data.focal_length_ratio = buf.getFloat(); return data; } } public static class CamSystemCalibrationData { private static final String[] CALIB_FMT_STRINGS = { "Calibration OTP format version = %x\n", "Main Native Sensor Resolution width = %dpx\n", "Main Native Sensor Resolution height = %dpx\n", "Main Calibration Resolution width = %dpx\n", "Main Calibration Resolution height = %dpx\n", "Main Focal length ratio = %f\n", "Aux Native Sensor Resolution width = %dpx\n", "Aux Native Sensor Resolution height = %dpx\n", "Aux Calibration Resolution width = %dpx\n", "Aux Calibration Resolution height = %dpx\n", "Aux Focal length ratio = %f\n", "Relative Rotation matrix [0] through [8] = %s\n", "Relative Geometric surface parameters [0] through [31] = %s\n", "Relative Principal point X axis offset (ox) = %fpx\n", "Relative Principal point Y axis offset (oy) = %fpx\n", "Relative position flag = %d\n", "Baseline distance = %fmm\n", "Main sensor mirror and flip setting = %d\n", "Aux sensor mirror and flip setting = %d\n", "Module orientation during calibration = %d\n", "Rotation flag = %d\n", "Main Normalized Focal length = %fpx\n", "Aux Normalized Focal length = %fpx" }; /* Version information */ int calibration_format_version; /* Main Camera Sensor specific calibration */ CamSensorCalibrationData main_cam_specific_calibration; /* Aux Camera Sensor specific calibration */ CamSensorCalibrationData aux_cam_specific_calibration; /* Relative viewpoint matching matrix w.r.t Main */ float[] relative_rotation_matrix = new float[9]; /* Relative geometric surface description parameters */ float[] relative_geometric_surface_parameters = new float[32]; /* Relative offset of sensor center from optical axis along horizontal dimension */ float relative_principle_point_x_offset; /* Relative offset of sensor center from optical axis along vertical dimension */ float relative_principle_point_y_offset; /* 0=Main Camera is on the left of Aux; 1=Main Camera is on the right of Aux */ short relative_position_flag; /* Camera separation in mm */ float relative_baseline_distance; /* calibration orientation fields */ short main_sensor_mirror_and_flip_setting; short aux_sensor_mirror_and_flip_setting; short module_orientation_during_calibration; short rotation_flag; private CamSystemCalibrationData() {} public static CamSystemCalibrationData createFromBytes(byte[] bytes) { if(bytes == null) return null; final ByteBuffer buf = ByteBuffer.wrap(bytes); buf.order(ByteOrder.LITTLE_ENDIAN); CamSystemCalibrationData data = createFromByteBuff(buf); if(DEBUG) { StringBuilder sb = new StringBuilder(); sb.append("OTP Calib Data:"); for(int i=0; i