Merge "SnapdragonCamera: Smile/blink/gaze detection" into camera.lnx.1.0-dev.1.0

6 files changed, 391 insertions, 11 deletions

diff --git a/src/com/android/camera/CaptureModule.java b/src/com/android/camera/CaptureModule.java
index 88037b829..cf26a761b 100644
--- a/src/com/android/camera/CaptureModule.java
+++ b/src/com/android/camera/CaptureModule.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
  * Not a Contribution.
  *
  * Copyright (C) 2012 The Android Open Source Project
@@ -245,6 +245,26 @@ public class CaptureModule implements CameraModule, PhotoController,
             new CaptureResult.Key<>("org.codeaurora.qcamera3.histogram.stats", int[].class);
     public static CameraCharacteristics.Key<Integer> isHdrScene =
             new CameraCharacteristics.Key<>("org.codeaurora.qcamera3.stats.is_hdr_scene", Integer.class);
+
+    public static CameraCharacteristics.Key<Byte> bsgcAvailable =
+            new CameraCharacteristics.Key<>("org.codeaurora.qcamera3.stats.bsgc_available", Byte.class);
+    public static CaptureResult.Key<byte[]> blinkDetected =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.blink_detected", byte[].class);
+    public static CaptureResult.Key<byte[]> blinkDegree =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.blink_degree", byte[].class);
+    public static CaptureResult.Key<byte[]> smileDegree =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.smile_degree", byte[].class);
+    public static CaptureResult.Key<byte[]> smileConfidence =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.smile_confidence", byte[].class);
+    public static CaptureResult.Key<byte[]> gazeAngle =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.gaze_angle", byte[].class);
+    public static CaptureResult.Key<int[]> gazeDirection =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.gaze_direction",
+                    int[].class);
+    public static CaptureResult.Key<byte[]> gazeDegree =
+            new CaptureResult.Key<>("org.codeaurora.qcamera3.stats.gaze_degree",
+                    byte[].class);
+
     private boolean[] mTakingPicture = new boolean[MAX_NUM_CAM];
     private int mControlAFMode = CameraMetadata.CONTROL_AF_MODE_CONTINUOUS_PICTURE;
     private int mLastResultAFState = -1;
@@ -322,6 +342,8 @@ public class CaptureModule implements CameraModule, PhotoController,
     private CaptureResult mPreviewCaptureResult;
     private Face[] mPreviewFaces = null;
     private Face[] mStickyFaces = null;
+    private ExtendedFace[] mExFaces = null;
+    private ExtendedFace[] mStickyExFaces = null;
     private Rect mBayerCameraRegion;
     private Handler mCameraHandler;
     private Handler mImageAvailableHandler;
@@ -551,7 +573,11 @@ public class CaptureModule implements CameraModule, PhotoController,
             if (id == getMainCameraId()) {
                 updateFocusStateChange(partialResult);
                 Face[] faces = partialResult.get(CaptureResult.STATISTICS_FACES);
-                updateFaceView(faces);
+                if (faces != null && isBsgcDetecionOn()) {
+                    updateFaceView(faces, getBsgcInfo(partialResult, faces.length));
+                } else {
+                    updateFaceView(faces, null);
+                }
             }
         }
 
@@ -563,7 +589,11 @@ public class CaptureModule implements CameraModule, PhotoController,
             if (id == getMainCameraId()) {
                 updateFocusStateChange(result);
                 Face[] faces = result.get(CaptureResult.STATISTICS_FACES);
-                updateFaceView(faces);
+                if (faces != null && isBsgcDetecionOn()) {
+                    updateFaceView(faces, getBsgcInfo(result, faces.length));
+                } else {
+                    updateFaceView(faces, null);
+                }
             }
             if (SettingsManager.getInstance().isHistogramSupport()) {
                 int[] histogramStats = result.get(CaptureModule.histogramStats);
@@ -803,6 +833,12 @@ public class CaptureModule implements CameraModule, PhotoController,
         return isBackCamera() && getCameraMode() == DUAL_MODE && value.equals("on");
     }
 
+    private boolean isBsgcDetecionOn() {
+        String value = mSettingsManager.getValue(SettingsManager.KEY_BSGC_DETECTION);
+        if (value == null) return false;
+        return  value.equals("enable");
+    }
+
     private boolean isRawCaptureOn() {
         String value = mSettingsManager.getValue(SettingsManager.KEY_SAVERAW);
         if (value == null) return  false;
@@ -2743,16 +2779,39 @@ public class CaptureModule implements CameraModule, PhotoController,
         return false;
     }
 
-    private void updateFaceView(final Face[] faces) {
+    private ExtendedFace[] getBsgcInfo(CaptureResult captureResult, int size) {
+        ExtendedFace []extendedFaces = new ExtendedFace[size];
+        byte[] blinkDetectedArray = captureResult.get(blinkDetected);
+        byte[] blinkDegreesArray = captureResult.get(blinkDegree);
+        int[] gazeDirectionArray = captureResult.get(gazeDirection);
+        byte[] gazeAngleArray = captureResult.get(gazeAngle);
+        byte[] smileDegreeArray = captureResult.get(smileDegree);
+        byte[] smileConfidenceArray = captureResult.get(smileConfidence);
+        for(int i=0;i<size;i++) {
+            ExtendedFace tmp = new ExtendedFace(i);
+            tmp.setBlinkDetected(blinkDetectedArray[i]);
+            tmp.setBlinkDegree(blinkDegreesArray[2*i], blinkDegreesArray[2*i+1]);
+            tmp.setGazeDirection(gazeDirectionArray[3*i], gazeDirectionArray[3*i+1], gazeDirectionArray[3*i+2]);
+            tmp.setGazeAngle(gazeAngleArray[i]);
+            tmp.setSmileDegree(smileDegreeArray[i]);
+            tmp.setSmileConfidence(smileConfidenceArray[i]);
+            extendedFaces[i] = tmp;
+        }
+        return extendedFaces;
+    }
+
+    private void updateFaceView(final Face[] faces, final ExtendedFace[] extendedFaces) {
         mPreviewFaces = faces;
+        mExFaces = extendedFaces;
         if (faces != null) {
             if (faces.length != 0) {
                 mStickyFaces = faces;
+                mStickyExFaces = extendedFaces;
             }
             mHandler.post(new Runnable() {
                 @Override
                 public void run() {
-                    mUI.onFaceDetection(faces);
+                    mUI.onFaceDetection(faces, extendedFaces);
                 }
             });
         }
diff --git a/src/com/android/camera/CaptureUI.java b/src/com/android/camera/CaptureUI.java
index 025a17aa1..6ec09effd 100755
--- a/src/com/android/camera/CaptureUI.java
+++ b/src/com/android/camera/CaptureUI.java
@@ -1432,11 +1432,13 @@ public class CaptureUI implements FocusOverlayManager.FocusUI,
         mFaceView.setDisplayOrientation(orientation);
         mFaceView.setMirror(mirror);
         mFaceView.setCameraBound(cameraBound);
+        mFaceView.setZoom(mModule.getZoomValue());
         mFaceView.resume();
     }
 
     public void updateFaceViewCameraBound(Rect cameraBound) {
         mFaceView.setCameraBound(cameraBound);
+        mFaceView.setZoom(mModule.getZoomValue());
     }
 
     public void onStopFaceDetection() {
@@ -1450,8 +1452,9 @@ public class CaptureUI implements FocusOverlayManager.FocusUI,
     public void onFaceDetection(Face[] faces, CameraManager.CameraProxy camera) {
     }
 
-    public void onFaceDetection(android.hardware.camera2.params.Face[] faces) {
-        mFaceView.setFaces(faces);
+    public void onFaceDetection(android.hardware.camera2.params.Face[] faces,
+                                ExtendedFace[] extendedFaces) {
+        mFaceView.setFaces(faces,extendedFaces);
     }
 
     public Point getSurfaceViewSize() {
diff --git a/src/com/android/camera/ExtendedFace.java b/src/com/android/camera/ExtendedFace.java
new file mode 100755
index 000000000..b5a087620
--- /dev/null
+++ b/src/com/android/camera/ExtendedFace.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2017, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials provided
+ *    with the distribution.
+ *  * Neither the name of The Linux Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.android.camera;
+
+import android.hardware.camera2.params.Face;
+
+public class ExtendedFace {
+    private int mSmileDegree = 0;
+    private int mSmileConfidence = 0;
+    private int mBlinkDetected = 0;
+    private int mLeyeBlink = 0;
+    private int mReyeBlink = 0;
+    private int mGazeAngle = 0;
+    private int mLeftrightGaze = 0;
+    private int mTopbottomGaze = 0;
+    private int mGazeDirection = 0;
+    private int mRollDirection = 0;
+    private int mId;
+
+    public ExtendedFace(int id) {
+        mId = id;
+    }
+
+    public int getBlinkDetected() {
+        return mBlinkDetected;
+    }
+
+    public int getLeyeBlink() {
+        return mLeyeBlink;
+    }
+
+    public int getReyeBlink() {
+        return mReyeBlink;
+    }
+
+    public int getSmileDegree() {
+        return mSmileDegree;
+    }
+
+    public int getSmileConfidence() {
+        return mSmileConfidence;
+    }
+
+    public int getLeftrightGaze() {
+        return mLeftrightGaze;
+    }
+
+    public int getTopbottomGaze() {
+        return mTopbottomGaze;
+    }
+
+    public int getGazeDirection() {
+        return mGazeDirection;
+    }
+
+    public int getRollDirection() {
+        return mRollDirection;
+    }
+
+    public void setBlinkDetected(int blinkDetected) {
+        this.mBlinkDetected = blinkDetected;
+    }
+
+    public void setBlinkDegree(byte left, byte right) {
+        this.mLeyeBlink = left;
+        this.mReyeBlink = right;
+    }
+
+    public void setSmileDegree(byte smileDegree) {
+        this.mSmileDegree = smileDegree;
+    }
+
+    public void setGazeDirection(int topbottomGaze, int leftrightGaze, int rollDirection) {
+        this.mTopbottomGaze = topbottomGaze;
+        this.mLeftrightGaze = leftrightGaze;
+        this.mRollDirection = rollDirection;
+    }
+
+    public void setGazeAngle(byte gazeAngle) {
+        this.mGazeAngle = gazeAngle;
+    }
+
+    public void setSmileConfidence(int smileConfidence) {
+        this.mSmileConfidence = smileConfidence;
+    }
+}
\ No newline at end of file
diff --git a/src/com/android/camera/SettingsManager.java b/src/com/android/camera/SettingsManager.java
index 7172b300c..1f05e3c6e 100755
--- a/src/com/android/camera/SettingsManager.java
+++ b/src/com/android/camera/SettingsManager.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -148,6 +148,7 @@ public class SettingsManager implements ListMenu.SettingsListener {
     public static final String KEY_ZOOM = "pref_camera2_zoom_key";
 
     public static final HashMap<String, Integer> KEY_ISO_INDEX = new HashMap<String, Integer>();
+    public static final String KEY_BSGC_DETECTION = "pref_camera2_bsgc_key";
 
     private static final String TAG = "SnapCam_SettingsManager";
 
@@ -612,6 +613,7 @@ public class SettingsManager implements ListMenu.SettingsListener {
         ListPreference histogram = mPreferenceGroup.findPreference(KEY_HISTOGRAM);
         ListPreference hdr = mPreferenceGroup.findPreference(KEY_HDR);
         ListPreference zoom = mPreferenceGroup.findPreference(KEY_ZOOM);
+        ListPreference bsgc = mPreferenceGroup.findPreference(KEY_BSGC_DETECTION);
 
         if (whiteBalance != null) {
             if (filterUnsupportedOptions(whiteBalance, getSupportedWhiteBalanceModes(cameraId))) {
@@ -626,6 +628,13 @@ public class SettingsManager implements ListMenu.SettingsListener {
             }
         }
 
+        if (bsgc != null) {
+            if (!isBsgcAvailable(mCameraId)) {
+                removePreference(mPreferenceGroup, KEY_BSGC_DETECTION);
+                mFilteredKeys.add(bsgc.getKey());
+            }
+        }
+
         if (colorEffect != null) {
             if (filterUnsupportedOptions(colorEffect, getSupportedColorEffects(cameraId))) {
                 mFilteredKeys.add(colorEffect.getKey());
@@ -1047,6 +1056,17 @@ public class SettingsManager implements ListMenu.SettingsListener {
         return false;
     }
 
+    public boolean isBsgcAvailable(int id) {
+        boolean ret = false;
+        try {
+            byte bsgc_available = mCharacteristics.get(id).get(CaptureModule.bsgcAvailable);
+            ret = bsgc_available == 1;
+        } catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+        return ret;
+    }
+
     public boolean isFacingFront(int id) {
         int facing = mCharacteristics.get(id).get(CameraCharacteristics.LENS_FACING);
         return facing == CameraCharacteristics.LENS_FACING_FRONT;
diff --git a/src/com/android/camera/ui/Camera2FaceView.java b/src/com/android/camera/ui/Camera2FaceView.java
index ea75c65e2..3b6d10541 100644..100755
--- a/src/com/android/camera/ui/Camera2FaceView.java
+++ b/src/com/android/camera/ui/Camera2FaceView.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -38,13 +38,21 @@ import android.os.Message;
 import android.util.AttributeSet;
 import android.util.Log;
 
+import com.android.camera.ExtendedFace;
 import com.android.camera.util.CameraUtil;
 
 public class Camera2FaceView extends FaceView {
 
+    private final int smile_threashold_no_smile = 30;
+    private final int smile_threashold_small_smile = 60;
+    private final int blink_threshold = 60;
+
     private Face[] mFaces;
+    private ExtendedFace[] mExFaces;
     private Face[] mPendingFaces;
+    private ExtendedFace[] mPendingExFaces;
     private Rect mCameraBound;
+    private float mZoom = 1.0f;
     private Handler mHandler = new Handler() {
         @Override
         public void handleMessage(Message msg) {
@@ -52,6 +60,7 @@ public class Camera2FaceView extends FaceView {
                 case MSG_SWITCH_FACES:
                     mStateSwitchPending = false;
                     mFaces = mPendingFaces;
+                    mExFaces = mPendingExFaces;
                     invalidate();
                     break;
             }
@@ -66,13 +75,18 @@ public class Camera2FaceView extends FaceView {
         mCameraBound = cameraBound;
     }
 
-    public void setFaces(Face[] faces) {
+    public void setZoom(float zoom) {
+        mZoom = zoom;
+    }
+
+    public void setFaces(Face[] faces, ExtendedFace[] extendedFaces) {
         if (LOGV) Log.v(TAG, "Num of faces=" + faces.length);
         if (mPause) return;
         if (mFaces != null) {
             if ((faces.length > 0 && mFaces.length == 0)
                     || (faces.length == 0 && mFaces.length > 0)) {
                 mPendingFaces = faces;
+                mPendingExFaces = extendedFaces;
                 if (!mStateSwitchPending) {
                     mStateSwitchPending = true;
                     mHandler.sendEmptyMessageDelayed(MSG_SWITCH_FACES, SWITCH_DELAY);
@@ -85,6 +99,7 @@ public class Camera2FaceView extends FaceView {
             mHandler.removeMessages(MSG_SWITCH_FACES);
         }
         mFaces = faces;
+        mExFaces = extendedFaces;
         if (!mBlocked && (mFaces != null) && (mFaces.length > 0) && mCameraBound != null) {
             invalidate();
         }
@@ -110,6 +125,12 @@ public class Camera2FaceView extends FaceView {
             translateMatrix.preTranslate(-mCameraBound.width() / 2f, -mCameraBound.height() / 2f);
             translateMatrix.postScale(2000f / mCameraBound.width(), 2000f / mCameraBound.height());
 
+            Matrix bsgcTranslateMatrix = new Matrix();
+            bsgcTranslateMatrix.preTranslate(-mCameraBound.width() / 2f * mZoom,
+                    -mCameraBound.height() / 2f * mZoom);
+            bsgcTranslateMatrix.postScale(2000f / mCameraBound.width(),
+                    2000f / mCameraBound.height());
+
             int dx = (getWidth() - rw) / 2;
             int dy = (getHeight() - rh) / 2;
 
@@ -122,6 +143,8 @@ public class Camera2FaceView extends FaceView {
             float rectWidth;
             float rectHeight;
             float diameter;
+            int extendFaceSize = 0;
+            extendFaceSize = mExFaces == null? 0 : mExFaces.length;
             for (int i = 0; i < mFaces.length; i++) {
                 if (mFaces[i].getScore() < 50) continue;
                 Rect faceBound = mFaces[i].getBounds();
@@ -137,6 +160,167 @@ public class Camera2FaceView extends FaceView {
                 diameter = rectHeight > rectWidth ? rectWidth : rectHeight;
 
                 canvas.drawCircle(mRect.centerX(), mRect.centerY(), diameter/2, mPaint);
+
+                if (i < extendFaceSize && mExFaces[i] != null) {
+                    ExtendedFace exFace = mExFaces[i];
+                    Face face = mFaces[i];
+                    float[] point = new float[4];
+                    int delta_x = faceBound.width() / 12;
+                    int delta_y = faceBound.height() / 12;
+
+                    delta_x = (int)(delta_x * mZoom);
+                    delta_y = (int)(delta_y * mZoom);
+
+                    Log.e(TAG, "blink: (" + exFace.getLeyeBlink()+ ", " +
+                            exFace.getReyeBlink() + ")");
+                    if (face.getLeftEyePosition() != null) {
+                        if ((mDisplayRotation == 0) ||
+                                (mDisplayRotation == 180)) {
+                            point[0] = face.getLeftEyePosition().x;
+                            point[1] = face.getLeftEyePosition().y - delta_y / 2;
+                            point[2] = face.getLeftEyePosition().x;
+                            point[3] = face.getLeftEyePosition().y + delta_y / 2;
+                        } else {
+                            point[0] = face.getLeftEyePosition().x - delta_x / 2;
+                            point[1] = face.getLeftEyePosition().y;
+                            point[2] = face.getLeftEyePosition().x + delta_x / 2;
+                            point[3] = face.getLeftEyePosition().y;
+                        }
+                        bsgcTranslateMatrix.mapPoints(point);
+                        mMatrix.mapPoints (point);
+                        if (exFace.getLeyeBlink() >= blink_threshold) {
+                            canvas.drawLine(point[0]+ dx, point[1]+ dy,
+                                    point[2]+ dx, point[3]+ dy, mPaint);
+                        }
+                    }
+                    if (face.getRightEyePosition() != null) {
+                        if ((mDisplayRotation == 0) ||
+                                (mDisplayRotation == 180)) {
+                            point[0] = face.getRightEyePosition().x;
+                            point[1] = face.getRightEyePosition().y - delta_y / 2;
+                            point[2] = face.getRightEyePosition().x;
+                            point[3] = face.getRightEyePosition().y + delta_y / 2;
+                        } else {
+                            point[0] = face.getRightEyePosition().x - delta_x / 2;
+                            point[1] = face.getRightEyePosition().y;
+                            point[2] = face.getRightEyePosition().x + delta_x / 2;
+                            point[3] = face.getRightEyePosition().y;
+                        }
+                        bsgcTranslateMatrix.mapPoints(point);
+                        mMatrix.mapPoints (point);
+                        if (exFace.getReyeBlink() >= blink_threshold) {
+                            //Add offset to the points if the rect has an offset
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] +dx, point[3] +dy, mPaint);
+                        }
+                    }
+
+                    if (exFace.getLeftrightGaze() != 0
+                            || exFace.getTopbottomGaze() != 0 ) {
+
+                        double length =
+                                Math.sqrt((face.getLeftEyePosition().x - face.getRightEyePosition().x) *
+                                        (face.getLeftEyePosition().x - face.getRightEyePosition().x) +
+                                        (face.getLeftEyePosition().y - face.getRightEyePosition().y) *
+                                                (face.getLeftEyePosition().y - face.getRightEyePosition().y)) / 2.0;
+                        double nGazeYaw = -exFace.getLeftrightGaze();
+                        double nGazePitch = -exFace.getTopbottomGaze();
+                        float gazeRollX =
+                                (float)((-Math.sin(nGazeYaw/180.0*Math.PI) *
+                                        Math.cos(-exFace.getRollDirection()/
+                                                180.0*Math.PI) +
+                                        Math.sin(nGazePitch/180.0*Math.PI) *
+                                                Math.cos(nGazeYaw/180.0*Math.PI) *
+                                                Math.sin(-exFace.getRollDirection()/
+                                                        180.0*Math.PI)) *
+                                        (-length) + 0.5);
+                        float gazeRollY =
+                                (float)((Math.sin(-nGazeYaw/180.0*Math.PI) *
+                                        Math.sin(-exFace.getRollDirection()/
+                                                180.0*Math.PI)-
+                                        Math.sin(nGazePitch/180.0*Math.PI) *
+                                                Math.cos(nGazeYaw/180.0*Math.PI) *
+                                                Math.cos(-exFace.getRollDirection()/
+                                                        180.0*Math.PI)) *
+                                        (-length) + 0.5);
+
+                        if (exFace.getLeyeBlink() < blink_threshold) {
+                            if ((mDisplayRotation == 90) ||
+                                    (mDisplayRotation == 270)) {
+                                point[0] = face.getLeftEyePosition().x;
+                                point[1] = face.getLeftEyePosition().y;
+                                point[2] = face.getLeftEyePosition().x + gazeRollX;
+                                point[3] = face.getLeftEyePosition().y + gazeRollY;
+                            } else {
+                                point[0] = face.getLeftEyePosition().x;
+                                point[1] = face.getLeftEyePosition().y;
+                                point[2] = face.getLeftEyePosition().x + gazeRollY;
+                                point[3] = face.getLeftEyePosition().y + gazeRollX;
+                            }
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints (point);
+                            canvas.drawLine(point[0] +dx, point[1] + dy,
+                                    point[2] + dx, point[3] +dy, mPaint);
+                        }
+
+                        if (exFace.getReyeBlink() < blink_threshold) {
+                            if ((mDisplayRotation == 90) ||
+                                    (mDisplayRotation == 270)) {
+                                point[0] = face.getRightEyePosition().x;
+                                point[1] = face.getRightEyePosition().y;
+                                point[2] = face.getRightEyePosition().x + gazeRollX;
+                                point[3] = face.getRightEyePosition().y + gazeRollY;
+                            } else {
+                                point[0] = face.getRightEyePosition().x;
+                                point[1] = face.getRightEyePosition().y;
+                                point[2] = face.getRightEyePosition().x + gazeRollY;
+                                point[3] = face.getRightEyePosition().y + gazeRollX;
+                            }
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints (point);
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] + dx, point[3] + dy, mPaint);
+                        }
+                    }
+
+                    if (face.getMouthPosition() != null) {
+                        Log.e(TAG, "smile: " + exFace.getSmileDegree() + "," +
+                                exFace.getSmileConfidence());
+                        if (exFace.getSmileDegree() < smile_threashold_no_smile) {
+                            point[0] = face.getMouthPosition().x + dx - delta_x;
+                            point[1] = face.getMouthPosition().y;
+                            point[2] = face.getMouthPosition().x + dx + delta_x;
+                            point[3] = face.getMouthPosition().y;
+                            Matrix faceMatrix = new Matrix();
+                            faceMatrix.preRotate(exFace.getRollDirection(),
+                                    face.getMouthPosition().x, face.getMouthPosition().y);
+                            faceMatrix.mapPoints(point);
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints(point);
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] + dx, point[3] + dy, mPaint);
+                        } else if (exFace.getSmileDegree() <
+                                smile_threashold_small_smile) {
+                            int rotation_mouth = 360 - mDisplayRotation;
+                            mRect.set(face.getMouthPosition().x-delta_x,
+                                    face.getMouthPosition().y-delta_y, face.getMouthPosition().x+delta_x,
+                                    face.getMouthPosition().y+delta_y);
+                            bsgcTranslateMatrix.mapRect(mRect);
+                            mMatrix.mapRect(mRect);
+                            mRect.offset(dx, dy);
+                            canvas.drawArc(mRect, rotation_mouth,
+                                    180, true, mPaint);
+                        } else {
+                            mRect.set(face.getMouthPosition().x-delta_x,
+                                    face.getMouthPosition().y-delta_y, face.getMouthPosition().x+delta_x,
+                                    face.getMouthPosition().y+delta_y);
+                            bsgcTranslateMatrix.mapRect(mRect);
+                            mMatrix.mapRect(mRect);
+                            mRect.offset(dx, dy);
+                            canvas.drawOval(mRect, mPaint);
+                        }
+                    }
+                }
             }
             canvas.restore();
         }
@@ -149,6 +333,7 @@ public class Camera2FaceView extends FaceView {
         // drawable.
         mColor = mFocusingColor;
         mFaces = null;
+        mExFaces = null;
         invalidate();
     }
 }
diff --git a/src/com/android/camera/ui/FaceView.java b/src/com/android/camera/ui/FaceView.java
index 9a310afdb..db37dbcb3 100644..100755
--- a/src/com/android/camera/ui/FaceView.java
+++ b/src/com/android/camera/ui/FaceView.java
@@ -72,7 +72,7 @@ public class FaceView extends View
 
     protected static final int MSG_SWITCH_FACES = 1;
     protected static final int SWITCH_DELAY = 70;
-    private int mDisplayRotation = 0;
+    protected int mDisplayRotation = 0;
     protected boolean mStateSwitchPending = false;
     private Handler mHandler = new Handler() {
         @Override