SnapdragonCamera: Smile/blink/gaze detection

Implement Smile/blink/gaze detection feature and
add option to enable it in develop options.

CRs-Fixed: 1087487
Change-Id: I3005ca9767fbe38323e86255238cd4f96f34fbe5

2 files changed, 188 insertions, 3 deletions

diff --git a/src/com/android/camera/ui/Camera2FaceView.java b/src/com/android/camera/ui/Camera2FaceView.java
index ea75c65e2..3b6d10541 100644..100755
--- a/src/com/android/camera/ui/Camera2FaceView.java
+++ b/src/com/android/camera/ui/Camera2FaceView.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -38,13 +38,21 @@ import android.os.Message;
 import android.util.AttributeSet;
 import android.util.Log;
 
+import com.android.camera.ExtendedFace;
 import com.android.camera.util.CameraUtil;
 
 public class Camera2FaceView extends FaceView {
 
+    private final int smile_threashold_no_smile = 30;
+    private final int smile_threashold_small_smile = 60;
+    private final int blink_threshold = 60;
+
     private Face[] mFaces;
+    private ExtendedFace[] mExFaces;
     private Face[] mPendingFaces;
+    private ExtendedFace[] mPendingExFaces;
     private Rect mCameraBound;
+    private float mZoom = 1.0f;
     private Handler mHandler = new Handler() {
         @Override
         public void handleMessage(Message msg) {
@@ -52,6 +60,7 @@ public class Camera2FaceView extends FaceView {
                 case MSG_SWITCH_FACES:
                     mStateSwitchPending = false;
                     mFaces = mPendingFaces;
+                    mExFaces = mPendingExFaces;
                     invalidate();
                     break;
             }
@@ -66,13 +75,18 @@ public class Camera2FaceView extends FaceView {
         mCameraBound = cameraBound;
     }
 
-    public void setFaces(Face[] faces) {
+    public void setZoom(float zoom) {
+        mZoom = zoom;
+    }
+
+    public void setFaces(Face[] faces, ExtendedFace[] extendedFaces) {
         if (LOGV) Log.v(TAG, "Num of faces=" + faces.length);
         if (mPause) return;
         if (mFaces != null) {
             if ((faces.length > 0 && mFaces.length == 0)
                     || (faces.length == 0 && mFaces.length > 0)) {
                 mPendingFaces = faces;
+                mPendingExFaces = extendedFaces;
                 if (!mStateSwitchPending) {
                     mStateSwitchPending = true;
                     mHandler.sendEmptyMessageDelayed(MSG_SWITCH_FACES, SWITCH_DELAY);
@@ -85,6 +99,7 @@ public class Camera2FaceView extends FaceView {
             mHandler.removeMessages(MSG_SWITCH_FACES);
         }
         mFaces = faces;
+        mExFaces = extendedFaces;
         if (!mBlocked && (mFaces != null) && (mFaces.length > 0) && mCameraBound != null) {
             invalidate();
         }
@@ -110,6 +125,12 @@ public class Camera2FaceView extends FaceView {
             translateMatrix.preTranslate(-mCameraBound.width() / 2f, -mCameraBound.height() / 2f);
             translateMatrix.postScale(2000f / mCameraBound.width(), 2000f / mCameraBound.height());
 
+            Matrix bsgcTranslateMatrix = new Matrix();
+            bsgcTranslateMatrix.preTranslate(-mCameraBound.width() / 2f * mZoom,
+                    -mCameraBound.height() / 2f * mZoom);
+            bsgcTranslateMatrix.postScale(2000f / mCameraBound.width(),
+                    2000f / mCameraBound.height());
+
             int dx = (getWidth() - rw) / 2;
             int dy = (getHeight() - rh) / 2;
 
@@ -122,6 +143,8 @@ public class Camera2FaceView extends FaceView {
             float rectWidth;
             float rectHeight;
             float diameter;
+            int extendFaceSize = 0;
+            extendFaceSize = mExFaces == null? 0 : mExFaces.length;
             for (int i = 0; i < mFaces.length; i++) {
                 if (mFaces[i].getScore() < 50) continue;
                 Rect faceBound = mFaces[i].getBounds();
@@ -137,6 +160,167 @@ public class Camera2FaceView extends FaceView {
                 diameter = rectHeight > rectWidth ? rectWidth : rectHeight;
 
                 canvas.drawCircle(mRect.centerX(), mRect.centerY(), diameter/2, mPaint);
+
+                if (i < extendFaceSize && mExFaces[i] != null) {
+                    ExtendedFace exFace = mExFaces[i];
+                    Face face = mFaces[i];
+                    float[] point = new float[4];
+                    int delta_x = faceBound.width() / 12;
+                    int delta_y = faceBound.height() / 12;
+
+                    delta_x = (int)(delta_x * mZoom);
+                    delta_y = (int)(delta_y * mZoom);
+
+                    Log.e(TAG, "blink: (" + exFace.getLeyeBlink()+ ", " +
+                            exFace.getReyeBlink() + ")");
+                    if (face.getLeftEyePosition() != null) {
+                        if ((mDisplayRotation == 0) ||
+                                (mDisplayRotation == 180)) {
+                            point[0] = face.getLeftEyePosition().x;
+                            point[1] = face.getLeftEyePosition().y - delta_y / 2;
+                            point[2] = face.getLeftEyePosition().x;
+                            point[3] = face.getLeftEyePosition().y + delta_y / 2;
+                        } else {
+                            point[0] = face.getLeftEyePosition().x - delta_x / 2;
+                            point[1] = face.getLeftEyePosition().y;
+                            point[2] = face.getLeftEyePosition().x + delta_x / 2;
+                            point[3] = face.getLeftEyePosition().y;
+                        }
+                        bsgcTranslateMatrix.mapPoints(point);
+                        mMatrix.mapPoints (point);
+                        if (exFace.getLeyeBlink() >= blink_threshold) {
+                            canvas.drawLine(point[0]+ dx, point[1]+ dy,
+                                    point[2]+ dx, point[3]+ dy, mPaint);
+                        }
+                    }
+                    if (face.getRightEyePosition() != null) {
+                        if ((mDisplayRotation == 0) ||
+                                (mDisplayRotation == 180)) {
+                            point[0] = face.getRightEyePosition().x;
+                            point[1] = face.getRightEyePosition().y - delta_y / 2;
+                            point[2] = face.getRightEyePosition().x;
+                            point[3] = face.getRightEyePosition().y + delta_y / 2;
+                        } else {
+                            point[0] = face.getRightEyePosition().x - delta_x / 2;
+                            point[1] = face.getRightEyePosition().y;
+                            point[2] = face.getRightEyePosition().x + delta_x / 2;
+                            point[3] = face.getRightEyePosition().y;
+                        }
+                        bsgcTranslateMatrix.mapPoints(point);
+                        mMatrix.mapPoints (point);
+                        if (exFace.getReyeBlink() >= blink_threshold) {
+                            //Add offset to the points if the rect has an offset
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] +dx, point[3] +dy, mPaint);
+                        }
+                    }
+
+                    if (exFace.getLeftrightGaze() != 0
+                            || exFace.getTopbottomGaze() != 0 ) {
+
+                        double length =
+                                Math.sqrt((face.getLeftEyePosition().x - face.getRightEyePosition().x) *
+                                        (face.getLeftEyePosition().x - face.getRightEyePosition().x) +
+                                        (face.getLeftEyePosition().y - face.getRightEyePosition().y) *
+                                                (face.getLeftEyePosition().y - face.getRightEyePosition().y)) / 2.0;
+                        double nGazeYaw = -exFace.getLeftrightGaze();
+                        double nGazePitch = -exFace.getTopbottomGaze();
+                        float gazeRollX =
+                                (float)((-Math.sin(nGazeYaw/180.0*Math.PI) *
+                                        Math.cos(-exFace.getRollDirection()/
+                                                180.0*Math.PI) +
+                                        Math.sin(nGazePitch/180.0*Math.PI) *
+                                                Math.cos(nGazeYaw/180.0*Math.PI) *
+                                                Math.sin(-exFace.getRollDirection()/
+                                                        180.0*Math.PI)) *
+                                        (-length) + 0.5);
+                        float gazeRollY =
+                                (float)((Math.sin(-nGazeYaw/180.0*Math.PI) *
+                                        Math.sin(-exFace.getRollDirection()/
+                                                180.0*Math.PI)-
+                                        Math.sin(nGazePitch/180.0*Math.PI) *
+                                                Math.cos(nGazeYaw/180.0*Math.PI) *
+                                                Math.cos(-exFace.getRollDirection()/
+                                                        180.0*Math.PI)) *
+                                        (-length) + 0.5);
+
+                        if (exFace.getLeyeBlink() < blink_threshold) {
+                            if ((mDisplayRotation == 90) ||
+                                    (mDisplayRotation == 270)) {
+                                point[0] = face.getLeftEyePosition().x;
+                                point[1] = face.getLeftEyePosition().y;
+                                point[2] = face.getLeftEyePosition().x + gazeRollX;
+                                point[3] = face.getLeftEyePosition().y + gazeRollY;
+                            } else {
+                                point[0] = face.getLeftEyePosition().x;
+                                point[1] = face.getLeftEyePosition().y;
+                                point[2] = face.getLeftEyePosition().x + gazeRollY;
+                                point[3] = face.getLeftEyePosition().y + gazeRollX;
+                            }
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints (point);
+                            canvas.drawLine(point[0] +dx, point[1] + dy,
+                                    point[2] + dx, point[3] +dy, mPaint);
+                        }
+
+                        if (exFace.getReyeBlink() < blink_threshold) {
+                            if ((mDisplayRotation == 90) ||
+                                    (mDisplayRotation == 270)) {
+                                point[0] = face.getRightEyePosition().x;
+                                point[1] = face.getRightEyePosition().y;
+                                point[2] = face.getRightEyePosition().x + gazeRollX;
+                                point[3] = face.getRightEyePosition().y + gazeRollY;
+                            } else {
+                                point[0] = face.getRightEyePosition().x;
+                                point[1] = face.getRightEyePosition().y;
+                                point[2] = face.getRightEyePosition().x + gazeRollY;
+                                point[3] = face.getRightEyePosition().y + gazeRollX;
+                            }
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints (point);
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] + dx, point[3] + dy, mPaint);
+                        }
+                    }
+
+                    if (face.getMouthPosition() != null) {
+                        Log.e(TAG, "smile: " + exFace.getSmileDegree() + "," +
+                                exFace.getSmileConfidence());
+                        if (exFace.getSmileDegree() < smile_threashold_no_smile) {
+                            point[0] = face.getMouthPosition().x + dx - delta_x;
+                            point[1] = face.getMouthPosition().y;
+                            point[2] = face.getMouthPosition().x + dx + delta_x;
+                            point[3] = face.getMouthPosition().y;
+                            Matrix faceMatrix = new Matrix();
+                            faceMatrix.preRotate(exFace.getRollDirection(),
+                                    face.getMouthPosition().x, face.getMouthPosition().y);
+                            faceMatrix.mapPoints(point);
+                            bsgcTranslateMatrix.mapPoints(point);
+                            mMatrix.mapPoints(point);
+                            canvas.drawLine(point[0] + dx, point[1] + dy,
+                                    point[2] + dx, point[3] + dy, mPaint);
+                        } else if (exFace.getSmileDegree() <
+                                smile_threashold_small_smile) {
+                            int rotation_mouth = 360 - mDisplayRotation;
+                            mRect.set(face.getMouthPosition().x-delta_x,
+                                    face.getMouthPosition().y-delta_y, face.getMouthPosition().x+delta_x,
+                                    face.getMouthPosition().y+delta_y);
+                            bsgcTranslateMatrix.mapRect(mRect);
+                            mMatrix.mapRect(mRect);
+                            mRect.offset(dx, dy);
+                            canvas.drawArc(mRect, rotation_mouth,
+                                    180, true, mPaint);
+                        } else {
+                            mRect.set(face.getMouthPosition().x-delta_x,
+                                    face.getMouthPosition().y-delta_y, face.getMouthPosition().x+delta_x,
+                                    face.getMouthPosition().y+delta_y);
+                            bsgcTranslateMatrix.mapRect(mRect);
+                            mMatrix.mapRect(mRect);
+                            mRect.offset(dx, dy);
+                            canvas.drawOval(mRect, mPaint);
+                        }
+                    }
+                }
             }
             canvas.restore();
         }
@@ -149,6 +333,7 @@ public class Camera2FaceView extends FaceView {
         // drawable.
         mColor = mFocusingColor;
         mFaces = null;
+        mExFaces = null;
         invalidate();
     }
 }
diff --git a/src/com/android/camera/ui/FaceView.java b/src/com/android/camera/ui/FaceView.java
index 9a310afdb..db37dbcb3 100644..100755
--- a/src/com/android/camera/ui/FaceView.java
+++ b/src/com/android/camera/ui/FaceView.java
@@ -72,7 +72,7 @@ public class FaceView extends View
 
     protected static final int MSG_SWITCH_FACES = 1;
     protected static final int SWITCH_DELAY = 70;
-    private int mDisplayRotation = 0;
+    protected int mDisplayRotation = 0;
     protected boolean mStateSwitchPending = false;
     private Handler mHandler = new Handler() {
         @Override