am 8a079b92: am 8d4e5229: Merge "Performance turning for aggregator 1. Move identification matching to secondary 2. In the first stage, do secondary matching only if there is no structured name 3. Aggregate all the raw contacts without structured name if they have mat

* commit '8a079b92788f590485203921a5c9be002823487b':
  Performance turning for aggregator 1. Move identification matching to secondary 2. In the first stage, do secondary matching only if there is no structured name 3. Aggregate all the raw contacts without structured name if they have matching secondary data 4. Remove approximate name matching in aggregator. (This will not impact the name variant, such as John and Johathan matching)

3 files changed, 176 insertions, 88 deletions

diff --git a/src/com/android/providers/contacts/aggregation/ContactAggregator2.java b/src/com/android/providers/contacts/aggregation/ContactAggregator2.java
index d70e34a8..9beb6c2a 100644
--- a/src/com/android/providers/contacts/aggregation/ContactAggregator2.java
+++ b/src/com/android/providers/contacts/aggregation/ContactAggregator2.java
@@ -16,6 +16,9 @@
 
 package com.android.providers.contacts.aggregation;
 
+import static com.android.providers.contacts.aggregation.util.RawContactMatcher.SCORE_THRESHOLD_PRIMARY;
+import static com.android.providers.contacts.aggregation.util.RawContactMatcher.SCORE_THRESHOLD_SECONDARY;
+import static com.android.providers.contacts.aggregation.util.RawContactMatcher.SCORE_THRESHOLD_SUGGEST;
 import android.database.Cursor;
 import android.database.sqlite.SQLiteDatabase;
 import android.provider.ContactsContract.AggregationExceptions;
@@ -54,13 +57,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import static com.android.providers.contacts.aggregation.util.RawContactMatcher
-        .SCORE_THRESHOLD_PRIMARY;
-import static com.android.providers.contacts.aggregation.util.RawContactMatcher
-        .SCORE_THRESHOLD_SECONDARY;
-import static com.android.providers.contacts.aggregation.util.RawContactMatcher
-        .SCORE_THRESHOLD_SUGGEST;
-
 /**
  * ContactAggregator2 deals with aggregating contact information with sufficient matching data
  * points. E.g., two John Doe contacts with same phone numbers are presumed to be the same
@@ -564,6 +560,9 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         try {
             while (c.moveToNext()) {
                 final long rId = c.getLong(IdentityLookupMatchQuery.RAW_CONTACT_ID);
+                if (rId == rawContactId) {
+                    continue;
+                }
                 final long contactId = c.getLong(IdentityLookupMatchQuery.CONTACT_ID);
                 final long accountId = c.getLong(IdentityLookupMatchQuery.ACCOUNT_ID);
                 matcher.matchIdentity(rId, contactId, accountId);
@@ -585,6 +584,9 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         try {
             while (c.moveToNext()) {
                 long rId =  c.getLong(NameLookupMatchQuery.RAW_CONTACT_ID);
+                if (rId == rawContactId) {
+                    continue;
+                }
                 long contactId = c.getLong(NameLookupMatchQuery.CONTACT_ID);
                 long accountId = c.getLong(NameLookupMatchQuery.ACCOUNT_ID);
                 String name = c.getString(NameLookupMatchQuery.NAME);
@@ -612,6 +614,9 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         try {
             while (c.moveToNext()) {
                 long rId = c.getLong(EmailLookupQuery.RAW_CONTACT_ID);
+                if (rId == rawContactId) {
+                    continue;
+                }
                 long contactId = c.getLong(EmailLookupQuery.CONTACT_ID);
                 long accountId = c.getLong(EmailLookupQuery.ACCOUNT_ID);
                 matcher.updateScoreWithEmailMatch(rId, contactId, accountId);
@@ -666,6 +671,9 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         try {
             while (c.moveToNext()) {
                 long rId = c.getLong(PhoneLookupQuery.RAW_CONTACT_ID);
+                if (rId == rawContactId) {
+                    continue;
+                }
                 long contactId = c.getLong(PhoneLookupQuery.CONTACT_ID);
                 long accountId = c.getLong(PhoneLookupQuery.ACCOUNT_ID);
                 matcher.updateScoreWithPhoneNumberMatch(rId, contactId, accountId);
@@ -789,28 +797,6 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         }
     }
 
-    private PhotoEntry getPhotoMetadata(SQLiteDatabase db, long photoFileId) {
-        if (photoFileId == 0) {
-            // Assume standard thumbnail size.  Don't bother getting a file size for priority;
-            // we should fall back to photo priority resolver if all we have are thumbnails.
-            int thumbDim = mContactsProvider.getMaxThumbnailDim();
-            return new PhotoEntry(thumbDim * thumbDim, 0);
-        } else {
-            Cursor c = db.query(Tables.PHOTO_FILES, PhotoFileQuery.COLUMNS, PhotoFiles._ID + "=?",
-                    new String[]{String.valueOf(photoFileId)}, null, null, null);
-            try {
-                if (c.getCount() == 1) {
-                    c.moveToFirst();
-                    int pixelCount =
-                            c.getInt(PhotoFileQuery.HEIGHT) * c.getInt(PhotoFileQuery.WIDTH);
-                    return new PhotoEntry(pixelCount, c.getInt(PhotoFileQuery.FILESIZE));
-                }
-            } finally {
-                c.close();
-            }
-        }
-        return new PhotoEntry(0, 0);
-    }
     /**
      * Finds contacts with data matches and returns a list of {@link MatchScore}'s in the
      * descending order of match score.
@@ -866,12 +852,20 @@ public class ContactAggregator2 extends AbstractContactAggregator {
      */
     private void updateMatchScores(SQLiteDatabase db, long rawContactId,
             MatchCandidateList candidates, RawContactMatcher matcher) {
+        //update primary score
         updateMatchScoresBasedOnExceptions(db, rawContactId, matcher);
-        updateMatchScoresBasedOnIdentityMatch(db, rawContactId, matcher);
         updateMatchScoresBasedOnNameMatches(db, rawContactId, matcher);
-        updateMatchScoresBasedOnEmailMatches(db, rawContactId, matcher);
-        updateMatchScoresBasedOnPhoneMatches(db, rawContactId, matcher);
-        updateMatchScoresBasedOnSecondaryData(db, rawContactId, candidates, matcher);
+        // update scores only if the raw contact doesn't have structured name
+        if (rawContactWithoutName(db, rawContactId)) {
+            updateMatchScoresBasedOnIdentityMatch(db, rawContactId, matcher);
+            updateMatchScoresBasedOnEmailMatches(db, rawContactId, matcher);
+            updateMatchScoresBasedOnPhoneMatches(db, rawContactId, matcher);
+            final List<Long> secondaryRawContactIds = matcher.prepareSecondaryMatchCandidates();
+            if (secondaryRawContactIds != null
+                    && secondaryRawContactIds.size() <= SECONDARY_HIT_LIMIT) {
+                updateScoreForCandidatesWithoutName(db, secondaryRawContactIds, matcher);
+            }
+        }
     }
 
     private void updateMatchScoresForSuggestionsBasedOnDataMatches(SQLiteDatabase db,
@@ -886,35 +880,53 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         }
     }
 
-    /**
-     * Update scores for matches with secondary data matching but insufficient primary scores.
-     * This method loads structured names for all candidate contacts and recomputes match scores
-     * using approximate matching.
-     */
-    private void updateMatchScoresBasedOnSecondaryData(SQLiteDatabase db,
-            long rawContactId, MatchCandidateList candidates, RawContactMatcher matcher) {
-        final List<Long> secondaryRawContactIds = matcher.prepareSecondaryMatchCandidates();
-        if (secondaryRawContactIds == null || secondaryRawContactIds.size() > SECONDARY_HIT_LIMIT) {
-            return;
+    private boolean rawContactWithoutName(SQLiteDatabase db, long rawContactId) {
+        String selection = RawContacts._ID + " =" + rawContactId;
+        final Cursor c = db.query(NullNameRawContactsIdsQuery.TABLE,
+                NullNameRawContactsIdsQuery.COLUMNS, selection, null, null, null, null);
+
+        try {
+            if (c.moveToFirst()) {
+                return TextUtils.isEmpty(c.getString(NullNameRawContactsIdsQuery.NAME));
+            }
+        } finally {
+            c.close();
         }
+        return false;
+    }
 
-        loadNameMatchCandidates(db, rawContactId, candidates, true);
+    /**
+     * Update scores for matches with secondary data matching but no structured name.
+     */
+    private void updateScoreForCandidatesWithoutName(SQLiteDatabase db,
+            List<Long> secondaryRawContactIds, RawContactMatcher matcher) {
 
         mSb.setLength(0);
+
         mSb.append(RawContacts._ID).append(" IN (");
         for (int i = 0; i < secondaryRawContactIds.size(); i++) {
             if (i != 0) {
-                mSb.append(',');
+                mSb.append(",");
             }
             mSb.append(secondaryRawContactIds.get(i));
         }
+        mSb.append( ")");
+        final Cursor c = db.query(NullNameRawContactsIdsQuery.TABLE,
+                NullNameRawContactsIdsQuery.COLUMNS, mSb.toString(), null, null, null, null);
 
-        // We only want to compare structured names to structured names
-        // at this stage, we need to ignore all other sources of name lookup data.
-        mSb.append(") AND " + STRUCTURED_NAME_BASED_LOOKUP_SQL);
-
-        matchAllCandidates(db, mSb.toString(), candidates, matcher,
-                RawContactMatcher.MATCHING_ALGORITHM_CONSERVATIVE, null);
+        try {
+            while (c.moveToNext()) {
+                Long rId = c.getLong(NullNameRawContactsIdsQuery.RAW_CONTACT_ID);
+                Long contactId = c.getLong(NullNameRawContactsIdsQuery.CONTACT_ID);
+                Long accountId = c.getLong(NullNameRawContactsIdsQuery.ACCOUNT_ID);
+                String name = c.getString(NullNameRawContactsIdsQuery.NAME);
+                if (TextUtils.isEmpty(name)) {
+                    matcher.matchNoName(rId, contactId, accountId);
+                }
+            }
+        } finally {
+            c.close();
+        }
     }
 
     protected interface IdentityLookupMatchQuery {
@@ -1026,4 +1038,18 @@ public class ContactAggregator2 extends AbstractContactAggregator {
         int ACCOUNT_ID = 2;
     }
 
+    protected interface NullNameRawContactsIdsQuery {
+        final String TABLE =  Tables.RAW_CONTACTS + " LEFT OUTER JOIN " +  Tables.NAME_LOOKUP
+                + " ON "+ RawContacts._ID + " = " + NameLookupColumns.RAW_CONTACT_ID
+                + " AND " + NameLookupColumns.NAME_TYPE + " = " + NameLookupType.NAME_EXACT;
+
+        final String[] COLUMNS = new String[] {
+                RawContacts._ID, RawContacts.CONTACT_ID, RawContactsColumns.ACCOUNT_ID,
+                NameLookupColumns.NORMALIZED_NAME};
+
+        int RAW_CONTACT_ID = 0;
+        int CONTACT_ID = 1;
+        int ACCOUNT_ID = 2;
+        int NAME = 3;
+    }
 }
diff --git a/src/com/android/providers/contacts/aggregation/util/RawContactMatcher.java b/src/com/android/providers/contacts/aggregation/util/RawContactMatcher.java
index 88aa2265..f39ae96c 100644
--- a/src/com/android/providers/contacts/aggregation/util/RawContactMatcher.java
+++ b/src/com/android/providers/contacts/aggregation/util/RawContactMatcher.java
@@ -30,12 +30,11 @@ import java.util.List;
 public class RawContactMatcher {
     private static final String TAG = "ContactMatcher";
 
-    // Best possible match score
-    public static final int MAX_SCORE = 100;
-
     // Suggest to aggregate contacts if their match score is equal or greater than this threshold
     public static final int SCORE_THRESHOLD_SUGGEST = 50;
 
+    public static final int SCORE_THRESHOLD_NO_NAME = 50;
+
     // Automatically aggregate contacts if their match score is equal or greater than this threshold
     public static final int SCORE_THRESHOLD_PRIMARY = 70;
 
@@ -49,6 +48,9 @@ public class RawContactMatcher {
     // Score for matching email addresses
     private static final int EMAIL_MATCH_SCORE = 71;
 
+    // Score for matching identity
+    private static final int IDENTITY_MATCH_SCORE = 71;
+
     // Score for matching nickname
     private static final int NICKNAME_MATCH_SCORE = 71;
 
@@ -180,13 +182,6 @@ public class RawContactMatcher {
     }
 
     /**
-     * Marks the contact as a full match, because we found an Identity match
-     */
-    public void matchIdentity(long rawContactId, long contactId, long accountId) {
-        updatePrimaryScore(rawContactId, contactId, accountId, MAX_SCORE);
-    }
-
-    /**
      * Checks if there is a match and updates the overall score for the
      * specified contact for a discovered match. The new score is determined
      * by the prior score, by the type of name we were looking for, the type
@@ -244,6 +239,10 @@ public class RawContactMatcher {
         updatePrimaryScore(rawContactId, contactId, accountId, score);
     }
 
+    public void matchIdentity(long rawContactId, long contactId, long accountId) {
+        updateSecondaryScore(rawContactId, contactId, accountId, IDENTITY_MATCH_SCORE);
+    }
+
     public void updateScoreWithPhoneNumberMatch(long rawContactId, long contactId, long accountId) {
         updateSecondaryScore(rawContactId, contactId, accountId, PHONE_MATCH_SCORE);
     }
@@ -278,9 +277,9 @@ public class RawContactMatcher {
         mScoreCount = 0;
     }
     /**
-     * Returns a list of IDs for raw contacts that are matched on secondary data elements
-     * (phone number, email address, nickname). We still need to obtain the approximate
-     * primary score for those contacts to determine if any of them should be aggregated.
+     * Returns a list of IDs for raw contacts that are only matched on secondary data elements
+     * (phone number, email address, nickname, identity). We need to check if they are missing
+     * structured name or not to decide if they should be aggregated.
      * <p>
      * May return null.
      */
@@ -295,7 +294,7 @@ public class RawContactMatcher {
 
             if (score.getSecondaryScore() >= SCORE_THRESHOLD_PRIMARY) {
                 if (rawContactIds == null) {
-                    rawContactIds = new ArrayList<Long>();
+                    rawContactIds = new ArrayList<>();
                 }
                 rawContactIds.add(score.getRawContactId());
             }
@@ -320,7 +319,9 @@ public class RawContactMatcher {
                 continue;
             }
 
-            if (score.getPrimaryScore() >= SCORE_THRESHOLD_SECONDARY) {
+            if (score.getPrimaryScore() >= SCORE_THRESHOLD_PRIMARY ||
+                    (score.getPrimaryScore() == SCORE_THRESHOLD_NO_NAME &&
+                            score.getSecondaryScore() > SCORE_THRESHOLD_SECONDARY)) {
                 matches.add(score);
             }
         }
@@ -351,4 +352,8 @@ public class RawContactMatcher {
     public String toString() {
         return mScoreList.subList(0, mScoreCount).toString();
     }
+
+    public void matchNoName(Long rawContactId, Long contactId, Long accountId) {
+        updatePrimaryScore(rawContactId, contactId, accountId, SCORE_THRESHOLD_NO_NAME);
+    }
 }
diff --git a/tests/src/com/android/providers/contacts/aggregation/ContactAggregator2Test.java b/tests/src/com/android/providers/contacts/aggregation/ContactAggregator2Test.java
index ac64cacf..67634120 100644
--- a/tests/src/com/android/providers/contacts/aggregation/ContactAggregator2Test.java
+++ b/tests/src/com/android/providers/contacts/aggregation/ContactAggregator2Test.java
@@ -293,10 +293,16 @@ public class ContactAggregator2Test extends BaseContactsProvider2Test {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
         insertPhoneNumber(rawContactId1, "(888)555-1231");
 
-        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
+        long rawContactId2 = RawContactUtil.createRawContactWithName(mResolver, "John", "Doe",
+                ACCOUNT_2);
         insertPhoneNumber(rawContactId2, "1(888)555-1231");
 
+        long rawContactId3 = RawContactUtil.createRawContact(mResolver, ACCOUNT_3);
+        insertPhoneNumber(rawContactId3, "1(888)555-1231");
+
         assertNotAggregated(rawContactId1, rawContactId2);
+        assertNotAggregated(rawContactId3, rawContactId2);
+        assertAggregated(rawContactId1, rawContactId3);
     }
 
     public void testAggregationBasedOnPhoneNumberWhenTargetAggregateHasNoName() {
@@ -333,26 +339,36 @@ public class ContactAggregator2Test extends BaseContactsProvider2Test {
         assertNotAggregated(rawContactId1, rawContactId2);
     }
 
-    public void testAggregationBasedOnPhoneNumberWithJustFirstName() {
+    public void testAggregationBasedOnEmailNoNameData() {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
-        DataUtil.insertStructuredName(mResolver, rawContactId1, "Chick", "Notnull");
-        insertPhoneNumber(rawContactId1, "(888)555-1236");
+        insertEmail(rawContactId1, "lightning@android.com");
 
         long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
-        DataUtil.insertStructuredName(mResolver, rawContactId2, "Chick", null);
-        insertPhoneNumber(rawContactId2, "1(888)555-1236");
+        insertEmail(rawContactId2, "lightning@android.com");
+
+        long rawContactId3 = RawContactUtil.createRawContactWithName(mResolver, "John", "Doe",
+                ACCOUNT_3);
+        insertEmail(rawContactId3, "lightning@android.com");
 
         assertAggregated(rawContactId1, rawContactId2);
+        assertNotAggregated(rawContactId1, rawContactId3);
+        assertNotAggregated(rawContactId2, rawContactId3);
     }
 
-    public void testAggregationBasedOnEmailNoNameData() {
+    public void testAggregationByIdentificationNoStructuredNameWithinDifferentAccounts() {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
-        insertEmail(rawContactId1, "lightning@android.com");
+        insertIdentity(rawContactId1, "jfamily", "google.com");
 
         long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
-        insertEmail(rawContactId2, "lightning@android.com");
+        insertIdentity(rawContactId2, "jfamily", "google.com");
 
-        assertNotAggregated(rawContactId1, rawContactId2);
+        long rawContactId3 = RawContactUtil.createRawContactWithName(mResolver, "John", "Doe",
+                ACCOUNT_3);
+        insertIdentity(rawContactId3, "jfamily ", "google.com");
+
+        assertAggregated(rawContactId1, rawContactId2);
+        assertNotAggregated(rawContactId1, rawContactId3);
+        assertNotAggregated(rawContactId2, rawContactId3);
     }
 
     public void testAggregationBasedOnEmailWhenTargetAggregateHasNoName() {
@@ -409,16 +425,27 @@ public class ContactAggregator2Test extends BaseContactsProvider2Test {
         assertAggregated(rawContactId1, rawContactId2, "Lawrence");
     }
 
-    public void testAggregationByNicknameNoStructuredName() {
+    public void testAggregationByNicknameNoStructuredNameWithinSameAccount() {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
         insertNickname(rawContactId1, "Frozone");
 
-        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
+        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
         insertNickname(rawContactId2, "Frozone");
 
         assertNotAggregated(rawContactId1, rawContactId2);
     }
 
+    public void testAggregationByNicknameNoStructuredNameWithinDifferentAccounts() {
+        long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
+        insertNickname(rawContactId1, "Frozone");
+
+        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
+        insertNickname(rawContactId2, "Frozone");
+
+        assertAggregated(rawContactId1, rawContactId2);
+    }
+
+
     public void testAggregationByNicknameWithDifferentNames() {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
         DataUtil.insertStructuredName(mResolver, rawContactId1, "Helen", "Parr");
@@ -445,16 +472,6 @@ public class ContactAggregator2Test extends BaseContactsProvider2Test {
         assertNotAggregated(rawContactId1, rawContactId2);
     }
 
-    public void testAggregationByIdentity() {
-        long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
-        insertIdentity(rawContactId1, "iden1", "namespace1");
-
-        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_2);
-        insertIdentity(rawContactId2, "iden1", "namespace1");
-
-        assertAggregated(rawContactId1, rawContactId2);
-    }
-
     public void testAggregationExceptionKeepIn() {
         long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
         DataUtil.insertStructuredName(mResolver, rawContactId1, "Johnk", "Smithk");
@@ -1792,6 +1809,46 @@ public class ContactAggregator2Test extends BaseContactsProvider2Test {
         assertNotAggregated(rawContactId4, rawContactId5);
     }
 
+    public void testFamilyMembersWithSimilarNameAndSameHomePhone() {
+        long rawContactId1 = RawContactUtil.createRawContactWithName(mResolver, "John", "Smith",
+                ACCOUNT_1);
+
+        insertPhoneNumber(rawContactId1,"1234", false, 3);
+        insertEmail(rawContactId1, "smithfamily@gmail.com");
+
+        long rawContactId2 = RawContactUtil.createRawContactWithName(mResolver, "Jane", "Smith",
+                ACCOUNT_1);
+        insertPhoneNumber(rawContactId2,"1234", false, 3);
+        insertPhoneNumber(rawContactId2,"8270", false, 4);
+        insertEmail(rawContactId2, "smithfamily@gmail.com");
+
+        long rawContactId3 = RawContactUtil.createRawContactWithName(mResolver, "Karen", "Smith",
+                ACCOUNT_1);
+        insertPhoneNumber(rawContactId3,"1234", false, 3);
+        insertEmail(rawContactId3, "smithfamily@gmail.com");
+
+        assertNotAggregated(rawContactId1, rawContactId2);
+        assertNotAggregated(rawContactId1, rawContactId3);
+        assertNotAggregated(rawContactId2, rawContactId3);
+    }
+
+    public void testNoNameContactsWithSameSecondaryData() {
+        long rawContactId1 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
+
+        insertPhoneNumber(rawContactId1,"1234", false, 3);
+        insertEmail(rawContactId1, "smithfamily@gmail.com");
+
+        long rawContactId2 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
+        insertPhoneNumber(rawContactId2,"1234", false, 3);
+
+        long rawContactId3 = RawContactUtil.createRawContact(mResolver, ACCOUNT_1);
+        insertEmail(rawContactId3, "smithfamily@gmail.com");
+
+        assertAggregated(rawContactId1, rawContactId2);
+        assertAggregated(rawContactId1, rawContactId3);
+        assertAggregated(rawContactId2, rawContactId3);
+    }
+
     private void assertSuggestions(long contactId, long... suggestions) {
         final Uri aggregateUri = ContentUris.withAppendedId(Contacts.CONTENT_URI, contactId);
         Uri uri = Uri.withAppendedPath(aggregateUri,