Fix a couple of bugs in imap synching

b/10111339
b/10125810
The first problem was that the imap BEFORE clause
is exclusive, so messages on the date given in
BEFORE will not be sent. Now, on the sync for the
most recent messages, we will just not specify a
BEFORE clause, so we can always get the most recent
messages even if our clock drifts from the server.

The second is that some imap servers do not accept
time information on the query dates, and that causes
errors. The imap spec defines the BEFORE and SINCE
clauses to come with a <date> only, not a time,
and although it seems that at least some imap servers
handle that, it can't be expected to always work.

Change-Id: Ibf41c6f7600b9f9537bc6d13b59873ee36798e1e

2 files changed, 45 insertions, 17 deletions

diff --git a/src/com/android/email/mail/store/ImapFolder.java b/src/com/android/email/mail/store/ImapFolder.java
index 7a9b927b3..67076c250 100644
--- a/src/com/android/email/mail/store/ImapFolder.java
+++ b/src/com/android/email/mail/store/ImapFolder.java
@@ -18,6 +18,7 @@ package com.android.email.mail.store;
 
 import android.content.Context;
 import android.text.TextUtils;
+import android.text.format.DateUtils;
 import android.util.Base64DataException;
 
 import com.android.email.mail.store.ImapStore.ImapException;
@@ -62,6 +63,7 @@ import java.util.Locale;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.TimeZone;
 
 class ImapFolder extends Folder {
     private final static Flag[] PERMANENT_FLAGS =
@@ -392,7 +394,6 @@ class ImapFolder extends Folder {
     @VisibleForTesting
     String[] searchForUids(String searchCriteria) throws MessagingException {
         checkOpen();
-        LogUtils.d(Logging.LOG_TAG, "searchForUids '" + searchCriteria + "'");
         try {
             try {
                 final String command = ImapConstants.UID_SEARCH + " " + searchCriteria;
@@ -497,21 +498,35 @@ class ImapFolder extends Folder {
     @VisibleForTesting
     public Message[] getMessages(long startDate, long endDate, MessageRetrievalListener listener)
             throws MessagingException {
-        LogUtils.d(Logging.LOG_TAG, "getMessages since " + startDate + " before " + endDate);
-        // Dates must be formatted like: 7-Feb-1994 21:52:25 -0800
+        // Dates must be formatted like: 7-Feb-1994. Time info within a date is not
+        // universally supported.
         // XXX can I limit the maximum number of results?
-        final String format = String.format("dd-LLL-yyyy HH:mm:ss Z");
-        final SimpleDateFormat formatter = new SimpleDateFormat(format);
+        final SimpleDateFormat formatter = new SimpleDateFormat("dd-LLL-yyyy", Locale.US);
+        formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
         final String sinceDateStr = formatter.format(endDate);
-        final String beforeDateStr = formatter.format(startDate);
 
-        if (startDate < endDate) {
-            throw new MessagingException(String.format("Invalid date range: %s - %s",
-                    sinceDateStr, beforeDateStr));
+        final StringBuilder queryParam = new StringBuilder();
+        queryParam.append( "1:* ");
+        // If the caller requests a startDate of zero, then ignore the BEFORE parameter.
+        // This makes sure that we can always query for the newest messages, even if our
+        // time is different from the imap server's time.
+        if (startDate != 0) {
+            final String beforeDateStr = formatter.format(startDate);
+            if (startDate < endDate) {
+                throw new MessagingException(String.format("Invalid date range: %s - %s",
+                        sinceDateStr, beforeDateStr));
+            }
+            queryParam.append("BEFORE \"")
+                .append(beforeDateStr)
+                .append("\" ");
         }
-        return getMessagesInternal(
-                searchForUids(String.format(Locale.US, "1:* BEFORE \"%s\" SINCE \"%s\"",
-                        beforeDateStr, sinceDateStr)), listener);
+        queryParam.append("SINCE \"")
+             .append(sinceDateStr)
+             .append("\"");
+
+        LogUtils.d(Logging.LOG_TAG, "getMessages dateRange " + queryParam.toString());
+
+        return getMessagesInternal(searchForUids(queryParam.toString()), listener);
     }
 
     @Override
diff --git a/src/com/android/email/service/ImapService.java b/src/com/android/email/service/ImapService.java
index 54fb7cc33..4b2b7c58a 100644
--- a/src/com/android/email/service/ImapService.java
+++ b/src/com/android/email/service/ImapService.java
@@ -409,9 +409,12 @@ public class ImapService extends Service {
         // 6. Save folder message count that we got earlier.
         mailbox.updateMessageCount(context, remoteMessageCount);
 
-        // 7. Figure out how big our sync window should be.
-        long startDate = System.currentTimeMillis();
-        long endDate = startDate - DEFAULT_SYNC_WINDOW_MILLIS;
+        // 7. Figure out how big our sync window should be. Leave startDate set to zero, this
+        // indicates we do not want any constraint on the BEFORE parameter sent in our query.
+        // This way, we will always be able to get the most recent messages, even if the
+        // imap server's date is different from ours.
+        long startDate = 0;
+        long endDate = System.currentTimeMillis() - DEFAULT_SYNC_WINDOW_MILLIS;
         LogUtils.d(Logging.LOG_TAG, "original window " + startDate + " - " + endDate);
         Cursor localOldestCursor = null;
         try {
@@ -440,7 +443,6 @@ public class ImapService extends Service {
         }
 
         // Get all messages in our query date range:
-        LogUtils.d(Logging.LOG_TAG, "loading range " + startDate + " - " + endDate);
         Message[] remoteMessages;
         remoteMessages = remoteFolder.getMessages(startDate, endDate, null);
 
@@ -461,8 +463,12 @@ public class ImapService extends Service {
             startDate = endDate - 1;
             Message[] additionalMessages = new Message[0];
             long windowIncreaseSize = INITIAL_WINDOW_SIZE_INCREASE;
-            while  (additionalMessages.length < additionalMessagesNeeded) {
+            while  (additionalMessages.length < additionalMessagesNeeded && endDate > 0) {
                 endDate = endDate - windowIncreaseSize;
+                if (endDate < 0) {
+                    LogUtils.d(Logging.LOG_TAG, "window size too large, this is the last attempt");
+                    endDate = 0;
+                }
                 LogUtils.d(Logging.LOG_TAG, "requesting additional messages from range " +
                         startDate + " - " + endDate);
                 additionalMessages = remoteFolder.getMessages(startDate, endDate, null);
@@ -475,6 +481,13 @@ public class ImapService extends Service {
             }
 
             LogUtils.d(Logging.LOG_TAG, "additionalMessages " + additionalMessages.length);
+            if (additionalMessages.length < additionalMessagesNeeded) {
+                // We have attempted to load a window that goes all the way back to time zero,
+                // but we still don't have as many messages as the server says are in the inbox.
+                // This is not expected to happen.
+                LogUtils.e(Logging.LOG_TAG, "expected to find " + additionalMessagesNeeded +
+                        " more messages, only got " + additionalMessages.length);
+            }
             int additionalToKeep = additionalMessages.length;
             if (additionalMessages.length > LOAD_MORE_MAX_INCREMENT) {
                 // We have way more additional messages than intended, drop some of them.