Remove usage legacy HTTP Client

Improved the "LookupUtils" to not use the deprecated HTTP classes
but "HttpURLConnection" instead. Contrary to the Apache HTTP client
the HttURLConnection is still existing in Android 6 and seems to be
more efficient besides being compatible with SSL SNI.

Furthermore set the missing "formattedNumber" attribute when the
"ContactBuilder" is creating the "ContactInfo" object after lookups.

Change-Id: I01be4fa09ceba7338aac63e2c718704fa1941ee5

11 files changed, 139 insertions, 151 deletions

diff --git a/src/com/android/dialer/lookup/ContactBuilder.java b/src/com/android/dialer/lookup/ContactBuilder.java
index 069045db2..994aa128a 100644
--- a/src/com/android/dialer/lookup/ContactBuilder.java
+++ b/src/com/android/dialer/lookup/ContactBuilder.java
@@ -207,6 +207,7 @@ public class ContactBuilder {
 
             ContactInfo info = new ContactInfo();
             info.name = mName.displayName;
+            info.formattedNumber = mFormattedNumber;
             info.normalizedNumber = mNormalizedNumber;
             info.number = mPhoneNumbers.get(0).number;
             info.type = mPhoneNumbers.get(0).type;
diff --git a/src/com/android/dialer/lookup/LookupUtils.java b/src/com/android/dialer/lookup/LookupUtils.java
index c0b84dc38..9d2678944 100644
--- a/src/com/android/dialer/lookup/LookupUtils.java
+++ b/src/com/android/dialer/lookup/LookupUtils.java
@@ -18,15 +18,20 @@ package com.android.dialer.lookup;
 
 import android.text.Html;
 
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.util.EntityUtils;
-
+import java.io.BufferedInputStream;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.IllegalCharsetNameException;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -34,35 +39,115 @@ public class LookupUtils {
     private static final String USER_AGENT =
             "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0";
 
-    public static String httpGet(HttpGet request) throws IOException {
-        HttpClient client = new DefaultHttpClient();
+    private static HttpURLConnection prepareHttpConnection(String url, Map<String, String> headers)
+            throws IOException {
+        // open connection
+        HttpURLConnection urlConnection = (HttpURLConnection) new URL(url).openConnection();
+        // set user agent (default value is null)
+        urlConnection.setRequestProperty("User-Agent", USER_AGENT);
+        // set all other headers if not null
+        if (headers != null) {
+            for (Map.Entry<String, String> header : headers.entrySet()) {
+                urlConnection.setRequestProperty(header.getKey(), header.getValue());
+            }
+        }
+
+        return urlConnection;
+    }
+
+    private static byte[] httpFetch(HttpURLConnection urlConnection) throws IOException {
+        // query url, read and return buffered response body
+        // we want to make sure that the connection gets closed here
+        InputStream is = new BufferedInputStream(urlConnection.getInputStream());
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        byte[] result = null;
+        try {
+            byte[] partial = new byte[4096];
+            int read;
+            while ((read = is.read(partial, 0, 4096)) != -1) {
+                baos.write(partial, 0, read);
+            }
+            result = baos.toByteArray();
+        } finally {
+            is.close();
+            baos.close();
+        }
+        return result;
+    }
+
+    private static Charset determineCharset(HttpURLConnection connection) {
+        String contentType = connection.getContentType();
+        if (contentType != null) {
+            String[] split = contentType.split(";");
+            for (int i = 0; i < split.length; i++) {
+                String trimmed = split[i].trim();
+                if (trimmed.startsWith("charset=")) {
+                    try {
+                        return Charset.forName(trimmed.substring(8));
+                    } catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
+                        // we don't know about this charset -> ignore
+                    }
+                }
+            }
+        }
+        return Charset.defaultCharset();
+    }
 
-        request.setHeader("User-Agent", USER_AGENT);
+    public static String httpGet(String url, Map<String, String> headers) throws IOException {
+        HttpURLConnection connection = prepareHttpConnection(url, headers);
+        try {
+            byte[] response = httpFetch(connection);
+            return new String(response, determineCharset(connection));
+        } finally {
+            connection.disconnect();
+        }
+    }
 
-        HttpResponse response = client.execute(request);
-        int status = response.getStatusLine().getStatusCode();
+    public static byte[] httpGetBytes(String url, Map<String, String> headers) throws IOException {
+        HttpURLConnection connection = prepareHttpConnection(url, headers);
+        try {
+            return httpFetch(connection);
+        } finally {
+            connection.disconnect();
+        }
+    }
 
-        // Android's org.apache.http doesn't have the RedirectStrategy class
-        if (status == HttpStatus.SC_MOVED_PERMANENTLY
-                || status == HttpStatus.SC_MOVED_TEMPORARILY) {
-            Header[] headers = response.getHeaders("Location");
+    public static String httpPost(String url, Map<String, String> headers, String postData)
+            throws IOException {
+        HttpURLConnection connection = prepareHttpConnection(url, headers);
 
-            if (headers != null && headers.length != 0) {
-                HttpGet newGet = new HttpGet(headers[headers.length - 1].getValue());
-                for (Header header : request.getAllHeaders()) {
-                    newGet.addHeader(header);
+        try {
+            // write postData to buffered output stream
+            if (postData != null) {
+                connection.setDoOutput(true);
+                BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+                        connection.getOutputStream()));
+                try {
+                    bw.write(postData, 0, postData.length());
+                    // close connection and re-throw exception
+                } finally {
+                    bw.close();
                 }
-                return httpGet(newGet);
-            } else {
-                throw new IOException("Empty redirection header");
             }
+            byte[] response = httpFetch(connection);
+            return new String(response, determineCharset(connection));
+        } finally {
+            connection.disconnect();
         }
+    }
 
-        if (status != HttpStatus.SC_OK) {
-            throw new IOException("HTTP failure (status " + status + ")");
+    public static List<String> allRegexResults(String input, String regex, boolean dotall) {
+        if (input == null) {
+            return null;
         }
+        Pattern pattern = Pattern.compile(regex, dotall ? Pattern.DOTALL : 0);
+        Matcher matcher = pattern.matcher(input);
 
-        return EntityUtils.toString(response.getEntity());
+        List<String> regexResults = new ArrayList<String>();
+        while (matcher.find()) {
+            regexResults.add(matcher.group(1).trim());
+        }
+        return regexResults;
     }
 
     public static String firstRegexResult(String input, String regex, boolean dotall) {
@@ -81,4 +166,3 @@ public class LookupUtils {
         return Html.fromHtml(input).toString().trim();
     }
 }
-
diff --git a/src/com/android/dialer/lookup/dastelefonbuch/TelefonbuchApi.java b/src/com/android/dialer/lookup/dastelefonbuch/TelefonbuchApi.java
index 4e619edc1..dfd578b98 100644
--- a/src/com/android/dialer/lookup/dastelefonbuch/TelefonbuchApi.java
+++ b/src/com/android/dialer/lookup/dastelefonbuch/TelefonbuchApi.java
@@ -21,8 +21,6 @@ import android.net.Uri;
 
 import com.android.dialer.lookup.LookupUtils;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 
 public class TelefonbuchApi {
@@ -48,7 +46,7 @@ public class TelefonbuchApi {
         // Cut out everything we're not interested in (scripts etc.) to
         // speed up the subsequent matching.
         String output = LookupUtils.firstRegexResult(
-                LookupUtils.httpGet(new HttpGet(uri.toString())),
+                LookupUtils.httpGet(uri.toString(), null),
                 ": Treffer(.*)Ende Treffer", true);
 
         String name = parseValue(output, NAME_REGEX, true, false);
diff --git a/src/com/android/dialer/lookup/gebeld/GebeldApi.java b/src/com/android/dialer/lookup/gebeld/GebeldApi.java
index effeb784b..ab650ecb7 100644
--- a/src/com/android/dialer/lookup/gebeld/GebeldApi.java
+++ b/src/com/android/dialer/lookup/gebeld/GebeldApi.java
@@ -21,8 +21,6 @@ import android.net.Uri;
 
 import com.android.dialer.lookup.LookupUtils;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -49,7 +47,7 @@ public class GebeldApi {
         // Cut out everything we're not interested in (scripts etc.) to
         // speed up the subsequent matching.
         String output = LookupUtils.firstRegexResult(
-                LookupUtils.httpGet(new HttpGet(uri.toString())),
+                LookupUtils.httpGet(uri.toString(), null),
                 "<div class=\"small-12 large-4 columns information\">(.*?)</div>", true);
 
         String name = null;
diff --git a/src/com/android/dialer/lookup/google/GoogleForwardLookup.java b/src/com/android/dialer/lookup/google/GoogleForwardLookup.java
index 215cbfd3b..4c701200d 100644
--- a/src/com/android/dialer/lookup/google/GoogleForwardLookup.java
+++ b/src/com/android/dialer/lookup/google/GoogleForwardLookup.java
@@ -19,6 +19,7 @@ package com.android.dialer.lookup.google;
 import com.android.dialer.calllog.ContactInfo;
 import com.android.dialer.lookup.ContactBuilder;
 import com.android.dialer.lookup.ForwardLookup;
+import com.android.dialer.lookup.LookupUtils;
 
 import android.content.Context;
 import android.content.pm.PackageManager;
@@ -31,14 +32,10 @@ import android.provider.ContactsContract.CommonDataKinds.Website;
 import android.text.Html;
 import android.util.Log;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.json.JSONArray;
 import org.json.JSONException;
@@ -129,10 +126,10 @@ public class GoogleForwardLookup extends ForwardLookup {
                 builder = builder.appendQueryParameter(QUERY_RANDOM,
                         getRandomNoiseString());
 
-                String httpResponse = httpGetRequest(
-                        builder.build().toString());
-
-                JSONArray results = new JSONArray(httpResponse);
+                Map<String, String> headers = new HashMap<String, String>();
+                headers.put("User-Agent", mUserAgent);
+                JSONArray results = new JSONArray(LookupUtils.httpGet(builder.build().toString(),
+                        headers));
 
                 if (DEBUG) Log.v(TAG, "Results: " + results);
 
@@ -251,49 +248,6 @@ public class GoogleForwardLookup extends ForwardLookup {
     }
 
     /**
-     * Fetch a URL and return the response as a string encoded in either
-     * UTF-8 or the charset specified in the Content-Type header.
-     *
-     * @param url URL
-     * @return Response from server
-     */
-    private String httpGetRequest(String url) throws IOException {
-        HttpClient client = new DefaultHttpClient();
-        HttpGet request = new HttpGet(url.toString());
-
-        request.setHeader("User-Agent", mUserAgent);
-
-        HttpResponse response = client.execute(request);
-
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        response.getEntity().writeTo(out);
-
-        String charset = getCharsetFromContentType(
-                response.getEntity().getContentType().getValue());
-
-        return new String(out.toByteArray(), charset);
-    }
-
-    /**
-     * Extract the content encoding from the HTTP 'Content-Type' header.
-     *
-     * @param contentType The 'Content-Type' header
-     * @return The charset or "UTF-8"
-     */
-    private static String getCharsetFromContentType(String contentType) {
-        String[] split = contentType.split(";");
-
-        for (int i = 0; i < split.length; i++) {
-            String trimmed = split[i].trim();
-            if (trimmed.startsWith("charset=")) {
-                return trimmed.substring(8);
-            }
-        }
-
-        return "UTF-8";
-    }
-
-    /**
      * Convert HTML to unformatted plain text.
      *
      * @param s HTML content
diff --git a/src/com/android/dialer/lookup/opencnam/OpenCnamReverseLookup.java b/src/com/android/dialer/lookup/opencnam/OpenCnamReverseLookup.java
index ad8507a3a..d1fd34ef0 100644
--- a/src/com/android/dialer/lookup/opencnam/OpenCnamReverseLookup.java
+++ b/src/com/android/dialer/lookup/opencnam/OpenCnamReverseLookup.java
@@ -29,8 +29,6 @@ import com.android.dialer.lookup.ReverseLookup;
 
 import cyanogenmod.providers.CMSettings;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 
 public class OpenCnamReverseLookup extends ReverseLookup {
@@ -106,6 +104,6 @@ public class OpenCnamReverseLookup extends ReverseLookup {
             builder.appendQueryParameter(AUTH_TOKEN, authToken);
         }
 
-        return LookupUtils.httpGet(new HttpGet(builder.build().toString()));
+        return LookupUtils.httpGet(builder.build().toString(), null);
     }
 }
diff --git a/src/com/android/dialer/lookup/openstreetmap/OpenStreetMapForwardLookup.java b/src/com/android/dialer/lookup/openstreetmap/OpenStreetMapForwardLookup.java
index 5bd5e723f..15af95f7b 100644
--- a/src/com/android/dialer/lookup/openstreetmap/OpenStreetMapForwardLookup.java
+++ b/src/com/android/dialer/lookup/openstreetmap/OpenStreetMapForwardLookup.java
@@ -28,13 +28,7 @@ import android.util.Log;
 import com.android.dialer.calllog.ContactInfo;
 import com.android.dialer.lookup.ContactBuilder;
 import com.android.dialer.lookup.ForwardLookup;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.util.EntityUtils;
+import com.android.dialer.lookup.LookupUtils;
 
 import org.json.JSONArray;
 import org.json.JSONException;
@@ -88,11 +82,7 @@ public class OpenStreetMapForwardLookup extends ForwardLookup {
                 RADIUS, lastLocation.getLatitude(), lastLocation.getLongitude());
 
         try {
-            String httpResponse = httpPostRequest(request);
-
-            JSONObject results = new JSONObject(httpResponse);
-
-            return getEntries(results);
+            return getEntries(new JSONObject(LookupUtils.httpPost(LOOKUP_URL, null, request)));
         } catch (IOException e) {
             Log.e(TAG, "Failed to execute query", e);
         } catch (JSONException e) {
@@ -178,13 +168,4 @@ public class OpenStreetMapForwardLookup extends ForwardLookup {
             return null;
         }
     }
-
-    private String httpPostRequest(String query) throws IOException {
-        HttpClient client = new DefaultHttpClient();
-        HttpPost post = new HttpPost(LOOKUP_URL);
-
-        post.setEntity(new StringEntity(query));
-
-        return EntityUtils.toString(client.execute(post).getEntity());
-    }
 }
diff --git a/src/com/android/dialer/lookup/whitepages/WhitePagesApi.java b/src/com/android/dialer/lookup/whitepages/WhitePagesApi.java
index 5b266bf9c..342e1901a 100644
--- a/src/com/android/dialer/lookup/whitepages/WhitePagesApi.java
+++ b/src/com/android/dialer/lookup/whitepages/WhitePagesApi.java
@@ -24,10 +24,10 @@ import android.util.Log;
 import com.android.dialer.lookup.LookupSettings;
 import com.android.dialer.lookup.LookupUtils;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -243,13 +243,13 @@ public class WhitePagesApi {
     }
 
     private static String httpGet(String url) throws IOException {
-        HttpGet get = new HttpGet(url);
-
+        Map<String, String> headers = null;
         if (mCookie != null) {
-            get.setHeader("Cookie", COOKIE + "=" + mCookie);
+            headers = new HashMap<String, String>();
+            headers.put("Cookie", COOKIE + "=" + mCookie);
         }
 
-        String output = LookupUtils.httpGet(get);
+        String output = LookupUtils.httpGet(url, headers);
         // If we can find a new cookie, use it
         Pattern p = Pattern.compile(COOKIE_REGEX, Pattern.DOTALL);
         Matcher m = p.matcher(output);
diff --git a/src/com/android/dialer/lookup/yellowpages/YellowPagesApi.java b/src/com/android/dialer/lookup/yellowpages/YellowPagesApi.java
index b52a67b1f..f17784303 100644
--- a/src/com/android/dialer/lookup/yellowpages/YellowPagesApi.java
+++ b/src/com/android/dialer/lookup/yellowpages/YellowPagesApi.java
@@ -22,8 +22,6 @@ import android.text.TextUtils;
 import com.android.dialer.lookup.LookupSettings;
 import com.android.dialer.lookup.LookupUtils;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -54,11 +52,11 @@ public class YellowPagesApi {
     }
 
     private void fetchPage() throws IOException {
-        mOutput = LookupUtils.httpGet(new HttpGet(mLookupUrl + mNumber));
+        mOutput = LookupUtils.httpGet(mLookupUrl + mNumber, null);
     }
 
     private String getPhotoUrl(String website) throws IOException {
-        String output = LookupUtils.httpGet(new HttpGet(website));
+        String output = LookupUtils.httpGet(website, null);
         String galleryRef = LookupUtils.firstRegexResult(output,
                 "href=\"([^\"]+gallery\\?lid=[^\"]+)\"", true);
         if (galleryRef == null) {
@@ -66,10 +64,8 @@ public class YellowPagesApi {
         }
 
         // Get first image
-        HttpGet get = new HttpGet("http://www.yellowpages.com" + galleryRef);
-        output = LookupUtils.httpGet(get);
-
-        return LookupUtils.firstRegexResult(output,
+        return LookupUtils.firstRegexResult(
+                LookupUtils.httpGet("http://www.yellowpages.com" + galleryRef, null),
                 "\"type\":\"image\",\"src\":\"([^\"]+)\"", true);
     }
 
diff --git a/src/com/android/dialer/lookup/yellowpages/YellowPagesReverseLookup.java b/src/com/android/dialer/lookup/yellowpages/YellowPagesReverseLookup.java
index eaaee5779..56fc837a8 100644
--- a/src/com/android/dialer/lookup/yellowpages/YellowPagesReverseLookup.java
+++ b/src/com/android/dialer/lookup/yellowpages/YellowPagesReverseLookup.java
@@ -18,6 +18,7 @@ package com.android.dialer.lookup.yellowpages;
 
 import com.android.dialer.calllog.ContactInfo;
 import com.android.dialer.lookup.ContactBuilder;
+import com.android.dialer.lookup.LookupUtils;
 import com.android.dialer.lookup.ReverseLookup;
 
 import android.content.ContentResolver;
@@ -28,13 +29,6 @@ import android.net.Uri;
 import android.provider.ContactsContract.CommonDataKinds.StructuredPostal;
 import android.util.Log;
 
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
-
-import java.io.ByteArrayOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 
@@ -61,23 +55,9 @@ public class YellowPagesReverseLookup extends ReverseLookup {
         String scheme = uri.getScheme();
 
         if (scheme.startsWith("http")) {
-            HttpClient client = new DefaultHttpClient();
-            HttpGet request = new HttpGet(uri.toString());
-
             try {
-                HttpResponse response = client.execute(request);
-
-                int responseCode = response.getStatusLine().getStatusCode();
-
-                ByteArrayOutputStream out = new ByteArrayOutputStream();
-                response.getEntity().writeTo(out);
-                byte[] responseBytes = out.toByteArray();
-
-                if (responseCode == HttpStatus.SC_OK) {
-                    Bitmap bmp = BitmapFactory.decodeByteArray(
-                            responseBytes, 0, responseBytes.length);
-                    return bmp;
-                }
+                byte[] response = LookupUtils.httpGetBytes(uri.toString(), null);
+                return BitmapFactory.decodeByteArray(response, 0, response.length);
             } catch (IOException e) {
                 Log.e(TAG, "Failed to retrieve image", e);
             }
diff --git a/src/com/android/dialer/lookup/zabasearch/ZabaSearchApi.java b/src/com/android/dialer/lookup/zabasearch/ZabaSearchApi.java
index 2e8b65abd..6edb50ab2 100644
--- a/src/com/android/dialer/lookup/zabasearch/ZabaSearchApi.java
+++ b/src/com/android/dialer/lookup/zabasearch/ZabaSearchApi.java
@@ -20,8 +20,6 @@ import android.text.TextUtils;
 
 import com.android.dialer.lookup.LookupUtils;
 
-import org.apache.http.client.methods.HttpGet;
-
 import java.io.IOException;
 
 public class ZabaSearchApi {
@@ -38,7 +36,7 @@ public class ZabaSearchApi {
     }
 
     private void fetchPage() throws IOException {
-        mOutput = LookupUtils.httpGet(new HttpGet(LOOKUP_URL + mNumber));
+        mOutput = LookupUtils.httpGet(LOOKUP_URL + mNumber, null);
     }
 
     private void buildContactInfo() {