diff options
author | Koushik Dutta <koushd@gmail.com> | 2013-03-21 02:04:14 -0700 |
---|---|---|
committer | Koushik Dutta <koushd@gmail.com> | 2013-03-21 02:04:14 -0700 |
commit | 1f8554df759a4ace36e4811ad41726c4e79398f9 (patch) | |
tree | c8d0520e17514929be96b1f86dd8008c85782905 | |
parent | f8e2cdf4bdad484cb1966cc6ce795a9706ae0390 (diff) | |
download | AndroidAsync-1f8554df759a4ace36e4811ad41726c4e79398f9.tar.gz AndroidAsync-1f8554df759a4ace36e4811ad41726c4e79398f9.tar.bz2 AndroidAsync-1f8554df759a4ace36e4811ad41726c4e79398f9.zip |
caching works!
20 files changed, 2314 insertions, 161 deletions
diff --git a/AndroidAsync/src/com/koushikdutta/async/AsyncSocketImpl.java b/AndroidAsync/src/com/koushikdutta/async/AsyncNetworkSocket.java index 943428b..45970ef 100644 --- a/AndroidAsync/src/com/koushikdutta/async/AsyncSocketImpl.java +++ b/AndroidAsync/src/com/koushikdutta/async/AsyncNetworkSocket.java @@ -12,8 +12,8 @@ import com.koushikdutta.async.callback.CompletedCallback; import com.koushikdutta.async.callback.DataCallback; import com.koushikdutta.async.callback.WritableCallback; -class AsyncSocketImpl implements AsyncSocket { - AsyncSocketImpl() { +public class AsyncNetworkSocket implements AsyncSocket { + AsyncNetworkSocket() { } public boolean isChunked() { diff --git a/AndroidAsync/src/com/koushikdutta/async/AsyncSSLSocket.java b/AndroidAsync/src/com/koushikdutta/async/AsyncSSLSocket.java index f144155..ee7bb99 100644 --- a/AndroidAsync/src/com/koushikdutta/async/AsyncSSLSocket.java +++ b/AndroidAsync/src/com/koushikdutta/async/AsyncSSLSocket.java @@ -24,7 +24,7 @@ import com.koushikdutta.async.callback.CompletedCallback; import com.koushikdutta.async.callback.DataCallback; import com.koushikdutta.async.callback.WritableCallback; -public class AsyncSSLSocket implements AsyncSocket { +public class AsyncSSLSocket implements WrapperSocket { AsyncSocket mSocket; BufferedDataEmitter mEmitter; BufferedDataSink mSink; @@ -382,4 +382,9 @@ public class AsyncSSLSocket implements AsyncSocket { public AsyncServer getServer() { return mSocket.getServer(); } + + @Override + public AsyncSocket getSocket() { + return mSocket; + } } diff --git a/AndroidAsync/src/com/koushikdutta/async/AsyncServer.java b/AndroidAsync/src/com/koushikdutta/async/AsyncServer.java index 63a3fb6..0b89c53 100644 --- a/AndroidAsync/src/com/koushikdutta/async/AsyncServer.java +++ b/AndroidAsync/src/com/koushikdutta/async/AsyncServer.java @@ -65,7 +65,7 @@ public class AsyncServer { public AsyncServer() { } - private void handleSocket(final AsyncSocketImpl handler) throws ClosedChannelException { + private void handleSocket(final AsyncNetworkSocket handler) throws ClosedChannelException { final ChannelWrapper sc = handler.getChannel(); SelectionKey ckey = sc.register(mSelector); ckey.attach(handler); @@ -285,7 +285,7 @@ public class AsyncServer { public AsyncSocket connectDatagram(final SocketAddress remote) throws IOException { final DatagramChannel socket = DatagramChannel.open(); - final AsyncSocketImpl handler = new AsyncSocketImpl(); + final AsyncNetworkSocket handler = new AsyncNetworkSocket(); handler.attach(socket); // ugh.. this should really be post to make it nonblocking... // but i want datagrams to be immediately writable. @@ -470,19 +470,19 @@ public class AsyncServer { sc.configureBlocking(false); SelectionKey ckey = sc.register(selector, SelectionKey.OP_READ); ListenCallback serverHandler = (ListenCallback) key.attachment(); - AsyncSocketImpl handler = new AsyncSocketImpl(); + AsyncNetworkSocket handler = new AsyncNetworkSocket(); handler.attach(sc); handler.setup(server, ckey); ckey.attach(handler); serverHandler.onAccepted(handler); } else if (key.isReadable()) { - AsyncSocketImpl handler = (AsyncSocketImpl) key.attachment(); + AsyncNetworkSocket handler = (AsyncNetworkSocket) key.attachment(); int transmitted = handler.onReadable(); server.onDataTransmitted(transmitted); } else if (key.isWritable()) { - AsyncSocketImpl handler = (AsyncSocketImpl) key.attachment(); + AsyncNetworkSocket handler = (AsyncNetworkSocket) key.attachment(); handler.onDataWritable(); } else if (key.isConnectable()) { @@ -491,7 +491,7 @@ public class AsyncServer { key.interestOps(SelectionKey.OP_READ); try { sc.finishConnect(); - AsyncSocketImpl newHandler = new AsyncSocketImpl(); + AsyncNetworkSocket newHandler = new AsyncNetworkSocket(); newHandler.setup(server, key); newHandler.attach(sc); key.attach(newHandler); diff --git a/AndroidAsync/src/com/koushikdutta/async/Util.java b/AndroidAsync/src/com/koushikdutta/async/Util.java index 42931c9..14bdf69 100644 --- a/AndroidAsync/src/com/koushikdutta/async/Util.java +++ b/AndroidAsync/src/com/koushikdutta/async/Util.java @@ -162,4 +162,15 @@ public class Util { bbl.add(bb); writeAll(sink, bbl, callback); } + + public static AsyncSocket getWrappedSocket(AsyncSocket socket, Class wrappedClass) { + if (wrappedClass.isInstance(socket)) + return socket; + while (socket instanceof WrapperSocket) { + socket = ((WrapperSocket)socket).getSocket(); + if (wrappedClass.isInstance(socket)) + return socket; + } + return null; + } } diff --git a/AndroidAsync/src/com/koushikdutta/async/WrapperSocket.java b/AndroidAsync/src/com/koushikdutta/async/WrapperSocket.java index 694cb15..4d4eb39 100644 --- a/AndroidAsync/src/com/koushikdutta/async/WrapperSocket.java +++ b/AndroidAsync/src/com/koushikdutta/async/WrapperSocket.java @@ -1,58 +1,5 @@ package com.koushikdutta.async; -import java.nio.ByteBuffer; - -import com.koushikdutta.async.callback.CompletedCallback; -import com.koushikdutta.async.callback.WritableCallback; - -public class WrapperSocket extends FilteredDataEmitter implements AsyncSocket { - private AsyncSocket mSocket; - public void setSocket(AsyncSocket socket) { - mSocket = socket; - setDataEmitter(mSocket); - } - - public AsyncSocket getSocket() { - return mSocket; - } - - @Override - public void write(ByteBuffer bb) { - mSocket.write(bb); - } - - @Override - public void write(ByteBufferList bb) { - mSocket.write(bb); - } - - @Override - public void setWriteableCallback(WritableCallback handler) { - mSocket.setWriteableCallback(handler); - } - - @Override - public WritableCallback getWriteableCallback() { - return getWriteableCallback(); - } - - @Override - public boolean isOpen() { - return mSocket.isOpen(); - } - - @Override - public void close() { - mSocket.close(); - } - - @Override - public void setClosedCallback(CompletedCallback handler) { - mSocket.setClosedCallback(handler); - } - - @Override - public CompletedCallback getClosedCallback() { - return mSocket.getClosedCallback(); - } +public interface WrapperSocket extends AsyncSocket { + public AsyncSocket getSocket(); } diff --git a/AndroidAsync/src/com/koushikdutta/async/WrapperSocketBase.java b/AndroidAsync/src/com/koushikdutta/async/WrapperSocketBase.java new file mode 100644 index 0000000..578d56e --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/WrapperSocketBase.java @@ -0,0 +1,59 @@ +package com.koushikdutta.async; + +import java.nio.ByteBuffer; + +import com.koushikdutta.async.callback.CompletedCallback; +import com.koushikdutta.async.callback.WritableCallback; + +public class WrapperSocketBase extends FilteredDataEmitter implements WrapperSocket { + private AsyncSocket mSocket; + public void setSocket(AsyncSocket socket) { + mSocket = socket; + setDataEmitter(mSocket); + } + + @Override + public AsyncSocket getSocket() { + return mSocket; + } + + @Override + public void write(ByteBuffer bb) { + mSocket.write(bb); + } + + @Override + public void write(ByteBufferList bb) { + mSocket.write(bb); + } + + @Override + public void setWriteableCallback(WritableCallback handler) { + mSocket.setWriteableCallback(handler); + } + + @Override + public WritableCallback getWriteableCallback() { + return getWriteableCallback(); + } + + @Override + public boolean isOpen() { + return mSocket.isOpen(); + } + + @Override + public void close() { + mSocket.close(); + } + + @Override + public void setClosedCallback(CompletedCallback handler) { + mSocket.setClosedCallback(handler); + } + + @Override + public CompletedCallback getClosedCallback() { + return mSocket.getClosedCallback(); + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClient.java b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClient.java index c77d446..c465bbe 100644 --- a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClient.java +++ b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClient.java @@ -8,19 +8,19 @@ import java.net.HttpURLConnection; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; -import java.util.Hashtable; import java.util.concurrent.CancellationException; import java.util.concurrent.TimeoutException; +import junit.framework.Assert; + import org.json.JSONException; import org.json.JSONObject; +import android.os.Bundle; import android.os.Handler; import android.os.Looper; import com.koushikdutta.async.AsyncSSLException; -import com.koushikdutta.async.AsyncSSLSocket; import com.koushikdutta.async.AsyncServer; import com.koushikdutta.async.AsyncSocket; import com.koushikdutta.async.ByteBufferList; @@ -45,16 +45,17 @@ public class AsyncHttpClient { } ArrayList<AsyncHttpClientMiddleware> mMiddleware = new ArrayList<AsyncHttpClientMiddleware>(); - public void addMiddleware(AsyncHttpClientMiddleware middleware) { + public void insertMiddleware(AsyncHttpClientMiddleware middleware) { synchronized (mMiddleware) { - mMiddleware.add(middleware); + mMiddleware.add(0, middleware); } } - private Hashtable<String, HashSet<AsyncSocket>> mSockets = new Hashtable<String, HashSet<AsyncSocket>>(); AsyncServer mServer; public AsyncHttpClient(AsyncServer server) { mServer = server; + insertMiddleware(new AsyncSocketMiddleware(this)); + insertMiddleware(new AsyncSSLSocketMiddleware(this)); } private static abstract class InternalConnectCallback implements ConnectCallback { @@ -102,19 +103,7 @@ public class AsyncHttpClient { return; } final URI uri = request.getUri(); - int port = uri.getPort(); - if (port == -1) { - if (uri.getScheme().equals("http")) - port = 80; - else if (uri.getScheme().equals("https")) - port = 443; - else { - reportConnectedCompleted(cancel, new Exception("invalid uri scheme"), null, callback); - return; - } - } - final String lookup = uri.getScheme() + "//" + uri.getHost() + ":" + port; - final int finalPort = port; + final Bundle state = new Bundle(); final InternalConnectCallback socketConnected = new InternalConnectCallback() { Object scheduled; @@ -145,9 +134,8 @@ public class AsyncHttpClient { reportConnectedCompleted(cancel, ex, null, callback); return; } + final AsyncHttpResponseImpl ret = new AsyncHttpResponseImpl(request) { - boolean keepalive = false; - boolean headersReceived; protected void onHeadersReceived() { try { if (cancel.isCanceled()) @@ -155,12 +143,11 @@ public class AsyncHttpClient { if (scheduled != null) mServer.removeAllCallbacks(scheduled); - headersReceived = true; RawHeaders headers = getRawHeaders(); - String kas = headers.get("Connection"); - if (kas != null && "keep-alive".toLowerCase().equals(kas.toLowerCase())) - keepalive = true; + for (AsyncHttpClientMiddleware middleware: mMiddleware) { + middleware.onHeadersReceived(state, getSocket(), request, getHeaders()); + } if ((headers.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM || headers.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) && request.getFollowRedirect()) { URI redirect = URI.create(headers.get("Location")); @@ -196,32 +183,12 @@ public class AsyncHttpClient { return; super.report(ex); if (!socket.isOpen() || ex != null) { - if (!headersReceived && ex != null) + if (getHeaders() == null && ex != null) reportConnectedCompleted(cancel, ex, null, callback); - return; } - if (!keepalive) { - socket.close(); - } - else { - HashSet<AsyncSocket> sockets = mSockets.get(lookup); - if (sockets == null) { - sockets = new HashSet<AsyncSocket>(); - mSockets.put(lookup, sockets); - } - final HashSet<AsyncSocket> ss = sockets; - synchronized (sockets) { - sockets.add(socket); - socket.setClosedCallback(new CompletedCallback() { - @Override - public void onCompleted(Exception ex) { - synchronized (ss) { - ss.remove(socket); - } - socket.setClosedCallback(null); - } - }); - } + + for (AsyncHttpClientMiddleware middleware: mMiddleware) { + middleware.onRequestComplete(state, socket, request, getHeaders(), ex); } } @@ -238,53 +205,21 @@ public class AsyncHttpClient { setSocket(null); return socket; } - - @Override - public boolean isReusedSocket() { - return reused; - } }; - // if this socket is not being reused, - // check to see if an AsyncSSLSocket needs to be wrapped around it. - if (!reused) { - if (request.getUri().getScheme().equals("https")) { - socket = new AsyncSSLSocket(socket, uri.getHost(), finalPort); - } + for (AsyncHttpClientMiddleware middleware: mMiddleware) { + socket = middleware.onSocket(state, socket, request); } - + ret.setSocket(socket); } }; - synchronized (mMiddleware) { - for (AsyncHttpClientMiddleware middleware: mMiddleware) { - if (middleware.getSocket(request, socketConnected)) - return; - } - } - - HashSet<AsyncSocket> sockets = mSockets.get(lookup); - if (sockets != null) { - synchronized (sockets) { - for (final AsyncSocket socket: sockets) { - if (socket.isOpen()) { - sockets.remove(socket); - socket.setClosedCallback(null); - mServer.post(new Runnable() { - @Override - public void run() { -// Log.i(LOGTAG, "Reusing socket."); - socketConnected.reused = true; - socketConnected.onConnectCompleted(null, socket); - } - }); - return; - } - } - } + for (AsyncHttpClientMiddleware middleware: mMiddleware) { + if (null != (cancel.socketCancelable = middleware.getSocket(state, request, socketConnected))) + return; } - cancel.socketCancelable = mServer.connectSocket(uri.getHost(), port, socketConnected); + Assert.fail(); } public Cancelable execute(URI uri, final HttpConnectCallback callback) { @@ -542,4 +477,8 @@ public class AsyncHttpClient { final AsyncHttpGet get = new AsyncHttpGet(uri); websocket(get, protocol, callback); } + + AsyncServer getServer() { + return mServer; + } } diff --git a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClientMiddleware.java b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClientMiddleware.java index 2198fb7..8963629 100644 --- a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClientMiddleware.java +++ b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpClientMiddleware.java @@ -1,9 +1,15 @@ package com.koushikdutta.async.http; +import android.os.Bundle; + import com.koushikdutta.async.AsyncSocket; +import com.koushikdutta.async.Cancelable; import com.koushikdutta.async.callback.ConnectCallback; +import com.koushikdutta.async.http.libcore.ResponseHeaders; public interface AsyncHttpClientMiddleware { - public boolean getSocket(final AsyncHttpRequest request, final ConnectCallback callback); - public AsyncSocket onSocket(AsyncSocket socket); + public Cancelable getSocket(Bundle state, AsyncHttpRequest request, final ConnectCallback callback); + public AsyncSocket onSocket(Bundle state, AsyncSocket socket, AsyncHttpRequest request); + public void onHeadersReceived(Bundle state, AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers); + public void onRequestComplete(Bundle state, AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers, Exception ex); } diff --git a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpResponse.java b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpResponse.java index 9e3a90a..08d59d6 100644 --- a/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpResponse.java +++ b/AndroidAsync/src/com/koushikdutta/async/http/AsyncHttpResponse.java @@ -12,5 +12,4 @@ public interface AsyncHttpResponse extends DataEmitter, DataSink { public ResponseHeaders getHeaders(); public void end(); public AsyncSocket detachSocket(); - public boolean isReusedSocket(); } diff --git a/AndroidAsync/src/com/koushikdutta/async/http/AsyncSSLSocketMiddleware.java b/AndroidAsync/src/com/koushikdutta/async/http/AsyncSSLSocketMiddleware.java new file mode 100644 index 0000000..ac6d302 --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/http/AsyncSSLSocketMiddleware.java @@ -0,0 +1,24 @@ +package com.koushikdutta.async.http; + +import java.net.URI; + +import com.koushikdutta.async.AsyncSSLSocket; +import com.koushikdutta.async.AsyncSocket; +import com.koushikdutta.async.callback.ConnectCallback; + +public class AsyncSSLSocketMiddleware extends AsyncSocketMiddleware { + public AsyncSSLSocketMiddleware(AsyncHttpClient client) { + super(client, "https", 443); + } + + @Override + protected ConnectCallback wrapCallback(final ConnectCallback callback, final URI uri, final int port) { + return new ConnectCallback() { + @Override + public void onConnectCompleted(Exception ex, AsyncSocket socket) { + socket = new AsyncSSLSocket(socket, uri.getHost(), port); + callback.onConnectCompleted(ex, socket); + } + }; + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/http/AsyncSocketMiddleware.java b/AndroidAsync/src/com/koushikdutta/async/http/AsyncSocketMiddleware.java index eb4159f..bb8f745 100644 --- a/AndroidAsync/src/com/koushikdutta/async/http/AsyncSocketMiddleware.java +++ b/AndroidAsync/src/com/koushikdutta/async/http/AsyncSocketMiddleware.java @@ -1,17 +1,126 @@ package com.koushikdutta.async.http; +import java.net.URI; +import java.util.HashSet; +import java.util.Hashtable; + +import android.os.Bundle; +import android.util.Log; + +import com.koushikdutta.async.AsyncNetworkSocket; import com.koushikdutta.async.AsyncSocket; +import com.koushikdutta.async.Cancelable; +import com.koushikdutta.async.SimpleCancelable; +import com.koushikdutta.async.callback.CompletedCallback; import com.koushikdutta.async.callback.ConnectCallback; +import com.koushikdutta.async.http.libcore.ResponseHeaders; public class AsyncSocketMiddleware implements AsyncHttpClientMiddleware { + protected AsyncSocketMiddleware(AsyncHttpClient client, String scheme, int defaultPort) { + mClient = client; + mScheme = scheme; + mDefaultPort = defaultPort; + } + + String mScheme; + int mDefaultPort; + + public AsyncSocketMiddleware(AsyncHttpClient client) { + this(client, "http", 80); + } + AsyncHttpClient mClient; + private Hashtable<String, HashSet<AsyncSocket>> mSockets = new Hashtable<String, HashSet<AsyncSocket>>(); + + protected ConnectCallback wrapCallback(ConnectCallback callback, URI uri, int port) { + return callback; + } + @Override - public boolean getSocket(AsyncHttpRequest request, ConnectCallback callback) { + public Cancelable getSocket(Bundle state, AsyncHttpRequest request, final ConnectCallback callback) { + final URI uri = request.getUri(); + final int port; + if (uri.getPort() == -1) { + if (!uri.getScheme().equals(mScheme)) + return null; + port = mDefaultPort; + } + else { + port = uri.getPort(); + } + final String lookup = mScheme + "//" + uri.getHost() + ":" + port; + state.putString("socket.lookup", lookup); + state.putString("socket.owner", getClass().getCanonicalName()); + + HashSet<AsyncSocket> sockets = mSockets.get(lookup); + if (sockets != null) { + synchronized (sockets) { + for (final AsyncSocket socket: sockets) { + if (socket.isOpen()) { + sockets.remove(socket); + socket.setClosedCallback(null); + mClient.getServer().post(new Runnable() { + @Override + public void run() { +// Log.i("AsyncHttpSocket", "Reusing socket."); + callback.onConnectCompleted(null, socket); + } + }); + // just a noop/dummy, as this can't actually be cancelled. + return new SimpleCancelable(); + } + } + } + } - return true; + ConnectCallback connectCallback = wrapCallback(callback, uri, port); + return mClient.getServer().connectSocket(uri.getHost(), port, connectCallback); + } + + @Override + public AsyncSocket onSocket(Bundle state, AsyncSocket socket, AsyncHttpRequest request) { + return socket; + } + + @Override + public void onHeadersReceived(Bundle state, AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers) { } @Override - public AsyncSocket onSocket(AsyncSocket socket) { - return null; + public void onRequestComplete(Bundle state, final AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers, Exception ex) { + if (!getClass().getCanonicalName().equals(state.getString("socket.owner"))) { +// Log.i("AsyncHttpSocket", getClass().getCanonicalName() + " Not keeping non-owned socket: " + state.getString("socket.owner")); + return; + } + if (ex != null || !socket.isOpen()) { + socket.close(); + return; + } + String kas = headers.getConnection(); + if (kas != null && "keep-alive".toLowerCase().equals(kas.toLowerCase())) { + String lookup = state.getString("socket.lookup"); + if (lookup == null) + return; + HashSet<AsyncSocket> sockets = mSockets.get(lookup); + if (sockets == null) { + sockets = new HashSet<AsyncSocket>(); + mSockets.put(lookup, sockets); + } + final HashSet<AsyncSocket> ss = sockets; + synchronized (sockets) { + sockets.add(socket); + socket.setClosedCallback(new CompletedCallback() { + @Override + public void onCompleted(Exception ex) { + synchronized (ss) { + ss.remove(socket); + } + socket.setClosedCallback(null); + } + }); + } + } + else { + socket.close(); + } } } diff --git a/AndroidAsync/src/com/koushikdutta/async/http/ResponseCacheMiddleware.java b/AndroidAsync/src/com/koushikdutta/async/http/ResponseCacheMiddleware.java new file mode 100644 index 0000000..6bba6ff --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/http/ResponseCacheMiddleware.java @@ -0,0 +1,741 @@ +package com.koushikdutta.async.http; + +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FilterInputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.math.BigInteger; +import java.net.CacheRequest; +import java.net.CacheResponse; +import java.net.SecureCacheResponse; +import java.net.URI; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.Principal; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLPeerUnverifiedException; + +import junit.framework.Assert; + +import android.os.Bundle; +import android.util.Base64; +import android.util.Log; + +import com.koushikdutta.async.AsyncServer; +import com.koushikdutta.async.AsyncSocket; +import com.koushikdutta.async.ByteBufferList; +import com.koushikdutta.async.Cancelable; +import com.koushikdutta.async.DataEmitter; +import com.koushikdutta.async.SimpleCancelable; +import com.koushikdutta.async.WrapperSocketBase; +import com.koushikdutta.async.callback.CompletedCallback; +import com.koushikdutta.async.callback.ConnectCallback; +import com.koushikdutta.async.callback.DataCallback; +import com.koushikdutta.async.callback.WritableCallback; +import com.koushikdutta.async.http.libcore.RawHeaders; +import com.koushikdutta.async.http.libcore.ResponseHeaders; +import com.koushikdutta.async.util.cache.Charsets; +import com.koushikdutta.async.util.cache.DiskLruCache; +import com.koushikdutta.async.util.cache.StrictLineReader; + +public class ResponseCacheMiddleware implements AsyncHttpClientMiddleware { + private DiskLruCache cache; + private static final int VERSION = 201105; + private static final int ENTRY_METADATA = 0; + private static final int ENTRY_BODY = 1; + private static final int ENTRY_COUNT = 2; + private AsyncHttpClient client; + + public ResponseCacheMiddleware(AsyncHttpClient client, File cacheDir) { + try { + this.client = client; + cache = DiskLruCache.open(cacheDir, VERSION, ENTRY_COUNT, 1024L * 1024L * 10L); + } + catch (IOException e) { + } + } + + boolean caching = true; + public void setCaching(boolean caching) { + this.caching = caching; + } + + public boolean getCaching() { + return caching; + } + + private static String uriToKey(URI uri) { + try { + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + byte[] md5bytes = messageDigest.digest(uri.toString().getBytes()); + return new BigInteger(1, md5bytes).toString(16); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + private class CachingSocket extends WrapperSocketBase { + CacheRequestImpl cacheRequest; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteBufferList cached; + @Override + public void onDataAvailable(DataEmitter emitter, ByteBufferList bb) { + if (cached != null) { + com.koushikdutta.async.Util.emitAllData(this, cached); + // couldn't emit it all, so just wait for another day... + if (cached.remaining() > 0) + return; + cached = null; + } + + // write to cache... any data not consumed needs to be retained for the next callback + OutputStream outputStream = this.outputStream; + try { + if (outputStream == null && cacheRequest != null) + outputStream = cacheRequest.getBody(); + if (outputStream != null) { + for (ByteBuffer b: bb) { + outputStream.write(b.array(), b.arrayOffset() + b.position(), b.remaining()); + } + } + } + catch (Exception e) { + e.printStackTrace(); + outputStream = null; + this.outputStream = null; + cacheRequest = null; + } + + super.onDataAvailable(emitter, bb); + + if (outputStream != null && bb.remaining() > 0) { + cached = new ByteBufferList(); + cached.add(bb); + bb.clear(); + } + } + } + + class CachedSocket implements AsyncSocket { + CacheResponse cacheResponse; + public CachedSocket(CacheResponse cacheResponse) { + this.cacheResponse = cacheResponse; + } + + @Override + public void setDataCallback(DataCallback callback) { + dataCallback = callback; + } + + DataCallback dataCallback; + @Override + public DataCallback getDataCallback() { + return dataCallback; + } + + @Override + public boolean isChunked() { + return false; + } + + boolean paused; + @Override + public void pause() { + paused = true; + } + + void report(Exception e) { + open = false; + if (endCallback != null) + endCallback.onCompleted(e); + if (closedCallback != null) + closedCallback.onCompleted(e); + } + + void spewInternal() { + if (pending.remaining() > 0) { + com.koushikdutta.async.Util.emitAllData(CachedSocket.this, pending); + if (pending.remaining() > 0) + return; + } + + // fill pending + try { + while (pending.remaining() == 0) { + ByteBuffer buffer = ByteBuffer.allocate(8192); + int read = cacheResponse.getBody().read(buffer.array()); + if (read == -1) { + report(null); + return; + } + buffer.limit(read); + pending.add(buffer); + com.koushikdutta.async.Util.emitAllData(CachedSocket.this, pending); + } + } + catch (IOException e) { + report(e); + } + } + + ByteBufferList pending = new ByteBufferList(); + void spew() { + getServer().post(new Runnable() { + @Override + public void run() { + spewInternal(); + } + }); + } + + @Override + public void resume() { + paused = false; + spew(); + } + + @Override + public boolean isPaused() { + return paused; + } + + @Override + public void setEndCallback(CompletedCallback callback) { + endCallback = callback; + } + + CompletedCallback endCallback; + @Override + public CompletedCallback getEndCallback() { + return endCallback; + } + + @Override + public void write(ByteBuffer bb) { + // it's gonna write headers and stuff... whatever + bb.limit(bb.position()); + } + + @Override + public void write(ByteBufferList bb) { + // it's gonna write headers and stuff... whatever + bb.clear(); + } + + @Override + public void setWriteableCallback(WritableCallback handler) { + } + + @Override + public WritableCallback getWriteableCallback() { + return null; + } + + boolean open; + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() { + open = false; + } + + @Override + public void setClosedCallback(CompletedCallback handler) { + closedCallback = handler; + } + + CompletedCallback closedCallback; + @Override + public CompletedCallback getClosedCallback() { + return closedCallback; + } + + @Override + public AsyncServer getServer() { + return client.getServer(); + } + } + +// private static final String LOGTAG = "AsyncHttpCache"; + + @Override + public Cancelable getSocket(Bundle state, AsyncHttpRequest request, final ConnectCallback callback) { + if (!caching) + return null; +// Log.i(LOGTAG, "getting cache socket: " + request.getUri().toString()); + + String key = uriToKey(request.getUri()); + DiskLruCache.Snapshot snapshot; + Entry entry; + try { + snapshot = cache.get(key); + if (snapshot == null) { +// Log.i(LOGTAG, "snapshot fail"); + return null; + } + entry = new Entry(snapshot.getInputStream(ENTRY_METADATA)); + } catch (IOException e) { + // Give up because the cache cannot be read. + return null; + } + + if (!entry.matches(request.getUri(), request.getMethod(), request.getHeaders().getHeaders().toMultimap())) { + snapshot.close(); + return null; + } + +// Log.i(LOGTAG, "Serving from cache"); + final CachedSocket socket = new CachedSocket(entry.isHttps() + ? new EntrySecureCacheResponse(entry, snapshot) + : new EntryCacheResponse(entry, snapshot)); + + client.getServer().post(new Runnable() { + @Override + public void run() { + callback.onConnectCompleted(null, socket); + socket.spewInternal(); + } + }); + return new SimpleCancelable(); + } + + @Override + public AsyncSocket onSocket(Bundle state, AsyncSocket socket, AsyncHttpRequest request) { + if (!caching) + return socket; + + // dont cache socket served from cache + if (com.koushikdutta.async.Util.getWrappedSocket(socket, CachedSocket.class) != null) + return socket; + + if (cache == null) + return socket; + + if (!request.getMethod().equals(AsyncHttpGet.METHOD)) + return socket; + + try { + CachingSocket ret = new CachingSocket(); + ret.setSocket(socket); + return ret; + } + catch (Exception e) { + return socket; + } + } + + @Override + public void onHeadersReceived(Bundle state, AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers) { +// Log.i(LOGTAG, "headers: " + request.getUri().toString()); + CachingSocket caching = (CachingSocket)com.koushikdutta.async.Util.getWrappedSocket(socket, CachingSocket.class); + if (caching == null) + return; + + if (!headers.isCacheable(request.getHeaders())) { + caching.outputStream = null; + caching.cacheRequest = null; + return; + } + + String key = uriToKey(request.getUri()); + RawHeaders varyHeaders = request.getHeaders().getHeaders().getAll(headers.getVaryFields()); + Entry entry = new Entry(request.getUri(), varyHeaders, request, headers); + DiskLruCache.Editor editor = null; + try { + editor = cache.edit(key); + if (editor == null) { +// Log.i(LOGTAG, "can't cache"); + caching.outputStream = null; + return; + } + entry.writeTo(editor); + caching.cacheRequest = new CacheRequestImpl(editor); + Assert.assertNotNull(caching.outputStream); + byte[] bytes = caching.outputStream.toByteArray(); + caching.outputStream = null; + caching.cacheRequest.getBody().write(bytes); + } + catch (IOException e) { +// Log.e(LOGTAG, "error", e); + caching.outputStream = null; + caching.cacheRequest = null; + } + } + + @Override + public void onRequestComplete(Bundle state, AsyncSocket socket, AsyncHttpRequest request, ResponseHeaders headers, Exception ex) { + CachingSocket caching = (CachingSocket)com.koushikdutta.async.Util.getWrappedSocket(socket, CachingSocket.class); + if (caching == null) + return; + +// Log.i(LOGTAG, "Cache done: " + ex); + try { + if (ex != null) + caching.cacheRequest.abort(); + else + caching.cacheRequest.getBody().close(); + } + catch (Exception e) { + } + + // reset for socket reuse + caching.outputStream = new ByteArrayOutputStream(); + caching.cacheRequest = null; + } + + + int writeSuccessCount; + int writeAbortCount; + + private final class CacheRequestImpl extends CacheRequest { + private final DiskLruCache.Editor editor; + private OutputStream cacheOut; + private boolean done; + private OutputStream body; + + public CacheRequestImpl(final DiskLruCache.Editor editor) throws IOException { + this.editor = editor; + this.cacheOut = editor.newOutputStream(ENTRY_BODY); + this.body = new FilterOutputStream(cacheOut) { + @Override public void close() throws IOException { + synchronized (ResponseCacheMiddleware.this) { + if (done) { + return; + } + done = true; + writeSuccessCount++; + } + super.close(); + editor.commit(); + } + + @Override + public void write(byte[] buffer, int offset, int length) throws IOException { + // Since we don't override "write(int oneByte)", we can write directly to "out" + // and avoid the inefficient implementation from the FilterOutputStream. + out.write(buffer, offset, length); + } + }; + } + + @Override public void abort() { + synchronized (ResponseCacheMiddleware.this) { + if (done) { + return; + } + done = true; + writeAbortCount++; + } + try { + cacheOut.close(); + } + catch (IOException e) { + } + try { + editor.abort(); + } catch (IOException ignored) { + } + } + + @Override public OutputStream getBody() throws IOException { + return body; + } + } + + private static final class Entry { + private final String uri; + private final RawHeaders varyHeaders; + private final String requestMethod; + private final RawHeaders responseHeaders; + private final String cipherSuite; + private final Certificate[] peerCertificates; + private final Certificate[] localCertificates; + + /* + * Reads an entry from an input stream. A typical entry looks like this: + * http://google.com/foo + * GET + * 2 + * Accept-Language: fr-CA + * Accept-Charset: UTF-8 + * HTTP/1.1 200 OK + * 3 + * Content-Type: image/png + * Content-Length: 100 + * Cache-Control: max-age=600 + * + * A typical HTTPS file looks like this: + * https://google.com/foo + * GET + * 2 + * Accept-Language: fr-CA + * Accept-Charset: UTF-8 + * HTTP/1.1 200 OK + * 3 + * Content-Type: image/png + * Content-Length: 100 + * Cache-Control: max-age=600 + * + * AES_256_WITH_MD5 + * 2 + * base64-encoded peerCertificate[0] + * base64-encoded peerCertificate[1] + * -1 + * + * The file is newline separated. The first two lines are the URL and + * the request method. Next is the number of HTTP Vary request header + * lines, followed by those lines. + * + * Next is the response status line, followed by the number of HTTP + * response header lines, followed by those lines. + * + * HTTPS responses also contain SSL session information. This begins + * with a blank line, and then a line containing the cipher suite. Next + * is the length of the peer certificate chain. These certificates are + * base64-encoded and appear each on their own line. The next line + * contains the length of the local certificate chain. These + * certificates are also base64-encoded and appear each on their own + * line. A length of -1 is used to encode a null array. + */ + public Entry(InputStream in) throws IOException { + try { + StrictLineReader reader = new StrictLineReader(in, Charsets.US_ASCII); + uri = reader.readLine(); + requestMethod = reader.readLine(); + varyHeaders = new RawHeaders(); + int varyRequestHeaderLineCount = reader.readInt(); + for (int i = 0; i < varyRequestHeaderLineCount; i++) { + varyHeaders.addLine(reader.readLine()); + } + + responseHeaders = new RawHeaders(); + responseHeaders.setStatusLine(reader.readLine()); + int responseHeaderLineCount = reader.readInt(); + for (int i = 0; i < responseHeaderLineCount; i++) { + responseHeaders.addLine(reader.readLine()); + } + +// if (isHttps()) { +// String blank = reader.readLine(); +// if (blank.length() != 0) { +// throw new IOException("expected \"\" but was \"" + blank + "\""); +// } +// cipherSuite = reader.readLine(); +// peerCertificates = readCertArray(reader); +// localCertificates = readCertArray(reader); +// } else { + cipherSuite = null; + peerCertificates = null; + localCertificates = null; +// } + } finally { + in.close(); + } + } + + public Entry(URI uri, RawHeaders varyHeaders, AsyncHttpRequest request, ResponseHeaders responseHeaders) { + this.uri = uri.toString(); + this.varyHeaders = varyHeaders; + this.requestMethod = request.getMethod(); + this.responseHeaders = responseHeaders.getHeaders(); + +// if (isHttps()) { +// HttpsURLConnection httpsConnection = (HttpsURLConnection) httpConnection; +// cipherSuite = httpsConnection.getCipherSuite(); +// Certificate[] peerCertificatesNonFinal = null; +// try { +// peerCertificatesNonFinal = httpsConnection.getServerCertificates(); +// } catch (SSLPeerUnverifiedException ignored) { +// } +// peerCertificates = peerCertificatesNonFinal; +// localCertificates = httpsConnection.getLocalCertificates(); +// } else { + cipherSuite = null; + peerCertificates = null; + localCertificates = null; +// } + } + + public void writeTo(DiskLruCache.Editor editor) throws IOException { + OutputStream out = editor.newOutputStream(ENTRY_METADATA); + Writer writer = new BufferedWriter(new OutputStreamWriter(out, Charsets.UTF_8)); + + writer.write(uri + '\n'); + writer.write(requestMethod + '\n'); + writer.write(Integer.toString(varyHeaders.length()) + '\n'); + for (int i = 0; i < varyHeaders.length(); i++) { + writer.write(varyHeaders.getFieldName(i) + ": " + + varyHeaders.getValue(i) + '\n'); + } + + writer.write(responseHeaders.getStatusLine() + '\n'); + writer.write(Integer.toString(responseHeaders.length()) + '\n'); + for (int i = 0; i < responseHeaders.length(); i++) { + writer.write(responseHeaders.getFieldName(i) + ": " + + responseHeaders.getValue(i) + '\n'); + } + + if (isHttps()) { + writer.write('\n'); + writer.write(cipherSuite + '\n'); + writeCertArray(writer, peerCertificates); + writeCertArray(writer, localCertificates); + } + writer.close(); + } + + private boolean isHttps() { + return uri.startsWith("https://"); + } + + private Certificate[] readCertArray(StrictLineReader reader) throws IOException { + int length = reader.readInt(); + if (length == -1) { + return null; + } + try { + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Certificate[] result = new Certificate[length]; + for (int i = 0; i < result.length; i++) { + String line = reader.readLine(); + byte[] bytes = Base64.decode(line, Base64.DEFAULT); + result[i] = certificateFactory.generateCertificate( + new ByteArrayInputStream(bytes)); + } + return result; + } catch (CertificateException e) { + throw new IOException(e.getMessage()); + } + } + + private void writeCertArray(Writer writer, Certificate[] certificates) throws IOException { + if (certificates == null) { + writer.write("-1\n"); + return; + } + try { + writer.write(Integer.toString(certificates.length) + '\n'); + for (Certificate certificate : certificates) { + byte[] bytes = certificate.getEncoded(); + String line = Base64.encodeToString(bytes, Base64.DEFAULT); + writer.write(line + '\n'); + } + } catch (CertificateEncodingException e) { + throw new IOException(e.getMessage()); + } + } + + public boolean matches(URI uri, String requestMethod, + Map<String, List<String>> requestHeaders) { + return this.uri.equals(uri.toString()) + && this.requestMethod.equals(requestMethod) + && new ResponseHeaders(uri, responseHeaders) + .varyMatches(varyHeaders.toMultimap(), requestHeaders); + } + } + + /** + * Returns an input stream that reads the body of a snapshot, closing the + * snapshot when the stream is closed. + */ + private static InputStream newBodyInputStream(final DiskLruCache.Snapshot snapshot) { + return new FilterInputStream(snapshot.getInputStream(ENTRY_BODY)) { + @Override public void close() throws IOException { + snapshot.close(); + super.close(); + } + }; + } + + static class EntryCacheResponse extends CacheResponse { + private final Entry entry; + private final DiskLruCache.Snapshot snapshot; + private final InputStream in; + + public EntryCacheResponse(Entry entry, DiskLruCache.Snapshot snapshot) { + this.entry = entry; + this.snapshot = snapshot; + this.in = newBodyInputStream(snapshot); + } + + @Override public Map<String, List<String>> getHeaders() { + return entry.responseHeaders.toMultimap(); + } + + @Override public InputStream getBody() { + return in; + } + } + + static class EntrySecureCacheResponse extends SecureCacheResponse { + private final Entry entry; + private final DiskLruCache.Snapshot snapshot; + private final InputStream in; + + public EntrySecureCacheResponse(Entry entry, DiskLruCache.Snapshot snapshot) { + this.entry = entry; + this.snapshot = snapshot; + this.in = newBodyInputStream(snapshot); + } + + @Override public Map<String, List<String>> getHeaders() { + return entry.responseHeaders.toMultimap(); + } + + @Override public InputStream getBody() { + return in; + } + + @Override public String getCipherSuite() { + return entry.cipherSuite; + } + + @Override public List<Certificate> getServerCertificateChain() + throws SSLPeerUnverifiedException { + if (entry.peerCertificates == null || entry.peerCertificates.length == 0) { + throw new SSLPeerUnverifiedException(null); + } + return Arrays.asList(entry.peerCertificates.clone()); + } + + @Override public Principal getPeerPrincipal() throws SSLPeerUnverifiedException { + if (entry.peerCertificates == null || entry.peerCertificates.length == 0) { + throw new SSLPeerUnverifiedException(null); + } + return ((X509Certificate) entry.peerCertificates[0]).getSubjectX500Principal(); + } + + @Override public List<Certificate> getLocalCertificateChain() { + if (entry.localCertificates == null || entry.localCertificates.length == 0) { + return null; + } + return Arrays.asList(entry.localCertificates.clone()); + } + + @Override public Principal getLocalPrincipal() { + if (entry.localCertificates == null || entry.localCertificates.length == 0) { + return null; + } + return ((X509Certificate) entry.localCertificates[0]).getSubjectX500Principal(); + } + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/Arrays.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/Arrays.java new file mode 100644 index 0000000..fc16d7b --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/Arrays.java @@ -0,0 +1,22 @@ +package com.koushikdutta.async.util.cache; + +import java.lang.reflect.Array; + +/* From java.util.Arrays */ +class Arrays { + @SuppressWarnings("unchecked") + static <T> T[] copyOfRange(T[] original, int start, int end) { + int originalLength = original.length; // For exception priority compatibility. + if (start > end) { + throw new IllegalArgumentException(); + } + if (start < 0 || start > originalLength) { + throw new ArrayIndexOutOfBoundsException(); + } + int resultLength = end - start; + int copyLength = Math.min(resultLength, originalLength - start); + T[] result = (T[]) Array.newInstance(original.getClass().getComponentType(), resultLength); + System.arraycopy(original, start, result, 0, copyLength); + return result; + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/Charsets.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/Charsets.java new file mode 100644 index 0000000..8116a52 --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/Charsets.java @@ -0,0 +1,9 @@ +package com.koushikdutta.async.util.cache; + +import java.nio.charset.Charset; + +/** From java.nio.charset.Charsets */ +public class Charsets { + public static final Charset US_ASCII = Charset.forName("US-ASCII"); + public static final Charset UTF_8 = Charset.forName("UTF-8"); +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/DiskLruCache.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/DiskLruCache.java new file mode 100644 index 0000000..9031d4f --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/DiskLruCache.java @@ -0,0 +1,960 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.koushikdutta.async.util.cache; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A cache that uses a bounded amount of space on a filesystem. Each cache + * entry has a string key and a fixed number of values. Each key must match + * the regex <strong>[a-z0-9_-]{1,64}</strong>. Values are byte sequences, + * accessible as streams or files. Each value must be between {@code 0} and + * {@code Integer.MAX_VALUE} bytes in length. + * + * <p>The cache stores its data in a directory on the filesystem. This + * directory must be exclusive to the cache; the cache may delete or overwrite + * files from its directory. It is an error for multiple processes to use the + * same cache directory at the same time. + * + * <p>This cache limits the number of bytes that it will store on the + * filesystem. When the number of stored bytes exceeds the limit, the cache will + * remove entries in the background until the limit is satisfied. The limit is + * not strict: the cache may temporarily exceed it while waiting for files to be + * deleted. The limit does not include filesystem overhead or the cache + * journal so space-sensitive applications should set a conservative limit. + * + * <p>Clients call {@link #edit} to create or update the values of an entry. An + * entry may have only one editor at one time; if a value is not available to be + * edited then {@link #edit} will return null. + * <ul> + * <li>When an entry is being <strong>created</strong> it is necessary to + * supply a full set of values; the empty value should be used as a + * placeholder if necessary. + * <li>When an entry is being <strong>edited</strong>, it is not necessary + * to supply data for every value; values default to their previous + * value. + * </ul> + * Every {@link #edit} call must be matched by a call to {@link Editor#commit} + * or {@link Editor#abort}. Committing is atomic: a read observes the full set + * of values as they were before or after the commit, but never a mix of values. + * + * <p>Clients call {@link #get} to read a snapshot of an entry. The read will + * observe the value at the time that {@link #get} was called. Updates and + * removals after the call do not impact ongoing reads. + * + * <p>This class is tolerant of some I/O errors. If files are missing from the + * filesystem, the corresponding entries will be dropped from the cache. If + * an error occurs while writing a cache value, the edit will fail silently. + * Callers should handle other problems by catching {@code IOException} and + * responding appropriately. + */ +public final class DiskLruCache implements Closeable { + static final String JOURNAL_FILE = "journal"; + static final String JOURNAL_FILE_TMP = "journal.tmp"; + static final String JOURNAL_FILE_BKP = "journal.bkp"; + static final String MAGIC = "libcore.io.DiskLruCache"; + static final String VERSION_1 = "1"; + static final long ANY_SEQUENCE_NUMBER = -1; + static final Pattern LEGAL_KEY_PATTERN = Pattern.compile("[a-z0-9_-]{1,64}"); + private static final String CLEAN = "CLEAN"; + private static final String DIRTY = "DIRTY"; + private static final String REMOVE = "REMOVE"; + private static final String READ = "READ"; + + /* + * This cache uses a journal file named "journal". A typical journal file + * looks like this: + * libcore.io.DiskLruCache + * 1 + * 100 + * 2 + * + * CLEAN 3400330d1dfc7f3f7f4b8d4d803dfcf6 832 21054 + * DIRTY 335c4c6028171cfddfbaae1a9c313c52 + * CLEAN 335c4c6028171cfddfbaae1a9c313c52 3934 2342 + * REMOVE 335c4c6028171cfddfbaae1a9c313c52 + * DIRTY 1ab96a171faeeee38496d8b330771a7a + * CLEAN 1ab96a171faeeee38496d8b330771a7a 1600 234 + * READ 335c4c6028171cfddfbaae1a9c313c52 + * READ 3400330d1dfc7f3f7f4b8d4d803dfcf6 + * + * The first five lines of the journal form its header. They are the + * constant string "libcore.io.DiskLruCache", the disk cache's version, + * the application's version, the value count, and a blank line. + * + * Each of the subsequent lines in the file is a record of the state of a + * cache entry. Each line contains space-separated values: a state, a key, + * and optional state-specific values. + * o DIRTY lines track that an entry is actively being created or updated. + * Every successful DIRTY action should be followed by a CLEAN or REMOVE + * action. DIRTY lines without a matching CLEAN or REMOVE indicate that + * temporary files may need to be deleted. + * o CLEAN lines track a cache entry that has been successfully published + * and may be read. A publish line is followed by the lengths of each of + * its values. + * o READ lines track accesses for LRU. + * o REMOVE lines track entries that have been deleted. + * + * The journal file is appended to as cache operations occur. The journal may + * occasionally be compacted by dropping redundant lines. A temporary file named + * "journal.tmp" will be used during compaction; that file should be deleted if + * it exists when the cache is opened. + */ + + private final File directory; + private final File journalFile; + private final File journalFileTmp; + private final File journalFileBkp; + private final int appVersion; + private long maxSize; + private final int valueCount; + private long size = 0; + private Writer journalWriter; + private final LinkedHashMap<String, Entry> lruEntries + = new LinkedHashMap<String, Entry>(0, 0.75f, true); + private int redundantOpCount; + + /** + * To differentiate between old and current snapshots, each entry is given + * a sequence number each time an edit is committed. A snapshot is stale if + * its sequence number is not equal to its entry's sequence number. + */ + private long nextSequenceNumber = 0; + + /** This cache uses a single background thread to evict entries. */ + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(0, 1, + 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); + private final Callable<Void> cleanupCallable = new Callable<Void>() { + public Void call() throws Exception { + synchronized (DiskLruCache.this) { + if (journalWriter == null) { + return null; // closed + } + trimToSize(); + if (journalRebuildRequired()) { + rebuildJournal(); + redundantOpCount = 0; + } + } + return null; + } + }; + + private DiskLruCache(File directory, int appVersion, int valueCount, long maxSize) { + this.directory = directory; + this.appVersion = appVersion; + this.journalFile = new File(directory, JOURNAL_FILE); + this.journalFileTmp = new File(directory, JOURNAL_FILE_TMP); + this.journalFileBkp = new File(directory, JOURNAL_FILE_BKP); + this.valueCount = valueCount; + this.maxSize = maxSize; + } + + /** + * Opens the cache in {@code directory}, creating a cache if none exists + * there. + * + * @param directory a writable directory + * @param appVersion + * @param valueCount the number of values per cache entry. Must be positive. + * @param maxSize the maximum number of bytes this cache should use to store + * @throws IOException if reading or writing the cache directory fails + */ + public static DiskLruCache open(File directory, int appVersion, int valueCount, long maxSize) + throws IOException { + if (maxSize <= 0) { + throw new IllegalArgumentException("maxSize <= 0"); + } + if (valueCount <= 0) { + throw new IllegalArgumentException("valueCount <= 0"); + } + + // if a bkp file exists, use it instead + File bkpFile = new File(directory, JOURNAL_FILE_BKP); + if (bkpFile.exists()) { + File journalFile = new File(directory, JOURNAL_FILE); + // if journal file also exists just delete backup file + if (journalFile.exists()) { + bkpFile.delete(); + } else { + renameTo(bkpFile, journalFile, false); + } + } + + // prefer to pick up where we left off + DiskLruCache cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); + if (cache.journalFile.exists()) { + try { + cache.readJournal(); + cache.processJournal(); + cache.journalWriter = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(cache.journalFile, true), Charsets.US_ASCII)); + return cache; + } catch (IOException journalIsCorrupt) { + System.out.println("DiskLruCache " + directory + " is corrupt: " + + journalIsCorrupt.getMessage() + ", removing"); + cache.delete(); + } + } + + // create a new empty cache + directory.mkdirs(); + cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); + cache.rebuildJournal(); + return cache; + } + + private void readJournal() throws IOException { + StrictLineReader reader = new StrictLineReader(new FileInputStream(journalFile), + Charsets.US_ASCII); + try { + String magic = reader.readLine(); + String version = reader.readLine(); + String appVersionString = reader.readLine(); + String valueCountString = reader.readLine(); + String blank = reader.readLine(); + if (!MAGIC.equals(magic) + || !VERSION_1.equals(version) + || !Integer.toString(appVersion).equals(appVersionString) + || !Integer.toString(valueCount).equals(valueCountString) + || !"".equals(blank)) { + throw new IOException("unexpected journal header: [" + + magic + ", " + version + ", " + valueCountString + ", " + blank + "]"); + } + + int lineCount = 0; + while (true) { + try { + readJournalLine(reader.readLine()); + lineCount++; + } catch (EOFException endOfJournal) { + break; + } + } + redundantOpCount = lineCount - lruEntries.size(); + } finally { + IoUtils.closeQuietly(reader); + } + } + + private void readJournalLine(String line) throws IOException { + int firstSpace = line.indexOf(' '); + if (firstSpace == -1) { + throw new IOException("unexpected journal line: " + line); + } + + int keyBegin = firstSpace + 1; + int secondSpace = line.indexOf(' ', keyBegin); + final String key; + if (secondSpace == -1) { + key = line.substring(keyBegin); + if (firstSpace == REMOVE.length() && line.startsWith(REMOVE)) { + lruEntries.remove(key); + return; + } + } else { + key = line.substring(keyBegin, secondSpace); + } + + Entry entry = lruEntries.get(key); + if (entry == null) { + entry = new Entry(key); + lruEntries.put(key, entry); + } + + if (secondSpace != -1 && firstSpace == CLEAN.length() && line.startsWith(CLEAN)) { + String[] parts = line.substring(secondSpace + 1).split(" "); + entry.readable = true; + entry.currentEditor = null; + entry.setLengths(parts); + } else if (secondSpace == -1 && firstSpace == DIRTY.length() && line.startsWith(DIRTY)) { + entry.currentEditor = new Editor(entry); + } else if (secondSpace == -1 && firstSpace == READ.length() && line.startsWith(READ)) { + // this work was already done by calling lruEntries.get() + } else { + throw new IOException("unexpected journal line: " + line); + } + } + + /** + * Computes the initial size and collects garbage as a part of opening the + * cache. Dirty entries are assumed to be inconsistent and will be deleted. + */ + private void processJournal() throws IOException { + deleteIfExists(journalFileTmp); + for (Iterator<Entry> i = lruEntries.values().iterator(); i.hasNext(); ) { + Entry entry = i.next(); + if (entry.currentEditor == null) { + for (int t = 0; t < valueCount; t++) { + size += entry.lengths[t]; + } + } else { + entry.currentEditor = null; + for (int t = 0; t < valueCount; t++) { + deleteIfExists(entry.getCleanFile(t)); + deleteIfExists(entry.getDirtyFile(t)); + } + i.remove(); + } + } + } + + /** + * Creates a new journal that omits redundant information. This replaces the + * current journal if it exists. + */ + private synchronized void rebuildJournal() throws IOException { + if (journalWriter != null) { + journalWriter.close(); + } + + Writer writer = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(journalFileTmp), Charsets.US_ASCII)); + try { + writer.write(MAGIC); + writer.write("\n"); + writer.write(VERSION_1); + writer.write("\n"); + writer.write(Integer.toString(appVersion)); + writer.write("\n"); + writer.write(Integer.toString(valueCount)); + writer.write("\n"); + writer.write("\n"); + + for (Entry entry : lruEntries.values()) { + if (entry.currentEditor != null) { + writer.write(DIRTY + ' ' + entry.key + '\n'); + } else { + writer.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); + } + } + } finally { + writer.close(); + } + + if (journalFile.exists()) { + renameTo(journalFile, journalFileBkp, true); + } + renameTo(journalFileTmp, journalFile, false); + journalFileBkp.delete(); + + journalWriter = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(journalFile, true), Charsets.US_ASCII)); + } + + private static void deleteIfExists(File file) throws IOException { + /*try { + Libcore.os.remove(file.getPath()); + } catch (ErrnoException errnoException) { + if (errnoException.errno != OsConstants.ENOENT) { + throw errnoException.rethrowAsIOException(); + } + }*/ + if (file.exists() && !file.delete()) { + throw new IOException(); + } + } + + private static void renameTo(File from, File to, boolean deleteDestination) + throws IOException { + if (deleteDestination) { + deleteIfExists(to); + } + if (!from.renameTo(to)) { + throw new IOException(); + } + } + + /** + * Returns a snapshot of the entry named {@code key}, or null if it doesn't + * exist is not currently readable. If a value is returned, it is moved to + * the head of the LRU queue. + */ + public synchronized Snapshot get(String key) throws IOException { + checkNotClosed(); + validateKey(key); + Entry entry = lruEntries.get(key); + if (entry == null) { + return null; + } + + if (!entry.readable) { + return null; + } + + /* + * Open all streams eagerly to guarantee that we see a single published + * snapshot. If we opened streams lazily then the streams could come + * from different edits. + */ + InputStream[] ins = new InputStream[valueCount]; + try { + for (int i = 0; i < valueCount; i++) { + ins[i] = new FileInputStream(entry.getCleanFile(i)); + } + } catch (FileNotFoundException e) { + // a file must have been deleted manually! + for (int i = 0; i < valueCount; i++) { + if (ins[i] != null) { + IoUtils.closeQuietly(ins[i]); + } else { + break; + } + } + return null; + } + + redundantOpCount++; + journalWriter.append(READ + ' ' + key + '\n'); + if (journalRebuildRequired()) { + executorService.submit(cleanupCallable); + } + + return new Snapshot(key, entry.sequenceNumber, ins, entry.lengths); + } + + /** + * Returns an editor for the entry named {@code key}, or null if another + * edit is in progress. + */ + public Editor edit(String key) throws IOException { + return edit(key, ANY_SEQUENCE_NUMBER); + } + + private synchronized Editor edit(String key, long expectedSequenceNumber) throws IOException { + checkNotClosed(); + validateKey(key); + Entry entry = lruEntries.get(key); + if (expectedSequenceNumber != ANY_SEQUENCE_NUMBER + && (entry == null || entry.sequenceNumber != expectedSequenceNumber)) { + return null; // snapshot is stale + } + if (entry == null) { + entry = new Entry(key); + lruEntries.put(key, entry); + } else if (entry.currentEditor != null) { + return null; // another edit is in progress + } + + Editor editor = new Editor(entry); + entry.currentEditor = editor; + + // flush the journal before creating files to prevent file leaks + journalWriter.write(DIRTY + ' ' + key + '\n'); + journalWriter.flush(); + return editor; + } + + /** + * Returns the directory where this cache stores its data. + */ + public File getDirectory() { + return directory; + } + + /** + * Returns the maximum number of bytes that this cache should use to store + * its data. + */ + public long getMaxSize() { + return maxSize; + } + + /** + * Changes the maximum number of bytes the cache can store and queues a job + * to trim the existing store, if necessary. + */ + public synchronized void setMaxSize(long maxSize) { + this.maxSize = maxSize; + executorService.submit(cleanupCallable); + } + + /** + * Returns the number of bytes currently being used to store the values in + * this cache. This may be greater than the max size if a background + * deletion is pending. + */ + public synchronized long size() { + return size; + } + + private synchronized void completeEdit(Editor editor, boolean success) throws IOException { + Entry entry = editor.entry; + if (entry.currentEditor != editor) { + throw new IllegalStateException(); + } + + // if this edit is creating the entry for the first time, every index must have a value + if (success && !entry.readable) { + for (int i = 0; i < valueCount; i++) { + if (!editor.written[i]) { + editor.abort(); + throw new IllegalStateException("Newly created entry didn't create value for index " + i); + } + if (!entry.getDirtyFile(i).exists()) { + editor.abort(); + return; + } + } + } + + for (int i = 0; i < valueCount; i++) { + File dirty = entry.getDirtyFile(i); + if (success) { + if (dirty.exists()) { + File clean = entry.getCleanFile(i); + dirty.renameTo(clean); + long oldLength = entry.lengths[i]; + long newLength = clean.length(); + entry.lengths[i] = newLength; + size = size - oldLength + newLength; + } + } else { + deleteIfExists(dirty); + } + } + + redundantOpCount++; + entry.currentEditor = null; + if (entry.readable | success) { + entry.readable = true; + journalWriter.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); + if (success) { + entry.sequenceNumber = nextSequenceNumber++; + } + } else { + lruEntries.remove(entry.key); + journalWriter.write(REMOVE + ' ' + entry.key + '\n'); + } + + if (size > maxSize || journalRebuildRequired()) { + executorService.submit(cleanupCallable); + } + } + + /** + * We only rebuild the journal when it will halve the size of the journal + * and eliminate at least 2000 ops. + */ + private boolean journalRebuildRequired() { + final int REDUNDANT_OP_COMPACT_THRESHOLD = 2000; + return redundantOpCount >= REDUNDANT_OP_COMPACT_THRESHOLD + && redundantOpCount >= lruEntries.size(); + } + + /** + * Drops the entry for {@code key} if it exists and can be removed. Entries + * actively being edited cannot be removed. + * + * @return true if an entry was removed. + */ + public synchronized boolean remove(String key) throws IOException { + checkNotClosed(); + validateKey(key); + Entry entry = lruEntries.get(key); + if (entry == null || entry.currentEditor != null) { + return false; + } + + for (int i = 0; i < valueCount; i++) { + File file = entry.getCleanFile(i); + if (!file.delete()) { + throw new IOException("failed to delete " + file); + } + size -= entry.lengths[i]; + entry.lengths[i] = 0; + } + + redundantOpCount++; + journalWriter.append(REMOVE + ' ' + key + '\n'); + lruEntries.remove(key); + + if (journalRebuildRequired()) { + executorService.submit(cleanupCallable); + } + + return true; + } + + /** + * Returns true if this cache has been closed. + */ + public boolean isClosed() { + return journalWriter == null; + } + + private void checkNotClosed() { + if (journalWriter == null) { + throw new IllegalStateException("cache is closed"); + } + } + + /** + * Force buffered operations to the filesystem. + */ + public synchronized void flush() throws IOException { + checkNotClosed(); + trimToSize(); + journalWriter.flush(); + } + + /** + * Closes this cache. Stored values will remain on the filesystem. + */ + public synchronized void close() throws IOException { + if (journalWriter == null) { + return; // already closed + } + for (Entry entry : new ArrayList<Entry>(lruEntries.values())) { + if (entry.currentEditor != null) { + entry.currentEditor.abort(); + } + } + trimToSize(); + journalWriter.close(); + journalWriter = null; + } + + private void trimToSize() throws IOException { + while (size > maxSize) { + Map.Entry<String, Entry> toEvict = lruEntries.entrySet().iterator().next();//lruEntries.eldest(); + remove(toEvict.getKey()); + } + } + + /** + * Closes the cache and deletes all of its stored values. This will delete + * all files in the cache directory including files that weren't created by + * the cache. + */ + public void delete() throws IOException { + close(); + IoUtils.deleteContents(directory); + } + + private void validateKey(String key) { + Matcher matcher = LEGAL_KEY_PATTERN.matcher(key); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "keys must match regex [a-z0-9_-]{1,64}: \"" + key + "\""); + } + } + + private static String inputStreamToString(InputStream in) throws IOException { + return Streams.readFully(new InputStreamReader(in, Charsets.UTF_8)); + } + + /** + * A snapshot of the values for an entry. + */ + public final class Snapshot implements Closeable { + private final String key; + private final long sequenceNumber; + private final InputStream[] ins; + private final long[] lengths; + + private Snapshot(String key, long sequenceNumber, InputStream[] ins, long[] lengths) { + this.key = key; + this.sequenceNumber = sequenceNumber; + this.ins = ins; + this.lengths = lengths; + } + + /** + * Returns an editor for this snapshot's entry, or null if either the + * entry has changed since this snapshot was created or if another edit + * is in progress. + */ + public Editor edit() throws IOException { + return DiskLruCache.this.edit(key, sequenceNumber); + } + + /** + * Returns the unbuffered stream with the value for {@code index}. + */ + public InputStream getInputStream(int index) { + return ins[index]; + } + + /** + * Returns the string value for {@code index}. + */ + public String getString(int index) throws IOException { + return inputStreamToString(getInputStream(index)); + } + + /** + * Returns the byte length of the value for {@code index}. + */ + public long getLength(int index) { + return lengths[index]; + } + + public void close() { + for (InputStream in : ins) { + IoUtils.closeQuietly(in); + } + } + } + + private static final OutputStream NULL_OUTPUT_STREAM = new OutputStream() { + @Override + public void write(int b) throws IOException { + //Eat all writes silently. Nom nom. + } + }; + + /** + * Edits the values for an entry. + */ + public final class Editor { + private final Entry entry; + private final boolean[] written; + private boolean hasErrors; + private boolean committed; + + private Editor(Entry entry) { + this.entry = entry; + this.written = (entry.readable) ? null : new boolean[valueCount]; + } + + /** + * Returns an unbuffered input stream to read the last committed value, + * or null if no value has been committed. + */ + public InputStream newInputStream(int index) throws IOException { + synchronized (DiskLruCache.this) { + if (entry.currentEditor != this) { + throw new IllegalStateException(); + } + if (!entry.readable) { + return null; + } + try { + return new FileInputStream(entry.getCleanFile(index)); + } catch (FileNotFoundException e) { + return null; + } + } + } + + /** + * Returns the last committed value as a string, or null if no value + * has been committed. + */ + public String getString(int index) throws IOException { + InputStream in = newInputStream(index); + return in != null ? inputStreamToString(in) : null; + } + + /** + * Returns a new unbuffered output stream to write the value at + * {@code index}. If the underlying output stream encounters errors + * when writing to the filesystem, this edit will be aborted when + * {@link #commit} is called. The returned output stream does not throw + * IOExceptions. + */ + public OutputStream newOutputStream(int index) throws IOException { + synchronized (DiskLruCache.this) { + if (entry.currentEditor != this) { + throw new IllegalStateException(); + } + if (!entry.readable) { + written[index] = true; + } + File dirtyFile = entry.getDirtyFile(index); + FileOutputStream outputStream; + try { + outputStream = new FileOutputStream(dirtyFile); + } catch (FileNotFoundException e) { + // Attempt to recreate the cache directory. + directory.mkdirs(); + try { + outputStream = new FileOutputStream(dirtyFile); + } catch (FileNotFoundException e2) { + // We are unable to recover. Silently eat the writes. + return NULL_OUTPUT_STREAM; + } + } + return new FaultHidingOutputStream(outputStream); + } + } + + /** + * Sets the value at {@code index} to {@code value}. + */ + public void set(int index, String value) throws IOException { + Writer writer = null; + try { + writer = new OutputStreamWriter(newOutputStream(index), Charsets.UTF_8); + writer.write(value); + } finally { + IoUtils.closeQuietly(writer); + } + } + + /** + * Commits this edit so it is visible to readers. This releases the + * edit lock so another edit may be started on the same key. + */ + public void commit() throws IOException { + if (hasErrors) { + completeEdit(this, false); + remove(entry.key); // the previous entry is stale + } else { + completeEdit(this, true); + } + committed = true; + } + + /** + * Aborts this edit. This releases the edit lock so another edit may be + * started on the same key. + */ + public void abort() throws IOException { + completeEdit(this, false); + } + + public void abortUnlessCommitted() { + if (!committed) { + try { + abort(); + } catch (IOException ignored) { + } + } + } + + private class FaultHidingOutputStream extends FilterOutputStream { + private FaultHidingOutputStream(OutputStream out) { + super(out); + } + + @Override public void write(int oneByte) { + try { + out.write(oneByte); + } catch (IOException e) { + hasErrors = true; + } + } + + @Override public void write(byte[] buffer, int offset, int length) { + try { + out.write(buffer, offset, length); + } catch (IOException e) { + hasErrors = true; + } + } + + @Override public void close() { + try { + out.close(); + } catch (IOException e) { + hasErrors = true; + } + } + + @Override public void flush() { + try { + out.flush(); + } catch (IOException e) { + hasErrors = true; + } + } + } + } + + private final class Entry { + private final String key; + + /** Lengths of this entry's files. */ + private final long[] lengths; + + /** True if this entry has ever been published */ + private boolean readable; + + /** The ongoing edit or null if this entry is not being edited. */ + private Editor currentEditor; + + /** The sequence number of the most recently committed edit to this entry. */ + private long sequenceNumber; + + private Entry(String key) { + this.key = key; + this.lengths = new long[valueCount]; + } + + public String getLengths() throws IOException { + StringBuilder result = new StringBuilder(); + for (long size : lengths) { + result.append(' ').append(size); + } + return result.toString(); + } + + /** + * Set lengths using decimal numbers like "10123". + */ + private void setLengths(String[] strings) throws IOException { + if (strings.length != valueCount) { + throw invalidLengths(strings); + } + + try { + for (int i = 0; i < strings.length; i++) { + lengths[i] = Long.parseLong(strings[i]); + } + } catch (NumberFormatException e) { + throw invalidLengths(strings); + } + } + + private IOException invalidLengths(String[] strings) throws IOException { + throw new IOException("unexpected journal line: " + java.util.Arrays.toString(strings)); + } + + public File getCleanFile(int i) { + return new File(directory, key + "." + i); + } + + public File getDirtyFile(int i) { + return new File(directory, key + "." + i + ".tmp"); + } + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/IOUtils.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/IOUtils.java new file mode 100644 index 0000000..cad50ba --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/IOUtils.java @@ -0,0 +1,34 @@ +package com.koushikdutta.async.util.cache; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; + +/** From libcore.io.IoUtils */ +class IoUtils { + static void deleteContents(File dir) throws IOException { + File[] files = dir.listFiles(); + if (files == null) { + throw new IllegalArgumentException("not a directory: " + dir); + } + for (File file : files) { + if (file.isDirectory()) { + deleteContents(file); + } + if (!file.delete()) { + throw new IOException("failed to delete file: " + file); + } + } + } + + static void closeQuietly(/*Auto*/Closeable closeable) { + if (closeable != null) { + try { + closeable.close(); + } catch (RuntimeException rethrown) { + throw rethrown; + } catch (Exception ignored) { + } + } + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/Streams.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/Streams.java new file mode 100644 index 0000000..c723f8f --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/Streams.java @@ -0,0 +1,22 @@ +package com.koushikdutta.async.util.cache; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringWriter; + +/** From libcore.io.Streams */ +class Streams { + static String readFully(Reader reader) throws IOException { + try { + StringWriter writer = new StringWriter(); + char[] buffer = new char[1024]; + int count; + while ((count = reader.read(buffer)) != -1) { + writer.write(buffer, 0, count); + } + return writer.toString(); + } finally { + reader.close(); + } + } +} diff --git a/AndroidAsync/src/com/koushikdutta/async/util/cache/StrictLineReader.java b/AndroidAsync/src/com/koushikdutta/async/util/cache/StrictLineReader.java new file mode 100644 index 0000000..541b89b --- /dev/null +++ b/AndroidAsync/src/com/koushikdutta/async/util/cache/StrictLineReader.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.koushikdutta.async.util.cache; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.EOFException; +import java.io.InputStream; +import java.io.IOException; +import java.nio.charset.Charset; + +/** + * Buffers input from an {@link InputStream} for reading lines. + * + * This class is used for buffered reading of lines. For purposes of this class, a line ends with + * "\n" or "\r\n". End of input is reported by throwing {@code EOFException}. Unterminated line at + * end of input is invalid and will be ignored, the caller may use {@code hasUnterminatedLine()} + * to detect it after catching the {@code EOFException}. + * + * This class is intended for reading input that strictly consists of lines, such as line-based + * cache entries or cache journal. Unlike the {@link BufferedReader} which in conjunction with + * {@link InputStreamReader} provides similar functionality, this class uses different + * end-of-input reporting and a more restrictive definition of a line. + * + * This class supports only charsets that encode '\r' and '\n' as a single byte with value 13 + * and 10, respectively, and the representation of no other character contains these values. + * We currently check in constructor that the charset is one of US-ASCII, UTF-8 and ISO-8859-1. + * The default charset is US_ASCII. + */ +public class StrictLineReader implements Closeable { + private static final byte CR = (byte)'\r'; + private static final byte LF = (byte)'\n'; + + private final InputStream in; + + /* + * Buffered data is stored in {@code buf}. As long as no exception occurs, 0 <= pos <= end + * and the data in the range [pos, end) is buffered for reading. At end of input, if there is + * an unterminated line, we set end == -1, otherwise end == pos. If the underlying + * {@code InputStream} throws an {@code IOException}, end may remain as either pos or -1. + */ + private byte[] buf; + private int pos; + private int end; + + /** + * Constructs a new {@code StrictLineReader} with the default capacity and charset. + * + * @param in the {@code InputStream} to read data from. + * @throws NullPointerException if {@code in} is null. + */ + public StrictLineReader(InputStream in) { + this(in, 8192); + } + + /** + * Constructs a new {@code LineReader} with the specified capacity and the default charset. + * + * @param in the {@code InputStream} to read data from. + * @param capacity the capacity of the buffer. + * @throws NullPointerException if {@code in} is null. + * @throws IllegalArgumentException for negative or zero {@code capacity}. + */ + public StrictLineReader(InputStream in, int capacity) { + this(in, capacity, Charsets.US_ASCII); + } + + /** + * Constructs a new {@code LineReader} with the specified charset and the default capacity. + * + * @param in the {@code InputStream} to read data from. + * @param charset the charset used to decode data. + * Only US-ASCII, UTF-8 and ISO-8859-1 is supported. + * @throws NullPointerException if {@code in} or {@code charset} is null. + * @throws IllegalArgumentException if the specified charset is not supported. + */ + public StrictLineReader(InputStream in, Charset charset) { + this(in, 8192, charset); + } + + /** + * Constructs a new {@code LineReader} with the specified capacity and charset. + * + * @param in the {@code InputStream} to read data from. + * @param capacity the capacity of the buffer. + * @param charset the charset used to decode data. + * Only US-ASCII, UTF-8 and ISO-8859-1 is supported. + * @throws NullPointerException if {@code in} or {@code charset} is null. + * @throws IllegalArgumentException if {@code capacity} is negative or zero + * or the specified charset is not supported. + */ + public StrictLineReader(InputStream in, int capacity, Charset charset) { + if (in == null) { + throw new NullPointerException("in == null"); + } else if (charset == null) { + throw new NullPointerException("charset == null"); + } + if (capacity < 0) { + throw new IllegalArgumentException("capacity <= 0"); + } + if (!(charset.equals(Charsets.US_ASCII) || charset.equals(Charsets.UTF_8))) { + throw new IllegalArgumentException("Unsupported encoding"); + } + + this.in = in; + buf = new byte[capacity]; + } + + /** + * Closes the reader by closing the underlying {@code InputStream} and + * marking this reader as closed. + * + * @throws IOException for errors when closing the underlying {@code InputStream}. + */ + @Override + public void close() throws IOException { + synchronized (in) { + if (buf != null) { + buf = null; + in.close(); + } + } + } + + /** + * Reads the next line. A line ends with {@code "\n"} or {@code "\r\n"}, + * this end of line marker is not included in the result. + * + * @return the next line from the input. + * @throws IOException for underlying {@code InputStream} errors. + * @throws EOFException for the end of source stream. + */ + public String readLine() throws IOException { + synchronized (in) { + if (buf == null) { + throw new IOException("LineReader is closed"); + } + + // Read more data if we are at the end of the buffered data. + // Though it's an error to read after an exception, we will let {@code fillBuf()} + // throw again if that happens; thus we need to handle end == -1 as well as end == pos. + if (pos >= end) { + fillBuf(); + } + // Try to find LF in the buffered data and return the line if successful. + for (int i = pos; i != end; ++i) { + if (buf[i] == LF) { + int lineEnd = (i != pos && buf[i - 1] == CR) ? i - 1 : i; + String res = new String(buf, pos, lineEnd - pos); + pos = i + 1; + return res; + } + } + + // Let's anticipate up to 80 characters on top of those already read. + ByteArrayOutputStream out = new ByteArrayOutputStream(end - pos + 80) { + @Override + public String toString() { + int length = (count > 0 && buf[count - 1] == CR) ? count - 1 : count; + return new String(buf, 0, length); + } + }; + + while (true) { + out.write(buf, pos, end - pos); + // Mark unterminated line in case fillBuf throws EOFException or IOException. + end = -1; + fillBuf(); + // Try to find LF in the buffered data and return the line if successful. + for (int i = pos; i != end; ++i) { + if (buf[i] == LF) { + if (i != pos) { + out.write(buf, pos, i - pos); + } + pos = i + 1; + return out.toString(); + } + } + } + } + } + + /** + * Read an {@code int} from a line containing its decimal representation. + * + * @return the value of the {@code int} from the next line. + * @throws IOException for underlying {@code InputStream} errors or conversion error. + * @throws EOFException for the end of source stream. + */ + public int readInt() throws IOException { + String intString = readLine(); + try { + return Integer.parseInt(intString); + } catch (NumberFormatException e) { + throw new IOException("expected an int but was \"" + intString + "\""); + } + } + + /** + * Check whether there was an unterminated line at end of input after the line reader reported + * end-of-input with EOFException. The value is meaningless in any other situation. + * + * @return true if there was an unterminated line at end of input. + */ + public boolean hasUnterminatedLine() { + return end == -1; + } + + /** + * Reads new input data into the buffer. Call only with pos == end or end == -1, + * depending on the desired outcome if the function throws. + * + * @throws IOException for underlying {@code InputStream} errors. + * @throws EOFException for the end of source stream. + */ + private void fillBuf() throws IOException { + int result = in.read(buf, 0, buf.length); + if (result == -1) { + throw new EOFException(); + } + pos = 0; + end = result; + } +} + diff --git a/AndroidAsyncSample/AndroidManifest.xml b/AndroidAsyncSample/AndroidManifest.xml index 776b9b1..a343479 100644 --- a/AndroidAsyncSample/AndroidManifest.xml +++ b/AndroidAsyncSample/AndroidManifest.xml @@ -7,6 +7,7 @@ android:minSdkVersion="8" android:targetSdkVersion="15" /> <uses-permission android:name="android.permission.INTERNET"/> + <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/> <application android:icon="@drawable/ic_launcher" diff --git a/AndroidAsyncSample/src/com/koushikdutta/async/sample/MainActivity.java b/AndroidAsyncSample/src/com/koushikdutta/async/sample/MainActivity.java index 9af863c..a7efd0b 100644 --- a/AndroidAsyncSample/src/com/koushikdutta/async/sample/MainActivity.java +++ b/AndroidAsyncSample/src/com/koushikdutta/async/sample/MainActivity.java @@ -11,19 +11,25 @@ import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.drawable.BitmapDrawable; import android.os.Bundle; -import android.util.Log; import android.view.Menu; +import android.view.MenuItem; +import android.view.MenuItem.OnMenuItemClickListener; import android.view.View; import android.view.View.OnClickListener; import android.widget.Button; import android.widget.ImageView; +import android.widget.Toast; import com.koushikdutta.async.http.AsyncHttpClient; import com.koushikdutta.async.http.AsyncHttpPost; import com.koushikdutta.async.http.AsyncHttpResponse; +import com.koushikdutta.async.http.ResponseCacheMiddleware; import com.koushikdutta.async.http.UrlEncodedFormBody; public class MainActivity extends Activity { + static boolean cacheAdded = false; + static ResponseCacheMiddleware cacher; + ImageView rommanager; ImageView tether; ImageView desksms; @@ -32,6 +38,12 @@ public class MainActivity extends Activity { @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); + + if (!cacheAdded) { + cacheAdded = true; + AsyncHttpClient.getDefaultInstance().insertMiddleware(cacher = new ResponseCacheMiddleware(AsyncHttpClient.getDefaultInstance(), getFileStreamPath("asynccache"))); + cacher.setCaching(false); + } setContentView(R.layout.activity_main); Button b = (Button)findViewById(R.id.go); @@ -46,11 +58,25 @@ public class MainActivity extends Activity { tether = (ImageView)findViewById(R.id.tether); desksms = (ImageView)findViewById(R.id.desksms); chart = (ImageView)findViewById(R.id.chart); + + showCacheToast(); } + void showCacheToast() { + boolean caching = cacher.getCaching(); + Toast.makeText(getApplicationContext(), "Caching: " + caching, Toast.LENGTH_SHORT).show(); + } + @Override public boolean onCreateOptionsMenu(Menu menu) { - getMenuInflater().inflate(R.menu.activity_main, menu); + menu.add("Toggle Caching").setOnMenuItemClickListener(new OnMenuItemClickListener() { + @Override + public boolean onMenuItemClick(MenuItem item) { + cacher.setCaching(!cacher.getCaching()); + showCacheToast(); + return true; + } + }); return true; } |