New tool merge two dex files into one.

This code isn't yet ready for general use, but I have booted a device
using a core.jar from the product of a merge. In particular, I still
need to better size the output dex.

The motivation is to support incremental builds in dx. Given a
"--incremental" flag, dx would create a dex file containing only those
classes newer than the target .dex. Then it would merge that small
.dex with the current .dex. Hopefully this yields a nice speedup
over dexing everything.

It would also be possible to use this to do parallel builds. We would
manually partition the input files, dx them, and then merge the result
together.

Change-Id: I8997fb3c8bfe73b64ef38a7cb3d6456fbe799d0f

16 files changed, 440 insertions, 0 deletions

diff --git a/dx/tests/115-merge/com/android/dx/merge/DexMergeTest.java b/dx/tests/115-merge/com/android/dx/merge/DexMergeTest.java
new file mode 100644
index 000000000..2cac97dc2
--- /dev/null
+++ b/dx/tests/115-merge/com/android/dx/merge/DexMergeTest.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.dx.merge;
+
+import dalvik.system.PathClassLoader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import junit.framework.TestCase;
+
+/**
+ * Test that DexMerge works by merging dex files, and then loading them into
+ * the current VM.
+ */
+public final class DexMergeTest extends TestCase {
+
+    public void testFillArrayData() throws Exception {
+        ClassLoader loader = mergeAndLoad(
+                "/testdata/Basic.dex",
+                "/testdata/FillArrayData.dex");
+
+        Class<?> basic = loader.loadClass("testdata.Basic");
+        assertEquals(1, basic.getDeclaredMethods().length);
+
+        Class<?> fillArrayData = loader.loadClass("testdata.FillArrayData");
+        assertTrue(Arrays.equals(
+                new byte[] { 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, -112, -23, 121 },
+                (byte[]) fillArrayData.getMethod("newByteArray").invoke(null)));
+        assertTrue(Arrays.equals(
+                new char[] { 0xFFFF, 0x4321, 0xABCD, 0, 'a', 'b', 'c' },
+                (char[]) fillArrayData.getMethod("newCharArray").invoke(null)));
+        assertTrue(Arrays.equals(
+                new long[] { 4660046610375530309L, 7540113804746346429L, -6246583658587674878L },
+                (long[]) fillArrayData.getMethod("newLongArray").invoke(null)));
+    }
+
+    public void testTryCatchFinally() throws Exception {
+        ClassLoader loader = mergeAndLoad(
+                "/testdata/Basic.dex",
+                "/testdata/TryCatchFinally.dex");
+
+        Class<?> basic = loader.loadClass("testdata.Basic");
+        assertEquals(1, basic.getDeclaredMethods().length);
+
+        Class<?> tryCatchFinally = loader.loadClass("testdata.TryCatchFinally");
+        tryCatchFinally.getDeclaredMethod("method").invoke(null);
+    }
+
+    public void testStaticValues() throws Exception {
+        ClassLoader loader = mergeAndLoad(
+                "/testdata/Basic.dex",
+                "/testdata/StaticValues.dex");
+
+        Class<?> basic = loader.loadClass("testdata.Basic");
+        assertEquals(1, basic.getDeclaredMethods().length);
+
+        Class<?> staticValues = loader.loadClass("testdata.StaticValues");
+        assertEquals((byte) 1, staticValues.getField("a").get(null));
+        assertEquals((short) 2, staticValues.getField("b").get(null));
+        assertEquals('C', staticValues.getField("c").get(null));
+        assertEquals(0xabcd1234, staticValues.getField("d").get(null));
+        assertEquals(4660046610375530309L,staticValues.getField("e").get(null));
+        assertEquals(0.5f, staticValues.getField("f").get(null));
+        assertEquals(-0.25, staticValues.getField("g").get(null));
+        assertEquals("this is a String", staticValues.getField("h").get(null));
+        assertEquals(String.class, staticValues.getField("i").get(null));
+        assertEquals("[0, 1]", Arrays.toString((int[]) staticValues.getField("j").get(null)));
+        assertEquals(null, staticValues.getField("k").get(null));
+        assertEquals(true, staticValues.getField("l").get(null));
+        assertEquals(false, staticValues.getField("m").get(null));
+    }
+
+    public ClassLoader mergeAndLoad(String dexAResource, String dexBResource) throws IOException {
+        File dexA = resourceToFile(dexAResource);
+        File dexB = resourceToFile(dexBResource);
+        File mergedDex = File.createTempFile("DexMergeTest", ".classes.dex");
+        new DexMerger(mergedDex, dexA, dexB).merge();
+        File mergedJar = dexToJar(mergedDex);
+        return new PathClassLoader(mergedJar.getPath(), getClass().getClassLoader());
+    }
+
+    private File resourceToFile(String resource) throws IOException {
+        File result = File.createTempFile("DexMergeTest", ".resource");
+        result.deleteOnExit();
+        FileOutputStream out = new FileOutputStream(result);
+        InputStream in = getClass().getResourceAsStream(resource);
+        if (in == null) {
+            throw new IllegalArgumentException("No such resource: " + resource);
+        }
+        copy(in, out);
+        out.close();
+        return result;
+    }
+
+    private File dexToJar(File dex) throws IOException {
+        File result = File.createTempFile("DexMergeTest", ".jar");
+        result.deleteOnExit();
+        JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(result));
+        jarOut.putNextEntry(new JarEntry("classes.dex"));
+        copy(new FileInputStream(dex), jarOut);
+        jarOut.closeEntry();
+        jarOut.close();
+        return result;
+    }
+
+    private void copy(InputStream in, OutputStream out) throws IOException {
+        byte[] buffer = new byte[1024];
+        int count;
+        while ((count = in.read(buffer)) != -1) {
+            out.write(buffer, 0, count);
+        }
+        in.close();
+    }
+}
diff --git a/dx/tests/115-merge/expected.txt b/dx/tests/115-merge/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/115-merge/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/115-merge/info.txt b/dx/tests/115-merge/info.txt
new file mode 100644
index 000000000..c1fa2e463
--- /dev/null
+++ b/dx/tests/115-merge/info.txt
@@ -0,0 +1,6 @@
+Merges two dex files into one and then loads the result.
+
+Because it loads the merged dex files, this JUnit test only works on a dalvikvm.
+The run script requires vogar, so you must have vogar on your $PATH to run this
+test. You'll also need a device or host VM for vogar to attach to.
+
diff --git a/dx/tests/115-merge/run b/dx/tests/115-merge/run
new file mode 100644
index 000000000..d1cf82d86
--- /dev/null
+++ b/dx/tests/115-merge/run
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Find dx.jar from dx in the android dev tree
+prog=`which dx`
+progdir=`dirname "${prog}"`
+
+javac testdata/*.java
+dx --dex --output=testdata/Basic.dex testdata/Basic.class
+dx --dex --output=testdata/FillArrayData.dex testdata/FillArrayData.class
+dx --dex --output=testdata/StaticValues.dex testdata/StaticValues.class
+dx --dex --output=testdata/TryCatchFinally.dex testdata/TryCatchFinally.class
+jar cfM resources.jar testdata/*.dex
+
+vogar --classpath resources.jar \
+  --classpath $progdir/../framework/dx.jar \
+  com/android/dx/merge/DexMergeTest.java > unit-out.txt
+
+if [ "$?" = "0" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi
diff --git a/dx/tests/115-merge/testdata/Basic.java b/dx/tests/115-merge/testdata/Basic.java
new file mode 100644
index 000000000..01a163536
--- /dev/null
+++ b/dx/tests/115-merge/testdata/Basic.java
@@ -0,0 +1,10 @@
+package testdata;
+
+public class Basic {
+
+    String field = "this is a field";
+
+    String method() {
+        return "this is a method result";
+    }
+}
diff --git a/dx/tests/115-merge/testdata/FillArrayData.java b/dx/tests/115-merge/testdata/FillArrayData.java
new file mode 100644
index 000000000..0ece9347e
--- /dev/null
+++ b/dx/tests/115-merge/testdata/FillArrayData.java
@@ -0,0 +1,16 @@
+package testdata;
+
+public class FillArrayData {
+
+    public static byte[] newByteArray() {
+        return new byte[] { 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, -112, -23, 121 };
+    }
+
+    public static char[] newCharArray() {
+        return new char[] { 0xFFFF, 0x4321, 0xABCD, 0, 'a', 'b', 'c' };
+    }
+
+    public static long[] newLongArray() {
+        return new long[] { 4660046610375530309L, 7540113804746346429L, -6246583658587674878L };
+    }
+}
diff --git a/dx/tests/115-merge/testdata/StaticValues.java b/dx/tests/115-merge/testdata/StaticValues.java
new file mode 100644
index 000000000..1a8648f9f
--- /dev/null
+++ b/dx/tests/115-merge/testdata/StaticValues.java
@@ -0,0 +1,17 @@
+package testdata;
+
+public class StaticValues {
+    public static final byte a = 1;
+    public static final short b = 2;
+    public static final char c = 'C';
+    public static final int d = 0xabcd1234;
+    public static final long e = 4660046610375530309L;
+    public static final float f = 0.5f;
+    public static final double g = -0.25;
+    public static final String h = "this is a String";
+    public static final Class<?> i = String.class;
+    public static final int[] j = { 0, 1 };
+    public static final String k = null;
+    public static final boolean l = true;
+    public static final boolean m = false;
+}
diff --git a/dx/tests/115-merge/testdata/TryCatchFinally.java b/dx/tests/115-merge/testdata/TryCatchFinally.java
new file mode 100644
index 000000000..4f3769e52
--- /dev/null
+++ b/dx/tests/115-merge/testdata/TryCatchFinally.java
@@ -0,0 +1,26 @@
+package testdata;
+
+public class TryCatchFinally {
+
+    public static void method() {
+        int count = 0;
+        try {
+            if (true) {
+                throw new NullPointerException();
+            }
+            throw new AssertionError();
+        } catch (IllegalStateException e) {
+            throw new AssertionError();
+        } catch (NullPointerException expected) {
+            count++;
+        } catch (RuntimeException e) {
+            throw new AssertionError();
+        } finally {
+            count++;
+        }
+
+        if (count != 2) {
+            throw new AssertionError();
+        }
+    }
+}
diff --git a/dx/tests/116-leb128/com/android/dx/util/Leb128UtilsTest.java b/dx/tests/116-leb128/com/android/dx/util/Leb128UtilsTest.java
new file mode 100644
index 000000000..f47bc86d1
--- /dev/null
+++ b/dx/tests/116-leb128/com/android/dx/util/Leb128UtilsTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.dx.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import junit.framework.TestCase;
+
+public final class Leb128UtilsTest extends TestCase {
+
+    public void testDecodeUnsignedLeb() throws IOException {
+        assertEquals(0, Leb128Utils.readUnsignedLeb128(newDataInput((byte) 0)));
+        assertEquals(1, Leb128Utils.readUnsignedLeb128(newDataInput((byte) 1)));
+        assertEquals(127, Leb128Utils.readUnsignedLeb128(newDataInput((byte) 0x7F)));
+        assertEquals(16256, Leb128Utils.readUnsignedLeb128(newDataInput((byte) 0x80, (byte) 0x7F)));
+    }
+
+    public void testEncodeUnsignedLeb() throws IOException {
+        assertEquals(new byte[] { 0 }, encodeUnsignedLeb(0));
+        assertEquals(new byte[] { 1 }, encodeUnsignedLeb(1));
+        assertEquals(new byte[] { 0x7F }, encodeUnsignedLeb(127));
+        assertEquals(new byte[] { (byte) 0x80, 0x7F }, encodeUnsignedLeb(16256));
+        assertEquals(new byte[] { (byte) 0xb4, 0x07 }, encodeUnsignedLeb(0x3b4));
+        assertEquals(new byte[] { (byte) 0x8c, 0x08 }, encodeUnsignedLeb(0x40c));
+    }
+
+    public void testDecodeSignedLeb() throws IOException {
+        assertEquals(0, Leb128Utils.readSignedLeb128(newDataInput((byte) 0)));
+        assertEquals(1, Leb128Utils.readSignedLeb128(newDataInput((byte) 1)));
+        assertEquals(-1, Leb128Utils.readSignedLeb128(newDataInput((byte) 0x7F)));
+        assertEquals(0x3C, Leb128Utils.readSignedLeb128(newDataInput((byte) 0x3C)));
+        assertEquals(-128, Leb128Utils.readSignedLeb128(newDataInput((byte) 0x80, (byte) 0x7F)));
+    }
+
+    public void testEncodeSignedLeb() throws IOException {
+        assertEquals(new byte[] { 0 }, encodeSignedLeb(0));
+        assertEquals(new byte[] { 1 }, encodeSignedLeb(1));
+        assertEquals(new byte[] { 0x7F }, encodeSignedLeb(-1));
+        assertEquals(new byte[] { (byte) 0x80, 0x7F }, encodeSignedLeb(-128));
+    }
+
+    private byte[] encodeSignedLeb(int value) {
+        byte[] buffer = new byte[5];
+        int length = Leb128Utils.writeSignedLeb128(buffer, 0, value);
+        return Arrays.copyOfRange(buffer, 0, length);
+    }
+
+    private byte[] encodeUnsignedLeb(int value) {
+        byte[] buffer = new byte[5];
+        int length = Leb128Utils.writeUnsignedLeb128(buffer, 0, value);
+        return Arrays.copyOfRange(buffer, 0, length);
+    }
+
+    public DataInputStream newDataInput(byte... bytes) {
+        return new DataInputStream(new ByteArrayInputStream(bytes));
+    }
+
+    private void assertEquals(byte[] expected, byte[] actual) {
+        assertTrue(Arrays.equals(expected, actual));
+    }
+}
diff --git a/dx/tests/116-leb128/expected.txt b/dx/tests/116-leb128/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/116-leb128/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/116-leb128/info.txt b/dx/tests/116-leb128/info.txt
new file mode 100644
index 000000000..1603ec3f6
--- /dev/null
+++ b/dx/tests/116-leb128/info.txt
@@ -0,0 +1,5 @@
+Performs little endian operations.
+
+The run script requires vogar, so you must have vogar on your $PATH to run this
+test. You'll also need a device or host VM for vogar to attach to.
+
diff --git a/dx/tests/116-leb128/run b/dx/tests/116-leb128/run
new file mode 100644
index 000000000..1e729ed90
--- /dev/null
+++ b/dx/tests/116-leb128/run
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Find dx.jar from dx in the android dev tree
+prog=`which dx`
+progdir=`dirname "${prog}"`
+
+vogar --classpath $progdir/../framework/dx.jar \
+  com/android/dx/util/Leb128UtilsTest.java > unit-out.txt
+
+if [ "$?" = "0" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi
diff --git a/dx/tests/117-modified-utf8/com/android/dx/util/Mutf8Test.java b/dx/tests/117-modified-utf8/com/android/dx/util/Mutf8Test.java
new file mode 100644
index 000000000..736b1bfe6
--- /dev/null
+++ b/dx/tests/117-modified-utf8/com/android/dx/util/Mutf8Test.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.dx.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.Arrays;
+import junit.framework.TestCase;
+
+public final class Mutf8Test extends TestCase {
+
+    public void testDecode() throws IOException {
+        File file = createTempFile(new byte[] { 'A', 'B', 'C', (byte) 0xc0, (byte) 0x80, 0, 'E' });
+        RandomAccessFile f = new RandomAccessFile(file, "r");
+        assertEquals('A', f.readByte());
+        assertEquals("BC\u0000", Mutf8.decode(f, new char[3]));
+        assertEquals('E', f.readByte());
+        file.delete();
+    }
+
+    public void testEncode() throws IOException {
+        assertEquals(Arrays.toString(new byte[] { 'B', 'C', (byte) 0xc0, (byte) 0x80 }),
+                Arrays.toString(Mutf8.encode("BC\u0000")));
+    }
+
+    private File createTempFile(byte[] contents) throws IOException {
+        File result = File.createTempFile(getClass().getName(), "test");
+        FileOutputStream out = new FileOutputStream(result);
+        out.write(contents);
+        out.close();
+        return result;
+    }
+}
diff --git a/dx/tests/117-modified-utf8/expected.txt b/dx/tests/117-modified-utf8/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/117-modified-utf8/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/117-modified-utf8/info.txt b/dx/tests/117-modified-utf8/info.txt
new file mode 100644
index 000000000..df11d9808
--- /dev/null
+++ b/dx/tests/117-modified-utf8/info.txt
@@ -0,0 +1,5 @@
+Performs modified UTF-8 operations.
+
+The run script requires vogar, so you must have vogar on your $PATH to run this
+test. You'll also need a device or host VM for vogar to attach to.
+
diff --git a/dx/tests/117-modified-utf8/run b/dx/tests/117-modified-utf8/run
new file mode 100644
index 000000000..a68999156
--- /dev/null
+++ b/dx/tests/117-modified-utf8/run
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Find dx.jar from dx in the android dev tree
+prog=`which dx`
+progdir=`dirname "${prog}"`
+
+vogar --classpath $progdir/../framework/dx.jar \
+  com/android/dx/util/Mutf8Test.java > unit-out.txt
+
+if [ "$?" = "0" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi