Merge "Support --num-threads with --multi-dex (take 2)" into lmp-mr1-dev

16 files changed, 662 insertions, 135 deletions

diff --git a/dx/src/com/android/dx/command/dexer/Main.java b/dx/src/com/android/dx/command/dexer/Main.java
index 06ffcedef..36b9e2059 100644
--- a/dx/src/com/android/dx/command/dexer/Main.java
+++ b/dx/src/com/android/dx/command/dexer/Main.java
@@ -64,12 +64,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.jar.Attributes;
@@ -179,11 +181,39 @@ public class Main {
     /** Library .dex files to merge into the output .dex. */
     private static final List<byte[]> libraryDexBuffers = new ArrayList<byte[]>();
 
-    /** thread pool object used for multi-threaded file processing */
-    private static ExecutorService threadPool;
+    /** Thread pool object used for multi-thread class translation. */
+    private static ExecutorService classTranslatorPool;
 
-    /** used to handle Errors for multi-threaded file processing */
-    private static List<Future<Void>> parallelProcessorFutures;
+    /** Single thread executor, for collecting results of parallel translation,
+     * and adding classes to dex file in original input file order. */
+    private static ExecutorService classDefItemConsumer;
+
+    /** Futures for {@code classDefItemConsumer} tasks. */
+    private static List<Future<Boolean>> addToDexFutures =
+            new ArrayList<Future<Boolean>>();
+
+    /** Thread pool object used for multi-thread dex conversion (to byte array).
+     * Used in combination with multi-dex support, to allow outputing
+     * a completed dex file, in parallel with continuing processing. */
+    private static ExecutorService dexOutPool;
+
+    /** Futures for {@code dexOutPool} task. */
+    private static List<Future<byte[]>> dexOutputFutures =
+            new ArrayList<Future<byte[]>>();
+
+    /** Lock object used to to coordinate dex file rotation, and
+     * multi-threaded translation. */
+    private static Object dexRotationLock = new Object();
+
+    /** Record the number if method indices "reserved" for files
+     * committed to translation in the context of the current dex
+     * file, but not yet added. */
+    private static int maxMethodIdsInProcess = 0;
+
+    /** Record the number if field indices "reserved" for files
+     * committed to translation in the context of the current dex
+     * file, but not yet added. */
+    private static int maxFieldIdsInProcess = 0;
 
     /** true if any files are successfully processed */
     private static volatile boolean anyFilesProcessed;
@@ -290,7 +320,7 @@ public class Main {
         byte[] outArray = null;
 
         if (!outputDex.isEmpty() || (args.humanOutName != null)) {
-            outArray = writeDex();
+            outArray = writeDex(outputDex);
 
             if (outArray == null) {
                 return 2;
@@ -325,13 +355,14 @@ public class Main {
     private static int runMultiDex() throws IOException {
 
         assert !args.incremental;
-        assert args.numThreads == 1;
 
         if (args.mainDexListFile != null) {
             classesInMainDex = new HashSet<String>();
             readPathsFromFile(args.mainDexListFile, classesInMainDex);
         }
 
+        dexOutPool = Executors.newFixedThreadPool(args.numThreads);
+
         if (!processAllFiles()) {
             return 1;
         }
@@ -342,14 +373,31 @@ public class Main {
 
         if (outputDex != null) {
             // this array is null if no classes were defined
-            dexOutputArrays.add(writeDex());
+
+            dexOutputFutures.add(dexOutPool.submit(new DexWriter(outputDex)));
 
             // Effectively free up the (often massive) DexFile memory.
             outputDex = null;
         }
+        try {
+            dexOutPool.shutdown();
+            if (!dexOutPool.awaitTermination(600L, TimeUnit.SECONDS)) {
+                throw new RuntimeException("Timed out waiting for dex writer threads.");
+            }
 
-        if (args.jarOutput) {
+            for (Future<byte[]> f : dexOutputFutures) {
+                dexOutputArrays.add(f.get());
+            }
 
+        } catch (InterruptedException ex) {
+            dexOutPool.shutdownNow();
+            throw new RuntimeException("A dex writer thread has been interrupted.");
+        } catch (Exception e) {
+            dexOutPool.shutdownNow();
+            throw new RuntimeException("Unexpected exception in dex writer thread");
+        }
+
+        if (args.jarOutput) {
             for (int i = 0; i < dexOutputArrays.size(); i++) {
                 outputResources.put(getDexFileName(i),
                         dexOutputArrays.get(i));
@@ -369,7 +417,6 @@ public class Main {
                     closeOutput(out);
                 }
             }
-
         }
 
         return 0;
@@ -475,10 +522,14 @@ public class Main {
         anyFilesProcessed = false;
         String[] fileNames = args.fileNames;
 
-        if (args.numThreads > 1) {
-            threadPool = Executors.newFixedThreadPool(args.numThreads);
-            parallelProcessorFutures = new ArrayList<Future<Void>>();
-        }
+        // translate classes in parallel
+        classTranslatorPool = new ThreadPoolExecutor(args.numThreads,
+               args.numThreads, 0, TimeUnit.SECONDS,
+               new ArrayBlockingQueue<Runnable>(2 * args.numThreads, true),
+               new ThreadPoolExecutor.CallerRunsPolicy());
+        // collect translated and write to dex in order
+        classDefItemConsumer = Executors.newSingleThreadExecutor();
+
 
         try {
             if (args.mainDexListFile != null) {
@@ -491,13 +542,25 @@ public class Main {
                     processOne(fileNames[i], mainPassFilter);
                 }
 
-                if (dexOutputArrays.size() > 0) {
+                if (dexOutputFutures.size() > 0) {
                     throw new DexException("Too many classes in " + Arguments.MAIN_DEX_LIST_OPTION
                             + ", main dex capacity exceeded");
                 }
 
                 if (args.minimalMainDex) {
                     // start second pass directly in a secondary dex file.
+
+                    // Wait for classes in progress to complete
+                    synchronized(dexRotationLock) {
+                        while(maxMethodIdsInProcess > 0 || maxFieldIdsInProcess > 0) {
+                            try {
+                                dexRotationLock.wait();
+                            } catch(InterruptedException ex) {
+                                /* ignore */
+                            }
+                        }
+                    }
+
                     rotateDexFile();
                 }
 
@@ -518,35 +581,36 @@ public class Main {
              */
         }
 
-        if (args.numThreads > 1) {
-            try {
-                threadPool.shutdown();
-                if (!threadPool.awaitTermination(600L, TimeUnit.SECONDS)) {
-                    throw new RuntimeException("Timed out waiting for threads.");
-                }
-            } catch (InterruptedException ex) {
-                threadPool.shutdownNow();
-                throw new RuntimeException("A thread has been interrupted.");
-            }
+        try {
+            classTranslatorPool.shutdown();
+            classTranslatorPool.awaitTermination(600L, TimeUnit.SECONDS);
+            classDefItemConsumer.shutdown();
+            classDefItemConsumer.awaitTermination(600L, TimeUnit.SECONDS);
 
-            try {
-              for (Future<?> future : parallelProcessorFutures) {
-                future.get();
-              }
-            } catch (ExecutionException e) {
-                Throwable cause = e.getCause();
-                // All Exceptions should have been handled in the ParallelProcessor, only Errors
-                // should remain
-                if (cause instanceof Error) {
-                    throw (Error) e.getCause();
-                } else {
-                    throw new AssertionError(e.getCause());
+            for (Future<Boolean> f : addToDexFutures) {
+                try {
+                    f.get();
+                } catch(ExecutionException ex) {
+                    // Catch any previously uncaught exceptions from
+                    // class translation and adding to dex.
+                    int count = errors.incrementAndGet();
+                    if (count < 10) {
+                        DxConsole.err.println("Uncaught translation error: " + ex.getCause());
+                    } else {
+                        throw new InterruptedException("Too many errors");
+                    }
                 }
-            } catch (InterruptedException e) {
-              // If we're here, it means all threads have completed cleanly, so there should not be
-              // any InterruptedException
-              throw new AssertionError(e);
             }
+
+        } catch (InterruptedException ie) {
+            classTranslatorPool.shutdownNow();
+            classDefItemConsumer.shutdownNow();
+            throw new RuntimeException("Translation has been interrupted", ie);
+        } catch (Exception e) {
+            classTranslatorPool.shutdownNow();
+            classDefItemConsumer.shutdownNow();
+            e.printStackTrace(System.out);
+            throw new RuntimeException("Unexpected exception in translator thread.", e);
         }
 
         int errorNum = errors.get();
@@ -582,7 +646,11 @@ public class Main {
 
     private static void rotateDexFile() {
         if (outputDex != null) {
-            dexOutputArrays.add(writeDex());
+            if (dexOutPool != null) {
+                dexOutputFutures.add(dexOutPool.submit(new DexWriter(outputDex)));
+            } else {
+                dexOutputArrays.add(writeDex(outputDex));
+            }
         }
 
         createDexFile();
@@ -599,47 +667,18 @@ public class Main {
     private static void processOne(String pathname, FileNameFilter filter) {
         ClassPathOpener opener;
 
-        opener = new ClassPathOpener(pathname, false, filter,
-                new ClassPathOpener.Consumer() {
-
-            @Override
-            public boolean processFileBytes(String name, long lastModified, byte[] bytes) {
-                return Main.processFileBytes(name, lastModified, bytes);
-            }
-
-            @Override
-            public void onException(Exception ex) {
-                if (ex instanceof StopProcessing) {
-                    throw (StopProcessing) ex;
-                } else if (ex instanceof SimException) {
-                    DxConsole.err.println("\nEXCEPTION FROM SIMULATION:");
-                    DxConsole.err.println(ex.getMessage() + "\n");
-                    DxConsole.err.println(((SimException) ex).getContext());
-                } else {
-                    DxConsole.err.println("\nUNEXPECTED TOP-LEVEL EXCEPTION:");
-                    ex.printStackTrace(DxConsole.err);
-                }
-                errors.incrementAndGet();
-            }
-
-            @Override
-            public void onProcessArchiveStart(File file) {
-                if (args.verbose) {
-                    DxConsole.out.println("processing archive " + file +
-                            "...");
-                }
-            }
-        });
+        opener = new ClassPathOpener(pathname, false, filter, new FileBytesConsumer());
 
-        if (args.numThreads > 1) {
-            parallelProcessorFutures.add(threadPool.submit(new ParallelProcessor(opener)));
-        } else {
-            if (opener.process()) {
-                anyFilesProcessed = true;
-            }
+        if (opener.process()) {
+          updateStatus(true);
         }
     }
 
+    private static void updateStatus(boolean res) {
+        anyFilesProcessed |= res;
+    }
+
+
     /**
      * Processes one file, which may be either a class or a resource.
      *
@@ -648,6 +687,7 @@ public class Main {
      * @return whether processing was successful
      */
     private static boolean processFileBytes(String name, long lastModified, byte[] bytes) {
+
         boolean isClass = name.endsWith(".class");
         boolean isClassesDex = name.equals(DexFormat.DEX_IN_JAR_NAME);
         boolean keepResources = (outputResources != null);
@@ -675,7 +715,10 @@ public class Main {
             if (lastModified < minimumFileAge) {
                 return true;
             }
-            return processClass(fixedName, bytes);
+            processClass(fixedName, bytes);
+            // Assume that an exception may occur. Status will be updated
+            // asynchronously, if the class compiles without error.
+            return false;
         } else if (isClassesDex) {
             synchronized (libraryDexBuffers) {
                 libraryDexBuffers.add(bytes);
@@ -702,42 +745,30 @@ public class Main {
             checkClassName(name);
         }
 
-        DirectClassFile cf =
-            new DirectClassFile(bytes, name, args.cfOptions.strictNameCheck);
+        try {
+            new DirectClassFileConsumer(name, bytes, null).call(
+                    new ClassParserTask(name, bytes).call());
+        } catch(Exception ex) {
+            throw new RuntimeException("Exception parsing classes", ex);
+        }
+
+        return true;
+    }
+
 
+    private static DirectClassFile parseClass(String name, byte[] bytes) {
+
+        DirectClassFile cf = new DirectClassFile(bytes, name,
+                args.cfOptions.strictNameCheck);
         cf.setAttributeFactory(StdAttributeFactory.THE_ONE);
-        cf.getMagic();
-
-        int numMethodIds = outputDex.getMethodIds().items().size();
-        int numFieldIds = outputDex.getFieldIds().items().size();
-        int constantPoolSize = cf.getConstantPool().size();
-
-        int maxMethodIdsInDex = numMethodIds + constantPoolSize + cf.getMethods().size() +
-                MAX_METHOD_ADDED_DURING_DEX_CREATION;
-        int maxFieldIdsInDex = numFieldIds + constantPoolSize + cf.getFields().size() +
-                MAX_FIELD_ADDED_DURING_DEX_CREATION;
-
-        if (args.multiDex
-            // Never switch to the next dex if current dex is already empty
-            && (outputDex.getClassDefs().items().size() > 0)
-            && ((maxMethodIdsInDex > args.maxNumberOfIdxPerDex) ||
-                (maxFieldIdsInDex > args.maxNumberOfIdxPerDex))) {
-            DexFile completeDex = outputDex;
-            rotateDexFile();
-            assert  (completeDex.getMethodIds().items().size() <= numMethodIds +
-                    MAX_METHOD_ADDED_DURING_DEX_CREATION) &&
-                    (completeDex.getFieldIds().items().size() <= numFieldIds +
-                    MAX_FIELD_ADDED_DURING_DEX_CREATION);
-        }
+        cf.getMagic(); // triggers the actual parsing
+        return cf;
+    }
 
+    private static ClassDefItem translateClass(byte[] bytes, DirectClassFile cf) {
         try {
-            ClassDefItem clazz =
-                CfTranslator.translate(cf, bytes, args.cfOptions, args.dexOptions, outputDex);
-            synchronized (outputDex) {
-                outputDex.add(clazz);
-            }
-            return true;
-
+            return CfTranslator.translate(cf, bytes, args.cfOptions,
+                    args.dexOptions, outputDex);
         } catch (ParseException ex) {
             DxConsole.err.println("\ntrouble processing:");
             if (args.debug) {
@@ -747,7 +778,14 @@ public class Main {
             }
         }
         errors.incrementAndGet();
-        return false;
+        return null;
+    }
+
+    private static boolean addClassToDex(ClassDefItem clazz) {
+        synchronized (outputDex) {
+            outputDex.add(clazz);
+        }
+        return true;
     }
 
     /**
@@ -797,7 +835,7 @@ public class Main {
      * @return {@code null-ok;} the converted {@code byte[]} or {@code null}
      * if there was a problem
      */
-    private static byte[] writeDex() {
+    private static byte[] writeDex(DexFile outputDex) {
         byte[] outArray = null;
 
         try {
@@ -836,7 +874,6 @@ public class Main {
             }
             return null;
         }
-
         return outArray;
     }
 
@@ -1552,12 +1589,6 @@ public class Main {
                 throw new UsageException();
             }
 
-            if (multiDex && numThreads != 1) {
-                System.out.println(NUM_THREADS_OPTION + " is ignored when used with "
-                    + MULTI_DEX_OPTION);
-                numThreads = 1;
-            }
-
             if (multiDex && incremental) {
                 System.err.println(INCREMENTAL_OPTION + " is not supported with "
                     + MULTI_DEX_OPTION);
@@ -1602,21 +1633,260 @@ public class Main {
         }
     }
 
-    /** Callable helper class to process files in multiple threads */
-    private static class ParallelProcessor implements Callable<Void> {
+    /**
+     * Callback class for processing input file bytes, produced by the
+     * ClassPathOpener.
+     */
+    private static class FileBytesConsumer implements ClassPathOpener.Consumer {
 
-        ClassPathOpener classPathOpener;
+        @Override
+        public boolean processFileBytes(String name, long lastModified,
+                byte[] bytes)   {
+            return Main.processFileBytes(name, lastModified, bytes);
+        }
 
-        private ParallelProcessor(ClassPathOpener classPathOpener) {
-            this.classPathOpener = classPathOpener;
+        @Override
+        public void onException(Exception ex) {
+            if (ex instanceof StopProcessing) {
+                throw (StopProcessing) ex;
+            } else if (ex instanceof SimException) {
+                DxConsole.err.println("\nEXCEPTION FROM SIMULATION:");
+                DxConsole.err.println(ex.getMessage() + "\n");
+                DxConsole.err.println(((SimException) ex).getContext());
+            } else {
+                DxConsole.err.println("\nUNEXPECTED TOP-LEVEL EXCEPTION:");
+                ex.printStackTrace(DxConsole.err);
+            }
+            errors.incrementAndGet();
         }
 
         @Override
-        public Void call() throws Exception {
-            if (classPathOpener.process()) {
-                anyFilesProcessed = true;
+        public void onProcessArchiveStart(File file) {
+            if (args.verbose) {
+                DxConsole.out.println("processing archive " + file + "...");
             }
-            return null;
+        }
+    }
+
+    /** Callable helper class to parse class bytes. */
+    private static class ClassParserTask implements Callable<DirectClassFile> {
+
+        String name;
+        byte[] bytes;
+
+        private ClassParserTask(String name, byte[] bytes) {
+            this.name = name;
+            this.bytes = bytes;
+        }
+
+        @Override
+        public DirectClassFile call() throws Exception {
+            DirectClassFile cf =  parseClass(name, bytes);
+
+            return cf;
+        }
+    }
+
+    /**
+     * Callable helper class used to sequentially collect the results of
+     * the (optionally parallel) translation phase, in correct input file order.
+     * This class is also responsible for coordinating dex file rotation
+     * with the ClassDefItemConsumer class.
+     * We maintain invariant that the number of indices used in the current
+     * dex file plus the max number of indices required by classes passed to
+     * the translation phase and not yet added to the dex file, is less than
+     * or equal to the dex file limit.
+     * For each parsed file, we estimate the maximum number of indices it may
+     * require. If passing the file to the translation phase would invalidate
+     * the invariant, we wait, until the next class is added to the dex file,
+     * and then reevaluate the invariant. If there are no further classes in
+     * the translation phase, we rotate the dex file.
+     */
+    private static class DirectClassFileConsumer implements Callable<Boolean> {
+
+        String name;
+        byte[] bytes;
+        Future<DirectClassFile> dcff;
+
+        private DirectClassFileConsumer(String name, byte[] bytes,
+                Future<DirectClassFile> dcff) {
+            this.name = name;
+            this.bytes = bytes;
+            this.dcff = dcff;
+        }
+
+        @Override
+        public Boolean call() throws Exception {
+
+            DirectClassFile cf = dcff.get();
+            return call(cf);
+        }
+
+        private Boolean call(DirectClassFile cf) {
+
+            int maxMethodIdsInClass = 0;
+            int maxFieldIdsInClass = 0;
+
+            if (args.multiDex) {
+
+                // Calculate max number of indices this class will add to the
+                // dex file.
+                // The constant pool contains at least one entry per method
+                // (name and signature), at least one entry per field (name
+                // and type), and at least per method/field reference (typed
+                // method ref).
+
+                int constantPoolSize = cf.getConstantPool().size();
+                maxMethodIdsInClass = constantPoolSize - cf.getFields().size()
+                        + MAX_METHOD_ADDED_DURING_DEX_CREATION;
+                maxFieldIdsInClass = constantPoolSize - cf.getMethods().size()
+                        + MAX_FIELD_ADDED_DURING_DEX_CREATION;
+                synchronized(dexRotationLock) {
+
+                    int numMethodIds;
+                    int numFieldIds;
+                    // Number of indices used in current dex file.
+                    synchronized(outputDex) {
+                        numMethodIds = outputDex.getMethodIds().items().size();
+                        numFieldIds = outputDex.getFieldIds().items().size();
+                    }
+                    // Wait until we're sure this class will fit in the current
+                    // dex file.
+                    while(((numMethodIds + maxMethodIdsInClass + maxMethodIdsInProcess
+                            > args.maxNumberOfIdxPerDex) ||
+                           (numFieldIds + maxFieldIdsInClass + maxFieldIdsInProcess
+                            > args.maxNumberOfIdxPerDex))) {
+
+                        if (maxMethodIdsInProcess > 0 || maxFieldIdsInProcess > 0) {
+                            // There are classes in the translation phase that
+                            // have not yet been added to the dex file, so we
+                            // wait for the next class to complete.
+                            try {
+                                dexRotationLock.wait();
+                            } catch(InterruptedException ex) {
+                                /* ignore */
+                            }
+                        } else if (outputDex.getClassDefs().items().size() > 0) {
+                            // There are no further classes in the translation
+                            // phase, and we have a full dex file. Rotate!
+                            rotateDexFile();
+                        } else {
+                            // The estimated number of indices is too large for
+                            // an empty dex file. We proceed hoping the actual
+                            // number of indices needed will fit.
+                            break;
+                        }
+                        synchronized(outputDex) {
+                            numMethodIds = outputDex.getMethodIds().items().size();
+                            numFieldIds = outputDex.getFieldIds().items().size();
+                        }
+                    }
+                    // Add our estimate to the total estimate for
+                    // classes under translation.
+                    maxMethodIdsInProcess += maxMethodIdsInClass;
+                    maxFieldIdsInProcess += maxFieldIdsInClass;
+                }
+            }
+
+            // Submit class to translation phase.
+            Future<ClassDefItem> cdif = classTranslatorPool.submit(
+                    new ClassTranslatorTask(name, bytes, cf));
+            Future<Boolean> res = classDefItemConsumer.submit(new ClassDefItemConsumer(
+                    name, cdif, maxMethodIdsInClass, maxFieldIdsInClass));
+            addToDexFutures.add(res);
+
+            return true;
+        }
+    }
+
+
+    /** Callable helper class to translate classes in parallel  */
+    private static class ClassTranslatorTask implements Callable<ClassDefItem> {
+
+        String name;
+        byte[] bytes;
+        DirectClassFile classFile;
+
+        private ClassTranslatorTask(String name, byte[] bytes,
+                DirectClassFile classFile) {
+            this.name = name;
+            this.bytes = bytes;
+            this.classFile = classFile;
+        }
+
+        @Override
+        public ClassDefItem call() {
+            ClassDefItem clazz = translateClass(bytes, classFile);
+            return clazz;
+        }
+    }
+
+    /**
+     * Callable helper class used to collect the results of
+     * the parallel translation phase, adding the translated classes to
+     * the current dex file in correct (deterministic) file order.
+     * This class is also responsible for coordinating dex file rotation
+     * with the DirectClassFileConsumer class.
+     */
+    private static class ClassDefItemConsumer implements Callable<Boolean> {
+
+        String name;
+        Future<ClassDefItem> futureClazz;
+        int maxMethodIdsInClass;
+        int maxFieldIdsInClass;
+
+        private ClassDefItemConsumer(String name, Future<ClassDefItem> futureClazz,
+                int maxMethodIdsInClass, int maxFieldIdsInClass) {
+            this.name = name;
+            this.futureClazz = futureClazz;
+            this.maxMethodIdsInClass = maxMethodIdsInClass;
+            this.maxFieldIdsInClass = maxFieldIdsInClass;
+        }
+
+        @Override
+        public Boolean call() throws Exception {
+            try {
+                ClassDefItem clazz = futureClazz.get();
+                if (clazz != null) {
+                    addClassToDex(clazz);
+                    updateStatus(true);
+                }
+                return true;
+            } catch(ExecutionException ex) {
+                // Rethrow previously uncaught translation exceptions.
+                // These, as well as any exceptions from addClassToDex,
+                // are handled and reported in processAllFiles().
+                Throwable t = ex.getCause();
+                throw (t instanceof Exception) ? (Exception) t : ex;
+            } finally {
+                if (args.multiDex) {
+                    // Having added our actual indicies to the dex file,
+                    // we subtract our original estimate from the total estimate,
+                    // and signal the translation phase, which may be paused
+                    // waiting to determine if more classes can be added to the
+                    // current dex file, or if a new dex file must be created.
+                    synchronized(dexRotationLock) {
+                        maxMethodIdsInProcess -= maxMethodIdsInClass;
+                        maxFieldIdsInProcess -= maxFieldIdsInClass;
+                        dexRotationLock.notifyAll();
+                    }
+                }
+            }
+        }
+    }
+
+    /** Callable helper class to convert dex files in worker threads */
+    private static class DexWriter implements Callable<byte[]> {
+
+        private DexFile dexFile;
+
+        private DexWriter(DexFile dexFile) {
+            this.dexFile = dexFile;
+        }
+
+        @Override
+        public byte[] call() throws IOException {
+            return writeDex(dexFile);
         }
     }
 }
diff --git a/dx/src/com/android/dx/dex/file/MixedItemSection.java b/dx/src/com/android/dx/dex/file/MixedItemSection.java
index 4edc6b62f..9053043f9 100644
--- a/dx/src/com/android/dx/dex/file/MixedItemSection.java
+++ b/dx/src/com/android/dx/dex/file/MixedItemSection.java
@@ -189,7 +189,7 @@ public final class MixedItemSection extends Section {
      * @param item {@code non-null;} the item to intern
      * @return {@code non-null;} the equivalent interned instance
      */
-    public <T extends OffsettedItem> T intern(T item) {
+    public synchronized <T extends OffsettedItem> T intern(T item) {
         throwIfPrepared();
 
         OffsettedItem result = interns.get(item);
diff --git a/dx/src/com/android/dx/dex/file/ProtoIdsSection.java b/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
index 4b1303b15..b7df10cdd 100644
--- a/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
@@ -86,7 +86,7 @@ public final class ProtoIdsSection extends UniformItemSection {
      * @param prototype {@code non-null;} the prototype to intern
      * @return {@code non-null;} the interned reference
      */
-    public ProtoIdItem intern(Prototype prototype) {
+    public synchronized ProtoIdItem intern(Prototype prototype) {
         if (prototype == null) {
             throw new NullPointerException("prototype == null");
         }
diff --git a/dx/src/com/android/dx/dex/file/StringIdsSection.java b/dx/src/com/android/dx/dex/file/StringIdsSection.java
index 7aff82ea9..6826c5a48 100644
--- a/dx/src/com/android/dx/dex/file/StringIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/StringIdsSection.java
@@ -117,7 +117,7 @@ public final class StringIdsSection
      * @param string {@code non-null;} the string to intern
      * @return {@code non-null;} the interned string
      */
-    public StringIdItem intern(StringIdItem string) {
+    public synchronized StringIdItem intern(StringIdItem string) {
         if (string == null) {
             throw new NullPointerException("string == null");
         }
@@ -140,7 +140,7 @@ public final class StringIdsSection
      *
      * @param nat {@code non-null;} the name-and-type
      */
-    public void intern(CstNat nat) {
+    public synchronized void intern(CstNat nat) {
         intern(nat.getName());
         intern(nat.getDescriptor());
     }
diff --git a/dx/src/com/android/dx/dex/file/TypeIdsSection.java b/dx/src/com/android/dx/dex/file/TypeIdsSection.java
index c3380f4b8..35d8e661a 100644
--- a/dx/src/com/android/dx/dex/file/TypeIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/TypeIdsSection.java
@@ -106,7 +106,7 @@ public final class TypeIdsSection extends UniformItemSection {
      * @param type {@code non-null;} the type to intern
      * @return {@code non-null;} the interned reference
      */
-    public TypeIdItem intern(Type type) {
+    public synchronized TypeIdItem intern(Type type) {
         if (type == null) {
             throw new NullPointerException("type == null");
         }
diff --git a/dx/tests/128-multidex-option-overflow/expected.txt b/dx/tests/128-multidex-option-overflow/expected.txt
index 3d7a649b8..ac448a63d 100644
--- a/dx/tests/128-multidex-option-overflow/expected.txt
+++ b/dx/tests/128-multidex-option-overflow/expected.txt
@@ -1,3 +1,2 @@
 classes2.dex
-classes3.dex
 classes.dex
diff --git a/dx/tests/129-numthread-deterministic/expected.txt b/dx/tests/129-numthread-deterministic/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/129-numthread-deterministic/info.txt b/dx/tests/129-numthread-deterministic/info.txt
new file mode 100644
index 000000000..e4885fab3
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/info.txt
@@ -0,0 +1,2 @@
+Test that dx generates deterministic output when using --num-threads
+
diff --git a/dx/tests/129-numthread-deterministic/run b/dx/tests/129-numthread-deterministic/run
new file mode 100644
index 000000000..fdc050649
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/run
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+mkdir src
+awk '
+BEGIN {
+    for (c = 1; c <= 500; c++) {
+        writeClass(c);
+    }
+}
+function writeClass(name) {
+    fileName = "src/Clazz" name ".java";
+    printf("public class Clazz%s {\n", name) > fileName;
+    for (i = 1; i <= 100; i++) {
+        printf("    int field%d;\n", i) > fileName;
+    }
+    for (i = 1; i <= 100; i++) {
+        printf("    void method%d(int param) { }\n", i) > fileName;
+    }
+    printf("}\n") > fileName;
+}'
+
+
+mkdir classes
+${JAVAC} -d classes `find src -name '*.java'`
+mkdir out
+dx -JXmx4g -JXms4g --dex --no-optimize --output=out classes
+mkdir out-multi
+dx -JXmx4g -JXms4g --dex --no-optimize --num-threads=4 --output=out-multi classes
+diff -r out out-multi > unit-out.txt
+
+if [ "$?" = "0" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi
+
diff --git a/dx/tests/130-numthread-multidex-deterministic/expected.txt b/dx/tests/130-numthread-multidex-deterministic/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/130-numthread-multidex-deterministic/info.txt b/dx/tests/130-numthread-multidex-deterministic/info.txt
new file mode 100644
index 000000000..e4885fab3
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/info.txt
@@ -0,0 +1,2 @@
+Test that dx generates deterministic output when using --num-threads
+
diff --git a/dx/tests/130-numthread-multidex-deterministic/run b/dx/tests/130-numthread-multidex-deterministic/run
new file mode 100644
index 000000000..b43ba1f54
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/run
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+mkdir src
+awk '
+BEGIN {
+    for (c = 1; c <= 1000; c++) {
+        writeClass(c);
+    }
+}
+function writeClass(name) {
+    fileName = "src/Clazz" name ".java";
+    printf("public class Clazz%s {\n", name) > fileName;
+    for (i = 1; i <= 100; i++) {
+        printf("    int field%d;\n", i) > fileName;
+    }
+    for (i = 1; i <= 100; i++) {
+        printf("    void method%d(int param) { }\n", i) > fileName;
+    }
+    printf("}\n") > fileName;
+}'
+
+
+mkdir classes
+${JAVAC} -d classes `find src -name '*.java'`
+mkdir out
+dx -JXmx4g -JXms4g --dex --no-optimize --multi-dex --output=out classes
+mkdir out-multi
+dx -JXmx4g -JXms4g --dex --no-optimize --multi-dex --num-threads=4 --output=out-multi classes
+diff -r out out-multi > unit-out.txt
+
+if [ "$?" = "0" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi
+
diff --git a/dx/tests/131-perf/ClassGen.java b/dx/tests/131-perf/ClassGen.java
new file mode 100644
index 000000000..c35bbcf82
--- /dev/null
+++ b/dx/tests/131-perf/ClassGen.java
@@ -0,0 +1,53 @@
+import java.io.File;
+import java.io.PrintWriter;
+
+public class ClassGen {
+
+    public static void main(String... args) {
+
+	int start = 1;
+        int end =   8024;
+	int fields =   4;
+        int methods =   6;
+	if (args.length > 0) {
+	    start = Integer.parseInt(args[0]);
+        }
+	if (args.length > 1) {
+	    end = Integer.parseInt(args[1]);
+        }
+	if (args.length > 2) {
+	    fields = Integer.parseInt(args[2]);
+        }
+	if (args.length > 3) {
+	    methods = Integer.parseInt(args[3]);
+        }
+
+	for (int file = start; file <= end; file++) {
+            try {
+	        File f = new File("src/Clazz" + file + ".java");
+	        PrintWriter pw = new PrintWriter(f);
+		pw.println("class Clazz" + file + " {");
+		for (int field = 1; field <= fields; field++) {
+		    pw.println("    public static int f" + field + ";");
+		}
+		for (int method = 1; method <= methods; method++) {
+		    pw.println("    boolean m" + method + "_" + (file%(end/2)) + "() {"
+);
+		    pw.println("      int max = Thread.MAX_PRIORITY;");
+		    pw.println("      for (int i = 0; i < max; i++) {");
+		    pw.println("        System.out.println(\"Hello from: \" + Clazz"
+                            + file + ".class + \".method" + method
+                            + "() \" + Clazz" + (end-file+1) + ".f1);");
+		    pw.println("        Thread.dumpStack();");
+		    pw.println("      }");
+		    pw.println("      return Thread.holdsLock(this);");
+		    pw.println("    }");
+		}
+		pw.println("}");
+                pw.close();
+            } catch(Exception ex) {
+		System.out.println("Ups");
+            }
+        }
+    }
+}
diff --git a/dx/tests/131-perf/expected.txt b/dx/tests/131-perf/expected.txt
new file mode 100644
index 000000000..54183385d
--- /dev/null
+++ b/dx/tests/131-perf/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/131-perf/info.txt b/dx/tests/131-perf/info.txt
new file mode 100644
index 000000000..f5b6a0cee
--- /dev/null
+++ b/dx/tests/131-perf/info.txt
@@ -0,0 +1,2 @@
+Script for --multi-dex --num-threads performance testing, default just test options can be used together
+
diff --git a/dx/tests/131-perf/run b/dx/tests/131-perf/run
new file mode 100644
index 000000000..e57545c03
--- /dev/null
+++ b/dx/tests/131-perf/run
@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+
+${JAVAC} ClassGen.java
+
+mkdir src
+mkdir classes
+
+# Heap size, min and max
+MEM=4g
+
+MULTIDEX="--multi-dex"
+THREADS="--num-threads=5"
+
+# Extra statistics, use to calibrate test.
+#EXTRA="--profile-concurrency"
+
+# Test smaller dex files
+#EXTRA="--set-max-idx-number=20000"
+
+# Test GC options
+#GC="-JXX:+UseConcMarkSweepGC"
+
+# Limit HW threads
+#TASKSET="taskset 0x00000fff
+
+# Number of classes, initial
+TEST_SIZE=1000
+
+# number of classes, max
+LIMIT=1000
+
+# Number of additional classes per test
+STEP=100
+
+# Number of fields per classes
+FIELDS=4
+
+# Number of methods per class
+METHODS=6
+
+
+first=1;
+while [ $TEST_SIZE -le $LIMIT ]; do
+  rm -rf out
+  mkdir out
+
+  sleep 2
+  java -classpath . ClassGen $first $TEST_SIZE $FIELDS $METHODS
+  first=`expr $TEST_SIZE + 1`
+
+  ${JAVAC} -d classes `find src -name '*.java'`
+  (cd classes; jar cf ../x.jar `find . -name '*.class'`)
+  sleep 3
+
+  start=`date +'%s%N'`
+  $TASKSET dx -JXmx$MEM -JXms$MEM $GC --dex $EXTRA --no-optimize $MULTIDEX $THREADS --output=out x.jar
+  end=`date +'%s%N'`
+  nsec=`expr $end - $start`
+  msec=`expr $nsec / 1000000`
+  echo "Classes/msec $TEST_SIZE $msec"
+
+  TEST_SIZE=`expr $TEST_SIZE + $STEP`
+done
+
+if [ "$?" = "1" ]; then
+    echo "Yay!"
+else
+    cat unit-out.txt
+fi