Tear down the code cache when it is full and restart from scratch.

Because the code cache may be wiped out after safe points now the patching of
inline cache for predicted chains is done through the compiler thread's work
queue.

11 files changed, 254 insertions, 69 deletions

diff --git a/vm/compiler/Compiler.c b/vm/compiler/Compiler.c
index 45d8317d0..014013db6 100644
--- a/vm/compiler/Compiler.c
+++ b/vm/compiler/Compiler.c
@@ -54,14 +54,16 @@ bool dvmCompilerWorkEnqueue(const u2 *pc, WorkOrderKind kind, void* info)
     int cc;
     int i;
     int numWork;
+    int oldStatus = dvmChangeStatus(NULL, THREAD_VMWAIT);
+    bool result = true;
 
     dvmLockMutex(&gDvmJit.compilerLock);
 
     /* Queue full */
     if (gDvmJit.compilerQueueLength == COMPILER_WORK_QUEUE_SIZE ||
         gDvmJit.codeCacheFull == true) {
-        dvmUnlockMutex(&gDvmJit.compilerLock);
-        return false;
+        result = false;
+        goto done;
     }
 
     for (numWork = gDvmJit.compilerQueueLength,
@@ -83,7 +85,8 @@ bool dvmCompilerWorkEnqueue(const u2 *pc, WorkOrderKind kind, void* info)
     newOrder->info = info;
     newOrder->result.codeAddress = NULL;
     newOrder->result.discardResult =
-        (kind == kWorkOrderTraceDebug) ? true : false;
+        (kind == kWorkOrderTraceDebug || kind == kWorkOrderICPatch) ?
+        true : false;
     gDvmJit.compilerWorkEnqueueIndex++;
     if (gDvmJit.compilerWorkEnqueueIndex == COMPILER_WORK_QUEUE_SIZE)
         gDvmJit.compilerWorkEnqueueIndex = 0;
@@ -93,7 +96,8 @@ bool dvmCompilerWorkEnqueue(const u2 *pc, WorkOrderKind kind, void* info)
 
 done:
     dvmUnlockMutex(&gDvmJit.compilerLock);
-    return true;
+    dvmChangeStatus(NULL, oldStatus);
+    return result;
 }
 
 /* Block until queue length is 0 */
@@ -108,6 +112,93 @@ void dvmCompilerDrainQueue(void)
     dvmChangeStatus(NULL, oldStatus);
 }
 
+bool dvmCompilerSetupCodeCache(void)
+{
+    extern void dvmCompilerTemplateStart(void);
+    extern void dmvCompilerTemplateEnd(void);
+
+    /* Allocate the code cache */
+    gDvmJit.codeCache = mmap(0, CODE_CACHE_SIZE,
+                          PROT_READ | PROT_WRITE | PROT_EXEC,
+                          MAP_PRIVATE | MAP_ANON, -1, 0);
+    if (gDvmJit.codeCache == MAP_FAILED) {
+        LOGE("Failed to create the code cache: %s\n", strerror(errno));
+        return false;
+    }
+
+    /* Copy the template code into the beginning of the code cache */
+    int templateSize = (intptr_t) dmvCompilerTemplateEnd -
+                       (intptr_t) dvmCompilerTemplateStart;
+    memcpy((void *) gDvmJit.codeCache,
+           (void *) dvmCompilerTemplateStart,
+           templateSize);
+
+    gDvmJit.templateSize = templateSize;
+    gDvmJit.codeCacheByteUsed = templateSize;
+
+    /* Only flush the part in the code cache that is being used now */
+    cacheflush((intptr_t) gDvmJit.codeCache,
+               (intptr_t) gDvmJit.codeCache + templateSize, 0);
+    return true;
+}
+
+static void resetCodeCache(void)
+{
+    Thread* self = dvmThreadSelf();
+    Thread* thread;
+
+    LOGD("Reset the JIT code cache (%d bytes used)", gDvmJit.codeCacheByteUsed);
+
+    /* Stop the world */
+    dvmSuspendAllThreads(SUSPEND_FOR_CC_RESET);
+
+    /* Wipe out the returnAddr field that soon will point to stale code */
+    for (thread = gDvm.threadList; thread != NULL; thread = thread->next) {
+        if (thread == self)
+            continue;
+
+        /* Crawl the Dalvik stack frames */
+        StackSaveArea *ssaPtr = ((StackSaveArea *) thread->curFrame) - 1;
+        while (ssaPtr != ((StackSaveArea *) NULL) - 1) {
+            ssaPtr->returnAddr = NULL;
+            ssaPtr = ((StackSaveArea *) ssaPtr->prevFrame) - 1;
+        };
+    }
+
+    /* Reset the JitEntry table contents to the initial unpopulated state */
+    dvmJitResetTable();
+
+#if 0
+    /*
+     * Uncomment the following code when testing/debugging.
+     *
+     * Wipe out the code cache content to force immediate crashes if
+     * stale JIT'ed code is invoked.
+     */
+    memset(gDvmJit.codeCache,
+           (intptr_t) gDvmJit.codeCache + gDvmJit.codeCacheByteUsed,
+           0);
+    cacheflush((intptr_t) gDvmJit.codeCache,
+               (intptr_t) gDvmJit.codeCache + gDvmJit.codeCacheByteUsed, 0);
+#endif
+
+    /* Reset the current mark of used bytes to the end of template code */
+    gDvmJit.codeCacheByteUsed = gDvmJit.templateSize;
+    gDvmJit.numCompilations = 0;
+
+    /* Reset the work queue */
+    memset(gDvmJit.compilerWorkQueue, 0,
+           sizeof(CompilerWorkOrder) * COMPILER_WORK_QUEUE_SIZE);
+    gDvmJit.compilerWorkEnqueueIndex = gDvmJit.compilerWorkDequeueIndex = 0;
+    gDvmJit.compilerQueueLength = 0;
+
+    /* All clear now */
+    gDvmJit.codeCacheFull = false;
+
+    /* Resume all threads */
+    dvmResumeAllThreads(SUSPEND_FOR_CC_RESET);
+}
+
 static void *compilerThreadStart(void *arg)
 {
     dvmChangeStatus(NULL, THREAD_VMWAIT);
@@ -156,11 +247,17 @@ static void *compilerThreadStart(void *arg)
                     if (!dvmCompilerDoWork(&work)) {
                         work.result.codeAddress = gDvmJit.interpretTemplate;
                     }
-                    dvmJitSetCodeAddr(work.pc, work.result.codeAddress,
-                                      work.result.instructionSet);
+                    if (!work.result.discardResult) {
+                        dvmJitSetCodeAddr(work.pc, work.result.codeAddress,
+                                          work.result.instructionSet);
+                    }
                 }
                 free(work.info);
                 dvmLockMutex(&gDvmJit.compilerLock);
+
+                if (gDvmJit.codeCacheFull == true) {
+                    resetCodeCache();
+                }
             } while (workQueueLength() != 0);
         }
     }
@@ -178,36 +275,6 @@ static void *compilerThreadStart(void *arg)
     return NULL;
 }
 
-bool dvmCompilerSetupCodeCache(void)
-{
-    extern void dvmCompilerTemplateStart(void);
-    extern void dmvCompilerTemplateEnd(void);
-
-    /* Allocate the code cache */
-    gDvmJit.codeCache = mmap(0, CODE_CACHE_SIZE,
-                          PROT_READ | PROT_WRITE | PROT_EXEC,
-                          MAP_PRIVATE | MAP_ANON, -1, 0);
-    if (gDvmJit.codeCache == MAP_FAILED) {
-        LOGE("Failed to create the code cache: %s\n", strerror(errno));
-        return false;
-    }
-
-    /* Copy the template code into the beginning of the code cache */
-    int templateSize = (intptr_t) dmvCompilerTemplateEnd -
-                       (intptr_t) dvmCompilerTemplateStart;
-    memcpy((void *) gDvmJit.codeCache,
-           (void *) dvmCompilerTemplateStart,
-           templateSize);
-
-    gDvmJit.templateSize = templateSize;
-    gDvmJit.codeCacheByteUsed = templateSize;
-
-    /* Only flush the part in the code cache that is being used now */
-    cacheflush((intptr_t) gDvmJit.codeCache,
-               (intptr_t) gDvmJit.codeCache + templateSize, 0);
-    return true;
-}
-
 bool dvmCompilerStartup(void)
 {
     /* Make sure the BBType enum is in sane state */
diff --git a/vm/compiler/Compiler.h b/vm/compiler/Compiler.h
index 06367a6ec..c66d5621a 100644
--- a/vm/compiler/Compiler.h
+++ b/vm/compiler/Compiler.h
@@ -38,7 +38,7 @@ typedef enum JitInstructionSetType {
 typedef struct JitTranslationInfo {
     void *codeAddress;
     JitInstructionSetType instructionSet;
-    bool discardResult;          // Used for debugging divergence
+    bool discardResult;         // Used for debugging divergence and IC patching
 } JitTranslationInfo;
 
 typedef enum WorkOrderKind {
@@ -46,6 +46,7 @@ typedef enum WorkOrderKind {
     kWorkOrderMethod = 1,       // Work is to compile a whole method
     kWorkOrderTrace = 2,        // Work is to compile code fragment(s)
     kWorkOrderTraceDebug = 3,   // Work is to compile/debug code fragment(s)
+    kWorkOrderICPatch = 4,      // Work is to patch a polymorphic callsite
 } WorkOrderKind;
 
 typedef struct CompilerWorkOrder {
diff --git a/vm/compiler/Utility.c b/vm/compiler/Utility.c
index 77a2397d3..26bb3d082 100644
--- a/vm/compiler/Utility.c
+++ b/vm/compiler/Utility.c
@@ -79,7 +79,8 @@ retry:
         currentArena->next = newArena;
         currentArena = newArena;
         numArenaBlocks++;
-        LOGD("Total arena pages for JIT: %d", numArenaBlocks);
+        if (numArenaBlocks > 10)
+            LOGD("Total arena pages for JIT: %d", numArenaBlocks);
         goto retry;
     }
     return NULL;
diff --git a/vm/compiler/codegen/CompilerCodegen.h b/vm/compiler/codegen/CompilerCodegen.h
index 3e9718c6d..ff39cd408 100644
--- a/vm/compiler/codegen/CompilerCodegen.h
+++ b/vm/compiler/codegen/CompilerCodegen.h
@@ -31,6 +31,9 @@ void dvmCompilerMIR2LIR(CompilationUnit *cUnit);
 /* Assemble LIR into machine code */
 void dvmCompilerAssembleLIR(CompilationUnit *cUnit, JitTranslationInfo *info);
 
+/* Patch inline cache content for polymorphic callsites */
+bool dvmJitPatchInlineCache(void *cellPtr, void *contentPtr);
+
 /* Implemented in the codegen/<target>/ArchUtility.c */
 void dvmCompilerCodegenDump(CompilationUnit *cUnit);
 
diff --git a/vm/compiler/codegen/arm/Assemble.c b/vm/compiler/codegen/arm/Assemble.c
index 26e227fd3..7444b3eba 100644
--- a/vm/compiler/codegen/arm/Assemble.c
+++ b/vm/compiler/codegen/arm/Assemble.c
@@ -1407,35 +1407,65 @@ const Method *dvmJitToPatchPredictedChain(const Method *method,
      */
     cell->counter = PREDICTED_CHAIN_COUNTER_AVOID;
 
-    /* Stop the world */
-    dvmSuspendAllThreads(SUSPEND_FOR_IC_PATCH);
+    PredictedChainingCell *newCell =
+        (PredictedChainingCell *) malloc(sizeof(PredictedChainingCell));
 
     int baseAddr = (int) cell + 4;   // PC is cur_addr + 4
     int branchOffset = tgtAddr - baseAddr;
 
-    COMPILER_TRACE_CHAINING(
-        LOGD("Jit Runtime: predicted chain %p from %s to %s (%s) patched",
-             cell, cell->clazz ? cell->clazz->descriptor : "NULL",
-             clazz->descriptor,
-             method->name));
+    newCell->branch = assembleChainingBranch(branchOffset, true);
+    newCell->clazz = clazz;
+    newCell->method = method;
 
-    cell->branch = assembleChainingBranch(branchOffset, true);
-    cell->clazz = clazz;
-    cell->method = method;
     /*
      * Reset the counter again in case other mutator threads got invoked
      * between the previous rest and dvmSuspendAllThreads call.
      */
-    cell->counter = PREDICTED_CHAIN_COUNTER_RECHAIN;
+    newCell->counter = PREDICTED_CHAIN_COUNTER_RECHAIN;
 
-    cacheflush((long) cell, (long) (cell+1), 0);
+    /*
+     * Enter the work order to the queue for the compiler thread to patch the
+     * chaining cell.
+     *
+     * No blocking call is added here because the patched result is not
+     * intended to be immediately consumed by the requesting thread. Its
+     * execution is simply resumed by chasing the class pointer to resolve the
+     * callsite.
+     */
+    dvmCompilerWorkEnqueue((const u2 *) cell, kWorkOrderICPatch, newCell);
+#endif
+done:
+    return method;
+}
+
+/*
+ * Patch the inline cache content based on the content passed from the work
+ * order.
+ */
+bool dvmJitPatchInlineCache(void *cellPtr, void *contentPtr)
+{
+    PredictedChainingCell *cellDest = (PredictedChainingCell *) cellPtr;
+    PredictedChainingCell *newContent = (PredictedChainingCell *) contentPtr;
+
+    /* Stop the world */
+    dvmSuspendAllThreads(SUSPEND_FOR_IC_PATCH);
+
+    COMPILER_TRACE_CHAINING(
+        LOGD("Jit Runtime: predicted chain %p from %s to %s (%s) patched",
+             cellDest, cellDest->clazz ? cellDest->clazz->descriptor : "NULL",
+             newContent->clazz->descriptor,
+             newContent->method->name));
+
+    /* Install the new cell content */
+    *cellDest = *newContent;
+
+    /* Then synchronize the I/D$ */
+    cacheflush((long) cellDest, (long) (cellDest+1), 0);
 
     /* All done - resume all other threads */
     dvmResumeAllThreads(SUSPEND_FOR_IC_PATCH);
-#endif
 
-done:
-    return method;
+    return true;
 }
 
 /*
diff --git a/vm/compiler/codegen/arm/CodegenDriver.c b/vm/compiler/codegen/arm/CodegenDriver.c
index f3648b9bd..010e8ca23 100644
--- a/vm/compiler/codegen/arm/CodegenDriver.c
+++ b/vm/compiler/codegen/arm/CodegenDriver.c
@@ -4117,6 +4117,9 @@ bool dvmCompilerDoWork(CompilerWorkOrder *work)
             gDvmJit.printMe = oldPrintMe;;
             break;
         }
+        case kWorkOrderICPatch:
+            res = dvmJitPatchInlineCache((void *) work->pc, work->info);
+            break;
         default:
             res = false;
             dvmAbort();
diff --git a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
index f768cc00e..0ecccfa9b 100644
--- a/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
+++ b/vm/compiler/template/armv5te/TEMPLATE_INVOKE_METHOD_NATIVE.S
@@ -51,6 +51,17 @@
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [rFP, #(offStackSaveArea_currentPc - sizeofStackSaveArea)]
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
+
+
 
diff --git a/vm/compiler/template/armv5te/footer.S b/vm/compiler/template/armv5te/footer.S
index c3732504e..dfbf9a876 100644
--- a/vm/compiler/template/armv5te/footer.S
+++ b/vm/compiler/template/armv5te/footer.S
@@ -32,8 +32,17 @@
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [r10, #offStackSaveArea_savedPc] @ reload rPC
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
 
 /*
  * On entry:
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S b/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
index 104e6ba50..af487fdd7 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv5te-vfp.S
@@ -441,8 +441,19 @@ dvmCompiler_TEMPLATE_INVOKE_METHOD_NATIVE:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [rFP, #(offStackSaveArea_currentPc - sizeofStackSaveArea)]
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
+
+
 
 
 /* ------------------------------ */
@@ -1406,8 +1417,17 @@ dvmCompiler_TEMPLATE_INTERPRET:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [r10, #offStackSaveArea_savedPc] @ reload rPC
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
 
 /*
  * On entry:
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S b/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
index 3040b193c..5715b9b7a 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv5te.S
@@ -441,8 +441,19 @@ dvmCompiler_TEMPLATE_INVOKE_METHOD_NATIVE:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [rFP, #(offStackSaveArea_currentPc - sizeofStackSaveArea)]
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
+
+
 
 
 /* ------------------------------ */
@@ -1131,8 +1142,17 @@ dvmCompiler_TEMPLATE_INTERPRET:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [r10, #offStackSaveArea_savedPc] @ reload rPC
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
 
 /*
  * On entry:
diff --git a/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S b/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
index 926e1d757..e3edf7d3c 100644
--- a/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
+++ b/vm/compiler/template/out/CompilerTemplateAsm-armv7-a.S
@@ -441,8 +441,19 @@ dvmCompiler_TEMPLATE_INVOKE_METHOD_NATIVE:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [rFP, #(offStackSaveArea_currentPc - sizeofStackSaveArea)]
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
+
+
 
 
 /* ------------------------------ */
@@ -1406,8 +1417,17 @@ dvmCompiler_TEMPLATE_INTERPRET:
     cmp     r1, #0                      @ null?
     str     r0, [r9, #offThread_jniLocal_topCookie] @ new top <- old top
     ldr     r0, [r10, #offStackSaveArea_savedPc] @ reload rPC
-    bne     .LhandleException             @ no, handle exception
-    bx      r2
+
+    @ r0 = dalvikCallsitePC
+    bne     .LhandleException           @ no, handle exception
+
+    cmp     r2, #0                      @ return chaining cell still exists?
+    bxne    r2                          @ yes - go ahead
+
+    @ continue executing the next instruction through the interpreter
+    ldr     r1, .LdvmJitToInterpNoChain @ defined in footer.S
+    add     rPC, r0, #6                 @ reconstruct new rPC (advance 6 bytes)
+    mov     pc, r1
 
 /*
  * On entry: