Reserve bits in the lock word for read barriers.

This prepares for the CC collector to use the standard object header
model by storing the read barrier state in the lock word.

Bug: 19355854
Bug: 12687968
Change-Id: Ia7585662dd2cebf0479a3e74f734afe5059fb70f

17 files changed, 432 insertions, 164 deletions

diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S
index fec1ce59bf..aff3880fa4 100644
--- a/runtime/arch/arm/quick_entrypoints_arm.S
+++ b/runtime/arch/arm/quick_entrypoints_arm.S
@@ -469,26 +469,33 @@ ENTRY art_quick_lock_object
 .Lretry_lock:
     ldr    r2, [r9, #THREAD_ID_OFFSET]
     ldrex  r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
-    cbnz   r1, .Lnot_unlocked         @ already thin locked
-    @ unlocked case - r2 holds thread id with count of 0
+    mov    r3, r1
+    and    r3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  @ zero the read barrier bits
+    cbnz   r3, .Lnot_unlocked         @ already thin locked
+    @ unlocked case - r1: original lock word that's zero except for the read barrier bits.
+    orr    r2, r1, r2                 @ r2 holds thread id with count of 0 with preserved read barrier bits
     strex  r3, r2, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
-    cbnz   r3, .Lstrex_fail           @ store failed, retry
+    cbnz   r3, .Llock_strex_fail      @ store failed, retry
     dmb    ish                        @ full (LoadLoad|LoadStore) memory barrier
     bx lr
-.Lstrex_fail:
-    b .Lretry_lock                    @ unlikely forward branch, need to reload and recheck r1/r2
-.Lnot_unlocked:
-    lsr    r3, r1, 30
+.Lnot_unlocked:  @ r1: original lock word, r2: thread_id with count of 0 and zero read barrier bits
+    lsr    r3, r1, LOCK_WORD_STATE_SHIFT
     cbnz   r3, .Lslow_lock            @ if either of the top two bits are set, go slow path
     eor    r2, r1, r2                 @ lock_word.ThreadId() ^ self->ThreadId()
     uxth   r2, r2                     @ zero top 16 bits
     cbnz   r2, .Lslow_lock            @ lock word and self thread id's match -> recursive lock
                                       @ else contention, go to slow path
-    add    r2, r1, #65536             @ increment count in lock word placing in r2 for storing
-    lsr    r1, r2, 30                 @ if either of the top two bits are set, we overflowed.
-    cbnz   r1, .Lslow_lock            @ if we overflow the count go slow path
-    str    r2, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ no need for strex as we hold the lock
+    mov    r3, r1                     @ copy the lock word to check count overflow.
+    and    r3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  @ zero the read barrier bits.
+    add    r2, r3, #LOCK_WORD_THIN_LOCK_COUNT_ONE  @ increment count in lock word placing in r2 to check overflow
+    lsr    r3, r2, LOCK_WORD_READ_BARRIER_STATE_SHIFT  @ if either of the upper two bits (28-29) are set, we overflowed.
+    cbnz   r3, .Lslow_lock            @ if we overflow the count go slow path
+    add    r2, r1, #LOCK_WORD_THIN_LOCK_COUNT_ONE  @ increment count for real
+    strex  r3, r2, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ strex necessary for read barrier bits
+    cbnz   r3, .Llock_strex_fail      @ strex failed, retry
     bx lr
+.Llock_strex_fail:
+    b      .Lretry_lock               @ retry
 .Lslow_lock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME r1, r2  @ save callee saves in case we block
     mov    r1, r9                     @ pass Thread::Current
@@ -505,23 +512,46 @@ END art_quick_lock_object
     .extern artUnlockObjectFromCode
 ENTRY art_quick_unlock_object
     cbz    r0, .Lslow_unlock
+.Lretry_unlock:
+#ifndef USE_READ_BARRIER
     ldr    r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
-    lsr    r2, r1, 30
+#else
+    ldrex  r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]  @ Need to use atomic instructions for read barrier
+#endif
+    lsr    r2, r1, #LOCK_WORD_STATE_SHIFT
     cbnz   r2, .Lslow_unlock          @ if either of the top two bits are set, go slow path
     ldr    r2, [r9, #THREAD_ID_OFFSET]
-    eor    r3, r1, r2                 @ lock_word.ThreadId() ^ self->ThreadId()
+    mov    r3, r1                     @ copy lock word to check thread id equality
+    and    r3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  @ zero the read barrier bits
+    eor    r3, r3, r2                 @ lock_word.ThreadId() ^ self->ThreadId()
     uxth   r3, r3                     @ zero top 16 bits
     cbnz   r3, .Lslow_unlock          @ do lock word and self thread id's match?
-    cmp    r1, #65536
+    mov    r3, r1                     @ copy lock word to detect transition to unlocked
+    and    r3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  @ zero the read barrier bits
+    cmp    r3, #LOCK_WORD_THIN_LOCK_COUNT_ONE
     bpl    .Lrecursive_thin_unlock
-    @ transition to unlocked, r3 holds 0
+    @ transition to unlocked
+    mov    r3, r1
+    and    r3, #LOCK_WORD_READ_BARRIER_STATE_MASK  @ r3: zero except for the preserved read barrier bits
     dmb    ish                        @ full (LoadStore|StoreStore) memory barrier
+#ifndef USE_READ_BARRIER
     str    r3, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
+#else
+    strex  r2, r3, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]  @ strex necessary for read barrier bits
+    cbnz   r2, .Lunlock_strex_fail    @ store failed, retry
+#endif
     bx     lr
-.Lrecursive_thin_unlock:
-    sub    r1, r1, #65536
+.Lrecursive_thin_unlock:  @ r1: original lock word
+    sub    r1, r1, #LOCK_WORD_THIN_LOCK_COUNT_ONE  @ decrement count
+#ifndef USE_READ_BARRIER
     str    r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
+#else
+    strex  r2, r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]  @ strex necessary for read barrier bits
+    cbnz   r2, .Lunlock_strex_fail    @ store failed, retry
+#endif
     bx     lr
+.Lunlock_strex_fail:
+    b      .Lretry_unlock             @ retry
 .Lslow_unlock:
     @ save callee saves in case exception allocation triggers GC
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME r1, r2
diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S
index 770073b5ba..382a4c2450 100644
--- a/runtime/arch/arm64/quick_entrypoints_arm64.S
+++ b/runtime/arch/arm64/quick_entrypoints_arm64.S
@@ -1000,25 +1000,33 @@ ENTRY art_quick_lock_object
 .Lretry_lock:
     ldr    w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop?
     ldxr   w1, [x4]
-    cbnz   w1, .Lnot_unlocked         // already thin locked
+    mov    x3, x1
+    and    w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  // zero the read barrier bits
+    cbnz   w3, .Lnot_unlocked         // already thin locked
+    // unlocked case - x1: original lock word that's zero except for the read barrier bits.
+    orr    x2, x1, x2                 // x2 holds thread id with count of 0 with preserved read barrier bits
     stxr   w3, w2, [x4]
-    cbnz   w3, .Lstrex_fail           // store failed, retry
+    cbnz   w3, .Llock_stxr_fail       // store failed, retry
     dmb    ishld                      // full (LoadLoad|LoadStore) memory barrier
     ret
-.Lstrex_fail:
-    b .Lretry_lock                    // unlikely forward branch, need to reload and recheck r1/r2
-.Lnot_unlocked:
-    lsr    w3, w1, 30
+.Lnot_unlocked:  // x1: original lock word
+    lsr    w3, w1, LOCK_WORD_STATE_SHIFT
     cbnz   w3, .Lslow_lock            // if either of the top two bits are set, go slow path
     eor    w2, w1, w2                 // lock_word.ThreadId() ^ self->ThreadId()
     uxth   w2, w2                     // zero top 16 bits
     cbnz   w2, .Lslow_lock            // lock word and self thread id's match -> recursive lock
                                       // else contention, go to slow path
-    add    w2, w1, #65536             // increment count in lock word placing in w2 for storing
-    lsr    w1, w2, 30                 // if either of the top two bits are set, we overflowed.
-    cbnz   w1, .Lslow_lock            // if we overflow the count go slow path
-    str    w2, [x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]  // no need for stxr as we hold the lock
+    mov    x3, x1                     // copy the lock word to check count overflow.
+    and    w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  // zero the read barrier bits.
+    add    w2, w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE  // increment count in lock word placing in w2 to check overflow
+    lsr    w3, w2, LOCK_WORD_READ_BARRIER_STATE_SHIFT  // if either of the upper two bits (28-29) are set, we overflowed.
+    cbnz   w3, .Lslow_lock            // if we overflow the count go slow path
+    add    w2, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE  // increment count for real
+    stxr   w3, w2, [x4]
+    cbnz   w3, .Llock_stxr_fail       // store failed, retry
     ret
+.Llock_stxr_fail:
+    b      .Lretry_lock               // retry
 .Lslow_lock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME  // save callee saves in case we block
     mov    x1, xSELF                  // pass Thread::Current
@@ -1036,23 +1044,47 @@ END art_quick_lock_object
     .extern artUnlockObjectFromCode
 ENTRY art_quick_unlock_object
     cbz    x0, .Lslow_unlock
-    ldr    w1, [x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
-    lsr    w2, w1, 30
+    add    x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET  // exclusive load/store has no immediate anymore
+.Lretry_unlock:
+#ifndef USE_READ_BARRIER
+    ldr    w1, [x4]
+#else
+    ldxr   w1, [x4]                   // Need to use atomic instructions for read barrier
+#endif
+    lsr    w2, w1, LOCK_WORD_STATE_SHIFT
     cbnz   w2, .Lslow_unlock          // if either of the top two bits are set, go slow path
     ldr    w2, [xSELF, #THREAD_ID_OFFSET]
-    eor    w3, w1, w2                 // lock_word.ThreadId() ^ self->ThreadId()
+    mov    x3, x1                     // copy lock word to check thread id equality
+    and    w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  // zero the read barrier bits
+    eor    w3, w3, w2                 // lock_word.ThreadId() ^ self->ThreadId()
     uxth   w3, w3                     // zero top 16 bits
     cbnz   w3, .Lslow_unlock          // do lock word and self thread id's match?
-    cmp    w1, #65536
+    mov    x3, x1                     // copy lock word to detect transition to unlocked
+    and    w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED  // zero the read barrier bits
+    cmp    w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE
     bpl    .Lrecursive_thin_unlock
-    // transition to unlocked, w3 holds 0
+    // transition to unlocked
+    mov    x3, x1
+    and    w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK  // w3: zero except for the preserved read barrier bits
     dmb    ish                        // full (LoadStore|StoreStore) memory barrier
-    str    w3, [x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
+#ifndef USE_READ_BARRIER
+    str    w3, [x4]
+#else
+    stxr   w2, w3, [x4]               // Need to use atomic instructions for read barrier
+    cbnz   w2, .Lunlock_stxr_fail     // store failed, retry
+#endif
     ret
-.Lrecursive_thin_unlock:
-    sub    w1, w1, #65536
-    str    w1, [x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET]
+.Lrecursive_thin_unlock:  // w1: original lock word
+    sub    w1, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE  // decrement count
+#ifndef USE_READ_BARRIER
+    str    w1, [x4]
+#else
+    stxr   w2, w1, [x4]               // Need to use atomic instructions for read barrier
+    cbnz   w2, .Lunlock_stxr_fail     // store failed, retry
+#endif
     ret
+.Lunlock_stxr_fail:
+    b      .Lretry_unlock               // retry
 .Lslow_unlock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME  // save callee saves in case exception allocation triggers GC
     mov    x1, xSELF                  // pass Thread::Current
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index c2acdd1668..c4374284b0 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -922,29 +922,39 @@ DEFINE_FUNCTION art_quick_lock_object
     jz   .Lslow_lock
 .Lretry_lock:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%eax), %ecx  // ecx := lock word
-    test LITERAL(0xC0000000), %ecx        // test the 2 high bits.
+    test LITERAL(LOCK_WORD_STATE_MASK), %ecx         // test the 2 high bits.
     jne  .Lslow_lock                      // slow path if either of the two high bits are set.
-    movl %fs:THREAD_ID_OFFSET, %edx       // edx := thread id
+    movl %ecx, %edx                       // save lock word (edx) to keep read barrier bits.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %ecx  // zero the read barrier bits.
     test %ecx, %ecx
     jnz  .Lalready_thin                   // lock word contains a thin lock
-    // unlocked case - %edx holds thread id with count of 0
+    // unlocked case - edx: original lock word, eax: obj.
     movl %eax, %ecx                       // remember object in case of retry
-    xor  %eax, %eax                       // eax == 0 for comparison with lock word in cmpxchg
-    lock cmpxchg  %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%ecx)
-    jnz  .Lcmpxchg_fail                   // cmpxchg failed retry
+    movl %edx, %eax                       // eax: lock word zero except for read barrier bits.
+    movl %fs:THREAD_ID_OFFSET, %edx       // load thread id.
+    or   %eax, %edx                       // edx: thread id with count of 0 + read barrier bits.
+    lock cmpxchg  %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%ecx)  // eax: old val, edx: new val.
+    jnz  .Llock_cmpxchg_fail              // cmpxchg failed retry
     ret
-.Lcmpxchg_fail:
-    movl  %ecx, %eax                      // restore eax
-    jmp  .Lretry_lock
-.Lalready_thin:
+.Lalready_thin:  // edx: lock word (with high 2 bits zero and original rb bits), eax: obj.
+    movl %fs:THREAD_ID_OFFSET, %ecx       // ecx := thread id
     cmpw %cx, %dx                         // do we hold the lock already?
     jne  .Lslow_lock
-    addl LITERAL(65536), %ecx             // increment recursion count
-    test LITERAL(0xC0000000), %ecx        // overflowed if either of top two bits are set
+    movl %edx, %ecx                       // copy the lock word to check count overflow.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %ecx  // zero the read barrier bits.
+    addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx  // increment recursion count for overflow check.
+    test LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK), %ecx  // overflowed if either of the upper two bits (28-29) are set.
     jne  .Lslow_lock                      // count overflowed so go slow
-    // update lockword, cmpxchg not necessary as we hold lock
-    movl %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%eax)
+    movl %eax, %ecx                       // save obj to use eax for cmpxchg.
+    movl %edx, %eax                       // copy the lock word as the old val for cmpxchg.
+    addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx  // increment recursion count again for real.
+    // update lockword, cmpxchg necessary for read barrier bits.
+    lock cmpxchg  %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%ecx)  // eax: old val, edx: new val.
+    jnz  .Llock_cmpxchg_fail              // cmpxchg failed retry
     ret
+.Llock_cmpxchg_fail:
+    movl  %ecx, %eax                      // restore eax
+    jmp  .Lretry_lock
 .Lslow_lock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME  ebx, ebx  // save ref containing registers for GC
     // Outgoing argument set up
@@ -963,20 +973,43 @@ END_FUNCTION art_quick_lock_object
 DEFINE_FUNCTION art_quick_unlock_object
     testl %eax, %eax                      // null check object/eax
     jz   .Lslow_unlock
+.Lretry_unlock:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%eax), %ecx  // ecx := lock word
     movl %fs:THREAD_ID_OFFSET, %edx       // edx := thread id
-    test LITERAL(0xC0000000), %ecx
+    test LITERAL(LOCK_WORD_STATE_MASK), %ecx
     jnz  .Lslow_unlock                    // lock word contains a monitor
     cmpw %cx, %dx                         // does the thread id match?
     jne  .Lslow_unlock
-    cmpl LITERAL(65536), %ecx
+    movl %ecx, %edx                       // copy the lock word to detect new count of 0.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %edx  // zero the read barrier bits.
+    cmpl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx
     jae  .Lrecursive_thin_unlock
-    movl LITERAL(0), MIRROR_OBJECT_LOCK_WORD_OFFSET(%eax)
+    // update lockword, cmpxchg necessary for read barrier bits.
+    movl %eax, %edx                       // edx: obj
+    movl %ecx, %eax                       // eax: old lock word.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK), %ecx  // ecx: new lock word zero except original rb bits.
+#ifndef USE_READ_BARRIER
+    movl %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edx)
+#else
+    lock cmpxchg  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edx)  // eax: old val, ecx: new val.
+    jnz  .Lunlock_cmpxchg_fail            // cmpxchg failed retry
+#endif
     ret
-.Lrecursive_thin_unlock:
-    subl LITERAL(65536), %ecx
-    mov  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%eax)
+.Lrecursive_thin_unlock:  // ecx: original lock word, eax: obj
+    // update lockword, cmpxchg necessary for read barrier bits.
+    movl %eax, %edx                       // edx: obj
+    movl %ecx, %eax                       // eax: old lock word.
+    subl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx  // ecx: new lock word with decremented count.
+#ifndef USE_READ_BARRIER
+    mov  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edx)
+#else
+    lock cmpxchg  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edx)  // eax: old val, ecx: new val.
+    jnz  .Lunlock_cmpxchg_fail            // cmpxchg failed retry
+#endif
     ret
+.Lunlock_cmpxchg_fail:  // edx: obj
+    movl %edx, %eax                       // restore eax
+    jmp  .Lretry_unlock
 .Lslow_unlock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME  ebx, ebx  // save ref containing registers for GC
     // Outgoing argument set up
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index c8655414c5..9b6b367e1c 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -955,24 +955,33 @@ DEFINE_FUNCTION art_quick_lock_object
     jz   .Lslow_lock
 .Lretry_lock:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi), %ecx  // ecx := lock word.
-    test LITERAL(0xC0000000), %ecx        // Test the 2 high bits.
+    test LITERAL(LOCK_WORD_STATE_MASK), %ecx         // Test the 2 high bits.
     jne  .Lslow_lock                      // Slow path if either of the two high bits are set.
-    movl %gs:THREAD_ID_OFFSET, %edx       // edx := thread id
+    movl %ecx, %edx                       // save lock word (edx) to keep read barrier bits.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %ecx  // zero the read barrier bits.
     test %ecx, %ecx
     jnz  .Lalready_thin                   // Lock word contains a thin lock.
-    // unlocked case - %edx holds thread id with count of 0
-    xor  %eax, %eax                       // eax == 0 for comparison with lock word in cmpxchg
+    // unlocked case - edx: original lock word, edi: obj.
+    movl %edx, %eax                       // eax: lock word zero except for read barrier bits.
+    movl %gs:THREAD_ID_OFFSET, %edx       // edx := thread id
+    or   %eax, %edx                       // edx: thread id with count of 0 + read barrier bits.
     lock cmpxchg  %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)
     jnz  .Lretry_lock                     // cmpxchg failed retry
     ret
-.Lalready_thin:
+.Lalready_thin:  // edx: lock word (with high 2 bits zero and original rb bits), edi: obj.
+    movl %gs:THREAD_ID_OFFSET, %ecx       // ecx := thread id
     cmpw %cx, %dx                         // do we hold the lock already?
     jne  .Lslow_lock
-    addl LITERAL(65536), %ecx             // increment recursion count
-    test LITERAL(0xC0000000), %ecx        // overflowed if either of top two bits are set
+    movl %edx, %ecx                       // copy the lock word to check count overflow.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %ecx  // zero the read barrier bits.
+    addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx  // increment recursion count
+    test LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK), %ecx  // overflowed if either of the upper two bits (28-29) are set
     jne  .Lslow_lock                      // count overflowed so go slow
-    // update lockword, cmpxchg not necessary as we hold lock
-    movl %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)
+    movl %edx, %eax                       // copy the lock word as the old val for cmpxchg.
+    addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx   // increment recursion count again for real.
+    // update lockword, cmpxchg necessary for read barrier bits.
+    lock cmpxchg  %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)  // eax: old val, edx: new val.
+    jnz  .Lretry_lock                     // cmpxchg failed retry
     ret
 .Lslow_lock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME
@@ -985,19 +994,37 @@ END_FUNCTION art_quick_lock_object
 DEFINE_FUNCTION art_quick_unlock_object
     testl %edi, %edi                      // null check object/edi
     jz   .Lslow_unlock
+.Lretry_unlock:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi), %ecx  // ecx := lock word
     movl %gs:THREAD_ID_OFFSET, %edx       // edx := thread id
-    test LITERAL(0xC0000000), %ecx
+    test LITERAL(LOCK_WORD_STATE_MASK), %ecx
     jnz  .Lslow_unlock                    // lock word contains a monitor
     cmpw %cx, %dx                         // does the thread id match?
     jne  .Lslow_unlock
-    cmpl LITERAL(65536), %ecx
+    movl %ecx, %edx                       // copy the lock word to detect new count of 0.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), %edx  // zero the read barrier bits.
+    cmpl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx
     jae  .Lrecursive_thin_unlock
-    movl LITERAL(0), MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)
+    // update lockword, cmpxchg necessary for read barrier bits.
+    movl %ecx, %eax                       // eax: old lock word.
+    andl LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK), %ecx  // ecx: new lock word zero except original rb bits.
+#ifndef USE_READ_BARRIER
+    movl %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)
+#else
+    lock cmpxchg  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)  // eax: old val, ecx: new val.
+    jnz  .Lretry_unlock                   // cmpxchg failed retry
+#endif
     ret
-.Lrecursive_thin_unlock:
-    subl LITERAL(65536), %ecx
+.Lrecursive_thin_unlock:  // ecx: original lock word, edi: obj
+    // update lockword, cmpxchg necessary for read barrier bits.
+    movl %ecx, %eax                       // eax: old lock word.
+    subl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx
+#ifndef USE_READ_BARRIER
     mov  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)
+#else
+    lock cmpxchg  %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi)  // eax: old val, ecx: new val.
+    jnz  .Lretry_unlock                   // cmpxchg failed retry
+#endif
     ret
 .Lslow_unlock:
     SETUP_REFS_ONLY_CALLEE_SAVE_FRAME
diff --git a/runtime/asm_support.h b/runtime/asm_support.h
index a35e05b87b..ee70fe7c81 100644
--- a/runtime/asm_support.h
+++ b/runtime/asm_support.h
@@ -18,6 +18,7 @@
 #define ART_RUNTIME_ASM_SUPPORT_H_
 
 #if defined(__cplusplus)
+#include "lock_word.h"
 #include "mirror/art_method.h"
 #include "mirror/class.h"
 #include "mirror/string.h"
@@ -156,6 +157,27 @@ ADD_TEST_EQ(MIRROR_ART_METHOD_QUICK_CODE_OFFSET_32,
 ADD_TEST_EQ(MIRROR_ART_METHOD_QUICK_CODE_OFFSET_64,
             art::mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(8).Int32Value())
 
+#define LOCK_WORD_STATE_SHIFT 30
+ADD_TEST_EQ(LOCK_WORD_STATE_SHIFT, static_cast<int32_t>(art::LockWord::kStateShift))
+
+#define LOCK_WORD_STATE_MASK 0xC0000000
+ADD_TEST_EQ(LOCK_WORD_STATE_MASK, static_cast<uint32_t>(art::LockWord::kStateMaskShifted))
+
+#define LOCK_WORD_READ_BARRIER_STATE_SHIFT 28
+ADD_TEST_EQ(LOCK_WORD_READ_BARRIER_STATE_SHIFT,
+            static_cast<int32_t>(art::LockWord::kReadBarrierStateShift))
+
+#define LOCK_WORD_READ_BARRIER_STATE_MASK 0x30000000
+ADD_TEST_EQ(LOCK_WORD_READ_BARRIER_STATE_MASK,
+            static_cast<int32_t>(art::LockWord::kReadBarrierStateMaskShifted))
+
+#define LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED 0xCFFFFFFF
+ADD_TEST_EQ(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED,
+            static_cast<uint32_t>(art::LockWord::kReadBarrierStateMaskShiftedToggled))
+
+#define LOCK_WORD_THIN_LOCK_COUNT_ONE 65536
+ADD_TEST_EQ(LOCK_WORD_THIN_LOCK_COUNT_ONE, static_cast<int32_t>(art::LockWord::kThinLockCountOne))
+
 #if defined(__cplusplus)
 }  // End of CheckAsmSupportOffsets.
 #endif
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 734c9351d5..057eed187f 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -484,14 +484,6 @@ inline mirror::Object* ConcurrentCopying::GetFwdPtr(mirror::Object* from_ref) {
   }
 }
 
-inline void ConcurrentCopying::SetFwdPtr(mirror::Object* from_ref, mirror::Object* to_ref) {
-  DCHECK(region_space_->IsInFromSpace(from_ref));
-  DCHECK(region_space_->IsInToSpace(to_ref) || heap_->GetNonMovingSpace()->HasAddress(to_ref));
-  LockWord lw = from_ref->GetLockWord(false);
-  DCHECK_NE(lw.GetState(), LockWord::kForwardingAddress);
-  from_ref->SetLockWord(LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)), false);
-}
-
 // The following visitors are that used to verify that there's no
 // references to the from-space left after marking.
 class ConcurrentCopyingVerifyNoFromSpaceRefsVisitor {
diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h
index d0e044639f..bbb551af23 100644
--- a/runtime/gc/collector/concurrent_copying.h
+++ b/runtime/gc/collector/concurrent_copying.h
@@ -230,8 +230,6 @@ class ConcurrentCopying : public GarbageCollector {
   bool IsOnAllocStack(mirror::Object* ref) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
   mirror::Object* GetFwdPtr(mirror::Object* from_ref)
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
-  void SetFwdPtr(mirror::Object* from_ref, mirror::Object* to_ref)
-      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
   void FlipThreadRoots() LOCKS_EXCLUDED(Locks::mutator_lock_);;
   void SwapStacks(Thread* self) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
   void RecordLiveStackFreezeSize(Thread* self);
diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc
index 234bce55b6..d1ce0bc68a 100644
--- a/runtime/gc/collector/mark_compact.cc
+++ b/runtime/gc/collector/mark_compact.cc
@@ -91,7 +91,7 @@ void MarkCompact::ForwardObject(mirror::Object* obj) {
   const size_t alloc_size = RoundUp(obj->SizeOf(), space::BumpPointerSpace::kAlignment);
   LockWord lock_word = obj->GetLockWord(false);
   // If we have a non empty lock word, store it and restore it later.
-  if (lock_word.GetValue() != LockWord().GetValue()) {
+  if (!LockWord::IsDefault(lock_word)) {
     // Set the bit in the bitmap so that we know to restore it later.
     objects_with_lockword_->Set(obj);
     lock_words_to_restore_.push_back(lock_word);
@@ -509,7 +509,7 @@ void MarkCompact::MoveObject(mirror::Object* obj, size_t len) {
   // Use memmove since there may be overlap.
   memmove(reinterpret_cast<void*>(dest_addr), reinterpret_cast<const void*>(obj), len);
   // Restore the saved lock word if needed.
-  LockWord lock_word;
+  LockWord lock_word = LockWord::Default();
   if (UNLIKELY(objects_with_lockword_->Test(obj))) {
     lock_word = lock_words_to_restore_.front();
     lock_words_to_restore_.pop_front();
diff --git a/runtime/lock_word-inl.h b/runtime/lock_word-inl.h
index c52578f8a2..d831bfbee2 100644
--- a/runtime/lock_word-inl.h
+++ b/runtime/lock_word-inl.h
@@ -24,17 +24,20 @@ namespace art {
 
 inline uint32_t LockWord::ThinLockOwner() const {
   DCHECK_EQ(GetState(), kThinLocked);
+  CheckReadBarrierState();
   return (value_ >> kThinLockOwnerShift) & kThinLockOwnerMask;
 }
 
 inline uint32_t LockWord::ThinLockCount() const {
   DCHECK_EQ(GetState(), kThinLocked);
+  CheckReadBarrierState();
   return (value_ >> kThinLockCountShift) & kThinLockCountMask;
 }
 
 inline Monitor* LockWord::FatLockMonitor() const {
   DCHECK_EQ(GetState(), kFatLocked);
-  MonitorId mon_id = value_ & ~(kStateMask << kStateShift);
+  CheckReadBarrierState();
+  MonitorId mon_id = (value_ >> kMonitorIdShift) & kMonitorIdMask;
   return MonitorPool::MonitorFromMonitorId(mon_id);
 }
 
@@ -47,14 +50,20 @@ inline LockWord::LockWord() : value_(0) {
   DCHECK_EQ(GetState(), kUnlocked);
 }
 
-inline LockWord::LockWord(Monitor* mon)
-    : value_(mon->GetMonitorId() | (kStateFat << kStateShift)) {
+inline LockWord::LockWord(Monitor* mon, uint32_t rb_state)
+    : value_(mon->GetMonitorId() | (rb_state << kReadBarrierStateShift) |
+             (kStateFat << kStateShift)) {
+#ifndef __LP64__
+  DCHECK_ALIGNED(mon, kMonitorIdAlignment);
+#endif
   DCHECK_EQ(FatLockMonitor(), mon);
   DCHECK_LE(mon->GetMonitorId(), static_cast<uint32_t>(kMaxMonitorId));
+  CheckReadBarrierState();
 }
 
 inline int32_t LockWord::GetHashCode() const {
   DCHECK_EQ(GetState(), kHashCode);
+  CheckReadBarrierState();
   return (value_ >> kHashShift) & kHashMask;
 }
 
diff --git a/runtime/lock_word.h b/runtime/lock_word.h
index 2d5c71bb93..46c3bd4a99 100644
--- a/runtime/lock_word.h
+++ b/runtime/lock_word.h
@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 #include "base/logging.h"
+#include "read_barrier.h"
 #include "utils.h"
 
 namespace art {
@@ -31,34 +32,43 @@ namespace mirror {
 class Monitor;
 
 /* The lock value itself as stored in mirror::Object::monitor_.  The two most significant bits of
- * the state. The three possible states are fat locked, thin/unlocked, and hash code.
- * When the lock word is in the "thin" state and its bits are formatted as follows:
+ * the state. The four possible states are fat locked, thin/unlocked, hash code, and forwarding
+ * address. When the lock word is in the "thin" state and its bits are formatted as follows:
  *
- *  |33|22222222221111|1111110000000000|
- *  |10|98765432109876|5432109876543210|
- *  |00| lock count   |thread id owner |
+ *  |33|22|222222221111|1111110000000000|
+ *  |10|98|765432109876|5432109876543210|
+ *  |00|rb| lock count |thread id owner |
  *
  * When the lock word is in the "fat" state and its bits are formatted as follows:
  *
- *  |33|222222222211111111110000000000|
- *  |10|987654321098765432109876543210|
- *  |01| MonitorId                    |
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |01|rb| MonitorId                  |
  *
  * When the lock word is in hash state and its bits are formatted as follows:
  *
- *  |33|222222222211111111110000000000|
- *  |10|987654321098765432109876543210|
- *  |10| HashCode                     |
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |10|rb| HashCode                   |
+ *
+ * When the lock word is in fowarding address state and its bits are formatted as follows:
+ *
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |11| ForwardingAddress             |
+ *
+ * The rb bits store the read barrier state.
  */
 class LockWord {
  public:
   enum SizeShiftsAndMasks {  // private marker to avoid generate-operator-out.py from processing.
     // Number of bits to encode the state, currently just fat or thin/unlocked or hash code.
     kStateSize = 2,
+    kReadBarrierStateSize = 2,
     // Number of bits to encode the thin lock owner.
     kThinLockOwnerSize = 16,
     // Remaining bits are the recursive lock count.
-    kThinLockCountSize = 32 - kThinLockOwnerSize - kStateSize,
+    kThinLockCountSize = 32 - kThinLockOwnerSize - kStateSize - kReadBarrierStateSize,
     // Thin lock bits. Owner in lowest bits.
 
     kThinLockOwnerShift = 0,
@@ -68,28 +78,41 @@ class LockWord {
     kThinLockCountShift = kThinLockOwnerSize + kThinLockOwnerShift,
     kThinLockCountMask = (1 << kThinLockCountSize) - 1,
     kThinLockMaxCount = kThinLockCountMask,
+    kThinLockCountOne = 1 << kThinLockCountShift,  // == 65536 (0x10000)
 
     // State in the highest bits.
-    kStateShift = kThinLockCountSize + kThinLockCountShift,
+    kStateShift = kReadBarrierStateSize + kThinLockCountSize + kThinLockCountShift,
     kStateMask = (1 << kStateSize) - 1,
+    kStateMaskShifted = kStateMask << kStateShift,
     kStateThinOrUnlocked = 0,
     kStateFat = 1,
     kStateHash = 2,
     kStateForwardingAddress = 3,
+    kReadBarrierStateShift = kThinLockCountSize + kThinLockCountShift,
+    kReadBarrierStateMask = (1 << kReadBarrierStateSize) - 1,
+    kReadBarrierStateMaskShifted = kReadBarrierStateMask << kReadBarrierStateShift,
+    kReadBarrierStateMaskShiftedToggled = ~kReadBarrierStateMaskShifted,
 
     // When the state is kHashCode, the non-state bits hold the hashcode.
     kHashShift = 0,
-    kHashSize = 32 - kStateSize,
+    kHashSize = 32 - kStateSize - kReadBarrierStateSize,
     kHashMask = (1 << kHashSize) - 1,
     kMaxHash = kHashMask,
+
+    kMonitorIdShift = kHashShift,
+    kMonitorIdSize = kHashSize,
+    kMonitorIdMask = kHashMask,
+    kMonitorIdAlignmentShift = 32 - kMonitorIdSize,
+    kMonitorIdAlignment = 1 << kMonitorIdAlignmentShift,
     kMaxMonitorId = kMaxHash
   };
 
-  static LockWord FromThinLockId(uint32_t thread_id, uint32_t count) {
+  static LockWord FromThinLockId(uint32_t thread_id, uint32_t count, uint32_t rb_state) {
     CHECK_LE(thread_id, static_cast<uint32_t>(kThinLockMaxOwner));
     CHECK_LE(count, static_cast<uint32_t>(kThinLockMaxCount));
     return LockWord((thread_id << kThinLockOwnerShift) | (count << kThinLockCountShift) |
-                     (kStateThinOrUnlocked << kStateShift));
+                    (rb_state << kReadBarrierStateShift) |
+                    (kStateThinOrUnlocked << kStateShift));
   }
 
   static LockWord FromForwardingAddress(size_t target) {
@@ -97,9 +120,23 @@ class LockWord {
     return LockWord((target >> kStateSize) | (kStateForwardingAddress << kStateShift));
   }
 
-  static LockWord FromHashCode(uint32_t hash_code) {
+  static LockWord FromHashCode(uint32_t hash_code, uint32_t rb_state) {
     CHECK_LE(hash_code, static_cast<uint32_t>(kMaxHash));
-    return LockWord((hash_code << kHashShift) | (kStateHash << kStateShift));
+    return LockWord((hash_code << kHashShift) |
+                    (rb_state << kReadBarrierStateShift) |
+                    (kStateHash << kStateShift));
+  }
+
+  static LockWord FromDefault(uint32_t rb_state) {
+    return LockWord(rb_state << kReadBarrierStateShift);
+  }
+
+  static bool IsDefault(LockWord lw) {
+    return LockWord().GetValue() == lw.GetValue();
+  }
+
+  static LockWord Default() {
+    return LockWord();
   }
 
   enum LockState {
@@ -111,6 +148,7 @@ class LockWord {
   };
 
   LockState GetState() const {
+    CheckReadBarrierState();
     if (UNLIKELY(value_ == 0)) {
       return kUnlocked;
     } else {
@@ -129,6 +167,10 @@ class LockWord {
     }
   }
 
+  uint32_t ReadBarrierState() const {
+    return (value_ >> kReadBarrierStateShift) & kReadBarrierStateMask;
+  }
+
   // Return the owner thin lock thread id.
   uint32_t ThinLockOwner() const;
 
@@ -141,25 +183,58 @@ class LockWord {
   // Return the forwarding address stored in the monitor.
   size_t ForwardingAddress() const;
 
+  // Constructor a lock word for inflation to use a Monitor.
+  explicit LockWord(Monitor* mon, uint32_t rb_state);
+
+  // Return the hash code stored in the lock word, must be kHashCode state.
+  int32_t GetHashCode() const;
+
+  template <bool kIncludeReadBarrierState>
+  static bool Equal(LockWord lw1, LockWord lw2) {
+    if (kIncludeReadBarrierState) {
+      return lw1.GetValue() == lw2.GetValue();
+    }
+    return lw1.GetValueWithoutReadBarrierState() == lw2.GetValueWithoutReadBarrierState();
+  }
+
+ private:
   // Default constructor with no lock ownership.
   LockWord();
 
-  // Constructor a lock word for inflation to use a Monitor.
-  explicit LockWord(Monitor* mon);
-
-  bool operator==(const LockWord& rhs) const {
-    return GetValue() == rhs.GetValue();
+  explicit LockWord(uint32_t val) : value_(val) {
+    CheckReadBarrierState();
   }
 
-  // Return the hash code stored in the lock word, must be kHashCode state.
-  int32_t GetHashCode() const;
+  // Disallow this in favor of explicit Equal() with the
+  // kIncludeReadBarrierState param to make clients be aware of the
+  // read barrier state.
+  bool operator==(const LockWord& rhs) = delete;
+
+  void CheckReadBarrierState() const {
+    if (kIsDebugBuild && ((value_ >> kStateShift) & kStateMask) != kStateForwardingAddress) {
+      uint32_t rb_state = ReadBarrierState();
+      if (!kUseReadBarrier) {
+        DCHECK_EQ(rb_state, 0U);
+      } else {
+        DCHECK(rb_state == ReadBarrier::white_ptr_ ||
+               rb_state == ReadBarrier::gray_ptr_ ||
+               rb_state == ReadBarrier::black_ptr_) << rb_state;
+      }
+    }
+  }
 
+  // Note GetValue() includes the read barrier bits and comparing (==)
+  // GetValue() between two lock words to compare the lock states may
+  // not work. Prefer Equal() or GetValueWithoutReadBarrierState().
   uint32_t GetValue() const {
+    CheckReadBarrierState();
     return value_;
   }
 
- private:
-  explicit LockWord(uint32_t val) : value_(val) {}
+  uint32_t GetValueWithoutReadBarrierState() const {
+    CheckReadBarrierState();
+    return value_ & ~(kReadBarrierStateMask << kReadBarrierStateShift);
+  }
 
   // Only Object should be converting LockWords to/from uints.
   friend class mirror::Object;
diff --git a/runtime/mirror/object.cc b/runtime/mirror/object.cc
index 9262a3e0b6..bbbdf98be8 100644
--- a/runtime/mirror/object.cc
+++ b/runtime/mirror/object.cc
@@ -159,7 +159,8 @@ int32_t Object::IdentityHashCode() const {
       case LockWord::kUnlocked: {
         // Try to compare and swap in a new hash, if we succeed we will return the hash on the next
         // loop iteration.
-        LockWord hash_word(LockWord::FromHashCode(GenerateIdentityHashCode()));
+        LockWord hash_word = LockWord::FromHashCode(GenerateIdentityHashCode(),
+                                                    lw.ReadBarrierState());
         DCHECK_EQ(hash_word.GetState(), LockWord::kHashCode);
         if (const_cast<Object*>(this)->CasLockWordWeakRelaxed(lw, hash_word)) {
           return hash_word.GetHashCode();
diff --git a/runtime/monitor.cc b/runtime/monitor.cc
index 5ed8c7ded9..45a971d22d 100644
--- a/runtime/monitor.cc
+++ b/runtime/monitor.cc
@@ -165,7 +165,7 @@ bool Monitor::Install(Thread* self) {
       return false;
     }
   }
-  LockWord fat(this);
+  LockWord fat(this, lw.ReadBarrierState());
   // Publish the updated lock word, which may race with other threads.
   bool success = GetObject()->CasLockWordWeakSequentiallyConsistent(lw, fat);
   // Lock profiling.
@@ -610,15 +610,22 @@ bool Monitor::Deflate(Thread* self, mirror::Object* obj) {
         return false;
       }
       // Deflate to a thin lock.
-      obj->SetLockWord(LockWord::FromThinLockId(owner->GetThreadId(), monitor->lock_count_), false);
+      LockWord new_lw = LockWord::FromThinLockId(owner->GetThreadId(), monitor->lock_count_,
+                                                 lw.ReadBarrierState());
+      // Assume no concurrent read barrier state changes as mutators are suspended.
+      obj->SetLockWord(new_lw, false);
       VLOG(monitor) << "Deflated " << obj << " to thin lock " << owner->GetTid() << " / "
           << monitor->lock_count_;
     } else if (monitor->HasHashCode()) {
-      obj->SetLockWord(LockWord::FromHashCode(monitor->GetHashCode()), false);
+      LockWord new_lw = LockWord::FromHashCode(monitor->GetHashCode(), lw.ReadBarrierState());
+      // Assume no concurrent read barrier state changes as mutators are suspended.
+      obj->SetLockWord(new_lw, false);
       VLOG(monitor) << "Deflated " << obj << " to hash monitor " << monitor->GetHashCode();
     } else {
       // No lock and no hash, just put an empty lock word inside the object.
-      obj->SetLockWord(LockWord(), false);
+      LockWord new_lw = LockWord::FromDefault(lw.ReadBarrierState());
+      // Assume no concurrent read barrier state changes as mutators are suspended.
+      obj->SetLockWord(new_lw, false);
       VLOG(monitor) << "Deflated" << obj << " to empty lock word";
     }
     // The monitor is deflated, mark the object as nullptr so that we know to delete it during the
@@ -704,7 +711,7 @@ mirror::Object* Monitor::MonitorEnter(Thread* self, mirror::Object* obj) {
     LockWord lock_word = h_obj->GetLockWord(true);
     switch (lock_word.GetState()) {
       case LockWord::kUnlocked: {
-        LockWord thin_locked(LockWord::FromThinLockId(thread_id, 0));
+        LockWord thin_locked(LockWord::FromThinLockId(thread_id, 0, lock_word.ReadBarrierState()));
         if (h_obj->CasLockWordWeakSequentiallyConsistent(lock_word, thin_locked)) {
           // CasLockWord enforces more than the acquire ordering we need here.
           return h_obj.Get();  // Success!
@@ -717,9 +724,18 @@ mirror::Object* Monitor::MonitorEnter(Thread* self, mirror::Object* obj) {
           // We own the lock, increase the recursion count.
           uint32_t new_count = lock_word.ThinLockCount() + 1;
           if (LIKELY(new_count <= LockWord::kThinLockMaxCount)) {
-            LockWord thin_locked(LockWord::FromThinLockId(thread_id, new_count));
-            h_obj->SetLockWord(thin_locked, true);
-            return h_obj.Get();  // Success!
+            LockWord thin_locked(LockWord::FromThinLockId(thread_id, new_count,
+                                                          lock_word.ReadBarrierState()));
+            if (!kUseReadBarrier) {
+              h_obj->SetLockWord(thin_locked, true);
+              return h_obj.Get();  // Success!
+            } else {
+              // Use CAS to preserve the read barrier state.
+              if (h_obj->CasLockWordWeakSequentiallyConsistent(lock_word, thin_locked)) {
+                return h_obj.Get();  // Success!
+              }
+            }
+            continue;  // Go again.
           } else {
             // We'd overflow the recursion count, so inflate the monitor.
             InflateThinLocked(self, h_obj, lock_word, 0);
@@ -762,43 +778,57 @@ bool Monitor::MonitorExit(Thread* self, mirror::Object* obj) {
   DCHECK(self != NULL);
   DCHECK(obj != NULL);
   obj = FakeUnlock(obj);
-  LockWord lock_word = obj->GetLockWord(true);
   StackHandleScope<1> hs(self);
   Handle<mirror::Object> h_obj(hs.NewHandle(obj));
-  switch (lock_word.GetState()) {
-    case LockWord::kHashCode:
-      // Fall-through.
-    case LockWord::kUnlocked:
-      FailedUnlock(h_obj.Get(), self, nullptr, nullptr);
-      return false;  // Failure.
-    case LockWord::kThinLocked: {
-      uint32_t thread_id = self->GetThreadId();
-      uint32_t owner_thread_id = lock_word.ThinLockOwner();
-      if (owner_thread_id != thread_id) {
-        // TODO: there's a race here with the owner dying while we unlock.
-        Thread* owner =
-            Runtime::Current()->GetThreadList()->FindThreadByThreadId(lock_word.ThinLockOwner());
-        FailedUnlock(h_obj.Get(), self, owner, nullptr);
+  while (true) {
+    LockWord lock_word = obj->GetLockWord(true);
+    switch (lock_word.GetState()) {
+      case LockWord::kHashCode:
+        // Fall-through.
+      case LockWord::kUnlocked:
+        FailedUnlock(h_obj.Get(), self, nullptr, nullptr);
         return false;  // Failure.
-      } else {
-        // We own the lock, decrease the recursion count.
-        if (lock_word.ThinLockCount() != 0) {
-          uint32_t new_count = lock_word.ThinLockCount() - 1;
-          LockWord thin_locked(LockWord::FromThinLockId(thread_id, new_count));
-          h_obj->SetLockWord(thin_locked, true);
+      case LockWord::kThinLocked: {
+        uint32_t thread_id = self->GetThreadId();
+        uint32_t owner_thread_id = lock_word.ThinLockOwner();
+        if (owner_thread_id != thread_id) {
+          // TODO: there's a race here with the owner dying while we unlock.
+          Thread* owner =
+              Runtime::Current()->GetThreadList()->FindThreadByThreadId(lock_word.ThinLockOwner());
+          FailedUnlock(h_obj.Get(), self, owner, nullptr);
+          return false;  // Failure.
         } else {
-          h_obj->SetLockWord(LockWord(), true);
+          // We own the lock, decrease the recursion count.
+          LockWord new_lw = LockWord::Default();
+          if (lock_word.ThinLockCount() != 0) {
+            uint32_t new_count = lock_word.ThinLockCount() - 1;
+            new_lw = LockWord::FromThinLockId(thread_id, new_count, lock_word.ReadBarrierState());
+          } else {
+            new_lw = LockWord::FromDefault(lock_word.ReadBarrierState());
+          }
+          if (!kUseReadBarrier) {
+            DCHECK_EQ(new_lw.ReadBarrierState(), 0U);
+            h_obj->SetLockWord(new_lw, true);
+            // Success!
+            return true;
+          } else {
+            // Use CAS to preserve the read barrier state.
+            if (h_obj->CasLockWordWeakSequentiallyConsistent(lock_word, new_lw)) {
+              // Success!
+              return true;
+            }
+          }
+          continue;  // Go again.
         }
-        return true;  // Success!
       }
-    }
-    case LockWord::kFatLocked: {
-      Monitor* mon = lock_word.FatLockMonitor();
-      return mon->Unlock(self);
-    }
-    default: {
-      LOG(FATAL) << "Invalid monitor state " << lock_word.GetState();
-      return false;
+      case LockWord::kFatLocked: {
+        Monitor* mon = lock_word.FatLockMonitor();
+        return mon->Unlock(self);
+      }
+      default: {
+        LOG(FATAL) << "Invalid monitor state " << lock_word.GetState();
+        return false;
+      }
     }
   }
 }
diff --git a/runtime/monitor.h b/runtime/monitor.h
index 0c5f8a43b1..2e065ae0c8 100644
--- a/runtime/monitor.h
+++ b/runtime/monitor.h
@@ -22,12 +22,16 @@
 
 #include <iosfwd>
 #include <list>
+#ifndef __LP64__
+#include <malloc.h>  // For memalign.
+#endif
 #include <vector>
 
 #include "atomic.h"
 #include "base/allocator.h"
 #include "base/mutex.h"
 #include "gc_root.h"
+#include "lock_word.h"
 #include "object_callbacks.h"
 #include "read_barrier_option.h"
 #include "thread_state.h"
@@ -127,8 +131,17 @@ class Monitor {
                                 uint32_t hash_code) NO_THREAD_SAFETY_ANALYSIS;
 
   static bool Deflate(Thread* self, mirror::Object* obj)
+      // Not exclusive because ImageWriter calls this during a Heap::VisitObjects() that
+      // does not allow a thread suspension in the middle. TODO: maybe make this exclusive.
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
+#ifndef __LP64__
+  void* operator new(size_t size) {
+    // Align Monitor* as per the monitor ID field size in the lock word.
+    return memalign(LockWord::kMonitorIdAlignment, size);
+  }
+#endif
+
  private:
   explicit Monitor(Thread* self, Thread* owner, mirror::Object* obj, int32_t hash_code)
         SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
diff --git a/runtime/monitor_pool.h b/runtime/monitor_pool.h
index 27678dcbdd..8ae5a54fe7 100644
--- a/runtime/monitor_pool.h
+++ b/runtime/monitor_pool.h
@@ -45,7 +45,9 @@ class MonitorPool {
   static Monitor* CreateMonitor(Thread* self, Thread* owner, mirror::Object* obj, int32_t hash_code)
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
 #ifndef __LP64__
-    return new Monitor(self, owner, obj, hash_code);
+    Monitor* mon = new Monitor(self, owner, obj, hash_code);
+    DCHECK_ALIGNED(mon, LockWord::kMonitorIdAlignment);
+    return mon;
 #else
     return GetMonitorPool()->CreateMonitorInPool(self, owner, obj, hash_code);
 #endif
@@ -71,7 +73,7 @@ class MonitorPool {
 
   static Monitor* MonitorFromMonitorId(MonitorId mon_id) {
 #ifndef __LP64__
-    return reinterpret_cast<Monitor*>(mon_id << 3);
+    return reinterpret_cast<Monitor*>(mon_id << LockWord::kMonitorIdAlignmentShift);
 #else
     return GetMonitorPool()->LookupMonitor(mon_id);
 #endif
@@ -79,7 +81,7 @@ class MonitorPool {
 
   static MonitorId MonitorIdFromMonitor(Monitor* mon) {
 #ifndef __LP64__
-    return reinterpret_cast<MonitorId>(mon) >> 3;
+    return reinterpret_cast<MonitorId>(mon) >> LockWord::kMonitorIdAlignmentShift;
 #else
     return mon->GetMonitorId();
 #endif
diff --git a/runtime/read_barrier_c.h b/runtime/read_barrier_c.h
index 49efaa2bd7..a2c4c36a2f 100644
--- a/runtime/read_barrier_c.h
+++ b/runtime/read_barrier_c.h
@@ -35,6 +35,10 @@
 #define USE_BAKER_OR_BROOKS_READ_BARRIER
 #endif
 
+#if defined(USE_BAKER_READ_BARRIER) || defined(USE_BROOKS_READ_BARRIER) || defined(USE_TABLE_LOOKUP_READ_BARRIER)
+#define USE_READ_BARRIER
+#endif
+
 #if defined(USE_BAKER_READ_BARRIER) && defined(USE_BROOKS_READ_BARRIER)
 #error "Only one of Baker or Brooks can be enabled at a time."
 #endif
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index 83c5ffb135..7c0e7d7eda 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -54,7 +54,7 @@ ThreadList::ThreadList()
     : suspend_all_count_(0), debug_suspend_all_count_(0),
       thread_exit_cond_("thread exit condition variable", *Locks::thread_list_lock_),
       suspend_all_historam_("suspend all histogram", 16, 64) {
-  CHECK(Monitor::IsValidLockWord(LockWord::FromThinLockId(kMaxThreadId, 1)));
+  CHECK(Monitor::IsValidLockWord(LockWord::FromThinLockId(kMaxThreadId, 1, 0U)));
 }
 
 ThreadList::~ThreadList() {
diff --git a/runtime/transaction_test.cc b/runtime/transaction_test.cc
index b80fe22cd5..5db51c8e93 100644
--- a/runtime/transaction_test.cc
+++ b/runtime/transaction_test.cc
@@ -63,7 +63,7 @@ class TransactionTest : public CommonRuntimeTest {
     ASSERT_TRUE(h_klass->IsVerified());
 
     mirror::Class::Status old_status = h_klass->GetStatus();
-    uint32_t old_lock_word = h_klass->GetLockWord(false).GetValue();
+    LockWord old_lock_word = h_klass->GetLockWord(false);
 
     Transaction transaction;
     Runtime::Current()->EnterTransactionMode(&transaction);
@@ -75,8 +75,8 @@ class TransactionTest : public CommonRuntimeTest {
     ASSERT_TRUE(transaction.IsAborted());
 
     // Check class's monitor get back to its original state without rolling back changes.
-    uint32_t new_lock_word = h_klass->GetLockWord(false).GetValue();
-    EXPECT_EQ(old_lock_word, new_lock_word);
+    LockWord new_lock_word = h_klass->GetLockWord(false);
+    EXPECT_TRUE(LockWord::Equal<false>(old_lock_word, new_lock_word));
 
     // Check class status is rolled back properly.
     soa.Self()->ClearException();
@@ -118,20 +118,20 @@ TEST_F(TransactionTest, Object_monitor) {
 
   // Lock object's monitor outside the transaction.
   h_obj->MonitorEnter(soa.Self());
-  uint32_t old_lock_word = h_obj->GetLockWord(false).GetValue();
+  LockWord old_lock_word = h_obj->GetLockWord(false);
 
   Transaction transaction;
   Runtime::Current()->EnterTransactionMode(&transaction);
   // Unlock object's monitor inside the transaction.
   h_obj->MonitorExit(soa.Self());
-  uint32_t new_lock_word = h_obj->GetLockWord(false).GetValue();
+  LockWord new_lock_word = h_obj->GetLockWord(false);
   Runtime::Current()->ExitTransactionMode();
 
   // Rolling back transaction's changes must not change monitor's state.
   transaction.Rollback();
-  uint32_t aborted_lock_word = h_obj->GetLockWord(false).GetValue();
-  EXPECT_NE(old_lock_word, new_lock_word);
-  EXPECT_EQ(aborted_lock_word, new_lock_word);
+  LockWord aborted_lock_word = h_obj->GetLockWord(false);
+  EXPECT_FALSE(LockWord::Equal<false>(old_lock_word, new_lock_word));
+  EXPECT_TRUE(LockWord::Equal<false>(aborted_lock_word, new_lock_word));
 }
 
 // Tests array's length is preserved after transaction rollback.