More of the concurrent copying collector.

Bug: 12687968
Change-Id: I62f70274d47df6d6cab714df95c518b750ce3105

1 files changed, 111 insertions, 7 deletions

diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h
index 0dc31e7bc1..c74fdedbf9 100644
--- a/runtime/read_barrier-inl.h
+++ b/runtime/read_barrier-inl.h
@@ -19,43 +19,147 @@
 
 #include "read_barrier.h"
 
+#include "gc/collector/concurrent_copying.h"
+#include "gc/heap.h"
 #include "mirror/object_reference.h"
+#include "mirror/reference.h"
+#include "runtime.h"
 
 namespace art {
 
-template <typename MirrorType, ReadBarrierOption kReadBarrierOption>
+template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
 inline MirrorType* ReadBarrier::Barrier(
     mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) {
-  // Unused for now.
-  UNUSED(obj, offset, ref_addr);
   const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
   if (with_read_barrier && kUseBakerReadBarrier) {
-    // To be implemented.
-    return ref_addr->AsMirrorPtr();
+    // The higher bits of the rb ptr, rb_ptr_high_bits (must be zero)
+    // is used to create artificial data dependency from the is_gray
+    // load to the ref field (ptr) load to avoid needing a load-load
+    // barrier between the two.
+    uintptr_t rb_ptr_high_bits;
+    bool is_gray = HasGrayReadBarrierPointer(obj, &rb_ptr_high_bits);
+    ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>(
+        rb_ptr_high_bits | reinterpret_cast<uintptr_t>(ref_addr));
+    MirrorType* ref = ref_addr->AsMirrorPtr();
+    if (is_gray) {
+      // Slow-path.
+      ref = reinterpret_cast<MirrorType*>(Mark(ref));
+    }
+    if (kEnableReadBarrierInvariantChecks) {
+      CHECK_EQ(rb_ptr_high_bits, 0U) << obj << " rb_ptr=" << obj->GetReadBarrierPointer();
+    }
+    AssertToSpaceInvariant(obj, offset, ref);
+    return ref;
   } else if (with_read_barrier && kUseBrooksReadBarrier) {
     // To be implemented.
     return ref_addr->AsMirrorPtr();
+  } else if (with_read_barrier && kUseTableLookupReadBarrier) {
+    MirrorType* ref = ref_addr->AsMirrorPtr();
+    MirrorType* old_ref = ref;
+    // The heap or the collector can be null at startup. TODO: avoid the need for this null check.
+    gc::Heap* heap = Runtime::Current()->GetHeap();
+    if (heap != nullptr && heap->GetReadBarrierTable()->IsSet(old_ref)) {
+      ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
+      // Update the field atomically. This may fail if mutator updates before us, but it's ok.
+      obj->CasFieldStrongSequentiallyConsistentObjectWithoutWriteBarrier<false, false>(
+          offset, old_ref, ref);
+    }
+    AssertToSpaceInvariant(obj, offset, ref);
+    return ref;
   } else {
     // No read barrier.
     return ref_addr->AsMirrorPtr();
   }
 }
 
-template <typename MirrorType, ReadBarrierOption kReadBarrierOption>
+template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
 inline MirrorType* ReadBarrier::BarrierForRoot(MirrorType** root) {
   MirrorType* ref = *root;
   const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
   if (with_read_barrier && kUseBakerReadBarrier) {
-    // To be implemented.
+    if (kMaybeDuringStartup && IsDuringStartup()) {
+      // During startup, the heap may not be initialized yet. Just
+      // return the given ref.
+      return ref;
+    }
+    // TODO: separate the read barrier code from the collector code more.
+    if (Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking()) {
+      ref = reinterpret_cast<MirrorType*>(Mark(ref));
+    }
+    AssertToSpaceInvariant(nullptr, MemberOffset(0), ref);
     return ref;
   } else if (with_read_barrier && kUseBrooksReadBarrier) {
     // To be implemented.
     return ref;
+  } else if (with_read_barrier && kUseTableLookupReadBarrier) {
+    if (kMaybeDuringStartup && IsDuringStartup()) {
+      // During startup, the heap may not be initialized yet. Just
+      // return the given ref.
+      return ref;
+    }
+    if (Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) {
+      MirrorType* old_ref = ref;
+      ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
+      // Update the field atomically. This may fail if mutator updates before us, but it's ok.
+      Atomic<mirror::Object*>* atomic_root = reinterpret_cast<Atomic<mirror::Object*>*>(root);
+      atomic_root->CompareExchangeStrongSequentiallyConsistent(old_ref, ref);
+    }
+    AssertToSpaceInvariant(nullptr, MemberOffset(0), ref);
+    return ref;
   } else {
     return ref;
   }
 }
 
+inline bool ReadBarrier::IsDuringStartup() {
+  gc::Heap* heap = Runtime::Current()->GetHeap();
+  if (heap == nullptr) {
+    // During startup, the heap can be null.
+    return true;
+  }
+  if (heap->CurrentCollectorType() != gc::kCollectorTypeCC) {
+    // CC isn't running.
+    return true;
+  }
+  gc::collector::ConcurrentCopying* collector = heap->ConcurrentCopyingCollector();
+  if (collector == nullptr) {
+    // During startup, the collector can be null.
+    return true;
+  }
+  return false;
+}
+
+inline void ReadBarrier::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset,
+                                                mirror::Object* ref) {
+  if (kEnableToSpaceInvariantChecks || kIsDebugBuild) {
+    if (ref == nullptr || IsDuringStartup()) {
+      return;
+    }
+    Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->
+        AssertToSpaceInvariant(obj, offset, ref);
+  }
+}
+
+inline mirror::Object* ReadBarrier::Mark(mirror::Object* obj) {
+  return Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->Mark(obj);
+}
+
+inline bool ReadBarrier::HasGrayReadBarrierPointer(mirror::Object* obj,
+                                                   uintptr_t* out_rb_ptr_high_bits) {
+  mirror::Object* rb_ptr = obj->GetReadBarrierPointer();
+  uintptr_t rb_ptr_bits = reinterpret_cast<uintptr_t>(rb_ptr);
+  uintptr_t rb_ptr_low_bits = rb_ptr_bits & rb_ptr_mask_;
+  if (kEnableReadBarrierInvariantChecks) {
+    CHECK(rb_ptr_low_bits == white_ptr_ || rb_ptr_low_bits == gray_ptr_ ||
+          rb_ptr_low_bits == black_ptr_)
+        << "obj=" << obj << " rb_ptr=" << rb_ptr << " " << PrettyTypeOf(obj);
+  }
+  bool is_gray = rb_ptr_low_bits == gray_ptr_;
+  // The high bits are supposed to be zero. We check this on the caller side.
+  *out_rb_ptr_high_bits = rb_ptr_bits & ~rb_ptr_mask_;
+  return is_gray;
+}
+
 }  // namespace art
 
 #endif  // ART_RUNTIME_READ_BARRIER_INL_H_