More allocation code optimizations.

- Inline Class::AllocObject() and Array::Alloc().
- Inline some short Mutex functions and add LIKELY/UNLIKELY to some
  Mutex functions.
- This change improves the Ritz MemAllocTest by ~6% on Nexus 4 and
  ~10% on host.

Bug: 9986565
Change-Id: I1606c74ddb21676cbc1de1a40e9b076fc23eaea4

11 files changed, 183 insertions, 177 deletions

diff --git a/runtime/base/mutex-inl.h b/runtime/base/mutex-inl.h
index 7e8365eaba..c0cfee2463 100644
--- a/runtime/base/mutex-inl.h
+++ b/runtime/base/mutex-inl.h
@@ -41,6 +41,54 @@ static inline int futex(volatile int *uaddr, int op, int val, const struct times
 }
 #endif  // ART_USE_FUTEXES
 
+#if defined(__APPLE__)
+
+// This works on Mac OS 10.6 but hasn't been tested on older releases.
+struct __attribute__((__may_alias__)) darwin_pthread_mutex_t {
+  long padding0;  // NOLINT(runtime/int) exact match to darwin type
+  int padding1;
+  uint32_t padding2;
+  int16_t padding3;
+  int16_t padding4;
+  uint32_t padding5;
+  pthread_t darwin_pthread_mutex_owner;
+  // ...other stuff we don't care about.
+};
+
+struct __attribute__((__may_alias__)) darwin_pthread_rwlock_t {
+  long padding0;  // NOLINT(runtime/int) exact match to darwin type
+  pthread_mutex_t padding1;
+  int padding2;
+  pthread_cond_t padding3;
+  pthread_cond_t padding4;
+  int padding5;
+  int padding6;
+  pthread_t darwin_pthread_rwlock_owner;
+  // ...other stuff we don't care about.
+};
+
+#endif  // __APPLE__
+
+#if defined(__GLIBC__)
+
+struct __attribute__((__may_alias__)) glibc_pthread_mutex_t {
+  int32_t padding0[2];
+  int owner;
+  // ...other stuff we don't care about.
+};
+
+struct __attribute__((__may_alias__)) glibc_pthread_rwlock_t {
+#ifdef __LP64__
+  int32_t padding0[6];
+#else
+  int32_t padding0[7];
+#endif
+  int writer;
+  // ...other stuff we don't care about.
+};
+
+#endif  // __GLIBC__
+
 class ScopedContentionRecorder {
  public:
   ScopedContentionRecorder(BaseMutex* mutex, uint64_t blocked_tid, uint64_t owner_tid)
@@ -185,6 +233,84 @@ inline void ReaderWriterMutex::SharedUnlock(Thread* self) {
 #endif
 }
 
+inline bool Mutex::IsExclusiveHeld(const Thread* self) const {
+  DCHECK(self == NULL || self == Thread::Current());
+  bool result = (GetExclusiveOwnerTid() == SafeGetTid(self));
+  if (kDebugLocking) {
+    // Sanity debug check that if we think it is locked we have it in our held mutexes.
+    if (result && self != NULL && level_ != kMonitorLock && !gAborting) {
+      CHECK_EQ(self->GetHeldMutex(level_), this);
+    }
+  }
+  return result;
+}
+
+inline uint64_t Mutex::GetExclusiveOwnerTid() const {
+#if ART_USE_FUTEXES
+  return exclusive_owner_;
+#elif defined(__BIONIC__)
+  return static_cast<uint64_t>((mutex_.value >> 16) & 0xffff);
+#elif defined(__GLIBC__)
+  return reinterpret_cast<const glibc_pthread_mutex_t*>(&mutex_)->owner;
+#elif defined(__APPLE__)
+  const darwin_pthread_mutex_t* dpmutex = reinterpret_cast<const darwin_pthread_mutex_t*>(&mutex_);
+  pthread_t owner = dpmutex->darwin_pthread_mutex_owner;
+  // 0 for unowned, -1 for PTHREAD_MTX_TID_SWITCHING
+  // TODO: should we make darwin_pthread_mutex_owner volatile and recheck until not -1?
+  if ((owner == (pthread_t)0) || (owner == (pthread_t)-1)) {
+    return 0;
+  }
+  uint64_t tid;
+  CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__);  // Requires Mac OS 10.6
+  return tid;
+#else
+#error unsupported C library
+#endif
+}
+
+inline bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const {
+  DCHECK(self == NULL || self == Thread::Current());
+  bool result = (GetExclusiveOwnerTid() == SafeGetTid(self));
+  if (kDebugLocking) {
+    // Sanity that if the pthread thinks we own the lock the Thread agrees.
+    if (self != NULL && result)  {
+      CHECK_EQ(self->GetHeldMutex(level_), this);
+    }
+  }
+  return result;
+}
+
+inline uint64_t ReaderWriterMutex::GetExclusiveOwnerTid() const {
+#if ART_USE_FUTEXES
+  int32_t state = state_;
+  if (state == 0) {
+    return 0;  // No owner.
+  } else if (state > 0) {
+    return -1;  // Shared.
+  } else {
+    return exclusive_owner_;
+  }
+#else
+#if defined(__BIONIC__)
+  return rwlock_.writerThreadId;
+#elif defined(__GLIBC__)
+  return reinterpret_cast<const glibc_pthread_rwlock_t*>(&rwlock_)->writer;
+#elif defined(__APPLE__)
+  const darwin_pthread_rwlock_t*
+      dprwlock = reinterpret_cast<const darwin_pthread_rwlock_t*>(&rwlock_);
+  pthread_t owner = dprwlock->darwin_pthread_rwlock_owner;
+  if (owner == (pthread_t)0) {
+    return 0;
+  }
+  uint64_t tid;
+  CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__);  // Requires Mac OS 10.6
+  return tid;
+#else
+#error unsupported C library
+#endif
+#endif
+}
+
 }  // namespace art
 
 #endif  // ART_RUNTIME_BASE_MUTEX_INL_H_
diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc
index b99e7c9281..b048bbb1ec 100644
--- a/runtime/base/mutex.cc
+++ b/runtime/base/mutex.cc
@@ -31,54 +31,6 @@
 
 namespace art {
 
-#if defined(__APPLE__)
-
-// This works on Mac OS 10.6 but hasn't been tested on older releases.
-struct __attribute__((__may_alias__)) darwin_pthread_mutex_t {
-  long padding0;  // NOLINT(runtime/int) exact match to darwin type
-  int padding1;
-  uint32_t padding2;
-  int16_t padding3;
-  int16_t padding4;
-  uint32_t padding5;
-  pthread_t darwin_pthread_mutex_owner;
-  // ...other stuff we don't care about.
-};
-
-struct __attribute__((__may_alias__)) darwin_pthread_rwlock_t {
-  long padding0;  // NOLINT(runtime/int) exact match to darwin type
-  pthread_mutex_t padding1;
-  int padding2;
-  pthread_cond_t padding3;
-  pthread_cond_t padding4;
-  int padding5;
-  int padding6;
-  pthread_t darwin_pthread_rwlock_owner;
-  // ...other stuff we don't care about.
-};
-
-#endif  // __APPLE__
-
-#if defined(__GLIBC__)
-
-struct __attribute__((__may_alias__)) glibc_pthread_mutex_t {
-  int32_t padding0[2];
-  int owner;
-  // ...other stuff we don't care about.
-};
-
-struct __attribute__((__may_alias__)) glibc_pthread_rwlock_t {
-#ifdef __LP64__
-  int32_t padding0[6];
-#else
-  int32_t padding0[7];
-#endif
-  int writer;
-  // ...other stuff we don't care about.
-};
-
-#endif  // __GLIBC__
-
 #if ART_USE_FUTEXES
 static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) {
   const int32_t one_sec = 1000 * 1000 * 1000;  // one second in nanoseconds.
@@ -346,7 +298,7 @@ void Mutex::ExclusiveLock(Thread* self) {
     bool done = false;
     do {
       int32_t cur_state = state_;
-      if (cur_state == 0) {
+      if (LIKELY(cur_state == 0)) {
         // Change state from 0 to 1.
         done = android_atomic_acquire_cas(0, 1, &state_) == 0;
       } else {
@@ -432,14 +384,14 @@ void Mutex::ExclusiveUnlock(Thread* self) {
   bool done = false;
   do {
     int32_t cur_state = state_;
-    if (cur_state == 1) {
+    if (LIKELY(cur_state == 1)) {
       // We're no longer the owner.
       exclusive_owner_ = 0;
       // Change state to 0.
       done = android_atomic_release_cas(cur_state, 0, &state_) == 0;
-      if (done) {  // Spurious fail?
+      if (LIKELY(done)) {  // Spurious fail?
         // Wake a contender
-        if (num_contenders_ > 0) {
+        if (UNLIKELY(num_contenders_ > 0)) {
           futex(&state_, FUTEX_WAKE, 1, NULL, NULL, 0);
         }
       }
@@ -461,41 +413,6 @@ void Mutex::ExclusiveUnlock(Thread* self) {
   }
 }
 
-bool Mutex::IsExclusiveHeld(const Thread* self) const {
-  DCHECK(self == NULL || self == Thread::Current());
-  bool result = (GetExclusiveOwnerTid() == SafeGetTid(self));
-  if (kDebugLocking) {
-    // Sanity debug check that if we think it is locked we have it in our held mutexes.
-    if (result && self != NULL && level_ != kMonitorLock && !gAborting) {
-      CHECK_EQ(self->GetHeldMutex(level_), this);
-    }
-  }
-  return result;
-}
-
-uint64_t Mutex::GetExclusiveOwnerTid() const {
-#if ART_USE_FUTEXES
-  return exclusive_owner_;
-#elif defined(__BIONIC__)
-  return static_cast<uint64_t>((mutex_.value >> 16) & 0xffff);
-#elif defined(__GLIBC__)
-  return reinterpret_cast<const glibc_pthread_mutex_t*>(&mutex_)->owner;
-#elif defined(__APPLE__)
-  const darwin_pthread_mutex_t* dpmutex = reinterpret_cast<const darwin_pthread_mutex_t*>(&mutex_);
-  pthread_t owner = dpmutex->darwin_pthread_mutex_owner;
-  // 0 for unowned, -1 for PTHREAD_MTX_TID_SWITCHING
-  // TODO: should we make darwin_pthread_mutex_owner volatile and recheck until not -1?
-  if ((owner == (pthread_t)0) || (owner == (pthread_t)-1)) {
-    return 0;
-  }
-  uint64_t tid;
-  CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__);  // Requires Mac OS 10.6
-  return tid;
-#else
-#error unsupported C library
-#endif
-}
-
 void Mutex::Dump(std::ostream& os) const {
   os << (recursive_ ? "recursive " : "non-recursive ")
       << name_
@@ -549,7 +466,7 @@ void ReaderWriterMutex::ExclusiveLock(Thread* self) {
   bool done = false;
   do {
     int32_t cur_state = state_;
-    if (cur_state == 0) {
+    if (LIKELY(cur_state == 0)) {
       // Change state from 0 to -1.
       done = android_atomic_acquire_cas(0, -1, &state_) == 0;
     } else {
@@ -583,14 +500,14 @@ void ReaderWriterMutex::ExclusiveUnlock(Thread* self) {
   bool done = false;
   do {
     int32_t cur_state = state_;
-    if (cur_state == -1) {
+    if (LIKELY(cur_state == -1)) {
       // We're no longer the owner.
       exclusive_owner_ = 0;
       // Change state from -1 to 0.
       done = android_atomic_release_cas(-1, 0, &state_) == 0;
-      if (done) {  // cmpxchg may fail due to noise?
+      if (LIKELY(done)) {  // cmpxchg may fail due to noise?
         // Wake any waiters.
-        if (num_pending_readers_ > 0 || num_pending_writers_ > 0) {
+        if (UNLIKELY(num_pending_readers_ > 0 || num_pending_writers_ > 0)) {
           futex(&state_, FUTEX_WAKE, -1, NULL, NULL, 0);
         }
       }
@@ -687,18 +604,6 @@ bool ReaderWriterMutex::SharedTryLock(Thread* self) {
   return true;
 }
 
-bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const {
-  DCHECK(self == NULL || self == Thread::Current());
-  bool result = (GetExclusiveOwnerTid() == SafeGetTid(self));
-  if (kDebugLocking) {
-    // Sanity that if the pthread thinks we own the lock the Thread agrees.
-    if (self != NULL && result)  {
-      CHECK_EQ(self->GetHeldMutex(level_), this);
-    }
-  }
-  return result;
-}
-
 bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const {
   DCHECK(self == NULL || self == Thread::Current());
   bool result;
@@ -710,37 +615,6 @@ bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const {
   return result;
 }
 
-uint64_t ReaderWriterMutex::GetExclusiveOwnerTid() const {
-#if ART_USE_FUTEXES
-  int32_t state = state_;
-  if (state == 0) {
-    return 0;  // No owner.
-  } else if (state > 0) {
-    return -1;  // Shared.
-  } else {
-    return exclusive_owner_;
-  }
-#else
-#if defined(__BIONIC__)
-  return rwlock_.writerThreadId;
-#elif defined(__GLIBC__)
-  return reinterpret_cast<const glibc_pthread_rwlock_t*>(&rwlock_)->writer;
-#elif defined(__APPLE__)
-  const darwin_pthread_rwlock_t*
-      dprwlock = reinterpret_cast<const darwin_pthread_rwlock_t*>(&rwlock_);
-  pthread_t owner = dprwlock->darwin_pthread_rwlock_owner;
-  if (owner == (pthread_t)0) {
-    return 0;
-  }
-  uint64_t tid;
-  CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__);  // Requires Mac OS 10.6
-  return tid;
-#else
-#error unsupported C library
-#endif
-#endif
-}
-
 void ReaderWriterMutex::Dump(std::ostream& os) const {
   os << name_
       << " level=" << static_cast<int>(level_)
diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc
index a174c0a1dc..c6d028eed5 100644
--- a/runtime/gc/space/large_object_space.cc
+++ b/runtime/gc/space/large_object_space.cc
@@ -17,6 +17,7 @@
 #include "large_object_space.h"
 
 #include "base/logging.h"
+#include "base/mutex-inl.h"
 #include "base/stl_util.h"
 #include "UniquePtr.h"
 #include "image.h"
diff --git a/runtime/indirect_reference_table_test.cc b/runtime/indirect_reference_table_test.cc
index bd2890c497..b6c6cb4be3 100644
--- a/runtime/indirect_reference_table_test.cc
+++ b/runtime/indirect_reference_table_test.cc
@@ -17,6 +17,7 @@
 #include "common_test.h"
 
 #include "indirect_reference_table.h"
+#include "mirror/object-inl.h"
 
 namespace art {
 
diff --git a/runtime/mirror/array-inl.h b/runtime/mirror/array-inl.h
index eb73c7dd38..c7b370f877 100644
--- a/runtime/mirror/array-inl.h
+++ b/runtime/mirror/array-inl.h
@@ -20,6 +20,8 @@
 #include "array.h"
 
 #include "class.h"
+#include "thread.h"
+#include "utils.h"
 
 namespace art {
 namespace mirror {
@@ -33,6 +35,39 @@ inline size_t Array::SizeOf() const {
   return header_size + data_size;
 }
 
+inline Array* Array::Alloc(Thread* self, Class* array_class, int32_t component_count,
+                           size_t component_size) {
+  DCHECK(array_class != NULL);
+  DCHECK_GE(component_count, 0);
+  DCHECK(array_class->IsArrayClass());
+
+  size_t header_size = sizeof(Object) + (component_size == sizeof(int64_t) ? 8 : 4);
+  size_t data_size = component_count * component_size;
+  size_t size = header_size + data_size;
+
+  // Check for overflow and throw OutOfMemoryError if this was an unreasonable request.
+  size_t component_shift = sizeof(size_t) * 8 - 1 - CLZ(component_size);
+  if (UNLIKELY(data_size >> component_shift != size_t(component_count) || size < data_size)) {
+    self->ThrowOutOfMemoryError(StringPrintf("%s of length %d would overflow",
+                                             PrettyDescriptor(array_class).c_str(),
+                                             component_count).c_str());
+    return NULL;
+  }
+
+  gc::Heap* heap = Runtime::Current()->GetHeap();
+  Array* array = down_cast<Array*>(heap->AllocObject(self, array_class, size));
+  if (LIKELY(array != NULL)) {
+    DCHECK(array->IsArrayInstance());
+    array->SetLength(component_count);
+  }
+  return array;
+}
+
+inline Array* Array::Alloc(Thread* self, Class* array_class, int32_t component_count) {
+  DCHECK(array_class->IsArrayClass());
+  return Alloc(self, array_class, component_count, array_class->GetComponentSize());
+}
+
 }  // namespace mirror
 }  // namespace art
 
diff --git a/runtime/mirror/array.cc b/runtime/mirror/array.cc
index 88cd309eeb..020085dbf0 100644
--- a/runtime/mirror/array.cc
+++ b/runtime/mirror/array.cc
@@ -32,39 +32,6 @@
 namespace art {
 namespace mirror {
 
-Array* Array::Alloc(Thread* self, Class* array_class, int32_t component_count,
-                    size_t component_size) {
-  DCHECK(array_class != NULL);
-  DCHECK_GE(component_count, 0);
-  DCHECK(array_class->IsArrayClass());
-
-  size_t header_size = sizeof(Object) + (component_size == sizeof(int64_t) ? 8 : 4);
-  size_t data_size = component_count * component_size;
-  size_t size = header_size + data_size;
-
-  // Check for overflow and throw OutOfMemoryError if this was an unreasonable request.
-  size_t component_shift = sizeof(size_t) * 8 - 1 - CLZ(component_size);
-  if (UNLIKELY(data_size >> component_shift != size_t(component_count) || size < data_size)) {
-    self->ThrowOutOfMemoryError(StringPrintf("%s of length %d would overflow",
-                                             PrettyDescriptor(array_class).c_str(),
-                                             component_count).c_str());
-    return NULL;
-  }
-
-  gc::Heap* heap = Runtime::Current()->GetHeap();
-  Array* array = down_cast<Array*>(heap->AllocObject(self, array_class, size));
-  if (array != NULL) {
-    DCHECK(array->IsArrayInstance());
-    array->SetLength(component_count);
-  }
-  return array;
-}
-
-Array* Array::Alloc(Thread* self, Class* array_class, int32_t component_count) {
-  DCHECK(array_class->IsArrayClass());
-  return Alloc(self, array_class, component_count, array_class->GetComponentSize());
-}
-
 // Create a multi-dimensional array of Objects or primitive types.
 //
 // We have to generate the names for X[], X[][], X[][][], and so on.  The
diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 1e1138745d..438ce81db2 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -342,6 +342,15 @@ inline void Class::SetName(String* name) {
   SetFieldObject(OFFSET_OF_OBJECT_MEMBER(Class, name_), name, false);
 }
 
+inline Object* Class::AllocObject(Thread* self) {
+  DCHECK(!IsArrayClass()) << PrettyClass(this);
+  DCHECK(IsInstantiable()) << PrettyClass(this);
+  // TODO: decide whether we want this check. It currently fails during bootstrap.
+  // DCHECK(!Runtime::Current()->IsStarted() || IsInitializing()) << PrettyClass(this);
+  DCHECK_GE(this->object_size_, sizeof(Object));
+  return Runtime::Current()->GetHeap()->AllocObject(self, this, this->object_size_);
+}
+
 }  // namespace mirror
 }  // namespace art
 
diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc
index 5e8b82793a..328c67deb1 100644
--- a/runtime/mirror/class.cc
+++ b/runtime/mirror/class.cc
@@ -118,15 +118,6 @@ void Class::SetDexCache(DexCache* new_dex_cache) {
   SetFieldObject(OFFSET_OF_OBJECT_MEMBER(Class, dex_cache_), new_dex_cache, false);
 }
 
-Object* Class::AllocObject(Thread* self) {
-  DCHECK(!IsArrayClass()) << PrettyClass(this);
-  DCHECK(IsInstantiable()) << PrettyClass(this);
-  // TODO: decide whether we want this check. It currently fails during bootstrap.
-  // DCHECK(!Runtime::Current()->IsStarted() || IsInitializing()) << PrettyClass(this);
-  DCHECK_GE(this->object_size_, sizeof(Object));
-  return Runtime::Current()->GetHeap()->AllocObject(self, this, this->object_size_);
-}
-
 void Class::SetClassSize(size_t new_class_size) {
   DCHECK_GE(new_class_size, GetClassSize()) << " class=" << PrettyTypeOf(this);
   SetField32(OFFSET_OF_OBJECT_MEMBER(Class, class_size_), new_class_size, false);
diff --git a/runtime/mirror/stack_trace_element.cc b/runtime/mirror/stack_trace_element.cc
index a505ed00a3..9d76c6bc11 100644
--- a/runtime/mirror/stack_trace_element.cc
+++ b/runtime/mirror/stack_trace_element.cc
@@ -17,6 +17,7 @@
 #include "stack_trace_element.h"
 
 #include "class.h"
+#include "class-inl.h"
 #include "gc/accounting/card_table-inl.h"
 #include "object-inl.h"
 #include "string.h"
diff --git a/runtime/mirror/string.cc b/runtime/mirror/string.cc
index f8a0e531eb..7d968c774d 100644
--- a/runtime/mirror/string.cc
+++ b/runtime/mirror/string.cc
@@ -17,6 +17,7 @@
 #include "string.h"
 
 #include "array.h"
+#include "class-inl.h"
 #include "gc/accounting/card_table-inl.h"
 #include "intern_table.h"
 #include "object-inl.h"
@@ -285,4 +286,3 @@ int32_t String::CompareTo(String* rhs) const {
 
 }  // namespace mirror
 }  // namespace art
-
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index 671924a620..aba81feeff 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "base/mutex.h"
+#include "base/mutex-inl.h"
 #include "base/timing_logger.h"
 #include "debugger.h"
 #include "thread.h"