Inflate contended lock word by suspending owner.

Bug 6961405.
Don't inflate monitors for Notify and NotifyAll.
Tidy lock word, handle recursive lock case alongside unlocked case and move
assembly out of line (except for ARM quick). Also handle null in out-of-line
assembly as the test is quick and the enter/exit code is already a safepoint.
To gain ownership of a monitor on behalf of another thread, monitor contenders
must not hold the monitor_lock_, so they wait on a condition variable.
Reduce size of per mutex contention log.
Be consistent in calling thin lock thread ids just thread ids.
Fix potential thread death races caused by the use of FindThreadByThreadId,
make it invariant that returned threads are either self or suspended now.

Code size reduction on ARM boot.oat 0.2%.
Old nexus 7 speedup 0.25%, new nexus 7 speedup 1.4%, nexus 10 speedup 2.24%,
nexus 4 speedup 2.09% on DeltaBlue.

Change-Id: Id52558b914f160d9c8578fdd7fc8199a9598576a

7 files changed, 3 insertions, 232 deletions

diff --git a/compiler/llvm/llvm_compilation_unit.cc b/compiler/llvm/llvm_compilation_unit.cc
index aa439ccbae..feb495e35f 100644
--- a/compiler/llvm/llvm_compilation_unit.cc
+++ b/compiler/llvm/llvm_compilation_unit.cc
@@ -82,7 +82,6 @@
 #include "ir_builder.h"
 #include "os.h"
 #include "runtime_support_builder_arm.h"
-#include "runtime_support_builder_thumb2.h"
 #include "runtime_support_builder_x86.h"
 #include "utils_llvm.h"
 
@@ -118,12 +117,10 @@ LlvmCompilationUnit::LlvmCompilationUnit(const CompilerLLVM* compiler_llvm, size
   default:
     runtime_support_.reset(new RuntimeSupportBuilder(*context_, *module_, *irb_));
     break;
+  case kThumb2:
   case kArm:
     runtime_support_.reset(new RuntimeSupportBuilderARM(*context_, *module_, *irb_));
     break;
-  case kThumb2:
-    runtime_support_.reset(new RuntimeSupportBuilderThumb2(*context_, *module_, *irb_));
-    break;
   case kX86:
     runtime_support_.reset(new RuntimeSupportBuilderX86(*context_, *module_, *irb_));
     break;
diff --git a/compiler/llvm/runtime_support_builder.cc b/compiler/llvm/runtime_support_builder.cc
index 24e283d309..c825fbf190 100644
--- a/compiler/llvm/runtime_support_builder.cc
+++ b/compiler/llvm/runtime_support_builder.cc
@@ -164,89 +164,13 @@ void RuntimeSupportBuilder::EmitTestSuspend() {
 /* Monitor */
 
 void RuntimeSupportBuilder::EmitLockObject(::llvm::Value* object) {
-  Value* monitor =
-      irb_.LoadFromObjectOffset(object,
-                                mirror::Object::MonitorOffset().Int32Value(),
-                                irb_.getJIntTy(),
-                                kTBAARuntimeInfo);
-
-  Value* real_monitor =
-      irb_.CreateAnd(monitor, ~(LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
-
-  // Is thin lock, unheld and not recursively acquired.
-  Value* unheld = irb_.CreateICmpEQ(real_monitor, irb_.getInt32(0));
-
-  Function* parent_func = irb_.GetInsertBlock()->getParent();
-  BasicBlock* bb_fast = BasicBlock::Create(context_, "lock_fast", parent_func);
-  BasicBlock* bb_slow = BasicBlock::Create(context_, "lock_slow", parent_func);
-  BasicBlock* bb_cont = BasicBlock::Create(context_, "lock_cont", parent_func);
-  irb_.CreateCondBr(unheld, bb_fast, bb_slow, kLikely);
-
-  irb_.SetInsertPoint(bb_fast);
-
-  // Calculate new monitor: new = old | (lock_id << LW_LOCK_OWNER_SHIFT)
-  Value* lock_id =
-      EmitLoadFromThreadOffset(Thread::ThinLockIdOffset().Int32Value(),
-                               irb_.getInt32Ty(), kTBAARuntimeInfo);
-
-  Value* owner = irb_.CreateShl(lock_id, LW_LOCK_OWNER_SHIFT);
-  Value* new_monitor = irb_.CreateOr(monitor, owner);
-
-  // Atomically update monitor.
-  Value* old_monitor =
-      irb_.CompareExchangeObjectOffset(object,
-                                       mirror::Object::MonitorOffset().Int32Value(),
-                                       monitor, new_monitor, kTBAARuntimeInfo);
-
-  Value* retry_slow_path = irb_.CreateICmpEQ(old_monitor, monitor);
-  irb_.CreateCondBr(retry_slow_path, bb_cont, bb_slow, kLikely);
-
-  irb_.SetInsertPoint(bb_slow);
   Function* slow_func = GetRuntimeSupportFunction(runtime_support::LockObject);
   irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
-  irb_.CreateBr(bb_cont);
-
-  irb_.SetInsertPoint(bb_cont);
 }
 
 void RuntimeSupportBuilder::EmitUnlockObject(::llvm::Value* object) {
-  Value* lock_id =
-      EmitLoadFromThreadOffset(Thread::ThinLockIdOffset().Int32Value(),
-                               irb_.getJIntTy(),
-                               kTBAARuntimeInfo);
-  Value* monitor =
-      irb_.LoadFromObjectOffset(object,
-                                mirror::Object::MonitorOffset().Int32Value(),
-                                irb_.getJIntTy(),
-                                kTBAARuntimeInfo);
-
-  Value* my_monitor = irb_.CreateShl(lock_id, LW_LOCK_OWNER_SHIFT);
-  Value* hash_state = irb_.CreateAnd(monitor, (LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
-  Value* real_monitor = irb_.CreateAnd(monitor, ~(LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
-
-  // Is thin lock, held by us and not recursively acquired
-  Value* is_fast_path = irb_.CreateICmpEQ(real_monitor, my_monitor);
-
-  Function* parent_func = irb_.GetInsertBlock()->getParent();
-  BasicBlock* bb_fast = BasicBlock::Create(context_, "unlock_fast", parent_func);
-  BasicBlock* bb_slow = BasicBlock::Create(context_, "unlock_slow", parent_func);
-  BasicBlock* bb_cont = BasicBlock::Create(context_, "unlock_cont", parent_func);
-  irb_.CreateCondBr(is_fast_path, bb_fast, bb_slow, kLikely);
-
-  irb_.SetInsertPoint(bb_fast);
-  // Set all bits to zero (except hash state)
-  irb_.StoreToObjectOffset(object,
-                           mirror::Object::MonitorOffset().Int32Value(),
-                           hash_state,
-                           kTBAARuntimeInfo);
-  irb_.CreateBr(bb_cont);
-
-  irb_.SetInsertPoint(bb_slow);
   Function* slow_func = GetRuntimeSupportFunction(runtime_support::UnlockObject);
   irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
-  irb_.CreateBr(bb_cont);
-
-  irb_.SetInsertPoint(bb_cont);
 }
 
 
diff --git a/compiler/llvm/runtime_support_builder.h b/compiler/llvm/runtime_support_builder.h
index e92ac0a908..898611af75 100644
--- a/compiler/llvm/runtime_support_builder.h
+++ b/compiler/llvm/runtime_support_builder.h
@@ -64,8 +64,8 @@ class RuntimeSupportBuilder {
   virtual void EmitTestSuspend();
 
   /* Monitor */
-  virtual void EmitLockObject(::llvm::Value* object);
-  virtual void EmitUnlockObject(::llvm::Value* object);
+  void EmitLockObject(::llvm::Value* object);
+  void EmitUnlockObject(::llvm::Value* object);
 
   /* MarkGCCard */
   virtual void EmitMarkGCCard(::llvm::Value* value, ::llvm::Value* target_addr);
diff --git a/compiler/llvm/runtime_support_builder_arm.cc b/compiler/llvm/runtime_support_builder_arm.cc
index 569d825272..cad46247fd 100644
--- a/compiler/llvm/runtime_support_builder_arm.cc
+++ b/compiler/llvm/runtime_support_builder_arm.cc
@@ -116,24 +116,5 @@ Value* RuntimeSupportBuilderARM::EmitSetCurrentThread(Value* thread) {
   return old_thread_register;
 }
 
-
-/* Monitor */
-
-void RuntimeSupportBuilderARM::EmitLockObject(Value* object) {
-  RuntimeSupportBuilder::EmitLockObject(object);
-  FunctionType* func_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
-                                            /*isVarArg=*/false);
-  InlineAsm* func = InlineAsm::get(func_ty, "dmb sy", "", true);
-  irb_.CreateCall(func);
-}
-
-void RuntimeSupportBuilderARM::EmitUnlockObject(Value* object) {
-  RuntimeSupportBuilder::EmitUnlockObject(object);
-  FunctionType* func_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
-                                            /*isVarArg=*/false);
-  InlineAsm* func = InlineAsm::get(func_ty, "dmb sy", "", true);
-  irb_.CreateCall(func);
-}
-
 }  // namespace llvm
 }  // namespace art
diff --git a/compiler/llvm/runtime_support_builder_arm.h b/compiler/llvm/runtime_support_builder_arm.h
index 5a353d7f30..0d01509be0 100644
--- a/compiler/llvm/runtime_support_builder_arm.h
+++ b/compiler/llvm/runtime_support_builder_arm.h
@@ -34,10 +34,6 @@ class RuntimeSupportBuilderARM : public RuntimeSupportBuilder {
   virtual void EmitStoreToThreadOffset(int64_t offset, ::llvm::Value* value,
                                        TBAASpecialType s_ty);
   virtual ::llvm::Value* EmitSetCurrentThread(::llvm::Value* thread);
-
-  /* Monitor */
-  virtual void EmitLockObject(::llvm::Value* object);
-  virtual void EmitUnlockObject(::llvm::Value* object);
 };
 
 }  // namespace llvm
diff --git a/compiler/llvm/runtime_support_builder_thumb2.cc b/compiler/llvm/runtime_support_builder_thumb2.cc
deleted file mode 100644
index eff29c8b04..0000000000
--- a/compiler/llvm/runtime_support_builder_thumb2.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "runtime_support_builder_thumb2.h"
-
-#include "ir_builder.h"
-#include "mirror/object.h"
-#include "monitor.h"
-#include "thread.h"
-#include "utils_llvm.h"
-
-#include <llvm/IR/DerivedTypes.h>
-#include <llvm/IR/Function.h>
-#include <llvm/IR/InlineAsm.h>
-#include <llvm/IR/Module.h>
-#include <llvm/IR/Type.h>
-
-#include <inttypes.h>
-#include <vector>
-
-using ::llvm::BasicBlock;
-using ::llvm::Function;
-using ::llvm::FunctionType;
-using ::llvm::InlineAsm;
-using ::llvm::Type;
-using ::llvm::Value;
-
-namespace art {
-namespace llvm {
-
-
-void RuntimeSupportBuilderThumb2::EmitLockObject(Value* object) {
-  FunctionType* func_ty = FunctionType::get(/*Result=*/irb_.getInt32Ty(),
-                                            /*Params=*/irb_.getJObjectTy(),
-                                            /*isVarArg=*/false);
-  // $0: result
-  // $1: object
-  // $2: temp
-  // $3: temp
-  std::string asms;
-  StringAppendF(&asms, "add $3, $1, #%" PRId32 "\n", mirror::Object::MonitorOffset().Int32Value());
-  StringAppendF(&asms, "ldr $2, [r9, #%" PRId32 "]\n", Thread::ThinLockIdOffset().Int32Value());
-  StringAppendF(&asms, "ldrex $0, [$3]\n");
-  StringAppendF(&asms, "lsl $2, $2, %d\n", LW_LOCK_OWNER_SHIFT);
-  StringAppendF(&asms, "bfi $2, $0, #0, #%d\n", LW_LOCK_OWNER_SHIFT - 1);
-  StringAppendF(&asms, "bfc $0, #%d, #%d\n", LW_HASH_STATE_SHIFT, LW_LOCK_OWNER_SHIFT - 1);
-  StringAppendF(&asms, "cmp $0, #0\n");
-  StringAppendF(&asms, "it eq\n");
-  StringAppendF(&asms, "strexeq $0, $2, [$3]\n");
-
-  InlineAsm* func = InlineAsm::get(func_ty, asms, "=&l,l,~l,~l", true);
-
-  Value* retry_slow_path = irb_.CreateCall(func, object);
-  retry_slow_path = irb_.CreateICmpNE(retry_slow_path, irb_.getJInt(0));
-
-  Function* parent_func = irb_.GetInsertBlock()->getParent();
-  BasicBlock* basic_block_lock = BasicBlock::Create(context_, "lock", parent_func);
-  BasicBlock* basic_block_cont = BasicBlock::Create(context_, "lock_cont", parent_func);
-  irb_.CreateCondBr(retry_slow_path, basic_block_lock, basic_block_cont, kUnlikely);
-
-  irb_.SetInsertPoint(basic_block_lock);
-  Function* slow_func = GetRuntimeSupportFunction(runtime_support::LockObject);
-  irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
-  irb_.CreateBr(basic_block_cont);
-
-  irb_.SetInsertPoint(basic_block_cont);
-  {  // Memory barrier
-    FunctionType* asm_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
-                                              /*isVarArg=*/false);
-    InlineAsm* func = InlineAsm::get(asm_ty, "dmb sy", "", true);
-    irb_.CreateCall(func);
-  }
-}
-
-
-}  // namespace llvm
-}  // namespace art
diff --git a/compiler/llvm/runtime_support_builder_thumb2.h b/compiler/llvm/runtime_support_builder_thumb2.h
deleted file mode 100644
index c47a2744ef..0000000000
--- a/compiler/llvm/runtime_support_builder_thumb2.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_
-#define ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_
-
-#include "runtime_support_builder_arm.h"
-
-namespace art {
-namespace llvm {
-
-class RuntimeSupportBuilderThumb2 : public RuntimeSupportBuilderARM {
- public:
-  RuntimeSupportBuilderThumb2(::llvm::LLVMContext& context, ::llvm::Module& module, IRBuilder& irb)
-    : RuntimeSupportBuilderARM(context, module, irb) {}
-
-  /* Monitor */
-  virtual void EmitLockObject(::llvm::Value* object);
-};
-
-}  // namespace llvm
-}  // namespace art
-
-#endif  // ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_