Use try lock to fix class resolution race

There was some possible deadlocks related to EnsureResolved caused by
acquiring an object lock.

Scenario:
Thread 1 acquires lock on obj1
Thread 1 begins to resolve / initialize class1
Thread 1 blocks since it sees that class1 is already being resolved and
gets preempted before it can acquire the object lock on class1
Thread 2 finishes resolving and initializing class1 and locks class1
Thread 2 blocks attempting to lock obj1
Thread 1 blocks attempting to lock class1
Deadlock

Fixed the deadlock by changing EnsureResolved to use a try lock for the
unresolved case.

Added a test.

Test: Device boot, test-art-host, monitor_test

Bug: 27417671

(cherry picked from commit a704eda0078989a73cac111ed309aca50d2e289b)

Change-Id: I1150b19bdc1a5cc87ae95eda4f2b6b4bca215a60

1 files changed, 23 insertions, 10 deletions

diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 1914733b7c..deaeb70342 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -2179,20 +2179,33 @@ mirror::Class* ClassLinker::EnsureResolved(Thread* self,
   }
 
   // Wait for the class if it has not already been linked.
-  if (!klass->IsResolved() && !klass->IsErroneous()) {
+  size_t index = 0;
+  // Maximum number of yield iterations until we start sleeping.
+  static const size_t kNumYieldIterations = 1000;
+  // How long each sleep is in us.
+  static const size_t kSleepDurationUS = 1000;  // 1 ms.
+  while (!klass->IsResolved() && !klass->IsErroneous()) {
     StackHandleScope<1> hs(self);
     HandleWrapper<mirror::Class> h_class(hs.NewHandleWrapper(&klass));
-    ObjectLock<mirror::Class> lock(self, h_class);
-    // Check for circular dependencies between classes.
-    if (!h_class->IsResolved() && h_class->GetClinitThreadId() == self->GetTid()) {
-      ThrowClassCircularityError(h_class.Get());
-      mirror::Class::SetStatus(h_class, mirror::Class::kStatusError, self);
-      return nullptr;
+    {
+      ObjectTryLock<mirror::Class> lock(self, h_class);
+      // Can not use a monitor wait here since it may block when returning and deadlock if another
+      // thread has locked klass.
+      if (lock.Acquired()) {
+        // Check for circular dependencies between classes, the lock is required for SetStatus.
+        if (!h_class->IsResolved() && h_class->GetClinitThreadId() == self->GetTid()) {
+          ThrowClassCircularityError(h_class.Get());
+          mirror::Class::SetStatus(h_class, mirror::Class::kStatusError, self);
+          return nullptr;
+        }
+      }
     }
-    // Wait for the pending initialization to complete.
-    while (!h_class->IsResolved() && !h_class->IsErroneous()) {
-      lock.WaitIgnoringInterrupts();
+    if (index < kNumYieldIterations) {
+      sched_yield();
+    } else {
+      usleep(kSleepDurationUS);
     }
+    ++index;
   }
 
   if (klass->IsErroneous()) {