Wait for threads to finish unregistering

There was a race where Thread::join would return before the thread was
unregistered. This caused a problem with Daemons.stop since the thread
list could get deleted before the daemon thread was removed from list_.
This caused occasional "Request to unregister unattached thread"
errors and warnings.

The fix is to wait until threads finish registering before destroying
the thread list. The only threads which can be unregistering at this
point are the daemons we stopped earlier during the runtime shutdown
process. The issue is that thread join finishes before we remove the
thread from the thread list.

Also some cleanup.

Bug: 18713034
Change-Id: I8921122fe8462643a6b814b5f00632481e3831fb

2 files changed, 10 insertions, 1 deletions

diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc
index 6e00cc79b5..13dcb8c634 100644
--- a/runtime/base/mutex.cc
+++ b/runtime/base/mutex.cc
@@ -57,6 +57,7 @@ Mutex* Locks::reference_queue_soft_references_lock_ = nullptr;
 Mutex* Locks::reference_queue_weak_references_lock_ = nullptr;
 Mutex* Locks::runtime_shutdown_lock_ = nullptr;
 Mutex* Locks::thread_list_lock_ = nullptr;
+ConditionVariable* Locks::thread_exit_cond_ = nullptr;
 Mutex* Locks::thread_suspend_count_lock_ = nullptr;
 Mutex* Locks::trace_lock_ = nullptr;
 Mutex* Locks::unexpected_signal_lock_ = nullptr;
@@ -1063,8 +1064,13 @@ void Locks::Init() {
     logging_lock_ = new Mutex("logging lock", current_lock_level, true);
 
     #undef UPDATE_CURRENT_LOCK_LEVEL
+
+    InitConditions();
   }
 }
 
+void Locks::InitConditions() {
+  thread_exit_cond_ = new ConditionVariable("thread exit condition variable", *thread_list_lock_);
+}
 
 }  // namespace art
diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h
index 45d2347ee2..3b052c0615 100644
--- a/runtime/base/mutex.h
+++ b/runtime/base/mutex.h
@@ -487,7 +487,7 @@ class SCOPED_LOCKABLE WriterMutexLock {
 class Locks {
  public:
   static void Init();
-
+  static void InitConditions() NO_THREAD_SAFETY_ANALYSIS;  // Condition variables.
   // Guards allocation entrypoint instrumenting.
   static Mutex* instrument_entrypoints_lock_;
 
@@ -575,6 +575,9 @@ class Locks {
   // attaching and detaching.
   static Mutex* thread_list_lock_ ACQUIRED_AFTER(deoptimization_lock_);
 
+  // Signaled when threads terminate. Used to determine when all non-daemons have terminated.
+  static ConditionVariable* thread_exit_cond_ GUARDED_BY(Locks::thread_list_lock_);
+
   // Guards maintaining loading library data structures.
   static Mutex* jni_libraries_lock_ ACQUIRED_AFTER(thread_list_lock_);