Implement dynamic TLS accesses and allocation

Initialize a thread's DTV to an empty zeroed DTV. Allocate the DTV and
any ELF module's TLS segment on-demand in __tls_get_addr. Use a generation
counter, incremented in the linker, to signal when threads should
update/reallocate their DTV objects.

A generation count of 0 always indicates the constant zero DTV.

Once a DTV is allocated, it isn't freed until the thread exits, because
a signal handler could interrupt the fast path of __tls_get_addr between
accessing the DTV slot and reading a field of the DTV. Bionic keeps a
linked list of DTV objects so it can free them at thread-exit.

Dynamic TLS memory is allocated using a BionicAllocator instance in
libc_shared_globals. For async-signal safety, access to the
linker/libc-shared state is protected by first blocking signals, then by
acquiring the reader-writer lock, TlsModules::rwlock. A write lock is
needed to allocate or free memory.

In pthread_exit, unconditionally block signals before freeing dynamic
TLS memory or freeing the shadow call stack.

ndk_cruft.cpp: Avoid including pthread_internal.h inside an extern "C".
(The header now includes a C++ template that doesn't compile inside
extern "C".)

Bug: http://b/78026329
Bug: http://b/123094171
Test: bionic unit tests
Change-Id: I3c9b12921c9e68b33dcc1d1dd276bff364eff5d7

2 files changed, 21 insertions, 17 deletions

diff --git a/linker/linker_soinfo.h b/linker/linker_soinfo.h
index 14571de5b..3499cf742 100644
--- a/linker/linker_soinfo.h
+++ b/linker/linker_soinfo.h
@@ -35,6 +35,7 @@
 
 #include "private/bionic_elf_tls.h"
 #include "linker_namespaces.h"
+#include "linker_tls.h"
 
 #define FLAG_LINKED           0x00000001
 #define FLAG_EXE              0x00000004 // The main executable
@@ -102,14 +103,9 @@ struct version_info {
 // TODO(dimitry): remove reference from soinfo member functions to this class.
 class VersionTracker;
 
-// The first ELF TLS module has ID 1. Zero is reserved for the first word of
-// the DTV, a generation count, and unresolved weak symbols also use module
-// ID 0.
-static constexpr size_t kUninitializedModuleId = 0;
-
 struct soinfo_tls {
   TlsSegment segment;
-  size_t module_id = kUninitializedModuleId;
+  size_t module_id = kTlsUninitializedModuleId;
 };
 
 #if defined(__work_around_b_24465209__)
diff --git a/linker/linker_tls.cpp b/linker/linker_tls.cpp
index 0d1796b63..a3aa9bfac 100644
--- a/linker/linker_tls.cpp
+++ b/linker/linker_tls.cpp
@@ -31,6 +31,7 @@
 #include <vector>
 
 #include "private/ScopedRWLock.h"
+#include "private/ScopedSignalBlocker.h"
 #include "private/bionic_defs.h"
 #include "private/bionic_elf_tls.h"
 #include "private/bionic_globals.h"
@@ -41,9 +42,6 @@
 static bool g_static_tls_finished;
 static std::vector<TlsModule> g_tls_modules;
 
-static inline size_t module_id_to_idx(size_t id) { return id - 1; }
-static inline size_t module_idx_to_id(size_t idx) { return idx + 1; }
-
 static size_t get_unused_module_index() {
   for (size_t i = 0; i < g_tls_modules.size(); ++i) {
     if (g_tls_modules[i].soinfo_ptr == nullptr) {
@@ -57,37 +55,47 @@ static size_t get_unused_module_index() {
 }
 
 static void register_tls_module(soinfo* si, size_t static_offset) {
+  TlsModules& libc_modules = __libc_shared_globals()->tls_modules;
+
   // The global TLS module table points at the std::vector of modules declared
   // in this file, so acquire a write lock before modifying the std::vector.
-  ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
+  ScopedSignalBlocker ssb;
+  ScopedWriteLock locker(&libc_modules.rwlock);
 
   size_t module_idx = get_unused_module_index();
 
   soinfo_tls* si_tls = si->get_tls();
-  si_tls->module_id = module_idx_to_id(module_idx);
+  si_tls->module_id = __tls_module_idx_to_id(module_idx);
+
+  const size_t new_generation = ++libc_modules.generation;
+  __libc_tls_generation_copy = new_generation;
+  if (libc_modules.generation_libc_so != nullptr) {
+    *libc_modules.generation_libc_so = new_generation;
+  }
 
   g_tls_modules[module_idx] = {
     .segment = si_tls->segment,
     .static_offset = static_offset,
-    .first_generation = ++__libc_shared_globals()->tls_modules.generation,
+    .first_generation = new_generation,
     .soinfo_ptr = si,
   };
 }
 
 static void unregister_tls_module(soinfo* si) {
+  ScopedSignalBlocker ssb;
   ScopedWriteLock locker(&__libc_shared_globals()->tls_modules.rwlock);
 
   soinfo_tls* si_tls = si->get_tls();
-  TlsModule& mod = g_tls_modules[module_id_to_idx(si_tls->module_id)];
+  TlsModule& mod = g_tls_modules[__tls_module_id_to_idx(si_tls->module_id)];
   CHECK(mod.static_offset == SIZE_MAX);
   CHECK(mod.soinfo_ptr == si);
   mod = {};
-  si_tls->module_id = kUninitializedModuleId;
+  si_tls->module_id = kTlsUninitializedModuleId;
 }
 
 // The reference is valid until a TLS module is registered or unregistered.
 const TlsModule& get_tls_module(size_t module_id) {
-  size_t module_idx = module_id_to_idx(module_id);
+  size_t module_idx = __tls_module_id_to_idx(module_id);
   CHECK(module_idx < g_tls_modules.size());
   return g_tls_modules[module_idx];
 }
@@ -123,7 +131,7 @@ void linker_finalize_static_tls() {
 
 void register_soinfo_tls(soinfo* si) {
   soinfo_tls* si_tls = si->get_tls();
-  if (si_tls == nullptr || si_tls->module_id != kUninitializedModuleId) {
+  if (si_tls == nullptr || si_tls->module_id != kTlsUninitializedModuleId) {
     return;
   }
   size_t static_offset = SIZE_MAX;
@@ -136,7 +144,7 @@ void register_soinfo_tls(soinfo* si) {
 
 void unregister_soinfo_tls(soinfo* si) {
   soinfo_tls* si_tls = si->get_tls();
-  if (si_tls == nullptr || si_tls->module_id == kUninitializedModuleId) {
+  if (si_tls == nullptr || si_tls->module_id == kTlsUninitializedModuleId) {
     return;
   }
   return unregister_tls_module(si);