Add minimal initialized TSD.

We use the minimal_initilized tsd (which requires no cleanup) for free()
specifically, if tsd hasn't been initialized yet.

Any other activity will transit the state from minimal to normal.  This is to
workaround the case where a thread has no malloc calls in its lifetime until
during thread termination, free() happens after tls destructors.

1 files changed, 22 insertions, 8 deletions

diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h
index 631fbf1f..155a2ec6 100644
--- a/include/jemalloc/internal/tsd.h
+++ b/include/jemalloc/internal/tsd.h
@@ -99,9 +99,10 @@ enum {
 	tsd_state_nominal_slow = 1, /* Initialized but on slow path. */
 	/* the above 2 nominal states should be lower values. */
 	tsd_state_nominal_max = 1, /* used for comparison only. */
-	tsd_state_purgatory = 2,
-	tsd_state_reincarnated = 3,
-	tsd_state_uninitialized = 4
+	tsd_state_minimal_initialized = 2,
+	tsd_state_purgatory = 3,
+	tsd_state_reincarnated = 4,
+	tsd_state_uninitialized = 5
 };
 
 /* Manually limit tsd_state_t to a single byte. */
@@ -190,7 +191,8 @@ JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd) {						\
 	assert(tsd->state == tsd_state_nominal ||			\
 	    tsd->state == tsd_state_nominal_slow ||			\
-	    tsd->state == tsd_state_reincarnated);			\
+	    tsd->state == tsd_state_reincarnated ||			\
+	    tsd->state == tsd_state_minimal_initialized);		\
 	return tsd_##n##p_get_unsafe(tsd);				\
 }
 MALLOC_TSD
@@ -225,7 +227,8 @@ MALLOC_TSD
 #define O(n, t, nt)							\
 JEMALLOC_ALWAYS_INLINE void						\
 tsd_##n##_set(tsd_t *tsd, t val) {					\
-	assert(tsd->state != tsd_state_reincarnated);			\
+	assert(tsd->state != tsd_state_reincarnated &&			\
+	    tsd->state != tsd_state_minimal_initialized);		\
 	*tsd_##n##p_get(tsd) = val;					\
 }
 MALLOC_TSD
@@ -248,7 +251,7 @@ tsd_fast(tsd_t *tsd) {
 }
 
 JEMALLOC_ALWAYS_INLINE tsd_t *
-tsd_fetch_impl(bool init, bool internal) {
+tsd_fetch_impl(bool init, bool minimal) {
 	tsd_t *tsd = tsd_get(init);
 
 	if (!init && tsd_get_allocates() && tsd == NULL) {
@@ -257,7 +260,7 @@ tsd_fetch_impl(bool init, bool internal) {
 	assert(tsd != NULL);
 
 	if (unlikely(tsd->state != tsd_state_nominal)) {
-		return tsd_fetch_slow(tsd, internal);
+		return tsd_fetch_slow(tsd, minimal);
 	}
 	assert(tsd_fast(tsd));
 	tsd_assert_fast(tsd);
@@ -265,11 +268,22 @@ tsd_fetch_impl(bool init, bool internal) {
 	return tsd;
 }
 
+/* Get a minimal TSD that requires no cleanup.  See comments in free(). */
 JEMALLOC_ALWAYS_INLINE tsd_t *
-tsd_internal_fetch(void) {
+tsd_fetch_min(void) {
 	return tsd_fetch_impl(true, true);
 }
 
+/* For internal background threads use only. */
+JEMALLOC_ALWAYS_INLINE tsd_t *
+tsd_internal_fetch(void) {
+	tsd_t *tsd = tsd_fetch_min();
+	/* Use reincarnated state to prevent full initialization. */
+	tsd->state = tsd_state_reincarnated;
+
+	return tsd;
+}
+
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_fetch(void) {
 	return tsd_fetch_impl(true, false);