Further updates to jemalloc code. am: c6954b2064 am: e1ad3f0e8b

am: 3bff253f72

Change-Id: I46c6e93a753024d4b9f31df3c4220d0fd5be21b1

10 files changed, 113 insertions, 21 deletions

diff --git a/Android.bp b/Android.bp
index 6031da43..bee6efa3 100644
--- a/Android.bp
+++ b/Android.bp
@@ -66,7 +66,8 @@ android_common_cflags = [
     // These parameters will be overridden by android_product_variables
     // for non-svelte configs.
     "-DANDROID_NUM_ARENAS=1",
-    "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=1",
+    // This value cannot go below 2.
+    "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=2",
     "-DANDROID_TCACHE_NSLOTS_LARGE=1",
 ]
 
@@ -74,6 +75,8 @@ android_product_variables = {
     // Only enable the tcache on non-svelte configurations, to save PSS.
     malloc_not_svelte: {
         cflags: [
+            "-DANDROID_ENABLE_TCACHE",
+
             "-UANDROID_NUM_ARENAS",
             "-DANDROID_NUM_ARENAS=2",
 
diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h
index 29948a2c..15886468 100644
--- a/include/jemalloc/internal/jemalloc_internal_defs.h
+++ b/include/jemalloc/internal/jemalloc_internal_defs.h
@@ -1,4 +1,6 @@
-#if !defined(__ANDROID__)
+/* Include cdefs to see if __BIONIC__ is set */
+#include <sys/cdefs.h>
+#if !defined(__BIONIC__)
 #include "jemalloc_internal_defs_host.h"
 #else
 /* include/jemalloc/internal/jemalloc_internal_defs.h.  Generated from jemalloc_internal_defs.h.in by configure.  */
@@ -297,12 +299,13 @@
  *                                 MADV_FREE, though typically with higher
  *                                 system overhead.
  */
-#define JEMALLOC_PURGE_MADVISE_FREE 
+/* MADV_FREE available since kernel 4.5 but not all devices support this yet. */
+/* #undef JEMALLOC_PURGE_MADVISE_FREE */
 #define JEMALLOC_PURGE_MADVISE_DONTNEED 
 #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS 
 
 /* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */
-/* #undef JEMALLOC_DEFINE_MADVISE_FREE */
+#define JEMALLOC_DEFINE_MADVISE_FREE 
 
 /*
  * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise.
diff --git a/include/jemalloc/internal/jemalloc_internal_defs_host.h b/include/jemalloc/internal/jemalloc_internal_defs_host.h
index 480a8356..38f91bc2 100644
--- a/include/jemalloc/internal/jemalloc_internal_defs_host.h
+++ b/include/jemalloc/internal/jemalloc_internal_defs_host.h
@@ -334,7 +334,9 @@
 #define LG_SIZEOF_INTMAX_T 3
 
 /* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */
+#if !defined(__ANDROID__)
 #define JEMALLOC_GLIBC_MALLOC_HOOK 
+#endif
 
 /* glibc memalign hook. */
 #define JEMALLOC_GLIBC_MEMALIGN_HOOK 
diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h
index b59d33a8..2124b095 100644
--- a/include/jemalloc/internal/rtree.h
+++ b/include/jemalloc/internal/rtree.h
@@ -336,7 +336,12 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	/* Fast path: L1 direct mapped cache. */
 	if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
 		rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
-		assert(leaf != NULL);
+		/* ANDROID CHANGE: Bad pointers return NULL */
+		/* assert(leaf != NULL); */
+		if (leaf == NULL) {
+			return NULL;
+		}
+		/* ANDROID END CHANGE */
 		uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);
 		return &leaf[subkey];
 	}
@@ -347,7 +352,12 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 #define RTREE_CACHE_CHECK_L2(i) do {					\
 	if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) {	\
 		rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf;	\
-		assert(leaf != NULL);					\
+		/* ANDROID CHANGE: Bad pointers return NULL */		\
+		/* assert(leaf != NULL); */				\
+		if (leaf == NULL) {					\
+			return NULL;					\
+		}							\
+		/* ANDROID END CHANGE */				\
 		if (i > 0) {						\
 			/* Bubble up by one. */				\
 			rtree_ctx->l2_cache[i].leafkey =		\
@@ -405,7 +415,10 @@ rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
     bool dependent) {
 	rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
 	    key, dependent, false);
-	if (!dependent && elm == NULL) {
+	/* ANDROID CHANGE: Bad pointers return NULL */
+	/* if (!dependent && elm == NULL) { */
+	if (elm == NULL) {
+	/* ANDROID END CHANGE */
 		return NULL;
 	}
 	assert(elm != NULL);
@@ -417,7 +430,10 @@ rtree_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
     uintptr_t key, bool dependent) {
 	rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
 	    dependent);
-	if (!dependent && elm == NULL) {
+	/* ANDROID CHANGE: Bad pointers return NULL */
+	/* if (!dependent && elm == NULL) { */
+	if (elm == NULL) {
+	/* ANDROID END CHANGE */
 		return NULL;
 	}
 	return rtree_leaf_elm_extent_read(tsdn, rtree, elm, dependent);
@@ -428,7 +444,9 @@ rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
     uintptr_t key, bool dependent) {
 	rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
 	    dependent);
-	if (!dependent && elm == NULL) {
+	/* ANDROID CHANGE: Bad pointers return NULL */
+	/* if (!dependent && elm == NULL) { */
+	if (elm == NULL) {
 		return NSIZES;
 	}
 	return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent);
@@ -444,7 +462,10 @@ rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
     uintptr_t key, bool dependent, extent_t **r_extent, szind_t *r_szind) {
 	rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
 	    dependent);
-	if (!dependent && elm == NULL) {
+	/* ANDROID CHANGE: Bad pointers return NULL */
+	/* if (!dependent && elm == NULL) { */
+	if (elm == NULL) {
+	/* ANDROID END CHANGE */
 		return true;
 	}
 	*r_extent = rtree_leaf_elm_extent_read(tsdn, rtree, elm, dependent);
@@ -457,7 +478,10 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
     uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) {
 	rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
 	    dependent);
-	if (!dependent && elm == NULL) {
+	/* ANDROID CHANGE: Bad pointers return NULL */
+	/* if (!dependent && elm == NULL) { */
+	if (elm == NULL) {
+	/* ANDROID END CHANGE */
 		return true;
 	}
 #ifdef RTREE_LEAF_COMPACT
diff --git a/include/jemalloc/jemalloc_rename.h b/include/jemalloc/jemalloc_rename.h
index c5cefd8d..2784243e 100644
--- a/include/jemalloc/jemalloc_rename.h
+++ b/include/jemalloc/jemalloc_rename.h
@@ -4,7 +4,7 @@
  * these macro definitions.
  */
 #ifndef JEMALLOC_NO_RENAME
-#if defined(__ANDROID__)
+#if defined(__BIONIC__)
 #  define je_aligned_alloc je_aligned_alloc
 #  define je_calloc je_calloc
 #  define je_dallocx je_dallocx
diff --git a/src/android_je_iterate.c b/src/android_je_iterate.c
index 3705cd63..0702f338 100644
--- a/src/android_je_iterate.c
+++ b/src/android_je_iterate.c
@@ -19,8 +19,54 @@ static bool malloc_disabled_tcache;
 
 int je_iterate(uintptr_t base, size_t size,
     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) {
-  // TODO: Figure out how to implement this functionality for jemalloc5.
-  return -1;
+  size_t pagesize = getpagesize();
+  tsd_t* tsd = tsd_fetch_min();
+  rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
+
+  // Make sure the pointer is aligned to at least 8 bytes.
+  uintptr_t ptr = (base + 7) & ~7;
+  uintptr_t end_ptr = ptr + size;
+  while (ptr < end_ptr) {
+    extent_t* extent = iealloc(tsd_tsdn(tsd), (void*)ptr);
+    if (extent == NULL) {
+      // Skip to the next page, guaranteed no other pointers on this page.
+      ptr += pagesize;
+      continue;
+    }
+
+    szind_t szind;
+    bool slab;
+    rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, ptr, true, &szind, &slab);
+    if (slab) {
+      // Small allocation.
+      szind_t binind = extent_szind_get(extent);
+      const bin_info_t* bin_info = &bin_infos[binind];
+      arena_slab_data_t* slab_data = extent_slab_data_get(extent);
+
+      uintptr_t first_ptr = (uintptr_t)extent_addr_get(extent);
+      size_t bin_size = bin_info->reg_size;
+      // Align the pointer to the bin size.
+      ptr = (ptr + bin_size - 1) & ~(bin_size - 1);
+      for (size_t bit = (ptr - first_ptr) / bin_size; bit < bin_info->bitmap_info.nbits; bit++) {
+        if (bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, bit)) {
+          uintptr_t allocated_ptr = first_ptr + bin_size * bit;
+          if (allocated_ptr >= end_ptr) {
+            break;
+          }
+          callback(allocated_ptr, bin_size, arg);
+        }
+      }
+    } else if (extent_state_get(extent) == extent_state_active) {
+      // Large allocation.
+      uintptr_t base_ptr = (uintptr_t)extent_addr_get(extent);
+      if (ptr <= base_ptr) {
+        // This extent is actually allocated and within the range to check.
+        callback(base_ptr, extent_usize_get(extent), arg);
+      }
+    }
+    ptr = (uintptr_t)extent_past_get(extent);
+  }
+  return 0;
 }
 
 static void je_malloc_disable_prefork() {
@@ -51,6 +97,9 @@ void je_malloc_disable() {
   pthread_mutex_lock(&malloc_disabled_lock);
   bool new_tcache = false;
   size_t old_len = sizeof(malloc_disabled_tcache);
+
+  // Disable the tcache (if not already disabled) so that we don't
+  // have to search the tcache for pointers.
   je_mallctl("thread.tcache.enabled",
       &malloc_disabled_tcache, &old_len,
       &new_tcache, sizeof(new_tcache));
@@ -60,6 +109,7 @@ void je_malloc_disable() {
 void je_malloc_enable() {
   jemalloc_postfork_parent();
   if (malloc_disabled_tcache) {
+    // Re-enable the tcache if it was enabled before the disabled call.
     je_mallctl("thread.tcache.enabled", NULL, NULL,
         &malloc_disabled_tcache, sizeof(malloc_disabled_tcache));
   }
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 9f4df5e7..0584362f 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -875,7 +875,7 @@ malloc_conf_init(void) {
 	const char *opts, *k, *v;
 	size_t klen, vlen;
 
-#if defined(__ANDROID__)
+#if defined(__BIONIC__)
 	/* For Android, do not look at files nor environment variables for
 	 * config data.
 	 */
@@ -1353,7 +1353,7 @@ static bool
 malloc_init_hard_recursible(void) {
 	malloc_init_state = malloc_init_recursible;
 
-#if defined(__ANDROID__) && defined(ANDROID_NUM_ARENAS)
+#if defined(__BIONIC__) && defined(ANDROID_NUM_ARENAS)
 	/* Hardcode since this value won't be used. */ 
 	ncpus = 2;
 #else
@@ -3341,7 +3341,7 @@ jemalloc_postfork_child(void) {
 
 /******************************************************************************/
 
-#if defined(__ANDROID__) && !defined(JEMALLOC_JET)
+#if defined(__BIONIC__) && !defined(JEMALLOC_JET)
 #include "android_je_iterate.c"
 #include "android_je_mallinfo.c"
 #endif
diff --git a/src/large.c b/src/large.c
index cbffd99b..4ea7f7a7 100644
--- a/src/large.c
+++ b/src/large.c
@@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	 */
 	is_zeroed = zero;
 	if (likely(!tsdn_null(tsdn))) {
-#if defined(__ANDROID__) && !defined(__LP64__)
+#if defined(__BIONIC__) && !defined(__LP64__) && !defined(JEMALLOC_JET) && !defined(JEMALLOC_INTEGRATION_TEST)
 		/* On 32 bit systems, using a per arena cache can exhaust
 		 * virtual address space. Force all huge allocations to
 		 * always take place in the first arena.
diff --git a/src/rtree.c b/src/rtree.c
index 53702cf7..01b6e80b 100644
--- a/src/rtree.c
+++ b/src/rtree.c
@@ -242,8 +242,11 @@ rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 
 #define RTREE_GET_CHILD(level) {					\
 		assert(level < RTREE_HEIGHT-1);				\
-		if (level != 0 && !dependent &&				\
-		    unlikely(!rtree_node_valid(node))) {		\
+		/* ANDROID CHANGE: Bad pointers return NULL */		\
+		/* if (level != 0 && !dependent && */			\
+		/*    unlikely(!rtree_node_valid(node))) { */		\
+		if (unlikely(!rtree_node_valid(node))) {		\
+		/* ANDROID END CHANGE */				\
 			return NULL;					\
 		}							\
 		uintptr_t subkey = rtree_subkey(key, level);		\
@@ -268,7 +271,10 @@ rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	 */
 #define RTREE_GET_LEAF(level) {						\
 		assert(level == RTREE_HEIGHT-1);			\
-		if (!dependent && unlikely(!rtree_leaf_valid(leaf))) {	\
+		/* ANDROID CHANGE: Bad pointers return NULL */		\
+		/* if (!dependent && unlikely(!rtree_leaf_valid(leaf))) {*/	\
+		if (unlikely(!rtree_leaf_valid(leaf))) {		\
+		/* ANDROID END CHANGE */				\
 			return NULL;					\
 		}							\
 		if (RTREE_CTX_NCACHE_L2 > 1) {				\
diff --git a/src/tcache.c b/src/tcache.c
index a769a6b1..b4320e42 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -9,7 +9,11 @@
 /******************************************************************************/
 /* Data. */
 
+#if !defined(__BIONIC__) || defined(ANDROID_ENABLE_TCACHE)
 bool	opt_tcache = true;
+#else
+bool	opt_tcache = false;
+#endif
 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
 
 cache_bin_info_t	*tcache_bin_info;