Move arena_slab_data_t's nfree into extent_t's e_bits.

Compact extent_t to 128 bytes on 64-bit systems by moving
arena_slab_data_t's nfree into extent_t's e_bits.

Cacheline-align extent_t structures so that they always cross the
minimum number of cacheline boundaries.

Re-order extent_t fields such that all fields except the slab bitmap
(and overlaid heap profiling context pointer) are in the first
cacheline.

This resolves #461.

4 files changed, 66 insertions, 23 deletions

diff --git a/include/jemalloc/internal/arena_structs_a.h b/include/jemalloc/internal/arena_structs_a.h
index ed265b20..07013a06 100644
--- a/include/jemalloc/internal/arena_structs_a.h
+++ b/include/jemalloc/internal/arena_structs_a.h
@@ -2,9 +2,6 @@
 #define JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H
 
 struct arena_slab_data_s {
-	/* Number of free regions in slab. */
-	unsigned	nfree;
-
 	/* Per region allocated/deallocated bitmap. */
 	bitmap_t	bitmap[BITMAP_GROUPS_MAX];
 };
diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
index 22229b50..99fa67c7 100644
--- a/include/jemalloc/internal/extent_inlines.h
+++ b/include/jemalloc/internal/extent_inlines.h
@@ -11,6 +11,7 @@ extent_state_t extent_state_get(const extent_t *extent);
 bool extent_zeroed_get(const extent_t *extent);
 bool extent_committed_get(const extent_t *extent);
 bool extent_slab_get(const extent_t *extent);
+unsigned extent_nfree_get(const extent_t *extent);
 void *extent_base_get(const extent_t *extent);
 void *extent_addr_get(const extent_t *extent);
 size_t extent_size_get(const extent_t *extent);
@@ -25,6 +26,9 @@ void extent_addr_set(extent_t *extent, void *addr);
 void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
 void extent_size_set(extent_t *extent, size_t size);
 void extent_szind_set(extent_t *extent, szind_t szind);
+void extent_nfree_set(extent_t *extent, unsigned nfree);
+void extent_nfree_inc(extent_t *extent);
+void extent_nfree_dec(extent_t *extent);
 void extent_sn_set(extent_t *extent, size_t sn);
 void extent_state_set(extent_t *extent, extent_state_t state);
 void extent_zeroed_set(extent_t *extent, bool zeroed);
@@ -112,6 +116,13 @@ extent_slab_get(const extent_t *extent) {
 	    EXTENT_BITS_SLAB_SHIFT);
 }
 
+JEMALLOC_INLINE unsigned
+extent_nfree_get(const extent_t *extent) {
+	assert(extent_slab_get(extent));
+	return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >>
+	    EXTENT_BITS_NFREE_SHIFT);
+}
+
 JEMALLOC_INLINE void *
 extent_base_get(const extent_t *extent) {
 	assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
@@ -211,6 +222,25 @@ extent_szind_set(extent_t *extent, szind_t szind) {
 }
 
 JEMALLOC_INLINE void
+extent_nfree_set(extent_t *extent, unsigned nfree) {
+	assert(extent_slab_get(extent));
+	extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) |
+	    ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
+}
+
+JEMALLOC_INLINE void
+extent_nfree_inc(extent_t *extent) {
+	assert(extent_slab_get(extent));
+	extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
+}
+
+JEMALLOC_INLINE void
+extent_nfree_dec(extent_t *extent) {
+	assert(extent_slab_get(extent));
+	extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
+}
+
+JEMALLOC_INLINE void
 extent_sn_set(extent_t *extent, size_t sn) {
 	extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
 	    ((uint64_t)sn << EXTENT_BITS_SN_SHIFT);
@@ -260,10 +290,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
 	extent_state_set(extent, state);
 	extent_zeroed_set(extent, zeroed);
 	extent_committed_set(extent, committed);
+	ql_elm_new(extent, ql_link);
 	if (config_prof) {
 		extent_prof_tctx_set(extent, NULL);
 	}
-	ql_elm_new(extent, ql_link);
 }
 
 JEMALLOC_INLINE void
diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h
index ddc04087..1527acb9 100644
--- a/include/jemalloc/internal/extent_structs.h
+++ b/include/jemalloc/internal/extent_structs.h
@@ -20,8 +20,9 @@ struct extent_s {
 	 * t: state
 	 * i: szind
 	 * n: sn
+	 * f: nfree
 	 *
-	 * nnnnnnnn ... nnnnnnni iiiiiiit tzcbaaaa aaaaaaaa
+	 * nnnnnnnn ... nnnnnfff fffffffi iiiiiiit tzcbaaaa aaaaaaaa
 	 *
 	 * arena_ind: Arena from which this extent came, or all 1 bits if
 	 *            unassociated.
@@ -47,6 +48,8 @@ struct extent_s {
 	 *        even for non-slabs, either due to large_pad or promotion of
 	 *        sampled small regions.
 	 *
+	 * nfree: Number of free regions in slab.
+	 *
 	 * sn: Serial number (potentially non-unique).
 	 *
 	 *     Serial numbers may wrap around if JEMALLOC_MUNMAP is defined, but
@@ -61,26 +64,35 @@ struct extent_s {
 	uint64_t		e_bits;
 #define EXTENT_BITS_ARENA_SHIFT		0
 #define EXTENT_BITS_ARENA_MASK \
-    (((1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT)
+    (((uint64_t)(1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT)
 
 #define EXTENT_BITS_SLAB_SHIFT		MALLOCX_ARENA_BITS
-#define EXTENT_BITS_SLAB_MASK		(0x1U << EXTENT_BITS_SLAB_SHIFT)
+#define EXTENT_BITS_SLAB_MASK \
+    ((uint64_t)0x1U << EXTENT_BITS_SLAB_SHIFT)
 
 #define EXTENT_BITS_COMMITTED_SHIFT	(MALLOCX_ARENA_BITS + 1)
-#define EXTENT_BITS_COMMITTED_MASK	(0x1U << EXTENT_BITS_COMMITTED_SHIFT)
+#define EXTENT_BITS_COMMITTED_MASK \
+    ((uint64_t)0x1U << EXTENT_BITS_COMMITTED_SHIFT)
 
 #define EXTENT_BITS_ZEROED_SHIFT	(MALLOCX_ARENA_BITS + 2)
-#define EXTENT_BITS_ZEROED_MASK		(0x1U << EXTENT_BITS_ZEROED_SHIFT)
+#define EXTENT_BITS_ZEROED_MASK \
+    ((uint64_t)0x1U << EXTENT_BITS_ZEROED_SHIFT)
 
 #define EXTENT_BITS_STATE_SHIFT		(MALLOCX_ARENA_BITS + 3)
-#define EXTENT_BITS_STATE_MASK		(0x3U << EXTENT_BITS_STATE_SHIFT)
+#define EXTENT_BITS_STATE_MASK \
+    ((uint64_t)0x3U << EXTENT_BITS_STATE_SHIFT)
 
 #define EXTENT_BITS_SZIND_SHIFT		(MALLOCX_ARENA_BITS + 5)
 #define EXTENT_BITS_SZIND_MASK \
-    (((1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT)
+    (((uint64_t)(1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT)
 
-#define EXTENT_BITS_SN_SHIFT \
+#define EXTENT_BITS_NFREE_SHIFT \
     (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES)
+#define EXTENT_BITS_NFREE_MASK \
+    ((uint64_t)((1U << (LG_SLAB_MAXREGS + 1)) - 1) << EXTENT_BITS_NFREE_SHIFT)
+
+#define EXTENT_BITS_SN_SHIFT \
+    (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES + (LG_SLAB_MAXREGS + 1))
 #define EXTENT_BITS_SN_MASK		(UINT64_MAX << EXTENT_BITS_SN_SHIFT)
 
 	/* Pointer to the extent that this structure is responsible for. */
@@ -89,17 +101,6 @@ struct extent_s {
 	/* Extent size. */
 	size_t			e_size;
 
-	union {
-		/* Small region slab metadata. */
-		arena_slab_data_t	e_slab_data;
-
-		/* Profile counters, used for large objects. */
-		union {
-			void		*e_prof_tctx_pun;
-			prof_tctx_t	*e_prof_tctx;
-		};
-	};
-
 	/*
 	 * List linkage, used by a variety of lists:
 	 * - arena_bin_t's slabs_full
@@ -112,6 +113,17 @@ struct extent_s {
 
 	/* Linkage for per size class sn/address-ordered heaps. */
 	phn(extent_t)		ph_link;
+
+	union {
+		/* Small region slab metadata. */
+		arena_slab_data_t	e_slab_data;
+
+		/* Profile counters, used for large objects. */
+		union {
+			void		*e_prof_tctx_pun;
+			prof_tctx_t	*e_prof_tctx;
+		};
+	};
 };
 typedef ql_head(extent_t) extent_list_t;
 typedef ph(extent_t) extent_heap_t;
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 64151c1b..5d03f5d0 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -175,6 +175,10 @@ extent_list_last
 extent_list_remove
 extent_list_replace
 extent_merge_wrapper
+extent_nfree_dec
+extent_nfree_get
+extent_nfree_inc
+extent_nfree_set
 extent_past_get
 extent_prof_tctx_get
 extent_prof_tctx_set