diff options
Diffstat (limited to 'include')
-rw-r--r-- | include/jemalloc/internal/arena_structs_a.h | 3 | ||||
-rw-r--r-- | include/jemalloc/internal/extent_inlines.h | 32 | ||||
-rw-r--r-- | include/jemalloc/internal/extent_structs.h | 50 | ||||
-rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 4 |
4 files changed, 66 insertions, 23 deletions
diff --git a/include/jemalloc/internal/arena_structs_a.h b/include/jemalloc/internal/arena_structs_a.h index ed265b20..07013a06 100644 --- a/include/jemalloc/internal/arena_structs_a.h +++ b/include/jemalloc/internal/arena_structs_a.h @@ -2,9 +2,6 @@ #define JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H struct arena_slab_data_s { - /* Number of free regions in slab. */ - unsigned nfree; - /* Per region allocated/deallocated bitmap. */ bitmap_t bitmap[BITMAP_GROUPS_MAX]; }; diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h index 22229b50..99fa67c7 100644 --- a/include/jemalloc/internal/extent_inlines.h +++ b/include/jemalloc/internal/extent_inlines.h @@ -11,6 +11,7 @@ extent_state_t extent_state_get(const extent_t *extent); bool extent_zeroed_get(const extent_t *extent); bool extent_committed_get(const extent_t *extent); bool extent_slab_get(const extent_t *extent); +unsigned extent_nfree_get(const extent_t *extent); void *extent_base_get(const extent_t *extent); void *extent_addr_get(const extent_t *extent); size_t extent_size_get(const extent_t *extent); @@ -25,6 +26,9 @@ void extent_addr_set(extent_t *extent, void *addr); void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment); void extent_size_set(extent_t *extent, size_t size); void extent_szind_set(extent_t *extent, szind_t szind); +void extent_nfree_set(extent_t *extent, unsigned nfree); +void extent_nfree_inc(extent_t *extent); +void extent_nfree_dec(extent_t *extent); void extent_sn_set(extent_t *extent, size_t sn); void extent_state_set(extent_t *extent, extent_state_t state); void extent_zeroed_set(extent_t *extent, bool zeroed); @@ -112,6 +116,13 @@ extent_slab_get(const extent_t *extent) { EXTENT_BITS_SLAB_SHIFT); } +JEMALLOC_INLINE unsigned +extent_nfree_get(const extent_t *extent) { + assert(extent_slab_get(extent)); + return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >> + EXTENT_BITS_NFREE_SHIFT); +} + JEMALLOC_INLINE void * extent_base_get(const extent_t *extent) { assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || @@ -211,6 +222,25 @@ extent_szind_set(extent_t *extent, szind_t szind) { } JEMALLOC_INLINE void +extent_nfree_set(extent_t *extent, unsigned nfree) { + assert(extent_slab_get(extent)); + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | + ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); +} + +JEMALLOC_INLINE void +extent_nfree_inc(extent_t *extent) { + assert(extent_slab_get(extent)); + extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); +} + +JEMALLOC_INLINE void +extent_nfree_dec(extent_t *extent) { + assert(extent_slab_get(extent)); + extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); +} + +JEMALLOC_INLINE void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); @@ -260,10 +290,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_state_set(extent, state); extent_zeroed_set(extent, zeroed); extent_committed_set(extent, committed); + ql_elm_new(extent, ql_link); if (config_prof) { extent_prof_tctx_set(extent, NULL); } - ql_elm_new(extent, ql_link); } JEMALLOC_INLINE void diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h index ddc04087..1527acb9 100644 --- a/include/jemalloc/internal/extent_structs.h +++ b/include/jemalloc/internal/extent_structs.h @@ -20,8 +20,9 @@ struct extent_s { * t: state * i: szind * n: sn + * f: nfree * - * nnnnnnnn ... nnnnnnni iiiiiiit tzcbaaaa aaaaaaaa + * nnnnnnnn ... nnnnnfff fffffffi iiiiiiit tzcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -47,6 +48,8 @@ struct extent_s { * even for non-slabs, either due to large_pad or promotion of * sampled small regions. * + * nfree: Number of free regions in slab. + * * sn: Serial number (potentially non-unique). * * Serial numbers may wrap around if JEMALLOC_MUNMAP is defined, but @@ -61,26 +64,35 @@ struct extent_s { uint64_t e_bits; #define EXTENT_BITS_ARENA_SHIFT 0 #define EXTENT_BITS_ARENA_MASK \ - (((1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT) + (((uint64_t)(1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT) #define EXTENT_BITS_SLAB_SHIFT MALLOCX_ARENA_BITS -#define EXTENT_BITS_SLAB_MASK (0x1U << EXTENT_BITS_SLAB_SHIFT) +#define EXTENT_BITS_SLAB_MASK \ + ((uint64_t)0x1U << EXTENT_BITS_SLAB_SHIFT) #define EXTENT_BITS_COMMITTED_SHIFT (MALLOCX_ARENA_BITS + 1) -#define EXTENT_BITS_COMMITTED_MASK (0x1U << EXTENT_BITS_COMMITTED_SHIFT) +#define EXTENT_BITS_COMMITTED_MASK \ + ((uint64_t)0x1U << EXTENT_BITS_COMMITTED_SHIFT) #define EXTENT_BITS_ZEROED_SHIFT (MALLOCX_ARENA_BITS + 2) -#define EXTENT_BITS_ZEROED_MASK (0x1U << EXTENT_BITS_ZEROED_SHIFT) +#define EXTENT_BITS_ZEROED_MASK \ + ((uint64_t)0x1U << EXTENT_BITS_ZEROED_SHIFT) #define EXTENT_BITS_STATE_SHIFT (MALLOCX_ARENA_BITS + 3) -#define EXTENT_BITS_STATE_MASK (0x3U << EXTENT_BITS_STATE_SHIFT) +#define EXTENT_BITS_STATE_MASK \ + ((uint64_t)0x3U << EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_SZIND_SHIFT (MALLOCX_ARENA_BITS + 5) #define EXTENT_BITS_SZIND_MASK \ - (((1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT) + (((uint64_t)(1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT) -#define EXTENT_BITS_SN_SHIFT \ +#define EXTENT_BITS_NFREE_SHIFT \ (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES) +#define EXTENT_BITS_NFREE_MASK \ + ((uint64_t)((1U << (LG_SLAB_MAXREGS + 1)) - 1) << EXTENT_BITS_NFREE_SHIFT) + +#define EXTENT_BITS_SN_SHIFT \ + (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES + (LG_SLAB_MAXREGS + 1)) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) /* Pointer to the extent that this structure is responsible for. */ @@ -89,17 +101,6 @@ struct extent_s { /* Extent size. */ size_t e_size; - union { - /* Small region slab metadata. */ - arena_slab_data_t e_slab_data; - - /* Profile counters, used for large objects. */ - union { - void *e_prof_tctx_pun; - prof_tctx_t *e_prof_tctx; - }; - }; - /* * List linkage, used by a variety of lists: * - arena_bin_t's slabs_full @@ -112,6 +113,17 @@ struct extent_s { /* Linkage for per size class sn/address-ordered heaps. */ phn(extent_t) ph_link; + + union { + /* Small region slab metadata. */ + arena_slab_data_t e_slab_data; + + /* Profile counters, used for large objects. */ + union { + void *e_prof_tctx_pun; + prof_tctx_t *e_prof_tctx; + }; + }; }; typedef ql_head(extent_t) extent_list_t; typedef ph(extent_t) extent_heap_t; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 64151c1b..5d03f5d0 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -175,6 +175,10 @@ extent_list_last extent_list_remove extent_list_replace extent_merge_wrapper +extent_nfree_dec +extent_nfree_get +extent_nfree_inc +extent_nfree_set extent_past_get extent_prof_tctx_get extent_prof_tctx_set |