Stop depending on JEMALLOC_N() for function interception during testing.

Instead, always define function pointers for interceptable functions,
but mark them const unless testing, so that the compiler can optimize
out the pointer dereferences.

6 files changed, 51 insertions, 167 deletions

diff --git a/src/arena.c b/src/arena.c
index edbd875f..045e6127 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -1407,20 +1407,12 @@ arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info, bool zero) {
 	}
 }
 
-#ifdef JEMALLOC_JET
-#undef arena_dalloc_junk_small
-#define arena_dalloc_junk_small JEMALLOC_N(n_arena_dalloc_junk_small)
-#endif
-void
-arena_dalloc_junk_small(void *ptr, const arena_bin_info_t *bin_info) {
+static void
+arena_dalloc_junk_small_impl(void *ptr, const arena_bin_info_t *bin_info) {
 	memset(ptr, JEMALLOC_FREE_JUNK, bin_info->reg_size);
 }
-#ifdef JEMALLOC_JET
-#undef arena_dalloc_junk_small
-#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
-arena_dalloc_junk_small_t *arena_dalloc_junk_small =
-    JEMALLOC_N(n_arena_dalloc_junk_small);
-#endif
+arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small =
+    arena_dalloc_junk_small_impl;
 
 static void *
 arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
diff --git a/src/large.c b/src/large.c
index f657ccbe..ed73dc22 100644
--- a/src/large.c
+++ b/src/large.c
@@ -68,26 +68,14 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	return extent_addr_get(extent);
 }
 
-#ifdef JEMALLOC_JET
-#undef large_dalloc_junk
-#define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk)
-#endif
-void
-large_dalloc_junk(void *ptr, size_t size) {
+static void
+large_dalloc_junk_impl(void *ptr, size_t size) {
 	memset(ptr, JEMALLOC_FREE_JUNK, size);
 }
-#ifdef JEMALLOC_JET
-#undef large_dalloc_junk
-#define large_dalloc_junk JEMALLOC_N(large_dalloc_junk)
-large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk);
-#endif
-
-#ifdef JEMALLOC_JET
-#undef large_dalloc_maybe_junk
-#define large_dalloc_maybe_junk JEMALLOC_N(n_large_dalloc_maybe_junk)
-#endif
-void
-large_dalloc_maybe_junk(void *ptr, size_t size) {
+large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk = large_dalloc_junk_impl;
+
+static void
+large_dalloc_maybe_junk_impl(void *ptr, size_t size) {
 	if (config_fill && have_dss && unlikely(opt_junk_free)) {
 		/*
 		 * Only bother junk filling if the extent isn't about to be
@@ -98,12 +86,8 @@ large_dalloc_maybe_junk(void *ptr, size_t size) {
 		}
 	}
 }
-#ifdef JEMALLOC_JET
-#undef large_dalloc_maybe_junk
-#define large_dalloc_maybe_junk JEMALLOC_N(large_dalloc_maybe_junk)
-large_dalloc_maybe_junk_t *large_dalloc_maybe_junk =
-    JEMALLOC_N(n_large_dalloc_maybe_junk);
-#endif
+large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk =
+    large_dalloc_maybe_junk_impl;
 
 static bool
 large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
diff --git a/src/nstime.c b/src/nstime.c
index 20c00422..e5412274 100644
--- a/src/nstime.c
+++ b/src/nstime.c
@@ -131,27 +131,15 @@ nstime_get(nstime_t *time) {
 }
 #endif
 
-#ifdef JEMALLOC_JET
-#undef nstime_monotonic
-#define nstime_monotonic JEMALLOC_N(n_nstime_monotonic)
-#endif
-bool
-nstime_monotonic(void) {
+static bool
+nstime_monotonic_impl(void) {
 	return NSTIME_MONOTONIC;
 #undef NSTIME_MONOTONIC
 }
-#ifdef JEMALLOC_JET
-#undef nstime_monotonic
-#define nstime_monotonic JEMALLOC_N(nstime_monotonic)
-nstime_monotonic_t *nstime_monotonic = JEMALLOC_N(n_nstime_monotonic);
-#endif
+nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl;
 
-#ifdef JEMALLOC_JET
-#undef nstime_update
-#define nstime_update JEMALLOC_N(n_nstime_update)
-#endif
-bool
-nstime_update(nstime_t *time) {
+static bool
+nstime_update_impl(nstime_t *time) {
 	nstime_t old_time;
 
 	nstime_copy(&old_time, time);
@@ -165,8 +153,4 @@ nstime_update(nstime_t *time) {
 
 	return false;
 }
-#ifdef JEMALLOC_JET
-#undef nstime_update
-#define nstime_update JEMALLOC_N(nstime_update)
-nstime_update_t *nstime_update = JEMALLOC_N(n_nstime_update);
-#endif
+nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl;
diff --git a/src/prof.c b/src/prof.c
index d60680c1..470d926f 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -932,9 +932,7 @@ prof_tdata_count(void) {
 
 	return tdata_count;
 }
-#endif
 
-#ifdef JEMALLOC_JET
 size_t
 prof_bt_count(void) {
 	size_t bt_count;
@@ -955,12 +953,8 @@ prof_bt_count(void) {
 }
 #endif
 
-#ifdef JEMALLOC_JET
-#undef prof_dump_open
-#define prof_dump_open JEMALLOC_N(prof_dump_open_impl)
-#endif
 static int
-prof_dump_open(bool propagate_err, const char *filename) {
+prof_dump_open_impl(bool propagate_err, const char *filename) {
 	int fd;
 
 	fd = creat(filename, 0644);
@@ -974,11 +968,7 @@ prof_dump_open(bool propagate_err, const char *filename) {
 
 	return fd;
 }
-#ifdef JEMALLOC_JET
-#undef prof_dump_open
-#define prof_dump_open JEMALLOC_N(prof_dump_open)
-prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
-#endif
+prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
 
 static bool
 prof_dump_flush(bool propagate_err) {
@@ -1331,12 +1321,9 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
 	return NULL;
 }
 
-#ifdef JEMALLOC_JET
-#undef prof_dump_header
-#define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
-#endif
 static bool
-prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) {
+prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
+    const prof_cnt_t *cnt_all) {
 	bool ret;
 
 	if (prof_dump_printf(propagate_err,
@@ -1353,11 +1340,7 @@ prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) {
 	malloc_mutex_unlock(tsdn, &tdatas_mtx);
 	return ret;
 }
-#ifdef JEMALLOC_JET
-#undef prof_dump_header
-#define prof_dump_header JEMALLOC_N(prof_dump_header)
-prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
-#endif
+prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
 
 static bool
 prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
diff --git a/src/rtree.c b/src/rtree.c
index 72786ff5..62df0143 100644
--- a/src/rtree.c
+++ b/src/rtree.c
@@ -25,65 +25,35 @@ rtree_new(rtree_t *rtree, bool zeroed) {
 	return false;
 }
 
-#ifdef JEMALLOC_JET
-#undef rtree_node_alloc
-#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc_impl)
-#endif
 static rtree_node_elm_t *
-rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
+rtree_node_alloc_impl(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
 	return (rtree_node_elm_t *)base_alloc(tsdn, b0get(), nelms *
 	    sizeof(rtree_node_elm_t), CACHELINE);
 }
-#ifdef JEMALLOC_JET
-#undef rtree_node_alloc
-#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc)
-rtree_node_alloc_t *rtree_node_alloc = JEMALLOC_N(rtree_node_alloc_impl);
-#endif
+rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc = rtree_node_alloc_impl;
 
-#ifdef JEMALLOC_JET
-#undef rtree_node_dalloc
-#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc_impl)
-#endif
-UNUSED static void
-rtree_node_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
+static void
+rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
 	/* Nodes are never deleted during normal operation. */
 	not_reached();
 }
-#ifdef JEMALLOC_JET
-#undef rtree_node_dalloc
-#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc)
-rtree_node_dalloc_t *rtree_node_dalloc = JEMALLOC_N(rtree_node_dalloc_impl);
-#endif
+UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
+    rtree_node_dalloc_impl;
 
-#ifdef JEMALLOC_JET
-#undef rtree_leaf_alloc
-#define rtree_leaf_alloc JEMALLOC_N(rtree_leaf_alloc_impl)
-#endif
 static rtree_leaf_elm_t *
-rtree_leaf_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
+rtree_leaf_alloc_impl(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
 	return (rtree_leaf_elm_t *)base_alloc(tsdn, b0get(), nelms *
 	    sizeof(rtree_leaf_elm_t), CACHELINE);
 }
-#ifdef JEMALLOC_JET
-#undef rtree_leaf_alloc
-#define rtree_leaf_alloc JEMALLOC_N(rtree_leaf_alloc)
-rtree_leaf_alloc_t *rtree_leaf_alloc = JEMALLOC_N(rtree_leaf_alloc_impl);
-#endif
+rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc = rtree_leaf_alloc_impl;
 
-#ifdef JEMALLOC_JET
-#undef rtree_leaf_dalloc
-#define rtree_leaf_dalloc JEMALLOC_N(rtree_leaf_dalloc_impl)
-#endif
-UNUSED static void
-rtree_leaf_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
+static void
+rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
 	/* Leaves are never deleted during normal operation. */
 	not_reached();
 }
-#ifdef JEMALLOC_JET
-#undef rtree_leaf_dalloc
-#define rtree_leaf_dalloc JEMALLOC_N(rtree_leaf_dalloc)
-rtree_leaf_dalloc_t *rtree_leaf_dalloc = JEMALLOC_N(rtree_leaf_dalloc_impl);
-#endif
+UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
+    rtree_leaf_dalloc_impl;
 
 #ifdef JEMALLOC_JET
 #  if RTREE_HEIGHT > 1
diff --git a/src/witness.c b/src/witness.c
index edb736bf..0e910dca 100644
--- a/src/witness.c
+++ b/src/witness.c
@@ -14,12 +14,9 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank,
 	witness->opaque = opaque;
 }
 
-#ifdef JEMALLOC_JET
-#undef witness_lock_error
-#define witness_lock_error JEMALLOC_N(n_witness_lock_error)
-#endif
-void
-witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) {
+static void
+witness_lock_error_impl(const witness_list_t *witnesses,
+    const witness_t *witness) {
 	witness_t *w;
 
 	malloc_printf("<jemalloc>: Lock rank order reversal:");
@@ -29,51 +26,28 @@ witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) {
 	malloc_printf(" %s(%u)\n", witness->name, witness->rank);
 	abort();
 }
-#ifdef JEMALLOC_JET
-#undef witness_lock_error
-#define witness_lock_error JEMALLOC_N(witness_lock_error)
-witness_lock_error_t *witness_lock_error = JEMALLOC_N(n_witness_lock_error);
-#endif
+witness_lock_error_t *JET_MUTABLE witness_lock_error = witness_lock_error_impl;
 
-#ifdef JEMALLOC_JET
-#undef witness_owner_error
-#define witness_owner_error JEMALLOC_N(n_witness_owner_error)
-#endif
-void
-witness_owner_error(const witness_t *witness) {
+static void
+witness_owner_error_impl(const witness_t *witness) {
 	malloc_printf("<jemalloc>: Should own %s(%u)\n", witness->name,
 	    witness->rank);
 	abort();
 }
-#ifdef JEMALLOC_JET
-#undef witness_owner_error
-#define witness_owner_error JEMALLOC_N(witness_owner_error)
-witness_owner_error_t *witness_owner_error = JEMALLOC_N(n_witness_owner_error);
-#endif
+witness_owner_error_t *JET_MUTABLE witness_owner_error =
+    witness_owner_error_impl;
 
-#ifdef JEMALLOC_JET
-#undef witness_not_owner_error
-#define witness_not_owner_error JEMALLOC_N(n_witness_not_owner_error)
-#endif
-void
-witness_not_owner_error(const witness_t *witness) {
+static void
+witness_not_owner_error_impl(const witness_t *witness) {
 	malloc_printf("<jemalloc>: Should not own %s(%u)\n", witness->name,
 	    witness->rank);
 	abort();
 }
-#ifdef JEMALLOC_JET
-#undef witness_not_owner_error
-#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error)
-witness_not_owner_error_t *witness_not_owner_error =
-    JEMALLOC_N(n_witness_not_owner_error);
-#endif
+witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error =
+    witness_not_owner_error_impl;
 
-#ifdef JEMALLOC_JET
-#undef witness_depth_error
-#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
-#endif
-void
-witness_depth_error(const witness_list_t *witnesses,
+static void
+witness_depth_error_impl(const witness_list_t *witnesses,
     witness_rank_t rank_inclusive, unsigned depth) {
 	witness_t *w;
 
@@ -85,11 +59,8 @@ witness_depth_error(const witness_list_t *witnesses,
 	malloc_printf("\n");
 	abort();
 }
-#ifdef JEMALLOC_JET
-#undef witness_depth_error
-#define witness_depth_error JEMALLOC_N(witness_depth_error)
-witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
-#endif
+witness_depth_error_t *JET_MUTABLE witness_depth_error =
+    witness_depth_error_impl;
 
 void
 witnesses_cleanup(tsd_t *tsd) {