Fix an extent coalesce bug.

When coalescing, we should take both extents off the LRU list; otherwise decay
can grab the existing outer extent through extents_evict.

2 files changed, 18 insertions, 7 deletions

diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
index 9f5c5cd2..9b8ddc27 100644
--- a/include/jemalloc/internal/extent_inlines.h
+++ b/include/jemalloc/internal/extent_inlines.h
@@ -356,6 +356,11 @@ extent_list_append(extent_list_t *list, extent_t *extent) {
 }
 
 static inline void
+extent_list_prepend(extent_list_t *list, extent_t *extent) {
+	ql_head_insert(list, extent, ql_link);
+}
+
+static inline void
 extent_list_replace(extent_list_t *list, extent_t *to_remove,
     extent_t *to_insert) {
 	ql_after_insert(to_remove, to_insert, ql_link);
diff --git a/src/extent.c b/src/extent.c
index 548a93e2..8b00ec94 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -1458,13 +1458,12 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
     bool growing_retained) {
 	assert(extent_can_coalesce(arena, extents, inner, outer));
 
-	if (forward && extents->delay_coalesce) {
+	if (extents->delay_coalesce) {
 		/*
-		 * The extent that remains after coalescing must occupy the
-		 * outer extent's position in the LRU.  For forward coalescing,
-		 * swap the inner extent into the LRU.
+		 * Remove outer from the LRU list so that it won't be show up in
+		 * decay through extents_evict.
 		 */
-		extent_list_replace(&extents->lru, outer, inner);
+		extent_list_remove(&extents->lru, outer);
 	}
 	extent_activate_locked(tsdn, arena, extents, outer,
 	    extents->delay_coalesce);
@@ -1474,9 +1473,16 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	    forward ? inner : outer, forward ? outer : inner, growing_retained);
 	malloc_mutex_lock(tsdn, &extents->mtx);
 
+	if (!err && extents->delay_coalesce) {
+		if (forward) {
+			extent_list_prepend(&extents->lru, inner);
+		} else {
+			extent_list_prepend(&extents->lru, outer);
+		}
+	}
 	if (err) {
-		if (forward && extents->delay_coalesce) {
-			extent_list_replace(&extents->lru, inner, outer);
+		if (extents->delay_coalesce) {
+			extent_list_prepend(&extents->lru, outer);
 		}
 		extent_deactivate_locked(tsdn, arena, extents, outer,
 		    extents->delay_coalesce);