diff options
author | Christopher Ferris <cferris@google.com> | 2018-09-10 16:02:28 -0700 |
---|---|---|
committer | Christopher Ferris <cferris@google.com> | 2018-09-14 12:14:54 -0700 |
commit | 03b5d1c549d393f0a4c86cde7377e098ebb0e686 (patch) | |
tree | 025d300d4ff64f7929d19c3808f8655b577caa7b | |
parent | 7eecc47efac8cef90e806aca2758ef0fca63cf7a (diff) | |
download | platform_external_jemalloc_new-master-cuttlefish-testing-release.tar.gz platform_external_jemalloc_new-master-cuttlefish-testing-release.tar.bz2 platform_external_jemalloc_new-master-cuttlefish-testing-release.zip |
Add android extensions.master-cuttlefish-testing-release
Bug: 62621531
Bug: 110158834
Test: Ran unit tests and benchmarks using libc.
Change-Id: Ie13ab8510c42f96b58496b0ab7e4f8c3a9cd2c6d
-rw-r--r-- | Android.bp | 113 | ||||
-rwxr-xr-x | android/run_jemalloc_tests.sh | 13 | ||||
-rw-r--r-- | include/jemalloc/internal/arena_types.h | 5 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal_defs.h | 6 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal_defs_host.h | 379 | ||||
-rw-r--r-- | include/jemalloc/internal/tcache_types.h | 16 | ||||
-rw-r--r-- | include/jemalloc/jemalloc.h | 25 | ||||
-rw-r--r-- | include/jemalloc/jemalloc_rename.h | 25 | ||||
-rw-r--r-- | src/android_je_iterate.c | 67 | ||||
-rw-r--r-- | src/android_je_mallinfo.c | 97 | ||||
-rw-r--r-- | src/jemalloc.c | 21 | ||||
-rw-r--r-- | src/large.c | 8 | ||||
-rw-r--r-- | src/pages.c | 21 |
13 files changed, 762 insertions, 34 deletions
@@ -27,8 +27,12 @@ common_cflags = [ "-fvisibility=hidden", "-O3", "-funroll-loops", + + // The following flags are for avoiding errors when compiling. "-Wno-unused-parameter", + "-Wno-unused-function", "-Wno-missing-field-initializers", + "-U_FORTIFY_SOURCE", ] @@ -37,11 +41,67 @@ common_c_local_includes = [ "include", ] +// These parameters change the way jemalloc works. +// ANDROID_NUM_ARENAS=XX +// The total number of arenas to create. +// ANDROID_TCACHE_NSLOTS_SMALL_MIN=XX +// The minimum number of small slots held in the tcache. This must be +// at least 1. +// ANDROID_TCACHE_NSLOTS_SMALL_MAX=XX +// The number of small slots held in the tcache. The higher this number +// is, the higher amount of PSS consumed. If this number is set too low +// then small allocations will take longer to complete. +// ANDROID_TCACHE_NSLOTS_LARGE=XX +// The number of large slots held in the tcache. The higher this number +// is, the higher amount of PSS consumed. If this number is set too low +// then large allocations will take longer to complete. +// ANDROID_LG_TCACHE_MAXCLASS_DEFAULT=XX +// 1 << XX is the maximum sized allocation that will be in the tcache. + +android_common_cflags = [ + "-DANDROID_TCACHE_NSLOTS_SMALL_MIN=1", + "-DANDROID_LG_TCACHE_MAXCLASS_DEFAULT=16", + + // Default some parameters to small values to minimize PSS. + // These parameters will be overridden by android_product_variables + // for non-svelte configs. + "-DANDROID_NUM_ARENAS=1", + "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=1", + "-DANDROID_TCACHE_NSLOTS_LARGE=1", +] + +android_product_variables = { + // Only enable the tcache on non-svelte configurations, to save PSS. + malloc_not_svelte: { + cflags: [ + "-UANDROID_NUM_ARENAS", + "-DANDROID_NUM_ARENAS=2", + + "-UANDROID_TCACHE_NSLOTS_SMALL_MAX", + "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=8", + + "-UANDROID_TCACHE_NSLOTS_LARGE", + "-DANDROID_TCACHE_NSLOTS_LARGE=16", + ], + }, +} + cc_defaults { name: "jemalloc5_defaults", defaults: ["linux_bionic_supported"], + host_supported: true, cflags: common_cflags, + target: { + android: { + cflags: android_common_cflags, + product_variables: android_product_variables, + }, + linux_glibc: { + enabled: true, + }, + }, + local_include_dirs: common_c_local_includes, stl: "none", } @@ -82,13 +142,21 @@ lib_src_files = [ //----------------------------------------------------------------------- // jemalloc static library //----------------------------------------------------------------------- -cc_library_static { +cc_library { name: "libjemalloc5", recovery_available: true, defaults: ["jemalloc5_defaults"], srcs: lib_src_files, + + target: { + android: { + shared: { + enabled: false, + }, + }, + }, } //----------------------------------------------------------------------- @@ -100,7 +168,6 @@ cc_library_static { defaults: ["jemalloc5_defaults"], cflags: [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_JET", ], @@ -129,7 +196,6 @@ cc_library_static { defaults: ["jemalloc5_defaults"], cflags: [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_UNIT_TEST", ], @@ -203,10 +269,11 @@ unit_tests = [ cc_test { name: "jemalloc5_unittests", + defaults: ["jemalloc5_defaults"], + gtest: false, cflags: common_cflags + [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_UNIT_TEST", ], @@ -219,6 +286,16 @@ cc_test { static_libs: ["libjemalloc5_unittest"], test_per_src: true, + + target: { + linux_glibc: { + enabled: true, + // The sanitizer does not work for these tests on the host. + sanitize: { + never: true, + }, + }, + }, } //----------------------------------------------------------------------- @@ -262,10 +339,11 @@ integration_tests = [ cc_test { name: "jemalloc5_integrationtests", + defaults: ["jemalloc5_defaults"], + gtest: false, cflags: common_cflags + [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_INTEGRATION_TEST", ], @@ -278,6 +356,18 @@ cc_test { static_libs: ["libjemalloc5_integrationtest"], test_per_src: true, + + target: { + linux_glibc: { + // The sanitizer does not work for these tests on the host. + sanitize: { + never: true, + }, + }, + }, + + // Needed for basic.cpp test. + stl: "libc++_static", } //----------------------------------------------------------------------- @@ -289,7 +379,6 @@ cc_library_static { defaults: ["jemalloc5_defaults"], cflags: [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_STRESS_TEST", "-DJEMALLOC_STRESS_TESTLIB", ], @@ -313,10 +402,11 @@ stress_tests = [ cc_test { name: "jemalloc5_stresstests", + defaults: ["jemalloc5_defaults"], + gtest: false, cflags: common_cflags + [ - "-U_FORTIFY_SOURCE", "-DJEMALLOC_STRESS_TEST", ], @@ -333,4 +423,13 @@ cc_test { ], test_per_src: true, + + target: { + linux_glibc: { + // The sanitizer does not work for these tests on the host. + sanitize: { + never: true, + }, + }, + }, } diff --git a/android/run_jemalloc_tests.sh b/android/run_jemalloc_tests.sh index ce1d7885..de085afc 100755 --- a/android/run_jemalloc_tests.sh +++ b/android/run_jemalloc_tests.sh @@ -1,4 +1,4 @@ -#!/system/bin/sh +#!/bin/sh # # Copyright (C) 2014 The Android Open Source Project # @@ -94,7 +94,7 @@ STRESS_TESTS=( \ "microbench" \ ) -TEST_DIRECTORIES=( "/data/nativetest" "/data/nativetest64" ) +TEST_DIRECTORIES=( "nativetest" "nativetest64" ) FAILING_TESTS=() function run_tests () { @@ -120,8 +120,17 @@ function run_tests () { fi } +if [[ "" == "$1" ]]; then + directory="/data" +else + directory=$1; +fi + +echo "Looking in $directory"; + EXIT_CODE=0 for test_dir in ${TEST_DIRECTORIES[@]}; do + test_dir="${directory}/${test_dir}" if [[ -d "${test_dir}" ]]; then run_tests "unit" "${test_dir}/${UNIT_TEST_DIR}" ${UNIT_TESTS[@]} run_tests "integration" "${test_dir}/${INTEGRATION_TEST_DIR}" ${INTEGRATION_TESTS[@]} diff --git a/include/jemalloc/internal/arena_types.h b/include/jemalloc/internal/arena_types.h index 70001b5f..423a5d88 100644 --- a/include/jemalloc/internal/arena_types.h +++ b/include/jemalloc/internal/arena_types.h @@ -6,8 +6,13 @@ #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* Default decay times in milliseconds. */ +#if defined(__ANDROID__) +#define DIRTY_DECAY_MS_DEFAULT ZD(0) +#define MUZZY_DECAY_MS_DEFAULT ZD(0) +#else #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) #define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000) +#endif /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h index 99d1ba3e..29948a2c 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/include/jemalloc/internal/jemalloc_internal_defs.h @@ -1,3 +1,6 @@ +#if !defined(__ANDROID__) +#include "jemalloc_internal_defs_host.h" +#else /* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ #ifndef JEMALLOC_INTERNAL_DEFS_H_ #define JEMALLOC_INTERNAL_DEFS_H_ @@ -182,7 +185,7 @@ * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage * segment (DSS). */ -#define JEMALLOC_DSS +/* #undef JEMALLOC_DSS */ /* Support memory filling (junk/zero). */ #define JEMALLOC_FILL @@ -381,3 +384,4 @@ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ +#endif diff --git a/include/jemalloc/internal/jemalloc_internal_defs_host.h b/include/jemalloc/internal/jemalloc_internal_defs_host.h new file mode 100644 index 00000000..480a8356 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_defs_host.h @@ -0,0 +1,379 @@ +/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +/* #undef JEMALLOC_PREFIX */ +/* #undef JEMALLOC_CPREFIX */ + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE___LIBC_CALLOC +#define JEMALLOC_OVERRIDE___LIBC_FREE +#define JEMALLOC_OVERRIDE___LIBC_MALLOC +#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN +#define JEMALLOC_OVERRIDE___LIBC_REALLOC +#define JEMALLOC_OVERRIDE___LIBC_VALLOC +/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#define JEMALLOC_PRIVATE_NAMESPACE je_ + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#ifdef __x86_64__ +#define CPU_SPINWAIT __asm__ volatile("pause") +#else +#define CPU_SPINWAIT +#endif +/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ +#define HAVE_CPU_SPINWAIT 1 + +/* + * Number of significant bits in virtual addresses. This may be less than the + * total number of bits in a pointer, e.g. on x64, for which the uppermost 16 + * bits are the same as bit 47. + */ +#ifdef __LP64__ +#define LG_VADDR 48 +#else +#define LG_VADDR 32 +#endif + +/* Defined if C11 atomics are available. */ +#define JEMALLOC_C11_ATOMICS 1 + +/* Defined if GCC __atomic atomics are available. */ +#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 + +/* Defined if GCC __sync atomics are available. */ +#define JEMALLOC_GCC_SYNC_ATOMICS 1 + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ + +/* + * Defined if __builtin_clz() and __builtin_clzl() are available. + */ +#define JEMALLOC_HAVE_BUILTIN_CLZ + +/* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +/* #undef JEMALLOC_OS_UNFAIR_LOCK */ + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +/* #undef JEMALLOC_OSSPIN */ + +/* Defined if syscall(2) is usable. */ +#define JEMALLOC_USE_SYSCALL + +/* + * Defined if secure_getenv(3) is available. + */ +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ + +/* + * Defined if issetugid(2) is available. + */ +/* #undef JEMALLOC_HAVE_ISSETUGID */ + +/* Defined if pthread_atfork(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_ATFORK + +/* Defined if pthread_setname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 + +/* + * Defined if mach_absolute_time() is available. + */ +/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */ + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#define JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +/* #undef JEMALLOC_MUTEX_INIT_CB */ + +/* Non-empty if the tls_model attribute is supported. */ +#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +/* #undef JEMALLOC_DEBUG */ + +/* JEMALLOC_STATS enables statistics calculation. */ +#define JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +/* #undef JEMALLOC_PROF */ + +/* Use libunwind for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBUNWIND */ + +/* Use libgcc for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBGCC */ + +/* Use gcc intrinsics for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_GCC */ + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage + * segment (DSS). + */ +#define JEMALLOC_DSS + +/* Support memory filling (junk/zero). */ +#define JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +/* #undef JEMALLOC_UTRACE */ + +/* Support optional abort() on OOM. */ +/* #undef JEMALLOC_XMALLOC */ + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +/* #undef JEMALLOC_LAZY_LOCK */ + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +/* #undef LG_QUANTUM */ + +/* One page is 2^LG_PAGE bytes. */ +#define LG_PAGE 12 + +/* + * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the + * system does not explicitly support huge pages; system calls that require + * explicit huge page support are separately configured. + */ +#define LG_HUGEPAGE 21 + +/* + * If defined, adjacent virtual memory mappings with identical attributes + * automatically coalesce, and they fragment when changes are made to subranges. + * This is the normal order of things for mmap()/munmap(), but on Windows + * VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e. + * mappings do *not* coalesce/fragment. + */ +#define JEMALLOC_MAPS_COALESCE + +/* + * If defined, retain memory for later reuse by default rather than using e.g. + * munmap() to unmap freed extents. This is enabled on 64-bit Linux because + * common sequences of mmap()/munmap() calls will cause virtual memory map + * holes. + */ +#define JEMALLOC_RETAIN + +/* TLS is used to map arenas and magazine caches to threads. */ +#define JEMALLOC_TLS + +/* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable + +/* + * ffs*() functions to use for bitmapping. Don't use these directly; instead, + * use ffs_*() from util.h. + */ +#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll +#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl +#define JEMALLOC_INTERNAL_FFS __builtin_ffs + +/* + * If defined, explicitly attempt to more uniformly distribute large allocation + * pointer alignments across all cache indices. + */ +#define JEMALLOC_CACHE_OBLIVIOUS + +/* + * If defined, enable logging facilities. We make this a configure option to + * avoid taking extra branches everywhere. + */ +/* #undef JEMALLOC_LOG */ + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +/* #undef JEMALLOC_ZONE */ + +/* + * Methods for determining whether the OS overcommits. + * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's + * /proc/sys/vm.overcommit_memory file. + * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl. + */ +/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ +#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY + +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#define JEMALLOC_HAVE_MADVISE_HUGE + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is + * defined, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched; + * otherwise this behaves similarly to + * MADV_FREE, though typically with higher + * system overhead. + */ +/* #undef JEMALLOC_PURGE_MADVISE_FREE */ +#define JEMALLOC_PURGE_MADVISE_DONTNEED +#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS + +/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ +/* #undef JEMALLOC_DEFINE_MADVISE_FREE */ + +/* + * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_DONTDUMP */ + +/* + * Defined if transparent huge pages (THPs) are supported via the + * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. + */ +/* #undef JEMALLOC_THP */ + +/* Define if operating system has alloca.h header. */ +#define JEMALLOC_HAS_ALLOCA_H 1 + +/* C99 restrict keyword supported. */ +#define JEMALLOC_HAS_RESTRICT 1 + +/* For use by hash code. */ +/* #undef JEMALLOC_BIG_ENDIAN */ + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#define LG_SIZEOF_INT 2 + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#ifdef __LP64__ +#define LG_SIZEOF_LONG 3 +#else +#define LG_SIZEOF_LONG 2 +#endif + +/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */ +#define LG_SIZEOF_LONG_LONG 3 + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#define LG_SIZEOF_INTMAX_T 3 + +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ +#define JEMALLOC_GLIBC_MALLOC_HOOK + +/* glibc memalign hook. */ +#define JEMALLOC_GLIBC_MEMALIGN_HOOK + +/* pthread support */ +#define JEMALLOC_HAVE_PTHREAD + +/* dlsym() support */ +#define JEMALLOC_HAVE_DLSYM + +/* Adaptive mutex support in pthreads. */ +#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP + +/* GNU specific sched_getcpu support */ +#define JEMALLOC_HAVE_SCHED_GETCPU + +/* GNU specific sched_setaffinity support */ +#define JEMALLOC_HAVE_SCHED_SETAFFINITY + +/* + * If defined, all the features necessary for background threads are present. + */ +#define JEMALLOC_BACKGROUND_THREAD 1 + +/* + * If defined, jemalloc symbols are not exported (doesn't work when + * JEMALLOC_PREFIX is not defined). + */ +/* #undef JEMALLOC_EXPORT */ + +/* config.malloc_conf options string. */ +#define JEMALLOC_CONFIG_MALLOC_CONF "" + +/* If defined, jemalloc takes the malloc/free/etc. symbol names. */ +#define JEMALLOC_IS_MALLOC 1 + +/* + * Defined if strerror_r returns char * if _GNU_SOURCE is defined. + */ +#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/include/jemalloc/internal/tcache_types.h b/include/jemalloc/internal/tcache_types.h index e49bc9d7..bcbb4b7b 100644 --- a/include/jemalloc/internal/tcache_types.h +++ b/include/jemalloc/internal/tcache_types.h @@ -19,7 +19,11 @@ typedef struct tcaches_s tcaches_t; /* * Absolute minimum number of cache slots for each small bin. */ +#if defined(ANDROID_TCACHE_NSLOTS_SMALL_MIN) +#define TCACHE_NSLOTS_SMALL_MIN ANDROID_TCACHE_NSLOTS_SMALL_MIN +#else #define TCACHE_NSLOTS_SMALL_MIN 20 +#endif /* * Absolute maximum number of cache slots for each small bin in the thread @@ -28,13 +32,25 @@ typedef struct tcaches_s tcaches_t; * * This constant must be an even number. */ +#if defined(ANDROID_TCACHE_NSLOTS_SMALL_MAX) +#define TCACHE_NSLOTS_SMALL_MAX ANDROID_TCACHE_NSLOTS_SMALL_MAX +#else #define TCACHE_NSLOTS_SMALL_MAX 200 +#endif /* Number of cache slots for large size classes. */ +#if defined(ANDROID_TCACHE_NSLOTS_LARGE) +#define TCACHE_NSLOTS_LARGE ANDROID_TCACHE_NSLOTS_LARGE +#else #define TCACHE_NSLOTS_LARGE 20 +#endif /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ +#if defined(ANDROID_LG_TCACHE_MAXCLASS_DEFAULT) +#define LG_TCACHE_MAXCLASS_DEFAULT ANDROID_LG_TCACHE_MAXCLASS_DEFAULT +#else #define LG_TCACHE_MAXCLASS_DEFAULT 15 +#endif /* * TCACHE_GC_SWEEP is the approximate number of allocation events between diff --git a/include/jemalloc/jemalloc.h b/include/jemalloc/jemalloc.h index c96dfc54..c0d12d2f 100644 --- a/include/jemalloc/jemalloc.h +++ b/include/jemalloc/jemalloc.h @@ -59,30 +59,7 @@ extern "C" { * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by * these macro definitions. */ -#ifndef JEMALLOC_NO_RENAME -# define je_aligned_alloc je_aligned_alloc -# define je_calloc je_calloc -# define je_dallocx je_dallocx -# define je_free je_free -# define je_mallctl je_mallctl -# define je_mallctlbymib je_mallctlbymib -# define je_mallctlnametomib je_mallctlnametomib -# define je_malloc je_malloc -# define je_malloc_conf je_malloc_conf -# define je_malloc_message je_malloc_message -# define je_malloc_stats_print je_malloc_stats_print -# define je_malloc_usable_size je_malloc_usable_size -# define je_mallocx je_mallocx -# define je_nallocx je_nallocx -# define je_posix_memalign je_posix_memalign -# define je_rallocx je_rallocx -# define je_realloc je_realloc -# define je_sallocx je_sallocx -# define je_sdallocx je_sdallocx -# define je_xallocx je_xallocx -# define je_memalign je_memalign -# define je_valloc je_valloc -#endif +#include "jemalloc_rename.h" #include <stdlib.h> #include <stdbool.h> diff --git a/include/jemalloc/jemalloc_rename.h b/include/jemalloc/jemalloc_rename.h index 28ebd37e..c5cefd8d 100644 --- a/include/jemalloc/jemalloc_rename.h +++ b/include/jemalloc/jemalloc_rename.h @@ -4,6 +4,7 @@ * these macro definitions. */ #ifndef JEMALLOC_NO_RENAME +#if defined(__ANDROID__) # define je_aligned_alloc je_aligned_alloc # define je_calloc je_calloc # define je_dallocx je_dallocx @@ -26,4 +27,28 @@ # define je_xallocx je_xallocx # define je_memalign je_memalign # define je_valloc je_valloc +#else +# define je_aligned_alloc aligned_alloc +# define je_calloc calloc +# define je_dallocx dallocx +# define je_free free +# define je_mallctl mallctl +# define je_mallctlbymib mallctlbymib +# define je_mallctlnametomib mallctlnametomib +# define je_malloc malloc +# define je_malloc_conf malloc_conf +# define je_malloc_message malloc_message +# define je_malloc_stats_print malloc_stats_print +# define je_malloc_usable_size malloc_usable_size +# define je_mallocx mallocx +# define je_nallocx nallocx +# define je_posix_memalign posix_memalign +# define je_rallocx rallocx +# define je_realloc realloc +# define je_sallocx sallocx +# define je_sdallocx sdallocx +# define je_xallocx xallocx +# define je_memalign memalign +# define je_valloc valloc +#endif #endif diff --git a/src/android_je_iterate.c b/src/android_je_iterate.c new file mode 100644 index 00000000..3705cd63 --- /dev/null +++ b/src/android_je_iterate.c @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +static pthread_mutex_t malloc_disabled_lock = PTHREAD_MUTEX_INITIALIZER; +static bool malloc_disabled_tcache; + +int je_iterate(uintptr_t base, size_t size, + void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) { + // TODO: Figure out how to implement this functionality for jemalloc5. + return -1; +} + +static void je_malloc_disable_prefork() { + pthread_mutex_lock(&malloc_disabled_lock); +} + +static void je_malloc_disable_postfork_parent() { + pthread_mutex_unlock(&malloc_disabled_lock); +} + +static void je_malloc_disable_postfork_child() { + pthread_mutex_init(&malloc_disabled_lock, NULL); +} + +void je_malloc_disable_init() { + if (pthread_atfork(je_malloc_disable_prefork, + je_malloc_disable_postfork_parent, je_malloc_disable_postfork_child) != 0) { + malloc_write("<jemalloc>: Error in pthread_atfork()\n"); + if (opt_abort) + abort(); + } +} + +void je_malloc_disable() { + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + pthread_once(&once_control, je_malloc_disable_init); + + pthread_mutex_lock(&malloc_disabled_lock); + bool new_tcache = false; + size_t old_len = sizeof(malloc_disabled_tcache); + je_mallctl("thread.tcache.enabled", + &malloc_disabled_tcache, &old_len, + &new_tcache, sizeof(new_tcache)); + jemalloc_prefork(); +} + +void je_malloc_enable() { + jemalloc_postfork_parent(); + if (malloc_disabled_tcache) { + je_mallctl("thread.tcache.enabled", NULL, NULL, + &malloc_disabled_tcache, sizeof(malloc_disabled_tcache)); + } + pthread_mutex_unlock(&malloc_disabled_lock); +} diff --git a/src/android_je_mallinfo.c b/src/android_je_mallinfo.c new file mode 100644 index 00000000..32661c21 --- /dev/null +++ b/src/android_je_mallinfo.c @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Only use bin locks since the stats are now all atomic and can be read + * without taking the stats lock. + */ +struct mallinfo je_mallinfo() { + struct mallinfo mi; + memset(&mi, 0, sizeof(mi)); + + malloc_mutex_lock(TSDN_NULL, &arenas_lock); + for (unsigned i = 0; i < narenas_auto; i++) { + arena_t* arena = atomic_load_p(&arenas[i], ATOMIC_ACQUIRE); + if (arena != NULL) { + mi.hblkhd += atomic_load_zu(&arena->stats.mapped, ATOMIC_ACQUIRE); + mi.uordblks += atomic_load_zu(&arena->stats.allocated_large, ATOMIC_ACQUIRE); + + for (unsigned j = 0; j < NBINS; j++) { + bin_t* bin = &arena->bins[j]; + + malloc_mutex_lock(TSDN_NULL, &bin->lock); + mi.uordblks += bin_infos[j].reg_size * bin->stats.curregs; + malloc_mutex_unlock(TSDN_NULL, &bin->lock); + } + } + } + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); + mi.fordblks = mi.hblkhd - mi.uordblks; + mi.usmblks = mi.hblkhd; + return mi; +} + +size_t __mallinfo_narenas() { + return narenas_auto; +} + +size_t __mallinfo_nbins() { + return NBINS; +} + +struct mallinfo __mallinfo_arena_info(size_t aidx) { + struct mallinfo mi; + memset(&mi, 0, sizeof(mi)); + + malloc_mutex_lock(TSDN_NULL, &arenas_lock); + if (aidx < narenas_auto) { + arena_t* arena = atomic_load_p(&arenas[aidx], ATOMIC_ACQUIRE); + if (arena != NULL) { + mi.hblkhd = atomic_load_zu(&arena->stats.mapped, ATOMIC_ACQUIRE); + mi.ordblks = atomic_load_zu(&arena->stats.allocated_large, ATOMIC_ACQUIRE); + + for (unsigned j = 0; j < NBINS; j++) { + bin_t* bin = &arena->bins[j]; + + malloc_mutex_lock(TSDN_NULL, &bin->lock); + mi.fsmblks += bin_infos[j].reg_size * bin->stats.curregs; + malloc_mutex_unlock(TSDN_NULL, &bin->lock); + } + } + } + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); + return mi; +} + +struct mallinfo __mallinfo_bin_info(size_t aidx, size_t bidx) { + struct mallinfo mi; + memset(&mi, 0, sizeof(mi)); + + malloc_mutex_lock(TSDN_NULL, &arenas_lock); + if (aidx < narenas_auto && bidx < NBINS) { + arena_t* arena = atomic_load_p(&arenas[aidx], ATOMIC_ACQUIRE); + if (arena != NULL) { + bin_t* bin = &arena->bins[bidx]; + + malloc_mutex_lock(TSDN_NULL, &bin->lock); + mi.ordblks = bin_infos[bidx].reg_size * bin->stats.curregs; + mi.uordblks = (size_t) bin->stats.nmalloc; + mi.fordblks = (size_t) bin->stats.ndalloc; + malloc_mutex_unlock(TSDN_NULL, &bin->lock); + } + } + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); + return mi; +} diff --git a/src/jemalloc.c b/src/jemalloc.c index f93c16fa..9f4df5e7 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -875,7 +875,14 @@ malloc_conf_init(void) { const char *opts, *k, *v; size_t klen, vlen; +#if defined(__ANDROID__) + /* For Android, do not look at files nor environment variables for + * config data. + */ + for (i = 0; i < 2; i++) { +#else for (i = 0; i < 4; i++) { +#endif /* Get runtime configuration. */ switch (i) { case 0: @@ -1346,7 +1353,12 @@ static bool malloc_init_hard_recursible(void) { malloc_init_state = malloc_init_recursible; +#if defined(__ANDROID__) && defined(ANDROID_NUM_ARENAS) + /* Hardcode since this value won't be used. */ + ncpus = 2; +#else ncpus = malloc_ncpus(); +#endif #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ @@ -1371,6 +1383,9 @@ malloc_init_hard_recursible(void) { static unsigned malloc_narenas_default(void) { +#if defined(ANDROID_NUM_ARENAS) + return ANDROID_NUM_ARENAS; +#else assert(ncpus > 0); /* * For SMP systems, create more than one arena per CPU by @@ -1381,6 +1396,7 @@ malloc_narenas_default(void) { } else { return 1; } +#endif } static percpu_arena_mode_t @@ -3324,3 +3340,8 @@ jemalloc_postfork_child(void) { } /******************************************************************************/ + +#if defined(__ANDROID__) && !defined(JEMALLOC_JET) +#include "android_je_iterate.c" +#include "android_je_mallinfo.c" +#endif diff --git a/src/large.c b/src/large.c index 27a2c679..cbffd99b 100644 --- a/src/large.c +++ b/src/large.c @@ -42,7 +42,15 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { +#if defined(__ANDROID__) && !defined(__LP64__) + /* On 32 bit systems, using a per arena cache can exhaust + * virtual address space. Force all huge allocations to + * always take place in the first arena. + */ + arena = arena_get(tsdn, 0, false); +#else arena = arena_choose(tsdn_tsd(tsdn), arena); +#endif } if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, arena, usize, alignment, &is_zeroed)) == NULL) { diff --git a/src/pages.c b/src/pages.c index 26002692..132aeeb7 100644 --- a/src/pages.c +++ b/src/pages.c @@ -16,6 +16,13 @@ #endif /******************************************************************************/ +/* Defines/includes needed for special android code. */ + +#if defined(__ANDROID__) +#include <sys/prctl.h> +#endif + +/******************************************************************************/ /* Data. */ /* Actual operating system page size, detected during bootstrap, <= PAGE. */ @@ -90,6 +97,13 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { ret = NULL; } #endif +#if defined(__ANDROID__) + if (ret != NULL) { + /* Name this memory as being used by libc */ + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ret, size, + "libc_malloc"); + } +#endif assert(ret == NULL || (addr == NULL && ret != addr) || (addr != NULL && ret == addr)); return ret; @@ -573,6 +587,11 @@ pages_boot(void) { mmap_flags = MAP_PRIVATE | MAP_ANON; #endif +#if defined(__ANDROID__) + /* Android always supports overcommits. */ + os_overcommits = true; +#else /* __ANDROID__ */ + #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT os_overcommits = os_overcommits_sysctl(); #elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY) @@ -586,6 +605,8 @@ pages_boot(void) { os_overcommits = false; #endif +#endif /* __ANDROID__ */ + init_thp_state(); /* Detect lazy purge runtime support. */ |