diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 7fecda7c..d5ca5e86 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -53,11 +53,22 @@ This manual describes jemalloc @jemalloc_version@. More information can be found at the jemalloc website. + + The following configuration options are enabled in libc's built-in + jemalloc: , + , , + , , and + . + Additionally, is enabled in development + versions of FreeBSD (controlled by the + MK_MALLOC_PRODUCTION make variable). + SYNOPSIS - #include <jemalloc/jemalloc.h> + #include <stdlib.h> +#include <malloc_np.h> Standard API @@ -3510,4 +3521,18 @@ malloc_conf = "narenas:1";]]> The posix_memalign() function conforms to IEEE Std 1003.1-2001 (POSIX.1). + + HISTORY + The malloc_usable_size() and + posix_memalign() functions first appeared in FreeBSD + 7.0. + + The aligned_alloc(), + malloc_stats_print(), and + mallctl*() functions first appeared in FreeBSD + 10.0. + + The *allocx() functions first appeared in FreeBSD + 11.0. + diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index 7d6053e2..a0e4f5af 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_DECLS_H #define JEMALLOC_INTERNAL_DECLS_H +#include "libc_private.h" +#include "namespace.h" + #include #ifdef _WIN32 # include diff --git a/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h new file mode 100644 index 00000000..0dab1296 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h @@ -0,0 +1,9 @@ +#ifndef __clang__ +# undef JEMALLOC_INTERNAL_UNREACHABLE +# define JEMALLOC_INTERNAL_UNREACHABLE abort + +# undef JEMALLOC_C11_ATOMICS +# undef JEMALLOC_GCC_ATOMIC_ATOMICS +# undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS +# undef JEMALLOC_GCC_U8_SYNC_ATOMICS +#endif diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in index 3418cbfa..53e30dc4 100644 --- a/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/include/jemalloc/internal/jemalloc_preamble.h.in @@ -8,6 +8,9 @@ #include #endif +#include "un-namespace.h" +#include "libc_private.h" + #define JEMALLOC_NO_DEMANGLE #ifdef JEMALLOC_JET # undef JEMALLOC_IS_MALLOC @@ -79,13 +82,7 @@ static const bool config_fill = false #endif ; -static const bool config_lazy_lock = -#ifdef JEMALLOC_LAZY_LOCK - true -#else - false -#endif - ; +static const bool config_lazy_lock = true; static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; static const bool config_prof = #ifdef JEMALLOC_PROF diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 7c24f072..94af1618 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -135,9 +135,6 @@ struct malloc_mutex_s { #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; -#else -# undef isthreaded /* Undo private_namespace.h definition. */ -# define isthreaded true #endif bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, @@ -145,6 +142,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); +bool malloc_mutex_first_thread(void); bool malloc_mutex_boot(void); void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); diff --git a/include/jemalloc/internal/test_hooks.h b/include/jemalloc/internal/test_hooks.h index a6351e59..0780c52f 100644 --- a/include/jemalloc/internal/test_hooks.h +++ b/include/jemalloc/internal/test_hooks.h @@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) -#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) -#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) -#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) -#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) -#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) -#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) -#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) /* Note that this is undef'd and re-define'd in src/prof.c. */ #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 9ba26004..ecfda5d6 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -198,7 +198,8 @@ struct tsd_s { t TSD_MANGLE(n); MALLOC_TSD #undef O -}; +/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */ +} JEMALLOC_ALIGNED(16); JEMALLOC_ALWAYS_INLINE uint8_t tsd_state_get(tsd_t *tsd) { diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h new file mode 100644 index 00000000..b752b0e7 --- /dev/null +++ b/include/jemalloc/jemalloc_FreeBSD.h @@ -0,0 +1,185 @@ +/* + * Override settings that were generated in jemalloc_defs.h as necessary. + */ + +#undef JEMALLOC_OVERRIDE_VALLOC + +#ifndef MALLOC_PRODUCTION +#define JEMALLOC_DEBUG +#endif + +#undef JEMALLOC_DSS + +#undef JEMALLOC_BACKGROUND_THREAD + +/* + * The following are architecture-dependent, so conditionally define them for + * each supported architecture. + */ +#undef JEMALLOC_TLS_MODEL +#undef LG_PAGE +#undef LG_VADDR +#undef LG_SIZEOF_PTR +#undef LG_SIZEOF_INT +#undef LG_SIZEOF_LONG +#undef LG_SIZEOF_INTMAX_T + +#ifdef __i386__ +# define LG_VADDR 32 +# define LG_SIZEOF_PTR 2 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __ia64__ +# define LG_VADDR 64 +# define LG_SIZEOF_PTR 3 +#endif +#ifdef __sparc64__ +# define LG_VADDR 64 +# define LG_SIZEOF_PTR 3 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __amd64__ +# define LG_VADDR 48 +# define LG_SIZEOF_PTR 3 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __arm__ +# define LG_VADDR 32 +# define LG_SIZEOF_PTR 2 +#endif +#ifdef __aarch64__ +# define LG_VADDR 48 +# define LG_SIZEOF_PTR 3 +#endif +#ifdef __mips__ +#ifdef __mips_n64 +# define LG_VADDR 64 +# define LG_SIZEOF_PTR 3 +#else +# define LG_VADDR 32 +# define LG_SIZEOF_PTR 2 +#endif +#endif +#ifdef __powerpc64__ +# define LG_VADDR 64 +# define LG_SIZEOF_PTR 3 +#elif defined(__powerpc__) +# define LG_VADDR 32 +# define LG_SIZEOF_PTR 2 +#endif +#ifdef __riscv +# define LG_VADDR 48 +# define LG_SIZEOF_PTR 3 +#endif + +#ifndef JEMALLOC_TLS_MODEL +# define JEMALLOC_TLS_MODEL /* Default. */ +#endif + +#define LG_PAGE PAGE_SHIFT +#define LG_SIZEOF_INT 2 +#define LG_SIZEOF_LONG LG_SIZEOF_PTR +#define LG_SIZEOF_INTMAX_T 3 + +#undef CPU_SPINWAIT +#include +#include +#define CPU_SPINWAIT cpu_spinwait() + +/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */ +#undef JEMALLOC_LAZY_LOCK +extern int __isthreaded; +#define isthreaded ((bool)__isthreaded) + +/* Mangle. */ +#undef je_malloc +#undef je_calloc +#undef je_posix_memalign +#undef je_aligned_alloc +#undef je_realloc +#undef je_free +#undef je_malloc_usable_size +#undef je_mallocx +#undef je_rallocx +#undef je_xallocx +#undef je_sallocx +#undef je_dallocx +#undef je_sdallocx +#undef je_nallocx +#undef je_mallctl +#undef je_mallctlnametomib +#undef je_mallctlbymib +#undef je_malloc_stats_print +#undef je_allocm +#undef je_rallocm +#undef je_sallocm +#undef je_dallocm +#undef je_nallocm +#define je_malloc __malloc +#define je_calloc __calloc +#define je_posix_memalign __posix_memalign +#define je_aligned_alloc __aligned_alloc +#define je_realloc __realloc +#define je_free __free +#define je_malloc_usable_size __malloc_usable_size +#define je_mallocx __mallocx +#define je_rallocx __rallocx +#define je_xallocx __xallocx +#define je_sallocx __sallocx +#define je_dallocx __dallocx +#define je_sdallocx __sdallocx +#define je_nallocx __nallocx +#define je_mallctl __mallctl +#define je_mallctlnametomib __mallctlnametomib +#define je_mallctlbymib __mallctlbymib +#define je_malloc_stats_print __malloc_stats_print +#define je_allocm __allocm +#define je_rallocm __rallocm +#define je_sallocm __sallocm +#define je_dallocm __dallocm +#define je_nallocm __nallocm +#define open _open +#define read _read +#define write _write +#define close _close +#define pthread_join _pthread_join +#define pthread_once _pthread_once +#define pthread_self _pthread_self +#define pthread_equal _pthread_equal +#define pthread_mutex_lock _pthread_mutex_lock +#define pthread_mutex_trylock _pthread_mutex_trylock +#define pthread_mutex_unlock _pthread_mutex_unlock +#define pthread_cond_init _pthread_cond_init +#define pthread_cond_wait _pthread_cond_wait +#define pthread_cond_timedwait _pthread_cond_timedwait +#define pthread_cond_signal _pthread_cond_signal + +#ifdef JEMALLOC_C_ +/* + * Define 'weak' symbols so that an application can have its own versions + * of malloc, calloc, realloc, free, et al. + */ +__weak_reference(__malloc, malloc); +__weak_reference(__calloc, calloc); +__weak_reference(__posix_memalign, posix_memalign); +__weak_reference(__aligned_alloc, aligned_alloc); +__weak_reference(__realloc, realloc); +__weak_reference(__free, free); +__weak_reference(__malloc_usable_size, malloc_usable_size); +__weak_reference(__mallocx, mallocx); +__weak_reference(__rallocx, rallocx); +__weak_reference(__xallocx, xallocx); +__weak_reference(__sallocx, sallocx); +__weak_reference(__dallocx, dallocx); +__weak_reference(__sdallocx, sdallocx); +__weak_reference(__nallocx, nallocx); +__weak_reference(__mallctl, mallctl); +__weak_reference(__mallctlnametomib, mallctlnametomib); +__weak_reference(__mallctlbymib, mallctlbymib); +__weak_reference(__malloc_stats_print, malloc_stats_print); +__weak_reference(__allocm, allocm); +__weak_reference(__rallocm, rallocm); +__weak_reference(__sallocm, sallocm); +__weak_reference(__dallocm, dallocm); +__weak_reference(__nallocm, nallocm); +#endif diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh index f9438912..47d032c1 100755 --- a/include/jemalloc/jemalloc_rename.sh +++ b/include/jemalloc/jemalloc_rename.sh @@ -19,4 +19,6 @@ done cat <: */ +const char *__malloc_options_1_0 = NULL; +__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); + /* Runtime configuration options. */ const char *je_malloc_conf #ifndef _WIN32 @@ -2660,25 +2664,6 @@ je_realloc(void *ptr, size_t arg_size) { LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); if (unlikely(size == 0)) { - if (ptr != NULL) { - /* realloc(ptr, 0) is equivalent to free(ptr). */ - UTRACE(ptr, 0, 0); - tcache_t *tcache; - tsd_t *tsd = tsd_fetch(); - if (tsd_reentrancy_level_get(tsd) == 0) { - tcache = tcache_get(tsd); - } else { - tcache = NULL; - } - - uintptr_t args[3] = {(uintptr_t)ptr, size}; - hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); - - ifree(tsd, ptr, tcache, true); - - LOG("core.realloc.exit", "result: %p", NULL); - return NULL; - } size = 1; } @@ -3750,6 +3735,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { * End non-standard functions. */ /******************************************************************************/ +/* + * Begin compatibility functions. + */ + +#define ALLOCM_LG_ALIGN(la) (la) +#define ALLOCM_ALIGN(a) (ffsl(a)-1) +#define ALLOCM_ZERO ((int)0x40) +#define ALLOCM_NO_MOVE ((int)0x80) + +#define ALLOCM_SUCCESS 0 +#define ALLOCM_ERR_OOM 1 +#define ALLOCM_ERR_NOT_MOVED 2 + +int +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) { + assert(ptr != NULL); + + void *p = je_mallocx(size, flags); + if (p == NULL) { + return (ALLOCM_ERR_OOM); + } + if (rsize != NULL) { + *rsize = isalloc(tsdn_fetch(), p); + } + *ptr = p; + return ALLOCM_SUCCESS; +} + +int +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { + assert(ptr != NULL); + assert(*ptr != NULL); + assert(size != 0); + assert(SIZE_T_MAX - size >= extra); + + int ret; + bool no_move = flags & ALLOCM_NO_MOVE; + + if (no_move) { + size_t usize = je_xallocx(*ptr, size, extra, flags); + ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; + if (rsize != NULL) { + *rsize = usize; + } + } else { + void *p = je_rallocx(*ptr, size+extra, flags); + if (p != NULL) { + *ptr = p; + ret = ALLOCM_SUCCESS; + } else { + ret = ALLOCM_ERR_OOM; + } + if (rsize != NULL) { + *rsize = isalloc(tsdn_fetch(), *ptr); + } + } + return ret; +} + +int +je_sallocm(const void *ptr, size_t *rsize, int flags) { + assert(rsize != NULL); + *rsize = je_sallocx(ptr, flags); + return ALLOCM_SUCCESS; +} + +int +je_dallocm(void *ptr, int flags) { + je_dallocx(ptr, flags); + return ALLOCM_SUCCESS; +} + +int +je_nallocm(size_t *rsize, size_t size, int flags) { + size_t usize = je_nallocx(size, flags); + if (usize == 0) { + return ALLOCM_ERR_OOM; + } + if (rsize != NULL) { + *rsize = usize; + } + return ALLOCM_SUCCESS; +} + +#undef ALLOCM_LG_ALIGN +#undef ALLOCM_ALIGN +#undef ALLOCM_ZERO +#undef ALLOCM_NO_MOVE + +#undef ALLOCM_SUCCESS +#undef ALLOCM_ERR_OOM +#undef ALLOCM_ERR_NOT_MOVED + +/* + * End compatibility functions. + */ +/******************************************************************************/ /* * The following functions are used by threading libraries for protection of * malloc during fork(). @@ -3919,4 +4001,11 @@ jemalloc_postfork_child(void) { ctl_postfork_child(tsd_tsdn(tsd)); } +void +_malloc_first_thread(void) +{ + + (void)malloc_mutex_first_thread(); +} + /******************************************************************************/ diff --git a/src/malloc_io.c b/src/malloc_io.c index d7cb0f52..cda589c4 100644 --- a/src/malloc_io.c +++ b/src/malloc_io.c @@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) { JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); +JEMALLOC_ATTR(visibility("hidden")) +void +wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { + + wrtmessage(NULL, s1); + wrtmessage(NULL, s2); + wrtmessage(NULL, s3); + wrtmessage(NULL, s4); +} + +void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, + const char *s4) = wrtmessage_1_0; +__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); + /* * Wrapper around malloc_message() that avoids the need for * je_malloc_message(...) throughout the code. diff --git a/src/mutex.c b/src/mutex.c index 3f920f5b..88a7730c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread, #ifdef JEMALLOC_MUTEX_INIT_CB JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void *(calloc_cb)(size_t, size_t)); + +#pragma weak _pthread_mutex_init_calloc_cb +int +_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)) +{ + + return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) + __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, + calloc_cb)); +} #endif void @@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1, } } +bool +malloc_mutex_first_thread(void) { + +#ifndef JEMALLOC_MUTEX_INIT_CB + return (malloc_mutex_first_thread()); +#else + return (false); +#endif +} + bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {