src/lib/libc/stdlib/jemalloc.c - diff

Return to jemalloc.c CVS log

Up to [cvs.NetBSD.org] / src / lib / libc / stdlib

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/lib/libc/stdlib/jemalloc.c between version 1.24.6.1 and 1.44.2.1

version 1.24.6.1, 2012/04/23 16:49:02

version 1.44.2.1, 2018/07/28 04:37:22

Line 143 __RCSID("$NetBSD$");

#ifdef __FreeBSD__

#include <machine/atomic.h>

#include <machine/cpufunc.h>

#endif

#include <machine/vmparam.h>

#endif

#include <errno.h>

#include <limits.h>

Line 163 __RCSID("$NetBSD$");

# include <reentrant.h>

# include "extern.h"

#define STRERROR_R(a, b, c) __strerror_r(a, b, c);

#define STRERROR_R(a, b, c) strerror_r_ss(a, b, c);

* A non localized version of strerror, that avoids bringing in

* stdio and the locale code. All the malloc messages are in English

* so why bother?

static int

__strerror_r(int e, char *s, size_t l)

{

int rval;

size_t slen;

if (e >= 0 && e < sys_nerr) {

slen = strlcpy(s, sys_errlist[e], l);

rval = 0;

} else {

slen = snprintf_ss(s, l, "Unknown error %u", e);

rval = EINVAL;

}

return slen >= l ? ERANGE : rval;

}

#endif

#ifdef __FreeBSD__

Line 216 __strerror_r(int e, char *s, size_t l)

Line 196 __strerror_r(int e, char *s, size_t l)

#define STRERROR_BUF 64

/* Minimum alignment of allocations is 2^QUANTUM_2POW_MIN bytes. */

* If you touch the TINY_MIN_2POW definition for any architecture, please

* make sure to adjust the corresponding definition for JEMALLOC_TINY_MIN_2POW

* in the gcc 4.8 tree in dist/gcc/tree-ssa-ccp.c and verify that a native

* gcc is still buildable!

#ifdef __i386__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

Line 225 __strerror_r(int e, char *s, size_t l)

Line 213 __strerror_r(int e, char *s, size_t l)

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 3

#endif

#ifdef __aarch64__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 3

# define NO_TLS

#endif

#ifdef __alpha__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 3

# define TINY_MIN_2POW 3

# define NO_TLS

#endif

#ifdef __sparc64__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 3

# define TINY_MIN_2POW 3

# define NO_TLS

#endif

#ifdef __amd64__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 3

# define TINY_MIN_2POW 3

#endif

#ifdef __arm__

# define QUANTUM_2POW_MIN 3

# define SIZEOF_PTR_2POW 2

# define USE_BRK

# ifdef __ARM_EABI__

# define TINY_MIN_2POW 3

# endif

# define NO_TLS

#endif

#ifdef __powerpc__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

# define USE_BRK

# define TINY_MIN_2POW 3

#endif

#if defined(__sparc__) && !defined(__sparc64__)

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

# define USE_BRK

#endif

#ifdef __or1k__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

# define USE_BRK

#endif

#ifdef __vax__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

Line 270 __strerror_r(int e, char *s, size_t l)

Line 275 __strerror_r(int e, char *s, size_t l)

# define SIZEOF_PTR_2POW 2

# define USE_BRK

#endif

#ifdef __mips__

#if defined(__mips__) || defined(__riscv__)

# ifdef _LP64

# define SIZEOF_PTR_2POW 3

# define TINY_MIN_2POW 3

# else

# define SIZEOF_PTR_2POW 2

# endif

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

# define USE_BRK

#endif

#ifdef __hppa__

# define QUANTUM_2POW_MIN 4

# define SIZEOF_PTR_2POW 2

# define TINY_MIN_2POW 4

# define SIZEOF_PTR_2POW 2

# define USE_BRK

#endif

Line 288 __strerror_r(int e, char *s, size_t l)

Line 299 __strerror_r(int e, char *s, size_t l)

# define SIZEOF_INT_2POW 2

#endif

/* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */

#if (!defined(PIC) && !defined(NO_TLS))

# define NO_TLS

#endif

* Size and alignment of memory chunks that are allocated by the OS's virtual

* memory system.

Line 308 __strerror_r(int e, char *s, size_t l)

Line 314 __strerror_r(int e, char *s, size_t l)

#define CACHELINE ((size_t)(1 << CACHELINE_2POW))

/* Smallest size class to support. */

#define TINY_MIN_2POW 1

#ifndef TINY_MIN_2POW

#define TINY_MIN_2POW 2

#endif

* Maximum size class that is a multiple of the quantum, but not (necessarily)

Line 365 static malloc_mutex_t init_lock = {_SPIN

Line 373 static malloc_mutex_t init_lock = {_SPIN

/* Set to true once the allocator has been initialized. */

static bool malloc_initialized = false;

#ifdef _REENTRANT

/* Used to avoid initialization races. */

static mutex_t init_lock = MUTEX_INITIALIZER;

#endif

/******************************************************************************/

Line 677 static size_t arena_maxclass; /* Max si

Line 687 static size_t arena_maxclass; /* Max si

* Chunks.

#ifdef _REENTRANT

/* Protects chunk-related data structures. */

static malloc_mutex_t chunks_mtx;

#endif

/* Tree of chunks that are stand-alone huge allocations. */

static chunk_tree_t huge;

Line 729 static void *base_pages;

Line 741 static void *base_pages;

static void *base_next_addr;

static void *base_past_addr; /* Addr immediately past base_pages. */

static chunk_node_t *base_chunk_nodes; /* LIFO cache of chunk nodes. */

#ifdef _REENTRANT

static malloc_mutex_t base_mtx;

#endif

#ifdef MALLOC_STATS

static size_t base_mapped;

#endif

Line 746 static size_t base_mapped;

Line 760 static size_t base_mapped;

static arena_t **arenas;

static unsigned narenas;

static unsigned next_arena;

#ifdef _REENTRANT

static malloc_mutex_t arenas_mtx; /* Protects arenas initialization. */

#endif

#ifndef NO_TLS

* Map of pthread_self() --> arenas[???], used for selecting an arena to use

* for allocations.

static __thread arena_t *arenas_map;

#ifndef NO_TLS

#define get_arenas_map() (arenas_map)

static __thread arena_t **arenas_map;

#define set_arenas_map(x) (arenas_map = x)

#else

static thread_key_t arenas_map_key;

static arena_t **arenas_map;

#define get_arenas_map() thr_getspecific(arenas_map_key)

#endif

#define set_arenas_map(x) thr_setspecific(arenas_map_key, x)

#if !defined(NO_TLS) || !defined(_REENTRANT)

# define get_arenas_map() (arenas_map)

# define set_arenas_map(x) (arenas_map = x)

#else

static thread_key_t arenas_map_key = -1;

static inline arena_t **

get_arenas_map(void)

{

if (!__isthreaded)

return arenas_map;

if (arenas_map_key == -1) {

(void)thr_keycreate(&arenas_map_key, NULL);

if (arenas_map != NULL) {

thr_setspecific(arenas_map_key, arenas_map);

arenas_map = NULL;

}

return thr_getspecific(arenas_map_key);

}

static __inline void

set_arenas_map(arena_t **a)

{

if (!__isthreaded) {

arenas_map = a;

return;

}

if (arenas_map_key == -1) {

(void)thr_keycreate(&arenas_map_key, NULL);

if (arenas_map != NULL) {

_DIAGASSERT(arenas_map == a);

arenas_map = NULL;

}

thr_setspecific(arenas_map_key, a);

}

#endif

#ifdef MALLOC_STATS

Line 816 static void wrtmessage(const char *p1, c

Line 872 static void wrtmessage(const char *p1, c

#ifdef MALLOC_STATS

static void malloc_printf(const char *format, ...);

#endif

static char *umax2s(uintmax_t x, char *s);

static char *size_t2s(size_t x, char *s);

static bool base_pages_alloc(size_t minsize);

static void *base_alloc(size_t size);

static chunk_node_t *base_chunk_node_alloc(void);

Line 978 malloc_printf(const char *format, ...)

Line 1034 malloc_printf(const char *format, ...)

* We don't want to depend on vsnprintf() for production builds, since that can

* cause unnecessary bloat for static binaries. umax2s() provides minimal

* cause unnecessary bloat for static binaries. size_t2s() provides minimal

* integer printing functionality, so that malloc_printf() use can be limited to

* MALLOC_STATS code.

#define UMAX2S_BUFSIZE 21

static char *

umax2s(uintmax_t x, char *s)

size_t2s(size_t x, char *s)

{

unsigned i;

/* Make sure UMAX2S_BUFSIZE is large enough. */

/* LINTED */

assert(sizeof(uintmax_t) <= 8);

assert(sizeof(size_t) <= 8);

i = UMAX2S_BUFSIZE - 1;

s[i] = '\0';

Line 1586 arena_chunk_comp(arena_chunk_t *a, arena

Line 1642 arena_chunk_comp(arena_chunk_t *a, arena

assert(a != NULL);

assert(b != NULL);

if (a->max_frun_npages < b->max_frun_npages)

return -1;

if (a->max_frun_npages > b->max_frun_npages)

return 1;

if ((uintptr_t)a < (uintptr_t)b)

return (-1);

else if (a == b)

Line 1643 arena_run_reg_alloc(arena_run_t *run, ar

Line 1704 arena_run_reg_alloc(arena_run_t *run, ar

+ (bin->reg_size * regind));

/* Clear bit. */

mask ^= (1 << bit);

mask ^= (1U << bit);

run->regs_mask[i] = mask;

return (ret);

Line 1660 arena_run_reg_alloc(arena_run_t *run, ar

Line 1721 arena_run_reg_alloc(arena_run_t *run, ar

+ (bin->reg_size * regind));

/* Clear bit. */

mask ^= (1 << bit);

mask ^= (1U << bit);

run->regs_mask[i] = mask;

Line 1775 arena_run_reg_dalloc(arena_run_t *run, a

Line 1836 arena_run_reg_dalloc(arena_run_t *run, a

if (elm < run->regs_minelm)

run->regs_minelm = elm;

bit = regind - (elm << (SIZEOF_INT_2POW + 3));

assert((run->regs_mask[elm] & (1 << bit)) == 0);

assert((run->regs_mask[elm] & (1U << bit)) == 0);

run->regs_mask[elm] |= (1 << bit);

run->regs_mask[elm] |= (1U << bit);

#undef SIZE_INV

#undef SIZE_INV_SHIFT

}

Line 1837 arena_chunk_alloc(arena_t *arena)

Line 1898 arena_chunk_alloc(arena_t *arena)

chunk->arena = arena;

/* LINTED */

RB_INSERT(arena_chunk_tree_s, &arena->chunks, chunk);

* Claim that no pages are in use, since the header is merely

* overhead.

Line 1859 arena_chunk_alloc(arena_t *arena)

Line 1917 arena_chunk_alloc(arena_t *arena)

chunk->map[chunk_npages - 1].npages = chunk_npages -

arena_chunk_header_npages;

chunk->map[chunk_npages - 1].pos = POS_FREE;

RB_INSERT(arena_chunk_tree_s, &arena->chunks, chunk);

}

return (chunk);

Line 1895 arena_chunk_dealloc(arena_t *arena, aren

Line 1955 arena_chunk_dealloc(arena_t *arena, aren

static arena_run_t *

arena_run_alloc(arena_t *arena, size_t size)

{

arena_chunk_t *chunk;

arena_chunk_t *chunk, *chunk_tmp;

arena_run_t *run;

unsigned need_npages, limit_pages, compl_need_npages;

unsigned need_npages;

assert(size <= (chunksize - (arena_chunk_header_npages <<

pagesize_2pow)));

assert((size & pagesize_mask) == 0);

* Search through arena's chunks in address order for a free run that is

* Search through the arena chunk tree for a large enough free run.

* large enough. Look for the first fit.

* Tree order ensures that any exact fit is picked immediately or

* otherwise the lowest address of the next size.

need_npages = (unsigned)(size >> pagesize_2pow);

limit_pages = chunk_npages - arena_chunk_header_npages;

compl_need_npages = limit_pages - need_npages;

/* LINTED */

RB_FOREACH(chunk, arena_chunk_tree_s, &arena->chunks) {

for (;;) {

chunk_tmp = RB_ROOT(&arena->chunks);

chunk = NULL;

while (chunk_tmp) {

if (chunk_tmp->max_frun_npages == need_npages) {

chunk = chunk_tmp;

break;

}

if (chunk_tmp->max_frun_npages < need_npages) {

chunk_tmp = RB_RIGHT(chunk_tmp, link);

continue;

}

chunk = chunk_tmp;

chunk_tmp = RB_LEFT(chunk, link);

}

if (chunk == NULL)

break;

* Avoid searching this chunk if there are not enough

* At this point, the chunk must have a cached run size large

* contiguous free pages for there to possibly be a large

* enough to fit the allocation.

* enough free run.

if (chunk->pages_used <= compl_need_npages &&

assert(need_npages <= chunk->max_frun_npages);

need_npages <= chunk->max_frun_npages) {

{

arena_chunk_map_t *mapelm;

unsigned i;

unsigned max_frun_npages = 0;

Line 1956 arena_run_alloc(arena_t *arena, size_t s

Line 2030 arena_run_alloc(arena_t *arena, size_t s

* chunk->min_frun_ind was already reset above (if

* necessary).

RB_REMOVE(arena_chunk_tree_s, &arena->chunks, chunk);

chunk->max_frun_npages = max_frun_npages;

RB_INSERT(arena_chunk_tree_s, &arena->chunks, chunk);

}

Line 2039 arena_run_dalloc(arena_t *arena, arena_r

Line 2115 arena_run_dalloc(arena_t *arena, arena_r

assert(chunk->map[run_ind + run_pages - 1].pos == POS_FREE);

}

if (chunk->map[run_ind].npages > chunk->max_frun_npages)

if (chunk->map[run_ind].npages > chunk->max_frun_npages) {

RB_REMOVE(arena_chunk_tree_s, &arena->chunks, chunk);

chunk->max_frun_npages = chunk->map[run_ind].npages;

RB_INSERT(arena_chunk_tree_s, &arena->chunks, chunk);

}

if (run_ind < chunk->min_frun_ind)

chunk->min_frun_ind = run_ind;

Line 3218 malloc_print_stats(void)

Line 3297 malloc_print_stats(void)

opt_xmalloc ? "X" : "x",

opt_zero ? "Z\n" : "z\n");

_malloc_message("CPUs: ", umax2s(ncpus, s), "\n", "");

_malloc_message("CPUs: ", size_t2s(ncpus, s), "\n", "");

_malloc_message("Max arenas: ", umax2s(narenas, s), "\n", "");

_malloc_message("Max arenas: ", size_t2s(narenas, s), "\n", "");

_malloc_message("Pointer size: ", umax2s(sizeof(void *), s),

_malloc_message("Pointer size: ", size_t2s(sizeof(void *), s),

"\n", "");

_malloc_message("Quantum size: ", umax2s(quantum, s), "\n", "");

_malloc_message("Quantum size: ", size_t2s(quantum, s), "\n", "");

_malloc_message("Max small size: ", umax2s(small_max, s), "\n",

_malloc_message("Max small size: ", size_t2s(small_max, s), "\n",

"");

_malloc_message("Chunk size: ", umax2s(chunksize, s), "", "");

_malloc_message("Chunk size: ", size_t2s(chunksize, s), "", "");

_malloc_message(" (2^", umax2s((size_t)opt_chunk_2pow, s),

_malloc_message(" (2^", size_t2s((size_t)opt_chunk_2pow, s),

")\n", "");

#ifdef MALLOC_STATS

Line 3617 malloc_init_hard(void)

Line 3696 malloc_init_hard(void)

opt_narenas_lshift += 2;

}

#ifdef NO_TLS

/* Initialize arena key. */

(void)thr_keycreate(&arenas_map_key, NULL);

#endif

/* Determine how many arenas to use. */

narenas = ncpus;

if (opt_narenas_lshift > 0) {

Line 3916 _malloc_prefork(void)

Line 3990 _malloc_prefork(void)

if (arenas[i] != NULL)

malloc_mutex_lock(&arenas[i]->mtx);

}

malloc_mutex_unlock(&arenas_mtx);

malloc_mutex_lock(&base_mtx);

Line 3934 _malloc_postfork(void)

Line 4007 _malloc_postfork(void)

malloc_mutex_unlock(&base_mtx);

malloc_mutex_lock(&arenas_mtx);

for (i = 0; i < narenas; i++) {

if (arenas[i] != NULL)

malloc_mutex_unlock(&arenas[i]->mtx);

CVSweb <webmaster@jp.NetBSD.org>