Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/lib/libc/stdlib/jemalloc.c,v rcsdiff: /ftp/cvs/cvsroot/src/lib/libc/stdlib/jemalloc.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.8 retrieving revision 1.19.6.2 diff -u -p -r1.8 -r1.19.6.2 --- src/lib/libc/stdlib/jemalloc.c 2007/10/16 15:12:16 1.8 +++ src/lib/libc/stdlib/jemalloc.c 2008/06/23 10:46:26 1.19.6.2 @@ -1,4 +1,4 @@ -/* $NetBSD: jemalloc.c,v 1.8 2007/10/16 15:12:16 yamt Exp $ */ +/* $NetBSD: jemalloc.c,v 1.19.6.2 2008/06/23 10:46:26 ad Exp $ */ /*- * Copyright (C) 2006,2007 Jason Evans . @@ -118,7 +118,7 @@ #include /* __FBSDID("$FreeBSD: src/lib/libc/stdlib/malloc.c,v 1.147 2007/06/15 22:00:16 jasone Exp $"); */ -__RCSID("$NetBSD: jemalloc.c,v 1.8 2007/10/16 15:12:16 yamt Exp $"); +__RCSID("$NetBSD: jemalloc.c,v 1.19.6.2 2008/06/23 10:46:26 ad Exp $"); #ifdef __FreeBSD__ #include "libc_private.h" @@ -161,13 +161,33 @@ __RCSID("$NetBSD: jemalloc.c,v 1.8 2007/ #ifdef __NetBSD__ # include -void _malloc_prefork(void); -void _malloc_postfork(void); -ssize_t _write(int, const void *, size_t); -const char *_getprogname(void); +# include "extern.h" + +#define STRERROR_R(a, b, c) __strerror_r(a, b, c); +/* + * A non localized version of strerror, that avoids bringing in + * stdio and the locale code. All the malloc messages are in English + * so why bother? + */ +static int +__strerror_r(int e, char *s, size_t l) +{ + int rval; + size_t slen; + + if (e >= 0 && e < sys_nerr) { + slen = strlcpy(s, sys_errlist[e], l); + rval = 0; + } else { + slen = snprintf_ss(s, l, "Unknown error %u", e); + rval = EINVAL; + } + return slen >= l ? ERANGE : rval; +} #endif #ifdef __FreeBSD__ +#define STRERROR_R(a, b, c) strerror_r(a, b, c); #include "un-namespace.h" #endif @@ -625,7 +645,7 @@ static unsigned ncpus; /* VM page size. */ static size_t pagesize; static size_t pagesize_mask; -static size_t pagesize_2pow; +static int pagesize_2pow; /* Various bin-related settings. */ static size_t bin_maxclass; /* Max size class for bins. */ @@ -757,9 +777,9 @@ static bool opt_junk = false; #endif static bool opt_hint = false; static bool opt_print_stats = false; -static size_t opt_quantum_2pow = QUANTUM_2POW_MIN; -static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT; -static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT; +static int opt_quantum_2pow = QUANTUM_2POW_MIN; +static int opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT; +static int opt_chunk_2pow = CHUNK_2POW_DEFAULT; static bool opt_utrace = false; static bool opt_sysv = false; static bool opt_xmalloc = false; @@ -804,7 +824,6 @@ static void *pages_map_align(void *addr, static void pages_unmap(void *addr, size_t size); static void *chunk_alloc(size_t size); static void chunk_dealloc(void *chunk, size_t size); -static arena_t *choose_arena_hard(void); static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size); static arena_chunk_t *arena_chunk_alloc(arena_t *arena); static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); @@ -926,10 +945,10 @@ static void wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4) { - _write(STDERR_FILENO, p1, strlen(p1)); - _write(STDERR_FILENO, p2, strlen(p2)); - _write(STDERR_FILENO, p3, strlen(p3)); - _write(STDERR_FILENO, p4, strlen(p4)); + write(STDERR_FILENO, p1, strlen(p1)); + write(STDERR_FILENO, p2, strlen(p2)); + write(STDERR_FILENO, p3, strlen(p3)); + write(STDERR_FILENO, p4, strlen(p4)); } void (*_malloc_message)(const char *p1, const char *p2, const char *p3, @@ -1227,8 +1246,8 @@ pages_map_align(void *addr, size_t size, if (munmap(ret, size) == -1) { char buf[STRERROR_BUF]; - strerror_r(errno, buf, sizeof(buf)); - _malloc_message(_getprogname(), + STRERROR_R(errno, buf, sizeof(buf)); + _malloc_message(getprogname(), ": (malloc) Error in munmap(): ", buf, "\n"); if (opt_abort) abort(); @@ -1255,8 +1274,8 @@ pages_unmap(void *addr, size_t size) if (munmap(addr, size) == -1) { char buf[STRERROR_BUF]; - strerror_r(errno, buf, sizeof(buf)); - _malloc_message(_getprogname(), + STRERROR_R(errno, buf, sizeof(buf)); + _malloc_message(getprogname(), ": (malloc) Error in munmap(): ", buf, "\n"); if (opt_abort) abort(); @@ -1503,66 +1522,54 @@ chunk_dealloc(void *chunk, size_t size) */ /* - * Choose an arena based on a per-thread value (fast-path code, calls slow-path - * code if necessary). + * Choose an arena based on a per-thread and (optimistically) per-CPU value. + * + * We maintain at least one block of arenas. Usually there are more. + * The blocks are $ncpu arenas in size. Whole blocks are 'hashed' + * amongst threads. To accomplish this, next_arena advances only in + * ncpu steps. */ -static inline arena_t * -choose_arena(void) +static __noinline arena_t * +choose_arena_hard(void) { - arena_t *ret; + unsigned i, curcpu; + arena_t **map; - /* - * We can only use TLS if this is a PIC library, since for the static - * library version, libc's malloc is used by TLS allocation, which - * introduces a bootstrapping issue. - */ - if (__isthreaded == false) { - /* - * Avoid the overhead of TLS for single-threaded operation. If the - * app switches to threaded mode, the initial thread may end up - * being assigned to some other arena, but this one-time switch - * shouldn't cause significant issues. - */ - return (arenas[0]); + /* Initialize the current block of arenas and advance to next. */ + malloc_mutex_lock(&arenas_mtx); + assert(next_arena % ncpus == 0); + assert(narenas % ncpus == 0); + map = &arenas[next_arena]; + set_arenas_map(map); + for (i = 0; i < ncpus; i++) { + if (arenas[next_arena] == NULL) + arenas_extend(next_arena); + next_arena = (next_arena + 1) % narenas; } + malloc_mutex_unlock(&arenas_mtx); - ret = get_arenas_map(); - if (ret == NULL) - ret = choose_arena_hard(); - - assert(ret != NULL); - return (ret); + /* + * If we were unable to allocate an arena above, then default to + * the first arena, which is always present. + */ + curcpu = thr_curcpu(); + if (map[curcpu] != NULL) + return map[curcpu]; + return arenas[0]; } -/* - * Choose an arena based on a per-thread value (slow-path code only, called - * only by choose_arena()). - */ -static arena_t * -choose_arena_hard(void) +static inline arena_t * +choose_arena(void) { - arena_t *ret; + unsigned curcpu; + arena_t **map; - assert(__isthreaded); - - /* Assign one of the arenas to this thread, in a round-robin fashion. */ - malloc_mutex_lock(&arenas_mtx); - ret = arenas[next_arena]; - if (ret == NULL) - ret = arenas_extend(next_arena); - if (ret == NULL) { - /* - * Make sure that this function never returns NULL, so that - * choose_arena() doesn't have to check for a NULL return - * value. - */ - ret = arenas[0]; - } - next_arena = (next_arena + 1) % narenas; - malloc_mutex_unlock(&arenas_mtx); - set_arenas_map(ret); + map = get_arenas_map(); + curcpu = thr_curcpu(); + if (__predict_true(map != NULL && map[curcpu] != NULL)) + return map[curcpu]; - return (ret); + return choose_arena_hard(); } #ifndef lint @@ -1740,7 +1747,7 @@ arena_run_reg_dalloc(arena_run_t *run, a * The page size is too large for us to use the lookup * table. Use real division. */ - regind = diff / size; + regind = (unsigned)(diff / size); } } else if (size <= ((sizeof(size_invs) / sizeof(unsigned)) << QUANTUM_2POW_MIN) + 2) { @@ -1753,7 +1760,7 @@ arena_run_reg_dalloc(arena_run_t *run, a * if the user increases small_max via the 'S' runtime * configuration option. */ - regind = diff / size; + regind = (unsigned)(diff / size); }; assert(diff == regind * size); assert(regind < bin->nregs); @@ -1779,7 +1786,7 @@ arena_run_split(arena_t *arena, arena_ru run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow); total_pages = chunk->map[run_ind].npages; - need_pages = (size >> pagesize_2pow); + need_pages = (unsigned)(size >> pagesize_2pow); assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; @@ -1894,7 +1901,7 @@ arena_run_alloc(arena_t *arena, size_t s * Search through arena's chunks in address order for a free run that is * large enough. Look for the first fit. */ - need_npages = (size >> pagesize_2pow); + need_npages = (unsigned)(size >> pagesize_2pow); limit_pages = chunk_npages - arena_chunk_header_npages; compl_need_npages = limit_pages - need_npages; /* LINTED */ @@ -1972,7 +1979,7 @@ arena_run_dalloc(arena_t *arena, arena_r >> pagesize_2pow); assert(run_ind >= arena_chunk_header_npages); assert(run_ind < (chunksize >> pagesize_2pow)); - run_pages = (size >> pagesize_2pow); + run_pages = (unsigned)(size >> pagesize_2pow); assert(run_pages == chunk->map[run_ind].npages); /* Subtract pages from count of pages used in chunk. */ @@ -2152,13 +2159,14 @@ arena_bin_run_size_calc(arena_bin_t *bin * header's mask length and the number of regions. */ try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size) - + 1; /* Counter-act the first line of the loop. */ + try_nregs = (unsigned)(((try_run_size - sizeof(arena_run_t)) / + bin->reg_size) + 1); /* Counter-act the first line of the loop. */ do { try_nregs--; try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) + ((try_nregs & ((1 << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0); - try_reg0_offset = try_run_size - (try_nregs * bin->reg_size); + try_reg0_offset = (unsigned)(try_run_size - + (try_nregs * bin->reg_size)); } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1)) > try_reg0_offset); @@ -2174,15 +2182,15 @@ arena_bin_run_size_calc(arena_bin_t *bin /* Try more aggressive settings. */ try_run_size += pagesize; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ + try_nregs = (unsigned)(((try_run_size - sizeof(arena_run_t)) / + bin->reg_size) + 1); /* Counter-act try_nregs-- in loop. */ do { try_nregs--; try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) + ((try_nregs & ((1 << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0); - try_reg0_offset = try_run_size - (try_nregs * - bin->reg_size); + try_reg0_offset = (unsigned)(try_run_size - (try_nregs * + bin->reg_size)); } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1)) > try_reg0_offset); } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL @@ -2318,7 +2326,7 @@ arena_palloc(arena_t *arena, size_t alig assert((size & pagesize_mask) == 0); assert((alignment & pagesize_mask) == 0); - npages = size >> pagesize_2pow; + npages = (unsigned)(size >> pagesize_2pow); malloc_mutex_lock(&arena->mtx); ret = (void *)arena_run_alloc(arena, alloc_size); @@ -2333,7 +2341,7 @@ arena_palloc(arena_t *arena, size_t alig assert((offset & pagesize_mask) == 0); assert(offset < alloc_size); if (offset == 0) { - pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> + pageind = (unsigned)(((uintptr_t)ret - (uintptr_t)chunk) >> pagesize_2pow); /* Update the map for the run to be kept. */ @@ -2344,13 +2352,13 @@ arena_palloc(arena_t *arena, size_t alig /* Trim trailing space. */ arena_palloc_trim(arena, chunk, pageind + npages, - (alloc_size - size) >> pagesize_2pow); + (unsigned)((alloc_size - size) >> pagesize_2pow)); } else { size_t leadsize, trailsize; leadsize = alignment - offset; ret = (void *)((uintptr_t)ret + leadsize); - pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> + pageind = (unsigned)(((uintptr_t)ret - (uintptr_t)chunk) >> pagesize_2pow); /* Update the map for the run to be kept. */ @@ -2360,15 +2368,16 @@ arena_palloc(arena_t *arena, size_t alig } /* Trim leading space. */ - arena_palloc_trim(arena, chunk, pageind - (leadsize >> - pagesize_2pow), leadsize >> pagesize_2pow); + arena_palloc_trim(arena, chunk, + (unsigned)(pageind - (leadsize >> pagesize_2pow)), + (unsigned)(leadsize >> pagesize_2pow)); trailsize = alloc_size - leadsize - size; if (trailsize != 0) { /* Trim trailing space. */ assert(trailsize < alloc_size); arena_palloc_trim(arena, chunk, pageind + npages, - trailsize >> pagesize_2pow); + (unsigned)(trailsize >> pagesize_2pow)); } } @@ -2402,7 +2411,8 @@ arena_salloc(const void *ptr) * affects this function, so we don't need to lock. */ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow); + pageind = (unsigned)(((uintptr_t)ptr - (uintptr_t)chunk) >> + pagesize_2pow); mapelm = &chunk->map[pageind]; if (mapelm->pos != 0 || ptr != (char *)((uintptr_t)chunk) + (pageind << pagesize_2pow)) { @@ -2482,7 +2492,8 @@ arena_dalloc(arena_t *arena, arena_chunk assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow); + pageind = (unsigned)(((uintptr_t)ptr - (uintptr_t)chunk) >> + pagesize_2pow); mapelm = &chunk->map[pageind]; if (mapelm->pos != 0 || ptr != (char *)((uintptr_t)chunk) + (pageind << pagesize_2pow)) { @@ -2666,7 +2677,7 @@ arenas_extend(unsigned ind) * by using arenas[0]. In practice, this is an extremely unlikely * failure. */ - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error initializing arena\n", "", ""); if (opt_abort) abort(); @@ -3294,7 +3305,7 @@ static bool malloc_init_hard(void) { unsigned i, j; - int linklen; + ssize_t linklen; char buf[PATH_MAX + 1]; const char *opts = ""; @@ -3358,8 +3369,8 @@ malloc_init_hard(void) } break; case 1: - if (issetugid() == 0 && (opts = - getenv("MALLOC_OPTIONS")) != NULL) { + if ((opts = getenv("MALLOC_OPTIONS")) != NULL && + issetugid() == 0) { /* * Do nothing; opts is already initialized to * the value of the MALLOC_OPTIONS environment @@ -3486,7 +3497,7 @@ malloc_init_hard(void) cbuf[0] = opts[j]; cbuf[1] = '\0'; - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Unsupported character in " "malloc options: '", cbuf, "'\n"); } @@ -3508,10 +3519,10 @@ malloc_init_hard(void) /* Set bin-related variables. */ bin_maxclass = (pagesize >> 1); assert(opt_quantum_2pow >= TINY_MIN_2POW); - ntbins = opt_quantum_2pow - TINY_MIN_2POW; + ntbins = (unsigned)(opt_quantum_2pow - TINY_MIN_2POW); assert(ntbins <= opt_quantum_2pow); - nqbins = (small_max >> opt_quantum_2pow); - nsbins = pagesize_2pow - opt_small_max_2pow - 1; + nqbins = (unsigned)(small_max >> opt_quantum_2pow); + nsbins = (unsigned)(pagesize_2pow - opt_small_max_2pow - 1); /* Set variables according to the value of opt_quantum_2pow. */ quantum = (1 << opt_quantum_2pow); @@ -3525,13 +3536,13 @@ malloc_init_hard(void) /* Set variables according to the value of opt_chunk_2pow. */ chunksize = (1LU << opt_chunk_2pow); chunksize_mask = chunksize - 1; - chunksize_2pow = opt_chunk_2pow; - chunk_npages = (chunksize >> pagesize_2pow); + chunksize_2pow = (unsigned)opt_chunk_2pow; + chunk_npages = (unsigned)(chunksize >> pagesize_2pow); { unsigned header_size; - header_size = sizeof(arena_chunk_t) + (sizeof(arena_chunk_map_t) - * (chunk_npages - 1)); + header_size = (unsigned)(sizeof(arena_chunk_t) + + (sizeof(arena_chunk_map_t) * (chunk_npages - 1))); arena_chunk_header_npages = (header_size >> pagesize_2pow); if ((header_size & pagesize_mask) != 0) arena_chunk_header_npages++; @@ -3606,7 +3617,7 @@ malloc_init_hard(void) * can handle. */ if (narenas * sizeof(arena_t *) > chunksize) - narenas = chunksize / sizeof(arena_t *); + narenas = (unsigned)(chunksize / sizeof(arena_t *)); } else if (opt_narenas_lshift < 0) { if ((narenas << opt_narenas_lshift) < narenas) narenas <<= opt_narenas_lshift; @@ -3678,7 +3689,7 @@ malloc(size_t size) RETURN: if (ret == NULL) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in malloc(): out of memory\n", "", ""); abort(); @@ -3690,9 +3701,6 @@ RETURN: return (ret); } -/* XXXAD */ -int posix_memalign(void **memptr, size_t alignment, size_t size); - int posix_memalign(void **memptr, size_t alignment, size_t size) { @@ -3706,7 +3714,7 @@ posix_memalign(void **memptr, size_t ali if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in posix_memalign(): " "invalid alignment\n", "", ""); abort(); @@ -3721,7 +3729,7 @@ posix_memalign(void **memptr, size_t ali if (result == NULL) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in posix_memalign(): out of memory\n", "", ""); abort(); @@ -3776,7 +3784,7 @@ calloc(size_t num, size_t size) RETURN: if (ret == NULL) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in calloc(): out of memory\n", "", ""); abort(); @@ -3811,7 +3819,7 @@ realloc(void *ptr, size_t size) if (ret == NULL) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in realloc(): out of " "memory\n", "", ""); abort(); @@ -3826,7 +3834,7 @@ realloc(void *ptr, size_t size) if (ret == NULL) { if (opt_xmalloc) { - _malloc_message(_getprogname(), + _malloc_message(getprogname(), ": (malloc) Error in realloc(): out of " "memory\n", "", ""); abort();