1#define JEMALLOC_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/atomic.h"
7#include "jemalloc/internal/ctl.h"
8#include "jemalloc/internal/extent_dss.h"
9#include "jemalloc/internal/extent_mmap.h"
10#include "jemalloc/internal/hook.h"
11#include "jemalloc/internal/jemalloc_internal_types.h"
12#include "jemalloc/internal/log.h"
13#include "jemalloc/internal/malloc_io.h"
14#include "jemalloc/internal/mutex.h"
15#include "jemalloc/internal/rtree.h"
16#include "jemalloc/internal/safety_check.h"
17#include "jemalloc/internal/sc.h"
18#include "jemalloc/internal/spin.h"
19#include "jemalloc/internal/sz.h"
20#include "jemalloc/internal/ticker.h"
21#include "jemalloc/internal/util.h"
22
23/******************************************************************************/
24/* Data. */
25
26/* Runtime configuration options. */
27const char *je_malloc_conf
28#ifndef _WIN32
29 JEMALLOC_ATTR(weak)
30#endif
31 ;
32bool opt_abort =
33#ifdef JEMALLOC_DEBUG
34 true
35#else
36 false
37#endif
38 ;
39bool opt_abort_conf =
40#ifdef JEMALLOC_DEBUG
41 true
42#else
43 false
44#endif
45 ;
46/* Intentionally default off, even with debug builds. */
47bool opt_confirm_conf = false;
48const char *opt_junk =
49#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
50 "true"
51#else
52 "false"
53#endif
54 ;
55bool opt_junk_alloc =
56#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
57 true
58#else
59 false
60#endif
61 ;
62bool opt_junk_free =
63#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
64 true
65#else
66 false
67#endif
68 ;
69
70bool opt_utrace = false;
71bool opt_xmalloc = false;
72bool opt_zero = false;
73unsigned opt_narenas = 0;
74
75unsigned ncpus;
76
77/* Protects arenas initialization. */
78malloc_mutex_t arenas_lock;
79/*
80 * Arenas that are used to service external requests. Not all elements of the
81 * arenas array are necessarily used; arenas are created lazily as needed.
82 *
83 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
84 * arenas. arenas[narenas_auto..narenas_total) are only used if the application
85 * takes some action to create them and allocate from them.
86 *
87 * Points to an arena_t.
88 */
89JEMALLOC_ALIGNED(CACHELINE)
90atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
91static atomic_u_t narenas_total; /* Use narenas_total_*(). */
92/* Below three are read-only after initialization. */
93static arena_t *a0; /* arenas[0]. */
94unsigned narenas_auto;
95unsigned manual_arena_base;
96
97typedef enum {
98 malloc_init_uninitialized = 3,
99 malloc_init_a0_initialized = 2,
100 malloc_init_recursible = 1,
101 malloc_init_initialized = 0 /* Common case --> jnz. */
102} malloc_init_t;
103static malloc_init_t malloc_init_state = malloc_init_uninitialized;
104
105/* False should be the common case. Set to true to trigger initialization. */
106bool malloc_slow = true;
107
108/* When malloc_slow is true, set the corresponding bits for sanity check. */
109enum {
110 flag_opt_junk_alloc = (1U),
111 flag_opt_junk_free = (1U << 1),
112 flag_opt_zero = (1U << 2),
113 flag_opt_utrace = (1U << 3),
114 flag_opt_xmalloc = (1U << 4)
115};
116static uint8_t malloc_slow_flags;
117
118#ifdef JEMALLOC_THREADED_INIT
119/* Used to let the initializing thread recursively allocate. */
120# define NO_INITIALIZER ((unsigned long)0)
121# define INITIALIZER pthread_self()
122# define IS_INITIALIZER (malloc_initializer == pthread_self())
123static pthread_t malloc_initializer = NO_INITIALIZER;
124#else
125# define NO_INITIALIZER false
126# define INITIALIZER true
127# define IS_INITIALIZER malloc_initializer
128static bool malloc_initializer = NO_INITIALIZER;
129#endif
130
131/* Used to avoid initialization races. */
132#ifdef _WIN32
133#if _WIN32_WINNT >= 0x0600
134static malloc_mutex_t init_lock = SRWLOCK_INIT;
135#else
136static malloc_mutex_t init_lock;
137static bool init_lock_initialized = false;
138
139JEMALLOC_ATTR(constructor)
140static void WINAPI
141_init_init_lock(void) {
142 /*
143 * If another constructor in the same binary is using mallctl to e.g.
144 * set up extent hooks, it may end up running before this one, and
145 * malloc_init_hard will crash trying to lock the uninitialized lock. So
146 * we force an initialization of the lock in malloc_init_hard as well.
147 * We don't try to care about atomicity of the accessed to the
148 * init_lock_initialized boolean, since it really only matters early in
149 * the process creation, before any separate thread normally starts
150 * doing anything.
151 */
152 if (!init_lock_initialized) {
153 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
154 malloc_mutex_rank_exclusive);
155 }
156 init_lock_initialized = true;
157}
158
159#ifdef _MSC_VER
160# pragma section(".CRT$XCU", read)
161JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
162static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
163#endif
164#endif
165#else
166static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
167#endif
168
169typedef struct {
170 void *p; /* Input pointer (as in realloc(p, s)). */
171 size_t s; /* Request size. */
172 void *r; /* Result pointer. */
173} malloc_utrace_t;
174
175#ifdef JEMALLOC_UTRACE
176# define UTRACE(a, b, c) do { \
177 if (unlikely(opt_utrace)) { \
178 int utrace_serrno = errno; \
179 malloc_utrace_t ut; \
180 ut.p = (a); \
181 ut.s = (b); \
182 ut.r = (c); \
183 utrace(&ut, sizeof(ut)); \
184 errno = utrace_serrno; \
185 } \
186} while (0)
187#else
188# define UTRACE(a, b, c)
189#endif
190
191/* Whether encountered any invalid config options. */
192static bool had_conf_error = false;
193
194/******************************************************************************/
195/*
196 * Function prototypes for static functions that are referenced prior to
197 * definition.
198 */
199
200static bool malloc_init_hard_a0(void);
201static bool malloc_init_hard(void);
202
203/******************************************************************************/
204/*
205 * Begin miscellaneous support functions.
206 */
207
208bool
209malloc_initialized(void) {
210 return (malloc_init_state == malloc_init_initialized);
211}
212
213JEMALLOC_ALWAYS_INLINE bool
214malloc_init_a0(void) {
215 if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
216 return malloc_init_hard_a0();
217 }
218 return false;
219}
220
221JEMALLOC_ALWAYS_INLINE bool
222malloc_init(void) {
223 if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
224 return true;
225 }
226 return false;
227}
228
229/*
230 * The a0*() functions are used instead of i{d,}alloc() in situations that
231 * cannot tolerate TLS variable access.
232 */
233
234static void *
235a0ialloc(size_t size, bool zero, bool is_internal) {
236 if (unlikely(malloc_init_a0())) {
237 return NULL;
238 }
239
240 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
241 is_internal, arena_get(TSDN_NULL, 0, true), true);
242}
243
244static void
245a0idalloc(void *ptr, bool is_internal) {
246 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
247}
248
249void *
250a0malloc(size_t size) {
251 return a0ialloc(size, false, true);
252}
253
254void
255a0dalloc(void *ptr) {
256 a0idalloc(ptr, true);
257}
258
259/*
260 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
261 * situations that cannot tolerate TLS variable access (TLS allocation and very
262 * early internal data structure initialization).
263 */
264
265void *
266bootstrap_malloc(size_t size) {
267 if (unlikely(size == 0)) {
268 size = 1;
269 }
270
271 return a0ialloc(size, false, false);
272}
273
274void *
275bootstrap_calloc(size_t num, size_t size) {
276 size_t num_size;
277
278 num_size = num * size;
279 if (unlikely(num_size == 0)) {
280 assert(num == 0 || size == 0);
281 num_size = 1;
282 }
283
284 return a0ialloc(num_size, true, false);
285}
286
287void
288bootstrap_free(void *ptr) {
289 if (unlikely(ptr == NULL)) {
290 return;
291 }
292
293 a0idalloc(ptr, false);
294}
295
296void
297arena_set(unsigned ind, arena_t *arena) {
298 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
299}
300
301static void
302narenas_total_set(unsigned narenas) {
303 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
304}
305
306static void
307narenas_total_inc(void) {
308 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
309}
310
311unsigned
312narenas_total_get(void) {
313 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
314}
315
316/* Create a new arena and insert it into the arenas array at index ind. */
317static arena_t *
318arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
319 arena_t *arena;
320
321 assert(ind <= narenas_total_get());
322 if (ind >= MALLOCX_ARENA_LIMIT) {
323 return NULL;
324 }
325 if (ind == narenas_total_get()) {
326 narenas_total_inc();
327 }
328
329 /*
330 * Another thread may have already initialized arenas[ind] if it's an
331 * auto arena.
332 */
333 arena = arena_get(tsdn, ind, false);
334 if (arena != NULL) {
335 assert(arena_is_auto(arena));
336 return arena;
337 }
338
339 /* Actually initialize the arena. */
340 arena = arena_new(tsdn, ind, extent_hooks);
341
342 return arena;
343}
344
345static void
346arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
347 if (ind == 0) {
348 return;
349 }
350 /*
351 * Avoid creating a new background thread just for the huge arena, which
352 * purges eagerly by default.
353 */
354 if (have_background_thread && !arena_is_huge(ind)) {
355 if (background_thread_create(tsdn_tsd(tsdn), ind)) {
356 malloc_printf("<jemalloc>: error in background thread "
357 "creation for arena %u. Abort.\n", ind);
358 abort();
359 }
360 }
361}
362
363arena_t *
364arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
365 arena_t *arena;
366
367 malloc_mutex_lock(tsdn, &arenas_lock);
368 arena = arena_init_locked(tsdn, ind, extent_hooks);
369 malloc_mutex_unlock(tsdn, &arenas_lock);
370
371 arena_new_create_background_thread(tsdn, ind);
372
373 return arena;
374}
375
376static void
377arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
378 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
379 arena_nthreads_inc(arena, internal);
380
381 if (internal) {
382 tsd_iarena_set(tsd, arena);
383 } else {
384 tsd_arena_set(tsd, arena);
385 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1,
386 ATOMIC_RELAXED);
387 tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
388 for (unsigned i = 0; i < SC_NBINS; i++) {
389 assert(bin_infos[i].n_shards > 0 &&
390 bin_infos[i].n_shards <= BIN_SHARDS_MAX);
391 bins->binshard[i] = shard % bin_infos[i].n_shards;
392 }
393 }
394}
395
396void
397arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) {
398 arena_t *oldarena, *newarena;
399
400 oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
401 newarena = arena_get(tsd_tsdn(tsd), newind, false);
402 arena_nthreads_dec(oldarena, false);
403 arena_nthreads_inc(newarena, false);
404 tsd_arena_set(tsd, newarena);
405}
406
407static void
408arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
409 arena_t *arena;
410
411 arena = arena_get(tsd_tsdn(tsd), ind, false);
412 arena_nthreads_dec(arena, internal);
413
414 if (internal) {
415 tsd_iarena_set(tsd, NULL);
416 } else {
417 tsd_arena_set(tsd, NULL);
418 }
419}
420
421arena_tdata_t *
422arena_tdata_get_hard(tsd_t *tsd, unsigned ind) {
423 arena_tdata_t *tdata, *arenas_tdata_old;
424 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
425 unsigned narenas_tdata_old, i;
426 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
427 unsigned narenas_actual = narenas_total_get();
428
429 /*
430 * Dissociate old tdata array (and set up for deallocation upon return)
431 * if it's too small.
432 */
433 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
434 arenas_tdata_old = arenas_tdata;
435 narenas_tdata_old = narenas_tdata;
436 arenas_tdata = NULL;
437 narenas_tdata = 0;
438 tsd_arenas_tdata_set(tsd, arenas_tdata);
439 tsd_narenas_tdata_set(tsd, narenas_tdata);
440 } else {
441 arenas_tdata_old = NULL;
442 narenas_tdata_old = 0;
443 }
444
445 /* Allocate tdata array if it's missing. */
446 if (arenas_tdata == NULL) {
447 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
448 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
449
450 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
451 *arenas_tdata_bypassp = true;
452 arenas_tdata = (arena_tdata_t *)a0malloc(
453 sizeof(arena_tdata_t) * narenas_tdata);
454 *arenas_tdata_bypassp = false;
455 }
456 if (arenas_tdata == NULL) {
457 tdata = NULL;
458 goto label_return;
459 }
460 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
461 tsd_arenas_tdata_set(tsd, arenas_tdata);
462 tsd_narenas_tdata_set(tsd, narenas_tdata);
463 }
464
465 /*
466 * Copy to tdata array. It's possible that the actual number of arenas
467 * has increased since narenas_total_get() was called above, but that
468 * causes no correctness issues unless two threads concurrently execute
469 * the arenas.create mallctl, which we trust mallctl synchronization to
470 * prevent.
471 */
472
473 /* Copy/initialize tickers. */
474 for (i = 0; i < narenas_actual; i++) {
475 if (i < narenas_tdata_old) {
476 ticker_copy(&arenas_tdata[i].decay_ticker,
477 &arenas_tdata_old[i].decay_ticker);
478 } else {
479 ticker_init(&arenas_tdata[i].decay_ticker,
480 DECAY_NTICKS_PER_UPDATE);
481 }
482 }
483 if (narenas_tdata > narenas_actual) {
484 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
485 * (narenas_tdata - narenas_actual));
486 }
487
488 /* Read the refreshed tdata array. */
489 tdata = &arenas_tdata[ind];
490label_return:
491 if (arenas_tdata_old != NULL) {
492 a0dalloc(arenas_tdata_old);
493 }
494 return tdata;
495}
496
497/* Slow path, called only by arena_choose(). */
498arena_t *
499arena_choose_hard(tsd_t *tsd, bool internal) {
500 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
501
502 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
503 unsigned choose = percpu_arena_choose();
504 ret = arena_get(tsd_tsdn(tsd), choose, true);
505 assert(ret != NULL);
506 arena_bind(tsd, arena_ind_get(ret), false);
507 arena_bind(tsd, arena_ind_get(ret), true);
508
509 return ret;
510 }
511
512 if (narenas_auto > 1) {
513 unsigned i, j, choose[2], first_null;
514 bool is_new_arena[2];
515
516 /*
517 * Determine binding for both non-internal and internal
518 * allocation.
519 *
520 * choose[0]: For application allocation.
521 * choose[1]: For internal metadata allocation.
522 */
523
524 for (j = 0; j < 2; j++) {
525 choose[j] = 0;
526 is_new_arena[j] = false;
527 }
528
529 first_null = narenas_auto;
530 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
531 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
532 for (i = 1; i < narenas_auto; i++) {
533 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
534 /*
535 * Choose the first arena that has the lowest
536 * number of threads assigned to it.
537 */
538 for (j = 0; j < 2; j++) {
539 if (arena_nthreads_get(arena_get(
540 tsd_tsdn(tsd), i, false), !!j) <
541 arena_nthreads_get(arena_get(
542 tsd_tsdn(tsd), choose[j], false),
543 !!j)) {
544 choose[j] = i;
545 }
546 }
547 } else if (first_null == narenas_auto) {
548 /*
549 * Record the index of the first uninitialized
550 * arena, in case all extant arenas are in use.
551 *
552 * NB: It is possible for there to be
553 * discontinuities in terms of initialized
554 * versus uninitialized arenas, due to the
555 * "thread.arena" mallctl.
556 */
557 first_null = i;
558 }
559 }
560
561 for (j = 0; j < 2; j++) {
562 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
563 choose[j], false), !!j) == 0 || first_null ==
564 narenas_auto) {
565 /*
566 * Use an unloaded arena, or the least loaded
567 * arena if all arenas are already initialized.
568 */
569 if (!!j == internal) {
570 ret = arena_get(tsd_tsdn(tsd),
571 choose[j], false);
572 }
573 } else {
574 arena_t *arena;
575
576 /* Initialize a new arena. */
577 choose[j] = first_null;
578 arena = arena_init_locked(tsd_tsdn(tsd),
579 choose[j],
580 (extent_hooks_t *)&extent_hooks_default);
581 if (arena == NULL) {
582 malloc_mutex_unlock(tsd_tsdn(tsd),
583 &arenas_lock);
584 return NULL;
585 }
586 is_new_arena[j] = true;
587 if (!!j == internal) {
588 ret = arena;
589 }
590 }
591 arena_bind(tsd, choose[j], !!j);
592 }
593 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
594
595 for (j = 0; j < 2; j++) {
596 if (is_new_arena[j]) {
597 assert(choose[j] > 0);
598 arena_new_create_background_thread(
599 tsd_tsdn(tsd), choose[j]);
600 }
601 }
602
603 } else {
604 ret = arena_get(tsd_tsdn(tsd), 0, false);
605 arena_bind(tsd, 0, false);
606 arena_bind(tsd, 0, true);
607 }
608
609 return ret;
610}
611
612void
613iarena_cleanup(tsd_t *tsd) {
614 arena_t *iarena;
615
616 iarena = tsd_iarena_get(tsd);
617 if (iarena != NULL) {
618 arena_unbind(tsd, arena_ind_get(iarena), true);
619 }
620}
621
622void
623arena_cleanup(tsd_t *tsd) {
624 arena_t *arena;
625
626 arena = tsd_arena_get(tsd);
627 if (arena != NULL) {
628 arena_unbind(tsd, arena_ind_get(arena), false);
629 }
630}
631
632void
633arenas_tdata_cleanup(tsd_t *tsd) {
634 arena_tdata_t *arenas_tdata;
635
636 /* Prevent tsd->arenas_tdata from being (re)created. */
637 *tsd_arenas_tdata_bypassp_get(tsd) = true;
638
639 arenas_tdata = tsd_arenas_tdata_get(tsd);
640 if (arenas_tdata != NULL) {
641 tsd_arenas_tdata_set(tsd, NULL);
642 a0dalloc(arenas_tdata);
643 }
644}
645
646static void
647stats_print_atexit(void) {
648 if (config_stats) {
649 tsdn_t *tsdn;
650 unsigned narenas, i;
651
652 tsdn = tsdn_fetch();
653
654 /*
655 * Merge stats from extant threads. This is racy, since
656 * individual threads do not lock when recording tcache stats
657 * events. As a consequence, the final stats may be slightly
658 * out of date by the time they are reported, if other threads
659 * continue to allocate.
660 */
661 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
662 arena_t *arena = arena_get(tsdn, i, false);
663 if (arena != NULL) {
664 tcache_t *tcache;
665
666 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
667 ql_foreach(tcache, &arena->tcache_ql, link) {
668 tcache_stats_merge(tsdn, tcache, arena);
669 }
670 malloc_mutex_unlock(tsdn,
671 &arena->tcache_ql_mtx);
672 }
673 }
674 }
675 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
676}
677
678/*
679 * Ensure that we don't hold any locks upon entry to or exit from allocator
680 * code (in a "broad" sense that doesn't count a reentrant allocation as an
681 * entrance or exit).
682 */
683JEMALLOC_ALWAYS_INLINE void
684check_entry_exit_locking(tsdn_t *tsdn) {
685 if (!config_debug) {
686 return;
687 }
688 if (tsdn_null(tsdn)) {
689 return;
690 }
691 tsd_t *tsd = tsdn_tsd(tsdn);
692 /*
693 * It's possible we hold locks at entry/exit if we're in a nested
694 * allocation.
695 */
696 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
697 if (reentrancy_level != 0) {
698 return;
699 }
700 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
701}
702
703/*
704 * End miscellaneous support functions.
705 */
706/******************************************************************************/
707/*
708 * Begin initialization functions.
709 */
710
711static char *
712jemalloc_secure_getenv(const char *name) {
713#ifdef JEMALLOC_HAVE_SECURE_GETENV
714 return secure_getenv(name);
715#else
716# ifdef JEMALLOC_HAVE_ISSETUGID
717 if (issetugid() != 0) {
718 return NULL;
719 }
720# endif
721 return getenv(name);
722#endif
723}
724
725static unsigned
726malloc_ncpus(void) {
727 long result;
728
729#ifdef _WIN32
730 SYSTEM_INFO si;
731 GetSystemInfo(&si);
732 result = si.dwNumberOfProcessors;
733#elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
734 /*
735 * glibc >= 2.6 has the CPU_COUNT macro.
736 *
737 * glibc's sysconf() uses isspace(). glibc allocates for the first time
738 * *before* setting up the isspace tables. Therefore we need a
739 * different method to get the number of CPUs.
740 */
741 {
742 cpu_set_t set;
743
744 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
745 result = CPU_COUNT(&set);
746 }
747#else
748 result = sysconf(_SC_NPROCESSORS_ONLN);
749#endif
750 return ((result == -1) ? 1 : (unsigned)result);
751}
752
753static void
754init_opt_stats_print_opts(const char *v, size_t vlen) {
755 size_t opts_len = strlen(opt_stats_print_opts);
756 assert(opts_len <= stats_print_tot_num_options);
757
758 for (size_t i = 0; i < vlen; i++) {
759 switch (v[i]) {
760#define OPTION(o, v, d, s) case o: break;
761 STATS_PRINT_OPTIONS
762#undef OPTION
763 default: continue;
764 }
765
766 if (strchr(opt_stats_print_opts, v[i]) != NULL) {
767 /* Ignore repeated. */
768 continue;
769 }
770
771 opt_stats_print_opts[opts_len++] = v[i];
772 opt_stats_print_opts[opts_len] = '\0';
773 assert(opts_len <= stats_print_tot_num_options);
774 }
775 assert(opts_len == strlen(opt_stats_print_opts));
776}
777
778/* Reads the next size pair in a multi-sized option. */
779static bool
780malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
781 size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
782 const char *cur = *slab_size_segment_cur;
783 char *end;
784 uintmax_t um;
785
786 set_errno(0);
787
788 /* First number, then '-' */
789 um = malloc_strtoumax(cur, &end, 0);
790 if (get_errno() != 0 || *end != '-') {
791 return true;
792 }
793 *slab_start = (size_t)um;
794 cur = end + 1;
795
796 /* Second number, then ':' */
797 um = malloc_strtoumax(cur, &end, 0);
798 if (get_errno() != 0 || *end != ':') {
799 return true;
800 }
801 *slab_end = (size_t)um;
802 cur = end + 1;
803
804 /* Last number */
805 um = malloc_strtoumax(cur, &end, 0);
806 if (get_errno() != 0) {
807 return true;
808 }
809 *new_size = (size_t)um;
810
811 /* Consume the separator if there is one. */
812 if (*end == '|') {
813 end++;
814 }
815
816 *vlen_left -= end - *slab_size_segment_cur;
817 *slab_size_segment_cur = end;
818
819 return false;
820}
821
822static bool
823malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
824 char const **v_p, size_t *vlen_p) {
825 bool accept;
826 const char *opts = *opts_p;
827
828 *k_p = opts;
829
830 for (accept = false; !accept;) {
831 switch (*opts) {
832 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
833 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
834 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
835 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
836 case 'Y': case 'Z':
837 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
838 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
839 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
840 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
841 case 'y': case 'z':
842 case '0': case '1': case '2': case '3': case '4': case '5':
843 case '6': case '7': case '8': case '9':
844 case '_':
845 opts++;
846 break;
847 case ':':
848 opts++;
849 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
850 *v_p = opts;
851 accept = true;
852 break;
853 case '\0':
854 if (opts != *opts_p) {
855 malloc_write("<jemalloc>: Conf string ends "
856 "with key\n");
857 }
858 return true;
859 default:
860 malloc_write("<jemalloc>: Malformed conf string\n");
861 return true;
862 }
863 }
864
865 for (accept = false; !accept;) {
866 switch (*opts) {
867 case ',':
868 opts++;
869 /*
870 * Look ahead one character here, because the next time
871 * this function is called, it will assume that end of
872 * input has been cleanly reached if no input remains,
873 * but we have optimistically already consumed the
874 * comma if one exists.
875 */
876 if (*opts == '\0') {
877 malloc_write("<jemalloc>: Conf string ends "
878 "with comma\n");
879 }
880 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
881 accept = true;
882 break;
883 case '\0':
884 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
885 accept = true;
886 break;
887 default:
888 opts++;
889 break;
890 }
891 }
892
893 *opts_p = opts;
894 return false;
895}
896
897static void
898malloc_abort_invalid_conf(void) {
899 assert(opt_abort_conf);
900 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf "
901 "value (see above).\n");
902 abort();
903}
904
905static void
906malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
907 size_t vlen) {
908 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
909 (int)vlen, v);
910 /* If abort_conf is set, error out after processing all options. */
911 const char *experimental = "experimental_";
912 if (strncmp(k, experimental, strlen(experimental)) == 0) {
913 /* However, tolerate experimental features. */
914 return;
915 }
916 had_conf_error = true;
917}
918
919static void
920malloc_slow_flag_init(void) {
921 /*
922 * Combine the runtime options into malloc_slow for fast path. Called
923 * after processing all the options.
924 */
925 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
926 | (opt_junk_free ? flag_opt_junk_free : 0)
927 | (opt_zero ? flag_opt_zero : 0)
928 | (opt_utrace ? flag_opt_utrace : 0)
929 | (opt_xmalloc ? flag_opt_xmalloc : 0);
930
931 malloc_slow = (malloc_slow_flags != 0);
932}
933
934/* Number of sources for initializing malloc_conf */
935#define MALLOC_CONF_NSOURCES 4
936
937static const char *
938obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
939 if (config_debug) {
940 static unsigned read_source = 0;
941 /*
942 * Each source should only be read once, to minimize # of
943 * syscalls on init.
944 */
945 assert(read_source++ == which_source);
946 }
947 assert(which_source < MALLOC_CONF_NSOURCES);
948
949 const char *ret;
950 switch (which_source) {
951 case 0:
952 ret = config_malloc_conf;
953 break;
954 case 1:
955 if (je_malloc_conf != NULL) {
956 /* Use options that were compiled into the program. */
957 ret = je_malloc_conf;
958 } else {
959 /* No configuration specified. */
960 ret = NULL;
961 }
962 break;
963 case 2: {
964 ssize_t linklen = 0;
965#ifndef _WIN32
966 int saved_errno = errno;
967 const char *linkname =
968# ifdef JEMALLOC_PREFIX
969 "/etc/"JEMALLOC_PREFIX"malloc.conf"
970# else
971 "/etc/malloc.conf"
972# endif
973 ;
974
975 /*
976 * Try to use the contents of the "/etc/malloc.conf" symbolic
977 * link's name.
978 */
979#ifndef JEMALLOC_READLINKAT
980 linklen = readlink(linkname, buf, PATH_MAX);
981#else
982 linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX);
983#endif
984 if (linklen == -1) {
985 /* No configuration specified. */
986 linklen = 0;
987 /* Restore errno. */
988 set_errno(saved_errno);
989 }
990#endif
991 buf[linklen] = '\0';
992 ret = buf;
993 break;
994 } case 3: {
995 const char *envname =
996#ifdef JEMALLOC_PREFIX
997 JEMALLOC_CPREFIX"MALLOC_CONF"
998#else
999 "MALLOC_CONF"
1000#endif
1001 ;
1002
1003 if ((ret = jemalloc_secure_getenv(envname)) != NULL) {
1004 /*
1005 * Do nothing; opts is already initialized to the value
1006 * of the MALLOC_CONF environment variable.
1007 */
1008 } else {
1009 /* No configuration specified. */
1010 ret = NULL;
1011 }
1012 break;
1013 } default:
1014 not_reached();
1015 ret = NULL;
1016 }
1017 return ret;
1018}
1019
1020static void
1021malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
1022 bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES],
1023 char buf[PATH_MAX + 1]) {
1024 static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
1025 "string specified via --with-malloc-conf",
1026 "string pointed to by the global variable malloc_conf",
1027 "\"name\" of the file referenced by the symbolic link named "
1028 "/etc/malloc.conf",
1029 "value of the environment variable MALLOC_CONF"
1030 };
1031 unsigned i;
1032 const char *opts, *k, *v;
1033 size_t klen, vlen;
1034
1035 for (i = 0; i < MALLOC_CONF_NSOURCES; i++) {
1036 /* Get runtime configuration. */
1037 if (initial_call) {
1038 opts_cache[i] = obtain_malloc_conf(i, buf);
1039 }
1040 opts = opts_cache[i];
1041 if (!initial_call && opt_confirm_conf) {
1042 malloc_printf(
1043 "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n",
1044 i + 1, opts_explain[i], opts != NULL ? opts : "");
1045 }
1046 if (opts == NULL) {
1047 continue;
1048 }
1049
1050 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1051 &vlen)) {
1052
1053#define CONF_ERROR(msg, k, klen, v, vlen) \
1054 if (!initial_call) { \
1055 malloc_conf_error( \
1056 msg, k, klen, v, vlen); \
1057 cur_opt_valid = false; \
1058 }
1059#define CONF_CONTINUE { \
1060 if (!initial_call && opt_confirm_conf \
1061 && cur_opt_valid) { \
1062 malloc_printf("<jemalloc>: -- " \
1063 "Set conf value: %.*s:%.*s" \
1064 "\n", (int)klen, k, \
1065 (int)vlen, v); \
1066 } \
1067 continue; \
1068 }
1069#define CONF_MATCH(n) \
1070 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1071#define CONF_MATCH_VALUE(n) \
1072 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1073#define CONF_HANDLE_BOOL(o, n) \
1074 if (CONF_MATCH(n)) { \
1075 if (CONF_MATCH_VALUE("true")) { \
1076 o = true; \
1077 } else if (CONF_MATCH_VALUE("false")) { \
1078 o = false; \
1079 } else { \
1080 CONF_ERROR("Invalid conf value",\
1081 k, klen, v, vlen); \
1082 } \
1083 CONF_CONTINUE; \
1084 }
1085 /*
1086 * One of the CONF_MIN macros below expands, in one of the use points,
1087 * to "unsigned integer < 0", which is always false, triggering the
1088 * GCC -Wtype-limits warning, which we disable here and re-enable below.
1089 */
1090 JEMALLOC_DIAGNOSTIC_PUSH
1091 JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
1092
1093#define CONF_DONT_CHECK_MIN(um, min) false
1094#define CONF_CHECK_MIN(um, min) ((um) < (min))
1095#define CONF_DONT_CHECK_MAX(um, max) false
1096#define CONF_CHECK_MAX(um, max) ((um) > (max))
1097#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
1098 if (CONF_MATCH(n)) { \
1099 uintmax_t um; \
1100 char *end; \
1101 \
1102 set_errno(0); \
1103 um = malloc_strtoumax(v, &end, 0); \
1104 if (get_errno() != 0 || (uintptr_t)end -\
1105 (uintptr_t)v != vlen) { \
1106 CONF_ERROR("Invalid conf value",\
1107 k, klen, v, vlen); \
1108 } else if (clip) { \
1109 if (check_min(um, (t)(min))) { \
1110 o = (t)(min); \
1111 } else if ( \
1112 check_max(um, (t)(max))) { \
1113 o = (t)(max); \
1114 } else { \
1115 o = (t)um; \
1116 } \
1117 } else { \
1118 if (check_min(um, (t)(min)) || \
1119 check_max(um, (t)(max))) { \
1120 CONF_ERROR( \
1121 "Out-of-range " \
1122 "conf value", \
1123 k, klen, v, vlen); \
1124 } else { \
1125 o = (t)um; \
1126 } \
1127 } \
1128 CONF_CONTINUE; \
1129 }
1130#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
1131 clip) \
1132 CONF_HANDLE_T_U(unsigned, o, n, min, max, \
1133 check_min, check_max, clip)
1134#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
1135 CONF_HANDLE_T_U(size_t, o, n, min, max, \
1136 check_min, check_max, clip)
1137#define CONF_HANDLE_SSIZE_T(o, n, min, max) \
1138 if (CONF_MATCH(n)) { \
1139 long l; \
1140 char *end; \
1141 \
1142 set_errno(0); \
1143 l = strtol(v, &end, 0); \
1144 if (get_errno() != 0 || (uintptr_t)end -\
1145 (uintptr_t)v != vlen) { \
1146 CONF_ERROR("Invalid conf value",\
1147 k, klen, v, vlen); \
1148 } else if (l < (ssize_t)(min) || l > \
1149 (ssize_t)(max)) { \
1150 CONF_ERROR( \
1151 "Out-of-range conf value", \
1152 k, klen, v, vlen); \
1153 } else { \
1154 o = l; \
1155 } \
1156 CONF_CONTINUE; \
1157 }
1158#define CONF_HANDLE_CHAR_P(o, n, d) \
1159 if (CONF_MATCH(n)) { \
1160 size_t cpylen = (vlen <= \
1161 sizeof(o)-1) ? vlen : \
1162 sizeof(o)-1; \
1163 strncpy(o, v, cpylen); \
1164 o[cpylen] = '\0'; \
1165 CONF_CONTINUE; \
1166 }
1167
1168 bool cur_opt_valid = true;
1169
1170 CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf")
1171 if (initial_call) {
1172 continue;
1173 }
1174
1175 CONF_HANDLE_BOOL(opt_abort, "abort")
1176 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
1177 if (strncmp("metadata_thp", k, klen) == 0) {
1178 int i;
1179 bool match = false;
1180 for (i = 0; i < metadata_thp_mode_limit; i++) {
1181 if (strncmp(metadata_thp_mode_names[i],
1182 v, vlen) == 0) {
1183 opt_metadata_thp = i;
1184 match = true;
1185 break;
1186 }
1187 }
1188 if (!match) {
1189 CONF_ERROR("Invalid conf value",
1190 k, klen, v, vlen);
1191 }
1192 CONF_CONTINUE;
1193 }
1194 CONF_HANDLE_BOOL(opt_retain, "retain")
1195 if (strncmp("dss", k, klen) == 0) {
1196 int i;
1197 bool match = false;
1198 for (i = 0; i < dss_prec_limit; i++) {
1199 if (strncmp(dss_prec_names[i], v, vlen)
1200 == 0) {
1201 if (extent_dss_prec_set(i)) {
1202 CONF_ERROR(
1203 "Error setting dss",
1204 k, klen, v, vlen);
1205 } else {
1206 opt_dss =
1207 dss_prec_names[i];
1208 match = true;
1209 break;
1210 }
1211 }
1212 }
1213 if (!match) {
1214 CONF_ERROR("Invalid conf value",
1215 k, klen, v, vlen);
1216 }
1217 CONF_CONTINUE;
1218 }
1219 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
1220 UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
1221 false)
1222 if (CONF_MATCH("bin_shards")) {
1223 const char *bin_shards_segment_cur = v;
1224 size_t vlen_left = vlen;
1225 do {
1226 size_t size_start;
1227 size_t size_end;
1228 size_t nshards;
1229 bool err = malloc_conf_multi_sizes_next(
1230 &bin_shards_segment_cur, &vlen_left,
1231 &size_start, &size_end, &nshards);
1232 if (err || bin_update_shard_size(
1233 bin_shard_sizes, size_start,
1234 size_end, nshards)) {
1235 CONF_ERROR(
1236 "Invalid settings for "
1237 "bin_shards", k, klen, v,
1238 vlen);
1239 break;
1240 }
1241 } while (vlen_left > 0);
1242 CONF_CONTINUE;
1243 }
1244 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
1245 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1246 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1247 SSIZE_MAX);
1248 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms,
1249 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1250 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1251 SSIZE_MAX);
1252 CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
1253 if (CONF_MATCH("stats_print_opts")) {
1254 init_opt_stats_print_opts(v, vlen);
1255 CONF_CONTINUE;
1256 }
1257 if (config_fill) {
1258 if (CONF_MATCH("junk")) {
1259 if (CONF_MATCH_VALUE("true")) {
1260 opt_junk = "true";
1261 opt_junk_alloc = opt_junk_free =
1262 true;
1263 } else if (CONF_MATCH_VALUE("false")) {
1264 opt_junk = "false";
1265 opt_junk_alloc = opt_junk_free =
1266 false;
1267 } else if (CONF_MATCH_VALUE("alloc")) {
1268 opt_junk = "alloc";
1269 opt_junk_alloc = true;
1270 opt_junk_free = false;
1271 } else if (CONF_MATCH_VALUE("free")) {
1272 opt_junk = "free";
1273 opt_junk_alloc = false;
1274 opt_junk_free = true;
1275 } else {
1276 CONF_ERROR(
1277 "Invalid conf value",
1278 k, klen, v, vlen);
1279 }
1280 CONF_CONTINUE;
1281 }
1282 CONF_HANDLE_BOOL(opt_zero, "zero")
1283 }
1284 if (config_utrace) {
1285 CONF_HANDLE_BOOL(opt_utrace, "utrace")
1286 }
1287 if (config_xmalloc) {
1288 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
1289 }
1290 CONF_HANDLE_BOOL(opt_tcache, "tcache")
1291 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max",
1292 -1, (sizeof(size_t) << 3) - 1)
1293
1294 /*
1295 * The runtime option of oversize_threshold remains
1296 * undocumented. It may be tweaked in the next major
1297 * release (6.0). The default value 8M is rather
1298 * conservative / safe. Tuning it further down may
1299 * improve fragmentation a bit more, but may also cause
1300 * contention on the huge arena.
1301 */
1302 CONF_HANDLE_SIZE_T(opt_oversize_threshold,
1303 "oversize_threshold", 0, SC_LARGE_MAXCLASS,
1304 CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false)
1305 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit,
1306 "lg_extent_max_active_fit", 0,
1307 (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN,
1308 CONF_CHECK_MAX, false)
1309
1310 if (strncmp("percpu_arena", k, klen) == 0) {
1311 bool match = false;
1312 for (int i = percpu_arena_mode_names_base; i <
1313 percpu_arena_mode_names_limit; i++) {
1314 if (strncmp(percpu_arena_mode_names[i],
1315 v, vlen) == 0) {
1316 if (!have_percpu_arena) {
1317 CONF_ERROR(
1318 "No getcpu support",
1319 k, klen, v, vlen);
1320 }
1321 opt_percpu_arena = i;
1322 match = true;
1323 break;
1324 }
1325 }
1326 if (!match) {
1327 CONF_ERROR("Invalid conf value",
1328 k, klen, v, vlen);
1329 }
1330 CONF_CONTINUE;
1331 }
1332 CONF_HANDLE_BOOL(opt_background_thread,
1333 "background_thread");
1334 CONF_HANDLE_SIZE_T(opt_max_background_threads,
1335 "max_background_threads", 1,
1336 opt_max_background_threads,
1337 CONF_CHECK_MIN, CONF_CHECK_MAX,
1338 true);
1339 if (CONF_MATCH("slab_sizes")) {
1340 bool err;
1341 const char *slab_size_segment_cur = v;
1342 size_t vlen_left = vlen;
1343 do {
1344 size_t slab_start;
1345 size_t slab_end;
1346 size_t pgs;
1347 err = malloc_conf_multi_sizes_next(
1348 &slab_size_segment_cur,
1349 &vlen_left, &slab_start, &slab_end,
1350 &pgs);
1351 if (!err) {
1352 sc_data_update_slab_size(
1353 sc_data, slab_start,
1354 slab_end, (int)pgs);
1355 } else {
1356 CONF_ERROR("Invalid settings "
1357 "for slab_sizes",
1358 k, klen, v, vlen);
1359 }
1360 } while (!err && vlen_left > 0);
1361 CONF_CONTINUE;
1362 }
1363 if (config_prof) {
1364 CONF_HANDLE_BOOL(opt_prof, "prof")
1365 CONF_HANDLE_CHAR_P(opt_prof_prefix,
1366 "prof_prefix", "jeprof")
1367 CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
1368 CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1369 "prof_thread_active_init")
1370 CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1371 "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
1372 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
1373 true)
1374 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
1375 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1376 "lg_prof_interval", -1,
1377 (sizeof(uint64_t) << 3) - 1)
1378 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
1379 CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
1380 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
1381 CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
1382 }
1383 if (config_log) {
1384 if (CONF_MATCH("log")) {
1385 size_t cpylen = (
1386 vlen <= sizeof(log_var_names) ?
1387 vlen : sizeof(log_var_names) - 1);
1388 strncpy(log_var_names, v, cpylen);
1389 log_var_names[cpylen] = '\0';
1390 CONF_CONTINUE;
1391 }
1392 }
1393 if (CONF_MATCH("thp")) {
1394 bool match = false;
1395 for (int i = 0; i < thp_mode_names_limit; i++) {
1396 if (strncmp(thp_mode_names[i],v, vlen)
1397 == 0) {
1398 if (!have_madvise_huge) {
1399 CONF_ERROR(
1400 "No THP support",
1401 k, klen, v, vlen);
1402 }
1403 opt_thp = i;
1404 match = true;
1405 break;
1406 }
1407 }
1408 if (!match) {
1409 CONF_ERROR("Invalid conf value",
1410 k, klen, v, vlen);
1411 }
1412 CONF_CONTINUE;
1413 }
1414 CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
1415#undef CONF_ERROR
1416#undef CONF_CONTINUE
1417#undef CONF_MATCH
1418#undef CONF_MATCH_VALUE
1419#undef CONF_HANDLE_BOOL
1420#undef CONF_DONT_CHECK_MIN
1421#undef CONF_CHECK_MIN
1422#undef CONF_DONT_CHECK_MAX
1423#undef CONF_CHECK_MAX
1424#undef CONF_HANDLE_T_U
1425#undef CONF_HANDLE_UNSIGNED
1426#undef CONF_HANDLE_SIZE_T
1427#undef CONF_HANDLE_SSIZE_T
1428#undef CONF_HANDLE_CHAR_P
1429 /* Re-enable diagnostic "-Wtype-limits" */
1430 JEMALLOC_DIAGNOSTIC_POP
1431 }
1432 if (opt_abort_conf && had_conf_error) {
1433 malloc_abort_invalid_conf();
1434 }
1435 }
1436 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
1437}
1438
1439static void
1440malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
1441 const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL};
1442 char buf[PATH_MAX + 1];
1443
1444 /* The first call only set the confirm_conf option and opts_cache */
1445 malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf);
1446 malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
1447 NULL);
1448}
1449
1450#undef MALLOC_CONF_NSOURCES
1451
1452static bool
1453malloc_init_hard_needed(void) {
1454 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1455 malloc_init_recursible)) {
1456 /*
1457 * Another thread initialized the allocator before this one
1458 * acquired init_lock, or this thread is the initializing
1459 * thread, and it is recursively allocating.
1460 */
1461 return false;
1462 }
1463#ifdef JEMALLOC_THREADED_INIT
1464 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1465 /* Busy-wait until the initializing thread completes. */
1466 spin_t spinner = SPIN_INITIALIZER;
1467 do {
1468 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1469 spin_adaptive(&spinner);
1470 malloc_mutex_lock(TSDN_NULL, &init_lock);
1471 } while (!malloc_initialized());
1472 return false;
1473 }
1474#endif
1475 return true;
1476}
1477
1478static bool
1479malloc_init_hard_a0_locked() {
1480 malloc_initializer = INITIALIZER;
1481
1482 JEMALLOC_DIAGNOSTIC_PUSH
1483 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
1484 sc_data_t sc_data = {0};
1485 JEMALLOC_DIAGNOSTIC_POP
1486
1487 /*
1488 * Ordering here is somewhat tricky; we need sc_boot() first, since that
1489 * determines what the size classes will be, and then
1490 * malloc_conf_init(), since any slab size tweaking will need to be done
1491 * before sz_boot and bin_boot, which assume that the values they read
1492 * out of sc_data_global are final.
1493 */
1494 sc_boot(&sc_data);
1495 unsigned bin_shard_sizes[SC_NBINS];
1496 bin_shard_sizes_boot(bin_shard_sizes);
1497 /*
1498 * prof_boot0 only initializes opt_prof_prefix. We need to do it before
1499 * we parse malloc_conf options, in case malloc_conf parsing overwrites
1500 * it.
1501 */
1502 if (config_prof) {
1503 prof_boot0();
1504 }
1505 malloc_conf_init(&sc_data, bin_shard_sizes);
1506 sz_boot(&sc_data);
1507 bin_boot(&sc_data, bin_shard_sizes);
1508
1509 if (opt_stats_print) {
1510 /* Print statistics at exit. */
1511 if (atexit(stats_print_atexit) != 0) {
1512 malloc_write("<jemalloc>: Error in atexit()\n");
1513 if (opt_abort) {
1514 abort();
1515 }
1516 }
1517 }
1518 if (pages_boot()) {
1519 return true;
1520 }
1521 if (base_boot(TSDN_NULL)) {
1522 return true;
1523 }
1524 if (extent_boot()) {
1525 return true;
1526 }
1527 if (ctl_boot()) {
1528 return true;
1529 }
1530 if (config_prof) {
1531 prof_boot1();
1532 }
1533 arena_boot(&sc_data);
1534 if (tcache_boot(TSDN_NULL)) {
1535 return true;
1536 }
1537 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
1538 malloc_mutex_rank_exclusive)) {
1539 return true;
1540 }
1541 hook_boot();
1542 /*
1543 * Create enough scaffolding to allow recursive allocation in
1544 * malloc_ncpus().
1545 */
1546 narenas_auto = 1;
1547 manual_arena_base = narenas_auto + 1;
1548 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1549 /*
1550 * Initialize one arena here. The rest are lazily created in
1551 * arena_choose_hard().
1552 */
1553 if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default)
1554 == NULL) {
1555 return true;
1556 }
1557 a0 = arena_get(TSDN_NULL, 0, false);
1558 malloc_init_state = malloc_init_a0_initialized;
1559
1560 return false;
1561}
1562
1563static bool
1564malloc_init_hard_a0(void) {
1565 bool ret;
1566
1567 malloc_mutex_lock(TSDN_NULL, &init_lock);
1568 ret = malloc_init_hard_a0_locked();
1569 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1570 return ret;
1571}
1572
1573/* Initialize data structures which may trigger recursive allocation. */
1574static bool
1575malloc_init_hard_recursible(void) {
1576 malloc_init_state = malloc_init_recursible;
1577
1578 ncpus = malloc_ncpus();
1579
1580#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
1581 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
1582 !defined(__native_client__))
1583 /* LinuxThreads' pthread_atfork() allocates. */
1584 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1585 jemalloc_postfork_child) != 0) {
1586 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1587 if (opt_abort) {
1588 abort();
1589 }
1590 return true;
1591 }
1592#endif
1593
1594 if (background_thread_boot0()) {
1595 return true;
1596 }
1597
1598 return false;
1599}
1600
1601static unsigned
1602malloc_narenas_default(void) {
1603 assert(ncpus > 0);
1604 /*
1605 * For SMP systems, create more than one arena per CPU by
1606 * default.
1607 */
1608 if (ncpus > 1) {
1609 return ncpus << 2;
1610 } else {
1611 return 1;
1612 }
1613}
1614
1615static percpu_arena_mode_t
1616percpu_arena_as_initialized(percpu_arena_mode_t mode) {
1617 assert(!malloc_initialized());
1618 assert(mode <= percpu_arena_disabled);
1619
1620 if (mode != percpu_arena_disabled) {
1621 mode += percpu_arena_mode_enabled_base;
1622 }
1623
1624 return mode;
1625}
1626
1627static bool
1628malloc_init_narenas(void) {
1629 assert(ncpus > 0);
1630
1631 if (opt_percpu_arena != percpu_arena_disabled) {
1632 if (!have_percpu_arena || malloc_getcpu() < 0) {
1633 opt_percpu_arena = percpu_arena_disabled;
1634 malloc_printf("<jemalloc>: perCPU arena getcpu() not "
1635 "available. Setting narenas to %u.\n", opt_narenas ?
1636 opt_narenas : malloc_narenas_default());
1637 if (opt_abort) {
1638 abort();
1639 }
1640 } else {
1641 if (ncpus >= MALLOCX_ARENA_LIMIT) {
1642 malloc_printf("<jemalloc>: narenas w/ percpu"
1643 "arena beyond limit (%d)\n", ncpus);
1644 if (opt_abort) {
1645 abort();
1646 }
1647 return true;
1648 }
1649 /* NB: opt_percpu_arena isn't fully initialized yet. */
1650 if (percpu_arena_as_initialized(opt_percpu_arena) ==
1651 per_phycpu_arena && ncpus % 2 != 0) {
1652 malloc_printf("<jemalloc>: invalid "
1653 "configuration -- per physical CPU arena "
1654 "with odd number (%u) of CPUs (no hyper "
1655 "threading?).\n", ncpus);
1656 if (opt_abort)
1657 abort();
1658 }
1659 unsigned n = percpu_arena_ind_limit(
1660 percpu_arena_as_initialized(opt_percpu_arena));
1661 if (opt_narenas < n) {
1662 /*
1663 * If narenas is specified with percpu_arena
1664 * enabled, actual narenas is set as the greater
1665 * of the two. percpu_arena_choose will be free
1666 * to use any of the arenas based on CPU
1667 * id. This is conservative (at a small cost)
1668 * but ensures correctness.
1669 *
1670 * If for some reason the ncpus determined at
1671 * boot is not the actual number (e.g. because
1672 * of affinity setting from numactl), reserving
1673 * narenas this way provides a workaround for
1674 * percpu_arena.
1675 */
1676 opt_narenas = n;
1677 }
1678 }
1679 }
1680 if (opt_narenas == 0) {
1681 opt_narenas = malloc_narenas_default();
1682 }
1683 assert(opt_narenas > 0);
1684
1685 narenas_auto = opt_narenas;
1686 /*
1687 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
1688 */
1689 if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
1690 narenas_auto = MALLOCX_ARENA_LIMIT - 1;
1691 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
1692 narenas_auto);
1693 }
1694 narenas_total_set(narenas_auto);
1695 if (arena_init_huge()) {
1696 narenas_total_inc();
1697 }
1698 manual_arena_base = narenas_total_get();
1699
1700 return false;
1701}
1702
1703static void
1704malloc_init_percpu(void) {
1705 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
1706}
1707
1708static bool
1709malloc_init_hard_finish(void) {
1710 if (malloc_mutex_boot()) {
1711 return true;
1712 }
1713
1714 malloc_init_state = malloc_init_initialized;
1715 malloc_slow_flag_init();
1716
1717 return false;
1718}
1719
1720static void
1721malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
1722 malloc_mutex_assert_owner(tsdn, &init_lock);
1723 malloc_mutex_unlock(tsdn, &init_lock);
1724 if (reentrancy_set) {
1725 assert(!tsdn_null(tsdn));
1726 tsd_t *tsd = tsdn_tsd(tsdn);
1727 assert(tsd_reentrancy_level_get(tsd) > 0);
1728 post_reentrancy(tsd);
1729 }
1730}
1731
1732static bool
1733malloc_init_hard(void) {
1734 tsd_t *tsd;
1735
1736#if defined(_WIN32) && _WIN32_WINNT < 0x0600
1737 _init_init_lock();
1738#endif
1739 malloc_mutex_lock(TSDN_NULL, &init_lock);
1740
1741#define UNLOCK_RETURN(tsdn, ret, reentrancy) \
1742 malloc_init_hard_cleanup(tsdn, reentrancy); \
1743 return ret;
1744
1745 if (!malloc_init_hard_needed()) {
1746 UNLOCK_RETURN(TSDN_NULL, false, false)
1747 }
1748
1749 if (malloc_init_state != malloc_init_a0_initialized &&
1750 malloc_init_hard_a0_locked()) {
1751 UNLOCK_RETURN(TSDN_NULL, true, false)
1752 }
1753
1754 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1755 /* Recursive allocation relies on functional tsd. */
1756 tsd = malloc_tsd_boot0();
1757 if (tsd == NULL) {
1758 return true;
1759 }
1760 if (malloc_init_hard_recursible()) {
1761 return true;
1762 }
1763
1764 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1765 /* Set reentrancy level to 1 during init. */
1766 pre_reentrancy(tsd, NULL);
1767 /* Initialize narenas before prof_boot2 (for allocation). */
1768 if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) {
1769 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1770 }
1771 if (config_prof && prof_boot2(tsd)) {
1772 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1773 }
1774
1775 malloc_init_percpu();
1776
1777 if (malloc_init_hard_finish()) {
1778 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1779 }
1780 post_reentrancy(tsd);
1781 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1782
1783 witness_assert_lockless(witness_tsd_tsdn(
1784 tsd_witness_tsdp_get_unsafe(tsd)));
1785 malloc_tsd_boot1();
1786 /* Update TSD after tsd_boot1. */
1787 tsd = tsd_fetch();
1788 if (opt_background_thread) {
1789 assert(have_background_thread);
1790 /*
1791 * Need to finish init & unlock first before creating background
1792 * threads (pthread_create depends on malloc). ctl_init (which
1793 * sets isthreaded) needs to be called without holding any lock.
1794 */
1795 background_thread_ctl_init(tsd_tsdn(tsd));
1796 if (background_thread_create(tsd, 0)) {
1797 return true;
1798 }
1799 }
1800#undef UNLOCK_RETURN
1801 return false;
1802}
1803
1804/*
1805 * End initialization functions.
1806 */
1807/******************************************************************************/
1808/*
1809 * Begin allocation-path internal functions and data structures.
1810 */
1811
1812/*
1813 * Settings determined by the documented behavior of the allocation functions.
1814 */
1815typedef struct static_opts_s static_opts_t;
1816struct static_opts_s {
1817 /* Whether or not allocation size may overflow. */
1818 bool may_overflow;
1819
1820 /*
1821 * Whether or not allocations (with alignment) of size 0 should be
1822 * treated as size 1.
1823 */
1824 bool bump_empty_aligned_alloc;
1825 /*
1826 * Whether to assert that allocations are not of size 0 (after any
1827 * bumping).
1828 */
1829 bool assert_nonempty_alloc;
1830
1831 /*
1832 * Whether or not to modify the 'result' argument to malloc in case of
1833 * error.
1834 */
1835 bool null_out_result_on_error;
1836 /* Whether to set errno when we encounter an error condition. */
1837 bool set_errno_on_error;
1838
1839 /*
1840 * The minimum valid alignment for functions requesting aligned storage.
1841 */
1842 size_t min_alignment;
1843
1844 /* The error string to use if we oom. */
1845 const char *oom_string;
1846 /* The error string to use if the passed-in alignment is invalid. */
1847 const char *invalid_alignment_string;
1848
1849 /*
1850 * False if we're configured to skip some time-consuming operations.
1851 *
1852 * This isn't really a malloc "behavior", but it acts as a useful
1853 * summary of several other static (or at least, static after program
1854 * initialization) options.
1855 */
1856 bool slow;
1857 /*
1858 * Return size.
1859 */
1860 bool usize;
1861};
1862
1863JEMALLOC_ALWAYS_INLINE void
1864static_opts_init(static_opts_t *static_opts) {
1865 static_opts->may_overflow = false;
1866 static_opts->bump_empty_aligned_alloc = false;
1867 static_opts->assert_nonempty_alloc = false;
1868 static_opts->null_out_result_on_error = false;
1869 static_opts->set_errno_on_error = false;
1870 static_opts->min_alignment = 0;
1871 static_opts->oom_string = "";
1872 static_opts->invalid_alignment_string = "";
1873 static_opts->slow = false;
1874 static_opts->usize = false;
1875}
1876
1877/*
1878 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we
1879 * should have one constant here per magic value there. Note however that the
1880 * representations need not be related.
1881 */
1882#define TCACHE_IND_NONE ((unsigned)-1)
1883#define TCACHE_IND_AUTOMATIC ((unsigned)-2)
1884#define ARENA_IND_AUTOMATIC ((unsigned)-1)
1885
1886typedef struct dynamic_opts_s dynamic_opts_t;
1887struct dynamic_opts_s {
1888 void **result;
1889 size_t usize;
1890 size_t num_items;
1891 size_t item_size;
1892 size_t alignment;
1893 bool zero;
1894 unsigned tcache_ind;
1895 unsigned arena_ind;
1896};
1897
1898JEMALLOC_ALWAYS_INLINE void
1899dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
1900 dynamic_opts->result = NULL;
1901 dynamic_opts->usize = 0;
1902 dynamic_opts->num_items = 0;
1903 dynamic_opts->item_size = 0;
1904 dynamic_opts->alignment = 0;
1905 dynamic_opts->zero = false;
1906 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC;
1907 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
1908}
1909
1910/* ind is ignored if dopts->alignment > 0. */
1911JEMALLOC_ALWAYS_INLINE void *
1912imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1913 size_t size, size_t usize, szind_t ind) {
1914 tcache_t *tcache;
1915 arena_t *arena;
1916
1917 /* Fill in the tcache. */
1918 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) {
1919 if (likely(!sopts->slow)) {
1920 /* Getting tcache ptr unconditionally. */
1921 tcache = tsd_tcachep_get(tsd);
1922 assert(tcache == tcache_get(tsd));
1923 } else {
1924 tcache = tcache_get(tsd);
1925 }
1926 } else if (dopts->tcache_ind == TCACHE_IND_NONE) {
1927 tcache = NULL;
1928 } else {
1929 tcache = tcaches_get(tsd, dopts->tcache_ind);
1930 }
1931
1932 /* Fill in the arena. */
1933 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) {
1934 /*
1935 * In case of automatic arena management, we defer arena
1936 * computation until as late as we can, hoping to fill the
1937 * allocation out of the tcache.
1938 */
1939 arena = NULL;
1940 } else {
1941 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true);
1942 }
1943
1944 if (unlikely(dopts->alignment != 0)) {
1945 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment,
1946 dopts->zero, tcache, arena);
1947 }
1948
1949 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false,
1950 arena, sopts->slow);
1951}
1952
1953JEMALLOC_ALWAYS_INLINE void *
1954imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1955 size_t usize, szind_t ind) {
1956 void *ret;
1957
1958 /*
1959 * For small allocations, sampling bumps the usize. If so, we allocate
1960 * from the ind_large bucket.
1961 */
1962 szind_t ind_large;
1963 size_t bumped_usize = usize;
1964
1965 if (usize <= SC_SMALL_MAXCLASS) {
1966 assert(((dopts->alignment == 0) ?
1967 sz_s2u(SC_LARGE_MINCLASS) :
1968 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment))
1969 == SC_LARGE_MINCLASS);
1970 ind_large = sz_size2index(SC_LARGE_MINCLASS);
1971 bumped_usize = sz_s2u(SC_LARGE_MINCLASS);
1972 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize,
1973 bumped_usize, ind_large);
1974 if (unlikely(ret == NULL)) {
1975 return NULL;
1976 }
1977 arena_prof_promote(tsd_tsdn(tsd), ret, usize);
1978 } else {
1979 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind);
1980 }
1981
1982 return ret;
1983}
1984
1985/*
1986 * Returns true if the allocation will overflow, and false otherwise. Sets
1987 * *size to the product either way.
1988 */
1989JEMALLOC_ALWAYS_INLINE bool
1990compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts,
1991 size_t *size) {
1992 /*
1993 * This function is just num_items * item_size, except that we may have
1994 * to check for overflow.
1995 */
1996
1997 if (!may_overflow) {
1998 assert(dopts->num_items == 1);
1999 *size = dopts->item_size;
2000 return false;
2001 }
2002
2003 /* A size_t with its high-half bits all set to 1. */
2004 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2);
2005
2006 *size = dopts->item_size * dopts->num_items;
2007
2008 if (unlikely(*size == 0)) {
2009 return (dopts->num_items != 0 && dopts->item_size != 0);
2010 }
2011
2012 /*
2013 * We got a non-zero size, but we don't know if we overflowed to get
2014 * there. To avoid having to do a divide, we'll be clever and note that
2015 * if both A and B can be represented in N/2 bits, then their product
2016 * can be represented in N bits (without the possibility of overflow).
2017 */
2018 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) {
2019 return false;
2020 }
2021 if (likely(*size / dopts->item_size == dopts->num_items)) {
2022 return false;
2023 }
2024 return true;
2025}
2026
2027JEMALLOC_ALWAYS_INLINE int
2028imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
2029 /* Where the actual allocated memory will live. */
2030 void *allocation = NULL;
2031 /* Filled in by compute_size_with_overflow below. */
2032 size_t size = 0;
2033 /*
2034 * For unaligned allocations, we need only ind. For aligned
2035 * allocations, or in case of stats or profiling we need usize.
2036 *
2037 * These are actually dead stores, in that their values are reset before
2038 * any branch on their value is taken. Sometimes though, it's
2039 * convenient to pass them as arguments before this point. To avoid
2040 * undefined behavior then, we initialize them with dummy stores.
2041 */
2042 szind_t ind = 0;
2043 size_t usize = 0;
2044
2045 /* Reentrancy is only checked on slow path. */
2046 int8_t reentrancy_level;
2047
2048 /* Compute the amount of memory the user wants. */
2049 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
2050 &size))) {
2051 goto label_oom;
2052 }
2053
2054 if (unlikely(dopts->alignment < sopts->min_alignment
2055 || (dopts->alignment & (dopts->alignment - 1)) != 0)) {
2056 goto label_invalid_alignment;
2057 }
2058
2059 /* This is the beginning of the "core" algorithm. */
2060
2061 if (dopts->alignment == 0) {
2062 ind = sz_size2index(size);
2063 if (unlikely(ind >= SC_NSIZES)) {
2064 goto label_oom;
2065 }
2066 if (config_stats || (config_prof && opt_prof) || sopts->usize) {
2067 usize = sz_index2size(ind);
2068 dopts->usize = usize;
2069 assert(usize > 0 && usize
2070 <= SC_LARGE_MAXCLASS);
2071 }
2072 } else {
2073 if (sopts->bump_empty_aligned_alloc) {
2074 if (unlikely(size == 0)) {
2075 size = 1;
2076 }
2077 }
2078 usize = sz_sa2u(size, dopts->alignment);
2079 dopts->usize = usize;
2080 if (unlikely(usize == 0
2081 || usize > SC_LARGE_MAXCLASS)) {
2082 goto label_oom;
2083 }
2084 }
2085 /* Validate the user input. */
2086 if (sopts->assert_nonempty_alloc) {
2087 assert (size != 0);
2088 }
2089
2090 check_entry_exit_locking(tsd_tsdn(tsd));
2091
2092 /*
2093 * If we need to handle reentrancy, we can do it out of a
2094 * known-initialized arena (i.e. arena 0).
2095 */
2096 reentrancy_level = tsd_reentrancy_level_get(tsd);
2097 if (sopts->slow && unlikely(reentrancy_level > 0)) {
2098 /*
2099 * We should never specify particular arenas or tcaches from
2100 * within our internal allocations.
2101 */
2102 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
2103 dopts->tcache_ind == TCACHE_IND_NONE);
2104 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
2105 dopts->tcache_ind = TCACHE_IND_NONE;
2106 /* We know that arena 0 has already been initialized. */
2107 dopts->arena_ind = 0;
2108 }
2109
2110 /* If profiling is on, get our profiling context. */
2111 if (config_prof && opt_prof) {
2112 /*
2113 * Note that if we're going down this path, usize must have been
2114 * initialized in the previous if statement.
2115 */
2116 prof_tctx_t *tctx = prof_alloc_prep(
2117 tsd, usize, prof_active_get_unlocked(), true);
2118
2119 alloc_ctx_t alloc_ctx;
2120 if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2121 alloc_ctx.slab = (usize
2122 <= SC_SMALL_MAXCLASS);
2123 allocation = imalloc_no_sample(
2124 sopts, dopts, tsd, usize, usize, ind);
2125 } else if ((uintptr_t)tctx > (uintptr_t)1U) {
2126 /*
2127 * Note that ind might still be 0 here. This is fine;
2128 * imalloc_sample ignores ind if dopts->alignment > 0.
2129 */
2130 allocation = imalloc_sample(
2131 sopts, dopts, tsd, usize, ind);
2132 alloc_ctx.slab = false;
2133 } else {
2134 allocation = NULL;
2135 }
2136
2137 if (unlikely(allocation == NULL)) {
2138 prof_alloc_rollback(tsd, tctx, true);
2139 goto label_oom;
2140 }
2141 prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
2142 } else {
2143 /*
2144 * If dopts->alignment > 0, then ind is still 0, but usize was
2145 * computed in the previous if statement. Down the positive
2146 * alignment path, imalloc_no_sample ignores ind and size
2147 * (relying only on usize).
2148 */
2149 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
2150 ind);
2151 if (unlikely(allocation == NULL)) {
2152 goto label_oom;
2153 }
2154 }
2155
2156 /*
2157 * Allocation has been done at this point. We still have some
2158 * post-allocation work to do though.
2159 */
2160 assert(dopts->alignment == 0
2161 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
2162
2163 if (config_stats) {
2164 assert(usize == isalloc(tsd_tsdn(tsd), allocation));
2165 *tsd_thread_allocatedp_get(tsd) += usize;
2166 }
2167
2168 if (sopts->slow) {
2169 UTRACE(0, size, allocation);
2170 }
2171
2172 /* Success! */
2173 check_entry_exit_locking(tsd_tsdn(tsd));
2174 *dopts->result = allocation;
2175 return 0;
2176
2177label_oom:
2178 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) {
2179 malloc_write(sopts->oom_string);
2180 abort();
2181 }
2182
2183 if (sopts->slow) {
2184 UTRACE(NULL, size, NULL);
2185 }
2186
2187 check_entry_exit_locking(tsd_tsdn(tsd));
2188
2189 if (sopts->set_errno_on_error) {
2190 set_errno(ENOMEM);
2191 }
2192
2193 if (sopts->null_out_result_on_error) {
2194 *dopts->result = NULL;
2195 }
2196
2197 return ENOMEM;
2198
2199 /*
2200 * This label is only jumped to by one goto; we move it out of line
2201 * anyways to avoid obscuring the non-error paths, and for symmetry with
2202 * the oom case.
2203 */
2204label_invalid_alignment:
2205 if (config_xmalloc && unlikely(opt_xmalloc)) {
2206 malloc_write(sopts->invalid_alignment_string);
2207 abort();
2208 }
2209
2210 if (sopts->set_errno_on_error) {
2211 set_errno(EINVAL);
2212 }
2213
2214 if (sopts->slow) {
2215 UTRACE(NULL, size, NULL);
2216 }
2217
2218 check_entry_exit_locking(tsd_tsdn(tsd));
2219
2220 if (sopts->null_out_result_on_error) {
2221 *dopts->result = NULL;
2222 }
2223
2224 return EINVAL;
2225}
2226
2227JEMALLOC_ALWAYS_INLINE bool
2228imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) {
2229 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) {
2230 if (config_xmalloc && unlikely(opt_xmalloc)) {
2231 malloc_write(sopts->oom_string);
2232 abort();
2233 }
2234 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL);
2235 set_errno(ENOMEM);
2236 *dopts->result = NULL;
2237
2238 return false;
2239 }
2240
2241 return true;
2242}
2243
2244/* Returns the errno-style error code of the allocation. */
2245JEMALLOC_ALWAYS_INLINE int
2246imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
2247 if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2248 return ENOMEM;
2249 }
2250
2251 /* We always need the tsd. Let's grab it right away. */
2252 tsd_t *tsd = tsd_fetch();
2253 assert(tsd);
2254 if (likely(tsd_fast(tsd))) {
2255 /* Fast and common path. */
2256 tsd_assert_fast(tsd);
2257 sopts->slow = false;
2258 return imalloc_body(sopts, dopts, tsd);
2259 } else {
2260 if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2261 return ENOMEM;
2262 }
2263
2264 sopts->slow = true;
2265 return imalloc_body(sopts, dopts, tsd);
2266 }
2267}
2268
2269JEMALLOC_NOINLINE
2270void *
2271malloc_default(size_t size) {
2272 void *ret;
2273 static_opts_t sopts;
2274 dynamic_opts_t dopts;
2275
2276 LOG("core.malloc.entry", "size: %zu", size);
2277
2278 static_opts_init(&sopts);
2279 dynamic_opts_init(&dopts);
2280
2281 sopts.null_out_result_on_error = true;
2282 sopts.set_errno_on_error = true;
2283 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n";
2284
2285 dopts.result = &ret;
2286 dopts.num_items = 1;
2287 dopts.item_size = size;
2288
2289 imalloc(&sopts, &dopts);
2290 /*
2291 * Note that this branch gets optimized away -- it immediately follows
2292 * the check on tsd_fast that sets sopts.slow.
2293 */
2294 if (sopts.slow) {
2295 uintptr_t args[3] = {size};
2296 hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args);
2297 }
2298
2299 LOG("core.malloc.exit", "result: %p", ret);
2300
2301 return ret;
2302}
2303
2304/******************************************************************************/
2305/*
2306 * Begin malloc(3)-compatible functions.
2307 */
2308
2309/*
2310 * malloc() fastpath.
2311 *
2312 * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit
2313 * tcache. If either of these is false, we tail-call to the slowpath,
2314 * malloc_default(). Tail-calling is used to avoid any caller-saved
2315 * registers.
2316 *
2317 * fastpath supports ticker and profiling, both of which will also
2318 * tail-call to the slowpath if they fire.
2319 */
2320JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2321void JEMALLOC_NOTHROW *
2322JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2323je_malloc(size_t size) {
2324 LOG("core.malloc.entry", "size: %zu", size);
2325
2326 if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
2327 return malloc_default(size);
2328 }
2329
2330 tsd_t *tsd = tsd_get(false);
2331 if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) {
2332 return malloc_default(size);
2333 }
2334
2335 tcache_t *tcache = tsd_tcachep_get(tsd);
2336
2337 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2338 return malloc_default(size);
2339 }
2340
2341 szind_t ind = sz_size2index_lookup(size);
2342 size_t usize;
2343 if (config_stats || config_prof) {
2344 usize = sz_index2size(ind);
2345 }
2346 /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */
2347 assert(ind < SC_NBINS);
2348 assert(size <= SC_SMALL_MAXCLASS);
2349
2350 if (config_prof) {
2351 int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
2352 bytes_until_sample -= usize;
2353 tsd_bytes_until_sample_set(tsd, bytes_until_sample);
2354
2355 if (unlikely(bytes_until_sample < 0)) {
2356 /*
2357 * Avoid a prof_active check on the fastpath.
2358 * If prof_active is false, set bytes_until_sample to
2359 * a large value. If prof_active is set to true,
2360 * bytes_until_sample will be reset.
2361 */
2362 if (!prof_active) {
2363 tsd_bytes_until_sample_set(tsd, SSIZE_MAX);
2364 }
2365 return malloc_default(size);
2366 }
2367 }
2368
2369 cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
2370 bool tcache_success;
2371 void* ret = cache_bin_alloc_easy(bin, &tcache_success);
2372
2373 if (tcache_success) {
2374 if (config_stats) {
2375 *tsd_thread_allocatedp_get(tsd) += usize;
2376 bin->tstats.nrequests++;
2377 }
2378 if (config_prof) {
2379 tcache->prof_accumbytes += usize;
2380 }
2381
2382 LOG("core.malloc.exit", "result: %p", ret);
2383
2384 /* Fastpath success */
2385 return ret;
2386 }
2387
2388 return malloc_default(size);
2389}
2390
2391JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2392JEMALLOC_ATTR(nonnull(1))
2393je_posix_memalign(void **memptr, size_t alignment, size_t size) {
2394 int ret;
2395 static_opts_t sopts;
2396 dynamic_opts_t dopts;
2397
2398 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, "
2399 "size: %zu", memptr, alignment, size);
2400
2401 static_opts_init(&sopts);
2402 dynamic_opts_init(&dopts);
2403
2404 sopts.bump_empty_aligned_alloc = true;
2405 sopts.min_alignment = sizeof(void *);
2406 sopts.oom_string =
2407 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2408 sopts.invalid_alignment_string =
2409 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2410
2411 dopts.result = memptr;
2412 dopts.num_items = 1;
2413 dopts.item_size = size;
2414 dopts.alignment = alignment;
2415
2416 ret = imalloc(&sopts, &dopts);
2417 if (sopts.slow) {
2418 uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment,
2419 (uintptr_t)size};
2420 hook_invoke_alloc(hook_alloc_posix_memalign, *memptr,
2421 (uintptr_t)ret, args);
2422 }
2423
2424 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret,
2425 *memptr);
2426
2427 return ret;
2428}
2429
2430JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2431void JEMALLOC_NOTHROW *
2432JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
2433je_aligned_alloc(size_t alignment, size_t size) {
2434 void *ret;
2435
2436 static_opts_t sopts;
2437 dynamic_opts_t dopts;
2438
2439 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n",
2440 alignment, size);
2441
2442 static_opts_init(&sopts);
2443 dynamic_opts_init(&dopts);
2444
2445 sopts.bump_empty_aligned_alloc = true;
2446 sopts.null_out_result_on_error = true;
2447 sopts.set_errno_on_error = true;
2448 sopts.min_alignment = 1;
2449 sopts.oom_string =
2450 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2451 sopts.invalid_alignment_string =
2452 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2453
2454 dopts.result = &ret;
2455 dopts.num_items = 1;
2456 dopts.item_size = size;
2457 dopts.alignment = alignment;
2458
2459 imalloc(&sopts, &dopts);
2460 if (sopts.slow) {
2461 uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size};
2462 hook_invoke_alloc(hook_alloc_aligned_alloc, ret,
2463 (uintptr_t)ret, args);
2464 }
2465
2466 LOG("core.aligned_alloc.exit", "result: %p", ret);
2467
2468 return ret;
2469}
2470
2471JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2472void JEMALLOC_NOTHROW *
2473JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
2474je_calloc(size_t num, size_t size) {
2475 void *ret;
2476 static_opts_t sopts;
2477 dynamic_opts_t dopts;
2478
2479 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size);
2480
2481 static_opts_init(&sopts);
2482 dynamic_opts_init(&dopts);
2483
2484 sopts.may_overflow = true;
2485 sopts.null_out_result_on_error = true;
2486 sopts.set_errno_on_error = true;
2487 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n";
2488
2489 dopts.result = &ret;
2490 dopts.num_items = num;
2491 dopts.item_size = size;
2492 dopts.zero = true;
2493
2494 imalloc(&sopts, &dopts);
2495 if (sopts.slow) {
2496 uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size};
2497 hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args);
2498 }
2499
2500 LOG("core.calloc.exit", "result: %p", ret);
2501
2502 return ret;
2503}
2504
2505static void *
2506irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2507 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
2508 void *p;
2509
2510 if (tctx == NULL) {
2511 return NULL;
2512 }
2513 if (usize <= SC_SMALL_MAXCLASS) {
2514 p = iralloc(tsd, old_ptr, old_usize,
2515 SC_LARGE_MINCLASS, 0, false, hook_args);
2516 if (p == NULL) {
2517 return NULL;
2518 }
2519 arena_prof_promote(tsd_tsdn(tsd), p, usize);
2520 } else {
2521 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2522 hook_args);
2523 }
2524
2525 return p;
2526}
2527
2528JEMALLOC_ALWAYS_INLINE void *
2529irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2530 alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
2531 void *p;
2532 bool prof_active;
2533 prof_tctx_t *old_tctx, *tctx;
2534
2535 prof_active = prof_active_get_unlocked();
2536 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
2537 tctx = prof_alloc_prep(tsd, usize, prof_active, true);
2538 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2539 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx,
2540 hook_args);
2541 } else {
2542 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2543 hook_args);
2544 }
2545 if (unlikely(p == NULL)) {
2546 prof_alloc_rollback(tsd, tctx, true);
2547 return NULL;
2548 }
2549 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
2550 old_tctx);
2551
2552 return p;
2553}
2554
2555JEMALLOC_ALWAYS_INLINE void
2556ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
2557 if (!slow_path) {
2558 tsd_assert_fast(tsd);
2559 }
2560 check_entry_exit_locking(tsd_tsdn(tsd));
2561 if (tsd_reentrancy_level_get(tsd) != 0) {
2562 assert(slow_path);
2563 }
2564
2565 assert(ptr != NULL);
2566 assert(malloc_initialized() || IS_INITIALIZER);
2567
2568 alloc_ctx_t alloc_ctx;
2569 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2570 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2571 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2572 assert(alloc_ctx.szind != SC_NSIZES);
2573
2574 size_t usize;
2575 if (config_prof && opt_prof) {
2576 usize = sz_index2size(alloc_ctx.szind);
2577 prof_free(tsd, ptr, usize, &alloc_ctx);
2578 } else if (config_stats) {
2579 usize = sz_index2size(alloc_ctx.szind);
2580 }
2581 if (config_stats) {
2582 *tsd_thread_deallocatedp_get(tsd) += usize;
2583 }
2584
2585 if (likely(!slow_path)) {
2586 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2587 false);
2588 } else {
2589 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2590 true);
2591 }
2592}
2593
2594JEMALLOC_ALWAYS_INLINE void
2595isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
2596 if (!slow_path) {
2597 tsd_assert_fast(tsd);
2598 }
2599 check_entry_exit_locking(tsd_tsdn(tsd));
2600 if (tsd_reentrancy_level_get(tsd) != 0) {
2601 assert(slow_path);
2602 }
2603
2604 assert(ptr != NULL);
2605 assert(malloc_initialized() || IS_INITIALIZER);
2606
2607 alloc_ctx_t alloc_ctx, *ctx;
2608 if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) {
2609 /*
2610 * When cache_oblivious is disabled and ptr is not page aligned,
2611 * the allocation was not sampled -- usize can be used to
2612 * determine szind directly.
2613 */
2614 alloc_ctx.szind = sz_size2index(usize);
2615 alloc_ctx.slab = true;
2616 ctx = &alloc_ctx;
2617 if (config_debug) {
2618 alloc_ctx_t dbg_ctx;
2619 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2620 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree,
2621 rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind,
2622 &dbg_ctx.slab);
2623 assert(dbg_ctx.szind == alloc_ctx.szind);
2624 assert(dbg_ctx.slab == alloc_ctx.slab);
2625 }
2626 } else if (config_prof && opt_prof) {
2627 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2628 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2629 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2630 assert(alloc_ctx.szind == sz_size2index(usize));
2631 ctx = &alloc_ctx;
2632 } else {
2633 ctx = NULL;
2634 }
2635
2636 if (config_prof && opt_prof) {
2637 prof_free(tsd, ptr, usize, ctx);
2638 }
2639 if (config_stats) {
2640 *tsd_thread_deallocatedp_get(tsd) += usize;
2641 }
2642
2643 if (likely(!slow_path)) {
2644 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
2645 } else {
2646 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
2647 }
2648}
2649
2650JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2651void JEMALLOC_NOTHROW *
2652JEMALLOC_ALLOC_SIZE(2)
2653je_realloc(void *ptr, size_t arg_size) {
2654 void *ret;
2655 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
2656 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
2657 size_t old_usize = 0;
2658 size_t size = arg_size;
2659
2660 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
2661
2662 if (unlikely(size == 0)) {
2663 if (ptr != NULL) {
2664 /* realloc(ptr, 0) is equivalent to free(ptr). */
2665 UTRACE(ptr, 0, 0);
2666 tcache_t *tcache;
2667 tsd_t *tsd = tsd_fetch();
2668 if (tsd_reentrancy_level_get(tsd) == 0) {
2669 tcache = tcache_get(tsd);
2670 } else {
2671 tcache = NULL;
2672 }
2673
2674 uintptr_t args[3] = {(uintptr_t)ptr, size};
2675 hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
2676
2677 ifree(tsd, ptr, tcache, true);
2678
2679 LOG("core.realloc.exit", "result: %p", NULL);
2680 return NULL;
2681 }
2682 size = 1;
2683 }
2684
2685 if (likely(ptr != NULL)) {
2686 assert(malloc_initialized() || IS_INITIALIZER);
2687 tsd_t *tsd = tsd_fetch();
2688
2689 check_entry_exit_locking(tsd_tsdn(tsd));
2690
2691
2692 hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr,
2693 (uintptr_t)arg_size, 0, 0}};
2694
2695 alloc_ctx_t alloc_ctx;
2696 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2697 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2698 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2699 assert(alloc_ctx.szind != SC_NSIZES);
2700 old_usize = sz_index2size(alloc_ctx.szind);
2701 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
2702 if (config_prof && opt_prof) {
2703 usize = sz_s2u(size);
2704 if (unlikely(usize == 0
2705 || usize > SC_LARGE_MAXCLASS)) {
2706 ret = NULL;
2707 } else {
2708 ret = irealloc_prof(tsd, ptr, old_usize, usize,
2709 &alloc_ctx, &hook_args);
2710 }
2711 } else {
2712 if (config_stats) {
2713 usize = sz_s2u(size);
2714 }
2715 ret = iralloc(tsd, ptr, old_usize, size, 0, false,
2716 &hook_args);
2717 }
2718 tsdn = tsd_tsdn(tsd);
2719 } else {
2720 /* realloc(NULL, size) is equivalent to malloc(size). */
2721 static_opts_t sopts;
2722 dynamic_opts_t dopts;
2723
2724 static_opts_init(&sopts);
2725 dynamic_opts_init(&dopts);
2726
2727 sopts.null_out_result_on_error = true;
2728 sopts.set_errno_on_error = true;
2729 sopts.oom_string =
2730 "<jemalloc>: Error in realloc(): out of memory\n";
2731
2732 dopts.result = &ret;
2733 dopts.num_items = 1;
2734 dopts.item_size = size;
2735
2736 imalloc(&sopts, &dopts);
2737 if (sopts.slow) {
2738 uintptr_t args[3] = {(uintptr_t)ptr, arg_size};
2739 hook_invoke_alloc(hook_alloc_realloc, ret,
2740 (uintptr_t)ret, args);
2741 }
2742
2743 return ret;
2744 }
2745
2746 if (unlikely(ret == NULL)) {
2747 if (config_xmalloc && unlikely(opt_xmalloc)) {
2748 malloc_write("<jemalloc>: Error in realloc(): "
2749 "out of memory\n");
2750 abort();
2751 }
2752 set_errno(ENOMEM);
2753 }
2754 if (config_stats && likely(ret != NULL)) {
2755 tsd_t *tsd;
2756
2757 assert(usize == isalloc(tsdn, ret));
2758 tsd = tsdn_tsd(tsdn);
2759 *tsd_thread_allocatedp_get(tsd) += usize;
2760 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2761 }
2762 UTRACE(ptr, size, ret);
2763 check_entry_exit_locking(tsdn);
2764
2765 LOG("core.realloc.exit", "result: %p", ret);
2766 return ret;
2767}
2768
2769JEMALLOC_NOINLINE
2770void
2771free_default(void *ptr) {
2772 UTRACE(ptr, 0, 0);
2773 if (likely(ptr != NULL)) {
2774 /*
2775 * We avoid setting up tsd fully (e.g. tcache, arena binding)
2776 * based on only free() calls -- other activities trigger the
2777 * minimal to full transition. This is because free() may
2778 * happen during thread shutdown after tls deallocation: if a
2779 * thread never had any malloc activities until then, a
2780 * fully-setup tsd won't be destructed properly.
2781 */
2782 tsd_t *tsd = tsd_fetch_min();
2783 check_entry_exit_locking(tsd_tsdn(tsd));
2784
2785 tcache_t *tcache;
2786 if (likely(tsd_fast(tsd))) {
2787 tsd_assert_fast(tsd);
2788 /* Unconditionally get tcache ptr on fast path. */
2789 tcache = tsd_tcachep_get(tsd);
2790 ifree(tsd, ptr, tcache, false);
2791 } else {
2792 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
2793 tcache = tcache_get(tsd);
2794 } else {
2795 tcache = NULL;
2796 }
2797 uintptr_t args_raw[3] = {(uintptr_t)ptr};
2798 hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
2799 ifree(tsd, ptr, tcache, true);
2800 }
2801 check_entry_exit_locking(tsd_tsdn(tsd));
2802 }
2803}
2804
2805JEMALLOC_ALWAYS_INLINE
2806bool free_fastpath(void *ptr, size_t size, bool size_hint) {
2807 tsd_t *tsd = tsd_get(false);
2808 if (unlikely(!tsd || !tsd_fast(tsd))) {
2809 return false;
2810 }
2811
2812 tcache_t *tcache = tsd_tcachep_get(tsd);
2813
2814 alloc_ctx_t alloc_ctx;
2815 /*
2816 * If !config_cache_oblivious, we can check PAGE alignment to
2817 * detect sampled objects. Otherwise addresses are
2818 * randomized, and we have to look it up in the rtree anyway.
2819 * See also isfree().
2820 */
2821 if (!size_hint || config_cache_oblivious) {
2822 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2823 bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree,
2824 rtree_ctx, (uintptr_t)ptr,
2825 &alloc_ctx.szind, &alloc_ctx.slab);
2826
2827 /* Note: profiled objects will have alloc_ctx.slab set */
2828 if (!res || !alloc_ctx.slab) {
2829 return false;
2830 }
2831 assert(alloc_ctx.szind != SC_NSIZES);
2832 } else {
2833 /*
2834 * Check for both sizes that are too large, and for sampled objects.
2835 * Sampled objects are always page-aligned. The sampled object check
2836 * will also check for null ptr.
2837 */
2838 if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) {
2839 return false;
2840 }
2841 alloc_ctx.szind = sz_size2index_lookup(size);
2842 }
2843
2844 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2845 return false;
2846 }
2847
2848 cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind);
2849 cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind];
2850 if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) {
2851 return false;
2852 }
2853
2854 if (config_stats) {
2855 size_t usize = sz_index2size(alloc_ctx.szind);
2856 *tsd_thread_deallocatedp_get(tsd) += usize;
2857 }
2858
2859 return true;
2860}
2861
2862JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2863je_free(void *ptr) {
2864 LOG("core.free.entry", "ptr: %p", ptr);
2865
2866 if (!free_fastpath(ptr, 0, false)) {
2867 free_default(ptr);
2868 }
2869
2870 LOG("core.free.exit", "");
2871}
2872
2873/*
2874 * End malloc(3)-compatible functions.
2875 */
2876/******************************************************************************/
2877/*
2878 * Begin non-standard override functions.
2879 */
2880
2881#ifdef JEMALLOC_OVERRIDE_MEMALIGN
2882JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2883void JEMALLOC_NOTHROW *
2884JEMALLOC_ATTR(malloc)
2885je_memalign(size_t alignment, size_t size) {
2886 void *ret;
2887 static_opts_t sopts;
2888 dynamic_opts_t dopts;
2889
2890 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment,
2891 size);
2892
2893 static_opts_init(&sopts);
2894 dynamic_opts_init(&dopts);
2895
2896 sopts.min_alignment = 1;
2897 sopts.oom_string =
2898 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2899 sopts.invalid_alignment_string =
2900 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2901 sopts.null_out_result_on_error = true;
2902
2903 dopts.result = &ret;
2904 dopts.num_items = 1;
2905 dopts.item_size = size;
2906 dopts.alignment = alignment;
2907
2908 imalloc(&sopts, &dopts);
2909 if (sopts.slow) {
2910 uintptr_t args[3] = {alignment, size};
2911 hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret,
2912 args);
2913 }
2914
2915 LOG("core.memalign.exit", "result: %p", ret);
2916 return ret;
2917}
2918#endif
2919
2920#ifdef JEMALLOC_OVERRIDE_VALLOC
2921JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2922void JEMALLOC_NOTHROW *
2923JEMALLOC_ATTR(malloc)
2924je_valloc(size_t size) {
2925 void *ret;
2926
2927 static_opts_t sopts;
2928 dynamic_opts_t dopts;
2929
2930 LOG("core.valloc.entry", "size: %zu\n", size);
2931
2932 static_opts_init(&sopts);
2933 dynamic_opts_init(&dopts);
2934
2935 sopts.null_out_result_on_error = true;
2936 sopts.min_alignment = PAGE;
2937 sopts.oom_string =
2938 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2939 sopts.invalid_alignment_string =
2940 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2941
2942 dopts.result = &ret;
2943 dopts.num_items = 1;
2944 dopts.item_size = size;
2945 dopts.alignment = PAGE;
2946
2947 imalloc(&sopts, &dopts);
2948 if (sopts.slow) {
2949 uintptr_t args[3] = {size};
2950 hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args);
2951 }
2952
2953 LOG("core.valloc.exit", "result: %p\n", ret);
2954 return ret;
2955}
2956#endif
2957
2958#if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)
2959/*
2960 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
2961 * to inconsistently reference libc's malloc(3)-compatible functions
2962 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
2963 *
2964 * These definitions interpose hooks in glibc. The functions are actually
2965 * passed an extra argument for the caller return address, which will be
2966 * ignored.
2967 */
2968JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
2969JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
2970JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
2971# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
2972JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
2973 je_memalign;
2974# endif
2975
2976# ifdef CPU_COUNT
2977/*
2978 * To enable static linking with glibc, the libc specific malloc interface must
2979 * be implemented also, so none of glibc's malloc.o functions are added to the
2980 * link.
2981 */
2982# define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
2983/* To force macro expansion of je_ prefix before stringification. */
2984# define PREALIAS(je_fn) ALIAS(je_fn)
2985# ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC
2986void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
2987# endif
2988# ifdef JEMALLOC_OVERRIDE___LIBC_FREE
2989void __libc_free(void* ptr) PREALIAS(je_free);
2990# endif
2991# ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC
2992void *__libc_malloc(size_t size) PREALIAS(je_malloc);
2993# endif
2994# ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN
2995void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
2996# endif
2997# ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC
2998void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
2999# endif
3000# ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC
3001void *__libc_valloc(size_t size) PREALIAS(je_valloc);
3002# endif
3003# ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN
3004int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
3005# endif
3006# undef PREALIAS
3007# undef ALIAS
3008# endif
3009#endif
3010
3011/*
3012 * End non-standard override functions.
3013 */
3014/******************************************************************************/
3015/*
3016 * Begin non-standard functions.
3017 */
3018
3019#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
3020
3021#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
3022#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
3023 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
3024
3025typedef struct {
3026 void *ptr;
3027 size_t size;
3028} smallocx_return_t;
3029
3030JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3031smallocx_return_t JEMALLOC_NOTHROW
3032/*
3033 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to:
3034 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
3035 */
3036JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
3037 (size_t size, int flags) {
3038 /*
3039 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be
3040 * used here because it makes writing beyond the `size`
3041 * of the `ptr` undefined behavior, but the objective
3042 * of this function is to allow writing beyond `size`
3043 * up to `smallocx_return_t::size`.
3044 */
3045 smallocx_return_t ret;
3046 static_opts_t sopts;
3047 dynamic_opts_t dopts;
3048
3049 LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags);
3050
3051 static_opts_init(&sopts);
3052 dynamic_opts_init(&dopts);
3053
3054 sopts.assert_nonempty_alloc = true;
3055 sopts.null_out_result_on_error = true;
3056 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3057 sopts.usize = true;
3058
3059 dopts.result = &ret.ptr;
3060 dopts.num_items = 1;
3061 dopts.item_size = size;
3062 if (unlikely(flags != 0)) {
3063 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3064 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3065 }
3066
3067 dopts.zero = MALLOCX_ZERO_GET(flags);
3068
3069 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3070 if ((flags & MALLOCX_TCACHE_MASK)
3071 == MALLOCX_TCACHE_NONE) {
3072 dopts.tcache_ind = TCACHE_IND_NONE;
3073 } else {
3074 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3075 }
3076 } else {
3077 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3078 }
3079
3080 if ((flags & MALLOCX_ARENA_MASK) != 0)
3081 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3082 }
3083
3084 imalloc(&sopts, &dopts);
3085 assert(dopts.usize == je_nallocx(size, flags));
3086 ret.size = dopts.usize;
3087
3088 LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size);
3089 return ret;
3090}
3091#undef JEMALLOC_SMALLOCX_CONCAT_HELPER
3092#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2
3093#endif
3094
3095JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3096void JEMALLOC_NOTHROW *
3097JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
3098je_mallocx(size_t size, int flags) {
3099 void *ret;
3100 static_opts_t sopts;
3101 dynamic_opts_t dopts;
3102
3103 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags);
3104
3105 static_opts_init(&sopts);
3106 dynamic_opts_init(&dopts);
3107
3108 sopts.assert_nonempty_alloc = true;
3109 sopts.null_out_result_on_error = true;
3110 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3111
3112 dopts.result = &ret;
3113 dopts.num_items = 1;
3114 dopts.item_size = size;
3115 if (unlikely(flags != 0)) {
3116 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3117 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3118 }
3119
3120 dopts.zero = MALLOCX_ZERO_GET(flags);
3121
3122 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3123 if ((flags & MALLOCX_TCACHE_MASK)
3124 == MALLOCX_TCACHE_NONE) {
3125 dopts.tcache_ind = TCACHE_IND_NONE;
3126 } else {
3127 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3128 }
3129 } else {
3130 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3131 }
3132
3133 if ((flags & MALLOCX_ARENA_MASK) != 0)
3134 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3135 }
3136
3137 imalloc(&sopts, &dopts);
3138 if (sopts.slow) {
3139 uintptr_t args[3] = {size, flags};
3140 hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret,
3141 args);
3142 }
3143
3144 LOG("core.mallocx.exit", "result: %p", ret);
3145 return ret;
3146}
3147
3148static void *
3149irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
3150 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
3151 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
3152 void *p;
3153
3154 if (tctx == NULL) {
3155 return NULL;
3156 }
3157 if (usize <= SC_SMALL_MAXCLASS) {
3158 p = iralloct(tsdn, old_ptr, old_usize,
3159 SC_LARGE_MINCLASS, alignment, zero, tcache,
3160 arena, hook_args);
3161 if (p == NULL) {
3162 return NULL;
3163 }
3164 arena_prof_promote(tsdn, p, usize);
3165 } else {
3166 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
3167 tcache, arena, hook_args);
3168 }
3169
3170 return p;
3171}
3172
3173JEMALLOC_ALWAYS_INLINE void *
3174irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
3175 size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
3176 arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
3177 void *p;
3178 bool prof_active;
3179 prof_tctx_t *old_tctx, *tctx;
3180
3181 prof_active = prof_active_get_unlocked();
3182 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
3183 tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
3184 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3185 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
3186 *usize, alignment, zero, tcache, arena, tctx, hook_args);
3187 } else {
3188 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
3189 zero, tcache, arena, hook_args);
3190 }
3191 if (unlikely(p == NULL)) {
3192 prof_alloc_rollback(tsd, tctx, false);
3193 return NULL;
3194 }
3195
3196 if (p == old_ptr && alignment != 0) {
3197 /*
3198 * The allocation did not move, so it is possible that the size
3199 * class is smaller than would guarantee the requested
3200 * alignment, and that the alignment constraint was
3201 * serendipitously satisfied. Additionally, old_usize may not
3202 * be the same as the current usize because of in-place large
3203 * reallocation. Therefore, query the actual value of usize.
3204 */
3205 *usize = isalloc(tsd_tsdn(tsd), p);
3206 }
3207 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
3208 old_usize, old_tctx);
3209
3210 return p;
3211}
3212
3213JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3214void JEMALLOC_NOTHROW *
3215JEMALLOC_ALLOC_SIZE(2)
3216je_rallocx(void *ptr, size_t size, int flags) {
3217 void *p;
3218 tsd_t *tsd;
3219 size_t usize;
3220 size_t old_usize;
3221 size_t alignment = MALLOCX_ALIGN_GET(flags);
3222 bool zero = flags & MALLOCX_ZERO;
3223 arena_t *arena;
3224 tcache_t *tcache;
3225
3226 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3227 size, flags);
3228
3229
3230 assert(ptr != NULL);
3231 assert(size != 0);
3232 assert(malloc_initialized() || IS_INITIALIZER);
3233 tsd = tsd_fetch();
3234 check_entry_exit_locking(tsd_tsdn(tsd));
3235
3236 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
3237 unsigned arena_ind = MALLOCX_ARENA_GET(flags);
3238 arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
3239 if (unlikely(arena == NULL)) {
3240 goto label_oom;
3241 }
3242 } else {
3243 arena = NULL;
3244 }
3245
3246 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3247 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3248 tcache = NULL;
3249 } else {
3250 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3251 }
3252 } else {
3253 tcache = tcache_get(tsd);
3254 }
3255
3256 alloc_ctx_t alloc_ctx;
3257 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3258 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3259 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3260 assert(alloc_ctx.szind != SC_NSIZES);
3261 old_usize = sz_index2size(alloc_ctx.szind);
3262 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3263
3264 hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags,
3265 0}};
3266 if (config_prof && opt_prof) {
3267 usize = (alignment == 0) ?
3268 sz_s2u(size) : sz_sa2u(size, alignment);
3269 if (unlikely(usize == 0
3270 || usize > SC_LARGE_MAXCLASS)) {
3271 goto label_oom;
3272 }
3273 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
3274 zero, tcache, arena, &alloc_ctx, &hook_args);
3275 if (unlikely(p == NULL)) {
3276 goto label_oom;
3277 }
3278 } else {
3279 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
3280 zero, tcache, arena, &hook_args);
3281 if (unlikely(p == NULL)) {
3282 goto label_oom;
3283 }
3284 if (config_stats) {
3285 usize = isalloc(tsd_tsdn(tsd), p);
3286 }
3287 }
3288 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
3289
3290 if (config_stats) {
3291 *tsd_thread_allocatedp_get(tsd) += usize;
3292 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3293 }
3294 UTRACE(ptr, size, p);
3295 check_entry_exit_locking(tsd_tsdn(tsd));
3296
3297 LOG("core.rallocx.exit", "result: %p", p);
3298 return p;
3299label_oom:
3300 if (config_xmalloc && unlikely(opt_xmalloc)) {
3301 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
3302 abort();
3303 }
3304 UTRACE(ptr, size, 0);
3305 check_entry_exit_locking(tsd_tsdn(tsd));
3306
3307 LOG("core.rallocx.exit", "result: %p", NULL);
3308 return NULL;
3309}
3310
3311JEMALLOC_ALWAYS_INLINE size_t
3312ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3313 size_t extra, size_t alignment, bool zero) {
3314 size_t newsize;
3315
3316 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero,
3317 &newsize)) {
3318 return old_usize;
3319 }
3320
3321 return newsize;
3322}
3323
3324static size_t
3325ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3326 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
3327 size_t usize;
3328
3329 if (tctx == NULL) {
3330 return old_usize;
3331 }
3332 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
3333 zero);
3334
3335 return usize;
3336}
3337
3338JEMALLOC_ALWAYS_INLINE size_t
3339ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
3340 size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) {
3341 size_t usize_max, usize;
3342 bool prof_active;
3343 prof_tctx_t *old_tctx, *tctx;
3344
3345 prof_active = prof_active_get_unlocked();
3346 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx);
3347 /*
3348 * usize isn't knowable before ixalloc() returns when extra is non-zero.
3349 * Therefore, compute its maximum possible value and use that in
3350 * prof_alloc_prep() to decide whether to capture a backtrace.
3351 * prof_realloc() will use the actual usize to decide whether to sample.
3352 */
3353 if (alignment == 0) {
3354 usize_max = sz_s2u(size+extra);
3355 assert(usize_max > 0
3356 && usize_max <= SC_LARGE_MAXCLASS);
3357 } else {
3358 usize_max = sz_sa2u(size+extra, alignment);
3359 if (unlikely(usize_max == 0
3360 || usize_max > SC_LARGE_MAXCLASS)) {
3361 /*
3362 * usize_max is out of range, and chances are that
3363 * allocation will fail, but use the maximum possible
3364 * value and carry on with prof_alloc_prep(), just in
3365 * case allocation succeeds.
3366 */
3367 usize_max = SC_LARGE_MAXCLASS;
3368 }
3369 }
3370 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
3371
3372 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3373 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
3374 size, extra, alignment, zero, tctx);
3375 } else {
3376 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3377 extra, alignment, zero);
3378 }
3379 if (usize == old_usize) {
3380 prof_alloc_rollback(tsd, tctx, false);
3381 return usize;
3382 }
3383 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
3384 old_tctx);
3385
3386 return usize;
3387}
3388
3389JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3390je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
3391 tsd_t *tsd;
3392 size_t usize, old_usize;
3393 size_t alignment = MALLOCX_ALIGN_GET(flags);
3394 bool zero = flags & MALLOCX_ZERO;
3395
3396 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
3397 "flags: %d", ptr, size, extra, flags);
3398
3399 assert(ptr != NULL);
3400 assert(size != 0);
3401 assert(SIZE_T_MAX - size >= extra);
3402 assert(malloc_initialized() || IS_INITIALIZER);
3403 tsd = tsd_fetch();
3404 check_entry_exit_locking(tsd_tsdn(tsd));
3405
3406 alloc_ctx_t alloc_ctx;
3407 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3408 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3409 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3410 assert(alloc_ctx.szind != SC_NSIZES);
3411 old_usize = sz_index2size(alloc_ctx.szind);
3412 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3413 /*
3414 * The API explicitly absolves itself of protecting against (size +
3415 * extra) numerical overflow, but we may need to clamp extra to avoid
3416 * exceeding SC_LARGE_MAXCLASS.
3417 *
3418 * Ordinarily, size limit checking is handled deeper down, but here we
3419 * have to check as part of (size + extra) clamping, since we need the
3420 * clamped value in the above helper functions.
3421 */
3422 if (unlikely(size > SC_LARGE_MAXCLASS)) {
3423 usize = old_usize;
3424 goto label_not_resized;
3425 }
3426 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) {
3427 extra = SC_LARGE_MAXCLASS - size;
3428 }
3429
3430 if (config_prof && opt_prof) {
3431 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
3432 alignment, zero, &alloc_ctx);
3433 } else {
3434 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3435 extra, alignment, zero);
3436 }
3437 if (unlikely(usize == old_usize)) {
3438 goto label_not_resized;
3439 }
3440
3441 if (config_stats) {
3442 *tsd_thread_allocatedp_get(tsd) += usize;
3443 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3444 }
3445label_not_resized:
3446 if (unlikely(!tsd_fast(tsd))) {
3447 uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};
3448 hook_invoke_expand(hook_expand_xallocx, ptr, old_usize,
3449 usize, (uintptr_t)usize, args);
3450 }
3451
3452 UTRACE(ptr, size, ptr);
3453 check_entry_exit_locking(tsd_tsdn(tsd));
3454
3455 LOG("core.xallocx.exit", "result: %zu", usize);
3456 return usize;
3457}
3458
3459JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3460JEMALLOC_ATTR(pure)
3461je_sallocx(const void *ptr, int flags) {
3462 size_t usize;
3463 tsdn_t *tsdn;
3464
3465 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3466
3467 assert(malloc_initialized() || IS_INITIALIZER);
3468 assert(ptr != NULL);
3469
3470 tsdn = tsdn_fetch();
3471 check_entry_exit_locking(tsdn);
3472
3473 if (config_debug || force_ivsalloc) {
3474 usize = ivsalloc(tsdn, ptr);
3475 assert(force_ivsalloc || usize != 0);
3476 } else {
3477 usize = isalloc(tsdn, ptr);
3478 }
3479
3480 check_entry_exit_locking(tsdn);
3481
3482 LOG("core.sallocx.exit", "result: %zu", usize);
3483 return usize;
3484}
3485
3486JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3487je_dallocx(void *ptr, int flags) {
3488 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3489
3490 assert(ptr != NULL);
3491 assert(malloc_initialized() || IS_INITIALIZER);
3492
3493 tsd_t *tsd = tsd_fetch();
3494 bool fast = tsd_fast(tsd);
3495 check_entry_exit_locking(tsd_tsdn(tsd));
3496
3497 tcache_t *tcache;
3498 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3499 /* Not allowed to be reentrant and specify a custom tcache. */
3500 assert(tsd_reentrancy_level_get(tsd) == 0);
3501 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3502 tcache = NULL;
3503 } else {
3504 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3505 }
3506 } else {
3507 if (likely(fast)) {
3508 tcache = tsd_tcachep_get(tsd);
3509 assert(tcache == tcache_get(tsd));
3510 } else {
3511 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3512 tcache = tcache_get(tsd);
3513 } else {
3514 tcache = NULL;
3515 }
3516 }
3517 }
3518
3519 UTRACE(ptr, 0, 0);
3520 if (likely(fast)) {
3521 tsd_assert_fast(tsd);
3522 ifree(tsd, ptr, tcache, false);
3523 } else {
3524 uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
3525 hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
3526 ifree(tsd, ptr, tcache, true);
3527 }
3528 check_entry_exit_locking(tsd_tsdn(tsd));
3529
3530 LOG("core.dallocx.exit", "");
3531}
3532
3533JEMALLOC_ALWAYS_INLINE size_t
3534inallocx(tsdn_t *tsdn, size_t size, int flags) {
3535 check_entry_exit_locking(tsdn);
3536
3537 size_t usize;
3538 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
3539 usize = sz_s2u(size);
3540 } else {
3541 usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
3542 }
3543 check_entry_exit_locking(tsdn);
3544 return usize;
3545}
3546
3547JEMALLOC_NOINLINE void
3548sdallocx_default(void *ptr, size_t size, int flags) {
3549 assert(ptr != NULL);
3550 assert(malloc_initialized() || IS_INITIALIZER);
3551
3552 tsd_t *tsd = tsd_fetch();
3553 bool fast = tsd_fast(tsd);
3554 size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
3555 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
3556 check_entry_exit_locking(tsd_tsdn(tsd));
3557
3558 tcache_t *tcache;
3559 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3560 /* Not allowed to be reentrant and specify a custom tcache. */
3561 assert(tsd_reentrancy_level_get(tsd) == 0);
3562 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3563 tcache = NULL;
3564 } else {
3565 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3566 }
3567 } else {
3568 if (likely(fast)) {
3569 tcache = tsd_tcachep_get(tsd);
3570 assert(tcache == tcache_get(tsd));
3571 } else {
3572 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3573 tcache = tcache_get(tsd);
3574 } else {
3575 tcache = NULL;
3576 }
3577 }
3578 }
3579
3580 UTRACE(ptr, 0, 0);
3581 if (likely(fast)) {
3582 tsd_assert_fast(tsd);
3583 isfree(tsd, ptr, usize, tcache, false);
3584 } else {
3585 uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags};
3586 hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw);
3587 isfree(tsd, ptr, usize, tcache, true);
3588 }
3589 check_entry_exit_locking(tsd_tsdn(tsd));
3590
3591}
3592
3593JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3594je_sdallocx(void *ptr, size_t size, int flags) {
3595 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3596 size, flags);
3597
3598 if (flags !=0 || !free_fastpath(ptr, size, true)) {
3599 sdallocx_default(ptr, size, flags);
3600 }
3601
3602 LOG("core.sdallocx.exit", "");
3603}
3604
3605void JEMALLOC_NOTHROW
3606je_sdallocx_noflags(void *ptr, size_t size) {
3607 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr,
3608 size);
3609
3610 if (!free_fastpath(ptr, size, true)) {
3611 sdallocx_default(ptr, size, 0);
3612 }
3613
3614 LOG("core.sdallocx.exit", "");
3615}
3616
3617JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3618JEMALLOC_ATTR(pure)
3619je_nallocx(size_t size, int flags) {
3620 size_t usize;
3621 tsdn_t *tsdn;
3622
3623 assert(size != 0);
3624
3625 if (unlikely(malloc_init())) {
3626 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3627 return 0;
3628 }
3629
3630 tsdn = tsdn_fetch();
3631 check_entry_exit_locking(tsdn);
3632
3633 usize = inallocx(tsdn, size, flags);
3634 if (unlikely(usize > SC_LARGE_MAXCLASS)) {
3635 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3636 return 0;
3637 }
3638
3639 check_entry_exit_locking(tsdn);
3640 LOG("core.nallocx.exit", "result: %zu", usize);
3641 return usize;
3642}
3643
3644JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3645je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
3646 size_t newlen) {
3647 int ret;
3648 tsd_t *tsd;
3649
3650 LOG("core.mallctl.entry", "name: %s", name);
3651
3652 if (unlikely(malloc_init())) {
3653 LOG("core.mallctl.exit", "result: %d", EAGAIN);
3654 return EAGAIN;
3655 }
3656
3657 tsd = tsd_fetch();
3658 check_entry_exit_locking(tsd_tsdn(tsd));
3659 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
3660 check_entry_exit_locking(tsd_tsdn(tsd));
3661
3662 LOG("core.mallctl.exit", "result: %d", ret);
3663 return ret;
3664}
3665
3666JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3667je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
3668 int ret;
3669
3670 LOG("core.mallctlnametomib.entry", "name: %s", name);
3671
3672 if (unlikely(malloc_init())) {
3673 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN);
3674 return EAGAIN;
3675 }
3676
3677 tsd_t *tsd = tsd_fetch();
3678 check_entry_exit_locking(tsd_tsdn(tsd));
3679 ret = ctl_nametomib(tsd, name, mibp, miblenp);
3680 check_entry_exit_locking(tsd_tsdn(tsd));
3681
3682 LOG("core.mallctlnametomib.exit", "result: %d", ret);
3683 return ret;
3684}
3685
3686JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3687je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
3688 void *newp, size_t newlen) {
3689 int ret;
3690 tsd_t *tsd;
3691
3692 LOG("core.mallctlbymib.entry", "");
3693
3694 if (unlikely(malloc_init())) {
3695 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN);
3696 return EAGAIN;
3697 }
3698
3699 tsd = tsd_fetch();
3700 check_entry_exit_locking(tsd_tsdn(tsd));
3701 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
3702 check_entry_exit_locking(tsd_tsdn(tsd));
3703 LOG("core.mallctlbymib.exit", "result: %d", ret);
3704 return ret;
3705}
3706
3707JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3708je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
3709 const char *opts) {
3710 tsdn_t *tsdn;
3711
3712 LOG("core.malloc_stats_print.entry", "");
3713
3714 tsdn = tsdn_fetch();
3715 check_entry_exit_locking(tsdn);
3716 stats_print(write_cb, cbopaque, opts);
3717 check_entry_exit_locking(tsdn);
3718 LOG("core.malloc_stats_print.exit", "");
3719}
3720
3721JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3722je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
3723 size_t ret;
3724 tsdn_t *tsdn;
3725
3726 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
3727
3728 assert(malloc_initialized() || IS_INITIALIZER);
3729
3730 tsdn = tsdn_fetch();
3731 check_entry_exit_locking(tsdn);
3732
3733 if (unlikely(ptr == NULL)) {
3734 ret = 0;
3735 } else {
3736 if (config_debug || force_ivsalloc) {
3737 ret = ivsalloc(tsdn, ptr);
3738 assert(force_ivsalloc || ret != 0);
3739 } else {
3740 ret = isalloc(tsdn, ptr);
3741 }
3742 }
3743
3744 check_entry_exit_locking(tsdn);
3745 LOG("core.malloc_usable_size.exit", "result: %zu", ret);
3746 return ret;
3747}
3748
3749/*
3750 * End non-standard functions.
3751 */
3752/******************************************************************************/
3753/*
3754 * The following functions are used by threading libraries for protection of
3755 * malloc during fork().
3756 */
3757
3758/*
3759 * If an application creates a thread before doing any allocation in the main
3760 * thread, then calls fork(2) in the main thread followed by memory allocation
3761 * in the child process, a race can occur that results in deadlock within the
3762 * child: the main thread may have forked while the created thread had
3763 * partially initialized the allocator. Ordinarily jemalloc prevents
3764 * fork/malloc races via the following functions it registers during
3765 * initialization using pthread_atfork(), but of course that does no good if
3766 * the allocator isn't fully initialized at fork time. The following library
3767 * constructor is a partial solution to this problem. It may still be possible
3768 * to trigger the deadlock described above, but doing so would involve forking
3769 * via a library constructor that runs before jemalloc's runs.
3770 */
3771#ifndef JEMALLOC_JET
3772JEMALLOC_ATTR(constructor)
3773static void
3774jemalloc_constructor(void) {
3775 malloc_init();
3776}
3777#endif
3778
3779#ifndef JEMALLOC_MUTEX_INIT_CB
3780void
3781jemalloc_prefork(void)
3782#else
3783JEMALLOC_EXPORT void
3784_malloc_prefork(void)
3785#endif
3786{
3787 tsd_t *tsd;
3788 unsigned i, j, narenas;
3789 arena_t *arena;
3790
3791#ifdef JEMALLOC_MUTEX_INIT_CB
3792 if (!malloc_initialized()) {
3793 return;
3794 }
3795#endif
3796 assert(malloc_initialized());
3797
3798 tsd = tsd_fetch();
3799
3800 narenas = narenas_total_get();
3801
3802 witness_prefork(tsd_witness_tsdp_get(tsd));
3803 /* Acquire all mutexes in a safe order. */
3804 ctl_prefork(tsd_tsdn(tsd));
3805 tcache_prefork(tsd_tsdn(tsd));
3806 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
3807 if (have_background_thread) {
3808 background_thread_prefork0(tsd_tsdn(tsd));
3809 }
3810 prof_prefork0(tsd_tsdn(tsd));
3811 if (have_background_thread) {
3812 background_thread_prefork1(tsd_tsdn(tsd));
3813 }
3814 /* Break arena prefork into stages to preserve lock order. */
3815 for (i = 0; i < 8; i++) {
3816 for (j = 0; j < narenas; j++) {
3817 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
3818 NULL) {
3819 switch (i) {
3820 case 0:
3821 arena_prefork0(tsd_tsdn(tsd), arena);
3822 break;
3823 case 1:
3824 arena_prefork1(tsd_tsdn(tsd), arena);
3825 break;
3826 case 2:
3827 arena_prefork2(tsd_tsdn(tsd), arena);
3828 break;
3829 case 3:
3830 arena_prefork3(tsd_tsdn(tsd), arena);
3831 break;
3832 case 4:
3833 arena_prefork4(tsd_tsdn(tsd), arena);
3834 break;
3835 case 5:
3836 arena_prefork5(tsd_tsdn(tsd), arena);
3837 break;
3838 case 6:
3839 arena_prefork6(tsd_tsdn(tsd), arena);
3840 break;
3841 case 7:
3842 arena_prefork7(tsd_tsdn(tsd), arena);
3843 break;
3844 default: not_reached();
3845 }
3846 }
3847 }
3848 }
3849 prof_prefork1(tsd_tsdn(tsd));
3850 tsd_prefork(tsd);
3851}
3852
3853#ifndef JEMALLOC_MUTEX_INIT_CB
3854void
3855jemalloc_postfork_parent(void)
3856#else
3857JEMALLOC_EXPORT void
3858_malloc_postfork(void)
3859#endif
3860{
3861 tsd_t *tsd;
3862 unsigned i, narenas;
3863
3864#ifdef JEMALLOC_MUTEX_INIT_CB
3865 if (!malloc_initialized()) {
3866 return;
3867 }
3868#endif
3869 assert(malloc_initialized());
3870
3871 tsd = tsd_fetch();
3872
3873 tsd_postfork_parent(tsd);
3874
3875 witness_postfork_parent(tsd_witness_tsdp_get(tsd));
3876 /* Release all mutexes, now that fork() has completed. */
3877 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3878 arena_t *arena;
3879
3880 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3881 arena_postfork_parent(tsd_tsdn(tsd), arena);
3882 }
3883 }
3884 prof_postfork_parent(tsd_tsdn(tsd));
3885 if (have_background_thread) {
3886 background_thread_postfork_parent(tsd_tsdn(tsd));
3887 }
3888 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
3889 tcache_postfork_parent(tsd_tsdn(tsd));
3890 ctl_postfork_parent(tsd_tsdn(tsd));
3891}
3892
3893void
3894jemalloc_postfork_child(void) {
3895 tsd_t *tsd;
3896 unsigned i, narenas;
3897
3898 assert(malloc_initialized());
3899
3900 tsd = tsd_fetch();
3901
3902 tsd_postfork_child(tsd);
3903
3904 witness_postfork_child(tsd_witness_tsdp_get(tsd));
3905 /* Release all mutexes, now that fork() has completed. */
3906 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3907 arena_t *arena;
3908
3909 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3910 arena_postfork_child(tsd_tsdn(tsd), arena);
3911 }
3912 }
3913 prof_postfork_child(tsd_tsdn(tsd));
3914 if (have_background_thread) {
3915 background_thread_postfork_child(tsd_tsdn(tsd));
3916 }
3917 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
3918 tcache_postfork_child(tsd_tsdn(tsd));
3919 ctl_postfork_child(tsd_tsdn(tsd));
3920}
3921
3922/******************************************************************************/
3923
3924/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
3925 * returns 1 if the allocation should be moved, and 0 if the allocation be kept.
3926 * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
3927JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3928get_defrag_hint(void* ptr) {
3929 assert(ptr != NULL);
3930 return iget_defrag_hint(TSDN_NULL, ptr);
3931}
3932