1#define JEMALLOC_TCACHE_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/mutex.h"
7#include "jemalloc/internal/safety_check.h"
8#include "jemalloc/internal/sc.h"
9
10/******************************************************************************/
11/* Data. */
12
13bool opt_tcache = true;
14ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
15
16cache_bin_info_t *tcache_bin_info;
17static unsigned stack_nelms; /* Total stack elms per tcache. */
18
19unsigned nhbins;
20size_t tcache_maxclass;
21
22tcaches_t *tcaches;
23
24/* Index of first element within tcaches that has never been used. */
25static unsigned tcaches_past;
26
27/* Head of singly linked list tracking available tcaches elements. */
28static tcaches_t *tcaches_avail;
29
30/* Protects tcaches{,_past,_avail}. */
31static malloc_mutex_t tcaches_mtx;
32
33/******************************************************************************/
34
35size_t
36tcache_salloc(tsdn_t *tsdn, const void *ptr) {
37 return arena_salloc(tsdn, ptr);
38}
39
40void
41tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
42 szind_t binind = tcache->next_gc_bin;
43
44 cache_bin_t *tbin;
45 if (binind < SC_NBINS) {
46 tbin = tcache_small_bin_get(tcache, binind);
47 } else {
48 tbin = tcache_large_bin_get(tcache, binind);
49 }
50 if (tbin->low_water > 0) {
51 /*
52 * Flush (ceiling) 3/4 of the objects below the low water mark.
53 */
54 if (binind < SC_NBINS) {
55 tcache_bin_flush_small(tsd, tcache, tbin, binind,
56 tbin->ncached - tbin->low_water + (tbin->low_water
57 >> 2));
58 /*
59 * Reduce fill count by 2X. Limit lg_fill_div such that
60 * the fill count is always at least 1.
61 */
62 cache_bin_info_t *tbin_info = &tcache_bin_info[binind];
63 if ((tbin_info->ncached_max >>
64 (tcache->lg_fill_div[binind] + 1)) >= 1) {
65 tcache->lg_fill_div[binind]++;
66 }
67 } else {
68 tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached
69 - tbin->low_water + (tbin->low_water >> 2), tcache);
70 }
71 } else if (tbin->low_water < 0) {
72 /*
73 * Increase fill count by 2X for small bins. Make sure
74 * lg_fill_div stays greater than 0.
75 */
76 if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) {
77 tcache->lg_fill_div[binind]--;
78 }
79 }
80 tbin->low_water = tbin->ncached;
81
82 tcache->next_gc_bin++;
83 if (tcache->next_gc_bin == nhbins) {
84 tcache->next_gc_bin = 0;
85 }
86}
87
88void *
89tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
90 cache_bin_t *tbin, szind_t binind, bool *tcache_success) {
91 void *ret;
92
93 assert(tcache->arena != NULL);
94 arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind,
95 config_prof ? tcache->prof_accumbytes : 0);
96 if (config_prof) {
97 tcache->prof_accumbytes = 0;
98 }
99 ret = cache_bin_alloc_easy(tbin, tcache_success);
100
101 return ret;
102}
103
104/* Enabled with --enable-extra-size-check. */
105static void
106tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind,
107 size_t nflush, extent_t **extents){
108 rtree_ctx_t rtree_ctx_fallback;
109 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
110
111 /*
112 * Verify that the items in the tcache all have the correct size; this
113 * is useful for catching sized deallocation bugs, also to fail early
114 * instead of corrupting metadata. Since this can be turned on for opt
115 * builds, avoid the branch in the loop.
116 */
117 szind_t szind;
118 size_t sz_sum = binind * nflush;
119 for (unsigned i = 0 ; i < nflush; i++) {
120 rtree_extent_szind_read(tsdn, &extents_rtree,
121 rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true,
122 &extents[i], &szind);
123 sz_sum -= szind;
124 }
125 if (sz_sum != 0) {
126 safety_check_fail("<jemalloc>: size mismatch in thread cache "
127 "detected, likely caused by sized deallocation bugs by "
128 "application. Abort.\n");
129 abort();
130 }
131}
132
133void
134tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
135 szind_t binind, unsigned rem) {
136 bool merged_stats = false;
137
138 assert(binind < SC_NBINS);
139 assert((cache_bin_sz_t)rem <= tbin->ncached);
140
141 arena_t *arena = tcache->arena;
142 assert(arena != NULL);
143 unsigned nflush = tbin->ncached - rem;
144 VARIABLE_ARRAY(extent_t *, item_extent, nflush);
145
146 /* Look up extent once per item. */
147 if (config_opt_safety_checks) {
148 tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind,
149 nflush, item_extent);
150 } else {
151 for (unsigned i = 0 ; i < nflush; i++) {
152 item_extent[i] = iealloc(tsd_tsdn(tsd),
153 *(tbin->avail - 1 - i));
154 }
155 }
156 while (nflush > 0) {
157 /* Lock the arena bin associated with the first object. */
158 extent_t *extent = item_extent[0];
159 unsigned bin_arena_ind = extent_arena_ind_get(extent);
160 arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind,
161 false);
162 unsigned binshard = extent_binshard_get(extent);
163 assert(binshard < bin_infos[binind].n_shards);
164 bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard];
165
166 if (config_prof && bin_arena == arena) {
167 if (arena_prof_accum(tsd_tsdn(tsd), arena,
168 tcache->prof_accumbytes)) {
169 prof_idump(tsd_tsdn(tsd));
170 }
171 tcache->prof_accumbytes = 0;
172 }
173
174 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
175 if (config_stats && bin_arena == arena && !merged_stats) {
176 merged_stats = true;
177 bin->stats.nflushes++;
178 bin->stats.nrequests += tbin->tstats.nrequests;
179 tbin->tstats.nrequests = 0;
180 }
181 unsigned ndeferred = 0;
182 for (unsigned i = 0; i < nflush; i++) {
183 void *ptr = *(tbin->avail - 1 - i);
184 extent = item_extent[i];
185 assert(ptr != NULL && extent != NULL);
186
187 if (extent_arena_ind_get(extent) == bin_arena_ind
188 && extent_binshard_get(extent) == binshard) {
189 arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
190 bin_arena, bin, binind, extent, ptr);
191 } else {
192 /*
193 * This object was allocated via a different
194 * arena bin than the one that is currently
195 * locked. Stash the object, so that it can be
196 * handled in a future pass.
197 */
198 *(tbin->avail - 1 - ndeferred) = ptr;
199 item_extent[ndeferred] = extent;
200 ndeferred++;
201 }
202 }
203 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
204 arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred);
205 nflush = ndeferred;
206 }
207 if (config_stats && !merged_stats) {
208 /*
209 * The flush loop didn't happen to flush to this thread's
210 * arena, so the stats didn't get merged. Manually do so now.
211 */
212 unsigned binshard;
213 bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind,
214 &binshard);
215 bin->stats.nflushes++;
216 bin->stats.nrequests += tbin->tstats.nrequests;
217 tbin->tstats.nrequests = 0;
218 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
219 }
220
221 memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
222 sizeof(void *));
223 tbin->ncached = rem;
224 if (tbin->ncached < tbin->low_water) {
225 tbin->low_water = tbin->ncached;
226 }
227}
228
229void
230tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
231 unsigned rem, tcache_t *tcache) {
232 bool merged_stats = false;
233
234 assert(binind < nhbins);
235 assert((cache_bin_sz_t)rem <= tbin->ncached);
236
237 arena_t *tcache_arena = tcache->arena;
238 assert(tcache_arena != NULL);
239 unsigned nflush = tbin->ncached - rem;
240 VARIABLE_ARRAY(extent_t *, item_extent, nflush);
241
242#ifndef JEMALLOC_EXTRA_SIZE_CHECK
243 /* Look up extent once per item. */
244 for (unsigned i = 0 ; i < nflush; i++) {
245 item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
246 }
247#else
248 tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush,
249 item_extent);
250#endif
251 while (nflush > 0) {
252 /* Lock the arena associated with the first object. */
253 extent_t *extent = item_extent[0];
254 unsigned locked_arena_ind = extent_arena_ind_get(extent);
255 arena_t *locked_arena = arena_get(tsd_tsdn(tsd),
256 locked_arena_ind, false);
257 bool idump;
258
259 if (config_prof) {
260 idump = false;
261 }
262
263 bool lock_large = !arena_is_auto(locked_arena);
264 if (lock_large) {
265 malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
266 }
267 for (unsigned i = 0; i < nflush; i++) {
268 void *ptr = *(tbin->avail - 1 - i);
269 assert(ptr != NULL);
270 extent = item_extent[i];
271 if (extent_arena_ind_get(extent) == locked_arena_ind) {
272 large_dalloc_prep_junked_locked(tsd_tsdn(tsd),
273 extent);
274 }
275 }
276 if ((config_prof || config_stats) &&
277 (locked_arena == tcache_arena)) {
278 if (config_prof) {
279 idump = arena_prof_accum(tsd_tsdn(tsd),
280 tcache_arena, tcache->prof_accumbytes);
281 tcache->prof_accumbytes = 0;
282 }
283 if (config_stats) {
284 merged_stats = true;
285 arena_stats_large_flush_nrequests_add(
286 tsd_tsdn(tsd), &tcache_arena->stats, binind,
287 tbin->tstats.nrequests);
288 tbin->tstats.nrequests = 0;
289 }
290 }
291 if (lock_large) {
292 malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
293 }
294
295 unsigned ndeferred = 0;
296 for (unsigned i = 0; i < nflush; i++) {
297 void *ptr = *(tbin->avail - 1 - i);
298 extent = item_extent[i];
299 assert(ptr != NULL && extent != NULL);
300
301 if (extent_arena_ind_get(extent) == locked_arena_ind) {
302 large_dalloc_finish(tsd_tsdn(tsd), extent);
303 } else {
304 /*
305 * This object was allocated via a different
306 * arena than the one that is currently locked.
307 * Stash the object, so that it can be handled
308 * in a future pass.
309 */
310 *(tbin->avail - 1 - ndeferred) = ptr;
311 item_extent[ndeferred] = extent;
312 ndeferred++;
313 }
314 }
315 if (config_prof && idump) {
316 prof_idump(tsd_tsdn(tsd));
317 }
318 arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush -
319 ndeferred);
320 nflush = ndeferred;
321 }
322 if (config_stats && !merged_stats) {
323 /*
324 * The flush loop didn't happen to flush to this thread's
325 * arena, so the stats didn't get merged. Manually do so now.
326 */
327 arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd),
328 &tcache_arena->stats, binind, tbin->tstats.nrequests);
329 tbin->tstats.nrequests = 0;
330 }
331
332 memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
333 sizeof(void *));
334 tbin->ncached = rem;
335 if (tbin->ncached < tbin->low_water) {
336 tbin->low_water = tbin->ncached;
337 }
338}
339
340void
341tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
342 assert(tcache->arena == NULL);
343 tcache->arena = arena;
344
345 if (config_stats) {
346 /* Link into list of extant tcaches. */
347 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
348
349 ql_elm_new(tcache, link);
350 ql_tail_insert(&arena->tcache_ql, tcache, link);
351 cache_bin_array_descriptor_init(
352 &tcache->cache_bin_array_descriptor, tcache->bins_small,
353 tcache->bins_large);
354 ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
355 &tcache->cache_bin_array_descriptor, link);
356
357 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
358 }
359}
360
361static void
362tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
363 arena_t *arena = tcache->arena;
364 assert(arena != NULL);
365 if (config_stats) {
366 /* Unlink from list of extant tcaches. */
367 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
368 if (config_debug) {
369 bool in_ql = false;
370 tcache_t *iter;
371 ql_foreach(iter, &arena->tcache_ql, link) {
372 if (iter == tcache) {
373 in_ql = true;
374 break;
375 }
376 }
377 assert(in_ql);
378 }
379 ql_remove(&arena->tcache_ql, tcache, link);
380 ql_remove(&arena->cache_bin_array_descriptor_ql,
381 &tcache->cache_bin_array_descriptor, link);
382 tcache_stats_merge(tsdn, tcache, arena);
383 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
384 }
385 tcache->arena = NULL;
386}
387
388void
389tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
390 tcache_arena_dissociate(tsdn, tcache);
391 tcache_arena_associate(tsdn, tcache, arena);
392}
393
394bool
395tsd_tcache_enabled_data_init(tsd_t *tsd) {
396 /* Called upon tsd initialization. */
397 tsd_tcache_enabled_set(tsd, opt_tcache);
398 tsd_slow_update(tsd);
399
400 if (opt_tcache) {
401 /* Trigger tcache init. */
402 tsd_tcache_data_init(tsd);
403 }
404
405 return false;
406}
407
408/* Initialize auto tcache (embedded in TSD). */
409static void
410tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
411 memset(&tcache->link, 0, sizeof(ql_elm(tcache_t)));
412 tcache->prof_accumbytes = 0;
413 tcache->next_gc_bin = 0;
414 tcache->arena = NULL;
415
416 ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
417
418 size_t stack_offset = 0;
419 assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
420 memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
421 memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));
422 unsigned i = 0;
423 for (; i < SC_NBINS; i++) {
424 tcache->lg_fill_div[i] = 1;
425 stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
426 /*
427 * avail points past the available space. Allocations will
428 * access the slots toward higher addresses (for the benefit of
429 * prefetch).
430 */
431 tcache_small_bin_get(tcache, i)->avail =
432 (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset);
433 }
434 for (; i < nhbins; i++) {
435 stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
436 tcache_large_bin_get(tcache, i)->avail =
437 (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset);
438 }
439 assert(stack_offset == stack_nelms * sizeof(void *));
440}
441
442/* Initialize auto tcache (embedded in TSD). */
443bool
444tsd_tcache_data_init(tsd_t *tsd) {
445 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
446 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
447 size_t size = stack_nelms * sizeof(void *);
448 /* Avoid false cacheline sharing. */
449 size = sz_sa2u(size, CACHELINE);
450
451 void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true,
452 NULL, true, arena_get(TSDN_NULL, 0, true));
453 if (avail_array == NULL) {
454 return true;
455 }
456
457 tcache_init(tsd, tcache, avail_array);
458 /*
459 * Initialization is a bit tricky here. After malloc init is done, all
460 * threads can rely on arena_choose and associate tcache accordingly.
461 * However, the thread that does actual malloc bootstrapping relies on
462 * functional tsd, and it can only rely on a0. In that case, we
463 * associate its tcache to a0 temporarily, and later on
464 * arena_choose_hard() will re-associate properly.
465 */
466 tcache->arena = NULL;
467 arena_t *arena;
468 if (!malloc_initialized()) {
469 /* If in initialization, assign to a0. */
470 arena = arena_get(tsd_tsdn(tsd), 0, false);
471 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
472 } else {
473 arena = arena_choose(tsd, NULL);
474 /* This may happen if thread.tcache.enabled is used. */
475 if (tcache->arena == NULL) {
476 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
477 }
478 }
479 assert(arena == tcache->arena);
480
481 return false;
482}
483
484/* Created manual tcache for tcache.create mallctl. */
485tcache_t *
486tcache_create_explicit(tsd_t *tsd) {
487 tcache_t *tcache;
488 size_t size, stack_offset;
489
490 size = sizeof(tcache_t);
491 /* Naturally align the pointer stacks. */
492 size = PTR_CEILING(size);
493 stack_offset = size;
494 size += stack_nelms * sizeof(void *);
495 /* Avoid false cacheline sharing. */
496 size = sz_sa2u(size, CACHELINE);
497
498 tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true,
499 arena_get(TSDN_NULL, 0, true));
500 if (tcache == NULL) {
501 return NULL;
502 }
503
504 tcache_init(tsd, tcache,
505 (void *)((uintptr_t)tcache + (uintptr_t)stack_offset));
506 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL));
507
508 return tcache;
509}
510
511static void
512tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
513 assert(tcache->arena != NULL);
514
515 for (unsigned i = 0; i < SC_NBINS; i++) {
516 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
517 tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
518
519 if (config_stats) {
520 assert(tbin->tstats.nrequests == 0);
521 }
522 }
523 for (unsigned i = SC_NBINS; i < nhbins; i++) {
524 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
525 tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
526
527 if (config_stats) {
528 assert(tbin->tstats.nrequests == 0);
529 }
530 }
531
532 if (config_prof && tcache->prof_accumbytes > 0 &&
533 arena_prof_accum(tsd_tsdn(tsd), tcache->arena,
534 tcache->prof_accumbytes)) {
535 prof_idump(tsd_tsdn(tsd));
536 }
537}
538
539void
540tcache_flush(tsd_t *tsd) {
541 assert(tcache_available(tsd));
542 tcache_flush_cache(tsd, tsd_tcachep_get(tsd));
543}
544
545static void
546tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
547 tcache_flush_cache(tsd, tcache);
548 arena_t *arena = tcache->arena;
549 tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
550
551 if (tsd_tcache) {
552 /* Release the avail array for the TSD embedded auto tcache. */
553 void *avail_array =
554 (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail -
555 (uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *));
556 idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true);
557 } else {
558 /* Release both the tcache struct and avail array. */
559 idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true);
560 }
561
562 /*
563 * The deallocation and tcache flush above may not trigger decay since
564 * we are on the tcache shutdown path (potentially with non-nominal
565 * tsd). Manually trigger decay to avoid pathological cases. Also
566 * include arena 0 because the tcache array is allocated from it.
567 */
568 arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false),
569 false, false);
570
571 if (arena_nthreads_get(arena, false) == 0 &&
572 !background_thread_enabled()) {
573 /* Force purging when no threads assigned to the arena anymore. */
574 arena_decay(tsd_tsdn(tsd), arena, false, true);
575 } else {
576 arena_decay(tsd_tsdn(tsd), arena, false, false);
577 }
578}
579
580/* For auto tcache (embedded in TSD) only. */
581void
582tcache_cleanup(tsd_t *tsd) {
583 tcache_t *tcache = tsd_tcachep_get(tsd);
584 if (!tcache_available(tsd)) {
585 assert(tsd_tcache_enabled_get(tsd) == false);
586 if (config_debug) {
587 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
588 }
589 return;
590 }
591 assert(tsd_tcache_enabled_get(tsd));
592 assert(tcache_small_bin_get(tcache, 0)->avail != NULL);
593
594 tcache_destroy(tsd, tcache, true);
595 if (config_debug) {
596 tcache_small_bin_get(tcache, 0)->avail = NULL;
597 }
598}
599
600void
601tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
602 unsigned i;
603
604 cassert(config_stats);
605
606 /* Merge and reset tcache stats. */
607 for (i = 0; i < SC_NBINS; i++) {
608 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
609 unsigned binshard;
610 bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard);
611 bin->stats.nrequests += tbin->tstats.nrequests;
612 malloc_mutex_unlock(tsdn, &bin->lock);
613 tbin->tstats.nrequests = 0;
614 }
615
616 for (; i < nhbins; i++) {
617 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
618 arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i,
619 tbin->tstats.nrequests);
620 tbin->tstats.nrequests = 0;
621 }
622}
623
624static bool
625tcaches_create_prep(tsd_t *tsd) {
626 bool err;
627
628 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
629
630 if (tcaches == NULL) {
631 tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
632 * (MALLOCX_TCACHE_MAX+1), CACHELINE);
633 if (tcaches == NULL) {
634 err = true;
635 goto label_return;
636 }
637 }
638
639 if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
640 err = true;
641 goto label_return;
642 }
643
644 err = false;
645label_return:
646 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
647 return err;
648}
649
650bool
651tcaches_create(tsd_t *tsd, unsigned *r_ind) {
652 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
653
654 bool err;
655
656 if (tcaches_create_prep(tsd)) {
657 err = true;
658 goto label_return;
659 }
660
661 tcache_t *tcache = tcache_create_explicit(tsd);
662 if (tcache == NULL) {
663 err = true;
664 goto label_return;
665 }
666
667 tcaches_t *elm;
668 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
669 if (tcaches_avail != NULL) {
670 elm = tcaches_avail;
671 tcaches_avail = tcaches_avail->next;
672 elm->tcache = tcache;
673 *r_ind = (unsigned)(elm - tcaches);
674 } else {
675 elm = &tcaches[tcaches_past];
676 elm->tcache = tcache;
677 *r_ind = tcaches_past;
678 tcaches_past++;
679 }
680 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
681
682 err = false;
683label_return:
684 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
685 return err;
686}
687
688static tcache_t *
689tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) {
690 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
691
692 if (elm->tcache == NULL) {
693 return NULL;
694 }
695 tcache_t *tcache = elm->tcache;
696 if (allow_reinit) {
697 elm->tcache = TCACHES_ELM_NEED_REINIT;
698 } else {
699 elm->tcache = NULL;
700 }
701
702 if (tcache == TCACHES_ELM_NEED_REINIT) {
703 return NULL;
704 }
705 return tcache;
706}
707
708void
709tcaches_flush(tsd_t *tsd, unsigned ind) {
710 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
711 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true);
712 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
713 if (tcache != NULL) {
714 /* Destroy the tcache; recreate in tcaches_get() if needed. */
715 tcache_destroy(tsd, tcache, false);
716 }
717}
718
719void
720tcaches_destroy(tsd_t *tsd, unsigned ind) {
721 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
722 tcaches_t *elm = &tcaches[ind];
723 tcache_t *tcache = tcaches_elm_remove(tsd, elm, false);
724 elm->next = tcaches_avail;
725 tcaches_avail = elm;
726 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
727 if (tcache != NULL) {
728 tcache_destroy(tsd, tcache, false);
729 }
730}
731
732bool
733tcache_boot(tsdn_t *tsdn) {
734 /* If necessary, clamp opt_lg_tcache_max. */
735 if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
736 SC_SMALL_MAXCLASS) {
737 tcache_maxclass = SC_SMALL_MAXCLASS;
738 } else {
739 tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
740 }
741
742 if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
743 malloc_mutex_rank_exclusive)) {
744 return true;
745 }
746
747 nhbins = sz_size2index(tcache_maxclass) + 1;
748
749 /* Initialize tcache_bin_info. */
750 tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
751 * sizeof(cache_bin_info_t), CACHELINE);
752 if (tcache_bin_info == NULL) {
753 return true;
754 }
755 stack_nelms = 0;
756 unsigned i;
757 for (i = 0; i < SC_NBINS; i++) {
758 if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
759 tcache_bin_info[i].ncached_max =
760 TCACHE_NSLOTS_SMALL_MIN;
761 } else if ((bin_infos[i].nregs << 1) <=
762 TCACHE_NSLOTS_SMALL_MAX) {
763 tcache_bin_info[i].ncached_max =
764 (bin_infos[i].nregs << 1);
765 } else {
766 tcache_bin_info[i].ncached_max =
767 TCACHE_NSLOTS_SMALL_MAX;
768 }
769 stack_nelms += tcache_bin_info[i].ncached_max;
770 }
771 for (; i < nhbins; i++) {
772 tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
773 stack_nelms += tcache_bin_info[i].ncached_max;
774 }
775
776 return false;
777}
778
779void
780tcache_prefork(tsdn_t *tsdn) {
781 if (!config_prof && opt_tcache) {
782 malloc_mutex_prefork(tsdn, &tcaches_mtx);
783 }
784}
785
786void
787tcache_postfork_parent(tsdn_t *tsdn) {
788 if (!config_prof && opt_tcache) {
789 malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
790 }
791}
792
793void
794tcache_postfork_child(tsdn_t *tsdn) {
795 if (!config_prof && opt_tcache) {
796 malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
797 }
798}
799