1 | #define JEMALLOC_TCACHE_C_ |
2 | #include "jemalloc/internal/jemalloc_preamble.h" |
3 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
4 | |
5 | #include "jemalloc/internal/assert.h" |
6 | #include "jemalloc/internal/mutex.h" |
7 | #include "jemalloc/internal/safety_check.h" |
8 | #include "jemalloc/internal/sc.h" |
9 | |
10 | /******************************************************************************/ |
11 | /* Data. */ |
12 | |
13 | bool opt_tcache = true; |
14 | ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; |
15 | |
16 | cache_bin_info_t *tcache_bin_info; |
17 | static unsigned stack_nelms; /* Total stack elms per tcache. */ |
18 | |
19 | unsigned nhbins; |
20 | size_t tcache_maxclass; |
21 | |
22 | tcaches_t *tcaches; |
23 | |
24 | /* Index of first element within tcaches that has never been used. */ |
25 | static unsigned tcaches_past; |
26 | |
27 | /* Head of singly linked list tracking available tcaches elements. */ |
28 | static tcaches_t *tcaches_avail; |
29 | |
30 | /* Protects tcaches{,_past,_avail}. */ |
31 | static malloc_mutex_t tcaches_mtx; |
32 | |
33 | /******************************************************************************/ |
34 | |
35 | size_t |
36 | tcache_salloc(tsdn_t *tsdn, const void *ptr) { |
37 | return arena_salloc(tsdn, ptr); |
38 | } |
39 | |
40 | void |
41 | tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { |
42 | szind_t binind = tcache->next_gc_bin; |
43 | |
44 | cache_bin_t *tbin; |
45 | if (binind < SC_NBINS) { |
46 | tbin = tcache_small_bin_get(tcache, binind); |
47 | } else { |
48 | tbin = tcache_large_bin_get(tcache, binind); |
49 | } |
50 | if (tbin->low_water > 0) { |
51 | /* |
52 | * Flush (ceiling) 3/4 of the objects below the low water mark. |
53 | */ |
54 | if (binind < SC_NBINS) { |
55 | tcache_bin_flush_small(tsd, tcache, tbin, binind, |
56 | tbin->ncached - tbin->low_water + (tbin->low_water |
57 | >> 2)); |
58 | /* |
59 | * Reduce fill count by 2X. Limit lg_fill_div such that |
60 | * the fill count is always at least 1. |
61 | */ |
62 | cache_bin_info_t *tbin_info = &tcache_bin_info[binind]; |
63 | if ((tbin_info->ncached_max >> |
64 | (tcache->lg_fill_div[binind] + 1)) >= 1) { |
65 | tcache->lg_fill_div[binind]++; |
66 | } |
67 | } else { |
68 | tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached |
69 | - tbin->low_water + (tbin->low_water >> 2), tcache); |
70 | } |
71 | } else if (tbin->low_water < 0) { |
72 | /* |
73 | * Increase fill count by 2X for small bins. Make sure |
74 | * lg_fill_div stays greater than 0. |
75 | */ |
76 | if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) { |
77 | tcache->lg_fill_div[binind]--; |
78 | } |
79 | } |
80 | tbin->low_water = tbin->ncached; |
81 | |
82 | tcache->next_gc_bin++; |
83 | if (tcache->next_gc_bin == nhbins) { |
84 | tcache->next_gc_bin = 0; |
85 | } |
86 | } |
87 | |
88 | void * |
89 | tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, |
90 | cache_bin_t *tbin, szind_t binind, bool *tcache_success) { |
91 | void *ret; |
92 | |
93 | assert(tcache->arena != NULL); |
94 | arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind, |
95 | config_prof ? tcache->prof_accumbytes : 0); |
96 | if (config_prof) { |
97 | tcache->prof_accumbytes = 0; |
98 | } |
99 | ret = cache_bin_alloc_easy(tbin, tcache_success); |
100 | |
101 | return ret; |
102 | } |
103 | |
104 | /* Enabled with --enable-extra-size-check. */ |
105 | static void |
106 | tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind, |
107 | size_t nflush, extent_t **extents){ |
108 | rtree_ctx_t rtree_ctx_fallback; |
109 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
110 | |
111 | /* |
112 | * Verify that the items in the tcache all have the correct size; this |
113 | * is useful for catching sized deallocation bugs, also to fail early |
114 | * instead of corrupting metadata. Since this can be turned on for opt |
115 | * builds, avoid the branch in the loop. |
116 | */ |
117 | szind_t szind; |
118 | size_t sz_sum = binind * nflush; |
119 | for (unsigned i = 0 ; i < nflush; i++) { |
120 | rtree_extent_szind_read(tsdn, &extents_rtree, |
121 | rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true, |
122 | &extents[i], &szind); |
123 | sz_sum -= szind; |
124 | } |
125 | if (sz_sum != 0) { |
126 | safety_check_fail("<jemalloc>: size mismatch in thread cache " |
127 | "detected, likely caused by sized deallocation bugs by " |
128 | "application. Abort.\n" ); |
129 | abort(); |
130 | } |
131 | } |
132 | |
133 | void |
134 | tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, |
135 | szind_t binind, unsigned rem) { |
136 | bool merged_stats = false; |
137 | |
138 | assert(binind < SC_NBINS); |
139 | assert((cache_bin_sz_t)rem <= tbin->ncached); |
140 | |
141 | arena_t *arena = tcache->arena; |
142 | assert(arena != NULL); |
143 | unsigned nflush = tbin->ncached - rem; |
144 | VARIABLE_ARRAY(extent_t *, item_extent, nflush); |
145 | |
146 | /* Look up extent once per item. */ |
147 | if (config_opt_safety_checks) { |
148 | tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, |
149 | nflush, item_extent); |
150 | } else { |
151 | for (unsigned i = 0 ; i < nflush; i++) { |
152 | item_extent[i] = iealloc(tsd_tsdn(tsd), |
153 | *(tbin->avail - 1 - i)); |
154 | } |
155 | } |
156 | while (nflush > 0) { |
157 | /* Lock the arena bin associated with the first object. */ |
158 | extent_t *extent = item_extent[0]; |
159 | unsigned bin_arena_ind = extent_arena_ind_get(extent); |
160 | arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, |
161 | false); |
162 | unsigned binshard = extent_binshard_get(extent); |
163 | assert(binshard < bin_infos[binind].n_shards); |
164 | bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard]; |
165 | |
166 | if (config_prof && bin_arena == arena) { |
167 | if (arena_prof_accum(tsd_tsdn(tsd), arena, |
168 | tcache->prof_accumbytes)) { |
169 | prof_idump(tsd_tsdn(tsd)); |
170 | } |
171 | tcache->prof_accumbytes = 0; |
172 | } |
173 | |
174 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
175 | if (config_stats && bin_arena == arena && !merged_stats) { |
176 | merged_stats = true; |
177 | bin->stats.nflushes++; |
178 | bin->stats.nrequests += tbin->tstats.nrequests; |
179 | tbin->tstats.nrequests = 0; |
180 | } |
181 | unsigned ndeferred = 0; |
182 | for (unsigned i = 0; i < nflush; i++) { |
183 | void *ptr = *(tbin->avail - 1 - i); |
184 | extent = item_extent[i]; |
185 | assert(ptr != NULL && extent != NULL); |
186 | |
187 | if (extent_arena_ind_get(extent) == bin_arena_ind |
188 | && extent_binshard_get(extent) == binshard) { |
189 | arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), |
190 | bin_arena, bin, binind, extent, ptr); |
191 | } else { |
192 | /* |
193 | * This object was allocated via a different |
194 | * arena bin than the one that is currently |
195 | * locked. Stash the object, so that it can be |
196 | * handled in a future pass. |
197 | */ |
198 | *(tbin->avail - 1 - ndeferred) = ptr; |
199 | item_extent[ndeferred] = extent; |
200 | ndeferred++; |
201 | } |
202 | } |
203 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
204 | arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred); |
205 | nflush = ndeferred; |
206 | } |
207 | if (config_stats && !merged_stats) { |
208 | /* |
209 | * The flush loop didn't happen to flush to this thread's |
210 | * arena, so the stats didn't get merged. Manually do so now. |
211 | */ |
212 | unsigned binshard; |
213 | bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, |
214 | &binshard); |
215 | bin->stats.nflushes++; |
216 | bin->stats.nrequests += tbin->tstats.nrequests; |
217 | tbin->tstats.nrequests = 0; |
218 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
219 | } |
220 | |
221 | memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * |
222 | sizeof(void *)); |
223 | tbin->ncached = rem; |
224 | if (tbin->ncached < tbin->low_water) { |
225 | tbin->low_water = tbin->ncached; |
226 | } |
227 | } |
228 | |
229 | void |
230 | tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, |
231 | unsigned rem, tcache_t *tcache) { |
232 | bool merged_stats = false; |
233 | |
234 | assert(binind < nhbins); |
235 | assert((cache_bin_sz_t)rem <= tbin->ncached); |
236 | |
237 | arena_t *tcache_arena = tcache->arena; |
238 | assert(tcache_arena != NULL); |
239 | unsigned nflush = tbin->ncached - rem; |
240 | VARIABLE_ARRAY(extent_t *, item_extent, nflush); |
241 | |
242 | #ifndef JEMALLOC_EXTRA_SIZE_CHECK |
243 | /* Look up extent once per item. */ |
244 | for (unsigned i = 0 ; i < nflush; i++) { |
245 | item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); |
246 | } |
247 | #else |
248 | tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, |
249 | item_extent); |
250 | #endif |
251 | while (nflush > 0) { |
252 | /* Lock the arena associated with the first object. */ |
253 | extent_t *extent = item_extent[0]; |
254 | unsigned locked_arena_ind = extent_arena_ind_get(extent); |
255 | arena_t *locked_arena = arena_get(tsd_tsdn(tsd), |
256 | locked_arena_ind, false); |
257 | bool idump; |
258 | |
259 | if (config_prof) { |
260 | idump = false; |
261 | } |
262 | |
263 | bool lock_large = !arena_is_auto(locked_arena); |
264 | if (lock_large) { |
265 | malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); |
266 | } |
267 | for (unsigned i = 0; i < nflush; i++) { |
268 | void *ptr = *(tbin->avail - 1 - i); |
269 | assert(ptr != NULL); |
270 | extent = item_extent[i]; |
271 | if (extent_arena_ind_get(extent) == locked_arena_ind) { |
272 | large_dalloc_prep_junked_locked(tsd_tsdn(tsd), |
273 | extent); |
274 | } |
275 | } |
276 | if ((config_prof || config_stats) && |
277 | (locked_arena == tcache_arena)) { |
278 | if (config_prof) { |
279 | idump = arena_prof_accum(tsd_tsdn(tsd), |
280 | tcache_arena, tcache->prof_accumbytes); |
281 | tcache->prof_accumbytes = 0; |
282 | } |
283 | if (config_stats) { |
284 | merged_stats = true; |
285 | arena_stats_large_flush_nrequests_add( |
286 | tsd_tsdn(tsd), &tcache_arena->stats, binind, |
287 | tbin->tstats.nrequests); |
288 | tbin->tstats.nrequests = 0; |
289 | } |
290 | } |
291 | if (lock_large) { |
292 | malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); |
293 | } |
294 | |
295 | unsigned ndeferred = 0; |
296 | for (unsigned i = 0; i < nflush; i++) { |
297 | void *ptr = *(tbin->avail - 1 - i); |
298 | extent = item_extent[i]; |
299 | assert(ptr != NULL && extent != NULL); |
300 | |
301 | if (extent_arena_ind_get(extent) == locked_arena_ind) { |
302 | large_dalloc_finish(tsd_tsdn(tsd), extent); |
303 | } else { |
304 | /* |
305 | * This object was allocated via a different |
306 | * arena than the one that is currently locked. |
307 | * Stash the object, so that it can be handled |
308 | * in a future pass. |
309 | */ |
310 | *(tbin->avail - 1 - ndeferred) = ptr; |
311 | item_extent[ndeferred] = extent; |
312 | ndeferred++; |
313 | } |
314 | } |
315 | if (config_prof && idump) { |
316 | prof_idump(tsd_tsdn(tsd)); |
317 | } |
318 | arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush - |
319 | ndeferred); |
320 | nflush = ndeferred; |
321 | } |
322 | if (config_stats && !merged_stats) { |
323 | /* |
324 | * The flush loop didn't happen to flush to this thread's |
325 | * arena, so the stats didn't get merged. Manually do so now. |
326 | */ |
327 | arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd), |
328 | &tcache_arena->stats, binind, tbin->tstats.nrequests); |
329 | tbin->tstats.nrequests = 0; |
330 | } |
331 | |
332 | memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * |
333 | sizeof(void *)); |
334 | tbin->ncached = rem; |
335 | if (tbin->ncached < tbin->low_water) { |
336 | tbin->low_water = tbin->ncached; |
337 | } |
338 | } |
339 | |
340 | void |
341 | tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
342 | assert(tcache->arena == NULL); |
343 | tcache->arena = arena; |
344 | |
345 | if (config_stats) { |
346 | /* Link into list of extant tcaches. */ |
347 | malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); |
348 | |
349 | ql_elm_new(tcache, link); |
350 | ql_tail_insert(&arena->tcache_ql, tcache, link); |
351 | cache_bin_array_descriptor_init( |
352 | &tcache->cache_bin_array_descriptor, tcache->bins_small, |
353 | tcache->bins_large); |
354 | ql_tail_insert(&arena->cache_bin_array_descriptor_ql, |
355 | &tcache->cache_bin_array_descriptor, link); |
356 | |
357 | malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); |
358 | } |
359 | } |
360 | |
361 | static void |
362 | tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { |
363 | arena_t *arena = tcache->arena; |
364 | assert(arena != NULL); |
365 | if (config_stats) { |
366 | /* Unlink from list of extant tcaches. */ |
367 | malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); |
368 | if (config_debug) { |
369 | bool in_ql = false; |
370 | tcache_t *iter; |
371 | ql_foreach(iter, &arena->tcache_ql, link) { |
372 | if (iter == tcache) { |
373 | in_ql = true; |
374 | break; |
375 | } |
376 | } |
377 | assert(in_ql); |
378 | } |
379 | ql_remove(&arena->tcache_ql, tcache, link); |
380 | ql_remove(&arena->cache_bin_array_descriptor_ql, |
381 | &tcache->cache_bin_array_descriptor, link); |
382 | tcache_stats_merge(tsdn, tcache, arena); |
383 | malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); |
384 | } |
385 | tcache->arena = NULL; |
386 | } |
387 | |
388 | void |
389 | tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
390 | tcache_arena_dissociate(tsdn, tcache); |
391 | tcache_arena_associate(tsdn, tcache, arena); |
392 | } |
393 | |
394 | bool |
395 | tsd_tcache_enabled_data_init(tsd_t *tsd) { |
396 | /* Called upon tsd initialization. */ |
397 | tsd_tcache_enabled_set(tsd, opt_tcache); |
398 | tsd_slow_update(tsd); |
399 | |
400 | if (opt_tcache) { |
401 | /* Trigger tcache init. */ |
402 | tsd_tcache_data_init(tsd); |
403 | } |
404 | |
405 | return false; |
406 | } |
407 | |
408 | /* Initialize auto tcache (embedded in TSD). */ |
409 | static void |
410 | tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { |
411 | memset(&tcache->link, 0, sizeof(ql_elm(tcache_t))); |
412 | tcache->prof_accumbytes = 0; |
413 | tcache->next_gc_bin = 0; |
414 | tcache->arena = NULL; |
415 | |
416 | ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR); |
417 | |
418 | size_t stack_offset = 0; |
419 | assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); |
420 | memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); |
421 | memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); |
422 | unsigned i = 0; |
423 | for (; i < SC_NBINS; i++) { |
424 | tcache->lg_fill_div[i] = 1; |
425 | stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); |
426 | /* |
427 | * avail points past the available space. Allocations will |
428 | * access the slots toward higher addresses (for the benefit of |
429 | * prefetch). |
430 | */ |
431 | tcache_small_bin_get(tcache, i)->avail = |
432 | (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset); |
433 | } |
434 | for (; i < nhbins; i++) { |
435 | stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); |
436 | tcache_large_bin_get(tcache, i)->avail = |
437 | (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset); |
438 | } |
439 | assert(stack_offset == stack_nelms * sizeof(void *)); |
440 | } |
441 | |
442 | /* Initialize auto tcache (embedded in TSD). */ |
443 | bool |
444 | tsd_tcache_data_init(tsd_t *tsd) { |
445 | tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); |
446 | assert(tcache_small_bin_get(tcache, 0)->avail == NULL); |
447 | size_t size = stack_nelms * sizeof(void *); |
448 | /* Avoid false cacheline sharing. */ |
449 | size = sz_sa2u(size, CACHELINE); |
450 | |
451 | void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, |
452 | NULL, true, arena_get(TSDN_NULL, 0, true)); |
453 | if (avail_array == NULL) { |
454 | return true; |
455 | } |
456 | |
457 | tcache_init(tsd, tcache, avail_array); |
458 | /* |
459 | * Initialization is a bit tricky here. After malloc init is done, all |
460 | * threads can rely on arena_choose and associate tcache accordingly. |
461 | * However, the thread that does actual malloc bootstrapping relies on |
462 | * functional tsd, and it can only rely on a0. In that case, we |
463 | * associate its tcache to a0 temporarily, and later on |
464 | * arena_choose_hard() will re-associate properly. |
465 | */ |
466 | tcache->arena = NULL; |
467 | arena_t *arena; |
468 | if (!malloc_initialized()) { |
469 | /* If in initialization, assign to a0. */ |
470 | arena = arena_get(tsd_tsdn(tsd), 0, false); |
471 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); |
472 | } else { |
473 | arena = arena_choose(tsd, NULL); |
474 | /* This may happen if thread.tcache.enabled is used. */ |
475 | if (tcache->arena == NULL) { |
476 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); |
477 | } |
478 | } |
479 | assert(arena == tcache->arena); |
480 | |
481 | return false; |
482 | } |
483 | |
484 | /* Created manual tcache for tcache.create mallctl. */ |
485 | tcache_t * |
486 | tcache_create_explicit(tsd_t *tsd) { |
487 | tcache_t *tcache; |
488 | size_t size, stack_offset; |
489 | |
490 | size = sizeof(tcache_t); |
491 | /* Naturally align the pointer stacks. */ |
492 | size = PTR_CEILING(size); |
493 | stack_offset = size; |
494 | size += stack_nelms * sizeof(void *); |
495 | /* Avoid false cacheline sharing. */ |
496 | size = sz_sa2u(size, CACHELINE); |
497 | |
498 | tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true, |
499 | arena_get(TSDN_NULL, 0, true)); |
500 | if (tcache == NULL) { |
501 | return NULL; |
502 | } |
503 | |
504 | tcache_init(tsd, tcache, |
505 | (void *)((uintptr_t)tcache + (uintptr_t)stack_offset)); |
506 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL)); |
507 | |
508 | return tcache; |
509 | } |
510 | |
511 | static void |
512 | tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { |
513 | assert(tcache->arena != NULL); |
514 | |
515 | for (unsigned i = 0; i < SC_NBINS; i++) { |
516 | cache_bin_t *tbin = tcache_small_bin_get(tcache, i); |
517 | tcache_bin_flush_small(tsd, tcache, tbin, i, 0); |
518 | |
519 | if (config_stats) { |
520 | assert(tbin->tstats.nrequests == 0); |
521 | } |
522 | } |
523 | for (unsigned i = SC_NBINS; i < nhbins; i++) { |
524 | cache_bin_t *tbin = tcache_large_bin_get(tcache, i); |
525 | tcache_bin_flush_large(tsd, tbin, i, 0, tcache); |
526 | |
527 | if (config_stats) { |
528 | assert(tbin->tstats.nrequests == 0); |
529 | } |
530 | } |
531 | |
532 | if (config_prof && tcache->prof_accumbytes > 0 && |
533 | arena_prof_accum(tsd_tsdn(tsd), tcache->arena, |
534 | tcache->prof_accumbytes)) { |
535 | prof_idump(tsd_tsdn(tsd)); |
536 | } |
537 | } |
538 | |
539 | void |
540 | tcache_flush(tsd_t *tsd) { |
541 | assert(tcache_available(tsd)); |
542 | tcache_flush_cache(tsd, tsd_tcachep_get(tsd)); |
543 | } |
544 | |
545 | static void |
546 | tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { |
547 | tcache_flush_cache(tsd, tcache); |
548 | arena_t *arena = tcache->arena; |
549 | tcache_arena_dissociate(tsd_tsdn(tsd), tcache); |
550 | |
551 | if (tsd_tcache) { |
552 | /* Release the avail array for the TSD embedded auto tcache. */ |
553 | void *avail_array = |
554 | (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail - |
555 | (uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *)); |
556 | idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true); |
557 | } else { |
558 | /* Release both the tcache struct and avail array. */ |
559 | idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); |
560 | } |
561 | |
562 | /* |
563 | * The deallocation and tcache flush above may not trigger decay since |
564 | * we are on the tcache shutdown path (potentially with non-nominal |
565 | * tsd). Manually trigger decay to avoid pathological cases. Also |
566 | * include arena 0 because the tcache array is allocated from it. |
567 | */ |
568 | arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), |
569 | false, false); |
570 | |
571 | if (arena_nthreads_get(arena, false) == 0 && |
572 | !background_thread_enabled()) { |
573 | /* Force purging when no threads assigned to the arena anymore. */ |
574 | arena_decay(tsd_tsdn(tsd), arena, false, true); |
575 | } else { |
576 | arena_decay(tsd_tsdn(tsd), arena, false, false); |
577 | } |
578 | } |
579 | |
580 | /* For auto tcache (embedded in TSD) only. */ |
581 | void |
582 | tcache_cleanup(tsd_t *tsd) { |
583 | tcache_t *tcache = tsd_tcachep_get(tsd); |
584 | if (!tcache_available(tsd)) { |
585 | assert(tsd_tcache_enabled_get(tsd) == false); |
586 | if (config_debug) { |
587 | assert(tcache_small_bin_get(tcache, 0)->avail == NULL); |
588 | } |
589 | return; |
590 | } |
591 | assert(tsd_tcache_enabled_get(tsd)); |
592 | assert(tcache_small_bin_get(tcache, 0)->avail != NULL); |
593 | |
594 | tcache_destroy(tsd, tcache, true); |
595 | if (config_debug) { |
596 | tcache_small_bin_get(tcache, 0)->avail = NULL; |
597 | } |
598 | } |
599 | |
600 | void |
601 | tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
602 | unsigned i; |
603 | |
604 | cassert(config_stats); |
605 | |
606 | /* Merge and reset tcache stats. */ |
607 | for (i = 0; i < SC_NBINS; i++) { |
608 | cache_bin_t *tbin = tcache_small_bin_get(tcache, i); |
609 | unsigned binshard; |
610 | bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard); |
611 | bin->stats.nrequests += tbin->tstats.nrequests; |
612 | malloc_mutex_unlock(tsdn, &bin->lock); |
613 | tbin->tstats.nrequests = 0; |
614 | } |
615 | |
616 | for (; i < nhbins; i++) { |
617 | cache_bin_t *tbin = tcache_large_bin_get(tcache, i); |
618 | arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i, |
619 | tbin->tstats.nrequests); |
620 | tbin->tstats.nrequests = 0; |
621 | } |
622 | } |
623 | |
624 | static bool |
625 | tcaches_create_prep(tsd_t *tsd) { |
626 | bool err; |
627 | |
628 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
629 | |
630 | if (tcaches == NULL) { |
631 | tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *) |
632 | * (MALLOCX_TCACHE_MAX+1), CACHELINE); |
633 | if (tcaches == NULL) { |
634 | err = true; |
635 | goto label_return; |
636 | } |
637 | } |
638 | |
639 | if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) { |
640 | err = true; |
641 | goto label_return; |
642 | } |
643 | |
644 | err = false; |
645 | label_return: |
646 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
647 | return err; |
648 | } |
649 | |
650 | bool |
651 | tcaches_create(tsd_t *tsd, unsigned *r_ind) { |
652 | witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); |
653 | |
654 | bool err; |
655 | |
656 | if (tcaches_create_prep(tsd)) { |
657 | err = true; |
658 | goto label_return; |
659 | } |
660 | |
661 | tcache_t *tcache = tcache_create_explicit(tsd); |
662 | if (tcache == NULL) { |
663 | err = true; |
664 | goto label_return; |
665 | } |
666 | |
667 | tcaches_t *elm; |
668 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
669 | if (tcaches_avail != NULL) { |
670 | elm = tcaches_avail; |
671 | tcaches_avail = tcaches_avail->next; |
672 | elm->tcache = tcache; |
673 | *r_ind = (unsigned)(elm - tcaches); |
674 | } else { |
675 | elm = &tcaches[tcaches_past]; |
676 | elm->tcache = tcache; |
677 | *r_ind = tcaches_past; |
678 | tcaches_past++; |
679 | } |
680 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
681 | |
682 | err = false; |
683 | label_return: |
684 | witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); |
685 | return err; |
686 | } |
687 | |
688 | static tcache_t * |
689 | tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { |
690 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); |
691 | |
692 | if (elm->tcache == NULL) { |
693 | return NULL; |
694 | } |
695 | tcache_t *tcache = elm->tcache; |
696 | if (allow_reinit) { |
697 | elm->tcache = TCACHES_ELM_NEED_REINIT; |
698 | } else { |
699 | elm->tcache = NULL; |
700 | } |
701 | |
702 | if (tcache == TCACHES_ELM_NEED_REINIT) { |
703 | return NULL; |
704 | } |
705 | return tcache; |
706 | } |
707 | |
708 | void |
709 | tcaches_flush(tsd_t *tsd, unsigned ind) { |
710 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
711 | tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); |
712 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
713 | if (tcache != NULL) { |
714 | /* Destroy the tcache; recreate in tcaches_get() if needed. */ |
715 | tcache_destroy(tsd, tcache, false); |
716 | } |
717 | } |
718 | |
719 | void |
720 | tcaches_destroy(tsd_t *tsd, unsigned ind) { |
721 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
722 | tcaches_t *elm = &tcaches[ind]; |
723 | tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); |
724 | elm->next = tcaches_avail; |
725 | tcaches_avail = elm; |
726 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
727 | if (tcache != NULL) { |
728 | tcache_destroy(tsd, tcache, false); |
729 | } |
730 | } |
731 | |
732 | bool |
733 | tcache_boot(tsdn_t *tsdn) { |
734 | /* If necessary, clamp opt_lg_tcache_max. */ |
735 | if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < |
736 | SC_SMALL_MAXCLASS) { |
737 | tcache_maxclass = SC_SMALL_MAXCLASS; |
738 | } else { |
739 | tcache_maxclass = (ZU(1) << opt_lg_tcache_max); |
740 | } |
741 | |
742 | if (malloc_mutex_init(&tcaches_mtx, "tcaches" , WITNESS_RANK_TCACHES, |
743 | malloc_mutex_rank_exclusive)) { |
744 | return true; |
745 | } |
746 | |
747 | nhbins = sz_size2index(tcache_maxclass) + 1; |
748 | |
749 | /* Initialize tcache_bin_info. */ |
750 | tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins |
751 | * sizeof(cache_bin_info_t), CACHELINE); |
752 | if (tcache_bin_info == NULL) { |
753 | return true; |
754 | } |
755 | stack_nelms = 0; |
756 | unsigned i; |
757 | for (i = 0; i < SC_NBINS; i++) { |
758 | if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { |
759 | tcache_bin_info[i].ncached_max = |
760 | TCACHE_NSLOTS_SMALL_MIN; |
761 | } else if ((bin_infos[i].nregs << 1) <= |
762 | TCACHE_NSLOTS_SMALL_MAX) { |
763 | tcache_bin_info[i].ncached_max = |
764 | (bin_infos[i].nregs << 1); |
765 | } else { |
766 | tcache_bin_info[i].ncached_max = |
767 | TCACHE_NSLOTS_SMALL_MAX; |
768 | } |
769 | stack_nelms += tcache_bin_info[i].ncached_max; |
770 | } |
771 | for (; i < nhbins; i++) { |
772 | tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; |
773 | stack_nelms += tcache_bin_info[i].ncached_max; |
774 | } |
775 | |
776 | return false; |
777 | } |
778 | |
779 | void |
780 | tcache_prefork(tsdn_t *tsdn) { |
781 | if (!config_prof && opt_tcache) { |
782 | malloc_mutex_prefork(tsdn, &tcaches_mtx); |
783 | } |
784 | } |
785 | |
786 | void |
787 | tcache_postfork_parent(tsdn_t *tsdn) { |
788 | if (!config_prof && opt_tcache) { |
789 | malloc_mutex_postfork_parent(tsdn, &tcaches_mtx); |
790 | } |
791 | } |
792 | |
793 | void |
794 | tcache_postfork_child(tsdn_t *tsdn) { |
795 | if (!config_prof && opt_tcache) { |
796 | malloc_mutex_postfork_child(tsdn, &tcaches_mtx); |
797 | } |
798 | } |
799 | |