1#define JEMALLOC_ARENA_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/div.h"
7#include "jemalloc/internal/extent_dss.h"
8#include "jemalloc/internal/extent_mmap.h"
9#include "jemalloc/internal/mutex.h"
10#include "jemalloc/internal/rtree.h"
11#include "jemalloc/internal/safety_check.h"
12#include "jemalloc/internal/util.h"
13
14JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
15
16/******************************************************************************/
17/* Data. */
18
19/*
20 * Define names for both unininitialized and initialized phases, so that
21 * options and mallctl processing are straightforward.
22 */
23const char *percpu_arena_mode_names[] = {
24 "percpu",
25 "phycpu",
26 "disabled",
27 "percpu",
28 "phycpu"
29};
30percpu_arena_mode_t opt_percpu_arena = PERCPU_ARENA_DEFAULT;
31
32ssize_t opt_dirty_decay_ms = DIRTY_DECAY_MS_DEFAULT;
33ssize_t opt_muzzy_decay_ms = MUZZY_DECAY_MS_DEFAULT;
34
35static atomic_zd_t dirty_decay_ms_default;
36static atomic_zd_t muzzy_decay_ms_default;
37
38const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
39#define STEP(step, h, x, y) \
40 h,
41 SMOOTHSTEP
42#undef STEP
43};
44
45static div_info_t arena_binind_div_info[SC_NBINS];
46
47size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
48size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
49static unsigned huge_arena_ind;
50
51/******************************************************************************/
52/*
53 * Function prototypes for static functions that are referenced prior to
54 * definition.
55 */
56
57static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena,
58 arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit,
59 size_t npages_decay_max, bool is_background_thread);
60static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena,
61 bool is_background_thread, bool all);
62static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
63 bin_t *bin);
64static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
65 bin_t *bin);
66
67/******************************************************************************/
68
69void
70arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
71 const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
72 size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
73 *nthreads += arena_nthreads_get(arena, false);
74 *dss = dss_prec_names[arena_dss_prec_get(arena)];
75 *dirty_decay_ms = arena_dirty_decay_ms_get(arena);
76 *muzzy_decay_ms = arena_muzzy_decay_ms_get(arena);
77 *nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED);
78 *ndirty += extents_npages_get(&arena->extents_dirty);
79 *nmuzzy += extents_npages_get(&arena->extents_muzzy);
80}
81
82void
83arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
84 const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
85 size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
86 bin_stats_t *bstats, arena_stats_large_t *lstats,
87 arena_stats_extents_t *estats) {
88 cassert(config_stats);
89
90 arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
91 muzzy_decay_ms, nactive, ndirty, nmuzzy);
92
93 size_t base_allocated, base_resident, base_mapped, metadata_thp;
94 base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
95 &base_mapped, &metadata_thp);
96
97 arena_stats_lock(tsdn, &arena->stats);
98
99 arena_stats_accum_zu(&astats->mapped, base_mapped
100 + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped));
101 arena_stats_accum_zu(&astats->retained,
102 extents_npages_get(&arena->extents_retained) << LG_PAGE);
103
104 atomic_store_zu(&astats->extent_avail,
105 atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED),
106 ATOMIC_RELAXED);
107
108 arena_stats_accum_u64(&astats->decay_dirty.npurge,
109 arena_stats_read_u64(tsdn, &arena->stats,
110 &arena->stats.decay_dirty.npurge));
111 arena_stats_accum_u64(&astats->decay_dirty.nmadvise,
112 arena_stats_read_u64(tsdn, &arena->stats,
113 &arena->stats.decay_dirty.nmadvise));
114 arena_stats_accum_u64(&astats->decay_dirty.purged,
115 arena_stats_read_u64(tsdn, &arena->stats,
116 &arena->stats.decay_dirty.purged));
117
118 arena_stats_accum_u64(&astats->decay_muzzy.npurge,
119 arena_stats_read_u64(tsdn, &arena->stats,
120 &arena->stats.decay_muzzy.npurge));
121 arena_stats_accum_u64(&astats->decay_muzzy.nmadvise,
122 arena_stats_read_u64(tsdn, &arena->stats,
123 &arena->stats.decay_muzzy.nmadvise));
124 arena_stats_accum_u64(&astats->decay_muzzy.purged,
125 arena_stats_read_u64(tsdn, &arena->stats,
126 &arena->stats.decay_muzzy.purged));
127
128 arena_stats_accum_zu(&astats->base, base_allocated);
129 arena_stats_accum_zu(&astats->internal, arena_internal_get(arena));
130 arena_stats_accum_zu(&astats->metadata_thp, metadata_thp);
131 arena_stats_accum_zu(&astats->resident, base_resident +
132 (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) +
133 extents_npages_get(&arena->extents_dirty) +
134 extents_npages_get(&arena->extents_muzzy)) << LG_PAGE)));
135 arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu(
136 &arena->stats.abandoned_vm, ATOMIC_RELAXED));
137
138 for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) {
139 uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats,
140 &arena->stats.lstats[i].nmalloc);
141 arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc);
142 arena_stats_accum_u64(&astats->nmalloc_large, nmalloc);
143
144 uint64_t ndalloc = arena_stats_read_u64(tsdn, &arena->stats,
145 &arena->stats.lstats[i].ndalloc);
146 arena_stats_accum_u64(&lstats[i].ndalloc, ndalloc);
147 arena_stats_accum_u64(&astats->ndalloc_large, ndalloc);
148
149 uint64_t nrequests = arena_stats_read_u64(tsdn, &arena->stats,
150 &arena->stats.lstats[i].nrequests);
151 arena_stats_accum_u64(&lstats[i].nrequests,
152 nmalloc + nrequests);
153 arena_stats_accum_u64(&astats->nrequests_large,
154 nmalloc + nrequests);
155
156 /* nfill == nmalloc for large currently. */
157 arena_stats_accum_u64(&lstats[i].nfills, nmalloc);
158 arena_stats_accum_u64(&astats->nfills_large, nmalloc);
159
160 uint64_t nflush = arena_stats_read_u64(tsdn, &arena->stats,
161 &arena->stats.lstats[i].nflushes);
162 arena_stats_accum_u64(&lstats[i].nflushes, nflush);
163 arena_stats_accum_u64(&astats->nflushes_large, nflush);
164
165 assert(nmalloc >= ndalloc);
166 assert(nmalloc - ndalloc <= SIZE_T_MAX);
167 size_t curlextents = (size_t)(nmalloc - ndalloc);
168 lstats[i].curlextents += curlextents;
169 arena_stats_accum_zu(&astats->allocated_large,
170 curlextents * sz_index2size(SC_NBINS + i));
171 }
172
173 for (pszind_t i = 0; i < SC_NPSIZES; i++) {
174 size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
175 retained_bytes;
176 dirty = extents_nextents_get(&arena->extents_dirty, i);
177 muzzy = extents_nextents_get(&arena->extents_muzzy, i);
178 retained = extents_nextents_get(&arena->extents_retained, i);
179 dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i);
180 muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i);
181 retained_bytes =
182 extents_nbytes_get(&arena->extents_retained, i);
183
184 atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED);
185 atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED);
186 atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED);
187 atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes,
188 ATOMIC_RELAXED);
189 atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes,
190 ATOMIC_RELAXED);
191 atomic_store_zu(&estats[i].retained_bytes, retained_bytes,
192 ATOMIC_RELAXED);
193 }
194
195 arena_stats_unlock(tsdn, &arena->stats);
196
197 /* tcache_bytes counts currently cached bytes. */
198 atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED);
199 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
200 cache_bin_array_descriptor_t *descriptor;
201 ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) {
202 szind_t i = 0;
203 for (; i < SC_NBINS; i++) {
204 cache_bin_t *tbin = &descriptor->bins_small[i];
205 arena_stats_accum_zu(&astats->tcache_bytes,
206 tbin->ncached * sz_index2size(i));
207 }
208 for (; i < nhbins; i++) {
209 cache_bin_t *tbin = &descriptor->bins_large[i];
210 arena_stats_accum_zu(&astats->tcache_bytes,
211 tbin->ncached * sz_index2size(i));
212 }
213 }
214 malloc_mutex_prof_read(tsdn,
215 &astats->mutex_prof_data[arena_prof_mutex_tcache_list],
216 &arena->tcache_ql_mtx);
217 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
218
219#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
220 malloc_mutex_lock(tsdn, &arena->mtx); \
221 malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \
222 &arena->mtx); \
223 malloc_mutex_unlock(tsdn, &arena->mtx);
224
225 /* Gather per arena mutex profiling data. */
226 READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
227 READ_ARENA_MUTEX_PROF_DATA(extent_avail_mtx,
228 arena_prof_mutex_extent_avail)
229 READ_ARENA_MUTEX_PROF_DATA(extents_dirty.mtx,
230 arena_prof_mutex_extents_dirty)
231 READ_ARENA_MUTEX_PROF_DATA(extents_muzzy.mtx,
232 arena_prof_mutex_extents_muzzy)
233 READ_ARENA_MUTEX_PROF_DATA(extents_retained.mtx,
234 arena_prof_mutex_extents_retained)
235 READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx,
236 arena_prof_mutex_decay_dirty)
237 READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx,
238 arena_prof_mutex_decay_muzzy)
239 READ_ARENA_MUTEX_PROF_DATA(base->mtx,
240 arena_prof_mutex_base)
241#undef READ_ARENA_MUTEX_PROF_DATA
242
243 nstime_copy(&astats->uptime, &arena->create_time);
244 nstime_update(&astats->uptime);
245 nstime_subtract(&astats->uptime, &arena->create_time);
246
247 for (szind_t i = 0; i < SC_NBINS; i++) {
248 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
249 bin_stats_merge(tsdn, &bstats[i],
250 &arena->bins[i].bin_shards[j]);
251 }
252 }
253}
254
255void
256arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
257 extent_hooks_t **r_extent_hooks, extent_t *extent) {
258 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
259 WITNESS_RANK_CORE, 0);
260
261 extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty,
262 extent);
263 if (arena_dirty_decay_ms_get(arena) == 0) {
264 arena_decay_dirty(tsdn, arena, false, true);
265 } else {
266 arena_background_thread_inactivity_check(tsdn, arena, false);
267 }
268}
269
270static void *
271arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) {
272 void *ret;
273 arena_slab_data_t *slab_data = extent_slab_data_get(slab);
274 size_t regind;
275
276 assert(extent_nfree_get(slab) > 0);
277 assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
278
279 regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info);
280 ret = (void *)((uintptr_t)extent_addr_get(slab) +
281 (uintptr_t)(bin_info->reg_size * regind));
282 extent_nfree_dec(slab);
283 return ret;
284}
285
286static void
287arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info,
288 unsigned cnt, void** ptrs) {
289 arena_slab_data_t *slab_data = extent_slab_data_get(slab);
290
291 assert(extent_nfree_get(slab) >= cnt);
292 assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
293
294#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE)
295 for (unsigned i = 0; i < cnt; i++) {
296 size_t regind = bitmap_sfu(slab_data->bitmap,
297 &bin_info->bitmap_info);
298 *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) +
299 (uintptr_t)(bin_info->reg_size * regind));
300 }
301#else
302 unsigned group = 0;
303 bitmap_t g = slab_data->bitmap[group];
304 unsigned i = 0;
305 while (i < cnt) {
306 while (g == 0) {
307 g = slab_data->bitmap[++group];
308 }
309 size_t shift = group << LG_BITMAP_GROUP_NBITS;
310 size_t pop = popcount_lu(g);
311 if (pop > (cnt - i)) {
312 pop = cnt - i;
313 }
314
315 /*
316 * Load from memory locations only once, outside the
317 * hot loop below.
318 */
319 uintptr_t base = (uintptr_t)extent_addr_get(slab);
320 uintptr_t regsize = (uintptr_t)bin_info->reg_size;
321 while (pop--) {
322 size_t bit = cfs_lu(&g);
323 size_t regind = shift + bit;
324 *(ptrs + i) = (void *)(base + regsize * regind);
325
326 i++;
327 }
328 slab_data->bitmap[group] = g;
329 }
330#endif
331 extent_nfree_sub(slab, cnt);
332}
333
334#ifndef JEMALLOC_JET
335static
336#endif
337size_t
338arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr) {
339 size_t diff, regind;
340
341 /* Freeing a pointer outside the slab can cause assertion failure. */
342 assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab));
343 assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab));
344 /* Freeing an interior pointer can cause assertion failure. */
345 assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) %
346 (uintptr_t)bin_infos[binind].reg_size == 0);
347
348 diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab));
349
350 /* Avoid doing division with a variable divisor. */
351 regind = div_compute(&arena_binind_div_info[binind], diff);
352
353 assert(regind < bin_infos[binind].nregs);
354
355 return regind;
356}
357
358static void
359arena_slab_reg_dalloc(extent_t *slab, arena_slab_data_t *slab_data, void *ptr) {
360 szind_t binind = extent_szind_get(slab);
361 const bin_info_t *bin_info = &bin_infos[binind];
362 size_t regind = arena_slab_regind(slab, binind, ptr);
363
364 assert(extent_nfree_get(slab) < bin_info->nregs);
365 /* Freeing an unallocated pointer can cause assertion failure. */
366 assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
367
368 bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
369 extent_nfree_inc(slab);
370}
371
372static void
373arena_nactive_add(arena_t *arena, size_t add_pages) {
374 atomic_fetch_add_zu(&arena->nactive, add_pages, ATOMIC_RELAXED);
375}
376
377static void
378arena_nactive_sub(arena_t *arena, size_t sub_pages) {
379 assert(atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) >= sub_pages);
380 atomic_fetch_sub_zu(&arena->nactive, sub_pages, ATOMIC_RELAXED);
381}
382
383static void
384arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
385 szind_t index, hindex;
386
387 cassert(config_stats);
388
389 if (usize < SC_LARGE_MINCLASS) {
390 usize = SC_LARGE_MINCLASS;
391 }
392 index = sz_size2index(usize);
393 hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0;
394
395 arena_stats_add_u64(tsdn, &arena->stats,
396 &arena->stats.lstats[hindex].nmalloc, 1);
397}
398
399static void
400arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
401 szind_t index, hindex;
402
403 cassert(config_stats);
404
405 if (usize < SC_LARGE_MINCLASS) {
406 usize = SC_LARGE_MINCLASS;
407 }
408 index = sz_size2index(usize);
409 hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0;
410
411 arena_stats_add_u64(tsdn, &arena->stats,
412 &arena->stats.lstats[hindex].ndalloc, 1);
413}
414
415static void
416arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize,
417 size_t usize) {
418 arena_large_dalloc_stats_update(tsdn, arena, oldusize);
419 arena_large_malloc_stats_update(tsdn, arena, usize);
420}
421
422static bool
423arena_may_have_muzzy(arena_t *arena) {
424 return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0));
425}
426
427extent_t *
428arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
429 size_t alignment, bool *zero) {
430 extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
431
432 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
433 WITNESS_RANK_CORE, 0);
434
435 szind_t szind = sz_size2index(usize);
436 size_t mapped_add;
437 bool commit = true;
438 extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks,
439 &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false,
440 szind, zero, &commit);
441 if (extent == NULL && arena_may_have_muzzy(arena)) {
442 extent = extents_alloc(tsdn, arena, &extent_hooks,
443 &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment,
444 false, szind, zero, &commit);
445 }
446 size_t size = usize + sz_large_pad;
447 if (extent == NULL) {
448 extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL,
449 usize, sz_large_pad, alignment, false, szind, zero,
450 &commit);
451 if (config_stats) {
452 /*
453 * extent may be NULL on OOM, but in that case
454 * mapped_add isn't used below, so there's no need to
455 * conditionlly set it to 0 here.
456 */
457 mapped_add = size;
458 }
459 } else if (config_stats) {
460 mapped_add = 0;
461 }
462
463 if (extent != NULL) {
464 if (config_stats) {
465 arena_stats_lock(tsdn, &arena->stats);
466 arena_large_malloc_stats_update(tsdn, arena, usize);
467 if (mapped_add != 0) {
468 arena_stats_add_zu(tsdn, &arena->stats,
469 &arena->stats.mapped, mapped_add);
470 }
471 arena_stats_unlock(tsdn, &arena->stats);
472 }
473 arena_nactive_add(arena, size >> LG_PAGE);
474 }
475
476 return extent;
477}
478
479void
480arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
481 if (config_stats) {
482 arena_stats_lock(tsdn, &arena->stats);
483 arena_large_dalloc_stats_update(tsdn, arena,
484 extent_usize_get(extent));
485 arena_stats_unlock(tsdn, &arena->stats);
486 }
487 arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
488}
489
490void
491arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
492 size_t oldusize) {
493 size_t usize = extent_usize_get(extent);
494 size_t udiff = oldusize - usize;
495
496 if (config_stats) {
497 arena_stats_lock(tsdn, &arena->stats);
498 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize);
499 arena_stats_unlock(tsdn, &arena->stats);
500 }
501 arena_nactive_sub(arena, udiff >> LG_PAGE);
502}
503
504void
505arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
506 size_t oldusize) {
507 size_t usize = extent_usize_get(extent);
508 size_t udiff = usize - oldusize;
509
510 if (config_stats) {
511 arena_stats_lock(tsdn, &arena->stats);
512 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize);
513 arena_stats_unlock(tsdn, &arena->stats);
514 }
515 arena_nactive_add(arena, udiff >> LG_PAGE);
516}
517
518static ssize_t
519arena_decay_ms_read(arena_decay_t *decay) {
520 return atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED);
521}
522
523static void
524arena_decay_ms_write(arena_decay_t *decay, ssize_t decay_ms) {
525 atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED);
526}
527
528static void
529arena_decay_deadline_init(arena_decay_t *decay) {
530 /*
531 * Generate a new deadline that is uniformly random within the next
532 * epoch after the current one.
533 */
534 nstime_copy(&decay->deadline, &decay->epoch);
535 nstime_add(&decay->deadline, &decay->interval);
536 if (arena_decay_ms_read(decay) > 0) {
537 nstime_t jitter;
538
539 nstime_init(&jitter, prng_range_u64(&decay->jitter_state,
540 nstime_ns(&decay->interval)));
541 nstime_add(&decay->deadline, &jitter);
542 }
543}
544
545static bool
546arena_decay_deadline_reached(const arena_decay_t *decay, const nstime_t *time) {
547 return (nstime_compare(&decay->deadline, time) <= 0);
548}
549
550static size_t
551arena_decay_backlog_npages_limit(const arena_decay_t *decay) {
552 uint64_t sum;
553 size_t npages_limit_backlog;
554 unsigned i;
555
556 /*
557 * For each element of decay_backlog, multiply by the corresponding
558 * fixed-point smoothstep decay factor. Sum the products, then divide
559 * to round down to the nearest whole number of pages.
560 */
561 sum = 0;
562 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
563 sum += decay->backlog[i] * h_steps[i];
564 }
565 npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP);
566
567 return npages_limit_backlog;
568}
569
570static void
571arena_decay_backlog_update_last(arena_decay_t *decay, size_t current_npages) {
572 size_t npages_delta = (current_npages > decay->nunpurged) ?
573 current_npages - decay->nunpurged : 0;
574 decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta;
575
576 if (config_debug) {
577 if (current_npages > decay->ceil_npages) {
578 decay->ceil_npages = current_npages;
579 }
580 size_t npages_limit = arena_decay_backlog_npages_limit(decay);
581 assert(decay->ceil_npages >= npages_limit);
582 if (decay->ceil_npages > npages_limit) {
583 decay->ceil_npages = npages_limit;
584 }
585 }
586}
587
588static void
589arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64,
590 size_t current_npages) {
591 if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) {
592 memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) *
593 sizeof(size_t));
594 } else {
595 size_t nadvance_z = (size_t)nadvance_u64;
596
597 assert((uint64_t)nadvance_z == nadvance_u64);
598
599 memmove(decay->backlog, &decay->backlog[nadvance_z],
600 (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t));
601 if (nadvance_z > 1) {
602 memset(&decay->backlog[SMOOTHSTEP_NSTEPS -
603 nadvance_z], 0, (nadvance_z-1) * sizeof(size_t));
604 }
605 }
606
607 arena_decay_backlog_update_last(decay, current_npages);
608}
609
610static void
611arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
612 extents_t *extents, size_t current_npages, size_t npages_limit,
613 bool is_background_thread) {
614 if (current_npages > npages_limit) {
615 arena_decay_to_limit(tsdn, arena, decay, extents, false,
616 npages_limit, current_npages - npages_limit,
617 is_background_thread);
618 }
619}
620
621static void
622arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time,
623 size_t current_npages) {
624 assert(arena_decay_deadline_reached(decay, time));
625
626 nstime_t delta;
627 nstime_copy(&delta, time);
628 nstime_subtract(&delta, &decay->epoch);
629
630 uint64_t nadvance_u64 = nstime_divide(&delta, &decay->interval);
631 assert(nadvance_u64 > 0);
632
633 /* Add nadvance_u64 decay intervals to epoch. */
634 nstime_copy(&delta, &decay->interval);
635 nstime_imultiply(&delta, nadvance_u64);
636 nstime_add(&decay->epoch, &delta);
637
638 /* Set a new deadline. */
639 arena_decay_deadline_init(decay);
640
641 /* Update the backlog. */
642 arena_decay_backlog_update(decay, nadvance_u64, current_npages);
643}
644
645static void
646arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
647 extents_t *extents, const nstime_t *time, bool is_background_thread) {
648 size_t current_npages = extents_npages_get(extents);
649 arena_decay_epoch_advance_helper(decay, time, current_npages);
650
651 size_t npages_limit = arena_decay_backlog_npages_limit(decay);
652 /* We may unlock decay->mtx when try_purge(). Finish logging first. */
653 decay->nunpurged = (npages_limit > current_npages) ? npages_limit :
654 current_npages;
655
656 if (!background_thread_enabled() || is_background_thread) {
657 arena_decay_try_purge(tsdn, arena, decay, extents,
658 current_npages, npages_limit, is_background_thread);
659 }
660}
661
662static void
663arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) {
664 arena_decay_ms_write(decay, decay_ms);
665 if (decay_ms > 0) {
666 nstime_init(&decay->interval, (uint64_t)decay_ms *
667 KQU(1000000));
668 nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS);
669 }
670
671 nstime_init(&decay->epoch, 0);
672 nstime_update(&decay->epoch);
673 decay->jitter_state = (uint64_t)(uintptr_t)decay;
674 arena_decay_deadline_init(decay);
675 decay->nunpurged = 0;
676 memset(decay->backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));
677}
678
679static bool
680arena_decay_init(arena_decay_t *decay, ssize_t decay_ms,
681 arena_stats_decay_t *stats) {
682 if (config_debug) {
683 for (size_t i = 0; i < sizeof(arena_decay_t); i++) {
684 assert(((char *)decay)[i] == 0);
685 }
686 decay->ceil_npages = 0;
687 }
688 if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY,
689 malloc_mutex_rank_exclusive)) {
690 return true;
691 }
692 decay->purging = false;
693 arena_decay_reinit(decay, decay_ms);
694 /* Memory is zeroed, so there is no need to clear stats. */
695 if (config_stats) {
696 decay->stats = stats;
697 }
698 return false;
699}
700
701static bool
702arena_decay_ms_valid(ssize_t decay_ms) {
703 if (decay_ms < -1) {
704 return false;
705 }
706 if (decay_ms == -1 || (uint64_t)decay_ms <= NSTIME_SEC_MAX *
707 KQU(1000)) {
708 return true;
709 }
710 return false;
711}
712
713static bool
714arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
715 extents_t *extents, bool is_background_thread) {
716 malloc_mutex_assert_owner(tsdn, &decay->mtx);
717
718 /* Purge all or nothing if the option is disabled. */
719 ssize_t decay_ms = arena_decay_ms_read(decay);
720 if (decay_ms <= 0) {
721 if (decay_ms == 0) {
722 arena_decay_to_limit(tsdn, arena, decay, extents, false,
723 0, extents_npages_get(extents),
724 is_background_thread);
725 }
726 return false;
727 }
728
729 nstime_t time;
730 nstime_init(&time, 0);
731 nstime_update(&time);
732 if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch, &time)
733 > 0)) {
734 /*
735 * Time went backwards. Move the epoch back in time and
736 * generate a new deadline, with the expectation that time
737 * typically flows forward for long enough periods of time that
738 * epochs complete. Unfortunately, this strategy is susceptible
739 * to clock jitter triggering premature epoch advances, but
740 * clock jitter estimation and compensation isn't feasible here
741 * because calls into this code are event-driven.
742 */
743 nstime_copy(&decay->epoch, &time);
744 arena_decay_deadline_init(decay);
745 } else {
746 /* Verify that time does not go backwards. */
747 assert(nstime_compare(&decay->epoch, &time) <= 0);
748 }
749
750 /*
751 * If the deadline has been reached, advance to the current epoch and
752 * purge to the new limit if necessary. Note that dirty pages created
753 * during the current epoch are not subject to purge until a future
754 * epoch, so as a result purging only happens during epoch advances, or
755 * being triggered by background threads (scheduled event).
756 */
757 bool advance_epoch = arena_decay_deadline_reached(decay, &time);
758 if (advance_epoch) {
759 arena_decay_epoch_advance(tsdn, arena, decay, extents, &time,
760 is_background_thread);
761 } else if (is_background_thread) {
762 arena_decay_try_purge(tsdn, arena, decay, extents,
763 extents_npages_get(extents),
764 arena_decay_backlog_npages_limit(decay),
765 is_background_thread);
766 }
767
768 return advance_epoch;
769}
770
771static ssize_t
772arena_decay_ms_get(arena_decay_t *decay) {
773 return arena_decay_ms_read(decay);
774}
775
776ssize_t
777arena_dirty_decay_ms_get(arena_t *arena) {
778 return arena_decay_ms_get(&arena->decay_dirty);
779}
780
781ssize_t
782arena_muzzy_decay_ms_get(arena_t *arena) {
783 return arena_decay_ms_get(&arena->decay_muzzy);
784}
785
786static bool
787arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
788 extents_t *extents, ssize_t decay_ms) {
789 if (!arena_decay_ms_valid(decay_ms)) {
790 return true;
791 }
792
793 malloc_mutex_lock(tsdn, &decay->mtx);
794 /*
795 * Restart decay backlog from scratch, which may cause many dirty pages
796 * to be immediately purged. It would conceptually be possible to map
797 * the old backlog onto the new backlog, but there is no justification
798 * for such complexity since decay_ms changes are intended to be
799 * infrequent, either between the {-1, 0, >0} states, or a one-time
800 * arbitrary change during initial arena configuration.
801 */
802 arena_decay_reinit(decay, decay_ms);
803 arena_maybe_decay(tsdn, arena, decay, extents, false);
804 malloc_mutex_unlock(tsdn, &decay->mtx);
805
806 return false;
807}
808
809bool
810arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
811 ssize_t decay_ms) {
812 return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty,
813 &arena->extents_dirty, decay_ms);
814}
815
816bool
817arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
818 ssize_t decay_ms) {
819 return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy,
820 &arena->extents_muzzy, decay_ms);
821}
822
823static size_t
824arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
825 extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit,
826 size_t npages_decay_max, extent_list_t *decay_extents) {
827 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
828 WITNESS_RANK_CORE, 0);
829
830 /* Stash extents according to npages_limit. */
831 size_t nstashed = 0;
832 extent_t *extent;
833 while (nstashed < npages_decay_max &&
834 (extent = extents_evict(tsdn, arena, r_extent_hooks, extents,
835 npages_limit)) != NULL) {
836 extent_list_append(decay_extents, extent);
837 nstashed += extent_size_get(extent) >> LG_PAGE;
838 }
839 return nstashed;
840}
841
842static size_t
843arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
844 extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents,
845 bool all, extent_list_t *decay_extents, bool is_background_thread) {
846 size_t nmadvise, nunmapped;
847 size_t npurged;
848
849 if (config_stats) {
850 nmadvise = 0;
851 nunmapped = 0;
852 }
853 npurged = 0;
854
855 ssize_t muzzy_decay_ms = arena_muzzy_decay_ms_get(arena);
856 for (extent_t *extent = extent_list_first(decay_extents); extent !=
857 NULL; extent = extent_list_first(decay_extents)) {
858 if (config_stats) {
859 nmadvise++;
860 }
861 size_t npages = extent_size_get(extent) >> LG_PAGE;
862 npurged += npages;
863 extent_list_remove(decay_extents, extent);
864 switch (extents_state_get(extents)) {
865 case extent_state_active:
866 not_reached();
867 case extent_state_dirty:
868 if (!all && muzzy_decay_ms != 0 &&
869 !extent_purge_lazy_wrapper(tsdn, arena,
870 r_extent_hooks, extent, 0,
871 extent_size_get(extent))) {
872 extents_dalloc(tsdn, arena, r_extent_hooks,
873 &arena->extents_muzzy, extent);
874 arena_background_thread_inactivity_check(tsdn,
875 arena, is_background_thread);
876 break;
877 }
878 /* Fall through. */
879 case extent_state_muzzy:
880 extent_dalloc_wrapper(tsdn, arena, r_extent_hooks,
881 extent);
882 if (config_stats) {
883 nunmapped += npages;
884 }
885 break;
886 case extent_state_retained:
887 default:
888 not_reached();
889 }
890 }
891
892 if (config_stats) {
893 arena_stats_lock(tsdn, &arena->stats);
894 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge,
895 1);
896 arena_stats_add_u64(tsdn, &arena->stats,
897 &decay->stats->nmadvise, nmadvise);
898 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged,
899 npurged);
900 arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped,
901 nunmapped << LG_PAGE);
902 arena_stats_unlock(tsdn, &arena->stats);
903 }
904
905 return npurged;
906}
907
908/*
909 * npages_limit: Decay at most npages_decay_max pages without violating the
910 * invariant: (extents_npages_get(extents) >= npages_limit). We need an upper
911 * bound on number of pages in order to prevent unbounded growth (namely in
912 * stashed), otherwise unbounded new pages could be added to extents during the
913 * current decay run, so that the purging thread never finishes.
914 */
915static void
916arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
917 extents_t *extents, bool all, size_t npages_limit, size_t npages_decay_max,
918 bool is_background_thread) {
919 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
920 WITNESS_RANK_CORE, 1);
921 malloc_mutex_assert_owner(tsdn, &decay->mtx);
922
923 if (decay->purging) {
924 return;
925 }
926 decay->purging = true;
927 malloc_mutex_unlock(tsdn, &decay->mtx);
928
929 extent_hooks_t *extent_hooks = extent_hooks_get(arena);
930
931 extent_list_t decay_extents;
932 extent_list_init(&decay_extents);
933
934 size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents,
935 npages_limit, npages_decay_max, &decay_extents);
936 if (npurge != 0) {
937 size_t npurged = arena_decay_stashed(tsdn, arena,
938 &extent_hooks, decay, extents, all, &decay_extents,
939 is_background_thread);
940 assert(npurged == npurge);
941 }
942
943 malloc_mutex_lock(tsdn, &decay->mtx);
944 decay->purging = false;
945}
946
947static bool
948arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
949 extents_t *extents, bool is_background_thread, bool all) {
950 if (all) {
951 malloc_mutex_lock(tsdn, &decay->mtx);
952 arena_decay_to_limit(tsdn, arena, decay, extents, all, 0,
953 extents_npages_get(extents), is_background_thread);
954 malloc_mutex_unlock(tsdn, &decay->mtx);
955
956 return false;
957 }
958
959 if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
960 /* No need to wait if another thread is in progress. */
961 return true;
962 }
963
964 bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents,
965 is_background_thread);
966 size_t npages_new;
967 if (epoch_advanced) {
968 /* Backlog is updated on epoch advance. */
969 npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1];
970 }
971 malloc_mutex_unlock(tsdn, &decay->mtx);
972
973 if (have_background_thread && background_thread_enabled() &&
974 epoch_advanced && !is_background_thread) {
975 background_thread_interval_check(tsdn, arena, decay,
976 npages_new);
977 }
978
979 return false;
980}
981
982static bool
983arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
984 bool all) {
985 return arena_decay_impl(tsdn, arena, &arena->decay_dirty,
986 &arena->extents_dirty, is_background_thread, all);
987}
988
989static bool
990arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
991 bool all) {
992 return arena_decay_impl(tsdn, arena, &arena->decay_muzzy,
993 &arena->extents_muzzy, is_background_thread, all);
994}
995
996void
997arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
998 if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
999 return;
1000 }
1001 arena_decay_muzzy(tsdn, arena, is_background_thread, all);
1002}
1003
1004static void
1005arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) {
1006 arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
1007
1008 extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
1009 arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, slab);
1010}
1011
1012static void
1013arena_bin_slabs_nonfull_insert(bin_t *bin, extent_t *slab) {
1014 assert(extent_nfree_get(slab) > 0);
1015 extent_heap_insert(&bin->slabs_nonfull, slab);
1016 if (config_stats) {
1017 bin->stats.nonfull_slabs++;
1018 }
1019}
1020
1021static void
1022arena_bin_slabs_nonfull_remove(bin_t *bin, extent_t *slab) {
1023 extent_heap_remove(&bin->slabs_nonfull, slab);
1024 if (config_stats) {
1025 bin->stats.nonfull_slabs--;
1026 }
1027}
1028
1029static extent_t *
1030arena_bin_slabs_nonfull_tryget(bin_t *bin) {
1031 extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull);
1032 if (slab == NULL) {
1033 return NULL;
1034 }
1035 if (config_stats) {
1036 bin->stats.reslabs++;
1037 bin->stats.nonfull_slabs--;
1038 }
1039 return slab;
1040}
1041
1042static void
1043arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, extent_t *slab) {
1044 assert(extent_nfree_get(slab) == 0);
1045 /*
1046 * Tracking extents is required by arena_reset, which is not allowed
1047 * for auto arenas. Bypass this step to avoid touching the extent
1048 * linkage (often results in cache misses) for auto arenas.
1049 */
1050 if (arena_is_auto(arena)) {
1051 return;
1052 }
1053 extent_list_append(&bin->slabs_full, slab);
1054}
1055
1056static void
1057arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) {
1058 if (arena_is_auto(arena)) {
1059 return;
1060 }
1061 extent_list_remove(&bin->slabs_full, slab);
1062}
1063
1064static void
1065arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) {
1066 extent_t *slab;
1067
1068 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
1069 if (bin->slabcur != NULL) {
1070 slab = bin->slabcur;
1071 bin->slabcur = NULL;
1072 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
1073 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
1074 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
1075 }
1076 while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) {
1077 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
1078 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
1079 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
1080 }
1081 for (slab = extent_list_first(&bin->slabs_full); slab != NULL;
1082 slab = extent_list_first(&bin->slabs_full)) {
1083 arena_bin_slabs_full_remove(arena, bin, slab);
1084 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
1085 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
1086 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
1087 }
1088 if (config_stats) {
1089 bin->stats.curregs = 0;
1090 bin->stats.curslabs = 0;
1091 }
1092 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
1093}
1094
1095void
1096arena_reset(tsd_t *tsd, arena_t *arena) {
1097 /*
1098 * Locking in this function is unintuitive. The caller guarantees that
1099 * no concurrent operations are happening in this arena, but there are
1100 * still reasons that some locking is necessary:
1101 *
1102 * - Some of the functions in the transitive closure of calls assume
1103 * appropriate locks are held, and in some cases these locks are
1104 * temporarily dropped to avoid lock order reversal or deadlock due to
1105 * reentry.
1106 * - mallctl("epoch", ...) may concurrently refresh stats. While
1107 * strictly speaking this is a "concurrent operation", disallowing
1108 * stats refreshes would impose an inconvenient burden.
1109 */
1110
1111 /* Large allocations. */
1112 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
1113
1114 for (extent_t *extent = extent_list_first(&arena->large); extent !=
1115 NULL; extent = extent_list_first(&arena->large)) {
1116 void *ptr = extent_base_get(extent);
1117 size_t usize;
1118
1119 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
1120 alloc_ctx_t alloc_ctx;
1121 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
1122 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
1123 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
1124 assert(alloc_ctx.szind != SC_NSIZES);
1125
1126 if (config_stats || (config_prof && opt_prof)) {
1127 usize = sz_index2size(alloc_ctx.szind);
1128 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
1129 }
1130 /* Remove large allocation from prof sample set. */
1131 if (config_prof && opt_prof) {
1132 prof_free(tsd, ptr, usize, &alloc_ctx);
1133 }
1134 large_dalloc(tsd_tsdn(tsd), extent);
1135 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
1136 }
1137 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
1138
1139 /* Bins. */
1140 for (unsigned i = 0; i < SC_NBINS; i++) {
1141 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
1142 arena_bin_reset(tsd, arena,
1143 &arena->bins[i].bin_shards[j]);
1144 }
1145 }
1146
1147 atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED);
1148}
1149
1150static void
1151arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) {
1152 /*
1153 * Iterate over the retained extents and destroy them. This gives the
1154 * extent allocator underlying the extent hooks an opportunity to unmap
1155 * all retained memory without having to keep its own metadata
1156 * structures. In practice, virtual memory for dss-allocated extents is
1157 * leaked here, so best practice is to avoid dss for arenas to be
1158 * destroyed, or provide custom extent hooks that track retained
1159 * dss-based extents for later reuse.
1160 */
1161 extent_hooks_t *extent_hooks = extent_hooks_get(arena);
1162 extent_t *extent;
1163 while ((extent = extents_evict(tsdn, arena, &extent_hooks,
1164 &arena->extents_retained, 0)) != NULL) {
1165 extent_destroy_wrapper(tsdn, arena, &extent_hooks, extent);
1166 }
1167}
1168
1169void
1170arena_destroy(tsd_t *tsd, arena_t *arena) {
1171 assert(base_ind_get(arena->base) >= narenas_auto);
1172 assert(arena_nthreads_get(arena, false) == 0);
1173 assert(arena_nthreads_get(arena, true) == 0);
1174
1175 /*
1176 * No allocations have occurred since arena_reset() was called.
1177 * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
1178 * extents, so only retained extents may remain.
1179 */
1180 assert(extents_npages_get(&arena->extents_dirty) == 0);
1181 assert(extents_npages_get(&arena->extents_muzzy) == 0);
1182
1183 /* Deallocate retained memory. */
1184 arena_destroy_retained(tsd_tsdn(tsd), arena);
1185
1186 /*
1187 * Remove the arena pointer from the arenas array. We rely on the fact
1188 * that there is no way for the application to get a dirty read from the
1189 * arenas array unless there is an inherent race in the application
1190 * involving access of an arena being concurrently destroyed. The
1191 * application must synchronize knowledge of the arena's validity, so as
1192 * long as we use an atomic write to update the arenas array, the
1193 * application will get a clean read any time after it synchronizes
1194 * knowledge that the arena is no longer valid.
1195 */
1196 arena_set(base_ind_get(arena->base), NULL);
1197
1198 /*
1199 * Destroy the base allocator, which manages all metadata ever mapped by
1200 * this arena.
1201 */
1202 base_delete(tsd_tsdn(tsd), arena->base);
1203}
1204
1205static extent_t *
1206arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
1207 extent_hooks_t **r_extent_hooks, const bin_info_t *bin_info,
1208 szind_t szind) {
1209 extent_t *slab;
1210 bool zero, commit;
1211
1212 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1213 WITNESS_RANK_CORE, 0);
1214
1215 zero = false;
1216 commit = true;
1217 slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL,
1218 bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit);
1219
1220 if (config_stats && slab != NULL) {
1221 arena_stats_mapped_add(tsdn, &arena->stats,
1222 bin_info->slab_size);
1223 }
1224
1225 return slab;
1226}
1227
1228static extent_t *
1229arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard,
1230 const bin_info_t *bin_info) {
1231 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
1232 WITNESS_RANK_CORE, 0);
1233
1234 extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
1235 szind_t szind = sz_size2index(bin_info->reg_size);
1236 bool zero = false;
1237 bool commit = true;
1238 extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks,
1239 &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true,
1240 binind, &zero, &commit);
1241 if (slab == NULL && arena_may_have_muzzy(arena)) {
1242 slab = extents_alloc(tsdn, arena, &extent_hooks,
1243 &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE,
1244 true, binind, &zero, &commit);
1245 }
1246 if (slab == NULL) {
1247 slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks,
1248 bin_info, szind);
1249 if (slab == NULL) {
1250 return NULL;
1251 }
1252 }
1253 assert(extent_slab_get(slab));
1254
1255 /* Initialize slab internals. */
1256 arena_slab_data_t *slab_data = extent_slab_data_get(slab);
1257 extent_nfree_binshard_set(slab, bin_info->nregs, binshard);
1258 bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false);
1259
1260 arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE);
1261
1262 return slab;
1263}
1264
1265static extent_t *
1266arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
1267 szind_t binind, unsigned binshard) {
1268 extent_t *slab;
1269 const bin_info_t *bin_info;
1270
1271 /* Look for a usable slab. */
1272 slab = arena_bin_slabs_nonfull_tryget(bin);
1273 if (slab != NULL) {
1274 return slab;
1275 }
1276 /* No existing slabs have any space available. */
1277
1278 bin_info = &bin_infos[binind];
1279
1280 /* Allocate a new slab. */
1281 malloc_mutex_unlock(tsdn, &bin->lock);
1282 /******************************/
1283 slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info);
1284 /********************************/
1285 malloc_mutex_lock(tsdn, &bin->lock);
1286 if (slab != NULL) {
1287 if (config_stats) {
1288 bin->stats.nslabs++;
1289 bin->stats.curslabs++;
1290 }
1291 return slab;
1292 }
1293
1294 /*
1295 * arena_slab_alloc() failed, but another thread may have made
1296 * sufficient memory available while this one dropped bin->lock above,
1297 * so search one more time.
1298 */
1299 slab = arena_bin_slabs_nonfull_tryget(bin);
1300 if (slab != NULL) {
1301 return slab;
1302 }
1303
1304 return NULL;
1305}
1306
1307/* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */
1308static void *
1309arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
1310 szind_t binind, unsigned binshard) {
1311 const bin_info_t *bin_info;
1312 extent_t *slab;
1313
1314 bin_info = &bin_infos[binind];
1315 if (!arena_is_auto(arena) && bin->slabcur != NULL) {
1316 arena_bin_slabs_full_insert(arena, bin, bin->slabcur);
1317 bin->slabcur = NULL;
1318 }
1319 slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard);
1320 if (bin->slabcur != NULL) {
1321 /*
1322 * Another thread updated slabcur while this one ran without the
1323 * bin lock in arena_bin_nonfull_slab_get().
1324 */
1325 if (extent_nfree_get(bin->slabcur) > 0) {
1326 void *ret = arena_slab_reg_alloc(bin->slabcur,
1327 bin_info);
1328 if (slab != NULL) {
1329 /*
1330 * arena_slab_alloc() may have allocated slab,
1331 * or it may have been pulled from
1332 * slabs_nonfull. Therefore it is unsafe to
1333 * make any assumptions about how slab has
1334 * previously been used, and
1335 * arena_bin_lower_slab() must be called, as if
1336 * a region were just deallocated from the slab.
1337 */
1338 if (extent_nfree_get(slab) == bin_info->nregs) {
1339 arena_dalloc_bin_slab(tsdn, arena, slab,
1340 bin);
1341 } else {
1342 arena_bin_lower_slab(tsdn, arena, slab,
1343 bin);
1344 }
1345 }
1346 return ret;
1347 }
1348
1349 arena_bin_slabs_full_insert(arena, bin, bin->slabcur);
1350 bin->slabcur = NULL;
1351 }
1352
1353 if (slab == NULL) {
1354 return NULL;
1355 }
1356 bin->slabcur = slab;
1357
1358 assert(extent_nfree_get(bin->slabcur) > 0);
1359
1360 return arena_slab_reg_alloc(slab, bin_info);
1361}
1362
1363/* Choose a bin shard and return the locked bin. */
1364bin_t *
1365arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
1366 unsigned *binshard) {
1367 bin_t *bin;
1368 if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
1369 *binshard = 0;
1370 } else {
1371 *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind];
1372 }
1373 assert(*binshard < bin_infos[binind].n_shards);
1374 bin = &arena->bins[binind].bin_shards[*binshard];
1375 malloc_mutex_lock(tsdn, &bin->lock);
1376
1377 return bin;
1378}
1379
1380void
1381arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
1382 cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) {
1383 unsigned i, nfill, cnt;
1384
1385 assert(tbin->ncached == 0);
1386
1387 if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) {
1388 prof_idump(tsdn);
1389 }
1390
1391 unsigned binshard;
1392 bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard);
1393
1394 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
1395 tcache->lg_fill_div[binind]); i < nfill; i += cnt) {
1396 extent_t *slab;
1397 if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) >
1398 0) {
1399 unsigned tofill = nfill - i;
1400 cnt = tofill < extent_nfree_get(slab) ?
1401 tofill : extent_nfree_get(slab);
1402 arena_slab_reg_alloc_batch(
1403 slab, &bin_infos[binind], cnt,
1404 tbin->avail - nfill + i);
1405 } else {
1406 cnt = 1;
1407 void *ptr = arena_bin_malloc_hard(tsdn, arena, bin,
1408 binind, binshard);
1409 /*
1410 * OOM. tbin->avail isn't yet filled down to its first
1411 * element, so the successful allocations (if any) must
1412 * be moved just before tbin->avail before bailing out.
1413 */
1414 if (ptr == NULL) {
1415 if (i > 0) {
1416 memmove(tbin->avail - i,
1417 tbin->avail - nfill,
1418 i * sizeof(void *));
1419 }
1420 break;
1421 }
1422 /* Insert such that low regions get used first. */
1423 *(tbin->avail - nfill + i) = ptr;
1424 }
1425 if (config_fill && unlikely(opt_junk_alloc)) {
1426 for (unsigned j = 0; j < cnt; j++) {
1427 void* ptr = *(tbin->avail - nfill + i + j);
1428 arena_alloc_junk_small(ptr, &bin_infos[binind],
1429 true);
1430 }
1431 }
1432 }
1433 if (config_stats) {
1434 bin->stats.nmalloc += i;
1435 bin->stats.nrequests += tbin->tstats.nrequests;
1436 bin->stats.curregs += i;
1437 bin->stats.nfills++;
1438 tbin->tstats.nrequests = 0;
1439 }
1440 malloc_mutex_unlock(tsdn, &bin->lock);
1441 tbin->ncached = i;
1442 arena_decay_tick(tsdn, arena);
1443}
1444
1445void
1446arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info, bool zero) {
1447 if (!zero) {
1448 memset(ptr, JEMALLOC_ALLOC_JUNK, bin_info->reg_size);
1449 }
1450}
1451
1452static void
1453arena_dalloc_junk_small_impl(void *ptr, const bin_info_t *bin_info) {
1454 memset(ptr, JEMALLOC_FREE_JUNK, bin_info->reg_size);
1455}
1456arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small =
1457 arena_dalloc_junk_small_impl;
1458
1459static void *
1460arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
1461 void *ret;
1462 bin_t *bin;
1463 size_t usize;
1464 extent_t *slab;
1465
1466 assert(binind < SC_NBINS);
1467 usize = sz_index2size(binind);
1468 unsigned binshard;
1469 bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard);
1470
1471 if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) {
1472 ret = arena_slab_reg_alloc(slab, &bin_infos[binind]);
1473 } else {
1474 ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard);
1475 }
1476
1477 if (ret == NULL) {
1478 malloc_mutex_unlock(tsdn, &bin->lock);
1479 return NULL;
1480 }
1481
1482 if (config_stats) {
1483 bin->stats.nmalloc++;
1484 bin->stats.nrequests++;
1485 bin->stats.curregs++;
1486 }
1487 malloc_mutex_unlock(tsdn, &bin->lock);
1488 if (config_prof && arena_prof_accum(tsdn, arena, usize)) {
1489 prof_idump(tsdn);
1490 }
1491
1492 if (!zero) {
1493 if (config_fill) {
1494 if (unlikely(opt_junk_alloc)) {
1495 arena_alloc_junk_small(ret,
1496 &bin_infos[binind], false);
1497 } else if (unlikely(opt_zero)) {
1498 memset(ret, 0, usize);
1499 }
1500 }
1501 } else {
1502 if (config_fill && unlikely(opt_junk_alloc)) {
1503 arena_alloc_junk_small(ret, &bin_infos[binind],
1504 true);
1505 }
1506 memset(ret, 0, usize);
1507 }
1508
1509 arena_decay_tick(tsdn, arena);
1510 return ret;
1511}
1512
1513void *
1514arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
1515 bool zero) {
1516 assert(!tsdn_null(tsdn) || arena != NULL);
1517
1518 if (likely(!tsdn_null(tsdn))) {
1519 arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size);
1520 }
1521 if (unlikely(arena == NULL)) {
1522 return NULL;
1523 }
1524
1525 if (likely(size <= SC_SMALL_MAXCLASS)) {
1526 return arena_malloc_small(tsdn, arena, ind, zero);
1527 }
1528 return large_malloc(tsdn, arena, sz_index2size(ind), zero);
1529}
1530
1531void *
1532arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
1533 bool zero, tcache_t *tcache) {
1534 void *ret;
1535
1536 if (usize <= SC_SMALL_MAXCLASS
1537 && (alignment < PAGE
1538 || (alignment == PAGE && (usize & PAGE_MASK) == 0))) {
1539 /* Small; alignment doesn't require special slab placement. */
1540 ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize),
1541 zero, tcache, true);
1542 } else {
1543 if (likely(alignment <= CACHELINE)) {
1544 ret = large_malloc(tsdn, arena, usize, zero);
1545 } else {
1546 ret = large_palloc(tsdn, arena, usize, alignment, zero);
1547 }
1548 }
1549 return ret;
1550}
1551
1552void
1553arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) {
1554 cassert(config_prof);
1555 assert(ptr != NULL);
1556 assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);
1557 assert(usize <= SC_SMALL_MAXCLASS);
1558
1559 if (config_opt_safety_checks) {
1560 safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS);
1561 }
1562
1563 rtree_ctx_t rtree_ctx_fallback;
1564 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
1565
1566 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,
1567 (uintptr_t)ptr, true);
1568 arena_t *arena = extent_arena_get(extent);
1569
1570 szind_t szind = sz_size2index(usize);
1571 extent_szind_set(extent, szind);
1572 rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
1573 szind, false);
1574
1575 prof_accum_cancel(tsdn, &arena->prof_accum, usize);
1576
1577 assert(isalloc(tsdn, ptr) == usize);
1578}
1579
1580static size_t
1581arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) {
1582 cassert(config_prof);
1583 assert(ptr != NULL);
1584
1585 extent_szind_set(extent, SC_NBINS);
1586 rtree_ctx_t rtree_ctx_fallback;
1587 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
1588 rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
1589 SC_NBINS, false);
1590
1591 assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);
1592
1593 return SC_LARGE_MINCLASS;
1594}
1595
1596void
1597arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
1598 bool slow_path) {
1599 cassert(config_prof);
1600 assert(opt_prof);
1601
1602 extent_t *extent = iealloc(tsdn, ptr);
1603 size_t usize = extent_usize_get(extent);
1604 size_t bumped_usize = arena_prof_demote(tsdn, extent, ptr);
1605 if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
1606 /*
1607 * Currently, we only do redzoning for small sampled
1608 * allocations.
1609 */
1610 assert(bumped_usize == SC_LARGE_MINCLASS);
1611 safety_check_verify_redzone(ptr, usize, bumped_usize);
1612 }
1613 if (bumped_usize <= tcache_maxclass && tcache != NULL) {
1614 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
1615 sz_size2index(bumped_usize), slow_path);
1616 } else {
1617 large_dalloc(tsdn, extent);
1618 }
1619}
1620
1621static void
1622arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, bin_t *bin) {
1623 /* Dissociate slab from bin. */
1624 if (slab == bin->slabcur) {
1625 bin->slabcur = NULL;
1626 } else {
1627 szind_t binind = extent_szind_get(slab);
1628 const bin_info_t *bin_info = &bin_infos[binind];
1629
1630 /*
1631 * The following block's conditional is necessary because if the
1632 * slab only contains one region, then it never gets inserted
1633 * into the non-full slabs heap.
1634 */
1635 if (bin_info->nregs == 1) {
1636 arena_bin_slabs_full_remove(arena, bin, slab);
1637 } else {
1638 arena_bin_slabs_nonfull_remove(bin, slab);
1639 }
1640 }
1641}
1642
1643static void
1644arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
1645 bin_t *bin) {
1646 assert(slab != bin->slabcur);
1647
1648 malloc_mutex_unlock(tsdn, &bin->lock);
1649 /******************************/
1650 arena_slab_dalloc(tsdn, arena, slab);
1651 /****************************/
1652 malloc_mutex_lock(tsdn, &bin->lock);
1653 if (config_stats) {
1654 bin->stats.curslabs--;
1655 }
1656}
1657
1658static void
1659arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
1660 bin_t *bin) {
1661 assert(extent_nfree_get(slab) > 0);
1662
1663 /*
1664 * Make sure that if bin->slabcur is non-NULL, it refers to the
1665 * oldest/lowest non-full slab. It is okay to NULL slabcur out rather
1666 * than proactively keeping it pointing at the oldest/lowest non-full
1667 * slab.
1668 */
1669 if (bin->slabcur != NULL && extent_snad_comp(bin->slabcur, slab) > 0) {
1670 /* Switch slabcur. */
1671 if (extent_nfree_get(bin->slabcur) > 0) {
1672 arena_bin_slabs_nonfull_insert(bin, bin->slabcur);
1673 } else {
1674 arena_bin_slabs_full_insert(arena, bin, bin->slabcur);
1675 }
1676 bin->slabcur = slab;
1677 if (config_stats) {
1678 bin->stats.reslabs++;
1679 }
1680 } else {
1681 arena_bin_slabs_nonfull_insert(bin, slab);
1682 }
1683}
1684
1685static void
1686arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
1687 szind_t binind, extent_t *slab, void *ptr, bool junked) {
1688 arena_slab_data_t *slab_data = extent_slab_data_get(slab);
1689 const bin_info_t *bin_info = &bin_infos[binind];
1690
1691 if (!junked && config_fill && unlikely(opt_junk_free)) {
1692 arena_dalloc_junk_small(ptr, bin_info);
1693 }
1694
1695 arena_slab_reg_dalloc(slab, slab_data, ptr);
1696 unsigned nfree = extent_nfree_get(slab);
1697 if (nfree == bin_info->nregs) {
1698 arena_dissociate_bin_slab(arena, slab, bin);
1699 arena_dalloc_bin_slab(tsdn, arena, slab, bin);
1700 } else if (nfree == 1 && slab != bin->slabcur) {
1701 arena_bin_slabs_full_remove(arena, bin, slab);
1702 arena_bin_lower_slab(tsdn, arena, slab, bin);
1703 }
1704
1705 if (config_stats) {
1706 bin->stats.ndalloc++;
1707 bin->stats.curregs--;
1708 }
1709}
1710
1711void
1712arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
1713 szind_t binind, extent_t *extent, void *ptr) {
1714 arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr,
1715 true);
1716}
1717
1718static void
1719arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) {
1720 szind_t binind = extent_szind_get(extent);
1721 unsigned binshard = extent_binshard_get(extent);
1722 bin_t *bin = &arena->bins[binind].bin_shards[binshard];
1723
1724 malloc_mutex_lock(tsdn, &bin->lock);
1725 arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr,
1726 false);
1727 malloc_mutex_unlock(tsdn, &bin->lock);
1728}
1729
1730void
1731arena_dalloc_small(tsdn_t *tsdn, void *ptr) {
1732 extent_t *extent = iealloc(tsdn, ptr);
1733 arena_t *arena = extent_arena_get(extent);
1734
1735 arena_dalloc_bin(tsdn, arena, extent, ptr);
1736 arena_decay_tick(tsdn, arena);
1737}
1738
1739bool
1740arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
1741 size_t extra, bool zero, size_t *newsize) {
1742 bool ret;
1743 /* Calls with non-zero extra had to clamp extra. */
1744 assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS);
1745
1746 extent_t *extent = iealloc(tsdn, ptr);
1747 if (unlikely(size > SC_LARGE_MAXCLASS)) {
1748 ret = true;
1749 goto done;
1750 }
1751
1752 size_t usize_min = sz_s2u(size);
1753 size_t usize_max = sz_s2u(size + extra);
1754 if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min
1755 <= SC_SMALL_MAXCLASS)) {
1756 /*
1757 * Avoid moving the allocation if the size class can be left the
1758 * same.
1759 */
1760 assert(bin_infos[sz_size2index(oldsize)].reg_size ==
1761 oldsize);
1762 if ((usize_max > SC_SMALL_MAXCLASS
1763 || sz_size2index(usize_max) != sz_size2index(oldsize))
1764 && (size > oldsize || usize_max < oldsize)) {
1765 ret = true;
1766 goto done;
1767 }
1768
1769 arena_decay_tick(tsdn, extent_arena_get(extent));
1770 ret = false;
1771 } else if (oldsize >= SC_LARGE_MINCLASS
1772 && usize_max >= SC_LARGE_MINCLASS) {
1773 ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max,
1774 zero);
1775 } else {
1776 ret = true;
1777 }
1778done:
1779 assert(extent == iealloc(tsdn, ptr));
1780 *newsize = extent_usize_get(extent);
1781
1782 return ret;
1783}
1784
1785static void *
1786arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
1787 size_t alignment, bool zero, tcache_t *tcache) {
1788 if (alignment == 0) {
1789 return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
1790 zero, tcache, true);
1791 }
1792 usize = sz_sa2u(usize, alignment);
1793 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
1794 return NULL;
1795 }
1796 return ipalloct(tsdn, usize, alignment, zero, tcache, arena);
1797}
1798
1799void *
1800arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
1801 size_t size, size_t alignment, bool zero, tcache_t *tcache,
1802 hook_ralloc_args_t *hook_args) {
1803 size_t usize = sz_s2u(size);
1804 if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
1805 return NULL;
1806 }
1807
1808 if (likely(usize <= SC_SMALL_MAXCLASS)) {
1809 /* Try to avoid moving the allocation. */
1810 UNUSED size_t newsize;
1811 if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero,
1812 &newsize)) {
1813 hook_invoke_expand(hook_args->is_realloc
1814 ? hook_expand_realloc : hook_expand_rallocx,
1815 ptr, oldsize, usize, (uintptr_t)ptr,
1816 hook_args->args);
1817 return ptr;
1818 }
1819 }
1820
1821 if (oldsize >= SC_LARGE_MINCLASS
1822 && usize >= SC_LARGE_MINCLASS) {
1823 return large_ralloc(tsdn, arena, ptr, usize,
1824 alignment, zero, tcache, hook_args);
1825 }
1826
1827 /*
1828 * size and oldsize are different enough that we need to move the
1829 * object. In that case, fall back to allocating new space and copying.
1830 */
1831 void *ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment,
1832 zero, tcache);
1833 if (ret == NULL) {
1834 return NULL;
1835 }
1836
1837 hook_invoke_alloc(hook_args->is_realloc
1838 ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
1839 hook_args->args);
1840 hook_invoke_dalloc(hook_args->is_realloc
1841 ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
1842
1843 /*
1844 * Junk/zero-filling were already done by
1845 * ipalloc()/arena_malloc().
1846 */
1847 size_t copysize = (usize < oldsize) ? usize : oldsize;
1848 memcpy(ret, ptr, copysize);
1849 isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
1850 return ret;
1851}
1852
1853dss_prec_t
1854arena_dss_prec_get(arena_t *arena) {
1855 return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
1856}
1857
1858bool
1859arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
1860 if (!have_dss) {
1861 return (dss_prec != dss_prec_disabled);
1862 }
1863 atomic_store_u(&arena->dss_prec, (unsigned)dss_prec, ATOMIC_RELEASE);
1864 return false;
1865}
1866
1867ssize_t
1868arena_dirty_decay_ms_default_get(void) {
1869 return atomic_load_zd(&dirty_decay_ms_default, ATOMIC_RELAXED);
1870}
1871
1872bool
1873arena_dirty_decay_ms_default_set(ssize_t decay_ms) {
1874 if (!arena_decay_ms_valid(decay_ms)) {
1875 return true;
1876 }
1877 atomic_store_zd(&dirty_decay_ms_default, decay_ms, ATOMIC_RELAXED);
1878 return false;
1879}
1880
1881ssize_t
1882arena_muzzy_decay_ms_default_get(void) {
1883 return atomic_load_zd(&muzzy_decay_ms_default, ATOMIC_RELAXED);
1884}
1885
1886bool
1887arena_muzzy_decay_ms_default_set(ssize_t decay_ms) {
1888 if (!arena_decay_ms_valid(decay_ms)) {
1889 return true;
1890 }
1891 atomic_store_zd(&muzzy_decay_ms_default, decay_ms, ATOMIC_RELAXED);
1892 return false;
1893}
1894
1895bool
1896arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
1897 size_t *new_limit) {
1898 assert(opt_retain);
1899
1900 pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0);
1901 if (new_limit != NULL) {
1902 size_t limit = *new_limit;
1903 /* Grow no more than the new limit. */
1904 if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) {
1905 return true;
1906 }
1907 }
1908
1909 malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
1910 if (old_limit != NULL) {
1911 *old_limit = sz_pind2sz(arena->retain_grow_limit);
1912 }
1913 if (new_limit != NULL) {
1914 arena->retain_grow_limit = new_ind;
1915 }
1916 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
1917
1918 return false;
1919}
1920
1921unsigned
1922arena_nthreads_get(arena_t *arena, bool internal) {
1923 return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
1924}
1925
1926void
1927arena_nthreads_inc(arena_t *arena, bool internal) {
1928 atomic_fetch_add_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
1929}
1930
1931void
1932arena_nthreads_dec(arena_t *arena, bool internal) {
1933 atomic_fetch_sub_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
1934}
1935
1936size_t
1937arena_extent_sn_next(arena_t *arena) {
1938 return atomic_fetch_add_zu(&arena->extent_sn_next, 1, ATOMIC_RELAXED);
1939}
1940
1941arena_t *
1942arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
1943 arena_t *arena;
1944 base_t *base;
1945 unsigned i;
1946
1947 if (ind == 0) {
1948 base = b0get();
1949 } else {
1950 base = base_new(tsdn, ind, extent_hooks);
1951 if (base == NULL) {
1952 return NULL;
1953 }
1954 }
1955
1956 unsigned nbins_total = 0;
1957 for (i = 0; i < SC_NBINS; i++) {
1958 nbins_total += bin_infos[i].n_shards;
1959 }
1960 size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total;
1961 arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE);
1962 if (arena == NULL) {
1963 goto label_error;
1964 }
1965
1966 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
1967 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
1968 arena->last_thd = NULL;
1969
1970 if (config_stats) {
1971 if (arena_stats_init(tsdn, &arena->stats)) {
1972 goto label_error;
1973 }
1974
1975 ql_new(&arena->tcache_ql);
1976 ql_new(&arena->cache_bin_array_descriptor_ql);
1977 if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
1978 WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
1979 goto label_error;
1980 }
1981 }
1982
1983 if (config_prof) {
1984 if (prof_accum_init(tsdn, &arena->prof_accum)) {
1985 goto label_error;
1986 }
1987 }
1988
1989 if (config_cache_oblivious) {
1990 /*
1991 * A nondeterministic seed based on the address of arena reduces
1992 * the likelihood of lockstep non-uniform cache index
1993 * utilization among identical concurrent processes, but at the
1994 * cost of test repeatability. For debug builds, instead use a
1995 * deterministic seed.
1996 */
1997 atomic_store_zu(&arena->offset_state, config_debug ? ind :
1998 (size_t)(uintptr_t)arena, ATOMIC_RELAXED);
1999 }
2000
2001 atomic_store_zu(&arena->extent_sn_next, 0, ATOMIC_RELAXED);
2002
2003 atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(),
2004 ATOMIC_RELAXED);
2005
2006 atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED);
2007
2008 extent_list_init(&arena->large);
2009 if (malloc_mutex_init(&arena->large_mtx, "arena_large",
2010 WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) {
2011 goto label_error;
2012 }
2013
2014 /*
2015 * Delay coalescing for dirty extents despite the disruptive effect on
2016 * memory layout for best-fit extent allocation, since cached extents
2017 * are likely to be reused soon after deallocation, and the cost of
2018 * merging/splitting extents is non-trivial.
2019 */
2020 if (extents_init(tsdn, &arena->extents_dirty, extent_state_dirty,
2021 true)) {
2022 goto label_error;
2023 }
2024 /*
2025 * Coalesce muzzy extents immediately, because operations on them are in
2026 * the critical path much less often than for dirty extents.
2027 */
2028 if (extents_init(tsdn, &arena->extents_muzzy, extent_state_muzzy,
2029 false)) {
2030 goto label_error;
2031 }
2032 /*
2033 * Coalesce retained extents immediately, in part because they will
2034 * never be evicted (and therefore there's no opportunity for delayed
2035 * coalescing), but also because operations on retained extents are not
2036 * in the critical path.
2037 */
2038 if (extents_init(tsdn, &arena->extents_retained, extent_state_retained,
2039 false)) {
2040 goto label_error;
2041 }
2042
2043 if (arena_decay_init(&arena->decay_dirty,
2044 arena_dirty_decay_ms_default_get(), &arena->stats.decay_dirty)) {
2045 goto label_error;
2046 }
2047 if (arena_decay_init(&arena->decay_muzzy,
2048 arena_muzzy_decay_ms_default_get(), &arena->stats.decay_muzzy)) {
2049 goto label_error;
2050 }
2051
2052 arena->extent_grow_next = sz_psz2ind(HUGEPAGE);
2053 arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS);
2054 if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
2055 WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
2056 goto label_error;
2057 }
2058
2059 extent_avail_new(&arena->extent_avail);
2060 if (malloc_mutex_init(&arena->extent_avail_mtx, "extent_avail",
2061 WITNESS_RANK_EXTENT_AVAIL, malloc_mutex_rank_exclusive)) {
2062 goto label_error;
2063 }
2064
2065 /* Initialize bins. */
2066 uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t);
2067 atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE);
2068 for (i = 0; i < SC_NBINS; i++) {
2069 unsigned nshards = bin_infos[i].n_shards;
2070 arena->bins[i].bin_shards = (bin_t *)bin_addr;
2071 bin_addr += nshards * sizeof(bin_t);
2072 for (unsigned j = 0; j < nshards; j++) {
2073 bool err = bin_init(&arena->bins[i].bin_shards[j]);
2074 if (err) {
2075 goto label_error;
2076 }
2077 }
2078 }
2079 assert(bin_addr == (uintptr_t)arena + arena_size);
2080
2081 arena->base = base;
2082 /* Set arena before creating background threads. */
2083 arena_set(ind, arena);
2084
2085 nstime_init(&arena->create_time, 0);
2086 nstime_update(&arena->create_time);
2087
2088 /* We don't support reentrancy for arena 0 bootstrapping. */
2089 if (ind != 0) {
2090 /*
2091 * If we're here, then arena 0 already exists, so bootstrapping
2092 * is done enough that we should have tsd.
2093 */
2094 assert(!tsdn_null(tsdn));
2095 pre_reentrancy(tsdn_tsd(tsdn), arena);
2096 if (test_hooks_arena_new_hook) {
2097 test_hooks_arena_new_hook();
2098 }
2099 post_reentrancy(tsdn_tsd(tsdn));
2100 }
2101
2102 return arena;
2103label_error:
2104 if (ind != 0) {
2105 base_delete(tsdn, base);
2106 }
2107 return NULL;
2108}
2109
2110arena_t *
2111arena_choose_huge(tsd_t *tsd) {
2112 /* huge_arena_ind can be 0 during init (will use a0). */
2113 if (huge_arena_ind == 0) {
2114 assert(!malloc_initialized());
2115 }
2116
2117 arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false);
2118 if (huge_arena == NULL) {
2119 /* Create the huge arena on demand. */
2120 assert(huge_arena_ind != 0);
2121 huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true);
2122 if (huge_arena == NULL) {
2123 return NULL;
2124 }
2125 /*
2126 * Purge eagerly for huge allocations, because: 1) number of
2127 * huge allocations is usually small, which means ticker based
2128 * decay is not reliable; and 2) less immediate reuse is
2129 * expected for huge allocations.
2130 */
2131 if (arena_dirty_decay_ms_default_get() > 0) {
2132 arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0);
2133 }
2134 if (arena_muzzy_decay_ms_default_get() > 0) {
2135 arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0);
2136 }
2137 }
2138
2139 return huge_arena;
2140}
2141
2142bool
2143arena_init_huge(void) {
2144 bool huge_enabled;
2145
2146 /* The threshold should be large size class. */
2147 if (opt_oversize_threshold > SC_LARGE_MAXCLASS ||
2148 opt_oversize_threshold < SC_LARGE_MINCLASS) {
2149 opt_oversize_threshold = 0;
2150 oversize_threshold = SC_LARGE_MAXCLASS + PAGE;
2151 huge_enabled = false;
2152 } else {
2153 /* Reserve the index for the huge arena. */
2154 huge_arena_ind = narenas_total_get();
2155 oversize_threshold = opt_oversize_threshold;
2156 huge_enabled = true;
2157 }
2158
2159 return huge_enabled;
2160}
2161
2162bool
2163arena_is_huge(unsigned arena_ind) {
2164 if (huge_arena_ind == 0) {
2165 return false;
2166 }
2167 return (arena_ind == huge_arena_ind);
2168}
2169
2170void
2171arena_boot(sc_data_t *sc_data) {
2172 arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
2173 arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms);
2174 for (unsigned i = 0; i < SC_NBINS; i++) {
2175 sc_t *sc = &sc_data->sc[i];
2176 div_init(&arena_binind_div_info[i],
2177 (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta));
2178 }
2179}
2180
2181void
2182arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
2183 malloc_mutex_prefork(tsdn, &arena->decay_dirty.mtx);
2184 malloc_mutex_prefork(tsdn, &arena->decay_muzzy.mtx);
2185}
2186
2187void
2188arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
2189 if (config_stats) {
2190 malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
2191 }
2192}
2193
2194void
2195arena_prefork2(tsdn_t *tsdn, arena_t *arena) {
2196 malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx);
2197}
2198
2199void
2200arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
2201 extents_prefork(tsdn, &arena->extents_dirty);
2202 extents_prefork(tsdn, &arena->extents_muzzy);
2203 extents_prefork(tsdn, &arena->extents_retained);
2204}
2205
2206void
2207arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
2208 malloc_mutex_prefork(tsdn, &arena->extent_avail_mtx);
2209}
2210
2211void
2212arena_prefork5(tsdn_t *tsdn, arena_t *arena) {
2213 base_prefork(tsdn, arena->base);
2214}
2215
2216void
2217arena_prefork6(tsdn_t *tsdn, arena_t *arena) {
2218 malloc_mutex_prefork(tsdn, &arena->large_mtx);
2219}
2220
2221void
2222arena_prefork7(tsdn_t *tsdn, arena_t *arena) {
2223 for (unsigned i = 0; i < SC_NBINS; i++) {
2224 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
2225 bin_prefork(tsdn, &arena->bins[i].bin_shards[j]);
2226 }
2227 }
2228}
2229
2230void
2231arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
2232 unsigned i;
2233
2234 for (i = 0; i < SC_NBINS; i++) {
2235 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
2236 bin_postfork_parent(tsdn,
2237 &arena->bins[i].bin_shards[j]);
2238 }
2239 }
2240 malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
2241 base_postfork_parent(tsdn, arena->base);
2242 malloc_mutex_postfork_parent(tsdn, &arena->extent_avail_mtx);
2243 extents_postfork_parent(tsdn, &arena->extents_dirty);
2244 extents_postfork_parent(tsdn, &arena->extents_muzzy);
2245 extents_postfork_parent(tsdn, &arena->extents_retained);
2246 malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx);
2247 malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx);
2248 malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx);
2249 if (config_stats) {
2250 malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
2251 }
2252}
2253
2254void
2255arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
2256 unsigned i;
2257
2258 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
2259 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
2260 if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) {
2261 arena_nthreads_inc(arena, false);
2262 }
2263 if (tsd_iarena_get(tsdn_tsd(tsdn)) == arena) {
2264 arena_nthreads_inc(arena, true);
2265 }
2266 if (config_stats) {
2267 ql_new(&arena->tcache_ql);
2268 ql_new(&arena->cache_bin_array_descriptor_ql);
2269 tcache_t *tcache = tcache_get(tsdn_tsd(tsdn));
2270 if (tcache != NULL && tcache->arena == arena) {
2271 ql_elm_new(tcache, link);
2272 ql_tail_insert(&arena->tcache_ql, tcache, link);
2273 cache_bin_array_descriptor_init(
2274 &tcache->cache_bin_array_descriptor,
2275 tcache->bins_small, tcache->bins_large);
2276 ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
2277 &tcache->cache_bin_array_descriptor, link);
2278 }
2279 }
2280
2281 for (i = 0; i < SC_NBINS; i++) {
2282 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
2283 bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]);
2284 }
2285 }
2286 malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
2287 base_postfork_child(tsdn, arena->base);
2288 malloc_mutex_postfork_child(tsdn, &arena->extent_avail_mtx);
2289 extents_postfork_child(tsdn, &arena->extents_dirty);
2290 extents_postfork_child(tsdn, &arena->extents_muzzy);
2291 extents_postfork_child(tsdn, &arena->extents_retained);
2292 malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx);
2293 malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx);
2294 malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx);
2295 if (config_stats) {
2296 malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
2297 }
2298}
2299