1 | #define JEMALLOC_ARENA_C_ |
2 | #include "jemalloc/internal/jemalloc_preamble.h" |
3 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
4 | |
5 | #include "jemalloc/internal/assert.h" |
6 | #include "jemalloc/internal/div.h" |
7 | #include "jemalloc/internal/extent_dss.h" |
8 | #include "jemalloc/internal/extent_mmap.h" |
9 | #include "jemalloc/internal/mutex.h" |
10 | #include "jemalloc/internal/rtree.h" |
11 | #include "jemalloc/internal/safety_check.h" |
12 | #include "jemalloc/internal/util.h" |
13 | |
14 | JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS |
15 | |
16 | /******************************************************************************/ |
17 | /* Data. */ |
18 | |
19 | /* |
20 | * Define names for both unininitialized and initialized phases, so that |
21 | * options and mallctl processing are straightforward. |
22 | */ |
23 | const char *percpu_arena_mode_names[] = { |
24 | "percpu" , |
25 | "phycpu" , |
26 | "disabled" , |
27 | "percpu" , |
28 | "phycpu" |
29 | }; |
30 | percpu_arena_mode_t opt_percpu_arena = PERCPU_ARENA_DEFAULT; |
31 | |
32 | ssize_t opt_dirty_decay_ms = DIRTY_DECAY_MS_DEFAULT; |
33 | ssize_t opt_muzzy_decay_ms = MUZZY_DECAY_MS_DEFAULT; |
34 | |
35 | static atomic_zd_t dirty_decay_ms_default; |
36 | static atomic_zd_t muzzy_decay_ms_default; |
37 | |
38 | const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { |
39 | #define STEP(step, h, x, y) \ |
40 | h, |
41 | SMOOTHSTEP |
42 | #undef STEP |
43 | }; |
44 | |
45 | static div_info_t arena_binind_div_info[SC_NBINS]; |
46 | |
47 | size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; |
48 | size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; |
49 | static unsigned huge_arena_ind; |
50 | |
51 | /******************************************************************************/ |
52 | /* |
53 | * Function prototypes for static functions that are referenced prior to |
54 | * definition. |
55 | */ |
56 | |
57 | static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, |
58 | arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit, |
59 | size_t npages_decay_max, bool is_background_thread); |
60 | static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, |
61 | bool is_background_thread, bool all); |
62 | static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, |
63 | bin_t *bin); |
64 | static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, |
65 | bin_t *bin); |
66 | |
67 | /******************************************************************************/ |
68 | |
69 | void |
70 | arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, |
71 | const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, |
72 | size_t *nactive, size_t *ndirty, size_t *nmuzzy) { |
73 | *nthreads += arena_nthreads_get(arena, false); |
74 | *dss = dss_prec_names[arena_dss_prec_get(arena)]; |
75 | *dirty_decay_ms = arena_dirty_decay_ms_get(arena); |
76 | *muzzy_decay_ms = arena_muzzy_decay_ms_get(arena); |
77 | *nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED); |
78 | *ndirty += extents_npages_get(&arena->extents_dirty); |
79 | *nmuzzy += extents_npages_get(&arena->extents_muzzy); |
80 | } |
81 | |
82 | void |
83 | arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, |
84 | const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, |
85 | size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, |
86 | bin_stats_t *bstats, arena_stats_large_t *lstats, |
87 | arena_stats_extents_t *estats) { |
88 | cassert(config_stats); |
89 | |
90 | arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, |
91 | muzzy_decay_ms, nactive, ndirty, nmuzzy); |
92 | |
93 | size_t base_allocated, base_resident, base_mapped, metadata_thp; |
94 | base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, |
95 | &base_mapped, &metadata_thp); |
96 | |
97 | arena_stats_lock(tsdn, &arena->stats); |
98 | |
99 | arena_stats_accum_zu(&astats->mapped, base_mapped |
100 | + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped)); |
101 | arena_stats_accum_zu(&astats->retained, |
102 | extents_npages_get(&arena->extents_retained) << LG_PAGE); |
103 | |
104 | atomic_store_zu(&astats->extent_avail, |
105 | atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED), |
106 | ATOMIC_RELAXED); |
107 | |
108 | arena_stats_accum_u64(&astats->decay_dirty.npurge, |
109 | arena_stats_read_u64(tsdn, &arena->stats, |
110 | &arena->stats.decay_dirty.npurge)); |
111 | arena_stats_accum_u64(&astats->decay_dirty.nmadvise, |
112 | arena_stats_read_u64(tsdn, &arena->stats, |
113 | &arena->stats.decay_dirty.nmadvise)); |
114 | arena_stats_accum_u64(&astats->decay_dirty.purged, |
115 | arena_stats_read_u64(tsdn, &arena->stats, |
116 | &arena->stats.decay_dirty.purged)); |
117 | |
118 | arena_stats_accum_u64(&astats->decay_muzzy.npurge, |
119 | arena_stats_read_u64(tsdn, &arena->stats, |
120 | &arena->stats.decay_muzzy.npurge)); |
121 | arena_stats_accum_u64(&astats->decay_muzzy.nmadvise, |
122 | arena_stats_read_u64(tsdn, &arena->stats, |
123 | &arena->stats.decay_muzzy.nmadvise)); |
124 | arena_stats_accum_u64(&astats->decay_muzzy.purged, |
125 | arena_stats_read_u64(tsdn, &arena->stats, |
126 | &arena->stats.decay_muzzy.purged)); |
127 | |
128 | arena_stats_accum_zu(&astats->base, base_allocated); |
129 | arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); |
130 | arena_stats_accum_zu(&astats->metadata_thp, metadata_thp); |
131 | arena_stats_accum_zu(&astats->resident, base_resident + |
132 | (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + |
133 | extents_npages_get(&arena->extents_dirty) + |
134 | extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); |
135 | arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu( |
136 | &arena->stats.abandoned_vm, ATOMIC_RELAXED)); |
137 | |
138 | for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { |
139 | uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, |
140 | &arena->stats.lstats[i].nmalloc); |
141 | arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc); |
142 | arena_stats_accum_u64(&astats->nmalloc_large, nmalloc); |
143 | |
144 | uint64_t ndalloc = arena_stats_read_u64(tsdn, &arena->stats, |
145 | &arena->stats.lstats[i].ndalloc); |
146 | arena_stats_accum_u64(&lstats[i].ndalloc, ndalloc); |
147 | arena_stats_accum_u64(&astats->ndalloc_large, ndalloc); |
148 | |
149 | uint64_t nrequests = arena_stats_read_u64(tsdn, &arena->stats, |
150 | &arena->stats.lstats[i].nrequests); |
151 | arena_stats_accum_u64(&lstats[i].nrequests, |
152 | nmalloc + nrequests); |
153 | arena_stats_accum_u64(&astats->nrequests_large, |
154 | nmalloc + nrequests); |
155 | |
156 | /* nfill == nmalloc for large currently. */ |
157 | arena_stats_accum_u64(&lstats[i].nfills, nmalloc); |
158 | arena_stats_accum_u64(&astats->nfills_large, nmalloc); |
159 | |
160 | uint64_t nflush = arena_stats_read_u64(tsdn, &arena->stats, |
161 | &arena->stats.lstats[i].nflushes); |
162 | arena_stats_accum_u64(&lstats[i].nflushes, nflush); |
163 | arena_stats_accum_u64(&astats->nflushes_large, nflush); |
164 | |
165 | assert(nmalloc >= ndalloc); |
166 | assert(nmalloc - ndalloc <= SIZE_T_MAX); |
167 | size_t curlextents = (size_t)(nmalloc - ndalloc); |
168 | lstats[i].curlextents += curlextents; |
169 | arena_stats_accum_zu(&astats->allocated_large, |
170 | curlextents * sz_index2size(SC_NBINS + i)); |
171 | } |
172 | |
173 | for (pszind_t i = 0; i < SC_NPSIZES; i++) { |
174 | size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, |
175 | retained_bytes; |
176 | dirty = extents_nextents_get(&arena->extents_dirty, i); |
177 | muzzy = extents_nextents_get(&arena->extents_muzzy, i); |
178 | retained = extents_nextents_get(&arena->extents_retained, i); |
179 | dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i); |
180 | muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i); |
181 | retained_bytes = |
182 | extents_nbytes_get(&arena->extents_retained, i); |
183 | |
184 | atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); |
185 | atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); |
186 | atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED); |
187 | atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes, |
188 | ATOMIC_RELAXED); |
189 | atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes, |
190 | ATOMIC_RELAXED); |
191 | atomic_store_zu(&estats[i].retained_bytes, retained_bytes, |
192 | ATOMIC_RELAXED); |
193 | } |
194 | |
195 | arena_stats_unlock(tsdn, &arena->stats); |
196 | |
197 | /* tcache_bytes counts currently cached bytes. */ |
198 | atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED); |
199 | malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); |
200 | cache_bin_array_descriptor_t *descriptor; |
201 | ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { |
202 | szind_t i = 0; |
203 | for (; i < SC_NBINS; i++) { |
204 | cache_bin_t *tbin = &descriptor->bins_small[i]; |
205 | arena_stats_accum_zu(&astats->tcache_bytes, |
206 | tbin->ncached * sz_index2size(i)); |
207 | } |
208 | for (; i < nhbins; i++) { |
209 | cache_bin_t *tbin = &descriptor->bins_large[i]; |
210 | arena_stats_accum_zu(&astats->tcache_bytes, |
211 | tbin->ncached * sz_index2size(i)); |
212 | } |
213 | } |
214 | malloc_mutex_prof_read(tsdn, |
215 | &astats->mutex_prof_data[arena_prof_mutex_tcache_list], |
216 | &arena->tcache_ql_mtx); |
217 | malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); |
218 | |
219 | #define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \ |
220 | malloc_mutex_lock(tsdn, &arena->mtx); \ |
221 | malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \ |
222 | &arena->mtx); \ |
223 | malloc_mutex_unlock(tsdn, &arena->mtx); |
224 | |
225 | /* Gather per arena mutex profiling data. */ |
226 | READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large); |
227 | READ_ARENA_MUTEX_PROF_DATA(extent_avail_mtx, |
228 | arena_prof_mutex_extent_avail) |
229 | READ_ARENA_MUTEX_PROF_DATA(extents_dirty.mtx, |
230 | arena_prof_mutex_extents_dirty) |
231 | READ_ARENA_MUTEX_PROF_DATA(extents_muzzy.mtx, |
232 | arena_prof_mutex_extents_muzzy) |
233 | READ_ARENA_MUTEX_PROF_DATA(extents_retained.mtx, |
234 | arena_prof_mutex_extents_retained) |
235 | READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx, |
236 | arena_prof_mutex_decay_dirty) |
237 | READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx, |
238 | arena_prof_mutex_decay_muzzy) |
239 | READ_ARENA_MUTEX_PROF_DATA(base->mtx, |
240 | arena_prof_mutex_base) |
241 | #undef READ_ARENA_MUTEX_PROF_DATA |
242 | |
243 | nstime_copy(&astats->uptime, &arena->create_time); |
244 | nstime_update(&astats->uptime); |
245 | nstime_subtract(&astats->uptime, &arena->create_time); |
246 | |
247 | for (szind_t i = 0; i < SC_NBINS; i++) { |
248 | for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { |
249 | bin_stats_merge(tsdn, &bstats[i], |
250 | &arena->bins[i].bin_shards[j]); |
251 | } |
252 | } |
253 | } |
254 | |
255 | void |
256 | arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, |
257 | extent_hooks_t **r_extent_hooks, extent_t *extent) { |
258 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
259 | WITNESS_RANK_CORE, 0); |
260 | |
261 | extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty, |
262 | extent); |
263 | if (arena_dirty_decay_ms_get(arena) == 0) { |
264 | arena_decay_dirty(tsdn, arena, false, true); |
265 | } else { |
266 | arena_background_thread_inactivity_check(tsdn, arena, false); |
267 | } |
268 | } |
269 | |
270 | static void * |
271 | arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) { |
272 | void *ret; |
273 | arena_slab_data_t *slab_data = extent_slab_data_get(slab); |
274 | size_t regind; |
275 | |
276 | assert(extent_nfree_get(slab) > 0); |
277 | assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); |
278 | |
279 | regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info); |
280 | ret = (void *)((uintptr_t)extent_addr_get(slab) + |
281 | (uintptr_t)(bin_info->reg_size * regind)); |
282 | extent_nfree_dec(slab); |
283 | return ret; |
284 | } |
285 | |
286 | static void |
287 | arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info, |
288 | unsigned cnt, void** ptrs) { |
289 | arena_slab_data_t *slab_data = extent_slab_data_get(slab); |
290 | |
291 | assert(extent_nfree_get(slab) >= cnt); |
292 | assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); |
293 | |
294 | #if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) |
295 | for (unsigned i = 0; i < cnt; i++) { |
296 | size_t regind = bitmap_sfu(slab_data->bitmap, |
297 | &bin_info->bitmap_info); |
298 | *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) + |
299 | (uintptr_t)(bin_info->reg_size * regind)); |
300 | } |
301 | #else |
302 | unsigned group = 0; |
303 | bitmap_t g = slab_data->bitmap[group]; |
304 | unsigned i = 0; |
305 | while (i < cnt) { |
306 | while (g == 0) { |
307 | g = slab_data->bitmap[++group]; |
308 | } |
309 | size_t shift = group << LG_BITMAP_GROUP_NBITS; |
310 | size_t pop = popcount_lu(g); |
311 | if (pop > (cnt - i)) { |
312 | pop = cnt - i; |
313 | } |
314 | |
315 | /* |
316 | * Load from memory locations only once, outside the |
317 | * hot loop below. |
318 | */ |
319 | uintptr_t base = (uintptr_t)extent_addr_get(slab); |
320 | uintptr_t regsize = (uintptr_t)bin_info->reg_size; |
321 | while (pop--) { |
322 | size_t bit = cfs_lu(&g); |
323 | size_t regind = shift + bit; |
324 | *(ptrs + i) = (void *)(base + regsize * regind); |
325 | |
326 | i++; |
327 | } |
328 | slab_data->bitmap[group] = g; |
329 | } |
330 | #endif |
331 | extent_nfree_sub(slab, cnt); |
332 | } |
333 | |
334 | #ifndef JEMALLOC_JET |
335 | static |
336 | #endif |
337 | size_t |
338 | arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr) { |
339 | size_t diff, regind; |
340 | |
341 | /* Freeing a pointer outside the slab can cause assertion failure. */ |
342 | assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab)); |
343 | assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab)); |
344 | /* Freeing an interior pointer can cause assertion failure. */ |
345 | assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) % |
346 | (uintptr_t)bin_infos[binind].reg_size == 0); |
347 | |
348 | diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)); |
349 | |
350 | /* Avoid doing division with a variable divisor. */ |
351 | regind = div_compute(&arena_binind_div_info[binind], diff); |
352 | |
353 | assert(regind < bin_infos[binind].nregs); |
354 | |
355 | return regind; |
356 | } |
357 | |
358 | static void |
359 | arena_slab_reg_dalloc(extent_t *slab, arena_slab_data_t *slab_data, void *ptr) { |
360 | szind_t binind = extent_szind_get(slab); |
361 | const bin_info_t *bin_info = &bin_infos[binind]; |
362 | size_t regind = arena_slab_regind(slab, binind, ptr); |
363 | |
364 | assert(extent_nfree_get(slab) < bin_info->nregs); |
365 | /* Freeing an unallocated pointer can cause assertion failure. */ |
366 | assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind)); |
367 | |
368 | bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind); |
369 | extent_nfree_inc(slab); |
370 | } |
371 | |
372 | static void |
373 | arena_nactive_add(arena_t *arena, size_t add_pages) { |
374 | atomic_fetch_add_zu(&arena->nactive, add_pages, ATOMIC_RELAXED); |
375 | } |
376 | |
377 | static void |
378 | arena_nactive_sub(arena_t *arena, size_t sub_pages) { |
379 | assert(atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) >= sub_pages); |
380 | atomic_fetch_sub_zu(&arena->nactive, sub_pages, ATOMIC_RELAXED); |
381 | } |
382 | |
383 | static void |
384 | arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { |
385 | szind_t index, hindex; |
386 | |
387 | cassert(config_stats); |
388 | |
389 | if (usize < SC_LARGE_MINCLASS) { |
390 | usize = SC_LARGE_MINCLASS; |
391 | } |
392 | index = sz_size2index(usize); |
393 | hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; |
394 | |
395 | arena_stats_add_u64(tsdn, &arena->stats, |
396 | &arena->stats.lstats[hindex].nmalloc, 1); |
397 | } |
398 | |
399 | static void |
400 | arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { |
401 | szind_t index, hindex; |
402 | |
403 | cassert(config_stats); |
404 | |
405 | if (usize < SC_LARGE_MINCLASS) { |
406 | usize = SC_LARGE_MINCLASS; |
407 | } |
408 | index = sz_size2index(usize); |
409 | hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; |
410 | |
411 | arena_stats_add_u64(tsdn, &arena->stats, |
412 | &arena->stats.lstats[hindex].ndalloc, 1); |
413 | } |
414 | |
415 | static void |
416 | arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, |
417 | size_t usize) { |
418 | arena_large_dalloc_stats_update(tsdn, arena, oldusize); |
419 | arena_large_malloc_stats_update(tsdn, arena, usize); |
420 | } |
421 | |
422 | static bool |
423 | arena_may_have_muzzy(arena_t *arena) { |
424 | return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0)); |
425 | } |
426 | |
427 | extent_t * |
428 | arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, |
429 | size_t alignment, bool *zero) { |
430 | extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; |
431 | |
432 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
433 | WITNESS_RANK_CORE, 0); |
434 | |
435 | szind_t szind = sz_size2index(usize); |
436 | size_t mapped_add; |
437 | bool commit = true; |
438 | extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, |
439 | &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, |
440 | szind, zero, &commit); |
441 | if (extent == NULL && arena_may_have_muzzy(arena)) { |
442 | extent = extents_alloc(tsdn, arena, &extent_hooks, |
443 | &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, |
444 | false, szind, zero, &commit); |
445 | } |
446 | size_t size = usize + sz_large_pad; |
447 | if (extent == NULL) { |
448 | extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL, |
449 | usize, sz_large_pad, alignment, false, szind, zero, |
450 | &commit); |
451 | if (config_stats) { |
452 | /* |
453 | * extent may be NULL on OOM, but in that case |
454 | * mapped_add isn't used below, so there's no need to |
455 | * conditionlly set it to 0 here. |
456 | */ |
457 | mapped_add = size; |
458 | } |
459 | } else if (config_stats) { |
460 | mapped_add = 0; |
461 | } |
462 | |
463 | if (extent != NULL) { |
464 | if (config_stats) { |
465 | arena_stats_lock(tsdn, &arena->stats); |
466 | arena_large_malloc_stats_update(tsdn, arena, usize); |
467 | if (mapped_add != 0) { |
468 | arena_stats_add_zu(tsdn, &arena->stats, |
469 | &arena->stats.mapped, mapped_add); |
470 | } |
471 | arena_stats_unlock(tsdn, &arena->stats); |
472 | } |
473 | arena_nactive_add(arena, size >> LG_PAGE); |
474 | } |
475 | |
476 | return extent; |
477 | } |
478 | |
479 | void |
480 | arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { |
481 | if (config_stats) { |
482 | arena_stats_lock(tsdn, &arena->stats); |
483 | arena_large_dalloc_stats_update(tsdn, arena, |
484 | extent_usize_get(extent)); |
485 | arena_stats_unlock(tsdn, &arena->stats); |
486 | } |
487 | arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); |
488 | } |
489 | |
490 | void |
491 | arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, |
492 | size_t oldusize) { |
493 | size_t usize = extent_usize_get(extent); |
494 | size_t udiff = oldusize - usize; |
495 | |
496 | if (config_stats) { |
497 | arena_stats_lock(tsdn, &arena->stats); |
498 | arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); |
499 | arena_stats_unlock(tsdn, &arena->stats); |
500 | } |
501 | arena_nactive_sub(arena, udiff >> LG_PAGE); |
502 | } |
503 | |
504 | void |
505 | arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, |
506 | size_t oldusize) { |
507 | size_t usize = extent_usize_get(extent); |
508 | size_t udiff = usize - oldusize; |
509 | |
510 | if (config_stats) { |
511 | arena_stats_lock(tsdn, &arena->stats); |
512 | arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); |
513 | arena_stats_unlock(tsdn, &arena->stats); |
514 | } |
515 | arena_nactive_add(arena, udiff >> LG_PAGE); |
516 | } |
517 | |
518 | static ssize_t |
519 | arena_decay_ms_read(arena_decay_t *decay) { |
520 | return atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED); |
521 | } |
522 | |
523 | static void |
524 | arena_decay_ms_write(arena_decay_t *decay, ssize_t decay_ms) { |
525 | atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED); |
526 | } |
527 | |
528 | static void |
529 | arena_decay_deadline_init(arena_decay_t *decay) { |
530 | /* |
531 | * Generate a new deadline that is uniformly random within the next |
532 | * epoch after the current one. |
533 | */ |
534 | nstime_copy(&decay->deadline, &decay->epoch); |
535 | nstime_add(&decay->deadline, &decay->interval); |
536 | if (arena_decay_ms_read(decay) > 0) { |
537 | nstime_t jitter; |
538 | |
539 | nstime_init(&jitter, prng_range_u64(&decay->jitter_state, |
540 | nstime_ns(&decay->interval))); |
541 | nstime_add(&decay->deadline, &jitter); |
542 | } |
543 | } |
544 | |
545 | static bool |
546 | arena_decay_deadline_reached(const arena_decay_t *decay, const nstime_t *time) { |
547 | return (nstime_compare(&decay->deadline, time) <= 0); |
548 | } |
549 | |
550 | static size_t |
551 | arena_decay_backlog_npages_limit(const arena_decay_t *decay) { |
552 | uint64_t sum; |
553 | size_t npages_limit_backlog; |
554 | unsigned i; |
555 | |
556 | /* |
557 | * For each element of decay_backlog, multiply by the corresponding |
558 | * fixed-point smoothstep decay factor. Sum the products, then divide |
559 | * to round down to the nearest whole number of pages. |
560 | */ |
561 | sum = 0; |
562 | for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) { |
563 | sum += decay->backlog[i] * h_steps[i]; |
564 | } |
565 | npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP); |
566 | |
567 | return npages_limit_backlog; |
568 | } |
569 | |
570 | static void |
571 | arena_decay_backlog_update_last(arena_decay_t *decay, size_t current_npages) { |
572 | size_t npages_delta = (current_npages > decay->nunpurged) ? |
573 | current_npages - decay->nunpurged : 0; |
574 | decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta; |
575 | |
576 | if (config_debug) { |
577 | if (current_npages > decay->ceil_npages) { |
578 | decay->ceil_npages = current_npages; |
579 | } |
580 | size_t npages_limit = arena_decay_backlog_npages_limit(decay); |
581 | assert(decay->ceil_npages >= npages_limit); |
582 | if (decay->ceil_npages > npages_limit) { |
583 | decay->ceil_npages = npages_limit; |
584 | } |
585 | } |
586 | } |
587 | |
588 | static void |
589 | arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64, |
590 | size_t current_npages) { |
591 | if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) { |
592 | memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) * |
593 | sizeof(size_t)); |
594 | } else { |
595 | size_t nadvance_z = (size_t)nadvance_u64; |
596 | |
597 | assert((uint64_t)nadvance_z == nadvance_u64); |
598 | |
599 | memmove(decay->backlog, &decay->backlog[nadvance_z], |
600 | (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t)); |
601 | if (nadvance_z > 1) { |
602 | memset(&decay->backlog[SMOOTHSTEP_NSTEPS - |
603 | nadvance_z], 0, (nadvance_z-1) * sizeof(size_t)); |
604 | } |
605 | } |
606 | |
607 | arena_decay_backlog_update_last(decay, current_npages); |
608 | } |
609 | |
610 | static void |
611 | arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
612 | extents_t *extents, size_t current_npages, size_t npages_limit, |
613 | bool is_background_thread) { |
614 | if (current_npages > npages_limit) { |
615 | arena_decay_to_limit(tsdn, arena, decay, extents, false, |
616 | npages_limit, current_npages - npages_limit, |
617 | is_background_thread); |
618 | } |
619 | } |
620 | |
621 | static void |
622 | arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time, |
623 | size_t current_npages) { |
624 | assert(arena_decay_deadline_reached(decay, time)); |
625 | |
626 | nstime_t delta; |
627 | nstime_copy(&delta, time); |
628 | nstime_subtract(&delta, &decay->epoch); |
629 | |
630 | uint64_t nadvance_u64 = nstime_divide(&delta, &decay->interval); |
631 | assert(nadvance_u64 > 0); |
632 | |
633 | /* Add nadvance_u64 decay intervals to epoch. */ |
634 | nstime_copy(&delta, &decay->interval); |
635 | nstime_imultiply(&delta, nadvance_u64); |
636 | nstime_add(&decay->epoch, &delta); |
637 | |
638 | /* Set a new deadline. */ |
639 | arena_decay_deadline_init(decay); |
640 | |
641 | /* Update the backlog. */ |
642 | arena_decay_backlog_update(decay, nadvance_u64, current_npages); |
643 | } |
644 | |
645 | static void |
646 | arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
647 | extents_t *extents, const nstime_t *time, bool is_background_thread) { |
648 | size_t current_npages = extents_npages_get(extents); |
649 | arena_decay_epoch_advance_helper(decay, time, current_npages); |
650 | |
651 | size_t npages_limit = arena_decay_backlog_npages_limit(decay); |
652 | /* We may unlock decay->mtx when try_purge(). Finish logging first. */ |
653 | decay->nunpurged = (npages_limit > current_npages) ? npages_limit : |
654 | current_npages; |
655 | |
656 | if (!background_thread_enabled() || is_background_thread) { |
657 | arena_decay_try_purge(tsdn, arena, decay, extents, |
658 | current_npages, npages_limit, is_background_thread); |
659 | } |
660 | } |
661 | |
662 | static void |
663 | arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) { |
664 | arena_decay_ms_write(decay, decay_ms); |
665 | if (decay_ms > 0) { |
666 | nstime_init(&decay->interval, (uint64_t)decay_ms * |
667 | KQU(1000000)); |
668 | nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS); |
669 | } |
670 | |
671 | nstime_init(&decay->epoch, 0); |
672 | nstime_update(&decay->epoch); |
673 | decay->jitter_state = (uint64_t)(uintptr_t)decay; |
674 | arena_decay_deadline_init(decay); |
675 | decay->nunpurged = 0; |
676 | memset(decay->backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); |
677 | } |
678 | |
679 | static bool |
680 | arena_decay_init(arena_decay_t *decay, ssize_t decay_ms, |
681 | arena_stats_decay_t *stats) { |
682 | if (config_debug) { |
683 | for (size_t i = 0; i < sizeof(arena_decay_t); i++) { |
684 | assert(((char *)decay)[i] == 0); |
685 | } |
686 | decay->ceil_npages = 0; |
687 | } |
688 | if (malloc_mutex_init(&decay->mtx, "decay" , WITNESS_RANK_DECAY, |
689 | malloc_mutex_rank_exclusive)) { |
690 | return true; |
691 | } |
692 | decay->purging = false; |
693 | arena_decay_reinit(decay, decay_ms); |
694 | /* Memory is zeroed, so there is no need to clear stats. */ |
695 | if (config_stats) { |
696 | decay->stats = stats; |
697 | } |
698 | return false; |
699 | } |
700 | |
701 | static bool |
702 | arena_decay_ms_valid(ssize_t decay_ms) { |
703 | if (decay_ms < -1) { |
704 | return false; |
705 | } |
706 | if (decay_ms == -1 || (uint64_t)decay_ms <= NSTIME_SEC_MAX * |
707 | KQU(1000)) { |
708 | return true; |
709 | } |
710 | return false; |
711 | } |
712 | |
713 | static bool |
714 | arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
715 | extents_t *extents, bool is_background_thread) { |
716 | malloc_mutex_assert_owner(tsdn, &decay->mtx); |
717 | |
718 | /* Purge all or nothing if the option is disabled. */ |
719 | ssize_t decay_ms = arena_decay_ms_read(decay); |
720 | if (decay_ms <= 0) { |
721 | if (decay_ms == 0) { |
722 | arena_decay_to_limit(tsdn, arena, decay, extents, false, |
723 | 0, extents_npages_get(extents), |
724 | is_background_thread); |
725 | } |
726 | return false; |
727 | } |
728 | |
729 | nstime_t time; |
730 | nstime_init(&time, 0); |
731 | nstime_update(&time); |
732 | if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch, &time) |
733 | > 0)) { |
734 | /* |
735 | * Time went backwards. Move the epoch back in time and |
736 | * generate a new deadline, with the expectation that time |
737 | * typically flows forward for long enough periods of time that |
738 | * epochs complete. Unfortunately, this strategy is susceptible |
739 | * to clock jitter triggering premature epoch advances, but |
740 | * clock jitter estimation and compensation isn't feasible here |
741 | * because calls into this code are event-driven. |
742 | */ |
743 | nstime_copy(&decay->epoch, &time); |
744 | arena_decay_deadline_init(decay); |
745 | } else { |
746 | /* Verify that time does not go backwards. */ |
747 | assert(nstime_compare(&decay->epoch, &time) <= 0); |
748 | } |
749 | |
750 | /* |
751 | * If the deadline has been reached, advance to the current epoch and |
752 | * purge to the new limit if necessary. Note that dirty pages created |
753 | * during the current epoch are not subject to purge until a future |
754 | * epoch, so as a result purging only happens during epoch advances, or |
755 | * being triggered by background threads (scheduled event). |
756 | */ |
757 | bool advance_epoch = arena_decay_deadline_reached(decay, &time); |
758 | if (advance_epoch) { |
759 | arena_decay_epoch_advance(tsdn, arena, decay, extents, &time, |
760 | is_background_thread); |
761 | } else if (is_background_thread) { |
762 | arena_decay_try_purge(tsdn, arena, decay, extents, |
763 | extents_npages_get(extents), |
764 | arena_decay_backlog_npages_limit(decay), |
765 | is_background_thread); |
766 | } |
767 | |
768 | return advance_epoch; |
769 | } |
770 | |
771 | static ssize_t |
772 | arena_decay_ms_get(arena_decay_t *decay) { |
773 | return arena_decay_ms_read(decay); |
774 | } |
775 | |
776 | ssize_t |
777 | arena_dirty_decay_ms_get(arena_t *arena) { |
778 | return arena_decay_ms_get(&arena->decay_dirty); |
779 | } |
780 | |
781 | ssize_t |
782 | arena_muzzy_decay_ms_get(arena_t *arena) { |
783 | return arena_decay_ms_get(&arena->decay_muzzy); |
784 | } |
785 | |
786 | static bool |
787 | arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
788 | extents_t *extents, ssize_t decay_ms) { |
789 | if (!arena_decay_ms_valid(decay_ms)) { |
790 | return true; |
791 | } |
792 | |
793 | malloc_mutex_lock(tsdn, &decay->mtx); |
794 | /* |
795 | * Restart decay backlog from scratch, which may cause many dirty pages |
796 | * to be immediately purged. It would conceptually be possible to map |
797 | * the old backlog onto the new backlog, but there is no justification |
798 | * for such complexity since decay_ms changes are intended to be |
799 | * infrequent, either between the {-1, 0, >0} states, or a one-time |
800 | * arbitrary change during initial arena configuration. |
801 | */ |
802 | arena_decay_reinit(decay, decay_ms); |
803 | arena_maybe_decay(tsdn, arena, decay, extents, false); |
804 | malloc_mutex_unlock(tsdn, &decay->mtx); |
805 | |
806 | return false; |
807 | } |
808 | |
809 | bool |
810 | arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena, |
811 | ssize_t decay_ms) { |
812 | return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty, |
813 | &arena->extents_dirty, decay_ms); |
814 | } |
815 | |
816 | bool |
817 | arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, |
818 | ssize_t decay_ms) { |
819 | return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy, |
820 | &arena->extents_muzzy, decay_ms); |
821 | } |
822 | |
823 | static size_t |
824 | arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, |
825 | extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit, |
826 | size_t npages_decay_max, extent_list_t *decay_extents) { |
827 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
828 | WITNESS_RANK_CORE, 0); |
829 | |
830 | /* Stash extents according to npages_limit. */ |
831 | size_t nstashed = 0; |
832 | extent_t *extent; |
833 | while (nstashed < npages_decay_max && |
834 | (extent = extents_evict(tsdn, arena, r_extent_hooks, extents, |
835 | npages_limit)) != NULL) { |
836 | extent_list_append(decay_extents, extent); |
837 | nstashed += extent_size_get(extent) >> LG_PAGE; |
838 | } |
839 | return nstashed; |
840 | } |
841 | |
842 | static size_t |
843 | arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, |
844 | extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, |
845 | bool all, extent_list_t *decay_extents, bool is_background_thread) { |
846 | size_t nmadvise, nunmapped; |
847 | size_t npurged; |
848 | |
849 | if (config_stats) { |
850 | nmadvise = 0; |
851 | nunmapped = 0; |
852 | } |
853 | npurged = 0; |
854 | |
855 | ssize_t muzzy_decay_ms = arena_muzzy_decay_ms_get(arena); |
856 | for (extent_t *extent = extent_list_first(decay_extents); extent != |
857 | NULL; extent = extent_list_first(decay_extents)) { |
858 | if (config_stats) { |
859 | nmadvise++; |
860 | } |
861 | size_t npages = extent_size_get(extent) >> LG_PAGE; |
862 | npurged += npages; |
863 | extent_list_remove(decay_extents, extent); |
864 | switch (extents_state_get(extents)) { |
865 | case extent_state_active: |
866 | not_reached(); |
867 | case extent_state_dirty: |
868 | if (!all && muzzy_decay_ms != 0 && |
869 | !extent_purge_lazy_wrapper(tsdn, arena, |
870 | r_extent_hooks, extent, 0, |
871 | extent_size_get(extent))) { |
872 | extents_dalloc(tsdn, arena, r_extent_hooks, |
873 | &arena->extents_muzzy, extent); |
874 | arena_background_thread_inactivity_check(tsdn, |
875 | arena, is_background_thread); |
876 | break; |
877 | } |
878 | /* Fall through. */ |
879 | case extent_state_muzzy: |
880 | extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, |
881 | extent); |
882 | if (config_stats) { |
883 | nunmapped += npages; |
884 | } |
885 | break; |
886 | case extent_state_retained: |
887 | default: |
888 | not_reached(); |
889 | } |
890 | } |
891 | |
892 | if (config_stats) { |
893 | arena_stats_lock(tsdn, &arena->stats); |
894 | arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge, |
895 | 1); |
896 | arena_stats_add_u64(tsdn, &arena->stats, |
897 | &decay->stats->nmadvise, nmadvise); |
898 | arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged, |
899 | npurged); |
900 | arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, |
901 | nunmapped << LG_PAGE); |
902 | arena_stats_unlock(tsdn, &arena->stats); |
903 | } |
904 | |
905 | return npurged; |
906 | } |
907 | |
908 | /* |
909 | * npages_limit: Decay at most npages_decay_max pages without violating the |
910 | * invariant: (extents_npages_get(extents) >= npages_limit). We need an upper |
911 | * bound on number of pages in order to prevent unbounded growth (namely in |
912 | * stashed), otherwise unbounded new pages could be added to extents during the |
913 | * current decay run, so that the purging thread never finishes. |
914 | */ |
915 | static void |
916 | arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
917 | extents_t *extents, bool all, size_t npages_limit, size_t npages_decay_max, |
918 | bool is_background_thread) { |
919 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
920 | WITNESS_RANK_CORE, 1); |
921 | malloc_mutex_assert_owner(tsdn, &decay->mtx); |
922 | |
923 | if (decay->purging) { |
924 | return; |
925 | } |
926 | decay->purging = true; |
927 | malloc_mutex_unlock(tsdn, &decay->mtx); |
928 | |
929 | extent_hooks_t *extent_hooks = extent_hooks_get(arena); |
930 | |
931 | extent_list_t decay_extents; |
932 | extent_list_init(&decay_extents); |
933 | |
934 | size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, |
935 | npages_limit, npages_decay_max, &decay_extents); |
936 | if (npurge != 0) { |
937 | size_t npurged = arena_decay_stashed(tsdn, arena, |
938 | &extent_hooks, decay, extents, all, &decay_extents, |
939 | is_background_thread); |
940 | assert(npurged == npurge); |
941 | } |
942 | |
943 | malloc_mutex_lock(tsdn, &decay->mtx); |
944 | decay->purging = false; |
945 | } |
946 | |
947 | static bool |
948 | arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, |
949 | extents_t *extents, bool is_background_thread, bool all) { |
950 | if (all) { |
951 | malloc_mutex_lock(tsdn, &decay->mtx); |
952 | arena_decay_to_limit(tsdn, arena, decay, extents, all, 0, |
953 | extents_npages_get(extents), is_background_thread); |
954 | malloc_mutex_unlock(tsdn, &decay->mtx); |
955 | |
956 | return false; |
957 | } |
958 | |
959 | if (malloc_mutex_trylock(tsdn, &decay->mtx)) { |
960 | /* No need to wait if another thread is in progress. */ |
961 | return true; |
962 | } |
963 | |
964 | bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, |
965 | is_background_thread); |
966 | size_t npages_new; |
967 | if (epoch_advanced) { |
968 | /* Backlog is updated on epoch advance. */ |
969 | npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1]; |
970 | } |
971 | malloc_mutex_unlock(tsdn, &decay->mtx); |
972 | |
973 | if (have_background_thread && background_thread_enabled() && |
974 | epoch_advanced && !is_background_thread) { |
975 | background_thread_interval_check(tsdn, arena, decay, |
976 | npages_new); |
977 | } |
978 | |
979 | return false; |
980 | } |
981 | |
982 | static bool |
983 | arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, |
984 | bool all) { |
985 | return arena_decay_impl(tsdn, arena, &arena->decay_dirty, |
986 | &arena->extents_dirty, is_background_thread, all); |
987 | } |
988 | |
989 | static bool |
990 | arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, |
991 | bool all) { |
992 | return arena_decay_impl(tsdn, arena, &arena->decay_muzzy, |
993 | &arena->extents_muzzy, is_background_thread, all); |
994 | } |
995 | |
996 | void |
997 | arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { |
998 | if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) { |
999 | return; |
1000 | } |
1001 | arena_decay_muzzy(tsdn, arena, is_background_thread, all); |
1002 | } |
1003 | |
1004 | static void |
1005 | arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { |
1006 | arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); |
1007 | |
1008 | extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; |
1009 | arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, slab); |
1010 | } |
1011 | |
1012 | static void |
1013 | arena_bin_slabs_nonfull_insert(bin_t *bin, extent_t *slab) { |
1014 | assert(extent_nfree_get(slab) > 0); |
1015 | extent_heap_insert(&bin->slabs_nonfull, slab); |
1016 | if (config_stats) { |
1017 | bin->stats.nonfull_slabs++; |
1018 | } |
1019 | } |
1020 | |
1021 | static void |
1022 | arena_bin_slabs_nonfull_remove(bin_t *bin, extent_t *slab) { |
1023 | extent_heap_remove(&bin->slabs_nonfull, slab); |
1024 | if (config_stats) { |
1025 | bin->stats.nonfull_slabs--; |
1026 | } |
1027 | } |
1028 | |
1029 | static extent_t * |
1030 | arena_bin_slabs_nonfull_tryget(bin_t *bin) { |
1031 | extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull); |
1032 | if (slab == NULL) { |
1033 | return NULL; |
1034 | } |
1035 | if (config_stats) { |
1036 | bin->stats.reslabs++; |
1037 | bin->stats.nonfull_slabs--; |
1038 | } |
1039 | return slab; |
1040 | } |
1041 | |
1042 | static void |
1043 | arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, extent_t *slab) { |
1044 | assert(extent_nfree_get(slab) == 0); |
1045 | /* |
1046 | * Tracking extents is required by arena_reset, which is not allowed |
1047 | * for auto arenas. Bypass this step to avoid touching the extent |
1048 | * linkage (often results in cache misses) for auto arenas. |
1049 | */ |
1050 | if (arena_is_auto(arena)) { |
1051 | return; |
1052 | } |
1053 | extent_list_append(&bin->slabs_full, slab); |
1054 | } |
1055 | |
1056 | static void |
1057 | arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { |
1058 | if (arena_is_auto(arena)) { |
1059 | return; |
1060 | } |
1061 | extent_list_remove(&bin->slabs_full, slab); |
1062 | } |
1063 | |
1064 | static void |
1065 | arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { |
1066 | extent_t *slab; |
1067 | |
1068 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
1069 | if (bin->slabcur != NULL) { |
1070 | slab = bin->slabcur; |
1071 | bin->slabcur = NULL; |
1072 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
1073 | arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); |
1074 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
1075 | } |
1076 | while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) { |
1077 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
1078 | arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); |
1079 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
1080 | } |
1081 | for (slab = extent_list_first(&bin->slabs_full); slab != NULL; |
1082 | slab = extent_list_first(&bin->slabs_full)) { |
1083 | arena_bin_slabs_full_remove(arena, bin, slab); |
1084 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
1085 | arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); |
1086 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
1087 | } |
1088 | if (config_stats) { |
1089 | bin->stats.curregs = 0; |
1090 | bin->stats.curslabs = 0; |
1091 | } |
1092 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
1093 | } |
1094 | |
1095 | void |
1096 | arena_reset(tsd_t *tsd, arena_t *arena) { |
1097 | /* |
1098 | * Locking in this function is unintuitive. The caller guarantees that |
1099 | * no concurrent operations are happening in this arena, but there are |
1100 | * still reasons that some locking is necessary: |
1101 | * |
1102 | * - Some of the functions in the transitive closure of calls assume |
1103 | * appropriate locks are held, and in some cases these locks are |
1104 | * temporarily dropped to avoid lock order reversal or deadlock due to |
1105 | * reentry. |
1106 | * - mallctl("epoch", ...) may concurrently refresh stats. While |
1107 | * strictly speaking this is a "concurrent operation", disallowing |
1108 | * stats refreshes would impose an inconvenient burden. |
1109 | */ |
1110 | |
1111 | /* Large allocations. */ |
1112 | malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); |
1113 | |
1114 | for (extent_t *extent = extent_list_first(&arena->large); extent != |
1115 | NULL; extent = extent_list_first(&arena->large)) { |
1116 | void *ptr = extent_base_get(extent); |
1117 | size_t usize; |
1118 | |
1119 | malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); |
1120 | alloc_ctx_t alloc_ctx; |
1121 | rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); |
1122 | rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, |
1123 | (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); |
1124 | assert(alloc_ctx.szind != SC_NSIZES); |
1125 | |
1126 | if (config_stats || (config_prof && opt_prof)) { |
1127 | usize = sz_index2size(alloc_ctx.szind); |
1128 | assert(usize == isalloc(tsd_tsdn(tsd), ptr)); |
1129 | } |
1130 | /* Remove large allocation from prof sample set. */ |
1131 | if (config_prof && opt_prof) { |
1132 | prof_free(tsd, ptr, usize, &alloc_ctx); |
1133 | } |
1134 | large_dalloc(tsd_tsdn(tsd), extent); |
1135 | malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); |
1136 | } |
1137 | malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); |
1138 | |
1139 | /* Bins. */ |
1140 | for (unsigned i = 0; i < SC_NBINS; i++) { |
1141 | for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { |
1142 | arena_bin_reset(tsd, arena, |
1143 | &arena->bins[i].bin_shards[j]); |
1144 | } |
1145 | } |
1146 | |
1147 | atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); |
1148 | } |
1149 | |
1150 | static void |
1151 | arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { |
1152 | /* |
1153 | * Iterate over the retained extents and destroy them. This gives the |
1154 | * extent allocator underlying the extent hooks an opportunity to unmap |
1155 | * all retained memory without having to keep its own metadata |
1156 | * structures. In practice, virtual memory for dss-allocated extents is |
1157 | * leaked here, so best practice is to avoid dss for arenas to be |
1158 | * destroyed, or provide custom extent hooks that track retained |
1159 | * dss-based extents for later reuse. |
1160 | */ |
1161 | extent_hooks_t *extent_hooks = extent_hooks_get(arena); |
1162 | extent_t *extent; |
1163 | while ((extent = extents_evict(tsdn, arena, &extent_hooks, |
1164 | &arena->extents_retained, 0)) != NULL) { |
1165 | extent_destroy_wrapper(tsdn, arena, &extent_hooks, extent); |
1166 | } |
1167 | } |
1168 | |
1169 | void |
1170 | arena_destroy(tsd_t *tsd, arena_t *arena) { |
1171 | assert(base_ind_get(arena->base) >= narenas_auto); |
1172 | assert(arena_nthreads_get(arena, false) == 0); |
1173 | assert(arena_nthreads_get(arena, true) == 0); |
1174 | |
1175 | /* |
1176 | * No allocations have occurred since arena_reset() was called. |
1177 | * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached |
1178 | * extents, so only retained extents may remain. |
1179 | */ |
1180 | assert(extents_npages_get(&arena->extents_dirty) == 0); |
1181 | assert(extents_npages_get(&arena->extents_muzzy) == 0); |
1182 | |
1183 | /* Deallocate retained memory. */ |
1184 | arena_destroy_retained(tsd_tsdn(tsd), arena); |
1185 | |
1186 | /* |
1187 | * Remove the arena pointer from the arenas array. We rely on the fact |
1188 | * that there is no way for the application to get a dirty read from the |
1189 | * arenas array unless there is an inherent race in the application |
1190 | * involving access of an arena being concurrently destroyed. The |
1191 | * application must synchronize knowledge of the arena's validity, so as |
1192 | * long as we use an atomic write to update the arenas array, the |
1193 | * application will get a clean read any time after it synchronizes |
1194 | * knowledge that the arena is no longer valid. |
1195 | */ |
1196 | arena_set(base_ind_get(arena->base), NULL); |
1197 | |
1198 | /* |
1199 | * Destroy the base allocator, which manages all metadata ever mapped by |
1200 | * this arena. |
1201 | */ |
1202 | base_delete(tsd_tsdn(tsd), arena->base); |
1203 | } |
1204 | |
1205 | static extent_t * |
1206 | arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, |
1207 | extent_hooks_t **r_extent_hooks, const bin_info_t *bin_info, |
1208 | szind_t szind) { |
1209 | extent_t *slab; |
1210 | bool zero, commit; |
1211 | |
1212 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
1213 | WITNESS_RANK_CORE, 0); |
1214 | |
1215 | zero = false; |
1216 | commit = true; |
1217 | slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, |
1218 | bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit); |
1219 | |
1220 | if (config_stats && slab != NULL) { |
1221 | arena_stats_mapped_add(tsdn, &arena->stats, |
1222 | bin_info->slab_size); |
1223 | } |
1224 | |
1225 | return slab; |
1226 | } |
1227 | |
1228 | static extent_t * |
1229 | arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, |
1230 | const bin_info_t *bin_info) { |
1231 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
1232 | WITNESS_RANK_CORE, 0); |
1233 | |
1234 | extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; |
1235 | szind_t szind = sz_size2index(bin_info->reg_size); |
1236 | bool zero = false; |
1237 | bool commit = true; |
1238 | extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, |
1239 | &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, |
1240 | binind, &zero, &commit); |
1241 | if (slab == NULL && arena_may_have_muzzy(arena)) { |
1242 | slab = extents_alloc(tsdn, arena, &extent_hooks, |
1243 | &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, |
1244 | true, binind, &zero, &commit); |
1245 | } |
1246 | if (slab == NULL) { |
1247 | slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks, |
1248 | bin_info, szind); |
1249 | if (slab == NULL) { |
1250 | return NULL; |
1251 | } |
1252 | } |
1253 | assert(extent_slab_get(slab)); |
1254 | |
1255 | /* Initialize slab internals. */ |
1256 | arena_slab_data_t *slab_data = extent_slab_data_get(slab); |
1257 | extent_nfree_binshard_set(slab, bin_info->nregs, binshard); |
1258 | bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); |
1259 | |
1260 | arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); |
1261 | |
1262 | return slab; |
1263 | } |
1264 | |
1265 | static extent_t * |
1266 | arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, |
1267 | szind_t binind, unsigned binshard) { |
1268 | extent_t *slab; |
1269 | const bin_info_t *bin_info; |
1270 | |
1271 | /* Look for a usable slab. */ |
1272 | slab = arena_bin_slabs_nonfull_tryget(bin); |
1273 | if (slab != NULL) { |
1274 | return slab; |
1275 | } |
1276 | /* No existing slabs have any space available. */ |
1277 | |
1278 | bin_info = &bin_infos[binind]; |
1279 | |
1280 | /* Allocate a new slab. */ |
1281 | malloc_mutex_unlock(tsdn, &bin->lock); |
1282 | /******************************/ |
1283 | slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info); |
1284 | /********************************/ |
1285 | malloc_mutex_lock(tsdn, &bin->lock); |
1286 | if (slab != NULL) { |
1287 | if (config_stats) { |
1288 | bin->stats.nslabs++; |
1289 | bin->stats.curslabs++; |
1290 | } |
1291 | return slab; |
1292 | } |
1293 | |
1294 | /* |
1295 | * arena_slab_alloc() failed, but another thread may have made |
1296 | * sufficient memory available while this one dropped bin->lock above, |
1297 | * so search one more time. |
1298 | */ |
1299 | slab = arena_bin_slabs_nonfull_tryget(bin); |
1300 | if (slab != NULL) { |
1301 | return slab; |
1302 | } |
1303 | |
1304 | return NULL; |
1305 | } |
1306 | |
1307 | /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ |
1308 | static void * |
1309 | arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, |
1310 | szind_t binind, unsigned binshard) { |
1311 | const bin_info_t *bin_info; |
1312 | extent_t *slab; |
1313 | |
1314 | bin_info = &bin_infos[binind]; |
1315 | if (!arena_is_auto(arena) && bin->slabcur != NULL) { |
1316 | arena_bin_slabs_full_insert(arena, bin, bin->slabcur); |
1317 | bin->slabcur = NULL; |
1318 | } |
1319 | slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard); |
1320 | if (bin->slabcur != NULL) { |
1321 | /* |
1322 | * Another thread updated slabcur while this one ran without the |
1323 | * bin lock in arena_bin_nonfull_slab_get(). |
1324 | */ |
1325 | if (extent_nfree_get(bin->slabcur) > 0) { |
1326 | void *ret = arena_slab_reg_alloc(bin->slabcur, |
1327 | bin_info); |
1328 | if (slab != NULL) { |
1329 | /* |
1330 | * arena_slab_alloc() may have allocated slab, |
1331 | * or it may have been pulled from |
1332 | * slabs_nonfull. Therefore it is unsafe to |
1333 | * make any assumptions about how slab has |
1334 | * previously been used, and |
1335 | * arena_bin_lower_slab() must be called, as if |
1336 | * a region were just deallocated from the slab. |
1337 | */ |
1338 | if (extent_nfree_get(slab) == bin_info->nregs) { |
1339 | arena_dalloc_bin_slab(tsdn, arena, slab, |
1340 | bin); |
1341 | } else { |
1342 | arena_bin_lower_slab(tsdn, arena, slab, |
1343 | bin); |
1344 | } |
1345 | } |
1346 | return ret; |
1347 | } |
1348 | |
1349 | arena_bin_slabs_full_insert(arena, bin, bin->slabcur); |
1350 | bin->slabcur = NULL; |
1351 | } |
1352 | |
1353 | if (slab == NULL) { |
1354 | return NULL; |
1355 | } |
1356 | bin->slabcur = slab; |
1357 | |
1358 | assert(extent_nfree_get(bin->slabcur) > 0); |
1359 | |
1360 | return arena_slab_reg_alloc(slab, bin_info); |
1361 | } |
1362 | |
1363 | /* Choose a bin shard and return the locked bin. */ |
1364 | bin_t * |
1365 | arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, |
1366 | unsigned *binshard) { |
1367 | bin_t *bin; |
1368 | if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { |
1369 | *binshard = 0; |
1370 | } else { |
1371 | *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; |
1372 | } |
1373 | assert(*binshard < bin_infos[binind].n_shards); |
1374 | bin = &arena->bins[binind].bin_shards[*binshard]; |
1375 | malloc_mutex_lock(tsdn, &bin->lock); |
1376 | |
1377 | return bin; |
1378 | } |
1379 | |
1380 | void |
1381 | arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, |
1382 | cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { |
1383 | unsigned i, nfill, cnt; |
1384 | |
1385 | assert(tbin->ncached == 0); |
1386 | |
1387 | if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { |
1388 | prof_idump(tsdn); |
1389 | } |
1390 | |
1391 | unsigned binshard; |
1392 | bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); |
1393 | |
1394 | for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> |
1395 | tcache->lg_fill_div[binind]); i < nfill; i += cnt) { |
1396 | extent_t *slab; |
1397 | if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > |
1398 | 0) { |
1399 | unsigned tofill = nfill - i; |
1400 | cnt = tofill < extent_nfree_get(slab) ? |
1401 | tofill : extent_nfree_get(slab); |
1402 | arena_slab_reg_alloc_batch( |
1403 | slab, &bin_infos[binind], cnt, |
1404 | tbin->avail - nfill + i); |
1405 | } else { |
1406 | cnt = 1; |
1407 | void *ptr = arena_bin_malloc_hard(tsdn, arena, bin, |
1408 | binind, binshard); |
1409 | /* |
1410 | * OOM. tbin->avail isn't yet filled down to its first |
1411 | * element, so the successful allocations (if any) must |
1412 | * be moved just before tbin->avail before bailing out. |
1413 | */ |
1414 | if (ptr == NULL) { |
1415 | if (i > 0) { |
1416 | memmove(tbin->avail - i, |
1417 | tbin->avail - nfill, |
1418 | i * sizeof(void *)); |
1419 | } |
1420 | break; |
1421 | } |
1422 | /* Insert such that low regions get used first. */ |
1423 | *(tbin->avail - nfill + i) = ptr; |
1424 | } |
1425 | if (config_fill && unlikely(opt_junk_alloc)) { |
1426 | for (unsigned j = 0; j < cnt; j++) { |
1427 | void* ptr = *(tbin->avail - nfill + i + j); |
1428 | arena_alloc_junk_small(ptr, &bin_infos[binind], |
1429 | true); |
1430 | } |
1431 | } |
1432 | } |
1433 | if (config_stats) { |
1434 | bin->stats.nmalloc += i; |
1435 | bin->stats.nrequests += tbin->tstats.nrequests; |
1436 | bin->stats.curregs += i; |
1437 | bin->stats.nfills++; |
1438 | tbin->tstats.nrequests = 0; |
1439 | } |
1440 | malloc_mutex_unlock(tsdn, &bin->lock); |
1441 | tbin->ncached = i; |
1442 | arena_decay_tick(tsdn, arena); |
1443 | } |
1444 | |
1445 | void |
1446 | arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info, bool zero) { |
1447 | if (!zero) { |
1448 | memset(ptr, JEMALLOC_ALLOC_JUNK, bin_info->reg_size); |
1449 | } |
1450 | } |
1451 | |
1452 | static void |
1453 | arena_dalloc_junk_small_impl(void *ptr, const bin_info_t *bin_info) { |
1454 | memset(ptr, JEMALLOC_FREE_JUNK, bin_info->reg_size); |
1455 | } |
1456 | arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small = |
1457 | arena_dalloc_junk_small_impl; |
1458 | |
1459 | static void * |
1460 | arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { |
1461 | void *ret; |
1462 | bin_t *bin; |
1463 | size_t usize; |
1464 | extent_t *slab; |
1465 | |
1466 | assert(binind < SC_NBINS); |
1467 | usize = sz_index2size(binind); |
1468 | unsigned binshard; |
1469 | bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); |
1470 | |
1471 | if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { |
1472 | ret = arena_slab_reg_alloc(slab, &bin_infos[binind]); |
1473 | } else { |
1474 | ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard); |
1475 | } |
1476 | |
1477 | if (ret == NULL) { |
1478 | malloc_mutex_unlock(tsdn, &bin->lock); |
1479 | return NULL; |
1480 | } |
1481 | |
1482 | if (config_stats) { |
1483 | bin->stats.nmalloc++; |
1484 | bin->stats.nrequests++; |
1485 | bin->stats.curregs++; |
1486 | } |
1487 | malloc_mutex_unlock(tsdn, &bin->lock); |
1488 | if (config_prof && arena_prof_accum(tsdn, arena, usize)) { |
1489 | prof_idump(tsdn); |
1490 | } |
1491 | |
1492 | if (!zero) { |
1493 | if (config_fill) { |
1494 | if (unlikely(opt_junk_alloc)) { |
1495 | arena_alloc_junk_small(ret, |
1496 | &bin_infos[binind], false); |
1497 | } else if (unlikely(opt_zero)) { |
1498 | memset(ret, 0, usize); |
1499 | } |
1500 | } |
1501 | } else { |
1502 | if (config_fill && unlikely(opt_junk_alloc)) { |
1503 | arena_alloc_junk_small(ret, &bin_infos[binind], |
1504 | true); |
1505 | } |
1506 | memset(ret, 0, usize); |
1507 | } |
1508 | |
1509 | arena_decay_tick(tsdn, arena); |
1510 | return ret; |
1511 | } |
1512 | |
1513 | void * |
1514 | arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, |
1515 | bool zero) { |
1516 | assert(!tsdn_null(tsdn) || arena != NULL); |
1517 | |
1518 | if (likely(!tsdn_null(tsdn))) { |
1519 | arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size); |
1520 | } |
1521 | if (unlikely(arena == NULL)) { |
1522 | return NULL; |
1523 | } |
1524 | |
1525 | if (likely(size <= SC_SMALL_MAXCLASS)) { |
1526 | return arena_malloc_small(tsdn, arena, ind, zero); |
1527 | } |
1528 | return large_malloc(tsdn, arena, sz_index2size(ind), zero); |
1529 | } |
1530 | |
1531 | void * |
1532 | arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, |
1533 | bool zero, tcache_t *tcache) { |
1534 | void *ret; |
1535 | |
1536 | if (usize <= SC_SMALL_MAXCLASS |
1537 | && (alignment < PAGE |
1538 | || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { |
1539 | /* Small; alignment doesn't require special slab placement. */ |
1540 | ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), |
1541 | zero, tcache, true); |
1542 | } else { |
1543 | if (likely(alignment <= CACHELINE)) { |
1544 | ret = large_malloc(tsdn, arena, usize, zero); |
1545 | } else { |
1546 | ret = large_palloc(tsdn, arena, usize, alignment, zero); |
1547 | } |
1548 | } |
1549 | return ret; |
1550 | } |
1551 | |
1552 | void |
1553 | arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) { |
1554 | cassert(config_prof); |
1555 | assert(ptr != NULL); |
1556 | assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); |
1557 | assert(usize <= SC_SMALL_MAXCLASS); |
1558 | |
1559 | if (config_opt_safety_checks) { |
1560 | safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS); |
1561 | } |
1562 | |
1563 | rtree_ctx_t rtree_ctx_fallback; |
1564 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
1565 | |
1566 | extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, |
1567 | (uintptr_t)ptr, true); |
1568 | arena_t *arena = extent_arena_get(extent); |
1569 | |
1570 | szind_t szind = sz_size2index(usize); |
1571 | extent_szind_set(extent, szind); |
1572 | rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, |
1573 | szind, false); |
1574 | |
1575 | prof_accum_cancel(tsdn, &arena->prof_accum, usize); |
1576 | |
1577 | assert(isalloc(tsdn, ptr) == usize); |
1578 | } |
1579 | |
1580 | static size_t |
1581 | arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { |
1582 | cassert(config_prof); |
1583 | assert(ptr != NULL); |
1584 | |
1585 | extent_szind_set(extent, SC_NBINS); |
1586 | rtree_ctx_t rtree_ctx_fallback; |
1587 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
1588 | rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, |
1589 | SC_NBINS, false); |
1590 | |
1591 | assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); |
1592 | |
1593 | return SC_LARGE_MINCLASS; |
1594 | } |
1595 | |
1596 | void |
1597 | arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, |
1598 | bool slow_path) { |
1599 | cassert(config_prof); |
1600 | assert(opt_prof); |
1601 | |
1602 | extent_t *extent = iealloc(tsdn, ptr); |
1603 | size_t usize = extent_usize_get(extent); |
1604 | size_t bumped_usize = arena_prof_demote(tsdn, extent, ptr); |
1605 | if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) { |
1606 | /* |
1607 | * Currently, we only do redzoning for small sampled |
1608 | * allocations. |
1609 | */ |
1610 | assert(bumped_usize == SC_LARGE_MINCLASS); |
1611 | safety_check_verify_redzone(ptr, usize, bumped_usize); |
1612 | } |
1613 | if (bumped_usize <= tcache_maxclass && tcache != NULL) { |
1614 | tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, |
1615 | sz_size2index(bumped_usize), slow_path); |
1616 | } else { |
1617 | large_dalloc(tsdn, extent); |
1618 | } |
1619 | } |
1620 | |
1621 | static void |
1622 | arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, bin_t *bin) { |
1623 | /* Dissociate slab from bin. */ |
1624 | if (slab == bin->slabcur) { |
1625 | bin->slabcur = NULL; |
1626 | } else { |
1627 | szind_t binind = extent_szind_get(slab); |
1628 | const bin_info_t *bin_info = &bin_infos[binind]; |
1629 | |
1630 | /* |
1631 | * The following block's conditional is necessary because if the |
1632 | * slab only contains one region, then it never gets inserted |
1633 | * into the non-full slabs heap. |
1634 | */ |
1635 | if (bin_info->nregs == 1) { |
1636 | arena_bin_slabs_full_remove(arena, bin, slab); |
1637 | } else { |
1638 | arena_bin_slabs_nonfull_remove(bin, slab); |
1639 | } |
1640 | } |
1641 | } |
1642 | |
1643 | static void |
1644 | arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, |
1645 | bin_t *bin) { |
1646 | assert(slab != bin->slabcur); |
1647 | |
1648 | malloc_mutex_unlock(tsdn, &bin->lock); |
1649 | /******************************/ |
1650 | arena_slab_dalloc(tsdn, arena, slab); |
1651 | /****************************/ |
1652 | malloc_mutex_lock(tsdn, &bin->lock); |
1653 | if (config_stats) { |
1654 | bin->stats.curslabs--; |
1655 | } |
1656 | } |
1657 | |
1658 | static void |
1659 | arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, |
1660 | bin_t *bin) { |
1661 | assert(extent_nfree_get(slab) > 0); |
1662 | |
1663 | /* |
1664 | * Make sure that if bin->slabcur is non-NULL, it refers to the |
1665 | * oldest/lowest non-full slab. It is okay to NULL slabcur out rather |
1666 | * than proactively keeping it pointing at the oldest/lowest non-full |
1667 | * slab. |
1668 | */ |
1669 | if (bin->slabcur != NULL && extent_snad_comp(bin->slabcur, slab) > 0) { |
1670 | /* Switch slabcur. */ |
1671 | if (extent_nfree_get(bin->slabcur) > 0) { |
1672 | arena_bin_slabs_nonfull_insert(bin, bin->slabcur); |
1673 | } else { |
1674 | arena_bin_slabs_full_insert(arena, bin, bin->slabcur); |
1675 | } |
1676 | bin->slabcur = slab; |
1677 | if (config_stats) { |
1678 | bin->stats.reslabs++; |
1679 | } |
1680 | } else { |
1681 | arena_bin_slabs_nonfull_insert(bin, slab); |
1682 | } |
1683 | } |
1684 | |
1685 | static void |
1686 | arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, |
1687 | szind_t binind, extent_t *slab, void *ptr, bool junked) { |
1688 | arena_slab_data_t *slab_data = extent_slab_data_get(slab); |
1689 | const bin_info_t *bin_info = &bin_infos[binind]; |
1690 | |
1691 | if (!junked && config_fill && unlikely(opt_junk_free)) { |
1692 | arena_dalloc_junk_small(ptr, bin_info); |
1693 | } |
1694 | |
1695 | arena_slab_reg_dalloc(slab, slab_data, ptr); |
1696 | unsigned nfree = extent_nfree_get(slab); |
1697 | if (nfree == bin_info->nregs) { |
1698 | arena_dissociate_bin_slab(arena, slab, bin); |
1699 | arena_dalloc_bin_slab(tsdn, arena, slab, bin); |
1700 | } else if (nfree == 1 && slab != bin->slabcur) { |
1701 | arena_bin_slabs_full_remove(arena, bin, slab); |
1702 | arena_bin_lower_slab(tsdn, arena, slab, bin); |
1703 | } |
1704 | |
1705 | if (config_stats) { |
1706 | bin->stats.ndalloc++; |
1707 | bin->stats.curregs--; |
1708 | } |
1709 | } |
1710 | |
1711 | void |
1712 | arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, |
1713 | szind_t binind, extent_t *extent, void *ptr) { |
1714 | arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, |
1715 | true); |
1716 | } |
1717 | |
1718 | static void |
1719 | arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { |
1720 | szind_t binind = extent_szind_get(extent); |
1721 | unsigned binshard = extent_binshard_get(extent); |
1722 | bin_t *bin = &arena->bins[binind].bin_shards[binshard]; |
1723 | |
1724 | malloc_mutex_lock(tsdn, &bin->lock); |
1725 | arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, |
1726 | false); |
1727 | malloc_mutex_unlock(tsdn, &bin->lock); |
1728 | } |
1729 | |
1730 | void |
1731 | arena_dalloc_small(tsdn_t *tsdn, void *ptr) { |
1732 | extent_t *extent = iealloc(tsdn, ptr); |
1733 | arena_t *arena = extent_arena_get(extent); |
1734 | |
1735 | arena_dalloc_bin(tsdn, arena, extent, ptr); |
1736 | arena_decay_tick(tsdn, arena); |
1737 | } |
1738 | |
1739 | bool |
1740 | arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, |
1741 | size_t , bool zero, size_t *newsize) { |
1742 | bool ret; |
1743 | /* Calls with non-zero extra had to clamp extra. */ |
1744 | assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS); |
1745 | |
1746 | extent_t *extent = iealloc(tsdn, ptr); |
1747 | if (unlikely(size > SC_LARGE_MAXCLASS)) { |
1748 | ret = true; |
1749 | goto done; |
1750 | } |
1751 | |
1752 | size_t usize_min = sz_s2u(size); |
1753 | size_t usize_max = sz_s2u(size + extra); |
1754 | if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min |
1755 | <= SC_SMALL_MAXCLASS)) { |
1756 | /* |
1757 | * Avoid moving the allocation if the size class can be left the |
1758 | * same. |
1759 | */ |
1760 | assert(bin_infos[sz_size2index(oldsize)].reg_size == |
1761 | oldsize); |
1762 | if ((usize_max > SC_SMALL_MAXCLASS |
1763 | || sz_size2index(usize_max) != sz_size2index(oldsize)) |
1764 | && (size > oldsize || usize_max < oldsize)) { |
1765 | ret = true; |
1766 | goto done; |
1767 | } |
1768 | |
1769 | arena_decay_tick(tsdn, extent_arena_get(extent)); |
1770 | ret = false; |
1771 | } else if (oldsize >= SC_LARGE_MINCLASS |
1772 | && usize_max >= SC_LARGE_MINCLASS) { |
1773 | ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max, |
1774 | zero); |
1775 | } else { |
1776 | ret = true; |
1777 | } |
1778 | done: |
1779 | assert(extent == iealloc(tsdn, ptr)); |
1780 | *newsize = extent_usize_get(extent); |
1781 | |
1782 | return ret; |
1783 | } |
1784 | |
1785 | static void * |
1786 | arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, |
1787 | size_t alignment, bool zero, tcache_t *tcache) { |
1788 | if (alignment == 0) { |
1789 | return arena_malloc(tsdn, arena, usize, sz_size2index(usize), |
1790 | zero, tcache, true); |
1791 | } |
1792 | usize = sz_sa2u(usize, alignment); |
1793 | if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { |
1794 | return NULL; |
1795 | } |
1796 | return ipalloct(tsdn, usize, alignment, zero, tcache, arena); |
1797 | } |
1798 | |
1799 | void * |
1800 | arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, |
1801 | size_t size, size_t alignment, bool zero, tcache_t *tcache, |
1802 | hook_ralloc_args_t *hook_args) { |
1803 | size_t usize = sz_s2u(size); |
1804 | if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { |
1805 | return NULL; |
1806 | } |
1807 | |
1808 | if (likely(usize <= SC_SMALL_MAXCLASS)) { |
1809 | /* Try to avoid moving the allocation. */ |
1810 | UNUSED size_t newsize; |
1811 | if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, |
1812 | &newsize)) { |
1813 | hook_invoke_expand(hook_args->is_realloc |
1814 | ? hook_expand_realloc : hook_expand_rallocx, |
1815 | ptr, oldsize, usize, (uintptr_t)ptr, |
1816 | hook_args->args); |
1817 | return ptr; |
1818 | } |
1819 | } |
1820 | |
1821 | if (oldsize >= SC_LARGE_MINCLASS |
1822 | && usize >= SC_LARGE_MINCLASS) { |
1823 | return large_ralloc(tsdn, arena, ptr, usize, |
1824 | alignment, zero, tcache, hook_args); |
1825 | } |
1826 | |
1827 | /* |
1828 | * size and oldsize are different enough that we need to move the |
1829 | * object. In that case, fall back to allocating new space and copying. |
1830 | */ |
1831 | void *ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment, |
1832 | zero, tcache); |
1833 | if (ret == NULL) { |
1834 | return NULL; |
1835 | } |
1836 | |
1837 | hook_invoke_alloc(hook_args->is_realloc |
1838 | ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, |
1839 | hook_args->args); |
1840 | hook_invoke_dalloc(hook_args->is_realloc |
1841 | ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); |
1842 | |
1843 | /* |
1844 | * Junk/zero-filling were already done by |
1845 | * ipalloc()/arena_malloc(). |
1846 | */ |
1847 | size_t copysize = (usize < oldsize) ? usize : oldsize; |
1848 | memcpy(ret, ptr, copysize); |
1849 | isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); |
1850 | return ret; |
1851 | } |
1852 | |
1853 | dss_prec_t |
1854 | arena_dss_prec_get(arena_t *arena) { |
1855 | return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE); |
1856 | } |
1857 | |
1858 | bool |
1859 | arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { |
1860 | if (!have_dss) { |
1861 | return (dss_prec != dss_prec_disabled); |
1862 | } |
1863 | atomic_store_u(&arena->dss_prec, (unsigned)dss_prec, ATOMIC_RELEASE); |
1864 | return false; |
1865 | } |
1866 | |
1867 | ssize_t |
1868 | arena_dirty_decay_ms_default_get(void) { |
1869 | return atomic_load_zd(&dirty_decay_ms_default, ATOMIC_RELAXED); |
1870 | } |
1871 | |
1872 | bool |
1873 | arena_dirty_decay_ms_default_set(ssize_t decay_ms) { |
1874 | if (!arena_decay_ms_valid(decay_ms)) { |
1875 | return true; |
1876 | } |
1877 | atomic_store_zd(&dirty_decay_ms_default, decay_ms, ATOMIC_RELAXED); |
1878 | return false; |
1879 | } |
1880 | |
1881 | ssize_t |
1882 | arena_muzzy_decay_ms_default_get(void) { |
1883 | return atomic_load_zd(&muzzy_decay_ms_default, ATOMIC_RELAXED); |
1884 | } |
1885 | |
1886 | bool |
1887 | arena_muzzy_decay_ms_default_set(ssize_t decay_ms) { |
1888 | if (!arena_decay_ms_valid(decay_ms)) { |
1889 | return true; |
1890 | } |
1891 | atomic_store_zd(&muzzy_decay_ms_default, decay_ms, ATOMIC_RELAXED); |
1892 | return false; |
1893 | } |
1894 | |
1895 | bool |
1896 | arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, |
1897 | size_t *new_limit) { |
1898 | assert(opt_retain); |
1899 | |
1900 | pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0); |
1901 | if (new_limit != NULL) { |
1902 | size_t limit = *new_limit; |
1903 | /* Grow no more than the new limit. */ |
1904 | if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { |
1905 | return true; |
1906 | } |
1907 | } |
1908 | |
1909 | malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx); |
1910 | if (old_limit != NULL) { |
1911 | *old_limit = sz_pind2sz(arena->retain_grow_limit); |
1912 | } |
1913 | if (new_limit != NULL) { |
1914 | arena->retain_grow_limit = new_ind; |
1915 | } |
1916 | malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx); |
1917 | |
1918 | return false; |
1919 | } |
1920 | |
1921 | unsigned |
1922 | arena_nthreads_get(arena_t *arena, bool internal) { |
1923 | return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED); |
1924 | } |
1925 | |
1926 | void |
1927 | arena_nthreads_inc(arena_t *arena, bool internal) { |
1928 | atomic_fetch_add_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED); |
1929 | } |
1930 | |
1931 | void |
1932 | arena_nthreads_dec(arena_t *arena, bool internal) { |
1933 | atomic_fetch_sub_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED); |
1934 | } |
1935 | |
1936 | size_t |
1937 | arena_extent_sn_next(arena_t *arena) { |
1938 | return atomic_fetch_add_zu(&arena->extent_sn_next, 1, ATOMIC_RELAXED); |
1939 | } |
1940 | |
1941 | arena_t * |
1942 | arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { |
1943 | arena_t *arena; |
1944 | base_t *base; |
1945 | unsigned i; |
1946 | |
1947 | if (ind == 0) { |
1948 | base = b0get(); |
1949 | } else { |
1950 | base = base_new(tsdn, ind, extent_hooks); |
1951 | if (base == NULL) { |
1952 | return NULL; |
1953 | } |
1954 | } |
1955 | |
1956 | unsigned nbins_total = 0; |
1957 | for (i = 0; i < SC_NBINS; i++) { |
1958 | nbins_total += bin_infos[i].n_shards; |
1959 | } |
1960 | size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; |
1961 | arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); |
1962 | if (arena == NULL) { |
1963 | goto label_error; |
1964 | } |
1965 | |
1966 | atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); |
1967 | atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); |
1968 | arena->last_thd = NULL; |
1969 | |
1970 | if (config_stats) { |
1971 | if (arena_stats_init(tsdn, &arena->stats)) { |
1972 | goto label_error; |
1973 | } |
1974 | |
1975 | ql_new(&arena->tcache_ql); |
1976 | ql_new(&arena->cache_bin_array_descriptor_ql); |
1977 | if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql" , |
1978 | WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) { |
1979 | goto label_error; |
1980 | } |
1981 | } |
1982 | |
1983 | if (config_prof) { |
1984 | if (prof_accum_init(tsdn, &arena->prof_accum)) { |
1985 | goto label_error; |
1986 | } |
1987 | } |
1988 | |
1989 | if (config_cache_oblivious) { |
1990 | /* |
1991 | * A nondeterministic seed based on the address of arena reduces |
1992 | * the likelihood of lockstep non-uniform cache index |
1993 | * utilization among identical concurrent processes, but at the |
1994 | * cost of test repeatability. For debug builds, instead use a |
1995 | * deterministic seed. |
1996 | */ |
1997 | atomic_store_zu(&arena->offset_state, config_debug ? ind : |
1998 | (size_t)(uintptr_t)arena, ATOMIC_RELAXED); |
1999 | } |
2000 | |
2001 | atomic_store_zu(&arena->extent_sn_next, 0, ATOMIC_RELAXED); |
2002 | |
2003 | atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(), |
2004 | ATOMIC_RELAXED); |
2005 | |
2006 | atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); |
2007 | |
2008 | extent_list_init(&arena->large); |
2009 | if (malloc_mutex_init(&arena->large_mtx, "arena_large" , |
2010 | WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) { |
2011 | goto label_error; |
2012 | } |
2013 | |
2014 | /* |
2015 | * Delay coalescing for dirty extents despite the disruptive effect on |
2016 | * memory layout for best-fit extent allocation, since cached extents |
2017 | * are likely to be reused soon after deallocation, and the cost of |
2018 | * merging/splitting extents is non-trivial. |
2019 | */ |
2020 | if (extents_init(tsdn, &arena->extents_dirty, extent_state_dirty, |
2021 | true)) { |
2022 | goto label_error; |
2023 | } |
2024 | /* |
2025 | * Coalesce muzzy extents immediately, because operations on them are in |
2026 | * the critical path much less often than for dirty extents. |
2027 | */ |
2028 | if (extents_init(tsdn, &arena->extents_muzzy, extent_state_muzzy, |
2029 | false)) { |
2030 | goto label_error; |
2031 | } |
2032 | /* |
2033 | * Coalesce retained extents immediately, in part because they will |
2034 | * never be evicted (and therefore there's no opportunity for delayed |
2035 | * coalescing), but also because operations on retained extents are not |
2036 | * in the critical path. |
2037 | */ |
2038 | if (extents_init(tsdn, &arena->extents_retained, extent_state_retained, |
2039 | false)) { |
2040 | goto label_error; |
2041 | } |
2042 | |
2043 | if (arena_decay_init(&arena->decay_dirty, |
2044 | arena_dirty_decay_ms_default_get(), &arena->stats.decay_dirty)) { |
2045 | goto label_error; |
2046 | } |
2047 | if (arena_decay_init(&arena->decay_muzzy, |
2048 | arena_muzzy_decay_ms_default_get(), &arena->stats.decay_muzzy)) { |
2049 | goto label_error; |
2050 | } |
2051 | |
2052 | arena->extent_grow_next = sz_psz2ind(HUGEPAGE); |
2053 | arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); |
2054 | if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow" , |
2055 | WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { |
2056 | goto label_error; |
2057 | } |
2058 | |
2059 | extent_avail_new(&arena->extent_avail); |
2060 | if (malloc_mutex_init(&arena->extent_avail_mtx, "extent_avail" , |
2061 | WITNESS_RANK_EXTENT_AVAIL, malloc_mutex_rank_exclusive)) { |
2062 | goto label_error; |
2063 | } |
2064 | |
2065 | /* Initialize bins. */ |
2066 | uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); |
2067 | atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); |
2068 | for (i = 0; i < SC_NBINS; i++) { |
2069 | unsigned nshards = bin_infos[i].n_shards; |
2070 | arena->bins[i].bin_shards = (bin_t *)bin_addr; |
2071 | bin_addr += nshards * sizeof(bin_t); |
2072 | for (unsigned j = 0; j < nshards; j++) { |
2073 | bool err = bin_init(&arena->bins[i].bin_shards[j]); |
2074 | if (err) { |
2075 | goto label_error; |
2076 | } |
2077 | } |
2078 | } |
2079 | assert(bin_addr == (uintptr_t)arena + arena_size); |
2080 | |
2081 | arena->base = base; |
2082 | /* Set arena before creating background threads. */ |
2083 | arena_set(ind, arena); |
2084 | |
2085 | nstime_init(&arena->create_time, 0); |
2086 | nstime_update(&arena->create_time); |
2087 | |
2088 | /* We don't support reentrancy for arena 0 bootstrapping. */ |
2089 | if (ind != 0) { |
2090 | /* |
2091 | * If we're here, then arena 0 already exists, so bootstrapping |
2092 | * is done enough that we should have tsd. |
2093 | */ |
2094 | assert(!tsdn_null(tsdn)); |
2095 | pre_reentrancy(tsdn_tsd(tsdn), arena); |
2096 | if (test_hooks_arena_new_hook) { |
2097 | test_hooks_arena_new_hook(); |
2098 | } |
2099 | post_reentrancy(tsdn_tsd(tsdn)); |
2100 | } |
2101 | |
2102 | return arena; |
2103 | label_error: |
2104 | if (ind != 0) { |
2105 | base_delete(tsdn, base); |
2106 | } |
2107 | return NULL; |
2108 | } |
2109 | |
2110 | arena_t * |
2111 | arena_choose_huge(tsd_t *tsd) { |
2112 | /* huge_arena_ind can be 0 during init (will use a0). */ |
2113 | if (huge_arena_ind == 0) { |
2114 | assert(!malloc_initialized()); |
2115 | } |
2116 | |
2117 | arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); |
2118 | if (huge_arena == NULL) { |
2119 | /* Create the huge arena on demand. */ |
2120 | assert(huge_arena_ind != 0); |
2121 | huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); |
2122 | if (huge_arena == NULL) { |
2123 | return NULL; |
2124 | } |
2125 | /* |
2126 | * Purge eagerly for huge allocations, because: 1) number of |
2127 | * huge allocations is usually small, which means ticker based |
2128 | * decay is not reliable; and 2) less immediate reuse is |
2129 | * expected for huge allocations. |
2130 | */ |
2131 | if (arena_dirty_decay_ms_default_get() > 0) { |
2132 | arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); |
2133 | } |
2134 | if (arena_muzzy_decay_ms_default_get() > 0) { |
2135 | arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); |
2136 | } |
2137 | } |
2138 | |
2139 | return huge_arena; |
2140 | } |
2141 | |
2142 | bool |
2143 | arena_init_huge(void) { |
2144 | bool huge_enabled; |
2145 | |
2146 | /* The threshold should be large size class. */ |
2147 | if (opt_oversize_threshold > SC_LARGE_MAXCLASS || |
2148 | opt_oversize_threshold < SC_LARGE_MINCLASS) { |
2149 | opt_oversize_threshold = 0; |
2150 | oversize_threshold = SC_LARGE_MAXCLASS + PAGE; |
2151 | huge_enabled = false; |
2152 | } else { |
2153 | /* Reserve the index for the huge arena. */ |
2154 | huge_arena_ind = narenas_total_get(); |
2155 | oversize_threshold = opt_oversize_threshold; |
2156 | huge_enabled = true; |
2157 | } |
2158 | |
2159 | return huge_enabled; |
2160 | } |
2161 | |
2162 | bool |
2163 | arena_is_huge(unsigned arena_ind) { |
2164 | if (huge_arena_ind == 0) { |
2165 | return false; |
2166 | } |
2167 | return (arena_ind == huge_arena_ind); |
2168 | } |
2169 | |
2170 | void |
2171 | arena_boot(sc_data_t *sc_data) { |
2172 | arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); |
2173 | arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); |
2174 | for (unsigned i = 0; i < SC_NBINS; i++) { |
2175 | sc_t *sc = &sc_data->sc[i]; |
2176 | div_init(&arena_binind_div_info[i], |
2177 | (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); |
2178 | } |
2179 | } |
2180 | |
2181 | void |
2182 | arena_prefork0(tsdn_t *tsdn, arena_t *arena) { |
2183 | malloc_mutex_prefork(tsdn, &arena->decay_dirty.mtx); |
2184 | malloc_mutex_prefork(tsdn, &arena->decay_muzzy.mtx); |
2185 | } |
2186 | |
2187 | void |
2188 | arena_prefork1(tsdn_t *tsdn, arena_t *arena) { |
2189 | if (config_stats) { |
2190 | malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx); |
2191 | } |
2192 | } |
2193 | |
2194 | void |
2195 | arena_prefork2(tsdn_t *tsdn, arena_t *arena) { |
2196 | malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx); |
2197 | } |
2198 | |
2199 | void |
2200 | arena_prefork3(tsdn_t *tsdn, arena_t *arena) { |
2201 | extents_prefork(tsdn, &arena->extents_dirty); |
2202 | extents_prefork(tsdn, &arena->extents_muzzy); |
2203 | extents_prefork(tsdn, &arena->extents_retained); |
2204 | } |
2205 | |
2206 | void |
2207 | arena_prefork4(tsdn_t *tsdn, arena_t *arena) { |
2208 | malloc_mutex_prefork(tsdn, &arena->extent_avail_mtx); |
2209 | } |
2210 | |
2211 | void |
2212 | arena_prefork5(tsdn_t *tsdn, arena_t *arena) { |
2213 | base_prefork(tsdn, arena->base); |
2214 | } |
2215 | |
2216 | void |
2217 | arena_prefork6(tsdn_t *tsdn, arena_t *arena) { |
2218 | malloc_mutex_prefork(tsdn, &arena->large_mtx); |
2219 | } |
2220 | |
2221 | void |
2222 | arena_prefork7(tsdn_t *tsdn, arena_t *arena) { |
2223 | for (unsigned i = 0; i < SC_NBINS; i++) { |
2224 | for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { |
2225 | bin_prefork(tsdn, &arena->bins[i].bin_shards[j]); |
2226 | } |
2227 | } |
2228 | } |
2229 | |
2230 | void |
2231 | arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { |
2232 | unsigned i; |
2233 | |
2234 | for (i = 0; i < SC_NBINS; i++) { |
2235 | for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { |
2236 | bin_postfork_parent(tsdn, |
2237 | &arena->bins[i].bin_shards[j]); |
2238 | } |
2239 | } |
2240 | malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); |
2241 | base_postfork_parent(tsdn, arena->base); |
2242 | malloc_mutex_postfork_parent(tsdn, &arena->extent_avail_mtx); |
2243 | extents_postfork_parent(tsdn, &arena->extents_dirty); |
2244 | extents_postfork_parent(tsdn, &arena->extents_muzzy); |
2245 | extents_postfork_parent(tsdn, &arena->extents_retained); |
2246 | malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx); |
2247 | malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); |
2248 | malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); |
2249 | if (config_stats) { |
2250 | malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); |
2251 | } |
2252 | } |
2253 | |
2254 | void |
2255 | arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { |
2256 | unsigned i; |
2257 | |
2258 | atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); |
2259 | atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); |
2260 | if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) { |
2261 | arena_nthreads_inc(arena, false); |
2262 | } |
2263 | if (tsd_iarena_get(tsdn_tsd(tsdn)) == arena) { |
2264 | arena_nthreads_inc(arena, true); |
2265 | } |
2266 | if (config_stats) { |
2267 | ql_new(&arena->tcache_ql); |
2268 | ql_new(&arena->cache_bin_array_descriptor_ql); |
2269 | tcache_t *tcache = tcache_get(tsdn_tsd(tsdn)); |
2270 | if (tcache != NULL && tcache->arena == arena) { |
2271 | ql_elm_new(tcache, link); |
2272 | ql_tail_insert(&arena->tcache_ql, tcache, link); |
2273 | cache_bin_array_descriptor_init( |
2274 | &tcache->cache_bin_array_descriptor, |
2275 | tcache->bins_small, tcache->bins_large); |
2276 | ql_tail_insert(&arena->cache_bin_array_descriptor_ql, |
2277 | &tcache->cache_bin_array_descriptor, link); |
2278 | } |
2279 | } |
2280 | |
2281 | for (i = 0; i < SC_NBINS; i++) { |
2282 | for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { |
2283 | bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]); |
2284 | } |
2285 | } |
2286 | malloc_mutex_postfork_child(tsdn, &arena->large_mtx); |
2287 | base_postfork_child(tsdn, arena->base); |
2288 | malloc_mutex_postfork_child(tsdn, &arena->extent_avail_mtx); |
2289 | extents_postfork_child(tsdn, &arena->extents_dirty); |
2290 | extents_postfork_child(tsdn, &arena->extents_muzzy); |
2291 | extents_postfork_child(tsdn, &arena->extents_retained); |
2292 | malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx); |
2293 | malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); |
2294 | malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); |
2295 | if (config_stats) { |
2296 | malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); |
2297 | } |
2298 | } |
2299 | |