1#define JEMALLOC_CTL_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/assert.h"
6#include "jemalloc/internal/ctl.h"
7#include "jemalloc/internal/extent_dss.h"
8#include "jemalloc/internal/extent_mmap.h"
9#include "jemalloc/internal/mutex.h"
10#include "jemalloc/internal/nstime.h"
11#include "jemalloc/internal/sc.h"
12#include "jemalloc/internal/util.h"
13
14/******************************************************************************/
15/* Data. */
16
17/*
18 * ctl_mtx protects the following:
19 * - ctl_stats->*
20 */
21static malloc_mutex_t ctl_mtx;
22static bool ctl_initialized;
23static ctl_stats_t *ctl_stats;
24static ctl_arenas_t *ctl_arenas;
25
26/******************************************************************************/
27/* Helpers for named and indexed nodes. */
28
29static const ctl_named_node_t *
30ctl_named_node(const ctl_node_t *node) {
31 return ((node->named) ? (const ctl_named_node_t *)node : NULL);
32}
33
34static const ctl_named_node_t *
35ctl_named_children(const ctl_named_node_t *node, size_t index) {
36 const ctl_named_node_t *children = ctl_named_node(node->children);
37
38 return (children ? &children[index] : NULL);
39}
40
41static const ctl_indexed_node_t *
42ctl_indexed_node(const ctl_node_t *node) {
43 return (!node->named ? (const ctl_indexed_node_t *)node : NULL);
44}
45
46/******************************************************************************/
47/* Function prototypes for non-inline static functions. */
48
49#define CTL_PROTO(n) \
50static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
51 void *oldp, size_t *oldlenp, void *newp, size_t newlen);
52
53#define INDEX_PROTO(n) \
54static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \
55 const size_t *mib, size_t miblen, size_t i);
56
57CTL_PROTO(version)
58CTL_PROTO(epoch)
59CTL_PROTO(background_thread)
60CTL_PROTO(max_background_threads)
61CTL_PROTO(thread_tcache_enabled)
62CTL_PROTO(thread_tcache_flush)
63CTL_PROTO(thread_prof_name)
64CTL_PROTO(thread_prof_active)
65CTL_PROTO(thread_arena)
66CTL_PROTO(thread_allocated)
67CTL_PROTO(thread_allocatedp)
68CTL_PROTO(thread_deallocated)
69CTL_PROTO(thread_deallocatedp)
70CTL_PROTO(config_cache_oblivious)
71CTL_PROTO(config_debug)
72CTL_PROTO(config_fill)
73CTL_PROTO(config_lazy_lock)
74CTL_PROTO(config_malloc_conf)
75CTL_PROTO(config_opt_safety_checks)
76CTL_PROTO(config_prof)
77CTL_PROTO(config_prof_libgcc)
78CTL_PROTO(config_prof_libunwind)
79CTL_PROTO(config_stats)
80CTL_PROTO(config_utrace)
81CTL_PROTO(config_xmalloc)
82CTL_PROTO(opt_abort)
83CTL_PROTO(opt_abort_conf)
84CTL_PROTO(opt_confirm_conf)
85CTL_PROTO(opt_metadata_thp)
86CTL_PROTO(opt_retain)
87CTL_PROTO(opt_dss)
88CTL_PROTO(opt_narenas)
89CTL_PROTO(opt_percpu_arena)
90CTL_PROTO(opt_oversize_threshold)
91CTL_PROTO(opt_background_thread)
92CTL_PROTO(opt_max_background_threads)
93CTL_PROTO(opt_dirty_decay_ms)
94CTL_PROTO(opt_muzzy_decay_ms)
95CTL_PROTO(opt_stats_print)
96CTL_PROTO(opt_stats_print_opts)
97CTL_PROTO(opt_junk)
98CTL_PROTO(opt_zero)
99CTL_PROTO(opt_utrace)
100CTL_PROTO(opt_xmalloc)
101CTL_PROTO(opt_tcache)
102CTL_PROTO(opt_thp)
103CTL_PROTO(opt_lg_extent_max_active_fit)
104CTL_PROTO(opt_lg_tcache_max)
105CTL_PROTO(opt_prof)
106CTL_PROTO(opt_prof_prefix)
107CTL_PROTO(opt_prof_active)
108CTL_PROTO(opt_prof_thread_active_init)
109CTL_PROTO(opt_lg_prof_sample)
110CTL_PROTO(opt_lg_prof_interval)
111CTL_PROTO(opt_prof_gdump)
112CTL_PROTO(opt_prof_final)
113CTL_PROTO(opt_prof_leak)
114CTL_PROTO(opt_prof_accum)
115CTL_PROTO(tcache_create)
116CTL_PROTO(tcache_flush)
117CTL_PROTO(tcache_destroy)
118CTL_PROTO(arena_i_initialized)
119CTL_PROTO(arena_i_decay)
120CTL_PROTO(arena_i_purge)
121CTL_PROTO(arena_i_reset)
122CTL_PROTO(arena_i_destroy)
123CTL_PROTO(arena_i_dss)
124CTL_PROTO(arena_i_dirty_decay_ms)
125CTL_PROTO(arena_i_muzzy_decay_ms)
126CTL_PROTO(arena_i_extent_hooks)
127CTL_PROTO(arena_i_retain_grow_limit)
128INDEX_PROTO(arena_i)
129CTL_PROTO(arenas_bin_i_size)
130CTL_PROTO(arenas_bin_i_nregs)
131CTL_PROTO(arenas_bin_i_slab_size)
132CTL_PROTO(arenas_bin_i_nshards)
133INDEX_PROTO(arenas_bin_i)
134CTL_PROTO(arenas_lextent_i_size)
135INDEX_PROTO(arenas_lextent_i)
136CTL_PROTO(arenas_narenas)
137CTL_PROTO(arenas_dirty_decay_ms)
138CTL_PROTO(arenas_muzzy_decay_ms)
139CTL_PROTO(arenas_quantum)
140CTL_PROTO(arenas_page)
141CTL_PROTO(arenas_tcache_max)
142CTL_PROTO(arenas_nbins)
143CTL_PROTO(arenas_nhbins)
144CTL_PROTO(arenas_nlextents)
145CTL_PROTO(arenas_create)
146CTL_PROTO(arenas_lookup)
147CTL_PROTO(prof_thread_active_init)
148CTL_PROTO(prof_active)
149CTL_PROTO(prof_dump)
150CTL_PROTO(prof_gdump)
151CTL_PROTO(prof_reset)
152CTL_PROTO(prof_interval)
153CTL_PROTO(lg_prof_sample)
154CTL_PROTO(prof_log_start)
155CTL_PROTO(prof_log_stop)
156CTL_PROTO(stats_arenas_i_small_allocated)
157CTL_PROTO(stats_arenas_i_small_nmalloc)
158CTL_PROTO(stats_arenas_i_small_ndalloc)
159CTL_PROTO(stats_arenas_i_small_nrequests)
160CTL_PROTO(stats_arenas_i_small_nfills)
161CTL_PROTO(stats_arenas_i_small_nflushes)
162CTL_PROTO(stats_arenas_i_large_allocated)
163CTL_PROTO(stats_arenas_i_large_nmalloc)
164CTL_PROTO(stats_arenas_i_large_ndalloc)
165CTL_PROTO(stats_arenas_i_large_nrequests)
166CTL_PROTO(stats_arenas_i_large_nfills)
167CTL_PROTO(stats_arenas_i_large_nflushes)
168CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
169CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
170CTL_PROTO(stats_arenas_i_bins_j_nrequests)
171CTL_PROTO(stats_arenas_i_bins_j_curregs)
172CTL_PROTO(stats_arenas_i_bins_j_nfills)
173CTL_PROTO(stats_arenas_i_bins_j_nflushes)
174CTL_PROTO(stats_arenas_i_bins_j_nslabs)
175CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
176CTL_PROTO(stats_arenas_i_bins_j_curslabs)
177CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs)
178INDEX_PROTO(stats_arenas_i_bins_j)
179CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)
180CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)
181CTL_PROTO(stats_arenas_i_lextents_j_nrequests)
182CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
183INDEX_PROTO(stats_arenas_i_lextents_j)
184CTL_PROTO(stats_arenas_i_extents_j_ndirty)
185CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)
186CTL_PROTO(stats_arenas_i_extents_j_nretained)
187CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
188CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
189CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
190INDEX_PROTO(stats_arenas_i_extents_j)
191CTL_PROTO(stats_arenas_i_nthreads)
192CTL_PROTO(stats_arenas_i_uptime)
193CTL_PROTO(stats_arenas_i_dss)
194CTL_PROTO(stats_arenas_i_dirty_decay_ms)
195CTL_PROTO(stats_arenas_i_muzzy_decay_ms)
196CTL_PROTO(stats_arenas_i_pactive)
197CTL_PROTO(stats_arenas_i_pdirty)
198CTL_PROTO(stats_arenas_i_pmuzzy)
199CTL_PROTO(stats_arenas_i_mapped)
200CTL_PROTO(stats_arenas_i_retained)
201CTL_PROTO(stats_arenas_i_extent_avail)
202CTL_PROTO(stats_arenas_i_dirty_npurge)
203CTL_PROTO(stats_arenas_i_dirty_nmadvise)
204CTL_PROTO(stats_arenas_i_dirty_purged)
205CTL_PROTO(stats_arenas_i_muzzy_npurge)
206CTL_PROTO(stats_arenas_i_muzzy_nmadvise)
207CTL_PROTO(stats_arenas_i_muzzy_purged)
208CTL_PROTO(stats_arenas_i_base)
209CTL_PROTO(stats_arenas_i_internal)
210CTL_PROTO(stats_arenas_i_metadata_thp)
211CTL_PROTO(stats_arenas_i_tcache_bytes)
212CTL_PROTO(stats_arenas_i_resident)
213CTL_PROTO(stats_arenas_i_abandoned_vm)
214INDEX_PROTO(stats_arenas_i)
215CTL_PROTO(stats_allocated)
216CTL_PROTO(stats_active)
217CTL_PROTO(stats_background_thread_num_threads)
218CTL_PROTO(stats_background_thread_num_runs)
219CTL_PROTO(stats_background_thread_run_interval)
220CTL_PROTO(stats_metadata)
221CTL_PROTO(stats_metadata_thp)
222CTL_PROTO(stats_resident)
223CTL_PROTO(stats_mapped)
224CTL_PROTO(stats_retained)
225CTL_PROTO(experimental_hooks_install)
226CTL_PROTO(experimental_hooks_remove)
227CTL_PROTO(experimental_utilization_query)
228CTL_PROTO(experimental_utilization_batch_query)
229CTL_PROTO(experimental_arenas_i_pactivep)
230INDEX_PROTO(experimental_arenas_i)
231
232#define MUTEX_STATS_CTL_PROTO_GEN(n) \
233CTL_PROTO(stats_##n##_num_ops) \
234CTL_PROTO(stats_##n##_num_wait) \
235CTL_PROTO(stats_##n##_num_spin_acq) \
236CTL_PROTO(stats_##n##_num_owner_switch) \
237CTL_PROTO(stats_##n##_total_wait_time) \
238CTL_PROTO(stats_##n##_max_wait_time) \
239CTL_PROTO(stats_##n##_max_num_thds)
240
241/* Global mutexes. */
242#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx)
243MUTEX_PROF_GLOBAL_MUTEXES
244#undef OP
245
246/* Per arena mutexes. */
247#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx)
248MUTEX_PROF_ARENA_MUTEXES
249#undef OP
250
251/* Arena bin mutexes. */
252MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)
253#undef MUTEX_STATS_CTL_PROTO_GEN
254
255CTL_PROTO(stats_mutexes_reset)
256
257/******************************************************************************/
258/* mallctl tree. */
259
260#define NAME(n) {true}, n
261#define CHILD(t, c) \
262 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \
263 (ctl_node_t *)c##_node, \
264 NULL
265#define CTL(c) 0, NULL, c##_ctl
266
267/*
268 * Only handles internal indexed nodes, since there are currently no external
269 * ones.
270 */
271#define INDEX(i) {false}, i##_index
272
273static const ctl_named_node_t thread_tcache_node[] = {
274 {NAME("enabled"), CTL(thread_tcache_enabled)},
275 {NAME("flush"), CTL(thread_tcache_flush)}
276};
277
278static const ctl_named_node_t thread_prof_node[] = {
279 {NAME("name"), CTL(thread_prof_name)},
280 {NAME("active"), CTL(thread_prof_active)}
281};
282
283static const ctl_named_node_t thread_node[] = {
284 {NAME("arena"), CTL(thread_arena)},
285 {NAME("allocated"), CTL(thread_allocated)},
286 {NAME("allocatedp"), CTL(thread_allocatedp)},
287 {NAME("deallocated"), CTL(thread_deallocated)},
288 {NAME("deallocatedp"), CTL(thread_deallocatedp)},
289 {NAME("tcache"), CHILD(named, thread_tcache)},
290 {NAME("prof"), CHILD(named, thread_prof)}
291};
292
293static const ctl_named_node_t config_node[] = {
294 {NAME("cache_oblivious"), CTL(config_cache_oblivious)},
295 {NAME("debug"), CTL(config_debug)},
296 {NAME("fill"), CTL(config_fill)},
297 {NAME("lazy_lock"), CTL(config_lazy_lock)},
298 {NAME("malloc_conf"), CTL(config_malloc_conf)},
299 {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)},
300 {NAME("prof"), CTL(config_prof)},
301 {NAME("prof_libgcc"), CTL(config_prof_libgcc)},
302 {NAME("prof_libunwind"), CTL(config_prof_libunwind)},
303 {NAME("stats"), CTL(config_stats)},
304 {NAME("utrace"), CTL(config_utrace)},
305 {NAME("xmalloc"), CTL(config_xmalloc)}
306};
307
308static const ctl_named_node_t opt_node[] = {
309 {NAME("abort"), CTL(opt_abort)},
310 {NAME("abort_conf"), CTL(opt_abort_conf)},
311 {NAME("confirm_conf"), CTL(opt_confirm_conf)},
312 {NAME("metadata_thp"), CTL(opt_metadata_thp)},
313 {NAME("retain"), CTL(opt_retain)},
314 {NAME("dss"), CTL(opt_dss)},
315 {NAME("narenas"), CTL(opt_narenas)},
316 {NAME("percpu_arena"), CTL(opt_percpu_arena)},
317 {NAME("oversize_threshold"), CTL(opt_oversize_threshold)},
318 {NAME("background_thread"), CTL(opt_background_thread)},
319 {NAME("max_background_threads"), CTL(opt_max_background_threads)},
320 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)},
321 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)},
322 {NAME("stats_print"), CTL(opt_stats_print)},
323 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)},
324 {NAME("junk"), CTL(opt_junk)},
325 {NAME("zero"), CTL(opt_zero)},
326 {NAME("utrace"), CTL(opt_utrace)},
327 {NAME("xmalloc"), CTL(opt_xmalloc)},
328 {NAME("tcache"), CTL(opt_tcache)},
329 {NAME("thp"), CTL(opt_thp)},
330 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)},
331 {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},
332 {NAME("prof"), CTL(opt_prof)},
333 {NAME("prof_prefix"), CTL(opt_prof_prefix)},
334 {NAME("prof_active"), CTL(opt_prof_active)},
335 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},
336 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},
337 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},
338 {NAME("prof_gdump"), CTL(opt_prof_gdump)},
339 {NAME("prof_final"), CTL(opt_prof_final)},
340 {NAME("prof_leak"), CTL(opt_prof_leak)},
341 {NAME("prof_accum"), CTL(opt_prof_accum)}
342};
343
344static const ctl_named_node_t tcache_node[] = {
345 {NAME("create"), CTL(tcache_create)},
346 {NAME("flush"), CTL(tcache_flush)},
347 {NAME("destroy"), CTL(tcache_destroy)}
348};
349
350static const ctl_named_node_t arena_i_node[] = {
351 {NAME("initialized"), CTL(arena_i_initialized)},
352 {NAME("decay"), CTL(arena_i_decay)},
353 {NAME("purge"), CTL(arena_i_purge)},
354 {NAME("reset"), CTL(arena_i_reset)},
355 {NAME("destroy"), CTL(arena_i_destroy)},
356 {NAME("dss"), CTL(arena_i_dss)},
357 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
358 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
359 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
360 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)}
361};
362static const ctl_named_node_t super_arena_i_node[] = {
363 {NAME(""), CHILD(named, arena_i)}
364};
365
366static const ctl_indexed_node_t arena_node[] = {
367 {INDEX(arena_i)}
368};
369
370static const ctl_named_node_t arenas_bin_i_node[] = {
371 {NAME("size"), CTL(arenas_bin_i_size)},
372 {NAME("nregs"), CTL(arenas_bin_i_nregs)},
373 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)},
374 {NAME("nshards"), CTL(arenas_bin_i_nshards)}
375};
376static const ctl_named_node_t super_arenas_bin_i_node[] = {
377 {NAME(""), CHILD(named, arenas_bin_i)}
378};
379
380static const ctl_indexed_node_t arenas_bin_node[] = {
381 {INDEX(arenas_bin_i)}
382};
383
384static const ctl_named_node_t arenas_lextent_i_node[] = {
385 {NAME("size"), CTL(arenas_lextent_i_size)}
386};
387static const ctl_named_node_t super_arenas_lextent_i_node[] = {
388 {NAME(""), CHILD(named, arenas_lextent_i)}
389};
390
391static const ctl_indexed_node_t arenas_lextent_node[] = {
392 {INDEX(arenas_lextent_i)}
393};
394
395static const ctl_named_node_t arenas_node[] = {
396 {NAME("narenas"), CTL(arenas_narenas)},
397 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)},
398 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)},
399 {NAME("quantum"), CTL(arenas_quantum)},
400 {NAME("page"), CTL(arenas_page)},
401 {NAME("tcache_max"), CTL(arenas_tcache_max)},
402 {NAME("nbins"), CTL(arenas_nbins)},
403 {NAME("nhbins"), CTL(arenas_nhbins)},
404 {NAME("bin"), CHILD(indexed, arenas_bin)},
405 {NAME("nlextents"), CTL(arenas_nlextents)},
406 {NAME("lextent"), CHILD(indexed, arenas_lextent)},
407 {NAME("create"), CTL(arenas_create)},
408 {NAME("lookup"), CTL(arenas_lookup)}
409};
410
411static const ctl_named_node_t prof_node[] = {
412 {NAME("thread_active_init"), CTL(prof_thread_active_init)},
413 {NAME("active"), CTL(prof_active)},
414 {NAME("dump"), CTL(prof_dump)},
415 {NAME("gdump"), CTL(prof_gdump)},
416 {NAME("reset"), CTL(prof_reset)},
417 {NAME("interval"), CTL(prof_interval)},
418 {NAME("lg_sample"), CTL(lg_prof_sample)},
419 {NAME("log_start"), CTL(prof_log_start)},
420 {NAME("log_stop"), CTL(prof_log_stop)}
421};
422static const ctl_named_node_t stats_arenas_i_small_node[] = {
423 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
424 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
425 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
426 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)},
427 {NAME("nfills"), CTL(stats_arenas_i_small_nfills)},
428 {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)}
429};
430
431static const ctl_named_node_t stats_arenas_i_large_node[] = {
432 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
433 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
434 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
435 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)},
436 {NAME("nfills"), CTL(stats_arenas_i_large_nfills)},
437 {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)}
438};
439
440#define MUTEX_PROF_DATA_NODE(prefix) \
441static const ctl_named_node_t stats_##prefix##_node[] = { \
442 {NAME("num_ops"), \
443 CTL(stats_##prefix##_num_ops)}, \
444 {NAME("num_wait"), \
445 CTL(stats_##prefix##_num_wait)}, \
446 {NAME("num_spin_acq"), \
447 CTL(stats_##prefix##_num_spin_acq)}, \
448 {NAME("num_owner_switch"), \
449 CTL(stats_##prefix##_num_owner_switch)}, \
450 {NAME("total_wait_time"), \
451 CTL(stats_##prefix##_total_wait_time)}, \
452 {NAME("max_wait_time"), \
453 CTL(stats_##prefix##_max_wait_time)}, \
454 {NAME("max_num_thds"), \
455 CTL(stats_##prefix##_max_num_thds)} \
456 /* Note that # of current waiting thread not provided. */ \
457};
458
459MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)
460
461static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
462 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
463 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
464 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},
465 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)},
466 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},
467 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)},
468 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)},
469 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)},
470 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)},
471 {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)},
472 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}
473};
474
475static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
476 {NAME(""), CHILD(named, stats_arenas_i_bins_j)}
477};
478
479static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
480 {INDEX(stats_arenas_i_bins_j)}
481};
482
483static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
484 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)},
485 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)},
486 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)},
487 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}
488};
489static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
490 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}
491};
492
493static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
494 {INDEX(stats_arenas_i_lextents_j)}
495};
496
497static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
498 {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)},
499 {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)},
500 {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)},
501 {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)},
502 {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)},
503 {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}
504};
505
506static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
507 {NAME(""), CHILD(named, stats_arenas_i_extents_j)}
508};
509
510static const ctl_indexed_node_t stats_arenas_i_extents_node[] = {
511 {INDEX(stats_arenas_i_extents_j)}
512};
513
514#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)
515MUTEX_PROF_ARENA_MUTEXES
516#undef OP
517
518static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
519#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},
520MUTEX_PROF_ARENA_MUTEXES
521#undef OP
522};
523
524static const ctl_named_node_t stats_arenas_i_node[] = {
525 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
526 {NAME("uptime"), CTL(stats_arenas_i_uptime)},
527 {NAME("dss"), CTL(stats_arenas_i_dss)},
528 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)},
529 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)},
530 {NAME("pactive"), CTL(stats_arenas_i_pactive)},
531 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
532 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
533 {NAME("mapped"), CTL(stats_arenas_i_mapped)},
534 {NAME("retained"), CTL(stats_arenas_i_retained)},
535 {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)},
536 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
537 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
538 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)},
539 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)},
540 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)},
541 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)},
542 {NAME("base"), CTL(stats_arenas_i_base)},
543 {NAME("internal"), CTL(stats_arenas_i_internal)},
544 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)},
545 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)},
546 {NAME("resident"), CTL(stats_arenas_i_resident)},
547 {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
548 {NAME("small"), CHILD(named, stats_arenas_i_small)},
549 {NAME("large"), CHILD(named, stats_arenas_i_large)},
550 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
551 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)},
552 {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)},
553 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}
554};
555static const ctl_named_node_t super_stats_arenas_i_node[] = {
556 {NAME(""), CHILD(named, stats_arenas_i)}
557};
558
559static const ctl_indexed_node_t stats_arenas_node[] = {
560 {INDEX(stats_arenas_i)}
561};
562
563static const ctl_named_node_t stats_background_thread_node[] = {
564 {NAME("num_threads"), CTL(stats_background_thread_num_threads)},
565 {NAME("num_runs"), CTL(stats_background_thread_num_runs)},
566 {NAME("run_interval"), CTL(stats_background_thread_run_interval)}
567};
568
569#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)
570MUTEX_PROF_GLOBAL_MUTEXES
571#undef OP
572
573static const ctl_named_node_t stats_mutexes_node[] = {
574#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},
575MUTEX_PROF_GLOBAL_MUTEXES
576#undef OP
577 {NAME("reset"), CTL(stats_mutexes_reset)}
578};
579#undef MUTEX_PROF_DATA_NODE
580
581static const ctl_named_node_t stats_node[] = {
582 {NAME("allocated"), CTL(stats_allocated)},
583 {NAME("active"), CTL(stats_active)},
584 {NAME("metadata"), CTL(stats_metadata)},
585 {NAME("metadata_thp"), CTL(stats_metadata_thp)},
586 {NAME("resident"), CTL(stats_resident)},
587 {NAME("mapped"), CTL(stats_mapped)},
588 {NAME("retained"), CTL(stats_retained)},
589 {NAME("background_thread"),
590 CHILD(named, stats_background_thread)},
591 {NAME("mutexes"), CHILD(named, stats_mutexes)},
592 {NAME("arenas"), CHILD(indexed, stats_arenas)}
593};
594
595static const ctl_named_node_t experimental_hooks_node[] = {
596 {NAME("install"), CTL(experimental_hooks_install)},
597 {NAME("remove"), CTL(experimental_hooks_remove)}
598};
599
600static const ctl_named_node_t experimental_utilization_node[] = {
601 {NAME("query"), CTL(experimental_utilization_query)},
602 {NAME("batch_query"), CTL(experimental_utilization_batch_query)}
603};
604
605static const ctl_named_node_t experimental_arenas_i_node[] = {
606 {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)}
607};
608static const ctl_named_node_t super_experimental_arenas_i_node[] = {
609 {NAME(""), CHILD(named, experimental_arenas_i)}
610};
611
612static const ctl_indexed_node_t experimental_arenas_node[] = {
613 {INDEX(experimental_arenas_i)}
614};
615
616static const ctl_named_node_t experimental_node[] = {
617 {NAME("hooks"), CHILD(named, experimental_hooks)},
618 {NAME("utilization"), CHILD(named, experimental_utilization)},
619 {NAME("arenas"), CHILD(indexed, experimental_arenas)}
620};
621
622static const ctl_named_node_t root_node[] = {
623 {NAME("version"), CTL(version)},
624 {NAME("epoch"), CTL(epoch)},
625 {NAME("background_thread"), CTL(background_thread)},
626 {NAME("max_background_threads"), CTL(max_background_threads)},
627 {NAME("thread"), CHILD(named, thread)},
628 {NAME("config"), CHILD(named, config)},
629 {NAME("opt"), CHILD(named, opt)},
630 {NAME("tcache"), CHILD(named, tcache)},
631 {NAME("arena"), CHILD(indexed, arena)},
632 {NAME("arenas"), CHILD(named, arenas)},
633 {NAME("prof"), CHILD(named, prof)},
634 {NAME("stats"), CHILD(named, stats)},
635 {NAME("experimental"), CHILD(named, experimental)}
636};
637static const ctl_named_node_t super_root_node[] = {
638 {NAME(""), CHILD(named, root)}
639};
640
641#undef NAME
642#undef CHILD
643#undef CTL
644#undef INDEX
645
646/******************************************************************************/
647
648/*
649 * Sets *dst + *src non-atomically. This is safe, since everything is
650 * synchronized by the ctl mutex.
651 */
652static void
653ctl_accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) {
654#ifdef JEMALLOC_ATOMIC_U64
655 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
656 uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED);
657 atomic_store_u64(dst, cur_dst + cur_src, ATOMIC_RELAXED);
658#else
659 *dst += *src;
660#endif
661}
662
663/* Likewise: with ctl mutex synchronization, reading is simple. */
664static uint64_t
665ctl_arena_stats_read_u64(arena_stats_u64_t *p) {
666#ifdef JEMALLOC_ATOMIC_U64
667 return atomic_load_u64(p, ATOMIC_RELAXED);
668#else
669 return *p;
670#endif
671}
672
673static void
674accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) {
675 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
676 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED);
677 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED);
678}
679
680/******************************************************************************/
681
682static unsigned
683arenas_i2a_impl(size_t i, bool compat, bool validate) {
684 unsigned a;
685
686 switch (i) {
687 case MALLCTL_ARENAS_ALL:
688 a = 0;
689 break;
690 case MALLCTL_ARENAS_DESTROYED:
691 a = 1;
692 break;
693 default:
694 if (compat && i == ctl_arenas->narenas) {
695 /*
696 * Provide deprecated backward compatibility for
697 * accessing the merged stats at index narenas rather
698 * than via MALLCTL_ARENAS_ALL. This is scheduled for
699 * removal in 6.0.0.
700 */
701 a = 0;
702 } else if (validate && i >= ctl_arenas->narenas) {
703 a = UINT_MAX;
704 } else {
705 /*
706 * This function should never be called for an index
707 * more than one past the range of indices that have
708 * initialized ctl data.
709 */
710 assert(i < ctl_arenas->narenas || (!validate && i ==
711 ctl_arenas->narenas));
712 a = (unsigned)i + 2;
713 }
714 break;
715 }
716
717 return a;
718}
719
720static unsigned
721arenas_i2a(size_t i) {
722 return arenas_i2a_impl(i, true, false);
723}
724
725static ctl_arena_t *
726arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) {
727 ctl_arena_t *ret;
728
729 assert(!compat || !init);
730
731 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)];
732 if (init && ret == NULL) {
733 if (config_stats) {
734 struct container_s {
735 ctl_arena_t ctl_arena;
736 ctl_arena_stats_t astats;
737 };
738 struct container_s *cont =
739 (struct container_s *)base_alloc(tsd_tsdn(tsd),
740 b0get(), sizeof(struct container_s), QUANTUM);
741 if (cont == NULL) {
742 return NULL;
743 }
744 ret = &cont->ctl_arena;
745 ret->astats = &cont->astats;
746 } else {
747 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(),
748 sizeof(ctl_arena_t), QUANTUM);
749 if (ret == NULL) {
750 return NULL;
751 }
752 }
753 ret->arena_ind = (unsigned)i;
754 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret;
755 }
756
757 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i));
758 return ret;
759}
760
761static ctl_arena_t *
762arenas_i(size_t i) {
763 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false);
764 assert(ret != NULL);
765 return ret;
766}
767
768static void
769ctl_arena_clear(ctl_arena_t *ctl_arena) {
770 ctl_arena->nthreads = 0;
771 ctl_arena->dss = dss_prec_names[dss_prec_limit];
772 ctl_arena->dirty_decay_ms = -1;
773 ctl_arena->muzzy_decay_ms = -1;
774 ctl_arena->pactive = 0;
775 ctl_arena->pdirty = 0;
776 ctl_arena->pmuzzy = 0;
777 if (config_stats) {
778 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t));
779 ctl_arena->astats->allocated_small = 0;
780 ctl_arena->astats->nmalloc_small = 0;
781 ctl_arena->astats->ndalloc_small = 0;
782 ctl_arena->astats->nrequests_small = 0;
783 ctl_arena->astats->nfills_small = 0;
784 ctl_arena->astats->nflushes_small = 0;
785 memset(ctl_arena->astats->bstats, 0, SC_NBINS *
786 sizeof(bin_stats_t));
787 memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) *
788 sizeof(arena_stats_large_t));
789 memset(ctl_arena->astats->estats, 0, SC_NPSIZES *
790 sizeof(arena_stats_extents_t));
791 }
792}
793
794static void
795ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
796 unsigned i;
797
798 if (config_stats) {
799 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads,
800 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
801 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
802 &ctl_arena->pdirty, &ctl_arena->pmuzzy,
803 &ctl_arena->astats->astats, ctl_arena->astats->bstats,
804 ctl_arena->astats->lstats, ctl_arena->astats->estats);
805
806 for (i = 0; i < SC_NBINS; i++) {
807 ctl_arena->astats->allocated_small +=
808 ctl_arena->astats->bstats[i].curregs *
809 sz_index2size(i);
810 ctl_arena->astats->nmalloc_small +=
811 ctl_arena->astats->bstats[i].nmalloc;
812 ctl_arena->astats->ndalloc_small +=
813 ctl_arena->astats->bstats[i].ndalloc;
814 ctl_arena->astats->nrequests_small +=
815 ctl_arena->astats->bstats[i].nrequests;
816 ctl_arena->astats->nfills_small +=
817 ctl_arena->astats->bstats[i].nfills;
818 ctl_arena->astats->nflushes_small +=
819 ctl_arena->astats->bstats[i].nflushes;
820 }
821 } else {
822 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads,
823 &ctl_arena->dss, &ctl_arena->dirty_decay_ms,
824 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
825 &ctl_arena->pdirty, &ctl_arena->pmuzzy);
826 }
827}
828
829static void
830ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
831 bool destroyed) {
832 unsigned i;
833
834 if (!destroyed) {
835 ctl_sdarena->nthreads += ctl_arena->nthreads;
836 ctl_sdarena->pactive += ctl_arena->pactive;
837 ctl_sdarena->pdirty += ctl_arena->pdirty;
838 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy;
839 } else {
840 assert(ctl_arena->nthreads == 0);
841 assert(ctl_arena->pactive == 0);
842 assert(ctl_arena->pdirty == 0);
843 assert(ctl_arena->pmuzzy == 0);
844 }
845
846 if (config_stats) {
847 ctl_arena_stats_t *sdstats = ctl_sdarena->astats;
848 ctl_arena_stats_t *astats = ctl_arena->astats;
849
850 if (!destroyed) {
851 accum_atomic_zu(&sdstats->astats.mapped,
852 &astats->astats.mapped);
853 accum_atomic_zu(&sdstats->astats.retained,
854 &astats->astats.retained);
855 accum_atomic_zu(&sdstats->astats.extent_avail,
856 &astats->astats.extent_avail);
857 }
858
859 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge,
860 &astats->astats.decay_dirty.npurge);
861 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise,
862 &astats->astats.decay_dirty.nmadvise);
863 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged,
864 &astats->astats.decay_dirty.purged);
865
866 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge,
867 &astats->astats.decay_muzzy.npurge);
868 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise,
869 &astats->astats.decay_muzzy.nmadvise);
870 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged,
871 &astats->astats.decay_muzzy.purged);
872
873#define OP(mtx) malloc_mutex_prof_merge( \
874 &(sdstats->astats.mutex_prof_data[ \
875 arena_prof_mutex_##mtx]), \
876 &(astats->astats.mutex_prof_data[ \
877 arena_prof_mutex_##mtx]));
878MUTEX_PROF_ARENA_MUTEXES
879#undef OP
880 if (!destroyed) {
881 accum_atomic_zu(&sdstats->astats.base,
882 &astats->astats.base);
883 accum_atomic_zu(&sdstats->astats.internal,
884 &astats->astats.internal);
885 accum_atomic_zu(&sdstats->astats.resident,
886 &astats->astats.resident);
887 accum_atomic_zu(&sdstats->astats.metadata_thp,
888 &astats->astats.metadata_thp);
889 } else {
890 assert(atomic_load_zu(
891 &astats->astats.internal, ATOMIC_RELAXED) == 0);
892 }
893
894 if (!destroyed) {
895 sdstats->allocated_small += astats->allocated_small;
896 } else {
897 assert(astats->allocated_small == 0);
898 }
899 sdstats->nmalloc_small += astats->nmalloc_small;
900 sdstats->ndalloc_small += astats->ndalloc_small;
901 sdstats->nrequests_small += astats->nrequests_small;
902 sdstats->nfills_small += astats->nfills_small;
903 sdstats->nflushes_small += astats->nflushes_small;
904
905 if (!destroyed) {
906 accum_atomic_zu(&sdstats->astats.allocated_large,
907 &astats->astats.allocated_large);
908 } else {
909 assert(atomic_load_zu(&astats->astats.allocated_large,
910 ATOMIC_RELAXED) == 0);
911 }
912 ctl_accum_arena_stats_u64(&sdstats->astats.nmalloc_large,
913 &astats->astats.nmalloc_large);
914 ctl_accum_arena_stats_u64(&sdstats->astats.ndalloc_large,
915 &astats->astats.ndalloc_large);
916 ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large,
917 &astats->astats.nrequests_large);
918 accum_atomic_zu(&sdstats->astats.abandoned_vm,
919 &astats->astats.abandoned_vm);
920
921 accum_atomic_zu(&sdstats->astats.tcache_bytes,
922 &astats->astats.tcache_bytes);
923
924 if (ctl_arena->arena_ind == 0) {
925 sdstats->astats.uptime = astats->astats.uptime;
926 }
927
928 /* Merge bin stats. */
929 for (i = 0; i < SC_NBINS; i++) {
930 sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
931 sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
932 sdstats->bstats[i].nrequests +=
933 astats->bstats[i].nrequests;
934 if (!destroyed) {
935 sdstats->bstats[i].curregs +=
936 astats->bstats[i].curregs;
937 } else {
938 assert(astats->bstats[i].curregs == 0);
939 }
940 sdstats->bstats[i].nfills += astats->bstats[i].nfills;
941 sdstats->bstats[i].nflushes +=
942 astats->bstats[i].nflushes;
943 sdstats->bstats[i].nslabs += astats->bstats[i].nslabs;
944 sdstats->bstats[i].reslabs += astats->bstats[i].reslabs;
945 if (!destroyed) {
946 sdstats->bstats[i].curslabs +=
947 astats->bstats[i].curslabs;
948 sdstats->bstats[i].nonfull_slabs +=
949 astats->bstats[i].nonfull_slabs;
950 } else {
951 assert(astats->bstats[i].curslabs == 0);
952 assert(astats->bstats[i].nonfull_slabs == 0);
953 }
954 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data,
955 &astats->bstats[i].mutex_data);
956 }
957
958 /* Merge stats for large allocations. */
959 for (i = 0; i < SC_NSIZES - SC_NBINS; i++) {
960 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc,
961 &astats->lstats[i].nmalloc);
962 ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc,
963 &astats->lstats[i].ndalloc);
964 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nrequests,
965 &astats->lstats[i].nrequests);
966 if (!destroyed) {
967 sdstats->lstats[i].curlextents +=
968 astats->lstats[i].curlextents;
969 } else {
970 assert(astats->lstats[i].curlextents == 0);
971 }
972 }
973
974 /* Merge extents stats. */
975 for (i = 0; i < SC_NPSIZES; i++) {
976 accum_atomic_zu(&sdstats->estats[i].ndirty,
977 &astats->estats[i].ndirty);
978 accum_atomic_zu(&sdstats->estats[i].nmuzzy,
979 &astats->estats[i].nmuzzy);
980 accum_atomic_zu(&sdstats->estats[i].nretained,
981 &astats->estats[i].nretained);
982 accum_atomic_zu(&sdstats->estats[i].dirty_bytes,
983 &astats->estats[i].dirty_bytes);
984 accum_atomic_zu(&sdstats->estats[i].muzzy_bytes,
985 &astats->estats[i].muzzy_bytes);
986 accum_atomic_zu(&sdstats->estats[i].retained_bytes,
987 &astats->estats[i].retained_bytes);
988 }
989 }
990}
991
992static void
993ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
994 unsigned i, bool destroyed) {
995 ctl_arena_t *ctl_arena = arenas_i(i);
996
997 ctl_arena_clear(ctl_arena);
998 ctl_arena_stats_amerge(tsdn, ctl_arena, arena);
999 /* Merge into sum stats as well. */
1000 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed);
1001}
1002
1003static unsigned
1004ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) {
1005 unsigned arena_ind;
1006 ctl_arena_t *ctl_arena;
1007
1008 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) !=
1009 NULL) {
1010 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
1011 arena_ind = ctl_arena->arena_ind;
1012 } else {
1013 arena_ind = ctl_arenas->narenas;
1014 }
1015
1016 /* Trigger stats allocation. */
1017 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) {
1018 return UINT_MAX;
1019 }
1020
1021 /* Initialize new arena. */
1022 if (arena_init(tsd_tsdn(tsd), arena_ind, extent_hooks) == NULL) {
1023 return UINT_MAX;
1024 }
1025
1026 if (arena_ind == ctl_arenas->narenas) {
1027 ctl_arenas->narenas++;
1028 }
1029
1030 return arena_ind;
1031}
1032
1033static void
1034ctl_background_thread_stats_read(tsdn_t *tsdn) {
1035 background_thread_stats_t *stats = &ctl_stats->background_thread;
1036 if (!have_background_thread ||
1037 background_thread_stats_read(tsdn, stats)) {
1038 memset(stats, 0, sizeof(background_thread_stats_t));
1039 nstime_init(&stats->run_interval, 0);
1040 }
1041}
1042
1043static void
1044ctl_refresh(tsdn_t *tsdn) {
1045 unsigned i;
1046 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL);
1047 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas);
1048
1049 /*
1050 * Clear sum stats, since they will be merged into by
1051 * ctl_arena_refresh().
1052 */
1053 ctl_arena_clear(ctl_sarena);
1054
1055 for (i = 0; i < ctl_arenas->narenas; i++) {
1056 tarenas[i] = arena_get(tsdn, i, false);
1057 }
1058
1059 for (i = 0; i < ctl_arenas->narenas; i++) {
1060 ctl_arena_t *ctl_arena = arenas_i(i);
1061 bool initialized = (tarenas[i] != NULL);
1062
1063 ctl_arena->initialized = initialized;
1064 if (initialized) {
1065 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i,
1066 false);
1067 }
1068 }
1069
1070 if (config_stats) {
1071 ctl_stats->allocated = ctl_sarena->astats->allocated_small +
1072 atomic_load_zu(&ctl_sarena->astats->astats.allocated_large,
1073 ATOMIC_RELAXED);
1074 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE);
1075 ctl_stats->metadata = atomic_load_zu(
1076 &ctl_sarena->astats->astats.base, ATOMIC_RELAXED) +
1077 atomic_load_zu(&ctl_sarena->astats->astats.internal,
1078 ATOMIC_RELAXED);
1079 ctl_stats->metadata_thp = atomic_load_zu(
1080 &ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED);
1081 ctl_stats->resident = atomic_load_zu(
1082 &ctl_sarena->astats->astats.resident, ATOMIC_RELAXED);
1083 ctl_stats->mapped = atomic_load_zu(
1084 &ctl_sarena->astats->astats.mapped, ATOMIC_RELAXED);
1085 ctl_stats->retained = atomic_load_zu(
1086 &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED);
1087
1088 ctl_background_thread_stats_read(tsdn);
1089
1090#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \
1091 malloc_mutex_lock(tsdn, &mtx); \
1092 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \
1093 malloc_mutex_unlock(tsdn, &mtx);
1094
1095 if (config_prof && opt_prof) {
1096 READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof,
1097 bt2gctx_mtx);
1098 }
1099 if (have_background_thread) {
1100 READ_GLOBAL_MUTEX_PROF_DATA(
1101 global_prof_mutex_background_thread,
1102 background_thread_lock);
1103 } else {
1104 memset(&ctl_stats->mutex_prof_data[
1105 global_prof_mutex_background_thread], 0,
1106 sizeof(mutex_prof_data_t));
1107 }
1108 /* We own ctl mutex already. */
1109 malloc_mutex_prof_read(tsdn,
1110 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl],
1111 &ctl_mtx);
1112#undef READ_GLOBAL_MUTEX_PROF_DATA
1113 }
1114 ctl_arenas->epoch++;
1115}
1116
1117static bool
1118ctl_init(tsd_t *tsd) {
1119 bool ret;
1120 tsdn_t *tsdn = tsd_tsdn(tsd);
1121
1122 malloc_mutex_lock(tsdn, &ctl_mtx);
1123 if (!ctl_initialized) {
1124 ctl_arena_t *ctl_sarena, *ctl_darena;
1125 unsigned i;
1126
1127 /*
1128 * Allocate demand-zeroed space for pointers to the full
1129 * range of supported arena indices.
1130 */
1131 if (ctl_arenas == NULL) {
1132 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn,
1133 b0get(), sizeof(ctl_arenas_t), QUANTUM);
1134 if (ctl_arenas == NULL) {
1135 ret = true;
1136 goto label_return;
1137 }
1138 }
1139
1140 if (config_stats && ctl_stats == NULL) {
1141 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
1142 sizeof(ctl_stats_t), QUANTUM);
1143 if (ctl_stats == NULL) {
1144 ret = true;
1145 goto label_return;
1146 }
1147 }
1148
1149 /*
1150 * Allocate space for the current full range of arenas
1151 * here rather than doing it lazily elsewhere, in order
1152 * to limit when OOM-caused errors can occur.
1153 */
1154 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false,
1155 true)) == NULL) {
1156 ret = true;
1157 goto label_return;
1158 }
1159 ctl_sarena->initialized = true;
1160
1161 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED,
1162 false, true)) == NULL) {
1163 ret = true;
1164 goto label_return;
1165 }
1166 ctl_arena_clear(ctl_darena);
1167 /*
1168 * Don't toggle ctl_darena to initialized until an arena is
1169 * actually destroyed, so that arena.<i>.initialized can be used
1170 * to query whether the stats are relevant.
1171 */
1172
1173 ctl_arenas->narenas = narenas_total_get();
1174 for (i = 0; i < ctl_arenas->narenas; i++) {
1175 if (arenas_i_impl(tsd, i, false, true) == NULL) {
1176 ret = true;
1177 goto label_return;
1178 }
1179 }
1180
1181 ql_new(&ctl_arenas->destroyed);
1182 ctl_refresh(tsdn);
1183
1184 ctl_initialized = true;
1185 }
1186
1187 ret = false;
1188label_return:
1189 malloc_mutex_unlock(tsdn, &ctl_mtx);
1190 return ret;
1191}
1192
1193static int
1194ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp,
1195 size_t *mibp, size_t *depthp) {
1196 int ret;
1197 const char *elm, *tdot, *dot;
1198 size_t elen, i, j;
1199 const ctl_named_node_t *node;
1200
1201 elm = name;
1202 /* Equivalent to strchrnul(). */
1203 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0');
1204 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1205 if (elen == 0) {
1206 ret = ENOENT;
1207 goto label_return;
1208 }
1209 node = super_root_node;
1210 for (i = 0; i < *depthp; i++) {
1211 assert(node);
1212 assert(node->nchildren > 0);
1213 if (ctl_named_node(node->children) != NULL) {
1214 const ctl_named_node_t *pnode = node;
1215
1216 /* Children are named. */
1217 for (j = 0; j < node->nchildren; j++) {
1218 const ctl_named_node_t *child =
1219 ctl_named_children(node, j);
1220 if (strlen(child->name) == elen &&
1221 strncmp(elm, child->name, elen) == 0) {
1222 node = child;
1223 if (nodesp != NULL) {
1224 nodesp[i] =
1225 (const ctl_node_t *)node;
1226 }
1227 mibp[i] = j;
1228 break;
1229 }
1230 }
1231 if (node == pnode) {
1232 ret = ENOENT;
1233 goto label_return;
1234 }
1235 } else {
1236 uintmax_t index;
1237 const ctl_indexed_node_t *inode;
1238
1239 /* Children are indexed. */
1240 index = malloc_strtoumax(elm, NULL, 10);
1241 if (index == UINTMAX_MAX || index > SIZE_T_MAX) {
1242 ret = ENOENT;
1243 goto label_return;
1244 }
1245
1246 inode = ctl_indexed_node(node->children);
1247 node = inode->index(tsdn, mibp, *depthp, (size_t)index);
1248 if (node == NULL) {
1249 ret = ENOENT;
1250 goto label_return;
1251 }
1252
1253 if (nodesp != NULL) {
1254 nodesp[i] = (const ctl_node_t *)node;
1255 }
1256 mibp[i] = (size_t)index;
1257 }
1258
1259 if (node->ctl != NULL) {
1260 /* Terminal node. */
1261 if (*dot != '\0') {
1262 /*
1263 * The name contains more elements than are
1264 * in this path through the tree.
1265 */
1266 ret = ENOENT;
1267 goto label_return;
1268 }
1269 /* Complete lookup successful. */
1270 *depthp = i + 1;
1271 break;
1272 }
1273
1274 /* Update elm. */
1275 if (*dot == '\0') {
1276 /* No more elements. */
1277 ret = ENOENT;
1278 goto label_return;
1279 }
1280 elm = &dot[1];
1281 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot :
1282 strchr(elm, '\0');
1283 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
1284 }
1285
1286 ret = 0;
1287label_return:
1288 return ret;
1289}
1290
1291int
1292ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
1293 void *newp, size_t newlen) {
1294 int ret;
1295 size_t depth;
1296 ctl_node_t const *nodes[CTL_MAX_DEPTH];
1297 size_t mib[CTL_MAX_DEPTH];
1298 const ctl_named_node_t *node;
1299
1300 if (!ctl_initialized && ctl_init(tsd)) {
1301 ret = EAGAIN;
1302 goto label_return;
1303 }
1304
1305 depth = CTL_MAX_DEPTH;
1306 ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth);
1307 if (ret != 0) {
1308 goto label_return;
1309 }
1310
1311 node = ctl_named_node(nodes[depth-1]);
1312 if (node != NULL && node->ctl) {
1313 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
1314 } else {
1315 /* The name refers to a partial path through the ctl tree. */
1316 ret = ENOENT;
1317 }
1318
1319label_return:
1320 return(ret);
1321}
1322
1323int
1324ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) {
1325 int ret;
1326
1327 if (!ctl_initialized && ctl_init(tsd)) {
1328 ret = EAGAIN;
1329 goto label_return;
1330 }
1331
1332 ret = ctl_lookup(tsd_tsdn(tsd), name, NULL, mibp, miblenp);
1333label_return:
1334 return(ret);
1335}
1336
1337int
1338ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
1339 size_t *oldlenp, void *newp, size_t newlen) {
1340 int ret;
1341 const ctl_named_node_t *node;
1342 size_t i;
1343
1344 if (!ctl_initialized && ctl_init(tsd)) {
1345 ret = EAGAIN;
1346 goto label_return;
1347 }
1348
1349 /* Iterate down the tree. */
1350 node = super_root_node;
1351 for (i = 0; i < miblen; i++) {
1352 assert(node);
1353 assert(node->nchildren > 0);
1354 if (ctl_named_node(node->children) != NULL) {
1355 /* Children are named. */
1356 if (node->nchildren <= mib[i]) {
1357 ret = ENOENT;
1358 goto label_return;
1359 }
1360 node = ctl_named_children(node, mib[i]);
1361 } else {
1362 const ctl_indexed_node_t *inode;
1363
1364 /* Indexed element. */
1365 inode = ctl_indexed_node(node->children);
1366 node = inode->index(tsd_tsdn(tsd), mib, miblen, mib[i]);
1367 if (node == NULL) {
1368 ret = ENOENT;
1369 goto label_return;
1370 }
1371 }
1372 }
1373
1374 /* Call the ctl function. */
1375 if (node && node->ctl) {
1376 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
1377 } else {
1378 /* Partial MIB. */
1379 ret = ENOENT;
1380 }
1381
1382label_return:
1383 return(ret);
1384}
1385
1386bool
1387ctl_boot(void) {
1388 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL,
1389 malloc_mutex_rank_exclusive)) {
1390 return true;
1391 }
1392
1393 ctl_initialized = false;
1394
1395 return false;
1396}
1397
1398void
1399ctl_prefork(tsdn_t *tsdn) {
1400 malloc_mutex_prefork(tsdn, &ctl_mtx);
1401}
1402
1403void
1404ctl_postfork_parent(tsdn_t *tsdn) {
1405 malloc_mutex_postfork_parent(tsdn, &ctl_mtx);
1406}
1407
1408void
1409ctl_postfork_child(tsdn_t *tsdn) {
1410 malloc_mutex_postfork_child(tsdn, &ctl_mtx);
1411}
1412
1413/******************************************************************************/
1414/* *_ctl() functions. */
1415
1416#define READONLY() do { \
1417 if (newp != NULL || newlen != 0) { \
1418 ret = EPERM; \
1419 goto label_return; \
1420 } \
1421} while (0)
1422
1423#define WRITEONLY() do { \
1424 if (oldp != NULL || oldlenp != NULL) { \
1425 ret = EPERM; \
1426 goto label_return; \
1427 } \
1428} while (0)
1429
1430#define READ_XOR_WRITE() do { \
1431 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \
1432 newlen != 0)) { \
1433 ret = EPERM; \
1434 goto label_return; \
1435 } \
1436} while (0)
1437
1438#define READ(v, t) do { \
1439 if (oldp != NULL && oldlenp != NULL) { \
1440 if (*oldlenp != sizeof(t)) { \
1441 size_t copylen = (sizeof(t) <= *oldlenp) \
1442 ? sizeof(t) : *oldlenp; \
1443 memcpy(oldp, (void *)&(v), copylen); \
1444 ret = EINVAL; \
1445 goto label_return; \
1446 } \
1447 *(t *)oldp = (v); \
1448 } \
1449} while (0)
1450
1451#define WRITE(v, t) do { \
1452 if (newp != NULL) { \
1453 if (newlen != sizeof(t)) { \
1454 ret = EINVAL; \
1455 goto label_return; \
1456 } \
1457 (v) = *(t *)newp; \
1458 } \
1459} while (0)
1460
1461#define MIB_UNSIGNED(v, i) do { \
1462 if (mib[i] > UINT_MAX) { \
1463 ret = EFAULT; \
1464 goto label_return; \
1465 } \
1466 v = (unsigned)mib[i]; \
1467} while (0)
1468
1469/*
1470 * There's a lot of code duplication in the following macros due to limitations
1471 * in how nested cpp macros are expanded.
1472 */
1473#define CTL_RO_CLGEN(c, l, n, v, t) \
1474static int \
1475n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1476 size_t *oldlenp, void *newp, size_t newlen) { \
1477 int ret; \
1478 t oldval; \
1479 \
1480 if (!(c)) { \
1481 return ENOENT; \
1482 } \
1483 if (l) { \
1484 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1485 } \
1486 READONLY(); \
1487 oldval = (v); \
1488 READ(oldval, t); \
1489 \
1490 ret = 0; \
1491label_return: \
1492 if (l) { \
1493 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1494 } \
1495 return ret; \
1496}
1497
1498#define CTL_RO_CGEN(c, n, v, t) \
1499static int \
1500n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1501 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1502 int ret; \
1503 t oldval; \
1504 \
1505 if (!(c)) { \
1506 return ENOENT; \
1507 } \
1508 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1509 READONLY(); \
1510 oldval = (v); \
1511 READ(oldval, t); \
1512 \
1513 ret = 0; \
1514label_return: \
1515 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1516 return ret; \
1517}
1518
1519#define CTL_RO_GEN(n, v, t) \
1520static int \
1521n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1522 size_t *oldlenp, void *newp, size_t newlen) { \
1523 int ret; \
1524 t oldval; \
1525 \
1526 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
1527 READONLY(); \
1528 oldval = (v); \
1529 READ(oldval, t); \
1530 \
1531 ret = 0; \
1532label_return: \
1533 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
1534 return ret; \
1535}
1536
1537/*
1538 * ctl_mtx is not acquired, under the assumption that no pertinent data will
1539 * mutate during the call.
1540 */
1541#define CTL_RO_NL_CGEN(c, n, v, t) \
1542static int \
1543n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1544 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1545 int ret; \
1546 t oldval; \
1547 \
1548 if (!(c)) { \
1549 return ENOENT; \
1550 } \
1551 READONLY(); \
1552 oldval = (v); \
1553 READ(oldval, t); \
1554 \
1555 ret = 0; \
1556label_return: \
1557 return ret; \
1558}
1559
1560#define CTL_RO_NL_GEN(n, v, t) \
1561static int \
1562n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1563 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1564 int ret; \
1565 t oldval; \
1566 \
1567 READONLY(); \
1568 oldval = (v); \
1569 READ(oldval, t); \
1570 \
1571 ret = 0; \
1572label_return: \
1573 return ret; \
1574}
1575
1576#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \
1577static int \
1578n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
1579 size_t *oldlenp, void *newp, size_t newlen) { \
1580 int ret; \
1581 t oldval; \
1582 \
1583 if (!(c)) { \
1584 return ENOENT; \
1585 } \
1586 READONLY(); \
1587 oldval = (m(tsd)); \
1588 READ(oldval, t); \
1589 \
1590 ret = 0; \
1591label_return: \
1592 return ret; \
1593}
1594
1595#define CTL_RO_CONFIG_GEN(n, t) \
1596static int \
1597n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
1598 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \
1599 int ret; \
1600 t oldval; \
1601 \
1602 READONLY(); \
1603 oldval = n; \
1604 READ(oldval, t); \
1605 \
1606 ret = 0; \
1607label_return: \
1608 return ret; \
1609}
1610
1611/******************************************************************************/
1612
1613CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
1614
1615static int
1616epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1617 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1618 int ret;
1619 UNUSED uint64_t newval;
1620
1621 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
1622 WRITE(newval, uint64_t);
1623 if (newp != NULL) {
1624 ctl_refresh(tsd_tsdn(tsd));
1625 }
1626 READ(ctl_arenas->epoch, uint64_t);
1627
1628 ret = 0;
1629label_return:
1630 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
1631 return ret;
1632}
1633
1634static int
1635background_thread_ctl(tsd_t *tsd, const size_t *mib,
1636 size_t miblen, void *oldp, size_t *oldlenp,
1637 void *newp, size_t newlen) {
1638 int ret;
1639 bool oldval;
1640
1641 if (!have_background_thread) {
1642 return ENOENT;
1643 }
1644 background_thread_ctl_init(tsd_tsdn(tsd));
1645
1646 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
1647 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
1648 if (newp == NULL) {
1649 oldval = background_thread_enabled();
1650 READ(oldval, bool);
1651 } else {
1652 if (newlen != sizeof(bool)) {
1653 ret = EINVAL;
1654 goto label_return;
1655 }
1656 oldval = background_thread_enabled();
1657 READ(oldval, bool);
1658
1659 bool newval = *(bool *)newp;
1660 if (newval == oldval) {
1661 ret = 0;
1662 goto label_return;
1663 }
1664
1665 background_thread_enabled_set(tsd_tsdn(tsd), newval);
1666 if (newval) {
1667 if (background_threads_enable(tsd)) {
1668 ret = EFAULT;
1669 goto label_return;
1670 }
1671 } else {
1672 if (background_threads_disable(tsd)) {
1673 ret = EFAULT;
1674 goto label_return;
1675 }
1676 }
1677 }
1678 ret = 0;
1679label_return:
1680 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
1681 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
1682
1683 return ret;
1684}
1685
1686static int
1687max_background_threads_ctl(tsd_t *tsd, const size_t *mib,
1688 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
1689 size_t newlen) {
1690 int ret;
1691 size_t oldval;
1692
1693 if (!have_background_thread) {
1694 return ENOENT;
1695 }
1696 background_thread_ctl_init(tsd_tsdn(tsd));
1697
1698 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
1699 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
1700 if (newp == NULL) {
1701 oldval = max_background_threads;
1702 READ(oldval, size_t);
1703 } else {
1704 if (newlen != sizeof(size_t)) {
1705 ret = EINVAL;
1706 goto label_return;
1707 }
1708 oldval = max_background_threads;
1709 READ(oldval, size_t);
1710
1711 size_t newval = *(size_t *)newp;
1712 if (newval == oldval) {
1713 ret = 0;
1714 goto label_return;
1715 }
1716 if (newval > opt_max_background_threads) {
1717 ret = EINVAL;
1718 goto label_return;
1719 }
1720
1721 if (background_thread_enabled()) {
1722 background_thread_enabled_set(tsd_tsdn(tsd), false);
1723 if (background_threads_disable(tsd)) {
1724 ret = EFAULT;
1725 goto label_return;
1726 }
1727 max_background_threads = newval;
1728 background_thread_enabled_set(tsd_tsdn(tsd), true);
1729 if (background_threads_enable(tsd)) {
1730 ret = EFAULT;
1731 goto label_return;
1732 }
1733 } else {
1734 max_background_threads = newval;
1735 }
1736 }
1737 ret = 0;
1738label_return:
1739 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
1740 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
1741
1742 return ret;
1743}
1744
1745/******************************************************************************/
1746
1747CTL_RO_CONFIG_GEN(config_cache_oblivious, bool)
1748CTL_RO_CONFIG_GEN(config_debug, bool)
1749CTL_RO_CONFIG_GEN(config_fill, bool)
1750CTL_RO_CONFIG_GEN(config_lazy_lock, bool)
1751CTL_RO_CONFIG_GEN(config_malloc_conf, const char *)
1752CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool)
1753CTL_RO_CONFIG_GEN(config_prof, bool)
1754CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
1755CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
1756CTL_RO_CONFIG_GEN(config_stats, bool)
1757CTL_RO_CONFIG_GEN(config_utrace, bool)
1758CTL_RO_CONFIG_GEN(config_xmalloc, bool)
1759
1760/******************************************************************************/
1761
1762CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
1763CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
1764CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
1765CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp],
1766 const char *)
1767CTL_RO_NL_GEN(opt_retain, opt_retain, bool)
1768CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
1769CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned)
1770CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena],
1771 const char *)
1772CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t)
1773CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool)
1774CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t)
1775CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t)
1776CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t)
1777CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
1778CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *)
1779CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
1780CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
1781CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
1782CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
1783CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool)
1784CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *)
1785CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit,
1786 size_t)
1787CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
1788CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
1789CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
1790CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
1791CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init,
1792 opt_prof_thread_active_init, bool)
1793CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
1794CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
1795CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
1796CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool)
1797CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool)
1798CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
1799
1800/******************************************************************************/
1801
1802static int
1803thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1804 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1805 int ret;
1806 arena_t *oldarena;
1807 unsigned newind, oldind;
1808
1809 oldarena = arena_choose(tsd, NULL);
1810 if (oldarena == NULL) {
1811 return EAGAIN;
1812 }
1813 newind = oldind = arena_ind_get(oldarena);
1814 WRITE(newind, unsigned);
1815 READ(oldind, unsigned);
1816
1817 if (newind != oldind) {
1818 arena_t *newarena;
1819
1820 if (newind >= narenas_total_get()) {
1821 /* New arena index is out of range. */
1822 ret = EFAULT;
1823 goto label_return;
1824 }
1825
1826 if (have_percpu_arena &&
1827 PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
1828 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) {
1829 /*
1830 * If perCPU arena is enabled, thread_arena
1831 * control is not allowed for the auto arena
1832 * range.
1833 */
1834 ret = EPERM;
1835 goto label_return;
1836 }
1837 }
1838
1839 /* Initialize arena if necessary. */
1840 newarena = arena_get(tsd_tsdn(tsd), newind, true);
1841 if (newarena == NULL) {
1842 ret = EAGAIN;
1843 goto label_return;
1844 }
1845 /* Set new arena/tcache associations. */
1846 arena_migrate(tsd, oldind, newind);
1847 if (tcache_available(tsd)) {
1848 tcache_arena_reassociate(tsd_tsdn(tsd),
1849 tsd_tcachep_get(tsd), newarena);
1850 }
1851 }
1852
1853 ret = 0;
1854label_return:
1855 return ret;
1856}
1857
1858CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get,
1859 uint64_t)
1860CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get,
1861 uint64_t *)
1862CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get,
1863 uint64_t)
1864CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp,
1865 tsd_thread_deallocatedp_get, uint64_t *)
1866
1867static int
1868thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib,
1869 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
1870 size_t newlen) {
1871 int ret;
1872 bool oldval;
1873
1874 oldval = tcache_enabled_get(tsd);
1875 if (newp != NULL) {
1876 if (newlen != sizeof(bool)) {
1877 ret = EINVAL;
1878 goto label_return;
1879 }
1880 tcache_enabled_set(tsd, *(bool *)newp);
1881 }
1882 READ(oldval, bool);
1883
1884 ret = 0;
1885label_return:
1886 return ret;
1887}
1888
1889static int
1890thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib,
1891 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
1892 size_t newlen) {
1893 int ret;
1894
1895 if (!tcache_available(tsd)) {
1896 ret = EFAULT;
1897 goto label_return;
1898 }
1899
1900 READONLY();
1901 WRITEONLY();
1902
1903 tcache_flush(tsd);
1904
1905 ret = 0;
1906label_return:
1907 return ret;
1908}
1909
1910static int
1911thread_prof_name_ctl(tsd_t *tsd, const size_t *mib,
1912 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
1913 size_t newlen) {
1914 int ret;
1915
1916 if (!config_prof) {
1917 return ENOENT;
1918 }
1919
1920 READ_XOR_WRITE();
1921
1922 if (newp != NULL) {
1923 if (newlen != sizeof(const char *)) {
1924 ret = EINVAL;
1925 goto label_return;
1926 }
1927
1928 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) !=
1929 0) {
1930 goto label_return;
1931 }
1932 } else {
1933 const char *oldname = prof_thread_name_get(tsd);
1934 READ(oldname, const char *);
1935 }
1936
1937 ret = 0;
1938label_return:
1939 return ret;
1940}
1941
1942static int
1943thread_prof_active_ctl(tsd_t *tsd, const size_t *mib,
1944 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
1945 size_t newlen) {
1946 int ret;
1947 bool oldval;
1948
1949 if (!config_prof) {
1950 return ENOENT;
1951 }
1952
1953 oldval = prof_thread_active_get(tsd);
1954 if (newp != NULL) {
1955 if (newlen != sizeof(bool)) {
1956 ret = EINVAL;
1957 goto label_return;
1958 }
1959 if (prof_thread_active_set(tsd, *(bool *)newp)) {
1960 ret = EAGAIN;
1961 goto label_return;
1962 }
1963 }
1964 READ(oldval, bool);
1965
1966 ret = 0;
1967label_return:
1968 return ret;
1969}
1970
1971/******************************************************************************/
1972
1973static int
1974tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1975 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1976 int ret;
1977 unsigned tcache_ind;
1978
1979 READONLY();
1980 if (tcaches_create(tsd, &tcache_ind)) {
1981 ret = EFAULT;
1982 goto label_return;
1983 }
1984 READ(tcache_ind, unsigned);
1985
1986 ret = 0;
1987label_return:
1988 return ret;
1989}
1990
1991static int
1992tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
1993 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
1994 int ret;
1995 unsigned tcache_ind;
1996
1997 WRITEONLY();
1998 tcache_ind = UINT_MAX;
1999 WRITE(tcache_ind, unsigned);
2000 if (tcache_ind == UINT_MAX) {
2001 ret = EFAULT;
2002 goto label_return;
2003 }
2004 tcaches_flush(tsd, tcache_ind);
2005
2006 ret = 0;
2007label_return:
2008 return ret;
2009}
2010
2011static int
2012tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2013 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2014 int ret;
2015 unsigned tcache_ind;
2016
2017 WRITEONLY();
2018 tcache_ind = UINT_MAX;
2019 WRITE(tcache_ind, unsigned);
2020 if (tcache_ind == UINT_MAX) {
2021 ret = EFAULT;
2022 goto label_return;
2023 }
2024 tcaches_destroy(tsd, tcache_ind);
2025
2026 ret = 0;
2027label_return:
2028 return ret;
2029}
2030
2031/******************************************************************************/
2032
2033static int
2034arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2035 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2036 int ret;
2037 tsdn_t *tsdn = tsd_tsdn(tsd);
2038 unsigned arena_ind;
2039 bool initialized;
2040
2041 READONLY();
2042 MIB_UNSIGNED(arena_ind, 1);
2043
2044 malloc_mutex_lock(tsdn, &ctl_mtx);
2045 initialized = arenas_i(arena_ind)->initialized;
2046 malloc_mutex_unlock(tsdn, &ctl_mtx);
2047
2048 READ(initialized, bool);
2049
2050 ret = 0;
2051label_return:
2052 return ret;
2053}
2054
2055static void
2056arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) {
2057 malloc_mutex_lock(tsdn, &ctl_mtx);
2058 {
2059 unsigned narenas = ctl_arenas->narenas;
2060
2061 /*
2062 * Access via index narenas is deprecated, and scheduled for
2063 * removal in 6.0.0.
2064 */
2065 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) {
2066 unsigned i;
2067 VARIABLE_ARRAY(arena_t *, tarenas, narenas);
2068
2069 for (i = 0; i < narenas; i++) {
2070 tarenas[i] = arena_get(tsdn, i, false);
2071 }
2072
2073 /*
2074 * No further need to hold ctl_mtx, since narenas and
2075 * tarenas contain everything needed below.
2076 */
2077 malloc_mutex_unlock(tsdn, &ctl_mtx);
2078
2079 for (i = 0; i < narenas; i++) {
2080 if (tarenas[i] != NULL) {
2081 arena_decay(tsdn, tarenas[i], false,
2082 all);
2083 }
2084 }
2085 } else {
2086 arena_t *tarena;
2087
2088 assert(arena_ind < narenas);
2089
2090 tarena = arena_get(tsdn, arena_ind, false);
2091
2092 /* No further need to hold ctl_mtx. */
2093 malloc_mutex_unlock(tsdn, &ctl_mtx);
2094
2095 if (tarena != NULL) {
2096 arena_decay(tsdn, tarena, false, all);
2097 }
2098 }
2099 }
2100}
2101
2102static int
2103arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2104 size_t *oldlenp, void *newp, size_t newlen) {
2105 int ret;
2106 unsigned arena_ind;
2107
2108 READONLY();
2109 WRITEONLY();
2110 MIB_UNSIGNED(arena_ind, 1);
2111 arena_i_decay(tsd_tsdn(tsd), arena_ind, false);
2112
2113 ret = 0;
2114label_return:
2115 return ret;
2116}
2117
2118static int
2119arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2120 size_t *oldlenp, void *newp, size_t newlen) {
2121 int ret;
2122 unsigned arena_ind;
2123
2124 READONLY();
2125 WRITEONLY();
2126 MIB_UNSIGNED(arena_ind, 1);
2127 arena_i_decay(tsd_tsdn(tsd), arena_ind, true);
2128
2129 ret = 0;
2130label_return:
2131 return ret;
2132}
2133
2134static int
2135arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
2136 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind,
2137 arena_t **arena) {
2138 int ret;
2139
2140 READONLY();
2141 WRITEONLY();
2142 MIB_UNSIGNED(*arena_ind, 1);
2143
2144 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false);
2145 if (*arena == NULL || arena_is_auto(*arena)) {
2146 ret = EFAULT;
2147 goto label_return;
2148 }
2149
2150 ret = 0;
2151label_return:
2152 return ret;
2153}
2154
2155static void
2156arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) {
2157 /* Temporarily disable the background thread during arena reset. */
2158 if (have_background_thread) {
2159 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
2160 if (background_thread_enabled()) {
2161 background_thread_info_t *info =
2162 background_thread_info_get(arena_ind);
2163 assert(info->state == background_thread_started);
2164 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2165 info->state = background_thread_paused;
2166 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2167 }
2168 }
2169}
2170
2171static void
2172arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) {
2173 if (have_background_thread) {
2174 if (background_thread_enabled()) {
2175 background_thread_info_t *info =
2176 background_thread_info_get(arena_ind);
2177 assert(info->state == background_thread_paused);
2178 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
2179 info->state = background_thread_started;
2180 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
2181 }
2182 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
2183 }
2184}
2185
2186static int
2187arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2188 size_t *oldlenp, void *newp, size_t newlen) {
2189 int ret;
2190 unsigned arena_ind;
2191 arena_t *arena;
2192
2193 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2194 newp, newlen, &arena_ind, &arena);
2195 if (ret != 0) {
2196 return ret;
2197 }
2198
2199 arena_reset_prepare_background_thread(tsd, arena_ind);
2200 arena_reset(tsd, arena);
2201 arena_reset_finish_background_thread(tsd, arena_ind);
2202
2203 return ret;
2204}
2205
2206static int
2207arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2208 size_t *oldlenp, void *newp, size_t newlen) {
2209 int ret;
2210 unsigned arena_ind;
2211 arena_t *arena;
2212 ctl_arena_t *ctl_darena, *ctl_arena;
2213
2214 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
2215 newp, newlen, &arena_ind, &arena);
2216 if (ret != 0) {
2217 goto label_return;
2218 }
2219
2220 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena,
2221 true) != 0) {
2222 ret = EFAULT;
2223 goto label_return;
2224 }
2225
2226 arena_reset_prepare_background_thread(tsd, arena_ind);
2227 /* Merge stats after resetting and purging arena. */
2228 arena_reset(tsd, arena);
2229 arena_decay(tsd_tsdn(tsd), arena, false, true);
2230 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED);
2231 ctl_darena->initialized = true;
2232 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true);
2233 /* Destroy arena. */
2234 arena_destroy(tsd, arena);
2235 ctl_arena = arenas_i(arena_ind);
2236 ctl_arena->initialized = false;
2237 /* Record arena index for later recycling via arenas.create. */
2238 ql_elm_new(ctl_arena, destroyed_link);
2239 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
2240 arena_reset_finish_background_thread(tsd, arena_ind);
2241
2242 assert(ret == 0);
2243label_return:
2244 return ret;
2245}
2246
2247static int
2248arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2249 size_t *oldlenp, void *newp, size_t newlen) {
2250 int ret;
2251 const char *dss = NULL;
2252 unsigned arena_ind;
2253 dss_prec_t dss_prec_old = dss_prec_limit;
2254 dss_prec_t dss_prec = dss_prec_limit;
2255
2256 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2257 WRITE(dss, const char *);
2258 MIB_UNSIGNED(arena_ind, 1);
2259 if (dss != NULL) {
2260 int i;
2261 bool match = false;
2262
2263 for (i = 0; i < dss_prec_limit; i++) {
2264 if (strcmp(dss_prec_names[i], dss) == 0) {
2265 dss_prec = i;
2266 match = true;
2267 break;
2268 }
2269 }
2270
2271 if (!match) {
2272 ret = EINVAL;
2273 goto label_return;
2274 }
2275 }
2276
2277 /*
2278 * Access via index narenas is deprecated, and scheduled for removal in
2279 * 6.0.0.
2280 */
2281 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
2282 ctl_arenas->narenas) {
2283 if (dss_prec != dss_prec_limit &&
2284 extent_dss_prec_set(dss_prec)) {
2285 ret = EFAULT;
2286 goto label_return;
2287 }
2288 dss_prec_old = extent_dss_prec_get();
2289 } else {
2290 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2291 if (arena == NULL || (dss_prec != dss_prec_limit &&
2292 arena_dss_prec_set(arena, dss_prec))) {
2293 ret = EFAULT;
2294 goto label_return;
2295 }
2296 dss_prec_old = arena_dss_prec_get(arena);
2297 }
2298
2299 dss = dss_prec_names[dss_prec_old];
2300 READ(dss, const char *);
2301
2302 ret = 0;
2303label_return:
2304 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2305 return ret;
2306}
2307
2308static int
2309arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
2310 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
2311 int ret;
2312 unsigned arena_ind;
2313 arena_t *arena;
2314
2315 MIB_UNSIGNED(arena_ind, 1);
2316 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2317 if (arena == NULL) {
2318 ret = EFAULT;
2319 goto label_return;
2320 }
2321
2322 if (oldp != NULL && oldlenp != NULL) {
2323 size_t oldval = dirty ? arena_dirty_decay_ms_get(arena) :
2324 arena_muzzy_decay_ms_get(arena);
2325 READ(oldval, ssize_t);
2326 }
2327 if (newp != NULL) {
2328 if (newlen != sizeof(ssize_t)) {
2329 ret = EINVAL;
2330 goto label_return;
2331 }
2332 if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) {
2333 /*
2334 * By default the huge arena purges eagerly. If it is
2335 * set to non-zero decay time afterwards, background
2336 * thread might be needed.
2337 */
2338 if (background_thread_create(tsd, arena_ind)) {
2339 ret = EFAULT;
2340 goto label_return;
2341 }
2342 }
2343 if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena,
2344 *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd),
2345 arena, *(ssize_t *)newp)) {
2346 ret = EFAULT;
2347 goto label_return;
2348 }
2349 }
2350
2351 ret = 0;
2352label_return:
2353 return ret;
2354}
2355
2356static int
2357arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2358 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2359 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2360 newlen, true);
2361}
2362
2363static int
2364arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2365 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2366 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2367 newlen, false);
2368}
2369
2370static int
2371arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2372 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2373 int ret;
2374 unsigned arena_ind;
2375 arena_t *arena;
2376
2377 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2378 MIB_UNSIGNED(arena_ind, 1);
2379 if (arena_ind < narenas_total_get()) {
2380 extent_hooks_t *old_extent_hooks;
2381 arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
2382 if (arena == NULL) {
2383 if (arena_ind >= narenas_auto) {
2384 ret = EFAULT;
2385 goto label_return;
2386 }
2387 old_extent_hooks =
2388 (extent_hooks_t *)&extent_hooks_default;
2389 READ(old_extent_hooks, extent_hooks_t *);
2390 if (newp != NULL) {
2391 /* Initialize a new arena as a side effect. */
2392 extent_hooks_t *new_extent_hooks
2393 JEMALLOC_CC_SILENCE_INIT(NULL);
2394 WRITE(new_extent_hooks, extent_hooks_t *);
2395 arena = arena_init(tsd_tsdn(tsd), arena_ind,
2396 new_extent_hooks);
2397 if (arena == NULL) {
2398 ret = EFAULT;
2399 goto label_return;
2400 }
2401 }
2402 } else {
2403 if (newp != NULL) {
2404 extent_hooks_t *new_extent_hooks
2405 JEMALLOC_CC_SILENCE_INIT(NULL);
2406 WRITE(new_extent_hooks, extent_hooks_t *);
2407 old_extent_hooks = extent_hooks_set(tsd, arena,
2408 new_extent_hooks);
2409 READ(old_extent_hooks, extent_hooks_t *);
2410 } else {
2411 old_extent_hooks = extent_hooks_get(arena);
2412 READ(old_extent_hooks, extent_hooks_t *);
2413 }
2414 }
2415 } else {
2416 ret = EFAULT;
2417 goto label_return;
2418 }
2419 ret = 0;
2420label_return:
2421 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2422 return ret;
2423}
2424
2425static int
2426arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib,
2427 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2428 size_t newlen) {
2429 int ret;
2430 unsigned arena_ind;
2431 arena_t *arena;
2432
2433 if (!opt_retain) {
2434 /* Only relevant when retain is enabled. */
2435 return ENOENT;
2436 }
2437
2438 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2439 MIB_UNSIGNED(arena_ind, 1);
2440 if (arena_ind < narenas_total_get() && (arena =
2441 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
2442 size_t old_limit, new_limit;
2443 if (newp != NULL) {
2444 WRITE(new_limit, size_t);
2445 }
2446 bool err = arena_retain_grow_limit_get_set(tsd, arena,
2447 &old_limit, newp != NULL ? &new_limit : NULL);
2448 if (!err) {
2449 READ(old_limit, size_t);
2450 ret = 0;
2451 } else {
2452 ret = EFAULT;
2453 }
2454 } else {
2455 ret = EFAULT;
2456 }
2457label_return:
2458 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2459 return ret;
2460}
2461
2462static const ctl_named_node_t *
2463arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
2464 size_t i) {
2465 const ctl_named_node_t *ret;
2466
2467 malloc_mutex_lock(tsdn, &ctl_mtx);
2468 switch (i) {
2469 case MALLCTL_ARENAS_ALL:
2470 case MALLCTL_ARENAS_DESTROYED:
2471 break;
2472 default:
2473 if (i > ctl_arenas->narenas) {
2474 ret = NULL;
2475 goto label_return;
2476 }
2477 break;
2478 }
2479
2480 ret = super_arena_i_node;
2481label_return:
2482 malloc_mutex_unlock(tsdn, &ctl_mtx);
2483 return ret;
2484}
2485
2486/******************************************************************************/
2487
2488static int
2489arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2490 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2491 int ret;
2492 unsigned narenas;
2493
2494 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2495 READONLY();
2496 if (*oldlenp != sizeof(unsigned)) {
2497 ret = EINVAL;
2498 goto label_return;
2499 }
2500 narenas = ctl_arenas->narenas;
2501 READ(narenas, unsigned);
2502
2503 ret = 0;
2504label_return:
2505 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2506 return ret;
2507}
2508
2509static int
2510arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib,
2511 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2512 size_t newlen, bool dirty) {
2513 int ret;
2514
2515 if (oldp != NULL && oldlenp != NULL) {
2516 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() :
2517 arena_muzzy_decay_ms_default_get());
2518 READ(oldval, ssize_t);
2519 }
2520 if (newp != NULL) {
2521 if (newlen != sizeof(ssize_t)) {
2522 ret = EINVAL;
2523 goto label_return;
2524 }
2525 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp)
2526 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) {
2527 ret = EFAULT;
2528 goto label_return;
2529 }
2530 }
2531
2532 ret = 0;
2533label_return:
2534 return ret;
2535}
2536
2537static int
2538arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2539 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2540 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2541 newlen, true);
2542}
2543
2544static int
2545arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2546 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2547 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
2548 newlen, false);
2549}
2550
2551CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
2552CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
2553CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t)
2554CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
2555CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned)
2556CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
2557CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
2558CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)
2559CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t)
2560static const ctl_named_node_t *
2561arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib,
2562 size_t miblen, size_t i) {
2563 if (i > SC_NBINS) {
2564 return NULL;
2565 }
2566 return super_arenas_bin_i_node;
2567}
2568
2569CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned)
2570CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]),
2571 size_t)
2572static const ctl_named_node_t *
2573arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib,
2574 size_t miblen, size_t i) {
2575 if (i > SC_NSIZES - SC_NBINS) {
2576 return NULL;
2577 }
2578 return super_arenas_lextent_i_node;
2579}
2580
2581static int
2582arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2583 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2584 int ret;
2585 extent_hooks_t *extent_hooks;
2586 unsigned arena_ind;
2587
2588 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2589
2590 extent_hooks = (extent_hooks_t *)&extent_hooks_default;
2591 WRITE(extent_hooks, extent_hooks_t *);
2592 if ((arena_ind = ctl_arena_init(tsd, extent_hooks)) == UINT_MAX) {
2593 ret = EAGAIN;
2594 goto label_return;
2595 }
2596 READ(arena_ind, unsigned);
2597
2598 ret = 0;
2599label_return:
2600 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2601 return ret;
2602}
2603
2604static int
2605arenas_lookup_ctl(tsd_t *tsd, const size_t *mib,
2606 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2607 size_t newlen) {
2608 int ret;
2609 unsigned arena_ind;
2610 void *ptr;
2611 extent_t *extent;
2612 arena_t *arena;
2613
2614 ptr = NULL;
2615 ret = EINVAL;
2616 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
2617 WRITE(ptr, void *);
2618 extent = iealloc(tsd_tsdn(tsd), ptr);
2619 if (extent == NULL)
2620 goto label_return;
2621
2622 arena = extent_arena_get(extent);
2623 if (arena == NULL)
2624 goto label_return;
2625
2626 arena_ind = arena_ind_get(arena);
2627 READ(arena_ind, unsigned);
2628
2629 ret = 0;
2630label_return:
2631 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
2632 return ret;
2633}
2634
2635/******************************************************************************/
2636
2637static int
2638prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib,
2639 size_t miblen, void *oldp, size_t *oldlenp, void *newp,
2640 size_t newlen) {
2641 int ret;
2642 bool oldval;
2643
2644 if (!config_prof) {
2645 return ENOENT;
2646 }
2647
2648 if (newp != NULL) {
2649 if (newlen != sizeof(bool)) {
2650 ret = EINVAL;
2651 goto label_return;
2652 }
2653 oldval = prof_thread_active_init_set(tsd_tsdn(tsd),
2654 *(bool *)newp);
2655 } else {
2656 oldval = prof_thread_active_init_get(tsd_tsdn(tsd));
2657 }
2658 READ(oldval, bool);
2659
2660 ret = 0;
2661label_return:
2662 return ret;
2663}
2664
2665static int
2666prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2667 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2668 int ret;
2669 bool oldval;
2670
2671 if (!config_prof) {
2672 return ENOENT;
2673 }
2674
2675 if (newp != NULL) {
2676 if (newlen != sizeof(bool)) {
2677 ret = EINVAL;
2678 goto label_return;
2679 }
2680 oldval = prof_active_set(tsd_tsdn(tsd), *(bool *)newp);
2681 } else {
2682 oldval = prof_active_get(tsd_tsdn(tsd));
2683 }
2684 READ(oldval, bool);
2685
2686 ret = 0;
2687label_return:
2688 return ret;
2689}
2690
2691static int
2692prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2693 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2694 int ret;
2695 const char *filename = NULL;
2696
2697 if (!config_prof) {
2698 return ENOENT;
2699 }
2700
2701 WRITEONLY();
2702 WRITE(filename, const char *);
2703
2704 if (prof_mdump(tsd, filename)) {
2705 ret = EFAULT;
2706 goto label_return;
2707 }
2708
2709 ret = 0;
2710label_return:
2711 return ret;
2712}
2713
2714static int
2715prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2716 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2717 int ret;
2718 bool oldval;
2719
2720 if (!config_prof) {
2721 return ENOENT;
2722 }
2723
2724 if (newp != NULL) {
2725 if (newlen != sizeof(bool)) {
2726 ret = EINVAL;
2727 goto label_return;
2728 }
2729 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp);
2730 } else {
2731 oldval = prof_gdump_get(tsd_tsdn(tsd));
2732 }
2733 READ(oldval, bool);
2734
2735 ret = 0;
2736label_return:
2737 return ret;
2738}
2739
2740static int
2741prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
2742 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
2743 int ret;
2744 size_t lg_sample = lg_prof_sample;
2745
2746 if (!config_prof) {
2747 return ENOENT;
2748 }
2749
2750 WRITEONLY();
2751 WRITE(lg_sample, size_t);
2752 if (lg_sample >= (sizeof(uint64_t) << 3)) {
2753 lg_sample = (sizeof(uint64_t) << 3) - 1;
2754 }
2755
2756 prof_reset(tsd, lg_sample);
2757
2758 ret = 0;
2759label_return:
2760 return ret;
2761}
2762
2763CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t)
2764CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
2765
2766static int
2767prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2768 size_t *oldlenp, void *newp, size_t newlen) {
2769 int ret;
2770
2771 const char *filename = NULL;
2772
2773 if (!config_prof) {
2774 return ENOENT;
2775 }
2776
2777 WRITEONLY();
2778 WRITE(filename, const char *);
2779
2780 if (prof_log_start(tsd_tsdn(tsd), filename)) {
2781 ret = EFAULT;
2782 goto label_return;
2783 }
2784
2785 ret = 0;
2786label_return:
2787 return ret;
2788}
2789
2790static int
2791prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
2792 size_t *oldlenp, void *newp, size_t newlen) {
2793 if (!config_prof) {
2794 return ENOENT;
2795 }
2796
2797 if (prof_log_stop(tsd_tsdn(tsd))) {
2798 return EFAULT;
2799 }
2800
2801 return 0;
2802}
2803
2804/******************************************************************************/
2805
2806CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
2807CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
2808CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
2809CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t)
2810CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
2811CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
2812CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
2813
2814CTL_RO_CGEN(config_stats, stats_background_thread_num_threads,
2815 ctl_stats->background_thread.num_threads, size_t)
2816CTL_RO_CGEN(config_stats, stats_background_thread_num_runs,
2817 ctl_stats->background_thread.num_runs, uint64_t)
2818CTL_RO_CGEN(config_stats, stats_background_thread_run_interval,
2819 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t)
2820
2821CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *)
2822CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms,
2823 ssize_t)
2824CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms,
2825 ssize_t)
2826CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned)
2827CTL_RO_GEN(stats_arenas_i_uptime,
2828 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t)
2829CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t)
2830CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
2831CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
2832CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
2833 atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED),
2834 size_t)
2835CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
2836 atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED),
2837 size_t)
2838CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
2839 atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail,
2840 ATOMIC_RELAXED),
2841 size_t)
2842
2843CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge,
2844 ctl_arena_stats_read_u64(
2845 &arenas_i(mib[2])->astats->astats.decay_dirty.npurge), uint64_t)
2846CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise,
2847 ctl_arena_stats_read_u64(
2848 &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t)
2849CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged,
2850 ctl_arena_stats_read_u64(
2851 &arenas_i(mib[2])->astats->astats.decay_dirty.purged), uint64_t)
2852
2853CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge,
2854 ctl_arena_stats_read_u64(
2855 &arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), uint64_t)
2856CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise,
2857 ctl_arena_stats_read_u64(
2858 &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t)
2859CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged,
2860 ctl_arena_stats_read_u64(
2861 &arenas_i(mib[2])->astats->astats.decay_muzzy.purged), uint64_t)
2862
2863CTL_RO_CGEN(config_stats, stats_arenas_i_base,
2864 atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED),
2865 size_t)
2866CTL_RO_CGEN(config_stats, stats_arenas_i_internal,
2867 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED),
2868 size_t)
2869CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp,
2870 atomic_load_zu(&arenas_i(mib[2])->astats->astats.metadata_thp,
2871 ATOMIC_RELAXED), size_t)
2872CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes,
2873 atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes,
2874 ATOMIC_RELAXED), size_t)
2875CTL_RO_CGEN(config_stats, stats_arenas_i_resident,
2876 atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED),
2877 size_t)
2878CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
2879 atomic_load_zu(&arenas_i(mib[2])->astats->astats.abandoned_vm,
2880 ATOMIC_RELAXED), size_t)
2881
2882CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
2883 arenas_i(mib[2])->astats->allocated_small, size_t)
2884CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,
2885 arenas_i(mib[2])->astats->nmalloc_small, uint64_t)
2886CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc,
2887 arenas_i(mib[2])->astats->ndalloc_small, uint64_t)
2888CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
2889 arenas_i(mib[2])->astats->nrequests_small, uint64_t)
2890CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills,
2891 arenas_i(mib[2])->astats->nfills_small, uint64_t)
2892CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes,
2893 arenas_i(mib[2])->astats->nflushes_small, uint64_t)
2894CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated,
2895 atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large,
2896 ATOMIC_RELAXED), size_t)
2897CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc,
2898 ctl_arena_stats_read_u64(
2899 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t)
2900CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc,
2901 ctl_arena_stats_read_u64(
2902 &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t)
2903CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
2904 ctl_arena_stats_read_u64(
2905 &arenas_i(mib[2])->astats->astats.nrequests_large), uint64_t)
2906/*
2907 * Note: "nmalloc_large" here instead of "nfills" in the read. This is
2908 * intentional (large has no batch fill).
2909 */
2910CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills,
2911 ctl_arena_stats_read_u64(
2912 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t)
2913CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes,
2914 ctl_arena_stats_read_u64(
2915 &arenas_i(mib[2])->astats->astats.nflushes_large), uint64_t)
2916
2917/* Lock profiling related APIs below. */
2918#define RO_MUTEX_CTL_GEN(n, l) \
2919CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \
2920 l.n_lock_ops, uint64_t) \
2921CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \
2922 l.n_wait_times, uint64_t) \
2923CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \
2924 l.n_spin_acquired, uint64_t) \
2925CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \
2926 l.n_owner_switches, uint64_t) \
2927CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \
2928 nstime_ns(&l.tot_wait_time), uint64_t) \
2929CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \
2930 nstime_ns(&l.max_wait_time), uint64_t) \
2931CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \
2932 l.max_n_thds, uint32_t)
2933
2934/* Global mutexes. */
2935#define OP(mtx) \
2936 RO_MUTEX_CTL_GEN(mutexes_##mtx, \
2937 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx])
2938MUTEX_PROF_GLOBAL_MUTEXES
2939#undef OP
2940
2941/* Per arena mutexes */
2942#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \
2943 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx])
2944MUTEX_PROF_ARENA_MUTEXES
2945#undef OP
2946
2947/* tcache bin mutex */
2948RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex,
2949 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data)
2950#undef RO_MUTEX_CTL_GEN
2951
2952/* Resets all mutex stats, including global, arena and bin mutexes. */
2953static int
2954stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
2955 size_t miblen, void *oldp, size_t *oldlenp,
2956 void *newp, size_t newlen) {
2957 if (!config_stats) {
2958 return ENOENT;
2959 }
2960
2961 tsdn_t *tsdn = tsd_tsdn(tsd);
2962
2963#define MUTEX_PROF_RESET(mtx) \
2964 malloc_mutex_lock(tsdn, &mtx); \
2965 malloc_mutex_prof_data_reset(tsdn, &mtx); \
2966 malloc_mutex_unlock(tsdn, &mtx);
2967
2968 /* Global mutexes: ctl and prof. */
2969 MUTEX_PROF_RESET(ctl_mtx);
2970 if (have_background_thread) {
2971 MUTEX_PROF_RESET(background_thread_lock);
2972 }
2973 if (config_prof && opt_prof) {
2974 MUTEX_PROF_RESET(bt2gctx_mtx);
2975 }
2976
2977
2978 /* Per arena mutexes. */
2979 unsigned n = narenas_total_get();
2980
2981 for (unsigned i = 0; i < n; i++) {
2982 arena_t *arena = arena_get(tsdn, i, false);
2983 if (!arena) {
2984 continue;
2985 }
2986 MUTEX_PROF_RESET(arena->large_mtx);
2987 MUTEX_PROF_RESET(arena->extent_avail_mtx);
2988 MUTEX_PROF_RESET(arena->extents_dirty.mtx);
2989 MUTEX_PROF_RESET(arena->extents_muzzy.mtx);
2990 MUTEX_PROF_RESET(arena->extents_retained.mtx);
2991 MUTEX_PROF_RESET(arena->decay_dirty.mtx);
2992 MUTEX_PROF_RESET(arena->decay_muzzy.mtx);
2993 MUTEX_PROF_RESET(arena->tcache_ql_mtx);
2994 MUTEX_PROF_RESET(arena->base->mtx);
2995
2996 for (szind_t i = 0; i < SC_NBINS; i++) {
2997 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
2998 bin_t *bin = &arena->bins[i].bin_shards[j];
2999 MUTEX_PROF_RESET(bin->lock);
3000 }
3001 }
3002 }
3003#undef MUTEX_PROF_RESET
3004 return 0;
3005}
3006
3007CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
3008 arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t)
3009CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc,
3010 arenas_i(mib[2])->astats->bstats[mib[4]].ndalloc, uint64_t)
3011CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests,
3012 arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t)
3013CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs,
3014 arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t)
3015CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills,
3016 arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t)
3017CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes,
3018 arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t)
3019CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs,
3020 arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t)
3021CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs,
3022 arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t)
3023CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs,
3024 arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t)
3025CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs,
3026 arenas_i(mib[2])->astats->bstats[mib[4]].nonfull_slabs, size_t)
3027
3028static const ctl_named_node_t *
3029stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib,
3030 size_t miblen, size_t j) {
3031 if (j > SC_NBINS) {
3032 return NULL;
3033 }
3034 return super_stats_arenas_i_bins_j_node;
3035}
3036
3037CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc,
3038 ctl_arena_stats_read_u64(
3039 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t)
3040CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
3041 ctl_arena_stats_read_u64(
3042 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t)
3043CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
3044 ctl_arena_stats_read_u64(
3045 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t)
3046CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
3047 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t)
3048
3049static const ctl_named_node_t *
3050stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib,
3051 size_t miblen, size_t j) {
3052 if (j > SC_NSIZES - SC_NBINS) {
3053 return NULL;
3054 }
3055 return super_stats_arenas_i_lextents_j_node;
3056}
3057
3058CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty,
3059 atomic_load_zu(
3060 &arenas_i(mib[2])->astats->estats[mib[4]].ndirty,
3061 ATOMIC_RELAXED), size_t);
3062CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy,
3063 atomic_load_zu(
3064 &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy,
3065 ATOMIC_RELAXED), size_t);
3066CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained,
3067 atomic_load_zu(
3068 &arenas_i(mib[2])->astats->estats[mib[4]].nretained,
3069 ATOMIC_RELAXED), size_t);
3070CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes,
3071 atomic_load_zu(
3072 &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes,
3073 ATOMIC_RELAXED), size_t);
3074CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes,
3075 atomic_load_zu(
3076 &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes,
3077 ATOMIC_RELAXED), size_t);
3078CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes,
3079 atomic_load_zu(
3080 &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes,
3081 ATOMIC_RELAXED), size_t);
3082
3083static const ctl_named_node_t *
3084stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib,
3085 size_t miblen, size_t j) {
3086 if (j >= SC_NPSIZES) {
3087 return NULL;
3088 }
3089 return super_stats_arenas_i_extents_j_node;
3090}
3091
3092static bool
3093ctl_arenas_i_verify(size_t i) {
3094 size_t a = arenas_i2a_impl(i, true, true);
3095 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) {
3096 return true;
3097 }
3098
3099 return false;
3100}
3101
3102static const ctl_named_node_t *
3103stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
3104 size_t miblen, size_t i) {
3105 const ctl_named_node_t *ret;
3106
3107 malloc_mutex_lock(tsdn, &ctl_mtx);
3108 if (ctl_arenas_i_verify(i)) {
3109 ret = NULL;
3110 goto label_return;
3111 }
3112
3113 ret = super_stats_arenas_i_node;
3114label_return:
3115 malloc_mutex_unlock(tsdn, &ctl_mtx);
3116 return ret;
3117}
3118
3119static int
3120experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3121 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3122 int ret;
3123 if (oldp == NULL || oldlenp == NULL|| newp == NULL) {
3124 ret = EINVAL;
3125 goto label_return;
3126 }
3127 /*
3128 * Note: this is a *private* struct. This is an experimental interface;
3129 * forcing the user to know the jemalloc internals well enough to
3130 * extract the ABI hopefully ensures nobody gets too comfortable with
3131 * this API, which can change at a moment's notice.
3132 */
3133 hooks_t hooks;
3134 WRITE(hooks, hooks_t);
3135 void *handle = hook_install(tsd_tsdn(tsd), &hooks);
3136 if (handle == NULL) {
3137 ret = EAGAIN;
3138 goto label_return;
3139 }
3140 READ(handle, void *);
3141
3142 ret = 0;
3143label_return:
3144 return ret;
3145}
3146
3147static int
3148experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
3149 void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3150 int ret;
3151 WRITEONLY();
3152 void *handle = NULL;
3153 WRITE(handle, void *);
3154 if (handle == NULL) {
3155 ret = EINVAL;
3156 goto label_return;
3157 }
3158 hook_remove(tsd_tsdn(tsd), handle);
3159 ret = 0;
3160label_return:
3161 return ret;
3162}
3163
3164/*
3165 * Output six memory utilization entries for an input pointer, the first one of
3166 * type (void *) and the remaining five of type size_t, describing the following
3167 * (in the same order):
3168 *
3169 * (a) memory address of the extent a potential reallocation would go into,
3170 * == the five fields below describe about the extent the pointer resides in ==
3171 * (b) number of free regions in the extent,
3172 * (c) number of regions in the extent,
3173 * (d) size of the extent in terms of bytes,
3174 * (e) total number of free regions in the bin the extent belongs to, and
3175 * (f) total number of regions in the bin the extent belongs to.
3176 *
3177 * Note that "(e)" and "(f)" are only available when stats are enabled;
3178 * otherwise their values are undefined.
3179 *
3180 * This API is mainly intended for small class allocations, where extents are
3181 * used as slab.
3182 *
3183 * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)"
3184 * will be zero (if stats are enabled; otherwise undefined). The other three
3185 * fields will be properly set though the values are trivial: "(b)" will be 0,
3186 * "(c)" will be 1, and "(d)" will be the usable size.
3187 *
3188 * The input pointer and size are respectively passed in by newp and newlen,
3189 * and the output fields and size are respectively oldp and *oldlenp.
3190 *
3191 * It can be beneficial to define the following macros to make it easier to
3192 * access the output:
3193 *
3194 * #define SLABCUR_READ(out) (*(void **)out)
3195 * #define COUNTS(out) ((size_t *)((void **)out + 1))
3196 * #define NFREE_READ(out) COUNTS(out)[0]
3197 * #define NREGS_READ(out) COUNTS(out)[1]
3198 * #define SIZE_READ(out) COUNTS(out)[2]
3199 * #define BIN_NFREE_READ(out) COUNTS(out)[3]
3200 * #define BIN_NREGS_READ(out) COUNTS(out)[4]
3201 *
3202 * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test
3203 * test_query in test/unit/extent_util.c for an example.
3204 *
3205 * For a typical defragmentation workflow making use of this API for
3206 * understanding the fragmentation level, please refer to the comment for
3207 * experimental_utilization_batch_query_ctl.
3208 *
3209 * It's up to the application how to determine the significance of
3210 * fragmentation relying on the outputs returned. Possible choices are:
3211 *
3212 * (a) if extent utilization ratio is below certain threshold,
3213 * (b) if extent memory consumption is above certain threshold,
3214 * (c) if extent utilization ratio is significantly below bin utilization ratio,
3215 * (d) if input pointer deviates a lot from potential reallocation address, or
3216 * (e) some selection/combination of the above.
3217 *
3218 * The caller needs to make sure that the input/output arguments are valid,
3219 * in particular, that the size of the output is correct, i.e.:
3220 *
3221 * *oldlenp = sizeof(void *) + sizeof(size_t) * 5
3222 *
3223 * Otherwise, the function immediately returns EINVAL without touching anything.
3224 *
3225 * In the rare case where there's no associated extent found for the input
3226 * pointer, the function zeros out all output fields and return. Please refer
3227 * to the comment for experimental_utilization_batch_query_ctl to understand the
3228 * motivation from C++.
3229 */
3230static int
3231experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib,
3232 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3233 int ret;
3234
3235 assert(sizeof(extent_util_stats_verbose_t)
3236 == sizeof(void *) + sizeof(size_t) * 5);
3237
3238 if (oldp == NULL || oldlenp == NULL
3239 || *oldlenp != sizeof(extent_util_stats_verbose_t)
3240 || newp == NULL) {
3241 ret = EINVAL;
3242 goto label_return;
3243 }
3244
3245 void *ptr = NULL;
3246 WRITE(ptr, void *);
3247 extent_util_stats_verbose_t *util_stats
3248 = (extent_util_stats_verbose_t *)oldp;
3249 extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr,
3250 &util_stats->nfree, &util_stats->nregs, &util_stats->size,
3251 &util_stats->bin_nfree, &util_stats->bin_nregs,
3252 &util_stats->slabcur_addr);
3253 ret = 0;
3254
3255label_return:
3256 return ret;
3257}
3258
3259/*
3260 * Given an input array of pointers, output three memory utilization entries of
3261 * type size_t for each input pointer about the extent it resides in:
3262 *
3263 * (a) number of free regions in the extent,
3264 * (b) number of regions in the extent, and
3265 * (c) size of the extent in terms of bytes.
3266 *
3267 * This API is mainly intended for small class allocations, where extents are
3268 * used as slab. In case of large class allocations, the outputs are trivial:
3269 * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size.
3270 *
3271 * Note that multiple input pointers may reside on a same extent so the output
3272 * fields may contain duplicates.
3273 *
3274 * The format of the input/output looks like:
3275 *
3276 * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions
3277 * | output[1]: 1st_extent_n_regions
3278 * | output[2]: 1st_extent_size
3279 * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions
3280 * | output[4]: 2nd_extent_n_regions
3281 * | output[5]: 2nd_extent_size
3282 * ... | ...
3283 *
3284 * The input array and size are respectively passed in by newp and newlen, and
3285 * the output array and size are respectively oldp and *oldlenp.
3286 *
3287 * It can be beneficial to define the following macros to make it easier to
3288 * access the output:
3289 *
3290 * #define NFREE_READ(out, i) out[(i) * 3]
3291 * #define NREGS_READ(out, i) out[(i) * 3 + 1]
3292 * #define SIZE_READ(out, i) out[(i) * 3 + 2]
3293 *
3294 * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit
3295 * test test_batch in test/unit/extent_util.c for a concrete example.
3296 *
3297 * A typical workflow would be composed of the following steps:
3298 *
3299 * (1) flush tcache: mallctl("thread.tcache.flush", ...)
3300 * (2) initialize input array of pointers to query fragmentation
3301 * (3) allocate output array to hold utilization statistics
3302 * (4) query utilization: mallctl("experimental.utilization.batch_query", ...)
3303 * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here
3304 * (6) disable tcache: mallctl("thread.tcache.enabled", ...)
3305 * (7) defragment allocations with significant fragmentation, e.g.:
3306 * for each allocation {
3307 * if it's fragmented {
3308 * malloc(...);
3309 * memcpy(...);
3310 * free(...);
3311 * }
3312 * }
3313 * (8) enable tcache: mallctl("thread.tcache.enabled", ...)
3314 *
3315 * The application can determine the significance of fragmentation themselves
3316 * relying on the statistics returned, both at the overall level i.e. step "(5)"
3317 * and at individual allocation level i.e. within step "(7)". Possible choices
3318 * are:
3319 *
3320 * (a) whether memory utilization ratio is below certain threshold,
3321 * (b) whether memory consumption is above certain threshold, or
3322 * (c) some combination of the two.
3323 *
3324 * The caller needs to make sure that the input/output arrays are valid and
3325 * their sizes are proper as well as matched, meaning:
3326 *
3327 * (a) newlen = n_pointers * sizeof(const void *)
3328 * (b) *oldlenp = n_pointers * sizeof(size_t) * 3
3329 * (c) n_pointers > 0
3330 *
3331 * Otherwise, the function immediately returns EINVAL without touching anything.
3332 *
3333 * In the rare case where there's no associated extent found for some pointers,
3334 * rather than immediately terminating the computation and raising an error,
3335 * the function simply zeros out the corresponding output fields and continues
3336 * the computation until all input pointers are handled. The motivations of
3337 * such a design are as follows:
3338 *
3339 * (a) The function always either processes nothing or processes everything, and
3340 * never leaves the output half touched and half untouched.
3341 *
3342 * (b) It facilitates usage needs especially common in C++. A vast variety of
3343 * C++ objects are instantiated with multiple dynamic memory allocations. For
3344 * example, std::string and std::vector typically use at least two allocations,
3345 * one for the metadata and one for the actual content. Other types may use
3346 * even more allocations. When inquiring about utilization statistics, the
3347 * caller often wants to examine into all such allocations, especially internal
3348 * one(s), rather than just the topmost one. The issue comes when some
3349 * implementations do certain optimizations to reduce/aggregate some internal
3350 * allocations, e.g. putting short strings directly into the metadata, and such
3351 * decisions are not known to the caller. Therefore, we permit pointers to
3352 * memory usages that may not be returned by previous malloc calls, and we
3353 * provide the caller a convenient way to identify such cases.
3354 */
3355static int
3356experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib,
3357 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3358 int ret;
3359
3360 assert(sizeof(extent_util_stats_t) == sizeof(size_t) * 3);
3361
3362 const size_t len = newlen / sizeof(const void *);
3363 if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0
3364 || newlen != len * sizeof(const void *)
3365 || *oldlenp != len * sizeof(extent_util_stats_t)) {
3366 ret = EINVAL;
3367 goto label_return;
3368 }
3369
3370 void **ptrs = (void **)newp;
3371 extent_util_stats_t *util_stats = (extent_util_stats_t *)oldp;
3372 size_t i;
3373 for (i = 0; i < len; ++i) {
3374 extent_util_stats_get(tsd_tsdn(tsd), ptrs[i],
3375 &util_stats[i].nfree, &util_stats[i].nregs,
3376 &util_stats[i].size);
3377 }
3378 ret = 0;
3379
3380label_return:
3381 return ret;
3382}
3383
3384static const ctl_named_node_t *
3385experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib,
3386 size_t miblen, size_t i) {
3387 const ctl_named_node_t *ret;
3388
3389 malloc_mutex_lock(tsdn, &ctl_mtx);
3390 if (ctl_arenas_i_verify(i)) {
3391 ret = NULL;
3392 goto label_return;
3393 }
3394 ret = super_experimental_arenas_i_node;
3395label_return:
3396 malloc_mutex_unlock(tsdn, &ctl_mtx);
3397 return ret;
3398}
3399
3400static int
3401experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib,
3402 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
3403 if (!config_stats) {
3404 return ENOENT;
3405 }
3406 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) {
3407 return EINVAL;
3408 }
3409
3410 unsigned arena_ind;
3411 arena_t *arena;
3412 int ret;
3413 size_t *pactivep;
3414
3415 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
3416 READONLY();
3417 MIB_UNSIGNED(arena_ind, 2);
3418 if (arena_ind < narenas_total_get() && (arena =
3419 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
3420#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \
3421 defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER)
3422 /* Expose the underlying counter for fast read. */
3423 pactivep = (size_t *)&(arena->nactive.repr);
3424 READ(pactivep, size_t *);
3425 ret = 0;
3426#else
3427 ret = EFAULT;
3428#endif
3429 } else {
3430 ret = EFAULT;
3431 }
3432label_return:
3433 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
3434 return ret;
3435}
3436