1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4/*
5 * This file is logically part of the PA module. While pa.c contains the core
6 * allocator functionality, this file contains boring integration functionality;
7 * things like the pre- and post- fork handlers, and stats merging for CTL
8 * refreshes.
9 */
10
11void
12pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
13 malloc_mutex_prefork(tsdn, &shard->pac.decay_dirty.mtx);
14 malloc_mutex_prefork(tsdn, &shard->pac.decay_muzzy.mtx);
15}
16
17void
18pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
19 if (shard->ever_used_hpa) {
20 sec_prefork2(tsdn, &shard->hpa_sec);
21 }
22}
23
24void
25pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
26 malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
27 if (shard->ever_used_hpa) {
28 hpa_shard_prefork3(tsdn, &shard->hpa_shard);
29 }
30}
31
32void
33pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
34 ecache_prefork(tsdn, &shard->pac.ecache_dirty);
35 ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
36 ecache_prefork(tsdn, &shard->pac.ecache_retained);
37 if (shard->ever_used_hpa) {
38 hpa_shard_prefork4(tsdn, &shard->hpa_shard);
39 }
40}
41
42void
43pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard) {
44 edata_cache_prefork(tsdn, &shard->edata_cache);
45}
46
47void
48pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
49 edata_cache_postfork_parent(tsdn, &shard->edata_cache);
50 ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
51 ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
52 ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
53 malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
54 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
55 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
56 if (shard->ever_used_hpa) {
57 sec_postfork_parent(tsdn, &shard->hpa_sec);
58 hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
59 }
60}
61
62void
63pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
64 edata_cache_postfork_child(tsdn, &shard->edata_cache);
65 ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
66 ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
67 ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
68 malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
69 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
70 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
71 if (shard->ever_used_hpa) {
72 sec_postfork_child(tsdn, &shard->hpa_sec);
73 hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
74 }
75}
76
77void
78pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty,
79 size_t *nmuzzy) {
80 *nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
81 *ndirty += ecache_npages_get(&shard->pac.ecache_dirty);
82 *nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy);
83}
84
85void
86pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
87 pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
88 hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
89 size_t *resident) {
90 cassert(config_stats);
91
92 pa_shard_stats_out->pac_stats.retained +=
93 ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
94 pa_shard_stats_out->edata_avail += atomic_load_zu(
95 &shard->edata_cache.count, ATOMIC_RELAXED);
96
97 size_t resident_pgs = 0;
98 resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
99 resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty);
100 *resident += (resident_pgs << LG_PAGE);
101
102 /* Dirty decay stats */
103 locked_inc_u64_unsynchronized(
104 &pa_shard_stats_out->pac_stats.decay_dirty.npurge,
105 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
106 &shard->pac.stats->decay_dirty.npurge));
107 locked_inc_u64_unsynchronized(
108 &pa_shard_stats_out->pac_stats.decay_dirty.nmadvise,
109 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
110 &shard->pac.stats->decay_dirty.nmadvise));
111 locked_inc_u64_unsynchronized(
112 &pa_shard_stats_out->pac_stats.decay_dirty.purged,
113 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
114 &shard->pac.stats->decay_dirty.purged));
115
116 /* Muzzy decay stats */
117 locked_inc_u64_unsynchronized(
118 &pa_shard_stats_out->pac_stats.decay_muzzy.npurge,
119 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
120 &shard->pac.stats->decay_muzzy.npurge));
121 locked_inc_u64_unsynchronized(
122 &pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise,
123 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
124 &shard->pac.stats->decay_muzzy.nmadvise));
125 locked_inc_u64_unsynchronized(
126 &pa_shard_stats_out->pac_stats.decay_muzzy.purged,
127 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
128 &shard->pac.stats->decay_muzzy.purged));
129
130 atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm,
131 atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));
132
133 for (pszind_t i = 0; i < SC_NPSIZES; i++) {
134 size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
135 retained_bytes;
136 dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
137 muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
138 retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
139 dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
140 muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
141 retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained,
142 i);
143
144 estats_out[i].ndirty = dirty;
145 estats_out[i].nmuzzy = muzzy;
146 estats_out[i].nretained = retained;
147 estats_out[i].dirty_bytes = dirty_bytes;
148 estats_out[i].muzzy_bytes = muzzy_bytes;
149 estats_out[i].retained_bytes = retained_bytes;
150 }
151
152 if (shard->ever_used_hpa) {
153 hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
154 sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
155 }
156}
157
158static void
159pa_shard_mtx_stats_read_single(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data,
160 malloc_mutex_t *mtx, int ind) {
161 malloc_mutex_lock(tsdn, mtx);
162 malloc_mutex_prof_read(tsdn, &mutex_prof_data[ind], mtx);
163 malloc_mutex_unlock(tsdn, mtx);
164}
165
166void
167pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
168 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]) {
169 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
170 &shard->edata_cache.mtx, arena_prof_mutex_extent_avail);
171 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
172 &shard->pac.ecache_dirty.mtx, arena_prof_mutex_extents_dirty);
173 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
174 &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
175 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
176 &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
177 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
178 &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
179 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
180 &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
181
182 if (shard->ever_used_hpa) {
183 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
184 &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
185 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
186 &shard->hpa_shard.grow_mtx,
187 arena_prof_mutex_hpa_shard_grow);
188 sec_mutex_stats_read(tsdn, &shard->hpa_sec,
189 &mutex_prof_data[arena_prof_mutex_hpa_sec]);
190 }
191}
192