1#ifndef JEMALLOC_INTERNAL_PA_H
2#define JEMALLOC_INTERNAL_PA_H
3
4#include "jemalloc/internal/base.h"
5#include "jemalloc/internal/decay.h"
6#include "jemalloc/internal/ecache.h"
7#include "jemalloc/internal/edata_cache.h"
8#include "jemalloc/internal/emap.h"
9#include "jemalloc/internal/hpa.h"
10#include "jemalloc/internal/lockedint.h"
11#include "jemalloc/internal/pac.h"
12#include "jemalloc/internal/pai.h"
13#include "jemalloc/internal/sec.h"
14
15/*
16 * The page allocator; responsible for acquiring pages of memory for
17 * allocations. It picks the implementation of the page allocator interface
18 * (i.e. a pai_t) to handle a given page-level allocation request. For now, the
19 * only such implementation is the PAC code ("page allocator classic"), but
20 * others will be coming soon.
21 */
22
23typedef struct pa_central_s pa_central_t;
24struct pa_central_s {
25 hpa_central_t hpa;
26};
27
28/*
29 * The stats for a particular pa_shard. Because of the way the ctl module
30 * handles stats epoch data collection (it has its own arena_stats, and merges
31 * the stats from each arena into it), this needs to live in the arena_stats_t;
32 * hence we define it here and let the pa_shard have a pointer (rather than the
33 * more natural approach of just embedding it in the pa_shard itself).
34 *
35 * We follow the arena_stats_t approach of marking the derived fields. These
36 * are the ones that are not maintained on their own; instead, their values are
37 * derived during those stats merges.
38 */
39typedef struct pa_shard_stats_s pa_shard_stats_t;
40struct pa_shard_stats_s {
41 /* Number of edata_t structs allocated by base, but not being used. */
42 size_t edata_avail; /* Derived. */
43 /*
44 * Stats specific to the PAC. For now, these are the only stats that
45 * exist, but there will eventually be other page allocators. Things
46 * like edata_avail make sense in a cross-PA sense, but things like
47 * npurges don't.
48 */
49 pac_stats_t pac_stats;
50};
51
52/*
53 * The local allocator handle. Keeps the state necessary to satisfy page-sized
54 * allocations.
55 *
56 * The contents are mostly internal to the PA module. The key exception is that
57 * arena decay code is allowed to grab pointers to the dirty and muzzy ecaches
58 * decay_ts, for a couple of queries, passing them back to a PA function, or
59 * acquiring decay.mtx and looking at decay.purging. The reasoning is that,
60 * while PA decides what and how to purge, the arena code decides when and where
61 * (e.g. on what thread). It's allowed to use the presence of another purger to
62 * decide.
63 * (The background thread code also touches some other decay internals, but
64 * that's not fundamental; its' just an artifact of a partial refactoring, and
65 * its accesses could be straightforwardly moved inside the decay module).
66 */
67typedef struct pa_shard_s pa_shard_t;
68struct pa_shard_s {
69 /* The central PA this shard is associated with. */
70 pa_central_t *central;
71
72 /*
73 * Number of pages in active extents.
74 *
75 * Synchronization: atomic.
76 */
77 atomic_zu_t nactive;
78
79 /*
80 * Whether or not we should prefer the hugepage allocator. Atomic since
81 * it may be concurrently modified by a thread setting extent hooks.
82 * Note that we still may do HPA operations in this arena; if use_hpa is
83 * changed from true to false, we'll free back to the hugepage allocator
84 * for those allocations.
85 */
86 atomic_b_t use_hpa;
87
88 /*
89 * If we never used the HPA to begin with, it wasn't initialized, and so
90 * we shouldn't try to e.g. acquire its mutexes during fork. This
91 * tracks that knowledge.
92 */
93 bool ever_used_hpa;
94
95 /* Allocates from a PAC. */
96 pac_t pac;
97
98 /*
99 * We place a small extent cache in front of the HPA, since we intend
100 * these configurations to use many fewer arenas, and therefore have a
101 * higher risk of hot locks.
102 */
103 sec_t hpa_sec;
104 hpa_shard_t hpa_shard;
105
106 /* The source of edata_t objects. */
107 edata_cache_t edata_cache;
108
109 unsigned ind;
110
111 malloc_mutex_t *stats_mtx;
112 pa_shard_stats_t *stats;
113
114 /* The emap this shard is tied to. */
115 emap_t *emap;
116
117 /* The base from which we get the ehooks and allocate metadat. */
118 base_t *base;
119};
120
121static inline bool
122pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
123 return ecache_npages_get(&shard->pac.ecache_muzzy) == 0 &&
124 pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
125}
126
127static inline ehooks_t *
128pa_shard_ehooks_get(pa_shard_t *shard) {
129 return base_ehooks_get(shard->base);
130}
131
132/* Returns true on error. */
133bool pa_central_init(pa_central_t *central, base_t *base, bool hpa,
134 hpa_hooks_t *hpa_hooks);
135
136/* Returns true on error. */
137bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
138 emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
139 malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold,
140 ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms);
141
142/*
143 * This isn't exposed to users; we allow late enablement of the HPA shard so
144 * that we can boot without worrying about the HPA, then turn it on in a0.
145 */
146bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
147 const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts);
148
149/*
150 * We stop using the HPA when custom extent hooks are installed, but still
151 * redirect deallocations to it.
152 */
153void pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard);
154
155/*
156 * This does the PA-specific parts of arena reset (i.e. freeing all active
157 * allocations).
158 */
159void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
160
161/*
162 * Destroy all the remaining retained extents. Should only be called after
163 * decaying all active, dirty, and muzzy extents to the retained state, as the
164 * last step in destroying the shard.
165 */
166void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
167
168/* Gets an edata for the given allocation. */
169edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
170 size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
171 bool *deferred_work_generated);
172/* Returns true on error, in which case nothing changed. */
173bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
174 size_t new_size, szind_t szind, bool zero, bool *deferred_work_generated);
175/*
176 * The same. Sets *generated_dirty to true if we produced new dirty pages, and
177 * false otherwise.
178 */
179bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
180 size_t new_size, szind_t szind, bool *deferred_work_generated);
181/*
182 * Frees the given edata back to the pa. Sets *generated_dirty if we produced
183 * new dirty pages (well, we always set it for now; but this need not be the
184 * case).
185 * (We could make generated_dirty the return value of course, but this is more
186 * consistent with the shrink pathway and our error codes here).
187 */
188void pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
189 bool *deferred_work_generated);
190bool pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
191 ssize_t decay_ms, pac_purge_eagerness_t eagerness);
192ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
193
194/*
195 * Do deferred work on this PA shard.
196 *
197 * Morally, this should do both PAC decay and the HPA deferred work. For now,
198 * though, the arena, background thread, and PAC modules are tightly interwoven
199 * in a way that's tricky to extricate, so we only do the HPA-specific parts.
200 */
201void pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard,
202 bool deferral_allowed);
203void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
204void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
205uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
206
207/******************************************************************************/
208/*
209 * Various bits of "boring" functionality that are still part of this module,
210 * but that we relegate to pa_extra.c, to keep the core logic in pa.c as
211 * readable as possible.
212 */
213
214/*
215 * These fork phases are synchronized with the arena fork phase numbering to
216 * make it easy to keep straight. That's why there's no prefork1.
217 */
218void pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard);
219void pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard);
220void pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard);
221void pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard);
222void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
223void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
224void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
225
226void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive,
227 size_t *ndirty, size_t *nmuzzy);
228
229void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
230 pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
231 hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
232 size_t *resident);
233
234/*
235 * Reads the PA-owned mutex stats into the output stats array, at the
236 * appropriate positions. Morally, these stats should really live in
237 * pa_shard_stats_t, but the indices are sort of baked into the various mutex
238 * prof macros. This would be a good thing to do at some point.
239 */
240void pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
241 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]);
242
243#endif /* JEMALLOC_INTERNAL_PA_H */
244