1 | #ifndef JEMALLOC_INTERNAL_HPA_H |
2 | #define JEMALLOC_INTERNAL_HPA_H |
3 | |
4 | #include "jemalloc/internal/exp_grow.h" |
5 | #include "jemalloc/internal/hpa_hooks.h" |
6 | #include "jemalloc/internal/hpa_opts.h" |
7 | #include "jemalloc/internal/pai.h" |
8 | #include "jemalloc/internal/psset.h" |
9 | |
10 | typedef struct hpa_central_s hpa_central_t; |
11 | struct hpa_central_s { |
12 | /* |
13 | * The mutex guarding most of the operations on the central data |
14 | * structure. |
15 | */ |
16 | malloc_mutex_t mtx; |
17 | /* |
18 | * Guards expansion of eden. We separate this from the regular mutex so |
19 | * that cheaper operations can still continue while we're doing the OS |
20 | * call. |
21 | */ |
22 | malloc_mutex_t grow_mtx; |
23 | /* |
24 | * Either NULL (if empty), or some integer multiple of a |
25 | * hugepage-aligned number of hugepages. We carve them off one at a |
26 | * time to satisfy new pageslab requests. |
27 | * |
28 | * Guarded by grow_mtx. |
29 | */ |
30 | void *eden; |
31 | size_t eden_len; |
32 | /* Source for metadata. */ |
33 | base_t *base; |
34 | /* Number of grow operations done on this hpa_central_t. */ |
35 | uint64_t age_counter; |
36 | |
37 | /* The HPA hooks. */ |
38 | hpa_hooks_t hooks; |
39 | }; |
40 | |
41 | typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t; |
42 | struct hpa_shard_nonderived_stats_s { |
43 | /* |
44 | * The number of times we've purged within a hugepage. |
45 | * |
46 | * Guarded by mtx. |
47 | */ |
48 | uint64_t npurge_passes; |
49 | /* |
50 | * The number of individual purge calls we perform (which should always |
51 | * be bigger than npurge_passes, since each pass purges at least one |
52 | * extent within a hugepage. |
53 | * |
54 | * Guarded by mtx. |
55 | */ |
56 | uint64_t npurges; |
57 | |
58 | /* |
59 | * The number of times we've hugified a pageslab. |
60 | * |
61 | * Guarded by mtx. |
62 | */ |
63 | uint64_t nhugifies; |
64 | /* |
65 | * The number of times we've dehugified a pageslab. |
66 | * |
67 | * Guarded by mtx. |
68 | */ |
69 | uint64_t ndehugifies; |
70 | }; |
71 | |
72 | /* Completely derived; only used by CTL. */ |
73 | typedef struct hpa_shard_stats_s hpa_shard_stats_t; |
74 | struct hpa_shard_stats_s { |
75 | psset_stats_t psset_stats; |
76 | hpa_shard_nonderived_stats_t nonderived_stats; |
77 | }; |
78 | |
79 | typedef struct hpa_shard_s hpa_shard_t; |
80 | struct hpa_shard_s { |
81 | /* |
82 | * pai must be the first member; we cast from a pointer to it to a |
83 | * pointer to the hpa_shard_t. |
84 | */ |
85 | pai_t pai; |
86 | |
87 | /* The central allocator we get our hugepages from. */ |
88 | hpa_central_t *central; |
89 | /* Protects most of this shard's state. */ |
90 | malloc_mutex_t mtx; |
91 | /* |
92 | * Guards the shard's access to the central allocator (preventing |
93 | * multiple threads operating on this shard from accessing the central |
94 | * allocator). |
95 | */ |
96 | malloc_mutex_t grow_mtx; |
97 | /* The base metadata allocator. */ |
98 | base_t *base; |
99 | |
100 | /* |
101 | * This edata cache is the one we use when allocating a small extent |
102 | * from a pageslab. The pageslab itself comes from the centralized |
103 | * allocator, and so will use its edata_cache. |
104 | */ |
105 | edata_cache_fast_t ecf; |
106 | |
107 | psset_t psset; |
108 | |
109 | /* |
110 | * How many grow operations have occurred. |
111 | * |
112 | * Guarded by grow_mtx. |
113 | */ |
114 | uint64_t age_counter; |
115 | |
116 | /* The arena ind we're associated with. */ |
117 | unsigned ind; |
118 | |
119 | /* |
120 | * Our emap. This is just a cache of the emap pointer in the associated |
121 | * hpa_central. |
122 | */ |
123 | emap_t *emap; |
124 | |
125 | /* The configuration choices for this hpa shard. */ |
126 | hpa_shard_opts_t opts; |
127 | |
128 | /* |
129 | * How many pages have we started but not yet finished purging in this |
130 | * hpa shard. |
131 | */ |
132 | size_t npending_purge; |
133 | |
134 | /* |
135 | * Those stats which are copied directly into the CTL-centric hpa shard |
136 | * stats. |
137 | */ |
138 | hpa_shard_nonderived_stats_t stats; |
139 | |
140 | /* |
141 | * Last time we performed purge on this shard. |
142 | */ |
143 | nstime_t last_purge; |
144 | }; |
145 | |
146 | /* |
147 | * Whether or not the HPA can be used given the current configuration. This is |
148 | * is not necessarily a guarantee that it backs its allocations by hugepages, |
149 | * just that it can function properly given the system it's running on. |
150 | */ |
151 | bool hpa_supported(); |
152 | bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks); |
153 | bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, |
154 | base_t *base, edata_cache_t *edata_cache, unsigned ind, |
155 | const hpa_shard_opts_t *opts); |
156 | |
157 | void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); |
158 | void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, |
159 | hpa_shard_stats_t *dst); |
160 | |
161 | /* |
162 | * Notify the shard that we won't use it for allocations much longer. Due to |
163 | * the possibility of races, we don't actually prevent allocations; just flush |
164 | * and disable the embedded edata_cache_small. |
165 | */ |
166 | void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard); |
167 | void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard); |
168 | |
169 | void hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard, |
170 | bool deferral_allowed); |
171 | void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard); |
172 | |
173 | /* |
174 | * We share the fork ordering with the PA and arena prefork handling; that's why |
175 | * these are 3 and 4 rather than 0 and 1. |
176 | */ |
177 | void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard); |
178 | void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard); |
179 | void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard); |
180 | void hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard); |
181 | |
182 | #endif /* JEMALLOC_INTERNAL_HPA_H */ |
183 | |