1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/extent_dss.h"
6#include "jemalloc/internal/spin.h"
7
8/******************************************************************************/
9/* Data. */
10
11const char *opt_dss = DSS_DEFAULT;
12
13const char *dss_prec_names[] = {
14 "disabled",
15 "primary",
16 "secondary",
17 "N/A"
18};
19
20/*
21 * Current dss precedence default, used when creating new arenas. NB: This is
22 * stored as unsigned rather than dss_prec_t because in principle there's no
23 * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use
24 * atomic operations to synchronize the setting.
25 */
26static atomic_u_t dss_prec_default = ATOMIC_INIT(
27 (unsigned)DSS_PREC_DEFAULT);
28
29/* Base address of the DSS. */
30static void *dss_base;
31/* Atomic boolean indicating whether a thread is currently extending DSS. */
32static atomic_b_t dss_extending;
33/* Atomic boolean indicating whether the DSS is exhausted. */
34static atomic_b_t dss_exhausted;
35/* Atomic current upper limit on DSS addresses. */
36static atomic_p_t dss_max;
37
38/******************************************************************************/
39
40static void *
41extent_dss_sbrk(intptr_t increment) {
42#ifdef JEMALLOC_DSS
43 return sbrk(increment);
44#else
45 not_implemented();
46 return NULL;
47#endif
48}
49
50dss_prec_t
51extent_dss_prec_get(void) {
52 dss_prec_t ret;
53
54 if (!have_dss) {
55 return dss_prec_disabled;
56 }
57 ret = (dss_prec_t)atomic_load_u(&dss_prec_default, ATOMIC_ACQUIRE);
58 return ret;
59}
60
61bool
62extent_dss_prec_set(dss_prec_t dss_prec) {
63 if (!have_dss) {
64 return (dss_prec != dss_prec_disabled);
65 }
66 atomic_store_u(&dss_prec_default, (unsigned)dss_prec, ATOMIC_RELEASE);
67 return false;
68}
69
70static void
71extent_dss_extending_start(void) {
72 spin_t spinner = SPIN_INITIALIZER;
73 while (true) {
74 bool expected = false;
75 if (atomic_compare_exchange_weak_b(&dss_extending, &expected,
76 true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) {
77 break;
78 }
79 spin_adaptive(&spinner);
80 }
81}
82
83static void
84extent_dss_extending_finish(void) {
85 assert(atomic_load_b(&dss_extending, ATOMIC_RELAXED));
86
87 atomic_store_b(&dss_extending, false, ATOMIC_RELEASE);
88}
89
90static void *
91extent_dss_max_update(void *new_addr) {
92 /*
93 * Get the current end of the DSS as max_cur and assure that dss_max is
94 * up to date.
95 */
96 void *max_cur = extent_dss_sbrk(0);
97 if (max_cur == (void *)-1) {
98 return NULL;
99 }
100 atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASE);
101 /* Fixed new_addr can only be supported if it is at the edge of DSS. */
102 if (new_addr != NULL && max_cur != new_addr) {
103 return NULL;
104 }
105 return max_cur;
106}
107
108void *
109extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
110 size_t alignment, bool *zero, bool *commit) {
111 edata_t *gap;
112
113 cassert(have_dss);
114 assert(size > 0);
115 assert(alignment == ALIGNMENT_CEILING(alignment, PAGE));
116
117 /*
118 * sbrk() uses a signed increment argument, so take care not to
119 * interpret a large allocation request as a negative increment.
120 */
121 if ((intptr_t)size < 0) {
122 return NULL;
123 }
124
125 gap = edata_cache_get(tsdn, &arena->pa_shard.edata_cache);
126 if (gap == NULL) {
127 return NULL;
128 }
129
130 extent_dss_extending_start();
131 if (!atomic_load_b(&dss_exhausted, ATOMIC_ACQUIRE)) {
132 /*
133 * The loop is necessary to recover from races with other
134 * threads that are using the DSS for something other than
135 * malloc.
136 */
137 while (true) {
138 void *max_cur = extent_dss_max_update(new_addr);
139 if (max_cur == NULL) {
140 goto label_oom;
141 }
142
143 bool head_state = opt_retain ? EXTENT_IS_HEAD :
144 EXTENT_NOT_HEAD;
145 /*
146 * Compute how much page-aligned gap space (if any) is
147 * necessary to satisfy alignment. This space can be
148 * recycled for later use.
149 */
150 void *gap_addr_page = (void *)(PAGE_CEILING(
151 (uintptr_t)max_cur));
152 void *ret = (void *)ALIGNMENT_CEILING(
153 (uintptr_t)gap_addr_page, alignment);
154 size_t gap_size_page = (uintptr_t)ret -
155 (uintptr_t)gap_addr_page;
156 if (gap_size_page != 0) {
157 edata_init(gap, arena_ind_get(arena),
158 gap_addr_page, gap_size_page, false,
159 SC_NSIZES, extent_sn_next(
160 &arena->pa_shard.pac),
161 extent_state_active, false, true,
162 EXTENT_PAI_PAC, head_state);
163 }
164 /*
165 * Compute the address just past the end of the desired
166 * allocation space.
167 */
168 void *dss_next = (void *)((uintptr_t)ret + size);
169 if ((uintptr_t)ret < (uintptr_t)max_cur ||
170 (uintptr_t)dss_next < (uintptr_t)max_cur) {
171 goto label_oom; /* Wrap-around. */
172 }
173 /* Compute the increment, including subpage bytes. */
174 void *gap_addr_subpage = max_cur;
175 size_t gap_size_subpage = (uintptr_t)ret -
176 (uintptr_t)gap_addr_subpage;
177 intptr_t incr = gap_size_subpage + size;
178
179 assert((uintptr_t)max_cur + incr == (uintptr_t)ret +
180 size);
181
182 /* Try to allocate. */
183 void *dss_prev = extent_dss_sbrk(incr);
184 if (dss_prev == max_cur) {
185 /* Success. */
186 atomic_store_p(&dss_max, dss_next,
187 ATOMIC_RELEASE);
188 extent_dss_extending_finish();
189
190 if (gap_size_page != 0) {
191 ehooks_t *ehooks = arena_get_ehooks(
192 arena);
193 extent_dalloc_gap(tsdn,
194 &arena->pa_shard.pac, ehooks, gap);
195 } else {
196 edata_cache_put(tsdn,
197 &arena->pa_shard.edata_cache, gap);
198 }
199 if (!*commit) {
200 *commit = pages_decommit(ret, size);
201 }
202 if (*zero && *commit) {
203 edata_t edata = {0};
204 ehooks_t *ehooks = arena_get_ehooks(
205 arena);
206
207 edata_init(&edata,
208 arena_ind_get(arena), ret, size,
209 size, false, SC_NSIZES,
210 extent_state_active, false, true,
211 EXTENT_PAI_PAC, head_state);
212 if (extent_purge_forced_wrapper(tsdn,
213 ehooks, &edata, 0, size)) {
214 memset(ret, 0, size);
215 }
216 }
217 return ret;
218 }
219 /*
220 * Failure, whether due to OOM or a race with a raw
221 * sbrk() call from outside the allocator.
222 */
223 if (dss_prev == (void *)-1) {
224 /* OOM. */
225 atomic_store_b(&dss_exhausted, true,
226 ATOMIC_RELEASE);
227 goto label_oom;
228 }
229 }
230 }
231label_oom:
232 extent_dss_extending_finish();
233 edata_cache_put(tsdn, &arena->pa_shard.edata_cache, gap);
234 return NULL;
235}
236
237static bool
238extent_in_dss_helper(void *addr, void *max) {
239 return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr <
240 (uintptr_t)max);
241}
242
243bool
244extent_in_dss(void *addr) {
245 cassert(have_dss);
246
247 return extent_in_dss_helper(addr, atomic_load_p(&dss_max,
248 ATOMIC_ACQUIRE));
249}
250
251bool
252extent_dss_mergeable(void *addr_a, void *addr_b) {
253 void *max;
254
255 cassert(have_dss);
256
257 if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b <
258 (uintptr_t)dss_base) {
259 return true;
260 }
261
262 max = atomic_load_p(&dss_max, ATOMIC_ACQUIRE);
263 return (extent_in_dss_helper(addr_a, max) ==
264 extent_in_dss_helper(addr_b, max));
265}
266
267void
268extent_dss_boot(void) {
269 cassert(have_dss);
270
271 dss_base = extent_dss_sbrk(0);
272 atomic_store_b(&dss_extending, false, ATOMIC_RELAXED);
273 atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXED);
274 atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXED);
275}
276
277/******************************************************************************/
278