1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/emap.h"
6#include "jemalloc/internal/extent_mmap.h"
7#include "jemalloc/internal/mutex.h"
8#include "jemalloc/internal/prof_recent.h"
9#include "jemalloc/internal/util.h"
10
11/******************************************************************************/
12
13void *
14large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) {
15 assert(usize == sz_s2u(usize));
16
17 return large_palloc(tsdn, arena, usize, CACHELINE, zero);
18}
19
20void *
21large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
22 bool zero) {
23 size_t ausize;
24 edata_t *edata;
25 UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);
26
27 assert(!tsdn_null(tsdn) || arena != NULL);
28
29 ausize = sz_sa2u(usize, alignment);
30 if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) {
31 return NULL;
32 }
33
34 if (likely(!tsdn_null(tsdn))) {
35 arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
36 }
37 if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn,
38 arena, usize, alignment, zero)) == NULL) {
39 return NULL;
40 }
41
42 /* See comments in arena_bin_slabs_full_insert(). */
43 if (!arena_is_auto(arena)) {
44 /* Insert edata into large. */
45 malloc_mutex_lock(tsdn, &arena->large_mtx);
46 edata_list_active_append(&arena->large, edata);
47 malloc_mutex_unlock(tsdn, &arena->large_mtx);
48 }
49
50 arena_decay_tick(tsdn, arena);
51 return edata_addr_get(edata);
52}
53
54static bool
55large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
56 arena_t *arena = arena_get_from_edata(edata);
57 ehooks_t *ehooks = arena_get_ehooks(arena);
58 size_t old_size = edata_size_get(edata);
59 size_t old_usize = edata_usize_get(edata);
60
61 assert(old_usize > usize);
62
63 if (ehooks_split_will_fail(ehooks)) {
64 return true;
65 }
66
67 bool deferred_work_generated = false;
68 bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size,
69 usize + sz_large_pad, sz_size2index(usize),
70 &deferred_work_generated);
71 if (err) {
72 return true;
73 }
74 if (deferred_work_generated) {
75 arena_handle_deferred_work(tsdn, arena);
76 }
77 arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
78
79 return false;
80}
81
82static bool
83large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
84 bool zero) {
85 arena_t *arena = arena_get_from_edata(edata);
86
87 size_t old_size = edata_size_get(edata);
88 size_t old_usize = edata_usize_get(edata);
89 size_t new_size = usize + sz_large_pad;
90
91 szind_t szind = sz_size2index(usize);
92
93 bool deferred_work_generated = false;
94 bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size,
95 szind, zero, &deferred_work_generated);
96
97 if (deferred_work_generated) {
98 arena_handle_deferred_work(tsdn, arena);
99 }
100
101 if (err) {
102 return true;
103 }
104
105 if (zero) {
106 if (opt_cache_oblivious) {
107 assert(sz_large_pad == PAGE);
108 /*
109 * Zero the trailing bytes of the original allocation's
110 * last page, since they are in an indeterminate state.
111 * There will always be trailing bytes, because ptr's
112 * offset from the beginning of the extent is a multiple
113 * of CACHELINE in [0 .. PAGE).
114 */
115 void *zbase = (void *)
116 ((uintptr_t)edata_addr_get(edata) + old_usize);
117 void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase +
118 PAGE));
119 size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase;
120 assert(nzero > 0);
121 memset(zbase, 0, nzero);
122 }
123 }
124 arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize);
125
126 return false;
127}
128
129bool
130large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
131 size_t usize_max, bool zero) {
132 size_t oldusize = edata_usize_get(edata);
133
134 /* The following should have been caught by callers. */
135 assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
136 /* Both allocation sizes must be large to avoid a move. */
137 assert(oldusize >= SC_LARGE_MINCLASS
138 && usize_max >= SC_LARGE_MINCLASS);
139
140 if (usize_max > oldusize) {
141 /* Attempt to expand the allocation in-place. */
142 if (!large_ralloc_no_move_expand(tsdn, edata, usize_max,
143 zero)) {
144 arena_decay_tick(tsdn, arena_get_from_edata(edata));
145 return false;
146 }
147 /* Try again, this time with usize_min. */
148 if (usize_min < usize_max && usize_min > oldusize &&
149 large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) {
150 arena_decay_tick(tsdn, arena_get_from_edata(edata));
151 return false;
152 }
153 }
154
155 /*
156 * Avoid moving the allocation if the existing extent size accommodates
157 * the new size.
158 */
159 if (oldusize >= usize_min && oldusize <= usize_max) {
160 arena_decay_tick(tsdn, arena_get_from_edata(edata));
161 return false;
162 }
163
164 /* Attempt to shrink the allocation in-place. */
165 if (oldusize > usize_max) {
166 if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) {
167 arena_decay_tick(tsdn, arena_get_from_edata(edata));
168 return false;
169 }
170 }
171 return true;
172}
173
174static void *
175large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
176 size_t alignment, bool zero) {
177 if (alignment <= CACHELINE) {
178 return large_malloc(tsdn, arena, usize, zero);
179 }
180 return large_palloc(tsdn, arena, usize, alignment, zero);
181}
182
183void *
184large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
185 size_t alignment, bool zero, tcache_t *tcache,
186 hook_ralloc_args_t *hook_args) {
187 edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
188
189 size_t oldusize = edata_usize_get(edata);
190 /* The following should have been caught by callers. */
191 assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
192 /* Both allocation sizes must be large to avoid a move. */
193 assert(oldusize >= SC_LARGE_MINCLASS
194 && usize >= SC_LARGE_MINCLASS);
195
196 /* Try to avoid moving the allocation. */
197 if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
198 hook_invoke_expand(hook_args->is_realloc
199 ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
200 usize, (uintptr_t)ptr, hook_args->args);
201 return edata_addr_get(edata);
202 }
203
204 /*
205 * usize and old size are different enough that we need to use a
206 * different size class. In that case, fall back to allocating new
207 * space and copying.
208 */
209 void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment,
210 zero);
211 if (ret == NULL) {
212 return NULL;
213 }
214
215 hook_invoke_alloc(hook_args->is_realloc
216 ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
217 hook_args->args);
218 hook_invoke_dalloc(hook_args->is_realloc
219 ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
220
221 size_t copysize = (usize < oldusize) ? usize : oldusize;
222 memcpy(ret, edata_addr_get(edata), copysize);
223 isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);
224 return ret;
225}
226
227/*
228 * locked indicates whether the arena's large_mtx is currently held.
229 */
230static void
231large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
232 bool locked) {
233 if (!locked) {
234 /* See comments in arena_bin_slabs_full_insert(). */
235 if (!arena_is_auto(arena)) {
236 malloc_mutex_lock(tsdn, &arena->large_mtx);
237 edata_list_active_remove(&arena->large, edata);
238 malloc_mutex_unlock(tsdn, &arena->large_mtx);
239 }
240 } else {
241 /* Only hold the large_mtx if necessary. */
242 if (!arena_is_auto(arena)) {
243 malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
244 edata_list_active_remove(&arena->large, edata);
245 }
246 }
247 arena_extent_dalloc_large_prep(tsdn, arena, edata);
248}
249
250static void
251large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
252 bool deferred_work_generated = false;
253 pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
254 if (deferred_work_generated) {
255 arena_handle_deferred_work(tsdn, arena);
256 }
257}
258
259void
260large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) {
261 large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true);
262}
263
264void
265large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) {
266 large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata);
267}
268
269void
270large_dalloc(tsdn_t *tsdn, edata_t *edata) {
271 arena_t *arena = arena_get_from_edata(edata);
272 large_dalloc_prep_impl(tsdn, arena, edata, false);
273 large_dalloc_finish_impl(tsdn, arena, edata);
274 arena_decay_tick(tsdn, arena);
275}
276
277size_t
278large_salloc(tsdn_t *tsdn, const edata_t *edata) {
279 return edata_usize_get(edata);
280}
281
282void
283large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
284 bool reset_recent) {
285 assert(prof_info != NULL);
286
287 prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata);
288 prof_info->alloc_tctx = alloc_tctx;
289
290 if ((uintptr_t)alloc_tctx > (uintptr_t)1U) {
291 nstime_copy(&prof_info->alloc_time,
292 edata_prof_alloc_time_get(edata));
293 prof_info->alloc_size = edata_prof_alloc_size_get(edata);
294 if (reset_recent) {
295 /*
296 * Reset the pointer on the recent allocation record,
297 * so that this allocation is recorded as released.
298 */
299 prof_recent_alloc_reset(tsd, edata);
300 }
301 }
302}
303
304static void
305large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) {
306 edata_prof_tctx_set(edata, tctx);
307}
308
309void
310large_prof_tctx_reset(edata_t *edata) {
311 large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U);
312}
313
314void
315large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) {
316 nstime_t t;
317 nstime_prof_init_update(&t);
318 edata_prof_alloc_time_set(edata, &t);
319 edata_prof_alloc_size_set(edata, size);
320 edata_prof_recent_alloc_init(edata);
321 large_prof_tctx_set(edata, tctx);
322}
323