1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/assert.h" |
5 | #include "jemalloc/internal/emap.h" |
6 | #include "jemalloc/internal/extent_mmap.h" |
7 | #include "jemalloc/internal/mutex.h" |
8 | #include "jemalloc/internal/prof_recent.h" |
9 | #include "jemalloc/internal/util.h" |
10 | |
11 | /******************************************************************************/ |
12 | |
13 | void * |
14 | large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) { |
15 | assert(usize == sz_s2u(usize)); |
16 | |
17 | return large_palloc(tsdn, arena, usize, CACHELINE, zero); |
18 | } |
19 | |
20 | void * |
21 | large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, |
22 | bool zero) { |
23 | size_t ausize; |
24 | edata_t *edata; |
25 | UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); |
26 | |
27 | assert(!tsdn_null(tsdn) || arena != NULL); |
28 | |
29 | ausize = sz_sa2u(usize, alignment); |
30 | if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { |
31 | return NULL; |
32 | } |
33 | |
34 | if (likely(!tsdn_null(tsdn))) { |
35 | arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); |
36 | } |
37 | if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, |
38 | arena, usize, alignment, zero)) == NULL) { |
39 | return NULL; |
40 | } |
41 | |
42 | /* See comments in arena_bin_slabs_full_insert(). */ |
43 | if (!arena_is_auto(arena)) { |
44 | /* Insert edata into large. */ |
45 | malloc_mutex_lock(tsdn, &arena->large_mtx); |
46 | edata_list_active_append(&arena->large, edata); |
47 | malloc_mutex_unlock(tsdn, &arena->large_mtx); |
48 | } |
49 | |
50 | arena_decay_tick(tsdn, arena); |
51 | return edata_addr_get(edata); |
52 | } |
53 | |
54 | static bool |
55 | large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { |
56 | arena_t *arena = arena_get_from_edata(edata); |
57 | ehooks_t *ehooks = arena_get_ehooks(arena); |
58 | size_t old_size = edata_size_get(edata); |
59 | size_t old_usize = edata_usize_get(edata); |
60 | |
61 | assert(old_usize > usize); |
62 | |
63 | if (ehooks_split_will_fail(ehooks)) { |
64 | return true; |
65 | } |
66 | |
67 | bool deferred_work_generated = false; |
68 | bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size, |
69 | usize + sz_large_pad, sz_size2index(usize), |
70 | &deferred_work_generated); |
71 | if (err) { |
72 | return true; |
73 | } |
74 | if (deferred_work_generated) { |
75 | arena_handle_deferred_work(tsdn, arena); |
76 | } |
77 | arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize); |
78 | |
79 | return false; |
80 | } |
81 | |
82 | static bool |
83 | large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, |
84 | bool zero) { |
85 | arena_t *arena = arena_get_from_edata(edata); |
86 | |
87 | size_t old_size = edata_size_get(edata); |
88 | size_t old_usize = edata_usize_get(edata); |
89 | size_t new_size = usize + sz_large_pad; |
90 | |
91 | szind_t szind = sz_size2index(usize); |
92 | |
93 | bool deferred_work_generated = false; |
94 | bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size, |
95 | szind, zero, &deferred_work_generated); |
96 | |
97 | if (deferred_work_generated) { |
98 | arena_handle_deferred_work(tsdn, arena); |
99 | } |
100 | |
101 | if (err) { |
102 | return true; |
103 | } |
104 | |
105 | if (zero) { |
106 | if (opt_cache_oblivious) { |
107 | assert(sz_large_pad == PAGE); |
108 | /* |
109 | * Zero the trailing bytes of the original allocation's |
110 | * last page, since they are in an indeterminate state. |
111 | * There will always be trailing bytes, because ptr's |
112 | * offset from the beginning of the extent is a multiple |
113 | * of CACHELINE in [0 .. PAGE). |
114 | */ |
115 | void *zbase = (void *) |
116 | ((uintptr_t)edata_addr_get(edata) + old_usize); |
117 | void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + |
118 | PAGE)); |
119 | size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; |
120 | assert(nzero > 0); |
121 | memset(zbase, 0, nzero); |
122 | } |
123 | } |
124 | arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize); |
125 | |
126 | return false; |
127 | } |
128 | |
129 | bool |
130 | large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, |
131 | size_t usize_max, bool zero) { |
132 | size_t oldusize = edata_usize_get(edata); |
133 | |
134 | /* The following should have been caught by callers. */ |
135 | assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); |
136 | /* Both allocation sizes must be large to avoid a move. */ |
137 | assert(oldusize >= SC_LARGE_MINCLASS |
138 | && usize_max >= SC_LARGE_MINCLASS); |
139 | |
140 | if (usize_max > oldusize) { |
141 | /* Attempt to expand the allocation in-place. */ |
142 | if (!large_ralloc_no_move_expand(tsdn, edata, usize_max, |
143 | zero)) { |
144 | arena_decay_tick(tsdn, arena_get_from_edata(edata)); |
145 | return false; |
146 | } |
147 | /* Try again, this time with usize_min. */ |
148 | if (usize_min < usize_max && usize_min > oldusize && |
149 | large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) { |
150 | arena_decay_tick(tsdn, arena_get_from_edata(edata)); |
151 | return false; |
152 | } |
153 | } |
154 | |
155 | /* |
156 | * Avoid moving the allocation if the existing extent size accommodates |
157 | * the new size. |
158 | */ |
159 | if (oldusize >= usize_min && oldusize <= usize_max) { |
160 | arena_decay_tick(tsdn, arena_get_from_edata(edata)); |
161 | return false; |
162 | } |
163 | |
164 | /* Attempt to shrink the allocation in-place. */ |
165 | if (oldusize > usize_max) { |
166 | if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) { |
167 | arena_decay_tick(tsdn, arena_get_from_edata(edata)); |
168 | return false; |
169 | } |
170 | } |
171 | return true; |
172 | } |
173 | |
174 | static void * |
175 | large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, |
176 | size_t alignment, bool zero) { |
177 | if (alignment <= CACHELINE) { |
178 | return large_malloc(tsdn, arena, usize, zero); |
179 | } |
180 | return large_palloc(tsdn, arena, usize, alignment, zero); |
181 | } |
182 | |
183 | void * |
184 | large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, |
185 | size_t alignment, bool zero, tcache_t *tcache, |
186 | hook_ralloc_args_t *hook_args) { |
187 | edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); |
188 | |
189 | size_t oldusize = edata_usize_get(edata); |
190 | /* The following should have been caught by callers. */ |
191 | assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); |
192 | /* Both allocation sizes must be large to avoid a move. */ |
193 | assert(oldusize >= SC_LARGE_MINCLASS |
194 | && usize >= SC_LARGE_MINCLASS); |
195 | |
196 | /* Try to avoid moving the allocation. */ |
197 | if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) { |
198 | hook_invoke_expand(hook_args->is_realloc |
199 | ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, |
200 | usize, (uintptr_t)ptr, hook_args->args); |
201 | return edata_addr_get(edata); |
202 | } |
203 | |
204 | /* |
205 | * usize and old size are different enough that we need to use a |
206 | * different size class. In that case, fall back to allocating new |
207 | * space and copying. |
208 | */ |
209 | void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, |
210 | zero); |
211 | if (ret == NULL) { |
212 | return NULL; |
213 | } |
214 | |
215 | hook_invoke_alloc(hook_args->is_realloc |
216 | ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, |
217 | hook_args->args); |
218 | hook_invoke_dalloc(hook_args->is_realloc |
219 | ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); |
220 | |
221 | size_t copysize = (usize < oldusize) ? usize : oldusize; |
222 | memcpy(ret, edata_addr_get(edata), copysize); |
223 | isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true); |
224 | return ret; |
225 | } |
226 | |
227 | /* |
228 | * locked indicates whether the arena's large_mtx is currently held. |
229 | */ |
230 | static void |
231 | large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata, |
232 | bool locked) { |
233 | if (!locked) { |
234 | /* See comments in arena_bin_slabs_full_insert(). */ |
235 | if (!arena_is_auto(arena)) { |
236 | malloc_mutex_lock(tsdn, &arena->large_mtx); |
237 | edata_list_active_remove(&arena->large, edata); |
238 | malloc_mutex_unlock(tsdn, &arena->large_mtx); |
239 | } |
240 | } else { |
241 | /* Only hold the large_mtx if necessary. */ |
242 | if (!arena_is_auto(arena)) { |
243 | malloc_mutex_assert_owner(tsdn, &arena->large_mtx); |
244 | edata_list_active_remove(&arena->large, edata); |
245 | } |
246 | } |
247 | arena_extent_dalloc_large_prep(tsdn, arena, edata); |
248 | } |
249 | |
250 | static void |
251 | large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { |
252 | bool deferred_work_generated = false; |
253 | pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated); |
254 | if (deferred_work_generated) { |
255 | arena_handle_deferred_work(tsdn, arena); |
256 | } |
257 | } |
258 | |
259 | void |
260 | large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) { |
261 | large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true); |
262 | } |
263 | |
264 | void |
265 | large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) { |
266 | large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata); |
267 | } |
268 | |
269 | void |
270 | large_dalloc(tsdn_t *tsdn, edata_t *edata) { |
271 | arena_t *arena = arena_get_from_edata(edata); |
272 | large_dalloc_prep_impl(tsdn, arena, edata, false); |
273 | large_dalloc_finish_impl(tsdn, arena, edata); |
274 | arena_decay_tick(tsdn, arena); |
275 | } |
276 | |
277 | size_t |
278 | large_salloc(tsdn_t *tsdn, const edata_t *edata) { |
279 | return edata_usize_get(edata); |
280 | } |
281 | |
282 | void |
283 | large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, |
284 | bool reset_recent) { |
285 | assert(prof_info != NULL); |
286 | |
287 | prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata); |
288 | prof_info->alloc_tctx = alloc_tctx; |
289 | |
290 | if ((uintptr_t)alloc_tctx > (uintptr_t)1U) { |
291 | nstime_copy(&prof_info->alloc_time, |
292 | edata_prof_alloc_time_get(edata)); |
293 | prof_info->alloc_size = edata_prof_alloc_size_get(edata); |
294 | if (reset_recent) { |
295 | /* |
296 | * Reset the pointer on the recent allocation record, |
297 | * so that this allocation is recorded as released. |
298 | */ |
299 | prof_recent_alloc_reset(tsd, edata); |
300 | } |
301 | } |
302 | } |
303 | |
304 | static void |
305 | large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) { |
306 | edata_prof_tctx_set(edata, tctx); |
307 | } |
308 | |
309 | void |
310 | large_prof_tctx_reset(edata_t *edata) { |
311 | large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U); |
312 | } |
313 | |
314 | void |
315 | large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) { |
316 | nstime_t t; |
317 | nstime_prof_init_update(&t); |
318 | edata_prof_alloc_time_set(edata, &t); |
319 | edata_prof_alloc_size_set(edata, size); |
320 | edata_prof_recent_alloc_init(edata); |
321 | large_prof_tctx_set(edata, tctx); |
322 | } |
323 | |