1 | #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H |
2 | #define JEMALLOC_INTERNAL_ARENA_INLINES_B_H |
3 | |
4 | #include "jemalloc/internal/jemalloc_internal_types.h" |
5 | #include "jemalloc/internal/mutex.h" |
6 | #include "jemalloc/internal/rtree.h" |
7 | #include "jemalloc/internal/sc.h" |
8 | #include "jemalloc/internal/sz.h" |
9 | #include "jemalloc/internal/ticker.h" |
10 | |
11 | JEMALLOC_ALWAYS_INLINE bool |
12 | arena_has_default_hooks(arena_t *arena) { |
13 | return (extent_hooks_get(arena) == &extent_hooks_default); |
14 | } |
15 | |
16 | JEMALLOC_ALWAYS_INLINE arena_t * |
17 | arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { |
18 | if (arena != NULL) { |
19 | return arena; |
20 | } |
21 | |
22 | /* |
23 | * For huge allocations, use the dedicated huge arena if both are true: |
24 | * 1) is using auto arena selection (i.e. arena == NULL), and 2) the |
25 | * thread is not assigned to a manual arena. |
26 | */ |
27 | if (unlikely(size >= oversize_threshold)) { |
28 | arena_t *tsd_arena = tsd_arena_get(tsd); |
29 | if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { |
30 | return arena_choose_huge(tsd); |
31 | } |
32 | } |
33 | |
34 | return arena_choose(tsd, NULL); |
35 | } |
36 | |
37 | JEMALLOC_ALWAYS_INLINE prof_tctx_t * |
38 | arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { |
39 | cassert(config_prof); |
40 | assert(ptr != NULL); |
41 | |
42 | /* Static check. */ |
43 | if (alloc_ctx == NULL) { |
44 | const extent_t *extent = iealloc(tsdn, ptr); |
45 | if (unlikely(!extent_slab_get(extent))) { |
46 | return large_prof_tctx_get(tsdn, extent); |
47 | } |
48 | } else { |
49 | if (unlikely(!alloc_ctx->slab)) { |
50 | return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr)); |
51 | } |
52 | } |
53 | return (prof_tctx_t *)(uintptr_t)1U; |
54 | } |
55 | |
56 | JEMALLOC_ALWAYS_INLINE void |
57 | arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, |
58 | alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { |
59 | cassert(config_prof); |
60 | assert(ptr != NULL); |
61 | |
62 | /* Static check. */ |
63 | if (alloc_ctx == NULL) { |
64 | extent_t *extent = iealloc(tsdn, ptr); |
65 | if (unlikely(!extent_slab_get(extent))) { |
66 | large_prof_tctx_set(tsdn, extent, tctx); |
67 | } |
68 | } else { |
69 | if (unlikely(!alloc_ctx->slab)) { |
70 | large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx); |
71 | } |
72 | } |
73 | } |
74 | |
75 | static inline void |
76 | arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { |
77 | cassert(config_prof); |
78 | assert(ptr != NULL); |
79 | |
80 | extent_t *extent = iealloc(tsdn, ptr); |
81 | assert(!extent_slab_get(extent)); |
82 | |
83 | large_prof_tctx_reset(tsdn, extent); |
84 | } |
85 | |
86 | JEMALLOC_ALWAYS_INLINE nstime_t |
87 | arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, |
88 | alloc_ctx_t *alloc_ctx) { |
89 | cassert(config_prof); |
90 | assert(ptr != NULL); |
91 | |
92 | extent_t *extent = iealloc(tsdn, ptr); |
93 | /* |
94 | * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're |
95 | * sure we have a sampled allocation. |
96 | */ |
97 | assert(!extent_slab_get(extent)); |
98 | return large_prof_alloc_time_get(extent); |
99 | } |
100 | |
101 | JEMALLOC_ALWAYS_INLINE void |
102 | arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, |
103 | nstime_t t) { |
104 | cassert(config_prof); |
105 | assert(ptr != NULL); |
106 | |
107 | extent_t *extent = iealloc(tsdn, ptr); |
108 | assert(!extent_slab_get(extent)); |
109 | large_prof_alloc_time_set(extent, t); |
110 | } |
111 | |
112 | JEMALLOC_ALWAYS_INLINE void |
113 | arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { |
114 | tsd_t *tsd; |
115 | ticker_t *decay_ticker; |
116 | |
117 | if (unlikely(tsdn_null(tsdn))) { |
118 | return; |
119 | } |
120 | tsd = tsdn_tsd(tsdn); |
121 | decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena)); |
122 | if (unlikely(decay_ticker == NULL)) { |
123 | return; |
124 | } |
125 | if (unlikely(ticker_ticks(decay_ticker, nticks))) { |
126 | arena_decay(tsdn, arena, false, false); |
127 | } |
128 | } |
129 | |
130 | JEMALLOC_ALWAYS_INLINE void |
131 | arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { |
132 | malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx); |
133 | malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx); |
134 | |
135 | arena_decay_ticks(tsdn, arena, 1); |
136 | } |
137 | |
138 | /* Purge a single extent to retained / unmapped directly. */ |
139 | JEMALLOC_ALWAYS_INLINE void |
140 | arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, |
141 | extent_t *extent) { |
142 | size_t extent_size = extent_size_get(extent); |
143 | extent_dalloc_wrapper(tsdn, arena, |
144 | r_extent_hooks, extent); |
145 | if (config_stats) { |
146 | /* Update stats accordingly. */ |
147 | arena_stats_lock(tsdn, &arena->stats); |
148 | arena_stats_add_u64(tsdn, &arena->stats, |
149 | &arena->decay_dirty.stats->nmadvise, 1); |
150 | arena_stats_add_u64(tsdn, &arena->stats, |
151 | &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); |
152 | arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, |
153 | extent_size); |
154 | arena_stats_unlock(tsdn, &arena->stats); |
155 | } |
156 | } |
157 | |
158 | JEMALLOC_ALWAYS_INLINE void * |
159 | arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, |
160 | tcache_t *tcache, bool slow_path) { |
161 | assert(!tsdn_null(tsdn) || tcache == NULL); |
162 | |
163 | if (likely(tcache != NULL)) { |
164 | if (likely(size <= SC_SMALL_MAXCLASS)) { |
165 | return tcache_alloc_small(tsdn_tsd(tsdn), arena, |
166 | tcache, size, ind, zero, slow_path); |
167 | } |
168 | if (likely(size <= tcache_maxclass)) { |
169 | return tcache_alloc_large(tsdn_tsd(tsdn), arena, |
170 | tcache, size, ind, zero, slow_path); |
171 | } |
172 | /* (size > tcache_maxclass) case falls through. */ |
173 | assert(size > tcache_maxclass); |
174 | } |
175 | |
176 | return arena_malloc_hard(tsdn, arena, size, ind, zero); |
177 | } |
178 | |
179 | JEMALLOC_ALWAYS_INLINE arena_t * |
180 | arena_aalloc(tsdn_t *tsdn, const void *ptr) { |
181 | return extent_arena_get(iealloc(tsdn, ptr)); |
182 | } |
183 | |
184 | JEMALLOC_ALWAYS_INLINE size_t |
185 | arena_salloc(tsdn_t *tsdn, const void *ptr) { |
186 | assert(ptr != NULL); |
187 | |
188 | rtree_ctx_t rtree_ctx_fallback; |
189 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
190 | |
191 | szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, |
192 | (uintptr_t)ptr, true); |
193 | assert(szind != SC_NSIZES); |
194 | |
195 | return sz_index2size(szind); |
196 | } |
197 | |
198 | JEMALLOC_ALWAYS_INLINE size_t |
199 | arena_vsalloc(tsdn_t *tsdn, const void *ptr) { |
200 | /* |
201 | * Return 0 if ptr is not within an extent managed by jemalloc. This |
202 | * function has two extra costs relative to isalloc(): |
203 | * - The rtree calls cannot claim to be dependent lookups, which induces |
204 | * rtree lookup load dependencies. |
205 | * - The lookup may fail, so there is an extra branch to check for |
206 | * failure. |
207 | */ |
208 | |
209 | rtree_ctx_t rtree_ctx_fallback; |
210 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
211 | |
212 | extent_t *extent; |
213 | szind_t szind; |
214 | if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx, |
215 | (uintptr_t)ptr, false, &extent, &szind)) { |
216 | return 0; |
217 | } |
218 | |
219 | if (extent == NULL) { |
220 | return 0; |
221 | } |
222 | assert(extent_state_get(extent) == extent_state_active); |
223 | /* Only slab members should be looked up via interior pointers. */ |
224 | assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); |
225 | |
226 | assert(szind != SC_NSIZES); |
227 | |
228 | return sz_index2size(szind); |
229 | } |
230 | |
231 | static inline void |
232 | arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) { |
233 | if (config_prof && unlikely(szind < SC_NBINS)) { |
234 | arena_dalloc_promoted(tsdn, ptr, NULL, true); |
235 | } else { |
236 | extent_t *extent = iealloc(tsdn, ptr); |
237 | large_dalloc(tsdn, extent); |
238 | } |
239 | } |
240 | |
241 | static inline void |
242 | arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { |
243 | assert(ptr != NULL); |
244 | |
245 | rtree_ctx_t rtree_ctx_fallback; |
246 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); |
247 | |
248 | szind_t szind; |
249 | bool slab; |
250 | rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, |
251 | true, &szind, &slab); |
252 | |
253 | if (config_debug) { |
254 | extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, |
255 | rtree_ctx, (uintptr_t)ptr, true); |
256 | assert(szind == extent_szind_get(extent)); |
257 | assert(szind < SC_NSIZES); |
258 | assert(slab == extent_slab_get(extent)); |
259 | } |
260 | |
261 | if (likely(slab)) { |
262 | /* Small allocation. */ |
263 | arena_dalloc_small(tsdn, ptr); |
264 | } else { |
265 | arena_dalloc_large_no_tcache(tsdn, ptr, szind); |
266 | } |
267 | } |
268 | |
269 | JEMALLOC_ALWAYS_INLINE void |
270 | arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, |
271 | bool slow_path) { |
272 | if (szind < nhbins) { |
273 | if (config_prof && unlikely(szind < SC_NBINS)) { |
274 | arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); |
275 | } else { |
276 | tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, |
277 | slow_path); |
278 | } |
279 | } else { |
280 | extent_t *extent = iealloc(tsdn, ptr); |
281 | large_dalloc(tsdn, extent); |
282 | } |
283 | } |
284 | |
285 | JEMALLOC_ALWAYS_INLINE void |
286 | arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, |
287 | alloc_ctx_t *alloc_ctx, bool slow_path) { |
288 | assert(!tsdn_null(tsdn) || tcache == NULL); |
289 | assert(ptr != NULL); |
290 | |
291 | if (unlikely(tcache == NULL)) { |
292 | arena_dalloc_no_tcache(tsdn, ptr); |
293 | return; |
294 | } |
295 | |
296 | szind_t szind; |
297 | bool slab; |
298 | rtree_ctx_t *rtree_ctx; |
299 | if (alloc_ctx != NULL) { |
300 | szind = alloc_ctx->szind; |
301 | slab = alloc_ctx->slab; |
302 | assert(szind != SC_NSIZES); |
303 | } else { |
304 | rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); |
305 | rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, |
306 | (uintptr_t)ptr, true, &szind, &slab); |
307 | } |
308 | |
309 | if (config_debug) { |
310 | rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); |
311 | extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, |
312 | rtree_ctx, (uintptr_t)ptr, true); |
313 | assert(szind == extent_szind_get(extent)); |
314 | assert(szind < SC_NSIZES); |
315 | assert(slab == extent_slab_get(extent)); |
316 | } |
317 | |
318 | if (likely(slab)) { |
319 | /* Small allocation. */ |
320 | tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, |
321 | slow_path); |
322 | } else { |
323 | arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); |
324 | } |
325 | } |
326 | |
327 | static inline void |
328 | arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { |
329 | assert(ptr != NULL); |
330 | assert(size <= SC_LARGE_MAXCLASS); |
331 | |
332 | szind_t szind; |
333 | bool slab; |
334 | if (!config_prof || !opt_prof) { |
335 | /* |
336 | * There is no risk of being confused by a promoted sampled |
337 | * object, so base szind and slab on the given size. |
338 | */ |
339 | szind = sz_size2index(size); |
340 | slab = (szind < SC_NBINS); |
341 | } |
342 | |
343 | if ((config_prof && opt_prof) || config_debug) { |
344 | rtree_ctx_t rtree_ctx_fallback; |
345 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, |
346 | &rtree_ctx_fallback); |
347 | |
348 | rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, |
349 | (uintptr_t)ptr, true, &szind, &slab); |
350 | |
351 | assert(szind == sz_size2index(size)); |
352 | assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); |
353 | |
354 | if (config_debug) { |
355 | extent_t *extent = rtree_extent_read(tsdn, |
356 | &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); |
357 | assert(szind == extent_szind_get(extent)); |
358 | assert(slab == extent_slab_get(extent)); |
359 | } |
360 | } |
361 | |
362 | if (likely(slab)) { |
363 | /* Small allocation. */ |
364 | arena_dalloc_small(tsdn, ptr); |
365 | } else { |
366 | arena_dalloc_large_no_tcache(tsdn, ptr, szind); |
367 | } |
368 | } |
369 | |
370 | JEMALLOC_ALWAYS_INLINE void |
371 | arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, |
372 | alloc_ctx_t *alloc_ctx, bool slow_path) { |
373 | assert(!tsdn_null(tsdn) || tcache == NULL); |
374 | assert(ptr != NULL); |
375 | assert(size <= SC_LARGE_MAXCLASS); |
376 | |
377 | if (unlikely(tcache == NULL)) { |
378 | arena_sdalloc_no_tcache(tsdn, ptr, size); |
379 | return; |
380 | } |
381 | |
382 | szind_t szind; |
383 | bool slab; |
384 | alloc_ctx_t local_ctx; |
385 | if (config_prof && opt_prof) { |
386 | if (alloc_ctx == NULL) { |
387 | /* Uncommon case and should be a static check. */ |
388 | rtree_ctx_t rtree_ctx_fallback; |
389 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, |
390 | &rtree_ctx_fallback); |
391 | rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, |
392 | (uintptr_t)ptr, true, &local_ctx.szind, |
393 | &local_ctx.slab); |
394 | assert(local_ctx.szind == sz_size2index(size)); |
395 | alloc_ctx = &local_ctx; |
396 | } |
397 | slab = alloc_ctx->slab; |
398 | szind = alloc_ctx->szind; |
399 | } else { |
400 | /* |
401 | * There is no risk of being confused by a promoted sampled |
402 | * object, so base szind and slab on the given size. |
403 | */ |
404 | szind = sz_size2index(size); |
405 | slab = (szind < SC_NBINS); |
406 | } |
407 | |
408 | if (config_debug) { |
409 | rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); |
410 | rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, |
411 | (uintptr_t)ptr, true, &szind, &slab); |
412 | extent_t *extent = rtree_extent_read(tsdn, |
413 | &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); |
414 | assert(szind == extent_szind_get(extent)); |
415 | assert(slab == extent_slab_get(extent)); |
416 | } |
417 | |
418 | if (likely(slab)) { |
419 | /* Small allocation. */ |
420 | tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, |
421 | slow_path); |
422 | } else { |
423 | arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); |
424 | } |
425 | } |
426 | |
427 | #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */ |
428 | |