1#ifndef JEMALLOC_INTERNAL_INLINES_B_H
2#define JEMALLOC_INTERNAL_INLINES_B_H
3
4#include "jemalloc/internal/extent.h"
5
6static inline void
7percpu_arena_update(tsd_t *tsd, unsigned cpu) {
8 assert(have_percpu_arena);
9 arena_t *oldarena = tsd_arena_get(tsd);
10 assert(oldarena != NULL);
11 unsigned oldind = arena_ind_get(oldarena);
12
13 if (oldind != cpu) {
14 unsigned newind = cpu;
15 arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
16 assert(newarena != NULL);
17
18 /* Set new arena/tcache associations. */
19 arena_migrate(tsd, oldarena, newarena);
20 tcache_t *tcache = tcache_get(tsd);
21 if (tcache != NULL) {
22 tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
23 tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow,
24 tcache, newarena);
25 }
26 }
27}
28
29
30/* Choose an arena based on a per-thread value. */
31static inline arena_t *
32arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
33 arena_t *ret;
34
35 if (arena != NULL) {
36 return arena;
37 }
38
39 /* During reentrancy, arena 0 is the safest bet. */
40 if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
41 return arena_get(tsd_tsdn(tsd), 0, true);
42 }
43
44 ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
45 if (unlikely(ret == NULL)) {
46 ret = arena_choose_hard(tsd, internal);
47 assert(ret);
48 if (tcache_available(tsd)) {
49 tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
50 tcache_t *tcache = tsd_tcachep_get(tsd);
51 if (tcache_slow->arena != NULL) {
52 /* See comments in tsd_tcache_data_init().*/
53 assert(tcache_slow->arena ==
54 arena_get(tsd_tsdn(tsd), 0, false));
55 if (tcache_slow->arena != ret) {
56 tcache_arena_reassociate(tsd_tsdn(tsd),
57 tcache_slow, tcache, ret);
58 }
59 } else {
60 tcache_arena_associate(tsd_tsdn(tsd),
61 tcache_slow, tcache, ret);
62 }
63 }
64 }
65
66 /*
67 * Note that for percpu arena, if the current arena is outside of the
68 * auto percpu arena range, (i.e. thread is assigned to a manually
69 * managed arena), then percpu arena is skipped.
70 */
71 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena) &&
72 !internal && (arena_ind_get(ret) <
73 percpu_arena_ind_limit(opt_percpu_arena)) && (ret->last_thd !=
74 tsd_tsdn(tsd))) {
75 unsigned ind = percpu_arena_choose();
76 if (arena_ind_get(ret) != ind) {
77 percpu_arena_update(tsd, ind);
78 ret = tsd_arena_get(tsd);
79 }
80 ret->last_thd = tsd_tsdn(tsd);
81 }
82
83 return ret;
84}
85
86static inline arena_t *
87arena_choose(tsd_t *tsd, arena_t *arena) {
88 return arena_choose_impl(tsd, arena, false);
89}
90
91static inline arena_t *
92arena_ichoose(tsd_t *tsd, arena_t *arena) {
93 return arena_choose_impl(tsd, arena, true);
94}
95
96static inline bool
97arena_is_auto(arena_t *arena) {
98 assert(narenas_auto > 0);
99
100 return (arena_ind_get(arena) < manual_arena_base);
101}
102
103#endif /* JEMALLOC_INTERNAL_INLINES_B_H */
104