1 | #ifndef JEMALLOC_INTERNAL_TCACHE_INLINES_H |
2 | #define JEMALLOC_INTERNAL_TCACHE_INLINES_H |
3 | |
4 | #include "jemalloc/internal/bin.h" |
5 | #include "jemalloc/internal/jemalloc_internal_types.h" |
6 | #include "jemalloc/internal/san.h" |
7 | #include "jemalloc/internal/sc.h" |
8 | #include "jemalloc/internal/sz.h" |
9 | #include "jemalloc/internal/util.h" |
10 | |
11 | static inline bool |
12 | tcache_enabled_get(tsd_t *tsd) { |
13 | return tsd_tcache_enabled_get(tsd); |
14 | } |
15 | |
16 | static inline void |
17 | tcache_enabled_set(tsd_t *tsd, bool enabled) { |
18 | bool was_enabled = tsd_tcache_enabled_get(tsd); |
19 | |
20 | if (!was_enabled && enabled) { |
21 | tsd_tcache_data_init(tsd); |
22 | } else if (was_enabled && !enabled) { |
23 | tcache_cleanup(tsd); |
24 | } |
25 | /* Commit the state last. Above calls check current state. */ |
26 | tsd_tcache_enabled_set(tsd, enabled); |
27 | tsd_slow_update(tsd); |
28 | } |
29 | |
30 | JEMALLOC_ALWAYS_INLINE bool |
31 | tcache_small_bin_disabled(szind_t ind, cache_bin_t *bin) { |
32 | assert(ind < SC_NBINS); |
33 | bool ret = (cache_bin_info_ncached_max(&tcache_bin_info[ind]) == 0); |
34 | if (ret && bin != NULL) { |
35 | /* small size class but cache bin disabled. */ |
36 | assert(ind >= nhbins); |
37 | assert((uintptr_t)(*bin->stack_head) == |
38 | cache_bin_preceding_junk); |
39 | } |
40 | |
41 | return ret; |
42 | } |
43 | |
44 | JEMALLOC_ALWAYS_INLINE void * |
45 | tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, |
46 | size_t size, szind_t binind, bool zero, bool slow_path) { |
47 | void *ret; |
48 | bool tcache_success; |
49 | |
50 | assert(binind < SC_NBINS); |
51 | cache_bin_t *bin = &tcache->bins[binind]; |
52 | ret = cache_bin_alloc(bin, &tcache_success); |
53 | assert(tcache_success == (ret != NULL)); |
54 | if (unlikely(!tcache_success)) { |
55 | bool tcache_hard_success; |
56 | arena = arena_choose(tsd, arena); |
57 | if (unlikely(arena == NULL)) { |
58 | return NULL; |
59 | } |
60 | if (unlikely(tcache_small_bin_disabled(binind, bin))) { |
61 | /* stats and zero are handled directly by the arena. */ |
62 | return arena_malloc_hard(tsd_tsdn(tsd), arena, size, |
63 | binind, zero); |
64 | } |
65 | tcache_bin_flush_stashed(tsd, tcache, bin, binind, |
66 | /* is_small */ true); |
67 | |
68 | ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, |
69 | bin, binind, &tcache_hard_success); |
70 | if (tcache_hard_success == false) { |
71 | return NULL; |
72 | } |
73 | } |
74 | |
75 | assert(ret); |
76 | if (unlikely(zero)) { |
77 | size_t usize = sz_index2size(binind); |
78 | assert(tcache_salloc(tsd_tsdn(tsd), ret) == usize); |
79 | memset(ret, 0, usize); |
80 | } |
81 | if (config_stats) { |
82 | bin->tstats.nrequests++; |
83 | } |
84 | return ret; |
85 | } |
86 | |
87 | JEMALLOC_ALWAYS_INLINE void * |
88 | tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, |
89 | szind_t binind, bool zero, bool slow_path) { |
90 | void *ret; |
91 | bool tcache_success; |
92 | |
93 | assert(binind >= SC_NBINS && binind < nhbins); |
94 | cache_bin_t *bin = &tcache->bins[binind]; |
95 | ret = cache_bin_alloc(bin, &tcache_success); |
96 | assert(tcache_success == (ret != NULL)); |
97 | if (unlikely(!tcache_success)) { |
98 | /* |
99 | * Only allocate one large object at a time, because it's quite |
100 | * expensive to create one and not use it. |
101 | */ |
102 | arena = arena_choose(tsd, arena); |
103 | if (unlikely(arena == NULL)) { |
104 | return NULL; |
105 | } |
106 | tcache_bin_flush_stashed(tsd, tcache, bin, binind, |
107 | /* is_small */ false); |
108 | |
109 | ret = large_malloc(tsd_tsdn(tsd), arena, sz_s2u(size), zero); |
110 | if (ret == NULL) { |
111 | return NULL; |
112 | } |
113 | } else { |
114 | if (unlikely(zero)) { |
115 | size_t usize = sz_index2size(binind); |
116 | assert(usize <= tcache_maxclass); |
117 | memset(ret, 0, usize); |
118 | } |
119 | |
120 | if (config_stats) { |
121 | bin->tstats.nrequests++; |
122 | } |
123 | } |
124 | |
125 | return ret; |
126 | } |
127 | |
128 | JEMALLOC_ALWAYS_INLINE void |
129 | tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, |
130 | bool slow_path) { |
131 | assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SC_SMALL_MAXCLASS); |
132 | |
133 | cache_bin_t *bin = &tcache->bins[binind]; |
134 | /* |
135 | * Not marking the branch unlikely because this is past free_fastpath() |
136 | * (which handles the most common cases), i.e. at this point it's often |
137 | * uncommon cases. |
138 | */ |
139 | if (cache_bin_nonfast_aligned(ptr)) { |
140 | /* Junk unconditionally, even if bin is full. */ |
141 | san_junk_ptr(ptr, sz_index2size(binind)); |
142 | if (cache_bin_stash(bin, ptr)) { |
143 | return; |
144 | } |
145 | assert(cache_bin_full(bin)); |
146 | /* Bin full; fall through into the flush branch. */ |
147 | } |
148 | |
149 | if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { |
150 | if (unlikely(tcache_small_bin_disabled(binind, bin))) { |
151 | arena_dalloc_small(tsd_tsdn(tsd), ptr); |
152 | return; |
153 | } |
154 | cache_bin_sz_t max = cache_bin_info_ncached_max( |
155 | &tcache_bin_info[binind]); |
156 | unsigned remain = max >> opt_lg_tcache_flush_small_div; |
157 | tcache_bin_flush_small(tsd, tcache, bin, binind, remain); |
158 | bool ret = cache_bin_dalloc_easy(bin, ptr); |
159 | assert(ret); |
160 | } |
161 | } |
162 | |
163 | JEMALLOC_ALWAYS_INLINE void |
164 | tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, |
165 | bool slow_path) { |
166 | |
167 | assert(tcache_salloc(tsd_tsdn(tsd), ptr) |
168 | > SC_SMALL_MAXCLASS); |
169 | assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); |
170 | |
171 | cache_bin_t *bin = &tcache->bins[binind]; |
172 | if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { |
173 | unsigned remain = cache_bin_info_ncached_max( |
174 | &tcache_bin_info[binind]) >> opt_lg_tcache_flush_large_div; |
175 | tcache_bin_flush_large(tsd, tcache, bin, binind, remain); |
176 | bool ret = cache_bin_dalloc_easy(bin, ptr); |
177 | assert(ret); |
178 | } |
179 | } |
180 | |
181 | JEMALLOC_ALWAYS_INLINE tcache_t * |
182 | tcaches_get(tsd_t *tsd, unsigned ind) { |
183 | tcaches_t *elm = &tcaches[ind]; |
184 | if (unlikely(elm->tcache == NULL)) { |
185 | malloc_printf("<jemalloc>: invalid tcache id (%u).\n" , ind); |
186 | abort(); |
187 | } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { |
188 | elm->tcache = tcache_create_explicit(tsd); |
189 | } |
190 | return elm->tcache; |
191 | } |
192 | |
193 | #endif /* JEMALLOC_INTERNAL_TCACHE_INLINES_H */ |
194 | |