1 | #ifndef JEMALLOC_INTERNAL_PROF_INLINES_B_H |
2 | #define JEMALLOC_INTERNAL_PROF_INLINES_B_H |
3 | |
4 | #include "jemalloc/internal/safety_check.h" |
5 | #include "jemalloc/internal/sz.h" |
6 | |
7 | JEMALLOC_ALWAYS_INLINE bool |
8 | prof_gdump_get_unlocked(void) { |
9 | /* |
10 | * No locking is used when reading prof_gdump_val in the fast path, so |
11 | * there are no guarantees regarding how long it will take for all |
12 | * threads to notice state changes. |
13 | */ |
14 | return prof_gdump_val; |
15 | } |
16 | |
17 | JEMALLOC_ALWAYS_INLINE prof_tdata_t * |
18 | prof_tdata_get(tsd_t *tsd, bool create) { |
19 | prof_tdata_t *tdata; |
20 | |
21 | cassert(config_prof); |
22 | |
23 | tdata = tsd_prof_tdata_get(tsd); |
24 | if (create) { |
25 | if (unlikely(tdata == NULL)) { |
26 | if (tsd_nominal(tsd)) { |
27 | tdata = prof_tdata_init(tsd); |
28 | tsd_prof_tdata_set(tsd, tdata); |
29 | } |
30 | } else if (unlikely(tdata->expired)) { |
31 | tdata = prof_tdata_reinit(tsd, tdata); |
32 | tsd_prof_tdata_set(tsd, tdata); |
33 | } |
34 | assert(tdata == NULL || tdata->attached); |
35 | } |
36 | |
37 | return tdata; |
38 | } |
39 | |
40 | JEMALLOC_ALWAYS_INLINE prof_tctx_t * |
41 | prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { |
42 | cassert(config_prof); |
43 | assert(ptr != NULL); |
44 | |
45 | return arena_prof_tctx_get(tsdn, ptr, alloc_ctx); |
46 | } |
47 | |
48 | JEMALLOC_ALWAYS_INLINE void |
49 | prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, |
50 | alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { |
51 | cassert(config_prof); |
52 | assert(ptr != NULL); |
53 | |
54 | arena_prof_tctx_set(tsdn, ptr, usize, alloc_ctx, tctx); |
55 | } |
56 | |
57 | JEMALLOC_ALWAYS_INLINE void |
58 | prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { |
59 | cassert(config_prof); |
60 | assert(ptr != NULL); |
61 | |
62 | arena_prof_tctx_reset(tsdn, ptr, tctx); |
63 | } |
64 | |
65 | JEMALLOC_ALWAYS_INLINE nstime_t |
66 | prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { |
67 | cassert(config_prof); |
68 | assert(ptr != NULL); |
69 | |
70 | return arena_prof_alloc_time_get(tsdn, ptr, alloc_ctx); |
71 | } |
72 | |
73 | JEMALLOC_ALWAYS_INLINE void |
74 | prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, |
75 | nstime_t t) { |
76 | cassert(config_prof); |
77 | assert(ptr != NULL); |
78 | |
79 | arena_prof_alloc_time_set(tsdn, ptr, alloc_ctx, t); |
80 | } |
81 | |
82 | JEMALLOC_ALWAYS_INLINE bool |
83 | prof_sample_check(tsd_t *tsd, size_t usize, bool update) { |
84 | ssize_t check = update ? 0 : usize; |
85 | |
86 | int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); |
87 | if (update) { |
88 | bytes_until_sample -= usize; |
89 | if (tsd_nominal(tsd)) { |
90 | tsd_bytes_until_sample_set(tsd, bytes_until_sample); |
91 | } |
92 | } |
93 | if (likely(bytes_until_sample >= check)) { |
94 | return true; |
95 | } |
96 | |
97 | return false; |
98 | } |
99 | |
100 | JEMALLOC_ALWAYS_INLINE bool |
101 | prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, |
102 | prof_tdata_t **tdata_out) { |
103 | prof_tdata_t *tdata; |
104 | |
105 | cassert(config_prof); |
106 | |
107 | /* Fastpath: no need to load tdata */ |
108 | if (likely(prof_sample_check(tsd, usize, update))) { |
109 | return true; |
110 | } |
111 | |
112 | bool booted = tsd_prof_tdata_get(tsd); |
113 | tdata = prof_tdata_get(tsd, true); |
114 | if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) { |
115 | tdata = NULL; |
116 | } |
117 | |
118 | if (tdata_out != NULL) { |
119 | *tdata_out = tdata; |
120 | } |
121 | |
122 | if (unlikely(tdata == NULL)) { |
123 | return true; |
124 | } |
125 | |
126 | /* |
127 | * If this was the first creation of tdata, then |
128 | * prof_tdata_get() reset bytes_until_sample, so decrement and |
129 | * check it again |
130 | */ |
131 | if (!booted && prof_sample_check(tsd, usize, update)) { |
132 | return true; |
133 | } |
134 | |
135 | if (tsd_reentrancy_level_get(tsd) > 0) { |
136 | return true; |
137 | } |
138 | /* Compute new sample threshold. */ |
139 | if (update) { |
140 | prof_sample_threshold_update(tdata); |
141 | } |
142 | return !tdata->active; |
143 | } |
144 | |
145 | JEMALLOC_ALWAYS_INLINE prof_tctx_t * |
146 | prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update) { |
147 | prof_tctx_t *ret; |
148 | prof_tdata_t *tdata; |
149 | prof_bt_t bt; |
150 | |
151 | assert(usize == sz_s2u(usize)); |
152 | |
153 | if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update, |
154 | &tdata))) { |
155 | ret = (prof_tctx_t *)(uintptr_t)1U; |
156 | } else { |
157 | bt_init(&bt, tdata->vec); |
158 | prof_backtrace(&bt); |
159 | ret = prof_lookup(tsd, &bt); |
160 | } |
161 | |
162 | return ret; |
163 | } |
164 | |
165 | JEMALLOC_ALWAYS_INLINE void |
166 | prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx, |
167 | prof_tctx_t *tctx) { |
168 | cassert(config_prof); |
169 | assert(ptr != NULL); |
170 | assert(usize == isalloc(tsdn, ptr)); |
171 | |
172 | if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { |
173 | prof_malloc_sample_object(tsdn, ptr, usize, tctx); |
174 | } else { |
175 | prof_tctx_set(tsdn, ptr, usize, alloc_ctx, |
176 | (prof_tctx_t *)(uintptr_t)1U); |
177 | } |
178 | } |
179 | |
180 | JEMALLOC_ALWAYS_INLINE void |
181 | prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, |
182 | bool prof_active, bool updated, const void *old_ptr, size_t old_usize, |
183 | prof_tctx_t *old_tctx) { |
184 | bool sampled, old_sampled, moved; |
185 | |
186 | cassert(config_prof); |
187 | assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); |
188 | |
189 | if (prof_active && !updated && ptr != NULL) { |
190 | assert(usize == isalloc(tsd_tsdn(tsd), ptr)); |
191 | if (prof_sample_accum_update(tsd, usize, true, NULL)) { |
192 | /* |
193 | * Don't sample. The usize passed to prof_alloc_prep() |
194 | * was larger than what actually got allocated, so a |
195 | * backtrace was captured for this allocation, even |
196 | * though its actual usize was insufficient to cross the |
197 | * sample threshold. |
198 | */ |
199 | prof_alloc_rollback(tsd, tctx, true); |
200 | tctx = (prof_tctx_t *)(uintptr_t)1U; |
201 | } |
202 | } |
203 | |
204 | sampled = ((uintptr_t)tctx > (uintptr_t)1U); |
205 | old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); |
206 | moved = (ptr != old_ptr); |
207 | |
208 | if (unlikely(sampled)) { |
209 | prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx); |
210 | } else if (moved) { |
211 | prof_tctx_set(tsd_tsdn(tsd), ptr, usize, NULL, |
212 | (prof_tctx_t *)(uintptr_t)1U); |
213 | } else if (unlikely(old_sampled)) { |
214 | /* |
215 | * prof_tctx_set() would work for the !moved case as well, but |
216 | * prof_tctx_reset() is slightly cheaper, and the proper thing |
217 | * to do here in the presence of explicit knowledge re: moved |
218 | * state. |
219 | */ |
220 | prof_tctx_reset(tsd_tsdn(tsd), ptr, tctx); |
221 | } else { |
222 | assert((uintptr_t)prof_tctx_get(tsd_tsdn(tsd), ptr, NULL) == |
223 | (uintptr_t)1U); |
224 | } |
225 | |
226 | /* |
227 | * The prof_free_sampled_object() call must come after the |
228 | * prof_malloc_sample_object() call, because tctx and old_tctx may be |
229 | * the same, in which case reversing the call order could cause the tctx |
230 | * to be prematurely destroyed as a side effect of momentarily zeroed |
231 | * counters. |
232 | */ |
233 | if (unlikely(old_sampled)) { |
234 | prof_free_sampled_object(tsd, ptr, old_usize, old_tctx); |
235 | } |
236 | } |
237 | |
238 | JEMALLOC_ALWAYS_INLINE void |
239 | prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { |
240 | prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx); |
241 | |
242 | cassert(config_prof); |
243 | assert(usize == isalloc(tsd_tsdn(tsd), ptr)); |
244 | |
245 | if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { |
246 | prof_free_sampled_object(tsd, ptr, usize, tctx); |
247 | } |
248 | } |
249 | |
250 | #endif /* JEMALLOC_INTERNAL_PROF_INLINES_B_H */ |
251 | |