1#ifndef JEMALLOC_INTERNAL_PROF_INLINES_H
2#define JEMALLOC_INTERNAL_PROF_INLINES_H
3
4#include "jemalloc/internal/safety_check.h"
5#include "jemalloc/internal/sz.h"
6#include "jemalloc/internal/thread_event.h"
7
8JEMALLOC_ALWAYS_INLINE void
9prof_active_assert() {
10 cassert(config_prof);
11 /*
12 * If opt_prof is off, then prof_active must always be off, regardless
13 * of whether prof_active_mtx is in effect or not.
14 */
15 assert(opt_prof || !prof_active_state);
16}
17
18JEMALLOC_ALWAYS_INLINE bool
19prof_active_get_unlocked(void) {
20 prof_active_assert();
21 /*
22 * Even if opt_prof is true, sampling can be temporarily disabled by
23 * setting prof_active to false. No locking is used when reading
24 * prof_active in the fast path, so there are no guarantees regarding
25 * how long it will take for all threads to notice state changes.
26 */
27 return prof_active_state;
28}
29
30JEMALLOC_ALWAYS_INLINE bool
31prof_gdump_get_unlocked(void) {
32 /*
33 * No locking is used when reading prof_gdump_val in the fast path, so
34 * there are no guarantees regarding how long it will take for all
35 * threads to notice state changes.
36 */
37 return prof_gdump_val;
38}
39
40JEMALLOC_ALWAYS_INLINE prof_tdata_t *
41prof_tdata_get(tsd_t *tsd, bool create) {
42 prof_tdata_t *tdata;
43
44 cassert(config_prof);
45
46 tdata = tsd_prof_tdata_get(tsd);
47 if (create) {
48 assert(tsd_reentrancy_level_get(tsd) == 0);
49 if (unlikely(tdata == NULL)) {
50 if (tsd_nominal(tsd)) {
51 tdata = prof_tdata_init(tsd);
52 tsd_prof_tdata_set(tsd, tdata);
53 }
54 } else if (unlikely(tdata->expired)) {
55 tdata = prof_tdata_reinit(tsd, tdata);
56 tsd_prof_tdata_set(tsd, tdata);
57 }
58 assert(tdata == NULL || tdata->attached);
59 }
60
61 return tdata;
62}
63
64JEMALLOC_ALWAYS_INLINE void
65prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
66 prof_info_t *prof_info) {
67 cassert(config_prof);
68 assert(ptr != NULL);
69 assert(prof_info != NULL);
70
71 arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, false);
72}
73
74JEMALLOC_ALWAYS_INLINE void
75prof_info_get_and_reset_recent(tsd_t *tsd, const void *ptr,
76 emap_alloc_ctx_t *alloc_ctx, prof_info_t *prof_info) {
77 cassert(config_prof);
78 assert(ptr != NULL);
79 assert(prof_info != NULL);
80
81 arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, true);
82}
83
84JEMALLOC_ALWAYS_INLINE void
85prof_tctx_reset(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
86 cassert(config_prof);
87 assert(ptr != NULL);
88
89 arena_prof_tctx_reset(tsd, ptr, alloc_ctx);
90}
91
92JEMALLOC_ALWAYS_INLINE void
93prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
94 cassert(config_prof);
95 assert(ptr != NULL);
96
97 arena_prof_tctx_reset_sampled(tsd, ptr);
98}
99
100JEMALLOC_ALWAYS_INLINE void
101prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
102 cassert(config_prof);
103 assert(edata != NULL);
104 assert((uintptr_t)tctx > (uintptr_t)1U);
105
106 arena_prof_info_set(tsd, edata, tctx, size);
107}
108
109JEMALLOC_ALWAYS_INLINE bool
110prof_sample_should_skip(tsd_t *tsd, bool sample_event) {
111 cassert(config_prof);
112
113 /* Fastpath: no need to load tdata */
114 if (likely(!sample_event)) {
115 return true;
116 }
117
118 /*
119 * sample_event is always obtained from the thread event module, and
120 * whenever it's true, it means that the thread event module has
121 * already checked the reentrancy level.
122 */
123 assert(tsd_reentrancy_level_get(tsd) == 0);
124
125 prof_tdata_t *tdata = prof_tdata_get(tsd, true);
126 if (unlikely(tdata == NULL)) {
127 return true;
128 }
129
130 return !tdata->active;
131}
132
133JEMALLOC_ALWAYS_INLINE prof_tctx_t *
134prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) {
135 prof_tctx_t *ret;
136
137 if (!prof_active ||
138 likely(prof_sample_should_skip(tsd, sample_event))) {
139 ret = (prof_tctx_t *)(uintptr_t)1U;
140 } else {
141 ret = prof_tctx_create(tsd);
142 }
143
144 return ret;
145}
146
147JEMALLOC_ALWAYS_INLINE void
148prof_malloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
149 emap_alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
150 cassert(config_prof);
151 assert(ptr != NULL);
152 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
153
154 if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) {
155 prof_malloc_sample_object(tsd, ptr, size, usize, tctx);
156 } else {
157 prof_tctx_reset(tsd, ptr, alloc_ctx);
158 }
159}
160
161JEMALLOC_ALWAYS_INLINE void
162prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
163 prof_tctx_t *tctx, bool prof_active, const void *old_ptr, size_t old_usize,
164 prof_info_t *old_prof_info, bool sample_event) {
165 bool sampled, old_sampled, moved;
166
167 cassert(config_prof);
168 assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
169
170 if (prof_active && ptr != NULL) {
171 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
172 if (prof_sample_should_skip(tsd, sample_event)) {
173 /*
174 * Don't sample. The usize passed to prof_alloc_prep()
175 * was larger than what actually got allocated, so a
176 * backtrace was captured for this allocation, even
177 * though its actual usize was insufficient to cross the
178 * sample threshold.
179 */
180 prof_alloc_rollback(tsd, tctx);
181 tctx = (prof_tctx_t *)(uintptr_t)1U;
182 }
183 }
184
185 sampled = ((uintptr_t)tctx > (uintptr_t)1U);
186 old_sampled = ((uintptr_t)old_prof_info->alloc_tctx > (uintptr_t)1U);
187 moved = (ptr != old_ptr);
188
189 if (unlikely(sampled)) {
190 prof_malloc_sample_object(tsd, ptr, size, usize, tctx);
191 } else if (moved) {
192 prof_tctx_reset(tsd, ptr, NULL);
193 } else if (unlikely(old_sampled)) {
194 /*
195 * prof_tctx_reset() would work for the !moved case as well,
196 * but prof_tctx_reset_sampled() is slightly cheaper, and the
197 * proper thing to do here in the presence of explicit
198 * knowledge re: moved state.
199 */
200 prof_tctx_reset_sampled(tsd, ptr);
201 } else {
202 prof_info_t prof_info;
203 prof_info_get(tsd, ptr, NULL, &prof_info);
204 assert((uintptr_t)prof_info.alloc_tctx == (uintptr_t)1U);
205 }
206
207 /*
208 * The prof_free_sampled_object() call must come after the
209 * prof_malloc_sample_object() call, because tctx and old_tctx may be
210 * the same, in which case reversing the call order could cause the tctx
211 * to be prematurely destroyed as a side effect of momentarily zeroed
212 * counters.
213 */
214 if (unlikely(old_sampled)) {
215 prof_free_sampled_object(tsd, old_usize, old_prof_info);
216 }
217}
218
219JEMALLOC_ALWAYS_INLINE size_t
220prof_sample_align(size_t orig_align) {
221 /*
222 * Enforce page alignment, so that sampled allocations can be identified
223 * w/o metadata lookup.
224 */
225 assert(opt_prof);
226 return (opt_cache_oblivious && orig_align < PAGE) ? PAGE :
227 orig_align;
228}
229
230JEMALLOC_ALWAYS_INLINE bool
231prof_sample_aligned(const void *ptr) {
232 return ((uintptr_t)ptr & PAGE_MASK) == 0;
233}
234
235JEMALLOC_ALWAYS_INLINE bool
236prof_sampled(tsd_t *tsd, const void *ptr) {
237 prof_info_t prof_info;
238 prof_info_get(tsd, ptr, NULL, &prof_info);
239 bool sampled = (uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U;
240 if (sampled) {
241 assert(prof_sample_aligned(ptr));
242 }
243 return sampled;
244}
245
246JEMALLOC_ALWAYS_INLINE void
247prof_free(tsd_t *tsd, const void *ptr, size_t usize,
248 emap_alloc_ctx_t *alloc_ctx) {
249 prof_info_t prof_info;
250 prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
251
252 cassert(config_prof);
253 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
254
255 if (unlikely((uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U)) {
256 assert(prof_sample_aligned(ptr));
257 prof_free_sampled_object(tsd, usize, &prof_info);
258 }
259}
260
261#endif /* JEMALLOC_INTERNAL_PROF_INLINES_H */
262