1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/ctl.h" |
5 | #include "jemalloc/internal/assert.h" |
6 | #include "jemalloc/internal/mutex.h" |
7 | #include "jemalloc/internal/counter.h" |
8 | #include "jemalloc/internal/prof_data.h" |
9 | #include "jemalloc/internal/prof_log.h" |
10 | #include "jemalloc/internal/prof_recent.h" |
11 | #include "jemalloc/internal/prof_stats.h" |
12 | #include "jemalloc/internal/prof_sys.h" |
13 | #include "jemalloc/internal/prof_hook.h" |
14 | #include "jemalloc/internal/thread_event.h" |
15 | |
16 | /* |
17 | * This file implements the profiling "APIs" needed by other parts of jemalloc, |
18 | * and also manages the relevant "operational" data, mainly options and mutexes; |
19 | * the core profiling data structures are encapsulated in prof_data.c. |
20 | */ |
21 | |
22 | /******************************************************************************/ |
23 | |
24 | /* Data. */ |
25 | |
26 | bool opt_prof = false; |
27 | bool opt_prof_active = true; |
28 | bool opt_prof_thread_active_init = true; |
29 | size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; |
30 | ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; |
31 | bool opt_prof_gdump = false; |
32 | bool opt_prof_final = false; |
33 | bool opt_prof_leak = false; |
34 | bool opt_prof_leak_error = false; |
35 | bool opt_prof_accum = false; |
36 | char opt_prof_prefix[PROF_DUMP_FILENAME_LEN]; |
37 | bool opt_prof_sys_thread_name = false; |
38 | bool opt_prof_unbias = true; |
39 | |
40 | /* Accessed via prof_sample_event_handler(). */ |
41 | static counter_accum_t prof_idump_accumulated; |
42 | |
43 | /* |
44 | * Initialized as opt_prof_active, and accessed via |
45 | * prof_active_[gs]et{_unlocked,}(). |
46 | */ |
47 | bool prof_active_state; |
48 | static malloc_mutex_t prof_active_mtx; |
49 | |
50 | /* |
51 | * Initialized as opt_prof_thread_active_init, and accessed via |
52 | * prof_thread_active_init_[gs]et(). |
53 | */ |
54 | static bool prof_thread_active_init; |
55 | static malloc_mutex_t prof_thread_active_init_mtx; |
56 | |
57 | /* |
58 | * Initialized as opt_prof_gdump, and accessed via |
59 | * prof_gdump_[gs]et{_unlocked,}(). |
60 | */ |
61 | bool prof_gdump_val; |
62 | static malloc_mutex_t prof_gdump_mtx; |
63 | |
64 | uint64_t prof_interval = 0; |
65 | |
66 | size_t lg_prof_sample; |
67 | |
68 | static uint64_t next_thr_uid; |
69 | static malloc_mutex_t next_thr_uid_mtx; |
70 | |
71 | /* Do not dump any profiles until bootstrapping is complete. */ |
72 | bool prof_booted = false; |
73 | |
74 | /* Logically a prof_backtrace_hook_t. */ |
75 | atomic_p_t prof_backtrace_hook; |
76 | |
77 | /* Logically a prof_dump_hook_t. */ |
78 | atomic_p_t prof_dump_hook; |
79 | |
80 | /******************************************************************************/ |
81 | |
82 | void |
83 | prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) { |
84 | cassert(config_prof); |
85 | |
86 | if (tsd_reentrancy_level_get(tsd) > 0) { |
87 | assert((uintptr_t)tctx == (uintptr_t)1U); |
88 | return; |
89 | } |
90 | |
91 | if ((uintptr_t)tctx > (uintptr_t)1U) { |
92 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
93 | tctx->prepared = false; |
94 | prof_tctx_try_destroy(tsd, tctx); |
95 | } |
96 | } |
97 | |
98 | void |
99 | prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, |
100 | size_t usize, prof_tctx_t *tctx) { |
101 | cassert(config_prof); |
102 | |
103 | if (opt_prof_sys_thread_name) { |
104 | prof_sys_thread_name_fetch(tsd); |
105 | } |
106 | |
107 | edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, |
108 | ptr); |
109 | prof_info_set(tsd, edata, tctx, size); |
110 | |
111 | szind_t szind = sz_size2index(usize); |
112 | |
113 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
114 | /* |
115 | * We need to do these map lookups while holding the lock, to avoid the |
116 | * possibility of races with prof_reset calls, which update the map and |
117 | * then acquire the lock. This actually still leaves a data race on the |
118 | * contents of the unbias map, but we have not yet gone through and |
119 | * atomic-ified the prof module, and compilers are not yet causing us |
120 | * issues. The key thing is to make sure that, if we read garbage data, |
121 | * the prof_reset call is about to mark our tctx as expired before any |
122 | * dumping of our corrupted output is attempted. |
123 | */ |
124 | size_t shifted_unbiased_cnt = prof_shifted_unbiased_cnt[szind]; |
125 | size_t unbiased_bytes = prof_unbiased_sz[szind]; |
126 | tctx->cnts.curobjs++; |
127 | tctx->cnts.curobjs_shifted_unbiased += shifted_unbiased_cnt; |
128 | tctx->cnts.curbytes += usize; |
129 | tctx->cnts.curbytes_unbiased += unbiased_bytes; |
130 | if (opt_prof_accum) { |
131 | tctx->cnts.accumobjs++; |
132 | tctx->cnts.accumobjs_shifted_unbiased += shifted_unbiased_cnt; |
133 | tctx->cnts.accumbytes += usize; |
134 | tctx->cnts.accumbytes_unbiased += unbiased_bytes; |
135 | } |
136 | bool record_recent = prof_recent_alloc_prepare(tsd, tctx); |
137 | tctx->prepared = false; |
138 | malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); |
139 | if (record_recent) { |
140 | assert(tctx == edata_prof_tctx_get(edata)); |
141 | prof_recent_alloc(tsd, edata, size, usize); |
142 | } |
143 | |
144 | if (opt_prof_stats) { |
145 | prof_stats_inc(tsd, szind, size); |
146 | } |
147 | } |
148 | |
149 | void |
150 | prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { |
151 | cassert(config_prof); |
152 | |
153 | assert(prof_info != NULL); |
154 | prof_tctx_t *tctx = prof_info->alloc_tctx; |
155 | assert((uintptr_t)tctx > (uintptr_t)1U); |
156 | |
157 | szind_t szind = sz_size2index(usize); |
158 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
159 | |
160 | assert(tctx->cnts.curobjs > 0); |
161 | assert(tctx->cnts.curbytes >= usize); |
162 | /* |
163 | * It's not correct to do equivalent asserts for unbiased bytes, because |
164 | * of the potential for races with prof.reset calls. The map contents |
165 | * should really be atomic, but we have not atomic-ified the prof module |
166 | * yet. |
167 | */ |
168 | tctx->cnts.curobjs--; |
169 | tctx->cnts.curobjs_shifted_unbiased -= prof_shifted_unbiased_cnt[szind]; |
170 | tctx->cnts.curbytes -= usize; |
171 | tctx->cnts.curbytes_unbiased -= prof_unbiased_sz[szind]; |
172 | |
173 | prof_try_log(tsd, usize, prof_info); |
174 | |
175 | prof_tctx_try_destroy(tsd, tctx); |
176 | |
177 | if (opt_prof_stats) { |
178 | prof_stats_dec(tsd, szind, prof_info->alloc_size); |
179 | } |
180 | } |
181 | |
182 | prof_tctx_t * |
183 | prof_tctx_create(tsd_t *tsd) { |
184 | if (!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0) { |
185 | return NULL; |
186 | } |
187 | |
188 | prof_tdata_t *tdata = prof_tdata_get(tsd, true); |
189 | if (tdata == NULL) { |
190 | return NULL; |
191 | } |
192 | |
193 | prof_bt_t bt; |
194 | bt_init(&bt, tdata->vec); |
195 | prof_backtrace(tsd, &bt); |
196 | return prof_lookup(tsd, &bt); |
197 | } |
198 | |
199 | /* |
200 | * The bodies of this function and prof_leakcheck() are compiled out unless heap |
201 | * profiling is enabled, so that it is possible to compile jemalloc with |
202 | * floating point support completely disabled. Avoiding floating point code is |
203 | * important on memory-constrained systems, but it also enables a workaround for |
204 | * versions of glibc that don't properly save/restore floating point registers |
205 | * during dynamic lazy symbol loading (which internally calls into whatever |
206 | * malloc implementation happens to be integrated into the application). Note |
207 | * that some compilers (e.g. gcc 4.8) may use floating point registers for fast |
208 | * memory moves, so jemalloc must be compiled with such optimizations disabled |
209 | * (e.g. |
210 | * -mno-sse) in order for the workaround to be complete. |
211 | */ |
212 | uint64_t |
213 | prof_sample_new_event_wait(tsd_t *tsd) { |
214 | #ifdef JEMALLOC_PROF |
215 | if (lg_prof_sample == 0) { |
216 | return TE_MIN_START_WAIT; |
217 | } |
218 | |
219 | /* |
220 | * Compute sample interval as a geometrically distributed random |
221 | * variable with mean (2^lg_prof_sample). |
222 | * |
223 | * __ __ |
224 | * | log(u) | 1 |
225 | * bytes_until_sample = | -------- |, where p = --------------- |
226 | * | log(1-p) | lg_prof_sample |
227 | * 2 |
228 | * |
229 | * For more information on the math, see: |
230 | * |
231 | * Non-Uniform Random Variate Generation |
232 | * Luc Devroye |
233 | * Springer-Verlag, New York, 1986 |
234 | * pp 500 |
235 | * (http://luc.devroye.org/rnbookindex.html) |
236 | * |
237 | * In the actual computation, there's a non-zero probability that our |
238 | * pseudo random number generator generates an exact 0, and to avoid |
239 | * log(0), we set u to 1.0 in case r is 0. Therefore u effectively is |
240 | * uniformly distributed in (0, 1] instead of [0, 1). Further, rather |
241 | * than taking the ceiling, we take the floor and then add 1, since |
242 | * otherwise bytes_until_sample would be 0 if u is exactly 1.0. |
243 | */ |
244 | uint64_t r = prng_lg_range_u64(tsd_prng_statep_get(tsd), 53); |
245 | double u = (r == 0U) ? 1.0 : (double)r * (1.0/9007199254740992.0L); |
246 | return (uint64_t)(log(u) / |
247 | log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) |
248 | + (uint64_t)1U; |
249 | #else |
250 | not_reached(); |
251 | return TE_MAX_START_WAIT; |
252 | #endif |
253 | } |
254 | |
255 | uint64_t |
256 | prof_sample_postponed_event_wait(tsd_t *tsd) { |
257 | /* |
258 | * The postponed wait time for prof sample event is computed as if we |
259 | * want a new wait time (i.e. as if the event were triggered). If we |
260 | * instead postpone to the immediate next allocation, like how we're |
261 | * handling the other events, then we can have sampling bias, if e.g. |
262 | * the allocation immediately following a reentrancy always comes from |
263 | * the same stack trace. |
264 | */ |
265 | return prof_sample_new_event_wait(tsd); |
266 | } |
267 | |
268 | void |
269 | prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed) { |
270 | cassert(config_prof); |
271 | assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); |
272 | if (prof_interval == 0 || !prof_active_get_unlocked()) { |
273 | return; |
274 | } |
275 | if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) { |
276 | prof_idump(tsd_tsdn(tsd)); |
277 | } |
278 | } |
279 | |
280 | static void |
281 | prof_fdump(void) { |
282 | tsd_t *tsd; |
283 | |
284 | cassert(config_prof); |
285 | assert(opt_prof_final); |
286 | |
287 | if (!prof_booted) { |
288 | return; |
289 | } |
290 | tsd = tsd_fetch(); |
291 | assert(tsd_reentrancy_level_get(tsd) == 0); |
292 | |
293 | prof_fdump_impl(tsd); |
294 | } |
295 | |
296 | static bool |
297 | prof_idump_accum_init(void) { |
298 | cassert(config_prof); |
299 | |
300 | return counter_accum_init(&prof_idump_accumulated, prof_interval); |
301 | } |
302 | |
303 | void |
304 | prof_idump(tsdn_t *tsdn) { |
305 | tsd_t *tsd; |
306 | prof_tdata_t *tdata; |
307 | |
308 | cassert(config_prof); |
309 | |
310 | if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) { |
311 | return; |
312 | } |
313 | tsd = tsdn_tsd(tsdn); |
314 | if (tsd_reentrancy_level_get(tsd) > 0) { |
315 | return; |
316 | } |
317 | |
318 | tdata = prof_tdata_get(tsd, true); |
319 | if (tdata == NULL) { |
320 | return; |
321 | } |
322 | if (tdata->enq) { |
323 | tdata->enq_idump = true; |
324 | return; |
325 | } |
326 | |
327 | prof_idump_impl(tsd); |
328 | } |
329 | |
330 | bool |
331 | prof_mdump(tsd_t *tsd, const char *filename) { |
332 | cassert(config_prof); |
333 | assert(tsd_reentrancy_level_get(tsd) == 0); |
334 | |
335 | if (!opt_prof || !prof_booted) { |
336 | return true; |
337 | } |
338 | |
339 | return prof_mdump_impl(tsd, filename); |
340 | } |
341 | |
342 | void |
343 | prof_gdump(tsdn_t *tsdn) { |
344 | tsd_t *tsd; |
345 | prof_tdata_t *tdata; |
346 | |
347 | cassert(config_prof); |
348 | |
349 | if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) { |
350 | return; |
351 | } |
352 | tsd = tsdn_tsd(tsdn); |
353 | if (tsd_reentrancy_level_get(tsd) > 0) { |
354 | return; |
355 | } |
356 | |
357 | tdata = prof_tdata_get(tsd, false); |
358 | if (tdata == NULL) { |
359 | return; |
360 | } |
361 | if (tdata->enq) { |
362 | tdata->enq_gdump = true; |
363 | return; |
364 | } |
365 | |
366 | prof_gdump_impl(tsd); |
367 | } |
368 | |
369 | static uint64_t |
370 | prof_thr_uid_alloc(tsdn_t *tsdn) { |
371 | uint64_t thr_uid; |
372 | |
373 | malloc_mutex_lock(tsdn, &next_thr_uid_mtx); |
374 | thr_uid = next_thr_uid; |
375 | next_thr_uid++; |
376 | malloc_mutex_unlock(tsdn, &next_thr_uid_mtx); |
377 | |
378 | return thr_uid; |
379 | } |
380 | |
381 | prof_tdata_t * |
382 | prof_tdata_init(tsd_t *tsd) { |
383 | return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0, |
384 | NULL, prof_thread_active_init_get(tsd_tsdn(tsd))); |
385 | } |
386 | |
387 | prof_tdata_t * |
388 | prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) { |
389 | uint64_t thr_uid = tdata->thr_uid; |
390 | uint64_t thr_discrim = tdata->thr_discrim + 1; |
391 | char *thread_name = (tdata->thread_name != NULL) ? |
392 | prof_thread_name_alloc(tsd, tdata->thread_name) : NULL; |
393 | bool active = tdata->active; |
394 | |
395 | prof_tdata_detach(tsd, tdata); |
396 | return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, |
397 | active); |
398 | } |
399 | |
400 | void |
401 | prof_tdata_cleanup(tsd_t *tsd) { |
402 | prof_tdata_t *tdata; |
403 | |
404 | if (!config_prof) { |
405 | return; |
406 | } |
407 | |
408 | tdata = tsd_prof_tdata_get(tsd); |
409 | if (tdata != NULL) { |
410 | prof_tdata_detach(tsd, tdata); |
411 | } |
412 | } |
413 | |
414 | bool |
415 | prof_active_get(tsdn_t *tsdn) { |
416 | bool prof_active_current; |
417 | |
418 | prof_active_assert(); |
419 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
420 | prof_active_current = prof_active_state; |
421 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
422 | return prof_active_current; |
423 | } |
424 | |
425 | bool |
426 | prof_active_set(tsdn_t *tsdn, bool active) { |
427 | bool prof_active_old; |
428 | |
429 | prof_active_assert(); |
430 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
431 | prof_active_old = prof_active_state; |
432 | prof_active_state = active; |
433 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
434 | prof_active_assert(); |
435 | return prof_active_old; |
436 | } |
437 | |
438 | const char * |
439 | prof_thread_name_get(tsd_t *tsd) { |
440 | assert(tsd_reentrancy_level_get(tsd) == 0); |
441 | |
442 | prof_tdata_t *tdata; |
443 | |
444 | tdata = prof_tdata_get(tsd, true); |
445 | if (tdata == NULL) { |
446 | return "" ; |
447 | } |
448 | return (tdata->thread_name != NULL ? tdata->thread_name : "" ); |
449 | } |
450 | |
451 | int |
452 | prof_thread_name_set(tsd_t *tsd, const char *thread_name) { |
453 | if (opt_prof_sys_thread_name) { |
454 | return ENOENT; |
455 | } else { |
456 | return prof_thread_name_set_impl(tsd, thread_name); |
457 | } |
458 | } |
459 | |
460 | bool |
461 | prof_thread_active_get(tsd_t *tsd) { |
462 | assert(tsd_reentrancy_level_get(tsd) == 0); |
463 | |
464 | prof_tdata_t *tdata; |
465 | |
466 | tdata = prof_tdata_get(tsd, true); |
467 | if (tdata == NULL) { |
468 | return false; |
469 | } |
470 | return tdata->active; |
471 | } |
472 | |
473 | bool |
474 | prof_thread_active_set(tsd_t *tsd, bool active) { |
475 | assert(tsd_reentrancy_level_get(tsd) == 0); |
476 | |
477 | prof_tdata_t *tdata; |
478 | |
479 | tdata = prof_tdata_get(tsd, true); |
480 | if (tdata == NULL) { |
481 | return true; |
482 | } |
483 | tdata->active = active; |
484 | return false; |
485 | } |
486 | |
487 | bool |
488 | prof_thread_active_init_get(tsdn_t *tsdn) { |
489 | bool active_init; |
490 | |
491 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
492 | active_init = prof_thread_active_init; |
493 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
494 | return active_init; |
495 | } |
496 | |
497 | bool |
498 | prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) { |
499 | bool active_init_old; |
500 | |
501 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
502 | active_init_old = prof_thread_active_init; |
503 | prof_thread_active_init = active_init; |
504 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
505 | return active_init_old; |
506 | } |
507 | |
508 | bool |
509 | prof_gdump_get(tsdn_t *tsdn) { |
510 | bool prof_gdump_current; |
511 | |
512 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
513 | prof_gdump_current = prof_gdump_val; |
514 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
515 | return prof_gdump_current; |
516 | } |
517 | |
518 | bool |
519 | prof_gdump_set(tsdn_t *tsdn, bool gdump) { |
520 | bool prof_gdump_old; |
521 | |
522 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
523 | prof_gdump_old = prof_gdump_val; |
524 | prof_gdump_val = gdump; |
525 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
526 | return prof_gdump_old; |
527 | } |
528 | |
529 | void |
530 | prof_backtrace_hook_set(prof_backtrace_hook_t hook) { |
531 | atomic_store_p(&prof_backtrace_hook, hook, ATOMIC_RELEASE); |
532 | } |
533 | |
534 | prof_backtrace_hook_t |
535 | prof_backtrace_hook_get() { |
536 | return (prof_backtrace_hook_t)atomic_load_p(&prof_backtrace_hook, |
537 | ATOMIC_ACQUIRE); |
538 | } |
539 | |
540 | void |
541 | prof_dump_hook_set(prof_dump_hook_t hook) { |
542 | atomic_store_p(&prof_dump_hook, hook, ATOMIC_RELEASE); |
543 | } |
544 | |
545 | prof_dump_hook_t |
546 | prof_dump_hook_get() { |
547 | return (prof_dump_hook_t)atomic_load_p(&prof_dump_hook, |
548 | ATOMIC_ACQUIRE); |
549 | } |
550 | |
551 | void |
552 | prof_boot0(void) { |
553 | cassert(config_prof); |
554 | |
555 | memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, |
556 | sizeof(PROF_PREFIX_DEFAULT)); |
557 | } |
558 | |
559 | void |
560 | prof_boot1(void) { |
561 | cassert(config_prof); |
562 | |
563 | /* |
564 | * opt_prof must be in its final state before any arenas are |
565 | * initialized, so this function must be executed early. |
566 | */ |
567 | if (opt_prof_leak_error && !opt_prof_leak) { |
568 | opt_prof_leak = true; |
569 | } |
570 | |
571 | if (opt_prof_leak && !opt_prof) { |
572 | /* |
573 | * Enable opt_prof, but in such a way that profiles are never |
574 | * automatically dumped. |
575 | */ |
576 | opt_prof = true; |
577 | opt_prof_gdump = false; |
578 | } else if (opt_prof) { |
579 | if (opt_lg_prof_interval >= 0) { |
580 | prof_interval = (((uint64_t)1U) << |
581 | opt_lg_prof_interval); |
582 | } |
583 | } |
584 | } |
585 | |
586 | bool |
587 | prof_boot2(tsd_t *tsd, base_t *base) { |
588 | cassert(config_prof); |
589 | |
590 | /* |
591 | * Initialize the global mutexes unconditionally to maintain correct |
592 | * stats when opt_prof is false. |
593 | */ |
594 | if (malloc_mutex_init(&prof_active_mtx, "prof_active" , |
595 | WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) { |
596 | return true; |
597 | } |
598 | if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump" , |
599 | WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) { |
600 | return true; |
601 | } |
602 | if (malloc_mutex_init(&prof_thread_active_init_mtx, |
603 | "prof_thread_active_init" , WITNESS_RANK_PROF_THREAD_ACTIVE_INIT, |
604 | malloc_mutex_rank_exclusive)) { |
605 | return true; |
606 | } |
607 | if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx" , |
608 | WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) { |
609 | return true; |
610 | } |
611 | if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas" , |
612 | WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) { |
613 | return true; |
614 | } |
615 | if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid" , |
616 | WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) { |
617 | return true; |
618 | } |
619 | if (malloc_mutex_init(&prof_stats_mtx, "prof_stats" , |
620 | WITNESS_RANK_PROF_STATS, malloc_mutex_rank_exclusive)) { |
621 | return true; |
622 | } |
623 | if (malloc_mutex_init(&prof_dump_filename_mtx, |
624 | "prof_dump_filename" , WITNESS_RANK_PROF_DUMP_FILENAME, |
625 | malloc_mutex_rank_exclusive)) { |
626 | return true; |
627 | } |
628 | if (malloc_mutex_init(&prof_dump_mtx, "prof_dump" , |
629 | WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) { |
630 | return true; |
631 | } |
632 | |
633 | if (opt_prof) { |
634 | lg_prof_sample = opt_lg_prof_sample; |
635 | prof_unbias_map_init(); |
636 | prof_active_state = opt_prof_active; |
637 | prof_gdump_val = opt_prof_gdump; |
638 | prof_thread_active_init = opt_prof_thread_active_init; |
639 | |
640 | if (prof_data_init(tsd)) { |
641 | return true; |
642 | } |
643 | |
644 | next_thr_uid = 0; |
645 | if (prof_idump_accum_init()) { |
646 | return true; |
647 | } |
648 | |
649 | if (opt_prof_final && opt_prof_prefix[0] != '\0' && |
650 | atexit(prof_fdump) != 0) { |
651 | malloc_write("<jemalloc>: Error in atexit()\n" ); |
652 | if (opt_abort) { |
653 | abort(); |
654 | } |
655 | } |
656 | |
657 | if (prof_log_init(tsd)) { |
658 | return true; |
659 | } |
660 | |
661 | if (prof_recent_init()) { |
662 | return true; |
663 | } |
664 | |
665 | prof_base = base; |
666 | |
667 | gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base, |
668 | PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE); |
669 | if (gctx_locks == NULL) { |
670 | return true; |
671 | } |
672 | for (unsigned i = 0; i < PROF_NCTX_LOCKS; i++) { |
673 | if (malloc_mutex_init(&gctx_locks[i], "prof_gctx" , |
674 | WITNESS_RANK_PROF_GCTX, |
675 | malloc_mutex_rank_exclusive)) { |
676 | return true; |
677 | } |
678 | } |
679 | |
680 | tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base, |
681 | PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t), CACHELINE); |
682 | if (tdata_locks == NULL) { |
683 | return true; |
684 | } |
685 | for (unsigned i = 0; i < PROF_NTDATA_LOCKS; i++) { |
686 | if (malloc_mutex_init(&tdata_locks[i], "prof_tdata" , |
687 | WITNESS_RANK_PROF_TDATA, |
688 | malloc_mutex_rank_exclusive)) { |
689 | return true; |
690 | } |
691 | } |
692 | |
693 | prof_unwind_init(); |
694 | prof_hooks_init(); |
695 | } |
696 | prof_booted = true; |
697 | |
698 | return false; |
699 | } |
700 | |
701 | void |
702 | prof_prefork0(tsdn_t *tsdn) { |
703 | if (config_prof && opt_prof) { |
704 | unsigned i; |
705 | |
706 | malloc_mutex_prefork(tsdn, &prof_dump_mtx); |
707 | malloc_mutex_prefork(tsdn, &bt2gctx_mtx); |
708 | malloc_mutex_prefork(tsdn, &tdatas_mtx); |
709 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) { |
710 | malloc_mutex_prefork(tsdn, &tdata_locks[i]); |
711 | } |
712 | malloc_mutex_prefork(tsdn, &log_mtx); |
713 | for (i = 0; i < PROF_NCTX_LOCKS; i++) { |
714 | malloc_mutex_prefork(tsdn, &gctx_locks[i]); |
715 | } |
716 | malloc_mutex_prefork(tsdn, &prof_recent_dump_mtx); |
717 | } |
718 | } |
719 | |
720 | void |
721 | prof_prefork1(tsdn_t *tsdn) { |
722 | if (config_prof && opt_prof) { |
723 | counter_prefork(tsdn, &prof_idump_accumulated); |
724 | malloc_mutex_prefork(tsdn, &prof_active_mtx); |
725 | malloc_mutex_prefork(tsdn, &prof_dump_filename_mtx); |
726 | malloc_mutex_prefork(tsdn, &prof_gdump_mtx); |
727 | malloc_mutex_prefork(tsdn, &prof_recent_alloc_mtx); |
728 | malloc_mutex_prefork(tsdn, &prof_stats_mtx); |
729 | malloc_mutex_prefork(tsdn, &next_thr_uid_mtx); |
730 | malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx); |
731 | } |
732 | } |
733 | |
734 | void |
735 | prof_postfork_parent(tsdn_t *tsdn) { |
736 | if (config_prof && opt_prof) { |
737 | unsigned i; |
738 | |
739 | malloc_mutex_postfork_parent(tsdn, |
740 | &prof_thread_active_init_mtx); |
741 | malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); |
742 | malloc_mutex_postfork_parent(tsdn, &prof_stats_mtx); |
743 | malloc_mutex_postfork_parent(tsdn, &prof_recent_alloc_mtx); |
744 | malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx); |
745 | malloc_mutex_postfork_parent(tsdn, &prof_dump_filename_mtx); |
746 | malloc_mutex_postfork_parent(tsdn, &prof_active_mtx); |
747 | counter_postfork_parent(tsdn, &prof_idump_accumulated); |
748 | malloc_mutex_postfork_parent(tsdn, &prof_recent_dump_mtx); |
749 | for (i = 0; i < PROF_NCTX_LOCKS; i++) { |
750 | malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]); |
751 | } |
752 | malloc_mutex_postfork_parent(tsdn, &log_mtx); |
753 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) { |
754 | malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]); |
755 | } |
756 | malloc_mutex_postfork_parent(tsdn, &tdatas_mtx); |
757 | malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx); |
758 | malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx); |
759 | } |
760 | } |
761 | |
762 | void |
763 | prof_postfork_child(tsdn_t *tsdn) { |
764 | if (config_prof && opt_prof) { |
765 | unsigned i; |
766 | |
767 | malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx); |
768 | malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx); |
769 | malloc_mutex_postfork_child(tsdn, &prof_stats_mtx); |
770 | malloc_mutex_postfork_child(tsdn, &prof_recent_alloc_mtx); |
771 | malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx); |
772 | malloc_mutex_postfork_child(tsdn, &prof_dump_filename_mtx); |
773 | malloc_mutex_postfork_child(tsdn, &prof_active_mtx); |
774 | counter_postfork_child(tsdn, &prof_idump_accumulated); |
775 | malloc_mutex_postfork_child(tsdn, &prof_recent_dump_mtx); |
776 | for (i = 0; i < PROF_NCTX_LOCKS; i++) { |
777 | malloc_mutex_postfork_child(tsdn, &gctx_locks[i]); |
778 | } |
779 | malloc_mutex_postfork_child(tsdn, &log_mtx); |
780 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) { |
781 | malloc_mutex_postfork_child(tsdn, &tdata_locks[i]); |
782 | } |
783 | malloc_mutex_postfork_child(tsdn, &tdatas_mtx); |
784 | malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx); |
785 | malloc_mutex_postfork_child(tsdn, &prof_dump_mtx); |
786 | } |
787 | } |
788 | |
789 | /******************************************************************************/ |
790 | |