1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/ckh.h"
6#include "jemalloc/internal/hash.h"
7#include "jemalloc/internal/malloc_io.h"
8#include "jemalloc/internal/prof_data.h"
9
10/*
11 * This file defines and manages the core profiling data structures.
12 *
13 * Conceptually, profiling data can be imagined as a table with three columns:
14 * thread, stack trace, and current allocation size. (When prof_accum is on,
15 * there's one additional column which is the cumulative allocation size.)
16 *
17 * Implementation wise, each thread maintains a hash recording the stack trace
18 * to allocation size correspondences, which are basically the individual rows
19 * in the table. In addition, two global "indices" are built to make data
20 * aggregation efficient (for dumping): bt2gctx and tdatas, which are basically
21 * the "grouped by stack trace" and "grouped by thread" views of the same table,
22 * respectively. Note that the allocation size is only aggregated to the two
23 * indices at dumping time, so as to optimize for performance.
24 */
25
26/******************************************************************************/
27
28malloc_mutex_t bt2gctx_mtx;
29malloc_mutex_t tdatas_mtx;
30malloc_mutex_t prof_dump_mtx;
31
32/*
33 * Table of mutexes that are shared among gctx's. These are leaf locks, so
34 * there is no problem with using them for more than one gctx at the same time.
35 * The primary motivation for this sharing though is that gctx's are ephemeral,
36 * and destroying mutexes causes complications for systems that allocate when
37 * creating/destroying mutexes.
38 */
39malloc_mutex_t *gctx_locks;
40static atomic_u_t cum_gctxs; /* Atomic counter. */
41
42/*
43 * Table of mutexes that are shared among tdata's. No operations require
44 * holding multiple tdata locks, so there is no problem with using them for more
45 * than one tdata at the same time, even though a gctx lock may be acquired
46 * while holding a tdata lock.
47 */
48malloc_mutex_t *tdata_locks;
49
50/*
51 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
52 * structure that knows about all backtraces currently captured.
53 */
54static ckh_t bt2gctx;
55
56/*
57 * Tree of all extant prof_tdata_t structures, regardless of state,
58 * {attached,detached,expired}.
59 */
60static prof_tdata_tree_t tdatas;
61
62size_t prof_unbiased_sz[PROF_SC_NSIZES];
63size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES];
64
65/******************************************************************************/
66/* Red-black trees. */
67
68static int
69prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
70 uint64_t a_thr_uid = a->thr_uid;
71 uint64_t b_thr_uid = b->thr_uid;
72 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
73 if (ret == 0) {
74 uint64_t a_thr_discrim = a->thr_discrim;
75 uint64_t b_thr_discrim = b->thr_discrim;
76 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
77 b_thr_discrim);
78 if (ret == 0) {
79 uint64_t a_tctx_uid = a->tctx_uid;
80 uint64_t b_tctx_uid = b->tctx_uid;
81 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
82 b_tctx_uid);
83 }
84 }
85 return ret;
86}
87
88rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
89 tctx_link, prof_tctx_comp)
90
91static int
92prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
93 unsigned a_len = a->bt.len;
94 unsigned b_len = b->bt.len;
95 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
96 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
97 if (ret == 0) {
98 ret = (a_len > b_len) - (a_len < b_len);
99 }
100 return ret;
101}
102
103rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
104 prof_gctx_comp)
105
106static int
107prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
108 int ret;
109 uint64_t a_uid = a->thr_uid;
110 uint64_t b_uid = b->thr_uid;
111
112 ret = ((a_uid > b_uid) - (a_uid < b_uid));
113 if (ret == 0) {
114 uint64_t a_discrim = a->thr_discrim;
115 uint64_t b_discrim = b->thr_discrim;
116
117 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
118 }
119 return ret;
120}
121
122rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
123 prof_tdata_comp)
124
125/******************************************************************************/
126
127static malloc_mutex_t *
128prof_gctx_mutex_choose(void) {
129 unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);
130
131 return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
132}
133
134static malloc_mutex_t *
135prof_tdata_mutex_choose(uint64_t thr_uid) {
136 return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
137}
138
139bool
140prof_data_init(tsd_t *tsd) {
141 tdata_tree_new(&tdatas);
142 return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS,
143 prof_bt_hash, prof_bt_keycomp);
144}
145
146static void
147prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
148 cassert(config_prof);
149 assert(tdata == prof_tdata_get(tsd, false));
150
151 if (tdata != NULL) {
152 assert(!tdata->enq);
153 tdata->enq = true;
154 }
155
156 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
157}
158
159static void
160prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
161 cassert(config_prof);
162 assert(tdata == prof_tdata_get(tsd, false));
163
164 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
165
166 if (tdata != NULL) {
167 bool idump, gdump;
168
169 assert(tdata->enq);
170 tdata->enq = false;
171 idump = tdata->enq_idump;
172 tdata->enq_idump = false;
173 gdump = tdata->enq_gdump;
174 tdata->enq_gdump = false;
175
176 if (idump) {
177 prof_idump(tsd_tsdn(tsd));
178 }
179 if (gdump) {
180 prof_gdump(tsd_tsdn(tsd));
181 }
182 }
183}
184
185static prof_gctx_t *
186prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
187 /*
188 * Create a single allocation that has space for vec of length bt->len.
189 */
190 size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
191 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
192 sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
193 true);
194 if (gctx == NULL) {
195 return NULL;
196 }
197 gctx->lock = prof_gctx_mutex_choose();
198 /*
199 * Set nlimbo to 1, in order to avoid a race condition with
200 * prof_tctx_destroy()/prof_gctx_try_destroy().
201 */
202 gctx->nlimbo = 1;
203 tctx_tree_new(&gctx->tctxs);
204 /* Duplicate bt. */
205 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
206 gctx->bt.vec = gctx->vec;
207 gctx->bt.len = bt->len;
208 return gctx;
209}
210
211static void
212prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self,
213 prof_gctx_t *gctx) {
214 cassert(config_prof);
215
216 /*
217 * Check that gctx is still unused by any thread cache before destroying
218 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
219 * condition with this function, as does prof_tctx_destroy() in order to
220 * avoid a race between the main body of prof_tctx_destroy() and entry
221 * into this function.
222 */
223 prof_enter(tsd, tdata_self);
224 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
225 assert(gctx->nlimbo != 0);
226 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
227 /* Remove gctx from bt2gctx. */
228 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
229 not_reached();
230 }
231 prof_leave(tsd, tdata_self);
232 /* Destroy gctx. */
233 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
234 idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
235 } else {
236 /*
237 * Compensate for increment in prof_tctx_destroy() or
238 * prof_lookup().
239 */
240 gctx->nlimbo--;
241 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
242 prof_leave(tsd, tdata_self);
243 }
244}
245
246static bool
247prof_gctx_should_destroy(prof_gctx_t *gctx) {
248 if (opt_prof_accum) {
249 return false;
250 }
251 if (!tctx_tree_empty(&gctx->tctxs)) {
252 return false;
253 }
254 if (gctx->nlimbo != 0) {
255 return false;
256 }
257 return true;
258}
259
260static bool
261prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
262 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
263 union {
264 prof_gctx_t *p;
265 void *v;
266 } gctx, tgctx;
267 union {
268 prof_bt_t *p;
269 void *v;
270 } btkey;
271 bool new_gctx;
272
273 prof_enter(tsd, tdata);
274 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
275 /* bt has never been seen before. Insert it. */
276 prof_leave(tsd, tdata);
277 tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
278 if (tgctx.v == NULL) {
279 return true;
280 }
281 prof_enter(tsd, tdata);
282 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
283 gctx.p = tgctx.p;
284 btkey.p = &gctx.p->bt;
285 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
286 /* OOM. */
287 prof_leave(tsd, tdata);
288 idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
289 true, true);
290 return true;
291 }
292 new_gctx = true;
293 } else {
294 new_gctx = false;
295 }
296 } else {
297 tgctx.v = NULL;
298 new_gctx = false;
299 }
300
301 if (!new_gctx) {
302 /*
303 * Increment nlimbo, in order to avoid a race condition with
304 * prof_tctx_destroy()/prof_gctx_try_destroy().
305 */
306 malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
307 gctx.p->nlimbo++;
308 malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
309 new_gctx = false;
310
311 if (tgctx.v != NULL) {
312 /* Lost race to insert. */
313 idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
314 true);
315 }
316 }
317 prof_leave(tsd, tdata);
318
319 *p_btkey = btkey.v;
320 *p_gctx = gctx.p;
321 *p_new_gctx = new_gctx;
322 return false;
323}
324
325prof_tctx_t *
326prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
327 union {
328 prof_tctx_t *p;
329 void *v;
330 } ret;
331 prof_tdata_t *tdata;
332 bool not_found;
333
334 cassert(config_prof);
335
336 tdata = prof_tdata_get(tsd, false);
337 assert(tdata != NULL);
338
339 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
340 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
341 if (!not_found) { /* Note double negative! */
342 ret.p->prepared = true;
343 }
344 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
345 if (not_found) {
346 void *btkey;
347 prof_gctx_t *gctx;
348 bool new_gctx, error;
349
350 /*
351 * This thread's cache lacks bt. Look for it in the global
352 * cache.
353 */
354 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
355 &new_gctx)) {
356 return NULL;
357 }
358
359 /* Link a prof_tctx_t into gctx for this thread. */
360 ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
361 sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
362 arena_ichoose(tsd, NULL), true);
363 if (ret.p == NULL) {
364 if (new_gctx) {
365 prof_gctx_try_destroy(tsd, tdata, gctx);
366 }
367 return NULL;
368 }
369 ret.p->tdata = tdata;
370 ret.p->thr_uid = tdata->thr_uid;
371 ret.p->thr_discrim = tdata->thr_discrim;
372 ret.p->recent_count = 0;
373 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
374 ret.p->gctx = gctx;
375 ret.p->tctx_uid = tdata->tctx_uid_next++;
376 ret.p->prepared = true;
377 ret.p->state = prof_tctx_state_initializing;
378 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
379 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
380 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
381 if (error) {
382 if (new_gctx) {
383 prof_gctx_try_destroy(tsd, tdata, gctx);
384 }
385 idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
386 return NULL;
387 }
388 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
389 ret.p->state = prof_tctx_state_nominal;
390 tctx_tree_insert(&gctx->tctxs, ret.p);
391 gctx->nlimbo--;
392 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
393 }
394
395 return ret.p;
396}
397
398/* Used in unit tests. */
399static prof_tdata_t *
400prof_tdata_count_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
401 void *arg) {
402 size_t *tdata_count = (size_t *)arg;
403
404 (*tdata_count)++;
405
406 return NULL;
407}
408
409/* Used in unit tests. */
410size_t
411prof_tdata_count(void) {
412 size_t tdata_count = 0;
413 tsdn_t *tsdn;
414
415 tsdn = tsdn_fetch();
416 malloc_mutex_lock(tsdn, &tdatas_mtx);
417 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
418 (void *)&tdata_count);
419 malloc_mutex_unlock(tsdn, &tdatas_mtx);
420
421 return tdata_count;
422}
423
424/* Used in unit tests. */
425size_t
426prof_bt_count(void) {
427 size_t bt_count;
428 tsd_t *tsd;
429 prof_tdata_t *tdata;
430
431 tsd = tsd_fetch();
432 tdata = prof_tdata_get(tsd, false);
433 if (tdata == NULL) {
434 return 0;
435 }
436
437 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
438 bt_count = ckh_count(&bt2gctx);
439 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
440
441 return bt_count;
442}
443
444char *
445prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) {
446 char *ret;
447 size_t size;
448
449 if (thread_name == NULL) {
450 return NULL;
451 }
452
453 size = strlen(thread_name) + 1;
454 if (size == 1) {
455 return "";
456 }
457
458 ret = iallocztm(tsd_tsdn(tsd), size, sz_size2index(size), false, NULL,
459 true, arena_get(TSDN_NULL, 0, true), true);
460 if (ret == NULL) {
461 return NULL;
462 }
463 memcpy(ret, thread_name, size);
464 return ret;
465}
466
467int
468prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name) {
469 assert(tsd_reentrancy_level_get(tsd) == 0);
470
471 prof_tdata_t *tdata;
472 unsigned i;
473 char *s;
474
475 tdata = prof_tdata_get(tsd, true);
476 if (tdata == NULL) {
477 return EAGAIN;
478 }
479
480 /* Validate input. */
481 if (thread_name == NULL) {
482 return EFAULT;
483 }
484 for (i = 0; thread_name[i] != '\0'; i++) {
485 char c = thread_name[i];
486 if (!isgraph(c) && !isblank(c)) {
487 return EFAULT;
488 }
489 }
490
491 s = prof_thread_name_alloc(tsd, thread_name);
492 if (s == NULL) {
493 return EAGAIN;
494 }
495
496 if (tdata->thread_name != NULL) {
497 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
498 true);
499 tdata->thread_name = NULL;
500 }
501 if (strlen(s) > 0) {
502 tdata->thread_name = s;
503 }
504 return 0;
505}
506
507JEMALLOC_FORMAT_PRINTF(3, 4)
508static void
509prof_dump_printf(write_cb_t *prof_dump_write, void *cbopaque,
510 const char *format, ...) {
511 va_list ap;
512 char buf[PROF_PRINTF_BUFSIZE];
513
514 va_start(ap, format);
515 malloc_vsnprintf(buf, sizeof(buf), format, ap);
516 va_end(ap);
517 prof_dump_write(cbopaque, buf);
518}
519
520/*
521 * Casting a double to a uint64_t may not necessarily be in range; this can be
522 * UB. I don't think this is practically possible with the cur counters, but
523 * plausibly could be with the accum counters.
524 */
525#ifdef JEMALLOC_PROF
526static uint64_t
527prof_double_uint64_cast(double d) {
528 /*
529 * Note: UINT64_MAX + 1 is exactly representable as a double on all
530 * reasonable platforms (certainly those we'll support). Writing this
531 * as !(a < b) instead of (a >= b) means that we're NaN-safe.
532 */
533 double rounded = round(d);
534 if (!(rounded < (double)UINT64_MAX)) {
535 return UINT64_MAX;
536 }
537 return (uint64_t)rounded;
538}
539#endif
540
541void prof_unbias_map_init() {
542 /* See the comment in prof_sample_new_event_wait */
543#ifdef JEMALLOC_PROF
544 for (szind_t i = 0; i < SC_NSIZES; i++) {
545 double sz = (double)sz_index2size(i);
546 double rate = (double)(ZU(1) << lg_prof_sample);
547 double div_val = 1.0 - exp(-sz / rate);
548 double unbiased_sz = sz / div_val;
549 /*
550 * The "true" right value for the unbiased count is
551 * 1.0/(1 - exp(-sz/rate)). The problem is, we keep the counts
552 * as integers (for a variety of reasons -- rounding errors
553 * could trigger asserts, and not all libcs can properly handle
554 * floating point arithmetic during malloc calls inside libc).
555 * Rounding to an integer, though, can lead to rounding errors
556 * of over 30% for sizes close to the sampling rate. So
557 * instead, we multiply by a constant, dividing the maximum
558 * possible roundoff error by that constant. To avoid overflow
559 * in summing up size_t values, the largest safe constant we can
560 * pick is the size of the smallest allocation.
561 */
562 double cnt_shift = (double)(ZU(1) << SC_LG_TINY_MIN);
563 double shifted_unbiased_cnt = cnt_shift / div_val;
564 prof_unbiased_sz[i] = (size_t)round(unbiased_sz);
565 prof_shifted_unbiased_cnt[i] = (size_t)round(
566 shifted_unbiased_cnt);
567 }
568#else
569 unreachable();
570#endif
571}
572
573/*
574 * The unbiasing story is long. The jeprof unbiasing logic was copied from
575 * pprof. Both shared an issue: they unbiased using the average size of the
576 * allocations at a particular stack trace. This can work out OK if allocations
577 * are mostly of the same size given some stack, but not otherwise. We now
578 * internally track what the unbiased results ought to be. We can't just report
579 * them as they are though; they'll still go through the jeprof unbiasing
580 * process. Instead, we figure out what values we can feed *into* jeprof's
581 * unbiasing mechanism that will lead to getting the right values out.
582 *
583 * It'll unbias count and aggregate size as:
584 *
585 * c_out = c_in * 1/(1-exp(-s_in/c_in/R)
586 * s_out = s_in * 1/(1-exp(-s_in/c_in/R)
587 *
588 * We want to solve for the values of c_in and s_in that will
589 * give the c_out and s_out that we've computed internally.
590 *
591 * Let's do a change of variables (both to make the math easier and to make it
592 * easier to write):
593 * x = s_in / c_in
594 * y = s_in
595 * k = 1/R.
596 *
597 * Then
598 * c_out = y/x * 1/(1-exp(-k*x))
599 * s_out = y * 1/(1-exp(-k*x))
600 *
601 * The first equation gives:
602 * y = x * c_out * (1-exp(-k*x))
603 * The second gives:
604 * y = s_out * (1-exp(-k*x))
605 * So we have
606 * x = s_out / c_out.
607 * And all the other values fall out from that.
608 *
609 * This is all a fair bit of work. The thing we get out of it is that we don't
610 * break backwards compatibility with jeprof (and the various tools that have
611 * copied its unbiasing logic). Eventually, we anticipate a v3 heap profile
612 * dump format based on JSON, at which point I think much of this logic can get
613 * cleaned up (since we'll be taking a compatibility break there anyways).
614 */
615static void
616prof_do_unbias(uint64_t c_out_shifted_i, uint64_t s_out_i, uint64_t *r_c_in,
617 uint64_t *r_s_in) {
618#ifdef JEMALLOC_PROF
619 if (c_out_shifted_i == 0 || s_out_i == 0) {
620 *r_c_in = 0;
621 *r_s_in = 0;
622 return;
623 }
624 /*
625 * See the note in prof_unbias_map_init() to see why we take c_out in a
626 * shifted form.
627 */
628 double c_out = (double)c_out_shifted_i
629 / (double)(ZU(1) << SC_LG_TINY_MIN);
630 double s_out = (double)s_out_i;
631 double R = (double)(ZU(1) << lg_prof_sample);
632
633 double x = s_out / c_out;
634 double y = s_out * (1.0 - exp(-x / R));
635
636 double c_in = y / x;
637 double s_in = y;
638
639 *r_c_in = prof_double_uint64_cast(c_in);
640 *r_s_in = prof_double_uint64_cast(s_in);
641#else
642 unreachable();
643#endif
644}
645
646static void
647prof_dump_print_cnts(write_cb_t *prof_dump_write, void *cbopaque,
648 const prof_cnt_t *cnts) {
649 uint64_t curobjs;
650 uint64_t curbytes;
651 uint64_t accumobjs;
652 uint64_t accumbytes;
653 if (opt_prof_unbias) {
654 prof_do_unbias(cnts->curobjs_shifted_unbiased,
655 cnts->curbytes_unbiased, &curobjs, &curbytes);
656 prof_do_unbias(cnts->accumobjs_shifted_unbiased,
657 cnts->accumbytes_unbiased, &accumobjs, &accumbytes);
658 } else {
659 curobjs = cnts->curobjs;
660 curbytes = cnts->curbytes;
661 accumobjs = cnts->accumobjs;
662 accumbytes = cnts->accumbytes;
663 }
664 prof_dump_printf(prof_dump_write, cbopaque,
665 "%"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]",
666 curobjs, curbytes, accumobjs, accumbytes);
667}
668
669static void
670prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
671 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
672
673 malloc_mutex_lock(tsdn, tctx->gctx->lock);
674
675 switch (tctx->state) {
676 case prof_tctx_state_initializing:
677 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
678 return;
679 case prof_tctx_state_nominal:
680 tctx->state = prof_tctx_state_dumping;
681 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
682
683 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
684
685 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
686 tdata->cnt_summed.curobjs_shifted_unbiased
687 += tctx->dump_cnts.curobjs_shifted_unbiased;
688 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
689 tdata->cnt_summed.curbytes_unbiased
690 += tctx->dump_cnts.curbytes_unbiased;
691 if (opt_prof_accum) {
692 tdata->cnt_summed.accumobjs +=
693 tctx->dump_cnts.accumobjs;
694 tdata->cnt_summed.accumobjs_shifted_unbiased +=
695 tctx->dump_cnts.accumobjs_shifted_unbiased;
696 tdata->cnt_summed.accumbytes +=
697 tctx->dump_cnts.accumbytes;
698 tdata->cnt_summed.accumbytes_unbiased +=
699 tctx->dump_cnts.accumbytes_unbiased;
700 }
701 break;
702 case prof_tctx_state_dumping:
703 case prof_tctx_state_purgatory:
704 not_reached();
705 }
706}
707
708static void
709prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
710 malloc_mutex_assert_owner(tsdn, gctx->lock);
711
712 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
713 gctx->cnt_summed.curobjs_shifted_unbiased
714 += tctx->dump_cnts.curobjs_shifted_unbiased;
715 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
716 gctx->cnt_summed.curbytes_unbiased += tctx->dump_cnts.curbytes_unbiased;
717 if (opt_prof_accum) {
718 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
719 gctx->cnt_summed.accumobjs_shifted_unbiased
720 += tctx->dump_cnts.accumobjs_shifted_unbiased;
721 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
722 gctx->cnt_summed.accumbytes_unbiased
723 += tctx->dump_cnts.accumbytes_unbiased;
724 }
725}
726
727static prof_tctx_t *
728prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
729 tsdn_t *tsdn = (tsdn_t *)arg;
730
731 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
732
733 switch (tctx->state) {
734 case prof_tctx_state_nominal:
735 /* New since dumping started; ignore. */
736 break;
737 case prof_tctx_state_dumping:
738 case prof_tctx_state_purgatory:
739 prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
740 break;
741 default:
742 not_reached();
743 }
744
745 return NULL;
746}
747
748typedef struct prof_dump_iter_arg_s prof_dump_iter_arg_t;
749struct prof_dump_iter_arg_s {
750 tsdn_t *tsdn;
751 write_cb_t *prof_dump_write;
752 void *cbopaque;
753};
754
755static prof_tctx_t *
756prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
757 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
758 malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
759
760 switch (tctx->state) {
761 case prof_tctx_state_initializing:
762 case prof_tctx_state_nominal:
763 /* Not captured by this dump. */
764 break;
765 case prof_tctx_state_dumping:
766 case prof_tctx_state_purgatory:
767 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
768 " t%"FMTu64": ", tctx->thr_uid);
769 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
770 &tctx->dump_cnts);
771 arg->prof_dump_write(arg->cbopaque, "\n");
772 break;
773 default:
774 not_reached();
775 }
776 return NULL;
777}
778
779static prof_tctx_t *
780prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
781 tsdn_t *tsdn = (tsdn_t *)arg;
782 prof_tctx_t *ret;
783
784 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
785
786 switch (tctx->state) {
787 case prof_tctx_state_nominal:
788 /* New since dumping started; ignore. */
789 break;
790 case prof_tctx_state_dumping:
791 tctx->state = prof_tctx_state_nominal;
792 break;
793 case prof_tctx_state_purgatory:
794 ret = tctx;
795 goto label_return;
796 default:
797 not_reached();
798 }
799
800 ret = NULL;
801label_return:
802 return ret;
803}
804
805static void
806prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
807 cassert(config_prof);
808
809 malloc_mutex_lock(tsdn, gctx->lock);
810
811 /*
812 * Increment nlimbo so that gctx won't go away before dump.
813 * Additionally, link gctx into the dump list so that it is included in
814 * prof_dump()'s second pass.
815 */
816 gctx->nlimbo++;
817 gctx_tree_insert(gctxs, gctx);
818
819 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
820
821 malloc_mutex_unlock(tsdn, gctx->lock);
822}
823
824typedef struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg_t;
825struct prof_gctx_merge_iter_arg_s {
826 tsdn_t *tsdn;
827 size_t *leak_ngctx;
828};
829
830static prof_gctx_t *
831prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
832 prof_gctx_merge_iter_arg_t *arg = (prof_gctx_merge_iter_arg_t *)opaque;
833
834 malloc_mutex_lock(arg->tsdn, gctx->lock);
835 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
836 (void *)arg->tsdn);
837 if (gctx->cnt_summed.curobjs != 0) {
838 (*arg->leak_ngctx)++;
839 }
840 malloc_mutex_unlock(arg->tsdn, gctx->lock);
841
842 return NULL;
843}
844
845static void
846prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
847 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
848 prof_gctx_t *gctx;
849
850 /*
851 * Standard tree iteration won't work here, because as soon as we
852 * decrement gctx->nlimbo and unlock gctx, another thread can
853 * concurrently destroy it, which will corrupt the tree. Therefore,
854 * tear down the tree one node at a time during iteration.
855 */
856 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
857 gctx_tree_remove(gctxs, gctx);
858 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
859 {
860 prof_tctx_t *next;
861
862 next = NULL;
863 do {
864 prof_tctx_t *to_destroy =
865 tctx_tree_iter(&gctx->tctxs, next,
866 prof_tctx_finish_iter,
867 (void *)tsd_tsdn(tsd));
868 if (to_destroy != NULL) {
869 next = tctx_tree_next(&gctx->tctxs,
870 to_destroy);
871 tctx_tree_remove(&gctx->tctxs,
872 to_destroy);
873 idalloctm(tsd_tsdn(tsd), to_destroy,
874 NULL, NULL, true, true);
875 } else {
876 next = NULL;
877 }
878 } while (next != NULL);
879 }
880 gctx->nlimbo--;
881 if (prof_gctx_should_destroy(gctx)) {
882 gctx->nlimbo++;
883 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
884 prof_gctx_try_destroy(tsd, tdata, gctx);
885 } else {
886 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
887 }
888 }
889}
890
891typedef struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg_t;
892struct prof_tdata_merge_iter_arg_s {
893 tsdn_t *tsdn;
894 prof_cnt_t *cnt_all;
895};
896
897static prof_tdata_t *
898prof_tdata_merge_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
899 void *opaque) {
900 prof_tdata_merge_iter_arg_t *arg =
901 (prof_tdata_merge_iter_arg_t *)opaque;
902
903 malloc_mutex_lock(arg->tsdn, tdata->lock);
904 if (!tdata->expired) {
905 size_t tabind;
906 union {
907 prof_tctx_t *p;
908 void *v;
909 } tctx;
910
911 tdata->dumping = true;
912 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
913 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
914 &tctx.v);) {
915 prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
916 }
917
918 arg->cnt_all->curobjs += tdata->cnt_summed.curobjs;
919 arg->cnt_all->curobjs_shifted_unbiased
920 += tdata->cnt_summed.curobjs_shifted_unbiased;
921 arg->cnt_all->curbytes += tdata->cnt_summed.curbytes;
922 arg->cnt_all->curbytes_unbiased
923 += tdata->cnt_summed.curbytes_unbiased;
924 if (opt_prof_accum) {
925 arg->cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
926 arg->cnt_all->accumobjs_shifted_unbiased
927 += tdata->cnt_summed.accumobjs_shifted_unbiased;
928 arg->cnt_all->accumbytes +=
929 tdata->cnt_summed.accumbytes;
930 arg->cnt_all->accumbytes_unbiased +=
931 tdata->cnt_summed.accumbytes_unbiased;
932 }
933 } else {
934 tdata->dumping = false;
935 }
936 malloc_mutex_unlock(arg->tsdn, tdata->lock);
937
938 return NULL;
939}
940
941static prof_tdata_t *
942prof_tdata_dump_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
943 void *opaque) {
944 if (!tdata->dumping) {
945 return NULL;
946 }
947
948 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
949 prof_dump_printf(arg->prof_dump_write, arg->cbopaque, " t%"FMTu64": ",
950 tdata->thr_uid);
951 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
952 &tdata->cnt_summed);
953 if (tdata->thread_name != NULL) {
954 arg->prof_dump_write(arg->cbopaque, " ");
955 arg->prof_dump_write(arg->cbopaque, tdata->thread_name);
956 }
957 arg->prof_dump_write(arg->cbopaque, "\n");
958 return NULL;
959}
960
961static void
962prof_dump_header(prof_dump_iter_arg_t *arg, const prof_cnt_t *cnt_all) {
963 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
964 "heap_v2/%"FMTu64"\n t*: ", ((uint64_t)1U << lg_prof_sample));
965 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, cnt_all);
966 arg->prof_dump_write(arg->cbopaque, "\n");
967
968 malloc_mutex_lock(arg->tsdn, &tdatas_mtx);
969 tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, arg);
970 malloc_mutex_unlock(arg->tsdn, &tdatas_mtx);
971}
972
973static void
974prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx,
975 const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
976 cassert(config_prof);
977 malloc_mutex_assert_owner(arg->tsdn, gctx->lock);
978
979 /* Avoid dumping such gctx's that have no useful data. */
980 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
981 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
982 assert(gctx->cnt_summed.curobjs == 0);
983 assert(gctx->cnt_summed.curbytes == 0);
984 /*
985 * These asserts would not be correct -- see the comment on races
986 * in prof.c
987 * assert(gctx->cnt_summed.curobjs_unbiased == 0);
988 * assert(gctx->cnt_summed.curbytes_unbiased == 0);
989 */
990 assert(gctx->cnt_summed.accumobjs == 0);
991 assert(gctx->cnt_summed.accumobjs_shifted_unbiased == 0);
992 assert(gctx->cnt_summed.accumbytes == 0);
993 assert(gctx->cnt_summed.accumbytes_unbiased == 0);
994 return;
995 }
996
997 arg->prof_dump_write(arg->cbopaque, "@");
998 for (unsigned i = 0; i < bt->len; i++) {
999 prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
1000 " %#"FMTxPTR, (uintptr_t)bt->vec[i]);
1001 }
1002
1003 arg->prof_dump_write(arg->cbopaque, "\n t*: ");
1004 prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
1005 &gctx->cnt_summed);
1006 arg->prof_dump_write(arg->cbopaque, "\n");
1007
1008 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, arg);
1009}
1010
1011/*
1012 * See prof_sample_new_event_wait() comment for why the body of this function
1013 * is conditionally compiled.
1014 */
1015static void
1016prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx) {
1017#ifdef JEMALLOC_PROF
1018 /*
1019 * Scaling is equivalent AdjustSamples() in jeprof, but the result may
1020 * differ slightly from what jeprof reports, because here we scale the
1021 * summary values, whereas jeprof scales each context individually and
1022 * reports the sums of the scaled values.
1023 */
1024 if (cnt_all->curbytes != 0) {
1025 double sample_period = (double)((uint64_t)1 << lg_prof_sample);
1026 double ratio = (((double)cnt_all->curbytes) /
1027 (double)cnt_all->curobjs) / sample_period;
1028 double scale_factor = 1.0 / (1.0 - exp(-ratio));
1029 uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
1030 * scale_factor);
1031 uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
1032 scale_factor);
1033
1034 malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
1035 " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
1036 curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
1037 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1038 malloc_printf(
1039 "<jemalloc>: Run jeprof on dump output for leak detail\n");
1040 if (opt_prof_leak_error) {
1041 malloc_printf(
1042 "<jemalloc>: Exiting with error code because memory"
1043 " leaks were detected\n");
1044 /*
1045 * Use _exit() with underscore to avoid calling atexit()
1046 * and entering endless cycle.
1047 */
1048 _exit(1);
1049 }
1050 }
1051#endif
1052}
1053
1054static prof_gctx_t *
1055prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
1056 prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
1057 malloc_mutex_lock(arg->tsdn, gctx->lock);
1058 prof_dump_gctx(arg, gctx, &gctx->bt, gctxs);
1059 malloc_mutex_unlock(arg->tsdn, gctx->lock);
1060 return NULL;
1061}
1062
1063static void
1064prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, prof_cnt_t *cnt_all,
1065 size_t *leak_ngctx, prof_gctx_tree_t *gctxs) {
1066 size_t tabind;
1067 union {
1068 prof_gctx_t *p;
1069 void *v;
1070 } gctx;
1071
1072 prof_enter(tsd, tdata);
1073
1074 /*
1075 * Put gctx's in limbo and clear their counters in preparation for
1076 * summing.
1077 */
1078 gctx_tree_new(gctxs);
1079 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
1080 prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
1081 }
1082
1083 /*
1084 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1085 * stats and merge them into the associated gctx's.
1086 */
1087 memset(cnt_all, 0, sizeof(prof_cnt_t));
1088 prof_tdata_merge_iter_arg_t prof_tdata_merge_iter_arg = {tsd_tsdn(tsd),
1089 cnt_all};
1090 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1091 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
1092 &prof_tdata_merge_iter_arg);
1093 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1094
1095 /* Merge tctx stats into gctx's. */
1096 *leak_ngctx = 0;
1097 prof_gctx_merge_iter_arg_t prof_gctx_merge_iter_arg = {tsd_tsdn(tsd),
1098 leak_ngctx};
1099 gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
1100 &prof_gctx_merge_iter_arg);
1101
1102 prof_leave(tsd, tdata);
1103}
1104
1105void
1106prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
1107 prof_tdata_t *tdata, bool leakcheck) {
1108 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_dump_mtx);
1109 prof_cnt_t cnt_all;
1110 size_t leak_ngctx;
1111 prof_gctx_tree_t gctxs;
1112 prof_dump_prep(tsd, tdata, &cnt_all, &leak_ngctx, &gctxs);
1113 prof_dump_iter_arg_t prof_dump_iter_arg = {tsd_tsdn(tsd),
1114 prof_dump_write, cbopaque};
1115 prof_dump_header(&prof_dump_iter_arg, &cnt_all);
1116 gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, &prof_dump_iter_arg);
1117 prof_gctx_finish(tsd, &gctxs);
1118 if (leakcheck) {
1119 prof_leakcheck(&cnt_all, leak_ngctx);
1120 }
1121}
1122
1123/* Used in unit tests. */
1124void
1125prof_cnt_all(prof_cnt_t *cnt_all) {
1126 tsd_t *tsd = tsd_fetch();
1127 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1128 if (tdata == NULL) {
1129 memset(cnt_all, 0, sizeof(prof_cnt_t));
1130 } else {
1131 size_t leak_ngctx;
1132 prof_gctx_tree_t gctxs;
1133 prof_dump_prep(tsd, tdata, cnt_all, &leak_ngctx, &gctxs);
1134 prof_gctx_finish(tsd, &gctxs);
1135 }
1136}
1137
1138void
1139prof_bt_hash(const void *key, size_t r_hash[2]) {
1140 prof_bt_t *bt = (prof_bt_t *)key;
1141
1142 cassert(config_prof);
1143
1144 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1145}
1146
1147bool
1148prof_bt_keycomp(const void *k1, const void *k2) {
1149 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1150 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1151
1152 cassert(config_prof);
1153
1154 if (bt1->len != bt2->len) {
1155 return false;
1156 }
1157 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1158}
1159
1160prof_tdata_t *
1161prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1162 char *thread_name, bool active) {
1163 assert(tsd_reentrancy_level_get(tsd) == 0);
1164
1165 prof_tdata_t *tdata;
1166
1167 cassert(config_prof);
1168
1169 /* Initialize an empty cache for this thread. */
1170 tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
1171 sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
1172 arena_get(TSDN_NULL, 0, true), true);
1173 if (tdata == NULL) {
1174 return NULL;
1175 }
1176
1177 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1178 tdata->thr_uid = thr_uid;
1179 tdata->thr_discrim = thr_discrim;
1180 tdata->thread_name = thread_name;
1181 tdata->attached = true;
1182 tdata->expired = false;
1183 tdata->tctx_uid_next = 0;
1184
1185 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
1186 prof_bt_keycomp)) {
1187 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1188 return NULL;
1189 }
1190
1191 tdata->enq = false;
1192 tdata->enq_idump = false;
1193 tdata->enq_gdump = false;
1194
1195 tdata->dumping = false;
1196 tdata->active = active;
1197
1198 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1199 tdata_tree_insert(&tdatas, tdata);
1200 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1201
1202 return tdata;
1203}
1204
1205static bool
1206prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
1207 if (tdata->attached && !even_if_attached) {
1208 return false;
1209 }
1210 if (ckh_count(&tdata->bt2tctx) != 0) {
1211 return false;
1212 }
1213 return true;
1214}
1215
1216static bool
1217prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
1218 bool even_if_attached) {
1219 malloc_mutex_assert_owner(tsdn, tdata->lock);
1220
1221 return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
1222}
1223
1224static void
1225prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1226 bool even_if_attached) {
1227 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
1228 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tdata->lock);
1229
1230 tdata_tree_remove(&tdatas, tdata);
1231
1232 assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
1233
1234 if (tdata->thread_name != NULL) {
1235 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
1236 true);
1237 }
1238 ckh_delete(tsd, &tdata->bt2tctx);
1239 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1240}
1241
1242static void
1243prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
1244 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1245 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1246 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1247}
1248
1249void
1250prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
1251 bool destroy_tdata;
1252
1253 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
1254 if (tdata->attached) {
1255 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
1256 true);
1257 /*
1258 * Only detach if !destroy_tdata, because detaching would allow
1259 * another thread to win the race to destroy tdata.
1260 */
1261 if (!destroy_tdata) {
1262 tdata->attached = false;
1263 }
1264 tsd_prof_tdata_set(tsd, NULL);
1265 } else {
1266 destroy_tdata = false;
1267 }
1268 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
1269 if (destroy_tdata) {
1270 prof_tdata_destroy(tsd, tdata, true);
1271 }
1272}
1273
1274static bool
1275prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
1276 bool destroy_tdata;
1277
1278 malloc_mutex_lock(tsdn, tdata->lock);
1279 if (!tdata->expired) {
1280 tdata->expired = true;
1281 destroy_tdata = prof_tdata_should_destroy(tsdn, tdata, false);
1282 } else {
1283 destroy_tdata = false;
1284 }
1285 malloc_mutex_unlock(tsdn, tdata->lock);
1286
1287 return destroy_tdata;
1288}
1289
1290static prof_tdata_t *
1291prof_tdata_reset_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
1292 void *arg) {
1293 tsdn_t *tsdn = (tsdn_t *)arg;
1294
1295 return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
1296}
1297
1298void
1299prof_reset(tsd_t *tsd, size_t lg_sample) {
1300 prof_tdata_t *next;
1301
1302 assert(lg_sample < (sizeof(uint64_t) << 3));
1303
1304 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
1305 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1306
1307 lg_prof_sample = lg_sample;
1308 prof_unbias_map_init();
1309
1310 next = NULL;
1311 do {
1312 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1313 prof_tdata_reset_iter, (void *)tsd);
1314 if (to_destroy != NULL) {
1315 next = tdata_tree_next(&tdatas, to_destroy);
1316 prof_tdata_destroy_locked(tsd, to_destroy, false);
1317 } else {
1318 next = NULL;
1319 }
1320 } while (next != NULL);
1321
1322 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1323 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1324}
1325
1326static bool
1327prof_tctx_should_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1328 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1329
1330 if (opt_prof_accum) {
1331 return false;
1332 }
1333 if (tctx->cnts.curobjs != 0) {
1334 return false;
1335 }
1336 if (tctx->prepared) {
1337 return false;
1338 }
1339 if (tctx->recent_count != 0) {
1340 return false;
1341 }
1342 return true;
1343}
1344
1345static void
1346prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1347 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1348
1349 assert(tctx->cnts.curobjs == 0);
1350 assert(tctx->cnts.curbytes == 0);
1351 /*
1352 * These asserts are not correct -- see the comment about races in
1353 * prof.c
1354 *
1355 * assert(tctx->cnts.curobjs_shifted_unbiased == 0);
1356 * assert(tctx->cnts.curbytes_unbiased == 0);
1357 */
1358 assert(!opt_prof_accum);
1359 assert(tctx->cnts.accumobjs == 0);
1360 assert(tctx->cnts.accumbytes == 0);
1361 /*
1362 * These ones are, since accumbyte counts never go down. Either
1363 * prof_accum is off (in which case these should never have changed from
1364 * their initial value of zero), or it's on (in which case we shouldn't
1365 * be destroying this tctx).
1366 */
1367 assert(tctx->cnts.accumobjs_shifted_unbiased == 0);
1368 assert(tctx->cnts.accumbytes_unbiased == 0);
1369
1370 prof_gctx_t *gctx = tctx->gctx;
1371
1372 {
1373 prof_tdata_t *tdata = tctx->tdata;
1374 tctx->tdata = NULL;
1375 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
1376 bool destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd),
1377 tdata, false);
1378 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
1379 if (destroy_tdata) {
1380 prof_tdata_destroy(tsd, tdata, false);
1381 }
1382 }
1383
1384 bool destroy_tctx, destroy_gctx;
1385
1386 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
1387 switch (tctx->state) {
1388 case prof_tctx_state_nominal:
1389 tctx_tree_remove(&gctx->tctxs, tctx);
1390 destroy_tctx = true;
1391 if (prof_gctx_should_destroy(gctx)) {
1392 /*
1393 * Increment gctx->nlimbo in order to keep another
1394 * thread from winning the race to destroy gctx while
1395 * this one has gctx->lock dropped. Without this, it
1396 * would be possible for another thread to:
1397 *
1398 * 1) Sample an allocation associated with gctx.
1399 * 2) Deallocate the sampled object.
1400 * 3) Successfully prof_gctx_try_destroy(gctx).
1401 *
1402 * The result would be that gctx no longer exists by the
1403 * time this thread accesses it in
1404 * prof_gctx_try_destroy().
1405 */
1406 gctx->nlimbo++;
1407 destroy_gctx = true;
1408 } else {
1409 destroy_gctx = false;
1410 }
1411 break;
1412 case prof_tctx_state_dumping:
1413 /*
1414 * A dumping thread needs tctx to remain valid until dumping
1415 * has finished. Change state such that the dumping thread will
1416 * complete destruction during a late dump iteration phase.
1417 */
1418 tctx->state = prof_tctx_state_purgatory;
1419 destroy_tctx = false;
1420 destroy_gctx = false;
1421 break;
1422 default:
1423 not_reached();
1424 destroy_tctx = false;
1425 destroy_gctx = false;
1426 }
1427 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1428 if (destroy_gctx) {
1429 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx);
1430 }
1431 if (destroy_tctx) {
1432 idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
1433 }
1434}
1435
1436void
1437prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
1438 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
1439 if (prof_tctx_should_destroy(tsd, tctx)) {
1440 /* tctx->tdata->lock will be released in prof_tctx_destroy(). */
1441 prof_tctx_destroy(tsd, tctx);
1442 } else {
1443 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
1444 }
1445}
1446
1447/******************************************************************************/
1448