1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5#include "jemalloc/internal/buf_writer.h"
6#include "jemalloc/internal/emitter.h"
7#include "jemalloc/internal/prof_data.h"
8#include "jemalloc/internal/prof_recent.h"
9
10ssize_t opt_prof_recent_alloc_max = PROF_RECENT_ALLOC_MAX_DEFAULT;
11malloc_mutex_t prof_recent_alloc_mtx; /* Protects the fields below */
12static atomic_zd_t prof_recent_alloc_max;
13static ssize_t prof_recent_alloc_count = 0;
14prof_recent_list_t prof_recent_alloc_list;
15
16malloc_mutex_t prof_recent_dump_mtx; /* Protects dumping. */
17
18static void
19prof_recent_alloc_max_init() {
20 atomic_store_zd(&prof_recent_alloc_max, opt_prof_recent_alloc_max,
21 ATOMIC_RELAXED);
22}
23
24static inline ssize_t
25prof_recent_alloc_max_get_no_lock() {
26 return atomic_load_zd(&prof_recent_alloc_max, ATOMIC_RELAXED);
27}
28
29static inline ssize_t
30prof_recent_alloc_max_get(tsd_t *tsd) {
31 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
32 return prof_recent_alloc_max_get_no_lock();
33}
34
35static inline ssize_t
36prof_recent_alloc_max_update(tsd_t *tsd, ssize_t max) {
37 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
38 ssize_t old_max = prof_recent_alloc_max_get(tsd);
39 atomic_store_zd(&prof_recent_alloc_max, max, ATOMIC_RELAXED);
40 return old_max;
41}
42
43static prof_recent_t *
44prof_recent_allocate_node(tsdn_t *tsdn) {
45 return (prof_recent_t *)iallocztm(tsdn, sizeof(prof_recent_t),
46 sz_size2index(sizeof(prof_recent_t)), false, NULL, true,
47 arena_get(tsdn, 0, false), true);
48}
49
50static void
51prof_recent_free_node(tsdn_t *tsdn, prof_recent_t *node) {
52 assert(node != NULL);
53 assert(isalloc(tsdn, node) == sz_s2u(sizeof(prof_recent_t)));
54 idalloctm(tsdn, node, NULL, NULL, true, true);
55}
56
57static inline void
58increment_recent_count(tsd_t *tsd, prof_tctx_t *tctx) {
59 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
60 ++tctx->recent_count;
61 assert(tctx->recent_count > 0);
62}
63
64bool
65prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx) {
66 cassert(config_prof);
67 assert(opt_prof && prof_booted);
68 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
69 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
70
71 /*
72 * Check whether last-N mode is turned on without trying to acquire the
73 * lock, so as to optimize for the following two scenarios:
74 * (1) Last-N mode is switched off;
75 * (2) Dumping, during which last-N mode is temporarily turned off so
76 * as not to block sampled allocations.
77 */
78 if (prof_recent_alloc_max_get_no_lock() == 0) {
79 return false;
80 }
81
82 /*
83 * Increment recent_count to hold the tctx so that it won't be gone
84 * even after tctx->tdata->lock is released. This acts as a
85 * "placeholder"; the real recording of the allocation requires a lock
86 * on prof_recent_alloc_mtx and is done in prof_recent_alloc (when
87 * tctx->tdata->lock has been released).
88 */
89 increment_recent_count(tsd, tctx);
90 return true;
91}
92
93static void
94decrement_recent_count(tsd_t *tsd, prof_tctx_t *tctx) {
95 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
96 assert(tctx != NULL);
97 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
98 assert(tctx->recent_count > 0);
99 --tctx->recent_count;
100 prof_tctx_try_destroy(tsd, tctx);
101}
102
103static inline edata_t *
104prof_recent_alloc_edata_get_no_lock(const prof_recent_t *n) {
105 return (edata_t *)atomic_load_p(&n->alloc_edata, ATOMIC_ACQUIRE);
106}
107
108edata_t *
109prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *n) {
110 cassert(config_prof);
111 return prof_recent_alloc_edata_get_no_lock(n);
112}
113
114static inline edata_t *
115prof_recent_alloc_edata_get(tsd_t *tsd, const prof_recent_t *n) {
116 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
117 return prof_recent_alloc_edata_get_no_lock(n);
118}
119
120static void
121prof_recent_alloc_edata_set(tsd_t *tsd, prof_recent_t *n, edata_t *edata) {
122 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
123 atomic_store_p(&n->alloc_edata, edata, ATOMIC_RELEASE);
124}
125
126void
127edata_prof_recent_alloc_init(edata_t *edata) {
128 cassert(config_prof);
129 edata_prof_recent_alloc_set_dont_call_directly(edata, NULL);
130}
131
132static inline prof_recent_t *
133edata_prof_recent_alloc_get_no_lock(const edata_t *edata) {
134 cassert(config_prof);
135 return edata_prof_recent_alloc_get_dont_call_directly(edata);
136}
137
138prof_recent_t *
139edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) {
140 cassert(config_prof);
141 return edata_prof_recent_alloc_get_no_lock(edata);
142}
143
144static inline prof_recent_t *
145edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) {
146 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
147 prof_recent_t *recent_alloc =
148 edata_prof_recent_alloc_get_no_lock(edata);
149 assert(recent_alloc == NULL ||
150 prof_recent_alloc_edata_get(tsd, recent_alloc) == edata);
151 return recent_alloc;
152}
153
154static prof_recent_t *
155edata_prof_recent_alloc_update_internal(tsd_t *tsd, edata_t *edata,
156 prof_recent_t *recent_alloc) {
157 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
158 prof_recent_t *old_recent_alloc =
159 edata_prof_recent_alloc_get(tsd, edata);
160 edata_prof_recent_alloc_set_dont_call_directly(edata, recent_alloc);
161 return old_recent_alloc;
162}
163
164static void
165edata_prof_recent_alloc_set(tsd_t *tsd, edata_t *edata,
166 prof_recent_t *recent_alloc) {
167 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
168 assert(recent_alloc != NULL);
169 prof_recent_t *old_recent_alloc =
170 edata_prof_recent_alloc_update_internal(tsd, edata, recent_alloc);
171 assert(old_recent_alloc == NULL);
172 prof_recent_alloc_edata_set(tsd, recent_alloc, edata);
173}
174
175static void
176edata_prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata,
177 prof_recent_t *recent_alloc) {
178 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
179 assert(recent_alloc != NULL);
180 prof_recent_t *old_recent_alloc =
181 edata_prof_recent_alloc_update_internal(tsd, edata, NULL);
182 assert(old_recent_alloc == recent_alloc);
183 assert(edata == prof_recent_alloc_edata_get(tsd, recent_alloc));
184 prof_recent_alloc_edata_set(tsd, recent_alloc, NULL);
185}
186
187/*
188 * This function should be called right before an allocation is released, so
189 * that the associated recent allocation record can contain the following
190 * information:
191 * (1) The allocation is released;
192 * (2) The time of the deallocation; and
193 * (3) The prof_tctx associated with the deallocation.
194 */
195void
196prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata) {
197 cassert(config_prof);
198 /*
199 * Check whether the recent allocation record still exists without
200 * trying to acquire the lock.
201 */
202 if (edata_prof_recent_alloc_get_no_lock(edata) == NULL) {
203 return;
204 }
205
206 prof_tctx_t *dalloc_tctx = prof_tctx_create(tsd);
207 /*
208 * In case dalloc_tctx is NULL, e.g. due to OOM, we will not record the
209 * deallocation time / tctx, which is handled later, after we check
210 * again when holding the lock.
211 */
212
213 if (dalloc_tctx != NULL) {
214 malloc_mutex_lock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock);
215 increment_recent_count(tsd, dalloc_tctx);
216 dalloc_tctx->prepared = false;
217 malloc_mutex_unlock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock);
218 }
219
220 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
221 /* Check again after acquiring the lock. */
222 prof_recent_t *recent = edata_prof_recent_alloc_get(tsd, edata);
223 if (recent != NULL) {
224 assert(nstime_equals_zero(&recent->dalloc_time));
225 assert(recent->dalloc_tctx == NULL);
226 if (dalloc_tctx != NULL) {
227 nstime_prof_update(&recent->dalloc_time);
228 recent->dalloc_tctx = dalloc_tctx;
229 dalloc_tctx = NULL;
230 }
231 edata_prof_recent_alloc_reset(tsd, edata, recent);
232 }
233 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
234
235 if (dalloc_tctx != NULL) {
236 /* We lost the rase - the allocation record was just gone. */
237 decrement_recent_count(tsd, dalloc_tctx);
238 }
239}
240
241static void
242prof_recent_alloc_evict_edata(tsd_t *tsd, prof_recent_t *recent_alloc) {
243 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
244 edata_t *edata = prof_recent_alloc_edata_get(tsd, recent_alloc);
245 if (edata != NULL) {
246 edata_prof_recent_alloc_reset(tsd, edata, recent_alloc);
247 }
248}
249
250static bool
251prof_recent_alloc_is_empty(tsd_t *tsd) {
252 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
253 if (ql_empty(&prof_recent_alloc_list)) {
254 assert(prof_recent_alloc_count == 0);
255 return true;
256 } else {
257 assert(prof_recent_alloc_count > 0);
258 return false;
259 }
260}
261
262static void
263prof_recent_alloc_assert_count(tsd_t *tsd) {
264 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
265 if (!config_debug) {
266 return;
267 }
268 ssize_t count = 0;
269 prof_recent_t *n;
270 ql_foreach(n, &prof_recent_alloc_list, link) {
271 ++count;
272 }
273 assert(count == prof_recent_alloc_count);
274 assert(prof_recent_alloc_max_get(tsd) == -1 ||
275 count <= prof_recent_alloc_max_get(tsd));
276}
277
278void
279prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize) {
280 cassert(config_prof);
281 assert(edata != NULL);
282 prof_tctx_t *tctx = edata_prof_tctx_get(edata);
283
284 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
285 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
286 prof_recent_alloc_assert_count(tsd);
287
288 /*
289 * Reserve a new prof_recent_t node if needed. If needed, we release
290 * the prof_recent_alloc_mtx lock and allocate. Then, rather than
291 * immediately checking for OOM, we regain the lock and try to make use
292 * of the reserve node if needed. There are six scenarios:
293 *
294 * \ now | no need | need but OOMed | need and allocated
295 * later \ | | |
296 * ------------------------------------------------------------
297 * no need | (1) | (2) | (3)
298 * ------------------------------------------------------------
299 * need | (4) | (5) | (6)
300 *
301 * First, "(4)" never happens, because we don't release the lock in the
302 * middle if there's no need for a new node; in such cases "(1)" always
303 * takes place, which is trivial.
304 *
305 * Out of the remaining four scenarios, "(6)" is the common case and is
306 * trivial. "(5)" is also trivial, in which case we'll rollback the
307 * effect of prof_recent_alloc_prepare() as expected.
308 *
309 * "(2)" / "(3)" occurs when the need for a new node is gone after we
310 * regain the lock. If the new node is successfully allocated, i.e. in
311 * the case of "(3)", we'll release it in the end; otherwise, i.e. in
312 * the case of "(2)", we do nothing - we're lucky that the OOM ends up
313 * doing no harm at all.
314 *
315 * Therefore, the only performance cost of the "release lock" ->
316 * "allocate" -> "regain lock" design is the "(3)" case, but it happens
317 * very rarely, so the cost is relatively small compared to the gain of
318 * not having to have the lock order of prof_recent_alloc_mtx above all
319 * the allocation locks.
320 */
321 prof_recent_t *reserve = NULL;
322 if (prof_recent_alloc_max_get(tsd) == -1 ||
323 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)) {
324 assert(prof_recent_alloc_max_get(tsd) != 0);
325 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
326 reserve = prof_recent_allocate_node(tsd_tsdn(tsd));
327 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
328 prof_recent_alloc_assert_count(tsd);
329 }
330
331 if (prof_recent_alloc_max_get(tsd) == 0) {
332 assert(prof_recent_alloc_is_empty(tsd));
333 goto label_rollback;
334 }
335
336 prof_tctx_t *old_alloc_tctx, *old_dalloc_tctx;
337 if (prof_recent_alloc_count == prof_recent_alloc_max_get(tsd)) {
338 /* If upper limit is reached, rotate the head. */
339 assert(prof_recent_alloc_max_get(tsd) != -1);
340 assert(!prof_recent_alloc_is_empty(tsd));
341 prof_recent_t *head = ql_first(&prof_recent_alloc_list);
342 old_alloc_tctx = head->alloc_tctx;
343 assert(old_alloc_tctx != NULL);
344 old_dalloc_tctx = head->dalloc_tctx;
345 prof_recent_alloc_evict_edata(tsd, head);
346 ql_rotate(&prof_recent_alloc_list, link);
347 } else {
348 /* Otherwise make use of the new node. */
349 assert(prof_recent_alloc_max_get(tsd) == -1 ||
350 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd));
351 if (reserve == NULL) {
352 goto label_rollback;
353 }
354 ql_elm_new(reserve, link);
355 ql_tail_insert(&prof_recent_alloc_list, reserve, link);
356 reserve = NULL;
357 old_alloc_tctx = NULL;
358 old_dalloc_tctx = NULL;
359 ++prof_recent_alloc_count;
360 }
361
362 /* Fill content into the tail node. */
363 prof_recent_t *tail = ql_last(&prof_recent_alloc_list, link);
364 assert(tail != NULL);
365 tail->size = size;
366 tail->usize = usize;
367 nstime_copy(&tail->alloc_time, edata_prof_alloc_time_get(edata));
368 tail->alloc_tctx = tctx;
369 nstime_init_zero(&tail->dalloc_time);
370 tail->dalloc_tctx = NULL;
371 edata_prof_recent_alloc_set(tsd, edata, tail);
372
373 assert(!prof_recent_alloc_is_empty(tsd));
374 prof_recent_alloc_assert_count(tsd);
375 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
376
377 if (reserve != NULL) {
378 prof_recent_free_node(tsd_tsdn(tsd), reserve);
379 }
380
381 /*
382 * Asynchronously handle the tctx of the old node, so that there's no
383 * simultaneous holdings of prof_recent_alloc_mtx and tdata->lock.
384 * In the worst case this may delay the tctx release but it's better
385 * than holding prof_recent_alloc_mtx for longer.
386 */
387 if (old_alloc_tctx != NULL) {
388 decrement_recent_count(tsd, old_alloc_tctx);
389 }
390 if (old_dalloc_tctx != NULL) {
391 decrement_recent_count(tsd, old_dalloc_tctx);
392 }
393 return;
394
395label_rollback:
396 assert(edata_prof_recent_alloc_get(tsd, edata) == NULL);
397 prof_recent_alloc_assert_count(tsd);
398 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
399 if (reserve != NULL) {
400 prof_recent_free_node(tsd_tsdn(tsd), reserve);
401 }
402 decrement_recent_count(tsd, tctx);
403}
404
405ssize_t
406prof_recent_alloc_max_ctl_read() {
407 cassert(config_prof);
408 /* Don't bother to acquire the lock. */
409 return prof_recent_alloc_max_get_no_lock();
410}
411
412static void
413prof_recent_alloc_restore_locked(tsd_t *tsd, prof_recent_list_t *to_delete) {
414 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
415 ssize_t max = prof_recent_alloc_max_get(tsd);
416 if (max == -1 || prof_recent_alloc_count <= max) {
417 /* Easy case - no need to alter the list. */
418 ql_new(to_delete);
419 prof_recent_alloc_assert_count(tsd);
420 return;
421 }
422
423 prof_recent_t *node;
424 ql_foreach(node, &prof_recent_alloc_list, link) {
425 if (prof_recent_alloc_count == max) {
426 break;
427 }
428 prof_recent_alloc_evict_edata(tsd, node);
429 --prof_recent_alloc_count;
430 }
431 assert(prof_recent_alloc_count == max);
432
433 ql_move(to_delete, &prof_recent_alloc_list);
434 if (max == 0) {
435 assert(node == NULL);
436 } else {
437 assert(node != NULL);
438 ql_split(to_delete, node, &prof_recent_alloc_list, link);
439 }
440 assert(!ql_empty(to_delete));
441 prof_recent_alloc_assert_count(tsd);
442}
443
444static void
445prof_recent_alloc_async_cleanup(tsd_t *tsd, prof_recent_list_t *to_delete) {
446 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_dump_mtx);
447 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
448 while (!ql_empty(to_delete)) {
449 prof_recent_t *node = ql_first(to_delete);
450 ql_remove(to_delete, node, link);
451 decrement_recent_count(tsd, node->alloc_tctx);
452 if (node->dalloc_tctx != NULL) {
453 decrement_recent_count(tsd, node->dalloc_tctx);
454 }
455 prof_recent_free_node(tsd_tsdn(tsd), node);
456 }
457}
458
459ssize_t
460prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max) {
461 cassert(config_prof);
462 assert(max >= -1);
463 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
464 prof_recent_alloc_assert_count(tsd);
465 const ssize_t old_max = prof_recent_alloc_max_update(tsd, max);
466 prof_recent_list_t to_delete;
467 prof_recent_alloc_restore_locked(tsd, &to_delete);
468 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
469 prof_recent_alloc_async_cleanup(tsd, &to_delete);
470 return old_max;
471}
472
473static void
474prof_recent_alloc_dump_bt(emitter_t *emitter, prof_tctx_t *tctx) {
475 char bt_buf[2 * sizeof(intptr_t) + 3];
476 char *s = bt_buf;
477 assert(tctx != NULL);
478 prof_bt_t *bt = &tctx->gctx->bt;
479 for (size_t i = 0; i < bt->len; ++i) {
480 malloc_snprintf(bt_buf, sizeof(bt_buf), "%p", bt->vec[i]);
481 emitter_json_value(emitter, emitter_type_string, &s);
482 }
483}
484
485static void
486prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) {
487 emitter_json_object_begin(emitter);
488
489 emitter_json_kv(emitter, "size", emitter_type_size, &node->size);
490 emitter_json_kv(emitter, "usize", emitter_type_size, &node->usize);
491 bool released = prof_recent_alloc_edata_get_no_lock(node) == NULL;
492 emitter_json_kv(emitter, "released", emitter_type_bool, &released);
493
494 emitter_json_kv(emitter, "alloc_thread_uid", emitter_type_uint64,
495 &node->alloc_tctx->thr_uid);
496 prof_tdata_t *alloc_tdata = node->alloc_tctx->tdata;
497 assert(alloc_tdata != NULL);
498 if (alloc_tdata->thread_name != NULL) {
499 emitter_json_kv(emitter, "alloc_thread_name",
500 emitter_type_string, &alloc_tdata->thread_name);
501 }
502 uint64_t alloc_time_ns = nstime_ns(&node->alloc_time);
503 emitter_json_kv(emitter, "alloc_time", emitter_type_uint64,
504 &alloc_time_ns);
505 emitter_json_array_kv_begin(emitter, "alloc_trace");
506 prof_recent_alloc_dump_bt(emitter, node->alloc_tctx);
507 emitter_json_array_end(emitter);
508
509 if (released && node->dalloc_tctx != NULL) {
510 emitter_json_kv(emitter, "dalloc_thread_uid",
511 emitter_type_uint64, &node->dalloc_tctx->thr_uid);
512 prof_tdata_t *dalloc_tdata = node->dalloc_tctx->tdata;
513 assert(dalloc_tdata != NULL);
514 if (dalloc_tdata->thread_name != NULL) {
515 emitter_json_kv(emitter, "dalloc_thread_name",
516 emitter_type_string, &dalloc_tdata->thread_name);
517 }
518 assert(!nstime_equals_zero(&node->dalloc_time));
519 uint64_t dalloc_time_ns = nstime_ns(&node->dalloc_time);
520 emitter_json_kv(emitter, "dalloc_time", emitter_type_uint64,
521 &dalloc_time_ns);
522 emitter_json_array_kv_begin(emitter, "dalloc_trace");
523 prof_recent_alloc_dump_bt(emitter, node->dalloc_tctx);
524 emitter_json_array_end(emitter);
525 }
526
527 emitter_json_object_end(emitter);
528}
529
530#define PROF_RECENT_PRINT_BUFSIZE 65536
531JEMALLOC_COLD
532void
533prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) {
534 cassert(config_prof);
535 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_dump_mtx);
536 buf_writer_t buf_writer;
537 buf_writer_init(tsd_tsdn(tsd), &buf_writer, write_cb, cbopaque, NULL,
538 PROF_RECENT_PRINT_BUFSIZE);
539 emitter_t emitter;
540 emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb,
541 &buf_writer);
542 prof_recent_list_t temp_list;
543
544 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
545 prof_recent_alloc_assert_count(tsd);
546 ssize_t dump_max = prof_recent_alloc_max_get(tsd);
547 ql_move(&temp_list, &prof_recent_alloc_list);
548 ssize_t dump_count = prof_recent_alloc_count;
549 prof_recent_alloc_count = 0;
550 prof_recent_alloc_assert_count(tsd);
551 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
552
553 emitter_begin(&emitter);
554 uint64_t sample_interval = (uint64_t)1U << lg_prof_sample;
555 emitter_json_kv(&emitter, "sample_interval", emitter_type_uint64,
556 &sample_interval);
557 emitter_json_kv(&emitter, "recent_alloc_max", emitter_type_ssize,
558 &dump_max);
559 emitter_json_array_kv_begin(&emitter, "recent_alloc");
560 prof_recent_t *node;
561 ql_foreach(node, &temp_list, link) {
562 prof_recent_alloc_dump_node(&emitter, node);
563 }
564 emitter_json_array_end(&emitter);
565 emitter_end(&emitter);
566
567 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
568 prof_recent_alloc_assert_count(tsd);
569 ql_concat(&temp_list, &prof_recent_alloc_list, link);
570 ql_move(&prof_recent_alloc_list, &temp_list);
571 prof_recent_alloc_count += dump_count;
572 prof_recent_alloc_restore_locked(tsd, &temp_list);
573 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx);
574
575 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer);
576 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_dump_mtx);
577
578 prof_recent_alloc_async_cleanup(tsd, &temp_list);
579}
580#undef PROF_RECENT_PRINT_BUFSIZE
581
582bool
583prof_recent_init() {
584 cassert(config_prof);
585 prof_recent_alloc_max_init();
586
587 if (malloc_mutex_init(&prof_recent_alloc_mtx, "prof_recent_alloc",
588 WITNESS_RANK_PROF_RECENT_ALLOC, malloc_mutex_rank_exclusive)) {
589 return true;
590 }
591
592 if (malloc_mutex_init(&prof_recent_dump_mtx, "prof_recent_dump",
593 WITNESS_RANK_PROF_RECENT_DUMP, malloc_mutex_rank_exclusive)) {
594 return true;
595 }
596
597 ql_new(&prof_recent_alloc_list);
598
599 return false;
600}
601