1#define JEMALLOC_PROF_SYS_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/buf_writer.h"
6#include "jemalloc/internal/ctl.h"
7#include "jemalloc/internal/prof_data.h"
8#include "jemalloc/internal/prof_sys.h"
9
10#ifdef JEMALLOC_PROF_LIBUNWIND
11#define UNW_LOCAL_ONLY
12#include <libunwind.h>
13#endif
14
15#ifdef JEMALLOC_PROF_LIBGCC
16/*
17 * We have a circular dependency -- jemalloc_internal.h tells us if we should
18 * use libgcc's unwinding functionality, but after we've included that, we've
19 * already hooked _Unwind_Backtrace. We'll temporarily disable hooking.
20 */
21#undef _Unwind_Backtrace
22#include <unwind.h>
23#define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
24#endif
25
26/******************************************************************************/
27
28malloc_mutex_t prof_dump_filename_mtx;
29
30bool prof_do_mock = false;
31
32static uint64_t prof_dump_seq;
33static uint64_t prof_dump_iseq;
34static uint64_t prof_dump_mseq;
35static uint64_t prof_dump_useq;
36
37static char *prof_prefix = NULL;
38
39/* The fallback allocator profiling functionality will use. */
40base_t *prof_base;
41
42void
43bt_init(prof_bt_t *bt, void **vec) {
44 cassert(config_prof);
45
46 bt->vec = vec;
47 bt->len = 0;
48}
49
50#ifdef JEMALLOC_PROF_LIBUNWIND
51static void
52prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
53 int nframes;
54
55 cassert(config_prof);
56 assert(*len == 0);
57 assert(vec != NULL);
58 assert(max_len == PROF_BT_MAX);
59
60 nframes = unw_backtrace(vec, PROF_BT_MAX);
61 if (nframes <= 0) {
62 return;
63 }
64 *len = nframes;
65}
66#elif (defined(JEMALLOC_PROF_LIBGCC))
67static _Unwind_Reason_Code
68prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) {
69 cassert(config_prof);
70
71 return _URC_NO_REASON;
72}
73
74static _Unwind_Reason_Code
75prof_unwind_callback(struct _Unwind_Context *context, void *arg) {
76 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
77 void *ip;
78
79 cassert(config_prof);
80
81 ip = (void *)_Unwind_GetIP(context);
82 if (ip == NULL) {
83 return _URC_END_OF_STACK;
84 }
85 data->vec[*data->len] = ip;
86 (*data->len)++;
87 if (*data->len == data->max) {
88 return _URC_END_OF_STACK;
89 }
90
91 return _URC_NO_REASON;
92}
93
94static void
95prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
96 prof_unwind_data_t data = {vec, len, max_len};
97
98 cassert(config_prof);
99 assert(vec != NULL);
100 assert(max_len == PROF_BT_MAX);
101
102 _Unwind_Backtrace(prof_unwind_callback, &data);
103}
104#elif (defined(JEMALLOC_PROF_GCC))
105static void
106prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
107#define BT_FRAME(i) \
108 if ((i) < max_len) { \
109 void *p; \
110 if (__builtin_frame_address(i) == 0) { \
111 return; \
112 } \
113 p = __builtin_return_address(i); \
114 if (p == NULL) { \
115 return; \
116 } \
117 vec[(i)] = p; \
118 *len = (i) + 1; \
119 } else { \
120 return; \
121 }
122
123 cassert(config_prof);
124 assert(vec != NULL);
125 assert(max_len == PROF_BT_MAX);
126
127 BT_FRAME(0)
128 BT_FRAME(1)
129 BT_FRAME(2)
130 BT_FRAME(3)
131 BT_FRAME(4)
132 BT_FRAME(5)
133 BT_FRAME(6)
134 BT_FRAME(7)
135 BT_FRAME(8)
136 BT_FRAME(9)
137
138 BT_FRAME(10)
139 BT_FRAME(11)
140 BT_FRAME(12)
141 BT_FRAME(13)
142 BT_FRAME(14)
143 BT_FRAME(15)
144 BT_FRAME(16)
145 BT_FRAME(17)
146 BT_FRAME(18)
147 BT_FRAME(19)
148
149 BT_FRAME(20)
150 BT_FRAME(21)
151 BT_FRAME(22)
152 BT_FRAME(23)
153 BT_FRAME(24)
154 BT_FRAME(25)
155 BT_FRAME(26)
156 BT_FRAME(27)
157 BT_FRAME(28)
158 BT_FRAME(29)
159
160 BT_FRAME(30)
161 BT_FRAME(31)
162 BT_FRAME(32)
163 BT_FRAME(33)
164 BT_FRAME(34)
165 BT_FRAME(35)
166 BT_FRAME(36)
167 BT_FRAME(37)
168 BT_FRAME(38)
169 BT_FRAME(39)
170
171 BT_FRAME(40)
172 BT_FRAME(41)
173 BT_FRAME(42)
174 BT_FRAME(43)
175 BT_FRAME(44)
176 BT_FRAME(45)
177 BT_FRAME(46)
178 BT_FRAME(47)
179 BT_FRAME(48)
180 BT_FRAME(49)
181
182 BT_FRAME(50)
183 BT_FRAME(51)
184 BT_FRAME(52)
185 BT_FRAME(53)
186 BT_FRAME(54)
187 BT_FRAME(55)
188 BT_FRAME(56)
189 BT_FRAME(57)
190 BT_FRAME(58)
191 BT_FRAME(59)
192
193 BT_FRAME(60)
194 BT_FRAME(61)
195 BT_FRAME(62)
196 BT_FRAME(63)
197 BT_FRAME(64)
198 BT_FRAME(65)
199 BT_FRAME(66)
200 BT_FRAME(67)
201 BT_FRAME(68)
202 BT_FRAME(69)
203
204 BT_FRAME(70)
205 BT_FRAME(71)
206 BT_FRAME(72)
207 BT_FRAME(73)
208 BT_FRAME(74)
209 BT_FRAME(75)
210 BT_FRAME(76)
211 BT_FRAME(77)
212 BT_FRAME(78)
213 BT_FRAME(79)
214
215 BT_FRAME(80)
216 BT_FRAME(81)
217 BT_FRAME(82)
218 BT_FRAME(83)
219 BT_FRAME(84)
220 BT_FRAME(85)
221 BT_FRAME(86)
222 BT_FRAME(87)
223 BT_FRAME(88)
224 BT_FRAME(89)
225
226 BT_FRAME(90)
227 BT_FRAME(91)
228 BT_FRAME(92)
229 BT_FRAME(93)
230 BT_FRAME(94)
231 BT_FRAME(95)
232 BT_FRAME(96)
233 BT_FRAME(97)
234 BT_FRAME(98)
235 BT_FRAME(99)
236
237 BT_FRAME(100)
238 BT_FRAME(101)
239 BT_FRAME(102)
240 BT_FRAME(103)
241 BT_FRAME(104)
242 BT_FRAME(105)
243 BT_FRAME(106)
244 BT_FRAME(107)
245 BT_FRAME(108)
246 BT_FRAME(109)
247
248 BT_FRAME(110)
249 BT_FRAME(111)
250 BT_FRAME(112)
251 BT_FRAME(113)
252 BT_FRAME(114)
253 BT_FRAME(115)
254 BT_FRAME(116)
255 BT_FRAME(117)
256 BT_FRAME(118)
257 BT_FRAME(119)
258
259 BT_FRAME(120)
260 BT_FRAME(121)
261 BT_FRAME(122)
262 BT_FRAME(123)
263 BT_FRAME(124)
264 BT_FRAME(125)
265 BT_FRAME(126)
266 BT_FRAME(127)
267#undef BT_FRAME
268}
269#else
270static void
271prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
272 cassert(config_prof);
273 not_reached();
274}
275#endif
276
277void
278prof_backtrace(tsd_t *tsd, prof_bt_t *bt) {
279 cassert(config_prof);
280 prof_backtrace_hook_t prof_backtrace_hook = prof_backtrace_hook_get();
281 assert(prof_backtrace_hook != NULL);
282
283 pre_reentrancy(tsd, NULL);
284 prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX);
285 post_reentrancy(tsd);
286}
287
288void
289prof_hooks_init() {
290 prof_backtrace_hook_set(&prof_backtrace_impl);
291 prof_dump_hook_set(NULL);
292}
293
294void
295prof_unwind_init() {
296#ifdef JEMALLOC_PROF_LIBGCC
297 /*
298 * Cause the backtracing machinery to allocate its internal
299 * state before enabling profiling.
300 */
301 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
302#endif
303}
304
305static int
306prof_sys_thread_name_read_impl(char *buf, size_t limit) {
307#if defined(JEMALLOC_HAVE_PTHREAD_GETNAME_NP)
308 return pthread_getname_np(pthread_self(), buf, limit);
309#elif defined(JEMALLOC_HAVE_PTHREAD_GET_NAME_NP)
310 pthread_get_name_np(pthread_self(), buf, limit);
311 return 0;
312#else
313 return ENOSYS;
314#endif
315}
316prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read =
317 prof_sys_thread_name_read_impl;
318
319void
320prof_sys_thread_name_fetch(tsd_t *tsd) {
321#define THREAD_NAME_MAX_LEN 16
322 char buf[THREAD_NAME_MAX_LEN];
323 if (!prof_sys_thread_name_read(buf, THREAD_NAME_MAX_LEN)) {
324 prof_thread_name_set_impl(tsd, buf);
325 }
326#undef THREAD_NAME_MAX_LEN
327}
328
329int
330prof_getpid(void) {
331#ifdef _WIN32
332 return GetCurrentProcessId();
333#else
334 return getpid();
335#endif
336}
337
338/*
339 * This buffer is rather large for stack allocation, so use a single buffer for
340 * all profile dumps; protected by prof_dump_mtx.
341 */
342static char prof_dump_buf[PROF_DUMP_BUFSIZE];
343
344typedef struct prof_dump_arg_s prof_dump_arg_t;
345struct prof_dump_arg_s {
346 /*
347 * Whether error should be handled locally: if true, then we print out
348 * error message as well as abort (if opt_abort is true) when an error
349 * occurred, and we also report the error back to the caller in the end;
350 * if false, then we only report the error back to the caller in the
351 * end.
352 */
353 const bool handle_error_locally;
354 /*
355 * Whether there has been an error in the dumping process, which could
356 * have happened either in file opening or in file writing. When an
357 * error has already occurred, we will stop further writing to the file.
358 */
359 bool error;
360 /* File descriptor of the dump file. */
361 int prof_dump_fd;
362};
363
364static void
365prof_dump_check_possible_error(prof_dump_arg_t *arg, bool err_cond,
366 const char *format, ...) {
367 assert(!arg->error);
368 if (!err_cond) {
369 return;
370 }
371
372 arg->error = true;
373 if (!arg->handle_error_locally) {
374 return;
375 }
376
377 va_list ap;
378 char buf[PROF_PRINTF_BUFSIZE];
379 va_start(ap, format);
380 malloc_vsnprintf(buf, sizeof(buf), format, ap);
381 va_end(ap);
382 malloc_write(buf);
383
384 if (opt_abort) {
385 abort();
386 }
387}
388
389static int
390prof_dump_open_file_impl(const char *filename, int mode) {
391 return creat(filename, mode);
392}
393prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file =
394 prof_dump_open_file_impl;
395
396static void
397prof_dump_open(prof_dump_arg_t *arg, const char *filename) {
398 arg->prof_dump_fd = prof_dump_open_file(filename, 0644);
399 prof_dump_check_possible_error(arg, arg->prof_dump_fd == -1,
400 "<jemalloc>: failed to open \"%s\"\n", filename);
401}
402
403prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file = malloc_write_fd;
404
405static void
406prof_dump_flush(void *opaque, const char *s) {
407 cassert(config_prof);
408 prof_dump_arg_t *arg = (prof_dump_arg_t *)opaque;
409 if (!arg->error) {
410 ssize_t err = prof_dump_write_file(arg->prof_dump_fd, s,
411 strlen(s));
412 prof_dump_check_possible_error(arg, err == -1,
413 "<jemalloc>: failed to write during heap profile flush\n");
414 }
415}
416
417static void
418prof_dump_close(prof_dump_arg_t *arg) {
419 if (arg->prof_dump_fd != -1) {
420 close(arg->prof_dump_fd);
421 }
422}
423
424#ifndef _WIN32
425JEMALLOC_FORMAT_PRINTF(1, 2)
426static int
427prof_open_maps_internal(const char *format, ...) {
428 int mfd;
429 va_list ap;
430 char filename[PATH_MAX + 1];
431
432 va_start(ap, format);
433 malloc_vsnprintf(filename, sizeof(filename), format, ap);
434 va_end(ap);
435
436#if defined(O_CLOEXEC)
437 mfd = open(filename, O_RDONLY | O_CLOEXEC);
438#else
439 mfd = open(filename, O_RDONLY);
440 if (mfd != -1) {
441 fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
442 }
443#endif
444
445 return mfd;
446}
447#endif
448
449static int
450prof_dump_open_maps_impl() {
451 int mfd;
452
453 cassert(config_prof);
454#if defined(__FreeBSD__) || defined(__DragonFly__)
455 mfd = prof_open_maps_internal("/proc/curproc/map");
456#elif defined(_WIN32)
457 mfd = -1; // Not implemented
458#else
459 int pid = prof_getpid();
460
461 mfd = prof_open_maps_internal("/proc/%d/task/%d/maps", pid, pid);
462 if (mfd == -1) {
463 mfd = prof_open_maps_internal("/proc/%d/maps", pid);
464 }
465#endif
466 return mfd;
467}
468prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps =
469 prof_dump_open_maps_impl;
470
471static ssize_t
472prof_dump_read_maps_cb(void *read_cbopaque, void *buf, size_t limit) {
473 int mfd = *(int *)read_cbopaque;
474 assert(mfd != -1);
475 return malloc_read_fd(mfd, buf, limit);
476}
477
478static void
479prof_dump_maps(buf_writer_t *buf_writer) {
480 int mfd = prof_dump_open_maps();
481 if (mfd == -1) {
482 return;
483 }
484
485 buf_writer_cb(buf_writer, "\nMAPPED_LIBRARIES:\n");
486 buf_writer_pipe(buf_writer, prof_dump_read_maps_cb, &mfd);
487 close(mfd);
488}
489
490static bool
491prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
492 bool leakcheck) {
493 cassert(config_prof);
494 assert(tsd_reentrancy_level_get(tsd) == 0);
495
496 prof_tdata_t * tdata = prof_tdata_get(tsd, true);
497 if (tdata == NULL) {
498 return true;
499 }
500
501 prof_dump_arg_t arg = {/* handle_error_locally */ !propagate_err,
502 /* error */ false, /* prof_dump_fd */ -1};
503
504 pre_reentrancy(tsd, NULL);
505 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
506
507 prof_dump_open(&arg, filename);
508 buf_writer_t buf_writer;
509 bool err = buf_writer_init(tsd_tsdn(tsd), &buf_writer, prof_dump_flush,
510 &arg, prof_dump_buf, PROF_DUMP_BUFSIZE);
511 assert(!err);
512 prof_dump_impl(tsd, buf_writer_cb, &buf_writer, tdata, leakcheck);
513 prof_dump_maps(&buf_writer);
514 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer);
515 prof_dump_close(&arg);
516
517 prof_dump_hook_t dump_hook = prof_dump_hook_get();
518 if (dump_hook != NULL) {
519 dump_hook(filename);
520 }
521 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
522 post_reentrancy(tsd);
523
524 return arg.error;
525}
526
527/*
528 * If profiling is off, then PROF_DUMP_FILENAME_LEN is 1, so we'll end up
529 * calling strncpy with a size of 0, which triggers a -Wstringop-truncation
530 * warning (strncpy can never actually be called in this case, since we bail out
531 * much earlier when config_prof is false). This function works around the
532 * warning to let us leave the warning on.
533 */
534static inline void
535prof_strncpy(char *UNUSED dest, const char *UNUSED src, size_t UNUSED size) {
536 cassert(config_prof);
537#ifdef JEMALLOC_PROF
538 strncpy(dest, src, size);
539#endif
540}
541
542static const char *
543prof_prefix_get(tsdn_t* tsdn) {
544 malloc_mutex_assert_owner(tsdn, &prof_dump_filename_mtx);
545
546 return prof_prefix == NULL ? opt_prof_prefix : prof_prefix;
547}
548
549static bool
550prof_prefix_is_empty(tsdn_t *tsdn) {
551 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
552 bool ret = (prof_prefix_get(tsdn)[0] == '\0');
553 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
554 return ret;
555}
556
557#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
558#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
559static void
560prof_dump_filename(tsd_t *tsd, char *filename, char v, uint64_t vseq) {
561 cassert(config_prof);
562
563 assert(tsd_reentrancy_level_get(tsd) == 0);
564 const char *prefix = prof_prefix_get(tsd_tsdn(tsd));
565
566 if (vseq != VSEQ_INVALID) {
567 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
568 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
569 "%s.%d.%"FMTu64".%c%"FMTu64".heap", prefix, prof_getpid(),
570 prof_dump_seq, v, vseq);
571 } else {
572 /* "<prefix>.<pid>.<seq>.<v>.heap" */
573 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
574 "%s.%d.%"FMTu64".%c.heap", prefix, prof_getpid(),
575 prof_dump_seq, v);
576 }
577 prof_dump_seq++;
578}
579
580void
581prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind) {
582 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
583 malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN,
584 "%s.%d.%"FMTu64".json", prof_prefix_get(tsdn), prof_getpid(), ind);
585 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
586}
587
588void
589prof_fdump_impl(tsd_t *tsd) {
590 char filename[DUMP_FILENAME_BUFSIZE];
591
592 assert(!prof_prefix_is_empty(tsd_tsdn(tsd)));
593 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
594 prof_dump_filename(tsd, filename, 'f', VSEQ_INVALID);
595 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
596 prof_dump(tsd, false, filename, opt_prof_leak);
597}
598
599bool
600prof_prefix_set(tsdn_t *tsdn, const char *prefix) {
601 cassert(config_prof);
602 ctl_mtx_assert_held(tsdn);
603 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
604 if (prof_prefix == NULL) {
605 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
606 /* Everything is still guarded by ctl_mtx. */
607 char *buffer = base_alloc(tsdn, prof_base,
608 PROF_DUMP_FILENAME_LEN, QUANTUM);
609 if (buffer == NULL) {
610 return true;
611 }
612 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
613 prof_prefix = buffer;
614 }
615 assert(prof_prefix != NULL);
616
617 prof_strncpy(prof_prefix, prefix, PROF_DUMP_FILENAME_LEN - 1);
618 prof_prefix[PROF_DUMP_FILENAME_LEN - 1] = '\0';
619 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
620
621 return false;
622}
623
624void
625prof_idump_impl(tsd_t *tsd) {
626 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
627 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') {
628 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
629 return;
630 }
631 char filename[PATH_MAX + 1];
632 prof_dump_filename(tsd, filename, 'i', prof_dump_iseq);
633 prof_dump_iseq++;
634 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
635 prof_dump(tsd, false, filename, false);
636}
637
638bool
639prof_mdump_impl(tsd_t *tsd, const char *filename) {
640 char filename_buf[DUMP_FILENAME_BUFSIZE];
641 if (filename == NULL) {
642 /* No filename specified, so automatically generate one. */
643 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
644 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') {
645 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
646 return true;
647 }
648 prof_dump_filename(tsd, filename_buf, 'm', prof_dump_mseq);
649 prof_dump_mseq++;
650 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx);
651 filename = filename_buf;
652 }
653 return prof_dump(tsd, true, filename, false);
654}
655
656void
657prof_gdump_impl(tsd_t *tsd) {
658 tsdn_t *tsdn = tsd_tsdn(tsd);
659 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx);
660 if (prof_prefix_get(tsdn)[0] == '\0') {
661 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
662 return;
663 }
664 char filename[DUMP_FILENAME_BUFSIZE];
665 prof_dump_filename(tsd, filename, 'u', prof_dump_useq);
666 prof_dump_useq++;
667 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx);
668 prof_dump(tsd, false, filename, false);
669}
670