1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/malloc_io.h"
5#include "jemalloc/internal/util.h"
6
7#ifdef assert
8# undef assert
9#endif
10#ifdef not_reached
11# undef not_reached
12#endif
13#ifdef not_implemented
14# undef not_implemented
15#endif
16#ifdef assert_not_implemented
17# undef assert_not_implemented
18#endif
19
20/*
21 * Define simple versions of assertion macros that won't recurse in case
22 * of assertion failures in malloc_*printf().
23 */
24#define assert(e) do { \
25 if (config_debug && !(e)) { \
26 malloc_write("<jemalloc>: Failed assertion\n"); \
27 abort(); \
28 } \
29} while (0)
30
31#define not_reached() do { \
32 if (config_debug) { \
33 malloc_write("<jemalloc>: Unreachable code reached\n"); \
34 abort(); \
35 } \
36 unreachable(); \
37} while (0)
38
39#define not_implemented() do { \
40 if (config_debug) { \
41 malloc_write("<jemalloc>: Not implemented\n"); \
42 abort(); \
43 } \
44} while (0)
45
46#define assert_not_implemented(e) do { \
47 if (unlikely(config_debug && !(e))) { \
48 not_implemented(); \
49 } \
50} while (0)
51
52/******************************************************************************/
53/* Function prototypes for non-inline static functions. */
54
55#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
56static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
57 size_t *slen_p);
58#define D2S_BUFSIZE (1 + U2S_BUFSIZE)
59static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
60#define O2S_BUFSIZE (1 + U2S_BUFSIZE)
61static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
62#define X2S_BUFSIZE (2 + U2S_BUFSIZE)
63static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
64 size_t *slen_p);
65
66/******************************************************************************/
67
68/* malloc_message() setup. */
69void
70wrtmessage(void *cbopaque, const char *s) {
71 malloc_write_fd(STDERR_FILENO, s, strlen(s));
72}
73
74JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
75
76/*
77 * Wrapper around malloc_message() that avoids the need for
78 * je_malloc_message(...) throughout the code.
79 */
80void
81malloc_write(const char *s) {
82 if (je_malloc_message != NULL) {
83 je_malloc_message(NULL, s);
84 } else {
85 wrtmessage(NULL, s);
86 }
87}
88
89/*
90 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
91 * provide a wrapper.
92 */
93int
94buferror(int err, char *buf, size_t buflen) {
95#ifdef _WIN32
96 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
97 (LPSTR)buf, (DWORD)buflen, NULL);
98 return 0;
99#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
100 char *b = strerror_r(err, buf, buflen);
101 if (b != buf) {
102 strncpy(buf, b, buflen);
103 buf[buflen-1] = '\0';
104 }
105 return 0;
106#else
107 return strerror_r(err, buf, buflen);
108#endif
109}
110
111uintmax_t
112malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
113 uintmax_t ret, digit;
114 unsigned b;
115 bool neg;
116 const char *p, *ns;
117
118 p = nptr;
119 if (base < 0 || base == 1 || base > 36) {
120 ns = p;
121 set_errno(EINVAL);
122 ret = UINTMAX_MAX;
123 goto label_return;
124 }
125 b = base;
126
127 /* Swallow leading whitespace and get sign, if any. */
128 neg = false;
129 while (true) {
130 switch (*p) {
131 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
132 p++;
133 break;
134 case '-':
135 neg = true;
136 JEMALLOC_FALLTHROUGH;
137 case '+':
138 p++;
139 JEMALLOC_FALLTHROUGH;
140 default:
141 goto label_prefix;
142 }
143 }
144
145 /* Get prefix, if any. */
146 label_prefix:
147 /*
148 * Note where the first non-whitespace/sign character is so that it is
149 * possible to tell whether any digits are consumed (e.g., " 0" vs.
150 * " -x").
151 */
152 ns = p;
153 if (*p == '0') {
154 switch (p[1]) {
155 case '0': case '1': case '2': case '3': case '4': case '5':
156 case '6': case '7':
157 if (b == 0) {
158 b = 8;
159 }
160 if (b == 8) {
161 p++;
162 }
163 break;
164 case 'X': case 'x':
165 switch (p[2]) {
166 case '0': case '1': case '2': case '3': case '4':
167 case '5': case '6': case '7': case '8': case '9':
168 case 'A': case 'B': case 'C': case 'D': case 'E':
169 case 'F':
170 case 'a': case 'b': case 'c': case 'd': case 'e':
171 case 'f':
172 if (b == 0) {
173 b = 16;
174 }
175 if (b == 16) {
176 p += 2;
177 }
178 break;
179 default:
180 break;
181 }
182 break;
183 default:
184 p++;
185 ret = 0;
186 goto label_return;
187 }
188 }
189 if (b == 0) {
190 b = 10;
191 }
192
193 /* Convert. */
194 ret = 0;
195 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
196 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
197 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
198 uintmax_t pret = ret;
199 ret *= b;
200 ret += digit;
201 if (ret < pret) {
202 /* Overflow. */
203 set_errno(ERANGE);
204 ret = UINTMAX_MAX;
205 goto label_return;
206 }
207 p++;
208 }
209 if (neg) {
210 ret = (uintmax_t)(-((intmax_t)ret));
211 }
212
213 if (p == ns) {
214 /* No conversion performed. */
215 set_errno(EINVAL);
216 ret = UINTMAX_MAX;
217 goto label_return;
218 }
219
220label_return:
221 if (endptr != NULL) {
222 if (p == ns) {
223 /* No characters were converted. */
224 *endptr = (char *)nptr;
225 } else {
226 *endptr = (char *)p;
227 }
228 }
229 return ret;
230}
231
232static char *
233u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
234 unsigned i;
235
236 i = U2S_BUFSIZE - 1;
237 s[i] = '\0';
238 switch (base) {
239 case 10:
240 do {
241 i--;
242 s[i] = "0123456789"[x % (uint64_t)10];
243 x /= (uint64_t)10;
244 } while (x > 0);
245 break;
246 case 16: {
247 const char *digits = (uppercase)
248 ? "0123456789ABCDEF"
249 : "0123456789abcdef";
250
251 do {
252 i--;
253 s[i] = digits[x & 0xf];
254 x >>= 4;
255 } while (x > 0);
256 break;
257 } default: {
258 const char *digits = (uppercase)
259 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
260 : "0123456789abcdefghijklmnopqrstuvwxyz";
261
262 assert(base >= 2 && base <= 36);
263 do {
264 i--;
265 s[i] = digits[x % (uint64_t)base];
266 x /= (uint64_t)base;
267 } while (x > 0);
268 }}
269
270 *slen_p = U2S_BUFSIZE - 1 - i;
271 return &s[i];
272}
273
274static char *
275d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
276 bool neg;
277
278 if ((neg = (x < 0))) {
279 x = -x;
280 }
281 s = u2s(x, 10, false, s, slen_p);
282 if (neg) {
283 sign = '-';
284 }
285 switch (sign) {
286 case '-':
287 if (!neg) {
288 break;
289 }
290 JEMALLOC_FALLTHROUGH;
291 case ' ':
292 case '+':
293 s--;
294 (*slen_p)++;
295 *s = sign;
296 break;
297 default: not_reached();
298 }
299 return s;
300}
301
302static char *
303o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
304 s = u2s(x, 8, false, s, slen_p);
305 if (alt_form && *s != '0') {
306 s--;
307 (*slen_p)++;
308 *s = '0';
309 }
310 return s;
311}
312
313static char *
314x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
315 s = u2s(x, 16, uppercase, s, slen_p);
316 if (alt_form) {
317 s -= 2;
318 (*slen_p) += 2;
319 memcpy(s, uppercase ? "0X" : "0x", 2);
320 }
321 return s;
322}
323
324JEMALLOC_COLD
325size_t
326malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
327 size_t i;
328 const char *f;
329
330#define APPEND_C(c) do { \
331 if (i < size) { \
332 str[i] = (c); \
333 } \
334 i++; \
335} while (0)
336#define APPEND_S(s, slen) do { \
337 if (i < size) { \
338 size_t cpylen = (slen <= size - i) ? slen : size - i; \
339 memcpy(&str[i], s, cpylen); \
340 } \
341 i += slen; \
342} while (0)
343#define APPEND_PADDED_S(s, slen, width, left_justify) do { \
344 /* Left padding. */ \
345 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \
346 (size_t)width - slen : 0); \
347 if (!left_justify && pad_len != 0) { \
348 size_t j; \
349 for (j = 0; j < pad_len; j++) { \
350 if (pad_zero) { \
351 APPEND_C('0'); \
352 } else { \
353 APPEND_C(' '); \
354 } \
355 } \
356 } \
357 /* Value. */ \
358 APPEND_S(s, slen); \
359 /* Right padding. */ \
360 if (left_justify && pad_len != 0) { \
361 size_t j; \
362 for (j = 0; j < pad_len; j++) { \
363 APPEND_C(' '); \
364 } \
365 } \
366} while (0)
367#define GET_ARG_NUMERIC(val, len) do { \
368 switch ((unsigned char)len) { \
369 case '?': \
370 val = va_arg(ap, int); \
371 break; \
372 case '?' | 0x80: \
373 val = va_arg(ap, unsigned int); \
374 break; \
375 case 'l': \
376 val = va_arg(ap, long); \
377 break; \
378 case 'l' | 0x80: \
379 val = va_arg(ap, unsigned long); \
380 break; \
381 case 'q': \
382 val = va_arg(ap, long long); \
383 break; \
384 case 'q' | 0x80: \
385 val = va_arg(ap, unsigned long long); \
386 break; \
387 case 'j': \
388 val = va_arg(ap, intmax_t); \
389 break; \
390 case 'j' | 0x80: \
391 val = va_arg(ap, uintmax_t); \
392 break; \
393 case 't': \
394 val = va_arg(ap, ptrdiff_t); \
395 break; \
396 case 'z': \
397 val = va_arg(ap, ssize_t); \
398 break; \
399 case 'z' | 0x80: \
400 val = va_arg(ap, size_t); \
401 break; \
402 case 'p': /* Synthetic; used for %p. */ \
403 val = va_arg(ap, uintptr_t); \
404 break; \
405 default: \
406 not_reached(); \
407 val = 0; \
408 } \
409} while (0)
410
411 i = 0;
412 f = format;
413 while (true) {
414 switch (*f) {
415 case '\0': goto label_out;
416 case '%': {
417 bool alt_form = false;
418 bool left_justify = false;
419 bool plus_space = false;
420 bool plus_plus = false;
421 int prec = -1;
422 int width = -1;
423 unsigned char len = '?';
424 char *s;
425 size_t slen;
426 bool first_width_digit = true;
427 bool pad_zero = false;
428
429 f++;
430 /* Flags. */
431 while (true) {
432 switch (*f) {
433 case '#':
434 assert(!alt_form);
435 alt_form = true;
436 break;
437 case '-':
438 assert(!left_justify);
439 left_justify = true;
440 break;
441 case ' ':
442 assert(!plus_space);
443 plus_space = true;
444 break;
445 case '+':
446 assert(!plus_plus);
447 plus_plus = true;
448 break;
449 default: goto label_width;
450 }
451 f++;
452 }
453 /* Width. */
454 label_width:
455 switch (*f) {
456 case '*':
457 width = va_arg(ap, int);
458 f++;
459 if (width < 0) {
460 left_justify = true;
461 width = -width;
462 }
463 break;
464 case '0':
465 if (first_width_digit) {
466 pad_zero = true;
467 }
468 JEMALLOC_FALLTHROUGH;
469 case '1': case '2': case '3': case '4':
470 case '5': case '6': case '7': case '8': case '9': {
471 uintmax_t uwidth;
472 set_errno(0);
473 uwidth = malloc_strtoumax(f, (char **)&f, 10);
474 assert(uwidth != UINTMAX_MAX || get_errno() !=
475 ERANGE);
476 width = (int)uwidth;
477 first_width_digit = false;
478 break;
479 } default:
480 break;
481 }
482 /* Width/precision separator. */
483 if (*f == '.') {
484 f++;
485 } else {
486 goto label_length;
487 }
488 /* Precision. */
489 switch (*f) {
490 case '*':
491 prec = va_arg(ap, int);
492 f++;
493 break;
494 case '0': case '1': case '2': case '3': case '4':
495 case '5': case '6': case '7': case '8': case '9': {
496 uintmax_t uprec;
497 set_errno(0);
498 uprec = malloc_strtoumax(f, (char **)&f, 10);
499 assert(uprec != UINTMAX_MAX || get_errno() !=
500 ERANGE);
501 prec = (int)uprec;
502 break;
503 }
504 default: break;
505 }
506 /* Length. */
507 label_length:
508 switch (*f) {
509 case 'l':
510 f++;
511 if (*f == 'l') {
512 len = 'q';
513 f++;
514 } else {
515 len = 'l';
516 }
517 break;
518 case 'q': case 'j': case 't': case 'z':
519 len = *f;
520 f++;
521 break;
522 default: break;
523 }
524 /* Conversion specifier. */
525 switch (*f) {
526 case '%':
527 /* %% */
528 APPEND_C(*f);
529 f++;
530 break;
531 case 'd': case 'i': {
532 intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
533 char buf[D2S_BUFSIZE];
534
535 /*
536 * Outputting negative, zero-padded numbers
537 * would require a nontrivial rework of the
538 * interaction between the width and padding
539 * (since 0 padding goes between the '-' and the
540 * number, while ' ' padding goes either before
541 * the - or after the number. Since we
542 * currently don't ever need 0-padded negative
543 * numbers, just don't bother supporting it.
544 */
545 assert(!pad_zero);
546
547 GET_ARG_NUMERIC(val, len);
548 s = d2s(val, (plus_plus ? '+' : (plus_space ?
549 ' ' : '-')), buf, &slen);
550 APPEND_PADDED_S(s, slen, width, left_justify);
551 f++;
552 break;
553 } case 'o': {
554 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
555 char buf[O2S_BUFSIZE];
556
557 GET_ARG_NUMERIC(val, len | 0x80);
558 s = o2s(val, alt_form, buf, &slen);
559 APPEND_PADDED_S(s, slen, width, left_justify);
560 f++;
561 break;
562 } case 'u': {
563 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
564 char buf[U2S_BUFSIZE];
565
566 GET_ARG_NUMERIC(val, len | 0x80);
567 s = u2s(val, 10, false, buf, &slen);
568 APPEND_PADDED_S(s, slen, width, left_justify);
569 f++;
570 break;
571 } case 'x': case 'X': {
572 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
573 char buf[X2S_BUFSIZE];
574
575 GET_ARG_NUMERIC(val, len | 0x80);
576 s = x2s(val, alt_form, *f == 'X', buf, &slen);
577 APPEND_PADDED_S(s, slen, width, left_justify);
578 f++;
579 break;
580 } case 'c': {
581 unsigned char val;
582 char buf[2];
583
584 assert(len == '?' || len == 'l');
585 assert_not_implemented(len != 'l');
586 val = va_arg(ap, int);
587 buf[0] = val;
588 buf[1] = '\0';
589 APPEND_PADDED_S(buf, 1, width, left_justify);
590 f++;
591 break;
592 } case 's':
593 assert(len == '?' || len == 'l');
594 assert_not_implemented(len != 'l');
595 s = va_arg(ap, char *);
596 slen = (prec < 0) ? strlen(s) : (size_t)prec;
597 APPEND_PADDED_S(s, slen, width, left_justify);
598 f++;
599 break;
600 case 'p': {
601 uintmax_t val;
602 char buf[X2S_BUFSIZE];
603
604 GET_ARG_NUMERIC(val, 'p');
605 s = x2s(val, true, false, buf, &slen);
606 APPEND_PADDED_S(s, slen, width, left_justify);
607 f++;
608 break;
609 } default: not_reached();
610 }
611 break;
612 } default: {
613 APPEND_C(*f);
614 f++;
615 break;
616 }}
617 }
618 label_out:
619 if (i < size) {
620 str[i] = '\0';
621 } else {
622 str[size - 1] = '\0';
623 }
624
625#undef APPEND_C
626#undef APPEND_S
627#undef APPEND_PADDED_S
628#undef GET_ARG_NUMERIC
629 return i;
630}
631
632JEMALLOC_FORMAT_PRINTF(3, 4)
633size_t
634malloc_snprintf(char *str, size_t size, const char *format, ...) {
635 size_t ret;
636 va_list ap;
637
638 va_start(ap, format);
639 ret = malloc_vsnprintf(str, size, format, ap);
640 va_end(ap);
641
642 return ret;
643}
644
645void
646malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
647 va_list ap) {
648 char buf[MALLOC_PRINTF_BUFSIZE];
649
650 if (write_cb == NULL) {
651 /*
652 * The caller did not provide an alternate write_cb callback
653 * function, so use the default one. malloc_write() is an
654 * inline function, so use malloc_message() directly here.
655 */
656 write_cb = (je_malloc_message != NULL) ? je_malloc_message :
657 wrtmessage;
658 }
659
660 malloc_vsnprintf(buf, sizeof(buf), format, ap);
661 write_cb(cbopaque, buf);
662}
663
664/*
665 * Print to a callback function in such a way as to (hopefully) avoid memory
666 * allocation.
667 */
668JEMALLOC_FORMAT_PRINTF(3, 4)
669void
670malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) {
671 va_list ap;
672
673 va_start(ap, format);
674 malloc_vcprintf(write_cb, cbopaque, format, ap);
675 va_end(ap);
676}
677
678/* Print to stderr in such a way as to avoid memory allocation. */
679JEMALLOC_FORMAT_PRINTF(1, 2)
680void
681malloc_printf(const char *format, ...) {
682 va_list ap;
683
684 va_start(ap, format);
685 malloc_vcprintf(NULL, NULL, format, ap);
686 va_end(ap);
687}
688
689/*
690 * Restore normal assertion macros, in order to make it possible to compile all
691 * C files as a single concatenation.
692 */
693#undef assert
694#undef not_reached
695#undef not_implemented
696#undef assert_not_implemented
697#include "jemalloc/internal/assert.h"
698