1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/malloc_io.h" |
5 | #include "jemalloc/internal/util.h" |
6 | |
7 | #ifdef assert |
8 | # undef assert |
9 | #endif |
10 | #ifdef not_reached |
11 | # undef not_reached |
12 | #endif |
13 | #ifdef not_implemented |
14 | # undef not_implemented |
15 | #endif |
16 | #ifdef assert_not_implemented |
17 | # undef assert_not_implemented |
18 | #endif |
19 | |
20 | /* |
21 | * Define simple versions of assertion macros that won't recurse in case |
22 | * of assertion failures in malloc_*printf(). |
23 | */ |
24 | #define assert(e) do { \ |
25 | if (config_debug && !(e)) { \ |
26 | malloc_write("<jemalloc>: Failed assertion\n"); \ |
27 | abort(); \ |
28 | } \ |
29 | } while (0) |
30 | |
31 | #define not_reached() do { \ |
32 | if (config_debug) { \ |
33 | malloc_write("<jemalloc>: Unreachable code reached\n"); \ |
34 | abort(); \ |
35 | } \ |
36 | unreachable(); \ |
37 | } while (0) |
38 | |
39 | #define not_implemented() do { \ |
40 | if (config_debug) { \ |
41 | malloc_write("<jemalloc>: Not implemented\n"); \ |
42 | abort(); \ |
43 | } \ |
44 | } while (0) |
45 | |
46 | #define assert_not_implemented(e) do { \ |
47 | if (unlikely(config_debug && !(e))) { \ |
48 | not_implemented(); \ |
49 | } \ |
50 | } while (0) |
51 | |
52 | /******************************************************************************/ |
53 | /* Function prototypes for non-inline static functions. */ |
54 | |
55 | #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) |
56 | static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, |
57 | size_t *slen_p); |
58 | #define D2S_BUFSIZE (1 + U2S_BUFSIZE) |
59 | static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); |
60 | #define O2S_BUFSIZE (1 + U2S_BUFSIZE) |
61 | static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); |
62 | #define X2S_BUFSIZE (2 + U2S_BUFSIZE) |
63 | static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, |
64 | size_t *slen_p); |
65 | |
66 | /******************************************************************************/ |
67 | |
68 | /* malloc_message() setup. */ |
69 | void |
70 | wrtmessage(void *cbopaque, const char *s) { |
71 | malloc_write_fd(STDERR_FILENO, s, strlen(s)); |
72 | } |
73 | |
74 | JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); |
75 | |
76 | /* |
77 | * Wrapper around malloc_message() that avoids the need for |
78 | * je_malloc_message(...) throughout the code. |
79 | */ |
80 | void |
81 | malloc_write(const char *s) { |
82 | if (je_malloc_message != NULL) { |
83 | je_malloc_message(NULL, s); |
84 | } else { |
85 | wrtmessage(NULL, s); |
86 | } |
87 | } |
88 | |
89 | /* |
90 | * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so |
91 | * provide a wrapper. |
92 | */ |
93 | int |
94 | buferror(int err, char *buf, size_t buflen) { |
95 | #ifdef _WIN32 |
96 | FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, |
97 | (LPSTR)buf, (DWORD)buflen, NULL); |
98 | return 0; |
99 | #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) |
100 | char *b = strerror_r(err, buf, buflen); |
101 | if (b != buf) { |
102 | strncpy(buf, b, buflen); |
103 | buf[buflen-1] = '\0'; |
104 | } |
105 | return 0; |
106 | #else |
107 | return strerror_r(err, buf, buflen); |
108 | #endif |
109 | } |
110 | |
111 | uintmax_t |
112 | malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { |
113 | uintmax_t ret, digit; |
114 | unsigned b; |
115 | bool neg; |
116 | const char *p, *ns; |
117 | |
118 | p = nptr; |
119 | if (base < 0 || base == 1 || base > 36) { |
120 | ns = p; |
121 | set_errno(EINVAL); |
122 | ret = UINTMAX_MAX; |
123 | goto label_return; |
124 | } |
125 | b = base; |
126 | |
127 | /* Swallow leading whitespace and get sign, if any. */ |
128 | neg = false; |
129 | while (true) { |
130 | switch (*p) { |
131 | case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': |
132 | p++; |
133 | break; |
134 | case '-': |
135 | neg = true; |
136 | JEMALLOC_FALLTHROUGH; |
137 | case '+': |
138 | p++; |
139 | JEMALLOC_FALLTHROUGH; |
140 | default: |
141 | goto label_prefix; |
142 | } |
143 | } |
144 | |
145 | /* Get prefix, if any. */ |
146 | label_prefix: |
147 | /* |
148 | * Note where the first non-whitespace/sign character is so that it is |
149 | * possible to tell whether any digits are consumed (e.g., " 0" vs. |
150 | * " -x"). |
151 | */ |
152 | ns = p; |
153 | if (*p == '0') { |
154 | switch (p[1]) { |
155 | case '0': case '1': case '2': case '3': case '4': case '5': |
156 | case '6': case '7': |
157 | if (b == 0) { |
158 | b = 8; |
159 | } |
160 | if (b == 8) { |
161 | p++; |
162 | } |
163 | break; |
164 | case 'X': case 'x': |
165 | switch (p[2]) { |
166 | case '0': case '1': case '2': case '3': case '4': |
167 | case '5': case '6': case '7': case '8': case '9': |
168 | case 'A': case 'B': case 'C': case 'D': case 'E': |
169 | case 'F': |
170 | case 'a': case 'b': case 'c': case 'd': case 'e': |
171 | case 'f': |
172 | if (b == 0) { |
173 | b = 16; |
174 | } |
175 | if (b == 16) { |
176 | p += 2; |
177 | } |
178 | break; |
179 | default: |
180 | break; |
181 | } |
182 | break; |
183 | default: |
184 | p++; |
185 | ret = 0; |
186 | goto label_return; |
187 | } |
188 | } |
189 | if (b == 0) { |
190 | b = 10; |
191 | } |
192 | |
193 | /* Convert. */ |
194 | ret = 0; |
195 | while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) |
196 | || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) |
197 | || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { |
198 | uintmax_t pret = ret; |
199 | ret *= b; |
200 | ret += digit; |
201 | if (ret < pret) { |
202 | /* Overflow. */ |
203 | set_errno(ERANGE); |
204 | ret = UINTMAX_MAX; |
205 | goto label_return; |
206 | } |
207 | p++; |
208 | } |
209 | if (neg) { |
210 | ret = (uintmax_t)(-((intmax_t)ret)); |
211 | } |
212 | |
213 | if (p == ns) { |
214 | /* No conversion performed. */ |
215 | set_errno(EINVAL); |
216 | ret = UINTMAX_MAX; |
217 | goto label_return; |
218 | } |
219 | |
220 | label_return: |
221 | if (endptr != NULL) { |
222 | if (p == ns) { |
223 | /* No characters were converted. */ |
224 | *endptr = (char *)nptr; |
225 | } else { |
226 | *endptr = (char *)p; |
227 | } |
228 | } |
229 | return ret; |
230 | } |
231 | |
232 | static char * |
233 | u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { |
234 | unsigned i; |
235 | |
236 | i = U2S_BUFSIZE - 1; |
237 | s[i] = '\0'; |
238 | switch (base) { |
239 | case 10: |
240 | do { |
241 | i--; |
242 | s[i] = "0123456789" [x % (uint64_t)10]; |
243 | x /= (uint64_t)10; |
244 | } while (x > 0); |
245 | break; |
246 | case 16: { |
247 | const char *digits = (uppercase) |
248 | ? "0123456789ABCDEF" |
249 | : "0123456789abcdef" ; |
250 | |
251 | do { |
252 | i--; |
253 | s[i] = digits[x & 0xf]; |
254 | x >>= 4; |
255 | } while (x > 0); |
256 | break; |
257 | } default: { |
258 | const char *digits = (uppercase) |
259 | ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
260 | : "0123456789abcdefghijklmnopqrstuvwxyz" ; |
261 | |
262 | assert(base >= 2 && base <= 36); |
263 | do { |
264 | i--; |
265 | s[i] = digits[x % (uint64_t)base]; |
266 | x /= (uint64_t)base; |
267 | } while (x > 0); |
268 | }} |
269 | |
270 | *slen_p = U2S_BUFSIZE - 1 - i; |
271 | return &s[i]; |
272 | } |
273 | |
274 | static char * |
275 | d2s(intmax_t x, char sign, char *s, size_t *slen_p) { |
276 | bool neg; |
277 | |
278 | if ((neg = (x < 0))) { |
279 | x = -x; |
280 | } |
281 | s = u2s(x, 10, false, s, slen_p); |
282 | if (neg) { |
283 | sign = '-'; |
284 | } |
285 | switch (sign) { |
286 | case '-': |
287 | if (!neg) { |
288 | break; |
289 | } |
290 | JEMALLOC_FALLTHROUGH; |
291 | case ' ': |
292 | case '+': |
293 | s--; |
294 | (*slen_p)++; |
295 | *s = sign; |
296 | break; |
297 | default: not_reached(); |
298 | } |
299 | return s; |
300 | } |
301 | |
302 | static char * |
303 | o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) { |
304 | s = u2s(x, 8, false, s, slen_p); |
305 | if (alt_form && *s != '0') { |
306 | s--; |
307 | (*slen_p)++; |
308 | *s = '0'; |
309 | } |
310 | return s; |
311 | } |
312 | |
313 | static char * |
314 | x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { |
315 | s = u2s(x, 16, uppercase, s, slen_p); |
316 | if (alt_form) { |
317 | s -= 2; |
318 | (*slen_p) += 2; |
319 | memcpy(s, uppercase ? "0X" : "0x" , 2); |
320 | } |
321 | return s; |
322 | } |
323 | |
324 | JEMALLOC_COLD |
325 | size_t |
326 | malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { |
327 | size_t i; |
328 | const char *f; |
329 | |
330 | #define APPEND_C(c) do { \ |
331 | if (i < size) { \ |
332 | str[i] = (c); \ |
333 | } \ |
334 | i++; \ |
335 | } while (0) |
336 | #define APPEND_S(s, slen) do { \ |
337 | if (i < size) { \ |
338 | size_t cpylen = (slen <= size - i) ? slen : size - i; \ |
339 | memcpy(&str[i], s, cpylen); \ |
340 | } \ |
341 | i += slen; \ |
342 | } while (0) |
343 | #define APPEND_PADDED_S(s, slen, width, left_justify) do { \ |
344 | /* Left padding. */ \ |
345 | size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ |
346 | (size_t)width - slen : 0); \ |
347 | if (!left_justify && pad_len != 0) { \ |
348 | size_t j; \ |
349 | for (j = 0; j < pad_len; j++) { \ |
350 | if (pad_zero) { \ |
351 | APPEND_C('0'); \ |
352 | } else { \ |
353 | APPEND_C(' '); \ |
354 | } \ |
355 | } \ |
356 | } \ |
357 | /* Value. */ \ |
358 | APPEND_S(s, slen); \ |
359 | /* Right padding. */ \ |
360 | if (left_justify && pad_len != 0) { \ |
361 | size_t j; \ |
362 | for (j = 0; j < pad_len; j++) { \ |
363 | APPEND_C(' '); \ |
364 | } \ |
365 | } \ |
366 | } while (0) |
367 | #define GET_ARG_NUMERIC(val, len) do { \ |
368 | switch ((unsigned char)len) { \ |
369 | case '?': \ |
370 | val = va_arg(ap, int); \ |
371 | break; \ |
372 | case '?' | 0x80: \ |
373 | val = va_arg(ap, unsigned int); \ |
374 | break; \ |
375 | case 'l': \ |
376 | val = va_arg(ap, long); \ |
377 | break; \ |
378 | case 'l' | 0x80: \ |
379 | val = va_arg(ap, unsigned long); \ |
380 | break; \ |
381 | case 'q': \ |
382 | val = va_arg(ap, long long); \ |
383 | break; \ |
384 | case 'q' | 0x80: \ |
385 | val = va_arg(ap, unsigned long long); \ |
386 | break; \ |
387 | case 'j': \ |
388 | val = va_arg(ap, intmax_t); \ |
389 | break; \ |
390 | case 'j' | 0x80: \ |
391 | val = va_arg(ap, uintmax_t); \ |
392 | break; \ |
393 | case 't': \ |
394 | val = va_arg(ap, ptrdiff_t); \ |
395 | break; \ |
396 | case 'z': \ |
397 | val = va_arg(ap, ssize_t); \ |
398 | break; \ |
399 | case 'z' | 0x80: \ |
400 | val = va_arg(ap, size_t); \ |
401 | break; \ |
402 | case 'p': /* Synthetic; used for %p. */ \ |
403 | val = va_arg(ap, uintptr_t); \ |
404 | break; \ |
405 | default: \ |
406 | not_reached(); \ |
407 | val = 0; \ |
408 | } \ |
409 | } while (0) |
410 | |
411 | i = 0; |
412 | f = format; |
413 | while (true) { |
414 | switch (*f) { |
415 | case '\0': goto label_out; |
416 | case '%': { |
417 | bool alt_form = false; |
418 | bool left_justify = false; |
419 | bool plus_space = false; |
420 | bool plus_plus = false; |
421 | int prec = -1; |
422 | int width = -1; |
423 | unsigned char len = '?'; |
424 | char *s; |
425 | size_t slen; |
426 | bool first_width_digit = true; |
427 | bool pad_zero = false; |
428 | |
429 | f++; |
430 | /* Flags. */ |
431 | while (true) { |
432 | switch (*f) { |
433 | case '#': |
434 | assert(!alt_form); |
435 | alt_form = true; |
436 | break; |
437 | case '-': |
438 | assert(!left_justify); |
439 | left_justify = true; |
440 | break; |
441 | case ' ': |
442 | assert(!plus_space); |
443 | plus_space = true; |
444 | break; |
445 | case '+': |
446 | assert(!plus_plus); |
447 | plus_plus = true; |
448 | break; |
449 | default: goto label_width; |
450 | } |
451 | f++; |
452 | } |
453 | /* Width. */ |
454 | label_width: |
455 | switch (*f) { |
456 | case '*': |
457 | width = va_arg(ap, int); |
458 | f++; |
459 | if (width < 0) { |
460 | left_justify = true; |
461 | width = -width; |
462 | } |
463 | break; |
464 | case '0': |
465 | if (first_width_digit) { |
466 | pad_zero = true; |
467 | } |
468 | JEMALLOC_FALLTHROUGH; |
469 | case '1': case '2': case '3': case '4': |
470 | case '5': case '6': case '7': case '8': case '9': { |
471 | uintmax_t uwidth; |
472 | set_errno(0); |
473 | uwidth = malloc_strtoumax(f, (char **)&f, 10); |
474 | assert(uwidth != UINTMAX_MAX || get_errno() != |
475 | ERANGE); |
476 | width = (int)uwidth; |
477 | first_width_digit = false; |
478 | break; |
479 | } default: |
480 | break; |
481 | } |
482 | /* Width/precision separator. */ |
483 | if (*f == '.') { |
484 | f++; |
485 | } else { |
486 | goto label_length; |
487 | } |
488 | /* Precision. */ |
489 | switch (*f) { |
490 | case '*': |
491 | prec = va_arg(ap, int); |
492 | f++; |
493 | break; |
494 | case '0': case '1': case '2': case '3': case '4': |
495 | case '5': case '6': case '7': case '8': case '9': { |
496 | uintmax_t uprec; |
497 | set_errno(0); |
498 | uprec = malloc_strtoumax(f, (char **)&f, 10); |
499 | assert(uprec != UINTMAX_MAX || get_errno() != |
500 | ERANGE); |
501 | prec = (int)uprec; |
502 | break; |
503 | } |
504 | default: break; |
505 | } |
506 | /* Length. */ |
507 | label_length: |
508 | switch (*f) { |
509 | case 'l': |
510 | f++; |
511 | if (*f == 'l') { |
512 | len = 'q'; |
513 | f++; |
514 | } else { |
515 | len = 'l'; |
516 | } |
517 | break; |
518 | case 'q': case 'j': case 't': case 'z': |
519 | len = *f; |
520 | f++; |
521 | break; |
522 | default: break; |
523 | } |
524 | /* Conversion specifier. */ |
525 | switch (*f) { |
526 | case '%': |
527 | /* %% */ |
528 | APPEND_C(*f); |
529 | f++; |
530 | break; |
531 | case 'd': case 'i': { |
532 | intmax_t val JEMALLOC_CC_SILENCE_INIT(0); |
533 | char buf[D2S_BUFSIZE]; |
534 | |
535 | /* |
536 | * Outputting negative, zero-padded numbers |
537 | * would require a nontrivial rework of the |
538 | * interaction between the width and padding |
539 | * (since 0 padding goes between the '-' and the |
540 | * number, while ' ' padding goes either before |
541 | * the - or after the number. Since we |
542 | * currently don't ever need 0-padded negative |
543 | * numbers, just don't bother supporting it. |
544 | */ |
545 | assert(!pad_zero); |
546 | |
547 | GET_ARG_NUMERIC(val, len); |
548 | s = d2s(val, (plus_plus ? '+' : (plus_space ? |
549 | ' ' : '-')), buf, &slen); |
550 | APPEND_PADDED_S(s, slen, width, left_justify); |
551 | f++; |
552 | break; |
553 | } case 'o': { |
554 | uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); |
555 | char buf[O2S_BUFSIZE]; |
556 | |
557 | GET_ARG_NUMERIC(val, len | 0x80); |
558 | s = o2s(val, alt_form, buf, &slen); |
559 | APPEND_PADDED_S(s, slen, width, left_justify); |
560 | f++; |
561 | break; |
562 | } case 'u': { |
563 | uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); |
564 | char buf[U2S_BUFSIZE]; |
565 | |
566 | GET_ARG_NUMERIC(val, len | 0x80); |
567 | s = u2s(val, 10, false, buf, &slen); |
568 | APPEND_PADDED_S(s, slen, width, left_justify); |
569 | f++; |
570 | break; |
571 | } case 'x': case 'X': { |
572 | uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); |
573 | char buf[X2S_BUFSIZE]; |
574 | |
575 | GET_ARG_NUMERIC(val, len | 0x80); |
576 | s = x2s(val, alt_form, *f == 'X', buf, &slen); |
577 | APPEND_PADDED_S(s, slen, width, left_justify); |
578 | f++; |
579 | break; |
580 | } case 'c': { |
581 | unsigned char val; |
582 | char buf[2]; |
583 | |
584 | assert(len == '?' || len == 'l'); |
585 | assert_not_implemented(len != 'l'); |
586 | val = va_arg(ap, int); |
587 | buf[0] = val; |
588 | buf[1] = '\0'; |
589 | APPEND_PADDED_S(buf, 1, width, left_justify); |
590 | f++; |
591 | break; |
592 | } case 's': |
593 | assert(len == '?' || len == 'l'); |
594 | assert_not_implemented(len != 'l'); |
595 | s = va_arg(ap, char *); |
596 | slen = (prec < 0) ? strlen(s) : (size_t)prec; |
597 | APPEND_PADDED_S(s, slen, width, left_justify); |
598 | f++; |
599 | break; |
600 | case 'p': { |
601 | uintmax_t val; |
602 | char buf[X2S_BUFSIZE]; |
603 | |
604 | GET_ARG_NUMERIC(val, 'p'); |
605 | s = x2s(val, true, false, buf, &slen); |
606 | APPEND_PADDED_S(s, slen, width, left_justify); |
607 | f++; |
608 | break; |
609 | } default: not_reached(); |
610 | } |
611 | break; |
612 | } default: { |
613 | APPEND_C(*f); |
614 | f++; |
615 | break; |
616 | }} |
617 | } |
618 | label_out: |
619 | if (i < size) { |
620 | str[i] = '\0'; |
621 | } else { |
622 | str[size - 1] = '\0'; |
623 | } |
624 | |
625 | #undef APPEND_C |
626 | #undef APPEND_S |
627 | #undef APPEND_PADDED_S |
628 | #undef GET_ARG_NUMERIC |
629 | return i; |
630 | } |
631 | |
632 | JEMALLOC_FORMAT_PRINTF(3, 4) |
633 | size_t |
634 | malloc_snprintf(char *str, size_t size, const char *format, ...) { |
635 | size_t ret; |
636 | va_list ap; |
637 | |
638 | va_start(ap, format); |
639 | ret = malloc_vsnprintf(str, size, format, ap); |
640 | va_end(ap); |
641 | |
642 | return ret; |
643 | } |
644 | |
645 | void |
646 | malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, |
647 | va_list ap) { |
648 | char buf[MALLOC_PRINTF_BUFSIZE]; |
649 | |
650 | if (write_cb == NULL) { |
651 | /* |
652 | * The caller did not provide an alternate write_cb callback |
653 | * function, so use the default one. malloc_write() is an |
654 | * inline function, so use malloc_message() directly here. |
655 | */ |
656 | write_cb = (je_malloc_message != NULL) ? je_malloc_message : |
657 | wrtmessage; |
658 | } |
659 | |
660 | malloc_vsnprintf(buf, sizeof(buf), format, ap); |
661 | write_cb(cbopaque, buf); |
662 | } |
663 | |
664 | /* |
665 | * Print to a callback function in such a way as to (hopefully) avoid memory |
666 | * allocation. |
667 | */ |
668 | JEMALLOC_FORMAT_PRINTF(3, 4) |
669 | void |
670 | malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) { |
671 | va_list ap; |
672 | |
673 | va_start(ap, format); |
674 | malloc_vcprintf(write_cb, cbopaque, format, ap); |
675 | va_end(ap); |
676 | } |
677 | |
678 | /* Print to stderr in such a way as to avoid memory allocation. */ |
679 | JEMALLOC_FORMAT_PRINTF(1, 2) |
680 | void |
681 | malloc_printf(const char *format, ...) { |
682 | va_list ap; |
683 | |
684 | va_start(ap, format); |
685 | malloc_vcprintf(NULL, NULL, format, ap); |
686 | va_end(ap); |
687 | } |
688 | |
689 | /* |
690 | * Restore normal assertion macros, in order to make it possible to compile all |
691 | * C files as a single concatenation. |
692 | */ |
693 | #undef assert |
694 | #undef not_reached |
695 | #undef not_implemented |
696 | #undef assert_not_implemented |
697 | #include "jemalloc/internal/assert.h" |
698 | |