1#ifndef JEMALLOC_INTERNAL_UTIL_H
2#define JEMALLOC_INTERNAL_UTIL_H
3
4#define UTIL_INLINE static inline
5
6/* Junk fill patterns. */
7#ifndef JEMALLOC_ALLOC_JUNK
8# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)
9#endif
10#ifndef JEMALLOC_FREE_JUNK
11# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a)
12#endif
13
14/*
15 * Wrap a cpp argument that contains commas such that it isn't broken up into
16 * multiple arguments.
17 */
18#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
19
20/* cpp macro definition stringification. */
21#define STRINGIFY_HELPER(x) #x
22#define STRINGIFY(x) STRINGIFY_HELPER(x)
23
24/*
25 * Silence compiler warnings due to uninitialized values. This is used
26 * wherever the compiler fails to recognize that the variable is never used
27 * uninitialized.
28 */
29#define JEMALLOC_CC_SILENCE_INIT(v) = v
30
31#ifdef __GNUC__
32# define likely(x) __builtin_expect(!!(x), 1)
33# define unlikely(x) __builtin_expect(!!(x), 0)
34#else
35# define likely(x) !!(x)
36# define unlikely(x) !!(x)
37#endif
38
39#if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
40# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
41#endif
42
43#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
44
45/* Set error code. */
46UTIL_INLINE void
47set_errno(int errnum) {
48#ifdef _WIN32
49 SetLastError(errnum);
50#else
51 errno = errnum;
52#endif
53}
54
55/* Get last error code. */
56UTIL_INLINE int
57get_errno(void) {
58#ifdef _WIN32
59 return GetLastError();
60#else
61 return errno;
62#endif
63}
64
65JEMALLOC_ALWAYS_INLINE void
66util_assume(bool b) {
67 if (!b) {
68 unreachable();
69 }
70}
71
72/* ptr should be valid. */
73JEMALLOC_ALWAYS_INLINE void
74util_prefetch_read(void *ptr) {
75 /*
76 * This should arguably be a config check; but any version of GCC so old
77 * that it doesn't support __builtin_prefetch is also too old to build
78 * jemalloc.
79 */
80#ifdef __GNUC__
81 if (config_debug) {
82 /* Enforce the "valid ptr" requirement. */
83 *(volatile char *)ptr;
84 }
85 __builtin_prefetch(ptr, /* read or write */ 0, /* locality hint */ 3);
86#else
87 *(volatile char *)ptr;
88#endif
89}
90
91JEMALLOC_ALWAYS_INLINE void
92util_prefetch_write(void *ptr) {
93#ifdef __GNUC__
94 if (config_debug) {
95 *(volatile char *)ptr;
96 }
97 /*
98 * The only difference from the read variant is that this has a 1 as the
99 * second argument (the write hint).
100 */
101 __builtin_prefetch(ptr, 1, 3);
102#else
103 *(volatile char *)ptr;
104#endif
105}
106
107JEMALLOC_ALWAYS_INLINE void
108util_prefetch_read_range(void *ptr, size_t sz) {
109 for (size_t i = 0; i < sz; i += CACHELINE) {
110 util_prefetch_read((void *)((uintptr_t)ptr + i));
111 }
112}
113
114JEMALLOC_ALWAYS_INLINE void
115util_prefetch_write_range(void *ptr, size_t sz) {
116 for (size_t i = 0; i < sz; i += CACHELINE) {
117 util_prefetch_write((void *)((uintptr_t)ptr + i));
118 }
119}
120
121#undef UTIL_INLINE
122
123#endif /* JEMALLOC_INTERNAL_UTIL_H */
124