1 | #ifndef JEMALLOC_INTERNAL_UTIL_H |
2 | #define JEMALLOC_INTERNAL_UTIL_H |
3 | |
4 | #define UTIL_INLINE static inline |
5 | |
6 | /* Junk fill patterns. */ |
7 | #ifndef JEMALLOC_ALLOC_JUNK |
8 | # define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) |
9 | #endif |
10 | #ifndef JEMALLOC_FREE_JUNK |
11 | # define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) |
12 | #endif |
13 | |
14 | /* |
15 | * Wrap a cpp argument that contains commas such that it isn't broken up into |
16 | * multiple arguments. |
17 | */ |
18 | #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ |
19 | |
20 | /* cpp macro definition stringification. */ |
21 | #define STRINGIFY_HELPER(x) #x |
22 | #define STRINGIFY(x) STRINGIFY_HELPER(x) |
23 | |
24 | /* |
25 | * Silence compiler warnings due to uninitialized values. This is used |
26 | * wherever the compiler fails to recognize that the variable is never used |
27 | * uninitialized. |
28 | */ |
29 | #define JEMALLOC_CC_SILENCE_INIT(v) = v |
30 | |
31 | #ifdef __GNUC__ |
32 | # define likely(x) __builtin_expect(!!(x), 1) |
33 | # define unlikely(x) __builtin_expect(!!(x), 0) |
34 | #else |
35 | # define likely(x) !!(x) |
36 | # define unlikely(x) !!(x) |
37 | #endif |
38 | |
39 | #if !defined(JEMALLOC_INTERNAL_UNREACHABLE) |
40 | # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure |
41 | #endif |
42 | |
43 | #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() |
44 | |
45 | /* Set error code. */ |
46 | UTIL_INLINE void |
47 | set_errno(int errnum) { |
48 | #ifdef _WIN32 |
49 | SetLastError(errnum); |
50 | #else |
51 | errno = errnum; |
52 | #endif |
53 | } |
54 | |
55 | /* Get last error code. */ |
56 | UTIL_INLINE int |
57 | get_errno(void) { |
58 | #ifdef _WIN32 |
59 | return GetLastError(); |
60 | #else |
61 | return errno; |
62 | #endif |
63 | } |
64 | |
65 | JEMALLOC_ALWAYS_INLINE void |
66 | util_assume(bool b) { |
67 | if (!b) { |
68 | unreachable(); |
69 | } |
70 | } |
71 | |
72 | /* ptr should be valid. */ |
73 | JEMALLOC_ALWAYS_INLINE void |
74 | util_prefetch_read(void *ptr) { |
75 | /* |
76 | * This should arguably be a config check; but any version of GCC so old |
77 | * that it doesn't support __builtin_prefetch is also too old to build |
78 | * jemalloc. |
79 | */ |
80 | #ifdef __GNUC__ |
81 | if (config_debug) { |
82 | /* Enforce the "valid ptr" requirement. */ |
83 | *(volatile char *)ptr; |
84 | } |
85 | __builtin_prefetch(ptr, /* read or write */ 0, /* locality hint */ 3); |
86 | #else |
87 | *(volatile char *)ptr; |
88 | #endif |
89 | } |
90 | |
91 | JEMALLOC_ALWAYS_INLINE void |
92 | util_prefetch_write(void *ptr) { |
93 | #ifdef __GNUC__ |
94 | if (config_debug) { |
95 | *(volatile char *)ptr; |
96 | } |
97 | /* |
98 | * The only difference from the read variant is that this has a 1 as the |
99 | * second argument (the write hint). |
100 | */ |
101 | __builtin_prefetch(ptr, 1, 3); |
102 | #else |
103 | *(volatile char *)ptr; |
104 | #endif |
105 | } |
106 | |
107 | JEMALLOC_ALWAYS_INLINE void |
108 | util_prefetch_read_range(void *ptr, size_t sz) { |
109 | for (size_t i = 0; i < sz; i += CACHELINE) { |
110 | util_prefetch_read((void *)((uintptr_t)ptr + i)); |
111 | } |
112 | } |
113 | |
114 | JEMALLOC_ALWAYS_INLINE void |
115 | util_prefetch_write_range(void *ptr, size_t sz) { |
116 | for (size_t i = 0; i < sz; i += CACHELINE) { |
117 | util_prefetch_write((void *)((uintptr_t)ptr + i)); |
118 | } |
119 | } |
120 | |
121 | #undef UTIL_INLINE |
122 | |
123 | #endif /* JEMALLOC_INTERNAL_UTIL_H */ |
124 | |