1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/nstime.h" |
5 | |
6 | #include "jemalloc/internal/assert.h" |
7 | |
8 | #define BILLION UINT64_C(1000000000) |
9 | #define MILLION UINT64_C(1000000) |
10 | |
11 | static void |
12 | nstime_set_initialized(nstime_t *time) { |
13 | #ifdef JEMALLOC_DEBUG |
14 | time->magic = NSTIME_MAGIC; |
15 | #endif |
16 | } |
17 | |
18 | static void |
19 | nstime_assert_initialized(const nstime_t *time) { |
20 | #ifdef JEMALLOC_DEBUG |
21 | /* |
22 | * Some parts (e.g. stats) rely on memset to zero initialize. Treat |
23 | * these as valid initialization. |
24 | */ |
25 | assert(time->magic == NSTIME_MAGIC || |
26 | (time->magic == 0 && time->ns == 0)); |
27 | #endif |
28 | } |
29 | |
30 | static void |
31 | nstime_pair_assert_initialized(const nstime_t *t1, const nstime_t *t2) { |
32 | nstime_assert_initialized(t1); |
33 | nstime_assert_initialized(t2); |
34 | } |
35 | |
36 | static void |
37 | nstime_initialize_operand(nstime_t *time) { |
38 | /* |
39 | * Operations like nstime_add may have the initial operand being zero |
40 | * initialized (covered by the assert below). Full-initialize needed |
41 | * before changing it to non-zero. |
42 | */ |
43 | nstime_assert_initialized(time); |
44 | nstime_set_initialized(time); |
45 | } |
46 | |
47 | void |
48 | nstime_init(nstime_t *time, uint64_t ns) { |
49 | nstime_set_initialized(time); |
50 | time->ns = ns; |
51 | } |
52 | |
53 | void |
54 | nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) { |
55 | nstime_set_initialized(time); |
56 | time->ns = sec * BILLION + nsec; |
57 | } |
58 | |
59 | uint64_t |
60 | nstime_ns(const nstime_t *time) { |
61 | nstime_assert_initialized(time); |
62 | return time->ns; |
63 | } |
64 | |
65 | uint64_t |
66 | nstime_msec(const nstime_t *time) { |
67 | nstime_assert_initialized(time); |
68 | return time->ns / MILLION; |
69 | } |
70 | |
71 | uint64_t |
72 | nstime_sec(const nstime_t *time) { |
73 | nstime_assert_initialized(time); |
74 | return time->ns / BILLION; |
75 | } |
76 | |
77 | uint64_t |
78 | nstime_nsec(const nstime_t *time) { |
79 | nstime_assert_initialized(time); |
80 | return time->ns % BILLION; |
81 | } |
82 | |
83 | void |
84 | nstime_copy(nstime_t *time, const nstime_t *source) { |
85 | /* Source is required to be initialized. */ |
86 | nstime_assert_initialized(source); |
87 | *time = *source; |
88 | nstime_assert_initialized(time); |
89 | } |
90 | |
91 | int |
92 | nstime_compare(const nstime_t *a, const nstime_t *b) { |
93 | nstime_pair_assert_initialized(a, b); |
94 | return (a->ns > b->ns) - (a->ns < b->ns); |
95 | } |
96 | |
97 | void |
98 | nstime_add(nstime_t *time, const nstime_t *addend) { |
99 | nstime_pair_assert_initialized(time, addend); |
100 | assert(UINT64_MAX - time->ns >= addend->ns); |
101 | |
102 | nstime_initialize_operand(time); |
103 | time->ns += addend->ns; |
104 | } |
105 | |
106 | void |
107 | nstime_iadd(nstime_t *time, uint64_t addend) { |
108 | nstime_assert_initialized(time); |
109 | assert(UINT64_MAX - time->ns >= addend); |
110 | |
111 | nstime_initialize_operand(time); |
112 | time->ns += addend; |
113 | } |
114 | |
115 | void |
116 | nstime_subtract(nstime_t *time, const nstime_t *subtrahend) { |
117 | nstime_pair_assert_initialized(time, subtrahend); |
118 | assert(nstime_compare(time, subtrahend) >= 0); |
119 | |
120 | /* No initialize operand -- subtraction must be initialized. */ |
121 | time->ns -= subtrahend->ns; |
122 | } |
123 | |
124 | void |
125 | nstime_isubtract(nstime_t *time, uint64_t subtrahend) { |
126 | nstime_assert_initialized(time); |
127 | assert(time->ns >= subtrahend); |
128 | |
129 | /* No initialize operand -- subtraction must be initialized. */ |
130 | time->ns -= subtrahend; |
131 | } |
132 | |
133 | void |
134 | nstime_imultiply(nstime_t *time, uint64_t multiplier) { |
135 | nstime_assert_initialized(time); |
136 | assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) << |
137 | 2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns)); |
138 | |
139 | nstime_initialize_operand(time); |
140 | time->ns *= multiplier; |
141 | } |
142 | |
143 | void |
144 | nstime_idivide(nstime_t *time, uint64_t divisor) { |
145 | nstime_assert_initialized(time); |
146 | assert(divisor != 0); |
147 | |
148 | nstime_initialize_operand(time); |
149 | time->ns /= divisor; |
150 | } |
151 | |
152 | uint64_t |
153 | nstime_divide(const nstime_t *time, const nstime_t *divisor) { |
154 | nstime_pair_assert_initialized(time, divisor); |
155 | assert(divisor->ns != 0); |
156 | |
157 | /* No initialize operand -- *time itself remains unchanged. */ |
158 | return time->ns / divisor->ns; |
159 | } |
160 | |
161 | /* Returns time since *past, w/o updating *past. */ |
162 | uint64_t |
163 | nstime_ns_since(const nstime_t *past) { |
164 | nstime_assert_initialized(past); |
165 | |
166 | nstime_t now; |
167 | nstime_copy(&now, past); |
168 | nstime_update(&now); |
169 | |
170 | assert(nstime_compare(&now, past) >= 0); |
171 | return now.ns - past->ns; |
172 | } |
173 | |
174 | #ifdef _WIN32 |
175 | # define NSTIME_MONOTONIC true |
176 | static void |
177 | nstime_get(nstime_t *time) { |
178 | FILETIME ft; |
179 | uint64_t ticks_100ns; |
180 | |
181 | GetSystemTimeAsFileTime(&ft); |
182 | ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; |
183 | |
184 | nstime_init(time, ticks_100ns * 100); |
185 | } |
186 | #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE) |
187 | # define NSTIME_MONOTONIC true |
188 | static void |
189 | nstime_get(nstime_t *time) { |
190 | struct timespec ts; |
191 | |
192 | clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); |
193 | nstime_init2(time, ts.tv_sec, ts.tv_nsec); |
194 | } |
195 | #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC) |
196 | # define NSTIME_MONOTONIC true |
197 | static void |
198 | nstime_get(nstime_t *time) { |
199 | struct timespec ts; |
200 | |
201 | clock_gettime(CLOCK_MONOTONIC, &ts); |
202 | nstime_init2(time, ts.tv_sec, ts.tv_nsec); |
203 | } |
204 | #elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME) |
205 | # define NSTIME_MONOTONIC true |
206 | static void |
207 | nstime_get(nstime_t *time) { |
208 | nstime_init(time, mach_absolute_time()); |
209 | } |
210 | #else |
211 | # define NSTIME_MONOTONIC false |
212 | static void |
213 | nstime_get(nstime_t *time) { |
214 | struct timeval tv; |
215 | |
216 | gettimeofday(&tv, NULL); |
217 | nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000); |
218 | } |
219 | #endif |
220 | |
221 | static bool |
222 | nstime_monotonic_impl(void) { |
223 | return NSTIME_MONOTONIC; |
224 | #undef NSTIME_MONOTONIC |
225 | } |
226 | nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl; |
227 | |
228 | prof_time_res_t opt_prof_time_res = |
229 | prof_time_res_default; |
230 | |
231 | const char *prof_time_res_mode_names[] = { |
232 | "default" , |
233 | "high" , |
234 | }; |
235 | |
236 | |
237 | static void |
238 | nstime_get_realtime(nstime_t *time) { |
239 | #if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32) |
240 | struct timespec ts; |
241 | |
242 | clock_gettime(CLOCK_REALTIME, &ts); |
243 | nstime_init2(time, ts.tv_sec, ts.tv_nsec); |
244 | #else |
245 | unreachable(); |
246 | #endif |
247 | } |
248 | |
249 | static void |
250 | nstime_prof_update_impl(nstime_t *time) { |
251 | nstime_t old_time; |
252 | |
253 | nstime_copy(&old_time, time); |
254 | |
255 | if (opt_prof_time_res == prof_time_res_high) { |
256 | nstime_get_realtime(time); |
257 | } else { |
258 | nstime_get(time); |
259 | } |
260 | } |
261 | nstime_prof_update_t *JET_MUTABLE nstime_prof_update = nstime_prof_update_impl; |
262 | |
263 | static void |
264 | nstime_update_impl(nstime_t *time) { |
265 | nstime_t old_time; |
266 | |
267 | nstime_copy(&old_time, time); |
268 | nstime_get(time); |
269 | |
270 | /* Handle non-monotonic clocks. */ |
271 | if (unlikely(nstime_compare(&old_time, time) > 0)) { |
272 | nstime_copy(time, &old_time); |
273 | } |
274 | } |
275 | nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl; |
276 | |
277 | void |
278 | nstime_init_update(nstime_t *time) { |
279 | nstime_init_zero(time); |
280 | nstime_update(time); |
281 | } |
282 | |
283 | void |
284 | nstime_prof_init_update(nstime_t *time) { |
285 | nstime_init_zero(time); |
286 | nstime_prof_update(time); |
287 | } |
288 | |
289 | |
290 | |