1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/nstime.h"
5
6#include "jemalloc/internal/assert.h"
7
8#define BILLION UINT64_C(1000000000)
9#define MILLION UINT64_C(1000000)
10
11static void
12nstime_set_initialized(nstime_t *time) {
13#ifdef JEMALLOC_DEBUG
14 time->magic = NSTIME_MAGIC;
15#endif
16}
17
18static void
19nstime_assert_initialized(const nstime_t *time) {
20#ifdef JEMALLOC_DEBUG
21 /*
22 * Some parts (e.g. stats) rely on memset to zero initialize. Treat
23 * these as valid initialization.
24 */
25 assert(time->magic == NSTIME_MAGIC ||
26 (time->magic == 0 && time->ns == 0));
27#endif
28}
29
30static void
31nstime_pair_assert_initialized(const nstime_t *t1, const nstime_t *t2) {
32 nstime_assert_initialized(t1);
33 nstime_assert_initialized(t2);
34}
35
36static void
37nstime_initialize_operand(nstime_t *time) {
38 /*
39 * Operations like nstime_add may have the initial operand being zero
40 * initialized (covered by the assert below). Full-initialize needed
41 * before changing it to non-zero.
42 */
43 nstime_assert_initialized(time);
44 nstime_set_initialized(time);
45}
46
47void
48nstime_init(nstime_t *time, uint64_t ns) {
49 nstime_set_initialized(time);
50 time->ns = ns;
51}
52
53void
54nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) {
55 nstime_set_initialized(time);
56 time->ns = sec * BILLION + nsec;
57}
58
59uint64_t
60nstime_ns(const nstime_t *time) {
61 nstime_assert_initialized(time);
62 return time->ns;
63}
64
65uint64_t
66nstime_msec(const nstime_t *time) {
67 nstime_assert_initialized(time);
68 return time->ns / MILLION;
69}
70
71uint64_t
72nstime_sec(const nstime_t *time) {
73 nstime_assert_initialized(time);
74 return time->ns / BILLION;
75}
76
77uint64_t
78nstime_nsec(const nstime_t *time) {
79 nstime_assert_initialized(time);
80 return time->ns % BILLION;
81}
82
83void
84nstime_copy(nstime_t *time, const nstime_t *source) {
85 /* Source is required to be initialized. */
86 nstime_assert_initialized(source);
87 *time = *source;
88 nstime_assert_initialized(time);
89}
90
91int
92nstime_compare(const nstime_t *a, const nstime_t *b) {
93 nstime_pair_assert_initialized(a, b);
94 return (a->ns > b->ns) - (a->ns < b->ns);
95}
96
97void
98nstime_add(nstime_t *time, const nstime_t *addend) {
99 nstime_pair_assert_initialized(time, addend);
100 assert(UINT64_MAX - time->ns >= addend->ns);
101
102 nstime_initialize_operand(time);
103 time->ns += addend->ns;
104}
105
106void
107nstime_iadd(nstime_t *time, uint64_t addend) {
108 nstime_assert_initialized(time);
109 assert(UINT64_MAX - time->ns >= addend);
110
111 nstime_initialize_operand(time);
112 time->ns += addend;
113}
114
115void
116nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
117 nstime_pair_assert_initialized(time, subtrahend);
118 assert(nstime_compare(time, subtrahend) >= 0);
119
120 /* No initialize operand -- subtraction must be initialized. */
121 time->ns -= subtrahend->ns;
122}
123
124void
125nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
126 nstime_assert_initialized(time);
127 assert(time->ns >= subtrahend);
128
129 /* No initialize operand -- subtraction must be initialized. */
130 time->ns -= subtrahend;
131}
132
133void
134nstime_imultiply(nstime_t *time, uint64_t multiplier) {
135 nstime_assert_initialized(time);
136 assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) <<
137 2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns));
138
139 nstime_initialize_operand(time);
140 time->ns *= multiplier;
141}
142
143void
144nstime_idivide(nstime_t *time, uint64_t divisor) {
145 nstime_assert_initialized(time);
146 assert(divisor != 0);
147
148 nstime_initialize_operand(time);
149 time->ns /= divisor;
150}
151
152uint64_t
153nstime_divide(const nstime_t *time, const nstime_t *divisor) {
154 nstime_pair_assert_initialized(time, divisor);
155 assert(divisor->ns != 0);
156
157 /* No initialize operand -- *time itself remains unchanged. */
158 return time->ns / divisor->ns;
159}
160
161/* Returns time since *past, w/o updating *past. */
162uint64_t
163nstime_ns_since(const nstime_t *past) {
164 nstime_assert_initialized(past);
165
166 nstime_t now;
167 nstime_copy(&now, past);
168 nstime_update(&now);
169
170 assert(nstime_compare(&now, past) >= 0);
171 return now.ns - past->ns;
172}
173
174#ifdef _WIN32
175# define NSTIME_MONOTONIC true
176static void
177nstime_get(nstime_t *time) {
178 FILETIME ft;
179 uint64_t ticks_100ns;
180
181 GetSystemTimeAsFileTime(&ft);
182 ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
183
184 nstime_init(time, ticks_100ns * 100);
185}
186#elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE)
187# define NSTIME_MONOTONIC true
188static void
189nstime_get(nstime_t *time) {
190 struct timespec ts;
191
192 clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
193 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
194}
195#elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC)
196# define NSTIME_MONOTONIC true
197static void
198nstime_get(nstime_t *time) {
199 struct timespec ts;
200
201 clock_gettime(CLOCK_MONOTONIC, &ts);
202 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
203}
204#elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME)
205# define NSTIME_MONOTONIC true
206static void
207nstime_get(nstime_t *time) {
208 nstime_init(time, mach_absolute_time());
209}
210#else
211# define NSTIME_MONOTONIC false
212static void
213nstime_get(nstime_t *time) {
214 struct timeval tv;
215
216 gettimeofday(&tv, NULL);
217 nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000);
218}
219#endif
220
221static bool
222nstime_monotonic_impl(void) {
223 return NSTIME_MONOTONIC;
224#undef NSTIME_MONOTONIC
225}
226nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl;
227
228prof_time_res_t opt_prof_time_res =
229 prof_time_res_default;
230
231const char *prof_time_res_mode_names[] = {
232 "default",
233 "high",
234};
235
236
237static void
238nstime_get_realtime(nstime_t *time) {
239#if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32)
240 struct timespec ts;
241
242 clock_gettime(CLOCK_REALTIME, &ts);
243 nstime_init2(time, ts.tv_sec, ts.tv_nsec);
244#else
245 unreachable();
246#endif
247}
248
249static void
250nstime_prof_update_impl(nstime_t *time) {
251 nstime_t old_time;
252
253 nstime_copy(&old_time, time);
254
255 if (opt_prof_time_res == prof_time_res_high) {
256 nstime_get_realtime(time);
257 } else {
258 nstime_get(time);
259 }
260}
261nstime_prof_update_t *JET_MUTABLE nstime_prof_update = nstime_prof_update_impl;
262
263static void
264nstime_update_impl(nstime_t *time) {
265 nstime_t old_time;
266
267 nstime_copy(&old_time, time);
268 nstime_get(time);
269
270 /* Handle non-monotonic clocks. */
271 if (unlikely(nstime_compare(&old_time, time) > 0)) {
272 nstime_copy(time, &old_time);
273 }
274}
275nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl;
276
277void
278nstime_init_update(nstime_t *time) {
279 nstime_init_zero(time);
280 nstime_update(time);
281}
282
283void
284nstime_prof_init_update(nstime_t *time) {
285 nstime_init_zero(time);
286 nstime_prof_update(time);
287}
288
289
290