1#ifndef JEMALLOC_INTERNAL_MUTEX_H
2#define JEMALLOC_INTERNAL_MUTEX_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/mutex_prof.h"
6#include "jemalloc/internal/tsd.h"
7#include "jemalloc/internal/witness.h"
8
9extern int64_t opt_mutex_max_spin;
10
11typedef enum {
12 /* Can only acquire one mutex of a given witness rank at a time. */
13 malloc_mutex_rank_exclusive,
14 /*
15 * Can acquire multiple mutexes of the same witness rank, but in
16 * address-ascending order only.
17 */
18 malloc_mutex_address_ordered
19} malloc_mutex_lock_order_t;
20
21typedef struct malloc_mutex_s malloc_mutex_t;
22struct malloc_mutex_s {
23 union {
24 struct {
25 /*
26 * prof_data is defined first to reduce cacheline
27 * bouncing: the data is not touched by the mutex holder
28 * during unlocking, while might be modified by
29 * contenders. Having it before the mutex itself could
30 * avoid prefetching a modified cacheline (for the
31 * unlocking thread).
32 */
33 mutex_prof_data_t prof_data;
34#ifdef _WIN32
35# if _WIN32_WINNT >= 0x0600
36 SRWLOCK lock;
37# else
38 CRITICAL_SECTION lock;
39# endif
40#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
41 os_unfair_lock lock;
42#elif (defined(JEMALLOC_MUTEX_INIT_CB))
43 pthread_mutex_t lock;
44 malloc_mutex_t *postponed_next;
45#else
46 pthread_mutex_t lock;
47#endif
48 /*
49 * Hint flag to avoid exclusive cache line contention
50 * during spin waiting
51 */
52 atomic_b_t locked;
53 };
54 /*
55 * We only touch witness when configured w/ debug. However we
56 * keep the field in a union when !debug so that we don't have
57 * to pollute the code base with #ifdefs, while avoid paying the
58 * memory cost.
59 */
60#if !defined(JEMALLOC_DEBUG)
61 witness_t witness;
62 malloc_mutex_lock_order_t lock_order;
63#endif
64 };
65
66#if defined(JEMALLOC_DEBUG)
67 witness_t witness;
68 malloc_mutex_lock_order_t lock_order;
69#endif
70};
71
72#ifdef _WIN32
73# if _WIN32_WINNT >= 0x0600
74# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
75# define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock)
76# define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
77# else
78# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
79# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
80# define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
81# endif
82#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
83# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
84# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
85# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
86#else
87# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
88# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
89# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
90#endif
91
92#define LOCK_PROF_DATA_INITIALIZER \
93 {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \
94 ATOMIC_INIT(0), 0, NULL, 0}
95
96#ifdef _WIN32
97# define MALLOC_MUTEX_INITIALIZER
98#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
99# if defined(JEMALLOC_DEBUG)
100# define MALLOC_MUTEX_INITIALIZER \
101 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
102 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
103# else
104# define MALLOC_MUTEX_INITIALIZER \
105 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
106 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
107# endif
108#elif (defined(JEMALLOC_MUTEX_INIT_CB))
109# if (defined(JEMALLOC_DEBUG))
110# define MALLOC_MUTEX_INITIALIZER \
111 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
112 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
113# else
114# define MALLOC_MUTEX_INITIALIZER \
115 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
116 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
117# endif
118
119#else
120# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
121# if defined(JEMALLOC_DEBUG)
122# define MALLOC_MUTEX_INITIALIZER \
123 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
124 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
125# else
126# define MALLOC_MUTEX_INITIALIZER \
127 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
128 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
129# endif
130#endif
131
132#ifdef JEMALLOC_LAZY_LOCK
133extern bool isthreaded;
134#else
135# undef isthreaded /* Undo private_namespace.h definition. */
136# define isthreaded true
137#endif
138
139bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
140 witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
141void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
142void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
143void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
144bool malloc_mutex_boot(void);
145void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
146
147void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
148
149static inline void
150malloc_mutex_lock_final(malloc_mutex_t *mutex) {
151 MALLOC_MUTEX_LOCK(mutex);
152 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
153}
154
155static inline bool
156malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
157 return MALLOC_MUTEX_TRYLOCK(mutex);
158}
159
160static inline void
161mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
162 if (config_stats) {
163 mutex_prof_data_t *data = &mutex->prof_data;
164 data->n_lock_ops++;
165 if (data->prev_owner != tsdn) {
166 data->prev_owner = tsdn;
167 data->n_owner_switches++;
168 }
169 }
170}
171
172/* Trylock: return false if the lock is successfully acquired. */
173static inline bool
174malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
175 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
176 if (isthreaded) {
177 if (malloc_mutex_trylock_final(mutex)) {
178 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
179 return true;
180 }
181 mutex_owner_stats_update(tsdn, mutex);
182 }
183 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
184
185 return false;
186}
187
188/* Aggregate lock prof data. */
189static inline void
190malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
191 nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
192 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
193 nstime_copy(&sum->max_wait_time, &data->max_wait_time);
194 }
195
196 sum->n_wait_times += data->n_wait_times;
197 sum->n_spin_acquired += data->n_spin_acquired;
198
199 if (sum->max_n_thds < data->max_n_thds) {
200 sum->max_n_thds = data->max_n_thds;
201 }
202 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
203 ATOMIC_RELAXED);
204 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
205 &data->n_waiting_thds, ATOMIC_RELAXED);
206 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
207 ATOMIC_RELAXED);
208 sum->n_owner_switches += data->n_owner_switches;
209 sum->n_lock_ops += data->n_lock_ops;
210}
211
212static inline void
213malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
214 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
215 if (isthreaded) {
216 if (malloc_mutex_trylock_final(mutex)) {
217 malloc_mutex_lock_slow(mutex);
218 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
219 }
220 mutex_owner_stats_update(tsdn, mutex);
221 }
222 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
223}
224
225static inline void
226malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
227 atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
228 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
229 if (isthreaded) {
230 MALLOC_MUTEX_UNLOCK(mutex);
231 }
232}
233
234static inline void
235malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
236 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
237}
238
239static inline void
240malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
241 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
242}
243
244static inline void
245malloc_mutex_prof_copy(mutex_prof_data_t *dst, mutex_prof_data_t *source) {
246 /*
247 * Not *really* allowed (we shouldn't be doing non-atomic loads of
248 * atomic data), but the mutex protection makes this safe, and writing
249 * a member-for-member copy is tedious for this situation.
250 */
251 *dst = *source;
252 /* n_wait_thds is not reported (modified w/o locking). */
253 atomic_store_u32(&dst->n_waiting_thds, 0, ATOMIC_RELAXED);
254}
255
256/* Copy the prof data from mutex for processing. */
257static inline void
258malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
259 malloc_mutex_t *mutex) {
260 /* Can only read holding the mutex. */
261 malloc_mutex_assert_owner(tsdn, mutex);
262 malloc_mutex_prof_copy(data, &mutex->prof_data);
263}
264
265static inline void
266malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
267 malloc_mutex_t *mutex) {
268 mutex_prof_data_t *source = &mutex->prof_data;
269 /* Can only read holding the mutex. */
270 malloc_mutex_assert_owner(tsdn, mutex);
271
272 nstime_add(&data->tot_wait_time, &source->tot_wait_time);
273 if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
274 nstime_copy(&data->max_wait_time, &source->max_wait_time);
275 }
276 data->n_wait_times += source->n_wait_times;
277 data->n_spin_acquired += source->n_spin_acquired;
278 if (data->max_n_thds < source->max_n_thds) {
279 data->max_n_thds = source->max_n_thds;
280 }
281 /* n_wait_thds is not reported. */
282 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
283 data->n_owner_switches += source->n_owner_switches;
284 data->n_lock_ops += source->n_lock_ops;
285}
286
287/* Compare the prof data and update to the maximum. */
288static inline void
289malloc_mutex_prof_max_update(tsdn_t *tsdn, mutex_prof_data_t *data,
290 malloc_mutex_t *mutex) {
291 mutex_prof_data_t *source = &mutex->prof_data;
292 /* Can only read holding the mutex. */
293 malloc_mutex_assert_owner(tsdn, mutex);
294
295 if (nstime_compare(&source->tot_wait_time, &data->tot_wait_time) > 0) {
296 nstime_copy(&data->tot_wait_time, &source->tot_wait_time);
297 }
298 if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
299 nstime_copy(&data->max_wait_time, &source->max_wait_time);
300 }
301 if (source->n_wait_times > data->n_wait_times) {
302 data->n_wait_times = source->n_wait_times;
303 }
304 if (source->n_spin_acquired > data->n_spin_acquired) {
305 data->n_spin_acquired = source->n_spin_acquired;
306 }
307 if (source->max_n_thds > data->max_n_thds) {
308 data->max_n_thds = source->max_n_thds;
309 }
310 if (source->n_owner_switches > data->n_owner_switches) {
311 data->n_owner_switches = source->n_owner_switches;
312 }
313 if (source->n_lock_ops > data->n_lock_ops) {
314 data->n_lock_ops = source->n_lock_ops;
315 }
316 /* n_wait_thds is not reported. */
317}
318
319#endif /* JEMALLOC_INTERNAL_MUTEX_H */
320