1#ifndef JEMALLOC_INTERNAL_MUTEX_H
2#define JEMALLOC_INTERNAL_MUTEX_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/mutex_prof.h"
6#include "jemalloc/internal/tsd.h"
7#include "jemalloc/internal/witness.h"
8
9typedef enum {
10 /* Can only acquire one mutex of a given witness rank at a time. */
11 malloc_mutex_rank_exclusive,
12 /*
13 * Can acquire multiple mutexes of the same witness rank, but in
14 * address-ascending order only.
15 */
16 malloc_mutex_address_ordered
17} malloc_mutex_lock_order_t;
18
19typedef struct malloc_mutex_s malloc_mutex_t;
20struct malloc_mutex_s {
21 union {
22 struct {
23 /*
24 * prof_data is defined first to reduce cacheline
25 * bouncing: the data is not touched by the mutex holder
26 * during unlocking, while might be modified by
27 * contenders. Having it before the mutex itself could
28 * avoid prefetching a modified cacheline (for the
29 * unlocking thread).
30 */
31 mutex_prof_data_t prof_data;
32#ifdef _WIN32
33# if _WIN32_WINNT >= 0x0600
34 SRWLOCK lock;
35# else
36 CRITICAL_SECTION lock;
37# endif
38#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
39 os_unfair_lock lock;
40#elif (defined(JEMALLOC_MUTEX_INIT_CB))
41 pthread_mutex_t lock;
42 malloc_mutex_t *postponed_next;
43#else
44 pthread_mutex_t lock;
45#endif
46 /*
47 * Hint flag to avoid exclusive cache line contention
48 * during spin waiting
49 */
50 atomic_b_t locked;
51 };
52 /*
53 * We only touch witness when configured w/ debug. However we
54 * keep the field in a union when !debug so that we don't have
55 * to pollute the code base with #ifdefs, while avoid paying the
56 * memory cost.
57 */
58#if !defined(JEMALLOC_DEBUG)
59 witness_t witness;
60 malloc_mutex_lock_order_t lock_order;
61#endif
62 };
63
64#if defined(JEMALLOC_DEBUG)
65 witness_t witness;
66 malloc_mutex_lock_order_t lock_order;
67#endif
68};
69
70/*
71 * Based on benchmark results, a fixed spin with this amount of retries works
72 * well for our critical sections.
73 */
74#define MALLOC_MUTEX_MAX_SPIN 250
75
76#ifdef _WIN32
77# if _WIN32_WINNT >= 0x0600
78# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
79# define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock)
80# define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
81# else
82# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
83# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
84# define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
85# endif
86#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
87# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
88# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
89# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
90#else
91# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
92# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
93# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
94#endif
95
96#define LOCK_PROF_DATA_INITIALIZER \
97 {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \
98 ATOMIC_INIT(0), 0, NULL, 0}
99
100#ifdef _WIN32
101# define MALLOC_MUTEX_INITIALIZER
102#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
103# if defined(JEMALLOC_DEBUG)
104# define MALLOC_MUTEX_INITIALIZER \
105 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
106 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
107# else
108# define MALLOC_MUTEX_INITIALIZER \
109 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
110 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
111# endif
112#elif (defined(JEMALLOC_MUTEX_INIT_CB))
113# if (defined(JEMALLOC_DEBUG))
114# define MALLOC_MUTEX_INITIALIZER \
115 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
116 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
117# else
118# define MALLOC_MUTEX_INITIALIZER \
119 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
120 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
121# endif
122
123#else
124# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
125# if defined(JEMALLOC_DEBUG)
126# define MALLOC_MUTEX_INITIALIZER \
127 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
128 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
129# else
130# define MALLOC_MUTEX_INITIALIZER \
131 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
132 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
133# endif
134#endif
135
136#ifdef JEMALLOC_LAZY_LOCK
137extern bool isthreaded;
138#else
139# undef isthreaded /* Undo private_namespace.h definition. */
140# define isthreaded true
141#endif
142
143bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
144 witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
145void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
146void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
147void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
148bool malloc_mutex_boot(void);
149void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
150
151void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
152
153static inline void
154malloc_mutex_lock_final(malloc_mutex_t *mutex) {
155 MALLOC_MUTEX_LOCK(mutex);
156 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
157}
158
159static inline bool
160malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
161 return MALLOC_MUTEX_TRYLOCK(mutex);
162}
163
164static inline void
165mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
166 if (config_stats) {
167 mutex_prof_data_t *data = &mutex->prof_data;
168 data->n_lock_ops++;
169 if (data->prev_owner != tsdn) {
170 data->prev_owner = tsdn;
171 data->n_owner_switches++;
172 }
173 }
174}
175
176/* Trylock: return false if the lock is successfully acquired. */
177static inline bool
178malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
179 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
180 if (isthreaded) {
181 if (malloc_mutex_trylock_final(mutex)) {
182 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
183 return true;
184 }
185 mutex_owner_stats_update(tsdn, mutex);
186 }
187 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
188
189 return false;
190}
191
192/* Aggregate lock prof data. */
193static inline void
194malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
195 nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
196 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
197 nstime_copy(&sum->max_wait_time, &data->max_wait_time);
198 }
199
200 sum->n_wait_times += data->n_wait_times;
201 sum->n_spin_acquired += data->n_spin_acquired;
202
203 if (sum->max_n_thds < data->max_n_thds) {
204 sum->max_n_thds = data->max_n_thds;
205 }
206 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
207 ATOMIC_RELAXED);
208 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
209 &data->n_waiting_thds, ATOMIC_RELAXED);
210 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
211 ATOMIC_RELAXED);
212 sum->n_owner_switches += data->n_owner_switches;
213 sum->n_lock_ops += data->n_lock_ops;
214}
215
216static inline void
217malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
218 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
219 if (isthreaded) {
220 if (malloc_mutex_trylock_final(mutex)) {
221 malloc_mutex_lock_slow(mutex);
222 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
223 }
224 mutex_owner_stats_update(tsdn, mutex);
225 }
226 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
227}
228
229static inline void
230malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
231 atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
232 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
233 if (isthreaded) {
234 MALLOC_MUTEX_UNLOCK(mutex);
235 }
236}
237
238static inline void
239malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
240 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
241}
242
243static inline void
244malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
245 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
246}
247
248/* Copy the prof data from mutex for processing. */
249static inline void
250malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
251 malloc_mutex_t *mutex) {
252 mutex_prof_data_t *source = &mutex->prof_data;
253 /* Can only read holding the mutex. */
254 malloc_mutex_assert_owner(tsdn, mutex);
255
256 /*
257 * Not *really* allowed (we shouldn't be doing non-atomic loads of
258 * atomic data), but the mutex protection makes this safe, and writing
259 * a member-for-member copy is tedious for this situation.
260 */
261 *data = *source;
262 /* n_wait_thds is not reported (modified w/o locking). */
263 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
264}
265
266static inline void
267malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
268 malloc_mutex_t *mutex) {
269 mutex_prof_data_t *source = &mutex->prof_data;
270 /* Can only read holding the mutex. */
271 malloc_mutex_assert_owner(tsdn, mutex);
272
273 nstime_add(&data->tot_wait_time, &source->tot_wait_time);
274 if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
275 nstime_copy(&data->max_wait_time, &source->max_wait_time);
276 }
277 data->n_wait_times += source->n_wait_times;
278 data->n_spin_acquired += source->n_spin_acquired;
279 if (data->max_n_thds < source->max_n_thds) {
280 data->max_n_thds = source->max_n_thds;
281 }
282 /* n_wait_thds is not reported. */
283 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
284 data->n_owner_switches += source->n_owner_switches;
285 data->n_lock_ops += source->n_lock_ops;
286}
287
288#endif /* JEMALLOC_INTERNAL_MUTEX_H */
289