1 | #ifndef JEMALLOC_INTERNAL_MUTEX_H |
2 | #define JEMALLOC_INTERNAL_MUTEX_H |
3 | |
4 | #include "jemalloc/internal/atomic.h" |
5 | #include "jemalloc/internal/mutex_prof.h" |
6 | #include "jemalloc/internal/tsd.h" |
7 | #include "jemalloc/internal/witness.h" |
8 | |
9 | extern int64_t opt_mutex_max_spin; |
10 | |
11 | typedef enum { |
12 | /* Can only acquire one mutex of a given witness rank at a time. */ |
13 | malloc_mutex_rank_exclusive, |
14 | /* |
15 | * Can acquire multiple mutexes of the same witness rank, but in |
16 | * address-ascending order only. |
17 | */ |
18 | malloc_mutex_address_ordered |
19 | } malloc_mutex_lock_order_t; |
20 | |
21 | typedef struct malloc_mutex_s malloc_mutex_t; |
22 | struct malloc_mutex_s { |
23 | union { |
24 | struct { |
25 | /* |
26 | * prof_data is defined first to reduce cacheline |
27 | * bouncing: the data is not touched by the mutex holder |
28 | * during unlocking, while might be modified by |
29 | * contenders. Having it before the mutex itself could |
30 | * avoid prefetching a modified cacheline (for the |
31 | * unlocking thread). |
32 | */ |
33 | mutex_prof_data_t prof_data; |
34 | #ifdef _WIN32 |
35 | # if _WIN32_WINNT >= 0x0600 |
36 | SRWLOCK lock; |
37 | # else |
38 | CRITICAL_SECTION lock; |
39 | # endif |
40 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
41 | os_unfair_lock lock; |
42 | #elif (defined(JEMALLOC_MUTEX_INIT_CB)) |
43 | pthread_mutex_t lock; |
44 | malloc_mutex_t *postponed_next; |
45 | #else |
46 | pthread_mutex_t lock; |
47 | #endif |
48 | /* |
49 | * Hint flag to avoid exclusive cache line contention |
50 | * during spin waiting |
51 | */ |
52 | atomic_b_t locked; |
53 | }; |
54 | /* |
55 | * We only touch witness when configured w/ debug. However we |
56 | * keep the field in a union when !debug so that we don't have |
57 | * to pollute the code base with #ifdefs, while avoid paying the |
58 | * memory cost. |
59 | */ |
60 | #if !defined(JEMALLOC_DEBUG) |
61 | witness_t witness; |
62 | malloc_mutex_lock_order_t lock_order; |
63 | #endif |
64 | }; |
65 | |
66 | #if defined(JEMALLOC_DEBUG) |
67 | witness_t witness; |
68 | malloc_mutex_lock_order_t lock_order; |
69 | #endif |
70 | }; |
71 | |
72 | #ifdef _WIN32 |
73 | # if _WIN32_WINNT >= 0x0600 |
74 | # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) |
75 | # define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) |
76 | # define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) |
77 | # else |
78 | # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) |
79 | # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) |
80 | # define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) |
81 | # endif |
82 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
83 | # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) |
84 | # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) |
85 | # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) |
86 | #else |
87 | # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) |
88 | # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) |
89 | # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) |
90 | #endif |
91 | |
92 | #define LOCK_PROF_DATA_INITIALIZER \ |
93 | {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ |
94 | ATOMIC_INIT(0), 0, NULL, 0} |
95 | |
96 | #ifdef _WIN32 |
97 | # define MALLOC_MUTEX_INITIALIZER |
98 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
99 | # if defined(JEMALLOC_DEBUG) |
100 | # define MALLOC_MUTEX_INITIALIZER \ |
101 | {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ |
102 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
103 | # else |
104 | # define MALLOC_MUTEX_INITIALIZER \ |
105 | {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ |
106 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
107 | # endif |
108 | #elif (defined(JEMALLOC_MUTEX_INIT_CB)) |
109 | # if (defined(JEMALLOC_DEBUG)) |
110 | # define MALLOC_MUTEX_INITIALIZER \ |
111 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ |
112 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
113 | # else |
114 | # define MALLOC_MUTEX_INITIALIZER \ |
115 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ |
116 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
117 | # endif |
118 | |
119 | #else |
120 | # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT |
121 | # if defined(JEMALLOC_DEBUG) |
122 | # define MALLOC_MUTEX_INITIALIZER \ |
123 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ |
124 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
125 | # else |
126 | # define MALLOC_MUTEX_INITIALIZER \ |
127 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ |
128 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
129 | # endif |
130 | #endif |
131 | |
132 | #ifdef JEMALLOC_LAZY_LOCK |
133 | extern bool isthreaded; |
134 | #else |
135 | # undef isthreaded /* Undo private_namespace.h definition. */ |
136 | # define isthreaded true |
137 | #endif |
138 | |
139 | bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, |
140 | witness_rank_t rank, malloc_mutex_lock_order_t lock_order); |
141 | void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); |
142 | void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); |
143 | void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); |
144 | bool malloc_mutex_boot(void); |
145 | void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); |
146 | |
147 | void malloc_mutex_lock_slow(malloc_mutex_t *mutex); |
148 | |
149 | static inline void |
150 | malloc_mutex_lock_final(malloc_mutex_t *mutex) { |
151 | MALLOC_MUTEX_LOCK(mutex); |
152 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
153 | } |
154 | |
155 | static inline bool |
156 | malloc_mutex_trylock_final(malloc_mutex_t *mutex) { |
157 | return MALLOC_MUTEX_TRYLOCK(mutex); |
158 | } |
159 | |
160 | static inline void |
161 | mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
162 | if (config_stats) { |
163 | mutex_prof_data_t *data = &mutex->prof_data; |
164 | data->n_lock_ops++; |
165 | if (data->prev_owner != tsdn) { |
166 | data->prev_owner = tsdn; |
167 | data->n_owner_switches++; |
168 | } |
169 | } |
170 | } |
171 | |
172 | /* Trylock: return false if the lock is successfully acquired. */ |
173 | static inline bool |
174 | malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
175 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
176 | if (isthreaded) { |
177 | if (malloc_mutex_trylock_final(mutex)) { |
178 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
179 | return true; |
180 | } |
181 | mutex_owner_stats_update(tsdn, mutex); |
182 | } |
183 | witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
184 | |
185 | return false; |
186 | } |
187 | |
188 | /* Aggregate lock prof data. */ |
189 | static inline void |
190 | malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { |
191 | nstime_add(&sum->tot_wait_time, &data->tot_wait_time); |
192 | if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { |
193 | nstime_copy(&sum->max_wait_time, &data->max_wait_time); |
194 | } |
195 | |
196 | sum->n_wait_times += data->n_wait_times; |
197 | sum->n_spin_acquired += data->n_spin_acquired; |
198 | |
199 | if (sum->max_n_thds < data->max_n_thds) { |
200 | sum->max_n_thds = data->max_n_thds; |
201 | } |
202 | uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, |
203 | ATOMIC_RELAXED); |
204 | uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( |
205 | &data->n_waiting_thds, ATOMIC_RELAXED); |
206 | atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, |
207 | ATOMIC_RELAXED); |
208 | sum->n_owner_switches += data->n_owner_switches; |
209 | sum->n_lock_ops += data->n_lock_ops; |
210 | } |
211 | |
212 | static inline void |
213 | malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
214 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
215 | if (isthreaded) { |
216 | if (malloc_mutex_trylock_final(mutex)) { |
217 | malloc_mutex_lock_slow(mutex); |
218 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
219 | } |
220 | mutex_owner_stats_update(tsdn, mutex); |
221 | } |
222 | witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
223 | } |
224 | |
225 | static inline void |
226 | malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
227 | atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); |
228 | witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
229 | if (isthreaded) { |
230 | MALLOC_MUTEX_UNLOCK(mutex); |
231 | } |
232 | } |
233 | |
234 | static inline void |
235 | malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
236 | witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
237 | } |
238 | |
239 | static inline void |
240 | malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
241 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
242 | } |
243 | |
244 | static inline void |
245 | malloc_mutex_prof_copy(mutex_prof_data_t *dst, mutex_prof_data_t *source) { |
246 | /* |
247 | * Not *really* allowed (we shouldn't be doing non-atomic loads of |
248 | * atomic data), but the mutex protection makes this safe, and writing |
249 | * a member-for-member copy is tedious for this situation. |
250 | */ |
251 | *dst = *source; |
252 | /* n_wait_thds is not reported (modified w/o locking). */ |
253 | atomic_store_u32(&dst->n_waiting_thds, 0, ATOMIC_RELAXED); |
254 | } |
255 | |
256 | /* Copy the prof data from mutex for processing. */ |
257 | static inline void |
258 | malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, |
259 | malloc_mutex_t *mutex) { |
260 | /* Can only read holding the mutex. */ |
261 | malloc_mutex_assert_owner(tsdn, mutex); |
262 | malloc_mutex_prof_copy(data, &mutex->prof_data); |
263 | } |
264 | |
265 | static inline void |
266 | malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, |
267 | malloc_mutex_t *mutex) { |
268 | mutex_prof_data_t *source = &mutex->prof_data; |
269 | /* Can only read holding the mutex. */ |
270 | malloc_mutex_assert_owner(tsdn, mutex); |
271 | |
272 | nstime_add(&data->tot_wait_time, &source->tot_wait_time); |
273 | if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { |
274 | nstime_copy(&data->max_wait_time, &source->max_wait_time); |
275 | } |
276 | data->n_wait_times += source->n_wait_times; |
277 | data->n_spin_acquired += source->n_spin_acquired; |
278 | if (data->max_n_thds < source->max_n_thds) { |
279 | data->max_n_thds = source->max_n_thds; |
280 | } |
281 | /* n_wait_thds is not reported. */ |
282 | atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); |
283 | data->n_owner_switches += source->n_owner_switches; |
284 | data->n_lock_ops += source->n_lock_ops; |
285 | } |
286 | |
287 | /* Compare the prof data and update to the maximum. */ |
288 | static inline void |
289 | malloc_mutex_prof_max_update(tsdn_t *tsdn, mutex_prof_data_t *data, |
290 | malloc_mutex_t *mutex) { |
291 | mutex_prof_data_t *source = &mutex->prof_data; |
292 | /* Can only read holding the mutex. */ |
293 | malloc_mutex_assert_owner(tsdn, mutex); |
294 | |
295 | if (nstime_compare(&source->tot_wait_time, &data->tot_wait_time) > 0) { |
296 | nstime_copy(&data->tot_wait_time, &source->tot_wait_time); |
297 | } |
298 | if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { |
299 | nstime_copy(&data->max_wait_time, &source->max_wait_time); |
300 | } |
301 | if (source->n_wait_times > data->n_wait_times) { |
302 | data->n_wait_times = source->n_wait_times; |
303 | } |
304 | if (source->n_spin_acquired > data->n_spin_acquired) { |
305 | data->n_spin_acquired = source->n_spin_acquired; |
306 | } |
307 | if (source->max_n_thds > data->max_n_thds) { |
308 | data->max_n_thds = source->max_n_thds; |
309 | } |
310 | if (source->n_owner_switches > data->n_owner_switches) { |
311 | data->n_owner_switches = source->n_owner_switches; |
312 | } |
313 | if (source->n_lock_ops > data->n_lock_ops) { |
314 | data->n_lock_ops = source->n_lock_ops; |
315 | } |
316 | /* n_wait_thds is not reported. */ |
317 | } |
318 | |
319 | #endif /* JEMALLOC_INTERNAL_MUTEX_H */ |
320 | |