1 | #ifndef JEMALLOC_INTERNAL_MUTEX_H |
2 | #define JEMALLOC_INTERNAL_MUTEX_H |
3 | |
4 | #include "jemalloc/internal/atomic.h" |
5 | #include "jemalloc/internal/mutex_prof.h" |
6 | #include "jemalloc/internal/tsd.h" |
7 | #include "jemalloc/internal/witness.h" |
8 | |
9 | typedef enum { |
10 | /* Can only acquire one mutex of a given witness rank at a time. */ |
11 | malloc_mutex_rank_exclusive, |
12 | /* |
13 | * Can acquire multiple mutexes of the same witness rank, but in |
14 | * address-ascending order only. |
15 | */ |
16 | malloc_mutex_address_ordered |
17 | } malloc_mutex_lock_order_t; |
18 | |
19 | typedef struct malloc_mutex_s malloc_mutex_t; |
20 | struct malloc_mutex_s { |
21 | union { |
22 | struct { |
23 | /* |
24 | * prof_data is defined first to reduce cacheline |
25 | * bouncing: the data is not touched by the mutex holder |
26 | * during unlocking, while might be modified by |
27 | * contenders. Having it before the mutex itself could |
28 | * avoid prefetching a modified cacheline (for the |
29 | * unlocking thread). |
30 | */ |
31 | mutex_prof_data_t prof_data; |
32 | #ifdef _WIN32 |
33 | # if _WIN32_WINNT >= 0x0600 |
34 | SRWLOCK lock; |
35 | # else |
36 | CRITICAL_SECTION lock; |
37 | # endif |
38 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
39 | os_unfair_lock lock; |
40 | #elif (defined(JEMALLOC_MUTEX_INIT_CB)) |
41 | pthread_mutex_t lock; |
42 | malloc_mutex_t *postponed_next; |
43 | #else |
44 | pthread_mutex_t lock; |
45 | #endif |
46 | /* |
47 | * Hint flag to avoid exclusive cache line contention |
48 | * during spin waiting |
49 | */ |
50 | atomic_b_t locked; |
51 | }; |
52 | /* |
53 | * We only touch witness when configured w/ debug. However we |
54 | * keep the field in a union when !debug so that we don't have |
55 | * to pollute the code base with #ifdefs, while avoid paying the |
56 | * memory cost. |
57 | */ |
58 | #if !defined(JEMALLOC_DEBUG) |
59 | witness_t witness; |
60 | malloc_mutex_lock_order_t lock_order; |
61 | #endif |
62 | }; |
63 | |
64 | #if defined(JEMALLOC_DEBUG) |
65 | witness_t witness; |
66 | malloc_mutex_lock_order_t lock_order; |
67 | #endif |
68 | }; |
69 | |
70 | /* |
71 | * Based on benchmark results, a fixed spin with this amount of retries works |
72 | * well for our critical sections. |
73 | */ |
74 | #define MALLOC_MUTEX_MAX_SPIN 250 |
75 | |
76 | #ifdef _WIN32 |
77 | # if _WIN32_WINNT >= 0x0600 |
78 | # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) |
79 | # define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) |
80 | # define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) |
81 | # else |
82 | # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) |
83 | # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) |
84 | # define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) |
85 | # endif |
86 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
87 | # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) |
88 | # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) |
89 | # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) |
90 | #else |
91 | # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) |
92 | # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) |
93 | # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) |
94 | #endif |
95 | |
96 | #define LOCK_PROF_DATA_INITIALIZER \ |
97 | {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ |
98 | ATOMIC_INIT(0), 0, NULL, 0} |
99 | |
100 | #ifdef _WIN32 |
101 | # define MALLOC_MUTEX_INITIALIZER |
102 | #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) |
103 | # if defined(JEMALLOC_DEBUG) |
104 | # define MALLOC_MUTEX_INITIALIZER \ |
105 | {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ |
106 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
107 | # else |
108 | # define MALLOC_MUTEX_INITIALIZER \ |
109 | {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ |
110 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
111 | # endif |
112 | #elif (defined(JEMALLOC_MUTEX_INIT_CB)) |
113 | # if (defined(JEMALLOC_DEBUG)) |
114 | # define MALLOC_MUTEX_INITIALIZER \ |
115 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ |
116 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
117 | # else |
118 | # define MALLOC_MUTEX_INITIALIZER \ |
119 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ |
120 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
121 | # endif |
122 | |
123 | #else |
124 | # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT |
125 | # if defined(JEMALLOC_DEBUG) |
126 | # define MALLOC_MUTEX_INITIALIZER \ |
127 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ |
128 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} |
129 | # else |
130 | # define MALLOC_MUTEX_INITIALIZER \ |
131 | {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ |
132 | WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} |
133 | # endif |
134 | #endif |
135 | |
136 | #ifdef JEMALLOC_LAZY_LOCK |
137 | extern bool isthreaded; |
138 | #else |
139 | # undef isthreaded /* Undo private_namespace.h definition. */ |
140 | # define isthreaded true |
141 | #endif |
142 | |
143 | bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, |
144 | witness_rank_t rank, malloc_mutex_lock_order_t lock_order); |
145 | void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); |
146 | void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); |
147 | void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); |
148 | bool malloc_mutex_boot(void); |
149 | void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); |
150 | |
151 | void malloc_mutex_lock_slow(malloc_mutex_t *mutex); |
152 | |
153 | static inline void |
154 | malloc_mutex_lock_final(malloc_mutex_t *mutex) { |
155 | MALLOC_MUTEX_LOCK(mutex); |
156 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
157 | } |
158 | |
159 | static inline bool |
160 | malloc_mutex_trylock_final(malloc_mutex_t *mutex) { |
161 | return MALLOC_MUTEX_TRYLOCK(mutex); |
162 | } |
163 | |
164 | static inline void |
165 | mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
166 | if (config_stats) { |
167 | mutex_prof_data_t *data = &mutex->prof_data; |
168 | data->n_lock_ops++; |
169 | if (data->prev_owner != tsdn) { |
170 | data->prev_owner = tsdn; |
171 | data->n_owner_switches++; |
172 | } |
173 | } |
174 | } |
175 | |
176 | /* Trylock: return false if the lock is successfully acquired. */ |
177 | static inline bool |
178 | malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
179 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
180 | if (isthreaded) { |
181 | if (malloc_mutex_trylock_final(mutex)) { |
182 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
183 | return true; |
184 | } |
185 | mutex_owner_stats_update(tsdn, mutex); |
186 | } |
187 | witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
188 | |
189 | return false; |
190 | } |
191 | |
192 | /* Aggregate lock prof data. */ |
193 | static inline void |
194 | malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { |
195 | nstime_add(&sum->tot_wait_time, &data->tot_wait_time); |
196 | if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { |
197 | nstime_copy(&sum->max_wait_time, &data->max_wait_time); |
198 | } |
199 | |
200 | sum->n_wait_times += data->n_wait_times; |
201 | sum->n_spin_acquired += data->n_spin_acquired; |
202 | |
203 | if (sum->max_n_thds < data->max_n_thds) { |
204 | sum->max_n_thds = data->max_n_thds; |
205 | } |
206 | uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, |
207 | ATOMIC_RELAXED); |
208 | uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( |
209 | &data->n_waiting_thds, ATOMIC_RELAXED); |
210 | atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, |
211 | ATOMIC_RELAXED); |
212 | sum->n_owner_switches += data->n_owner_switches; |
213 | sum->n_lock_ops += data->n_lock_ops; |
214 | } |
215 | |
216 | static inline void |
217 | malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
218 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
219 | if (isthreaded) { |
220 | if (malloc_mutex_trylock_final(mutex)) { |
221 | malloc_mutex_lock_slow(mutex); |
222 | atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); |
223 | } |
224 | mutex_owner_stats_update(tsdn, mutex); |
225 | } |
226 | witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
227 | } |
228 | |
229 | static inline void |
230 | malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
231 | atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); |
232 | witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
233 | if (isthreaded) { |
234 | MALLOC_MUTEX_UNLOCK(mutex); |
235 | } |
236 | } |
237 | |
238 | static inline void |
239 | malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
240 | witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
241 | } |
242 | |
243 | static inline void |
244 | malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { |
245 | witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); |
246 | } |
247 | |
248 | /* Copy the prof data from mutex for processing. */ |
249 | static inline void |
250 | malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, |
251 | malloc_mutex_t *mutex) { |
252 | mutex_prof_data_t *source = &mutex->prof_data; |
253 | /* Can only read holding the mutex. */ |
254 | malloc_mutex_assert_owner(tsdn, mutex); |
255 | |
256 | /* |
257 | * Not *really* allowed (we shouldn't be doing non-atomic loads of |
258 | * atomic data), but the mutex protection makes this safe, and writing |
259 | * a member-for-member copy is tedious for this situation. |
260 | */ |
261 | *data = *source; |
262 | /* n_wait_thds is not reported (modified w/o locking). */ |
263 | atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); |
264 | } |
265 | |
266 | static inline void |
267 | malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, |
268 | malloc_mutex_t *mutex) { |
269 | mutex_prof_data_t *source = &mutex->prof_data; |
270 | /* Can only read holding the mutex. */ |
271 | malloc_mutex_assert_owner(tsdn, mutex); |
272 | |
273 | nstime_add(&data->tot_wait_time, &source->tot_wait_time); |
274 | if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { |
275 | nstime_copy(&data->max_wait_time, &source->max_wait_time); |
276 | } |
277 | data->n_wait_times += source->n_wait_times; |
278 | data->n_spin_acquired += source->n_spin_acquired; |
279 | if (data->max_n_thds < source->max_n_thds) { |
280 | data->max_n_thds = source->max_n_thds; |
281 | } |
282 | /* n_wait_thds is not reported. */ |
283 | atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); |
284 | data->n_owner_switches += source->n_owner_switches; |
285 | data->n_lock_ops += source->n_lock_ops; |
286 | } |
287 | |
288 | #endif /* JEMALLOC_INTERNAL_MUTEX_H */ |
289 | |