1/*
2 * Implementation of the Global Interpreter Lock (GIL).
3 */
4
5#include <stdlib.h>
6#include <errno.h>
7
8#include "pycore_atomic.h"
9
10
11/*
12 Notes about the implementation:
13
14 - The GIL is just a boolean variable (locked) whose access is protected
15 by a mutex (gil_mutex), and whose changes are signalled by a condition
16 variable (gil_cond). gil_mutex is taken for short periods of time,
17 and therefore mostly uncontended.
18
19 - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
20 able to release the GIL on demand by another thread. A volatile boolean
21 variable (gil_drop_request) is used for that purpose, which is checked
22 at every turn of the eval loop. That variable is set after a wait of
23 `interval` microseconds on `gil_cond` has timed out.
24
25 [Actually, another volatile boolean variable (eval_breaker) is used
26 which ORs several conditions into one. Volatile booleans are
27 sufficient as inter-thread signalling means since Python is run
28 on cache-coherent architectures only.]
29
30 - A thread wanting to take the GIL will first let pass a given amount of
31 time (`interval` microseconds) before setting gil_drop_request. This
32 encourages a defined switching period, but doesn't enforce it since
33 opcodes can take an arbitrary time to execute.
34
35 The `interval` value is available for the user to read and modify
36 using the Python API `sys.{get,set}switchinterval()`.
37
38 - When a thread releases the GIL and gil_drop_request is set, that thread
39 ensures that another GIL-awaiting thread gets scheduled.
40 It does so by waiting on a condition variable (switch_cond) until
41 the value of last_holder is changed to something else than its
42 own thread state pointer, indicating that another thread was able to
43 take the GIL.
44
45 This is meant to prohibit the latency-adverse behaviour on multi-core
46 machines where one thread would speculatively release the GIL, but still
47 run and end up being the first to re-acquire it, making the "timeslices"
48 much longer than expected.
49 (Note: this mechanism is enabled with FORCE_SWITCHING above)
50*/
51
52#include "condvar.h"
53
54#define MUTEX_INIT(mut) \
55 if (PyMUTEX_INIT(&(mut))) { \
56 Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
57#define MUTEX_FINI(mut) \
58 if (PyMUTEX_FINI(&(mut))) { \
59 Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
60#define MUTEX_LOCK(mut) \
61 if (PyMUTEX_LOCK(&(mut))) { \
62 Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
63#define MUTEX_UNLOCK(mut) \
64 if (PyMUTEX_UNLOCK(&(mut))) { \
65 Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
66
67#define COND_INIT(cond) \
68 if (PyCOND_INIT(&(cond))) { \
69 Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
70#define COND_FINI(cond) \
71 if (PyCOND_FINI(&(cond))) { \
72 Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
73#define COND_SIGNAL(cond) \
74 if (PyCOND_SIGNAL(&(cond))) { \
75 Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
76#define COND_WAIT(cond, mut) \
77 if (PyCOND_WAIT(&(cond), &(mut))) { \
78 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
79#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
80 { \
81 int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
82 if (r < 0) \
83 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
84 if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
85 timeout_result = 1; \
86 else \
87 timeout_result = 0; \
88 } \
89
90
91#define DEFAULT_INTERVAL 5000
92
93static void _gil_initialize(struct _gil_runtime_state *gil)
94{
95 _Py_atomic_int uninitialized = {-1};
96 gil->locked = uninitialized;
97 gil->interval = DEFAULT_INTERVAL;
98}
99
100static int gil_created(struct _gil_runtime_state *gil)
101{
102 return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0);
103}
104
105static void create_gil(struct _gil_runtime_state *gil)
106{
107 MUTEX_INIT(gil->mutex);
108#ifdef FORCE_SWITCHING
109 MUTEX_INIT(gil->switch_mutex);
110#endif
111 COND_INIT(gil->cond);
112#ifdef FORCE_SWITCHING
113 COND_INIT(gil->switch_cond);
114#endif
115 _Py_atomic_store_relaxed(&gil->last_holder, 0);
116 _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);
117 _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release);
118}
119
120static void destroy_gil(struct _gil_runtime_state *gil)
121{
122 /* some pthread-like implementations tie the mutex to the cond
123 * and must have the cond destroyed first.
124 */
125 COND_FINI(gil->cond);
126 MUTEX_FINI(gil->mutex);
127#ifdef FORCE_SWITCHING
128 COND_FINI(gil->switch_cond);
129 MUTEX_FINI(gil->switch_mutex);
130#endif
131 _Py_atomic_store_explicit(&gil->locked, -1,
132 _Py_memory_order_release);
133 _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
134}
135
136static void recreate_gil(struct _gil_runtime_state *gil)
137{
138 _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
139 /* XXX should we destroy the old OS resources here? */
140 create_gil(gil);
141}
142
143static void
144drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,
145 PyThreadState *tstate)
146{
147#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
148 struct _gil_runtime_state *gil = &ceval2->gil;
149#else
150 struct _gil_runtime_state *gil = &ceval->gil;
151#endif
152 if (!_Py_atomic_load_relaxed(&gil->locked)) {
153 Py_FatalError("drop_gil: GIL is not locked");
154 }
155
156 /* tstate is allowed to be NULL (early interpreter init) */
157 if (tstate != NULL) {
158 /* Sub-interpreter support: threads might have been switched
159 under our feet using PyThreadState_Swap(). Fix the GIL last
160 holder variable so that our heuristics work. */
161 _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
162 }
163
164 MUTEX_LOCK(gil->mutex);
165 _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);
166 _Py_atomic_store_relaxed(&gil->locked, 0);
167 COND_SIGNAL(gil->cond);
168 MUTEX_UNLOCK(gil->mutex);
169
170#ifdef FORCE_SWITCHING
171 if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) {
172 MUTEX_LOCK(gil->switch_mutex);
173 /* Not switched yet => wait */
174 if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate)
175 {
176 assert(is_tstate_valid(tstate));
177 RESET_GIL_DROP_REQUEST(tstate->interp);
178 /* NOTE: if COND_WAIT does not atomically start waiting when
179 releasing the mutex, another thread can run through, take
180 the GIL and drop it again, and reset the condition
181 before we even had a chance to wait for it. */
182 COND_WAIT(gil->switch_cond, gil->switch_mutex);
183 }
184 MUTEX_UNLOCK(gil->switch_mutex);
185 }
186#endif
187}
188
189
190/* Check if a Python thread must exit immediately, rather than taking the GIL
191 if Py_Finalize() has been called.
192
193 When this function is called by a daemon thread after Py_Finalize() has been
194 called, the GIL does no longer exist.
195
196 tstate must be non-NULL. */
197static inline int
198tstate_must_exit(PyThreadState *tstate)
199{
200 /* bpo-39877: Access _PyRuntime directly rather than using
201 tstate->interp->runtime to support calls from Python daemon threads.
202 After Py_Finalize() has been called, tstate can be a dangling pointer:
203 point to PyThreadState freed memory. */
204 PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime);
205 return (finalizing != NULL && finalizing != tstate);
206}
207
208
209/* Take the GIL.
210
211 The function saves errno at entry and restores its value at exit.
212
213 tstate must be non-NULL. */
214static void
215take_gil(PyThreadState *tstate)
216{
217 int err = errno;
218
219 assert(tstate != NULL);
220
221 if (tstate_must_exit(tstate)) {
222 /* bpo-39877: If Py_Finalize() has been called and tstate is not the
223 thread which called Py_Finalize(), exit immediately the thread.
224
225 This code path can be reached by a daemon thread after Py_Finalize()
226 completes. In this case, tstate is a dangling pointer: points to
227 PyThreadState freed memory. */
228 PyThread_exit_thread();
229 }
230
231 assert(is_tstate_valid(tstate));
232 PyInterpreterState *interp = tstate->interp;
233 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
234 struct _ceval_state *ceval2 = &interp->ceval;
235#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
236 struct _gil_runtime_state *gil = &ceval2->gil;
237#else
238 struct _gil_runtime_state *gil = &ceval->gil;
239#endif
240
241 /* Check that _PyEval_InitThreads() was called to create the lock */
242 assert(gil_created(gil));
243
244 MUTEX_LOCK(gil->mutex);
245
246 if (!_Py_atomic_load_relaxed(&gil->locked)) {
247 goto _ready;
248 }
249
250 while (_Py_atomic_load_relaxed(&gil->locked)) {
251 unsigned long saved_switchnum = gil->switch_number;
252
253 unsigned long interval = (gil->interval >= 1 ? gil->interval : 1);
254 int timed_out = 0;
255 COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);
256
257 /* If we timed out and no switch occurred in the meantime, it is time
258 to ask the GIL-holding thread to drop it. */
259 if (timed_out &&
260 _Py_atomic_load_relaxed(&gil->locked) &&
261 gil->switch_number == saved_switchnum)
262 {
263 if (tstate_must_exit(tstate)) {
264 MUTEX_UNLOCK(gil->mutex);
265 PyThread_exit_thread();
266 }
267 assert(is_tstate_valid(tstate));
268
269 SET_GIL_DROP_REQUEST(interp);
270 }
271 }
272
273_ready:
274#ifdef FORCE_SWITCHING
275 /* This mutex must be taken before modifying gil->last_holder:
276 see drop_gil(). */
277 MUTEX_LOCK(gil->switch_mutex);
278#endif
279 /* We now hold the GIL */
280 _Py_atomic_store_relaxed(&gil->locked, 1);
281 _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);
282
283 if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) {
284 _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
285 ++gil->switch_number;
286 }
287
288#ifdef FORCE_SWITCHING
289 COND_SIGNAL(gil->switch_cond);
290 MUTEX_UNLOCK(gil->switch_mutex);
291#endif
292
293 if (tstate_must_exit(tstate)) {
294 /* bpo-36475: If Py_Finalize() has been called and tstate is not
295 the thread which called Py_Finalize(), exit immediately the
296 thread.
297
298 This code path can be reached by a daemon thread which was waiting
299 in take_gil() while the main thread called
300 wait_for_thread_shutdown() from Py_Finalize(). */
301 MUTEX_UNLOCK(gil->mutex);
302 drop_gil(ceval, ceval2, tstate);
303 PyThread_exit_thread();
304 }
305 assert(is_tstate_valid(tstate));
306
307 if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {
308 RESET_GIL_DROP_REQUEST(interp);
309 }
310 else {
311 /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there
312 is a pending signal: signal received by another thread which cannot
313 handle signals.
314
315 Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */
316 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
317 }
318
319 /* Don't access tstate if the thread must exit */
320 if (tstate->async_exc != NULL) {
321 _PyEval_SignalAsyncExc(tstate->interp);
322 }
323
324 MUTEX_UNLOCK(gil->mutex);
325
326 errno = err;
327}
328
329void _PyEval_SetSwitchInterval(unsigned long microseconds)
330{
331#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
332 PyInterpreterState *interp = PyInterpreterState_Get();
333 struct _gil_runtime_state *gil = &interp->ceval.gil;
334#else
335 struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
336#endif
337 gil->interval = microseconds;
338}
339
340unsigned long _PyEval_GetSwitchInterval()
341{
342#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
343 PyInterpreterState *interp = PyInterpreterState_Get();
344 struct _gil_runtime_state *gil = &interp->ceval.gil;
345#else
346 struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
347#endif
348 return gil->interval;
349}
350