1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/assert.h" |
5 | |
6 | JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS |
7 | |
8 | /******************************************************************************/ |
9 | /* Data. */ |
10 | |
11 | /* This option should be opt-in only. */ |
12 | #define BACKGROUND_THREAD_DEFAULT false |
13 | /* Read-only after initialization. */ |
14 | bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; |
15 | size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; |
16 | |
17 | /* Used for thread creation, termination and stats. */ |
18 | malloc_mutex_t background_thread_lock; |
19 | /* Indicates global state. Atomic because decay reads this w/o locking. */ |
20 | atomic_b_t background_thread_enabled_state; |
21 | size_t n_background_threads; |
22 | size_t max_background_threads; |
23 | /* Thread info per-index. */ |
24 | background_thread_info_t *background_thread_info; |
25 | |
26 | /******************************************************************************/ |
27 | |
28 | #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER |
29 | |
30 | static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, |
31 | void *(*)(void *), void *__restrict); |
32 | |
33 | static void |
34 | pthread_create_wrapper_init(void) { |
35 | #ifdef JEMALLOC_LAZY_LOCK |
36 | if (!isthreaded) { |
37 | isthreaded = true; |
38 | } |
39 | #endif |
40 | } |
41 | |
42 | int |
43 | pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr, |
44 | void *(*start_routine)(void *), void *__restrict arg) { |
45 | pthread_create_wrapper_init(); |
46 | |
47 | return pthread_create_fptr(thread, attr, start_routine, arg); |
48 | } |
49 | #endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */ |
50 | |
51 | #ifndef JEMALLOC_BACKGROUND_THREAD |
52 | #define NOT_REACHED { not_reached(); } |
53 | bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED |
54 | bool background_threads_enable(tsd_t *tsd) NOT_REACHED |
55 | bool background_threads_disable(tsd_t *tsd) NOT_REACHED |
56 | bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED |
57 | void background_thread_wakeup_early(background_thread_info_t *info, |
58 | nstime_t *remaining_sleep) NOT_REACHED |
59 | void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED |
60 | void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED |
61 | void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED |
62 | void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED |
63 | bool background_thread_stats_read(tsdn_t *tsdn, |
64 | background_thread_stats_t *stats) NOT_REACHED |
65 | void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED |
66 | #undef NOT_REACHED |
67 | #else |
68 | |
69 | static bool background_thread_enabled_at_fork; |
70 | |
71 | static void |
72 | background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { |
73 | background_thread_wakeup_time_set(tsdn, info, 0); |
74 | info->npages_to_purge_new = 0; |
75 | if (config_stats) { |
76 | info->tot_n_runs = 0; |
77 | nstime_init_zero(&info->tot_sleep_time); |
78 | } |
79 | } |
80 | |
81 | static inline bool |
82 | set_current_thread_affinity(int cpu) { |
83 | #ifdef __OpenBSD__ |
84 | return false; |
85 | #else |
86 | #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) |
87 | cpu_set_t cpuset; |
88 | #else |
89 | # ifndef __NetBSD__ |
90 | cpuset_t cpuset; |
91 | # else |
92 | cpuset_t *cpuset; |
93 | # endif |
94 | #endif |
95 | |
96 | #ifndef __NetBSD__ |
97 | CPU_ZERO(&cpuset); |
98 | CPU_SET(cpu, &cpuset); |
99 | #else |
100 | cpuset = cpuset_create(); |
101 | #endif |
102 | |
103 | #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) |
104 | return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0); |
105 | #else |
106 | # ifndef __NetBSD__ |
107 | int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t), |
108 | &cpuset); |
109 | # else |
110 | int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), |
111 | cpuset); |
112 | cpuset_destroy(cpuset); |
113 | # endif |
114 | return ret != 0; |
115 | #endif |
116 | #endif |
117 | } |
118 | |
119 | #define BILLION UINT64_C(1000000000) |
120 | /* Minimal sleep interval 100 ms. */ |
121 | #define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10) |
122 | |
123 | static void |
124 | background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, |
125 | uint64_t interval) { |
126 | if (config_stats) { |
127 | info->tot_n_runs++; |
128 | } |
129 | info->npages_to_purge_new = 0; |
130 | |
131 | struct timeval tv; |
132 | /* Specific clock required by timedwait. */ |
133 | gettimeofday(&tv, NULL); |
134 | nstime_t before_sleep; |
135 | nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000); |
136 | |
137 | int ret; |
138 | if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) { |
139 | background_thread_wakeup_time_set(tsdn, info, |
140 | BACKGROUND_THREAD_INDEFINITE_SLEEP); |
141 | ret = pthread_cond_wait(&info->cond, &info->mtx.lock); |
142 | assert(ret == 0); |
143 | } else { |
144 | assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS && |
145 | interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP); |
146 | /* We need malloc clock (can be different from tv). */ |
147 | nstime_t next_wakeup; |
148 | nstime_init_update(&next_wakeup); |
149 | nstime_iadd(&next_wakeup, interval); |
150 | assert(nstime_ns(&next_wakeup) < |
151 | BACKGROUND_THREAD_INDEFINITE_SLEEP); |
152 | background_thread_wakeup_time_set(tsdn, info, |
153 | nstime_ns(&next_wakeup)); |
154 | |
155 | nstime_t ts_wakeup; |
156 | nstime_copy(&ts_wakeup, &before_sleep); |
157 | nstime_iadd(&ts_wakeup, interval); |
158 | struct timespec ts; |
159 | ts.tv_sec = (size_t)nstime_sec(&ts_wakeup); |
160 | ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup); |
161 | |
162 | assert(!background_thread_indefinite_sleep(info)); |
163 | ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts); |
164 | assert(ret == ETIMEDOUT || ret == 0); |
165 | } |
166 | if (config_stats) { |
167 | gettimeofday(&tv, NULL); |
168 | nstime_t after_sleep; |
169 | nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000); |
170 | if (nstime_compare(&after_sleep, &before_sleep) > 0) { |
171 | nstime_subtract(&after_sleep, &before_sleep); |
172 | nstime_add(&info->tot_sleep_time, &after_sleep); |
173 | } |
174 | } |
175 | } |
176 | |
177 | static bool |
178 | background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) { |
179 | if (unlikely(info->state == background_thread_paused)) { |
180 | malloc_mutex_unlock(tsdn, &info->mtx); |
181 | /* Wait on global lock to update status. */ |
182 | malloc_mutex_lock(tsdn, &background_thread_lock); |
183 | malloc_mutex_unlock(tsdn, &background_thread_lock); |
184 | malloc_mutex_lock(tsdn, &info->mtx); |
185 | return true; |
186 | } |
187 | |
188 | return false; |
189 | } |
190 | |
191 | static inline void |
192 | background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, |
193 | unsigned ind) { |
194 | uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX; |
195 | unsigned narenas = narenas_total_get(); |
196 | bool slept_indefinitely = background_thread_indefinite_sleep(info); |
197 | |
198 | for (unsigned i = ind; i < narenas; i += max_background_threads) { |
199 | arena_t *arena = arena_get(tsdn, i, false); |
200 | if (!arena) { |
201 | continue; |
202 | } |
203 | /* |
204 | * If thread was woken up from the indefinite sleep, don't |
205 | * do the work instantly, but rather check when the deferred |
206 | * work that caused this thread to wake up is scheduled for. |
207 | */ |
208 | if (!slept_indefinitely) { |
209 | arena_do_deferred_work(tsdn, arena); |
210 | } |
211 | if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) { |
212 | /* Min interval will be used. */ |
213 | continue; |
214 | } |
215 | uint64_t ns_arena_deferred = pa_shard_time_until_deferred_work( |
216 | tsdn, &arena->pa_shard); |
217 | if (ns_arena_deferred < ns_until_deferred) { |
218 | ns_until_deferred = ns_arena_deferred; |
219 | } |
220 | } |
221 | |
222 | uint64_t sleep_ns; |
223 | if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) { |
224 | sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP; |
225 | } else { |
226 | sleep_ns = |
227 | (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS) |
228 | ? BACKGROUND_THREAD_MIN_INTERVAL_NS |
229 | : ns_until_deferred; |
230 | |
231 | } |
232 | |
233 | background_thread_sleep(tsdn, info, sleep_ns); |
234 | } |
235 | |
236 | static bool |
237 | background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { |
238 | if (info == &background_thread_info[0]) { |
239 | malloc_mutex_assert_owner(tsd_tsdn(tsd), |
240 | &background_thread_lock); |
241 | } else { |
242 | malloc_mutex_assert_not_owner(tsd_tsdn(tsd), |
243 | &background_thread_lock); |
244 | } |
245 | |
246 | pre_reentrancy(tsd, NULL); |
247 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
248 | bool has_thread; |
249 | assert(info->state != background_thread_paused); |
250 | if (info->state == background_thread_started) { |
251 | has_thread = true; |
252 | info->state = background_thread_stopped; |
253 | pthread_cond_signal(&info->cond); |
254 | } else { |
255 | has_thread = false; |
256 | } |
257 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
258 | |
259 | if (!has_thread) { |
260 | post_reentrancy(tsd); |
261 | return false; |
262 | } |
263 | void *ret; |
264 | if (pthread_join(info->thread, &ret)) { |
265 | post_reentrancy(tsd); |
266 | return true; |
267 | } |
268 | assert(ret == NULL); |
269 | n_background_threads--; |
270 | post_reentrancy(tsd); |
271 | |
272 | return false; |
273 | } |
274 | |
275 | static void *background_thread_entry(void *ind_arg); |
276 | |
277 | static int |
278 | background_thread_create_signals_masked(pthread_t *thread, |
279 | const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { |
280 | /* |
281 | * Mask signals during thread creation so that the thread inherits |
282 | * an empty signal set. |
283 | */ |
284 | sigset_t set; |
285 | sigfillset(&set); |
286 | sigset_t oldset; |
287 | int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); |
288 | if (mask_err != 0) { |
289 | return mask_err; |
290 | } |
291 | int create_err = pthread_create_wrapper(thread, attr, start_routine, |
292 | arg); |
293 | /* |
294 | * Restore the signal mask. Failure to restore the signal mask here |
295 | * changes program behavior. |
296 | */ |
297 | int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL); |
298 | if (restore_err != 0) { |
299 | malloc_printf("<jemalloc>: background thread creation " |
300 | "failed (%d), and signal mask restoration failed " |
301 | "(%d)\n" , create_err, restore_err); |
302 | if (opt_abort) { |
303 | abort(); |
304 | } |
305 | } |
306 | return create_err; |
307 | } |
308 | |
309 | static bool |
310 | check_background_thread_creation(tsd_t *tsd, unsigned *n_created, |
311 | bool *created_threads) { |
312 | bool ret = false; |
313 | if (likely(*n_created == n_background_threads)) { |
314 | return ret; |
315 | } |
316 | |
317 | tsdn_t *tsdn = tsd_tsdn(tsd); |
318 | malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx); |
319 | for (unsigned i = 1; i < max_background_threads; i++) { |
320 | if (created_threads[i]) { |
321 | continue; |
322 | } |
323 | background_thread_info_t *info = &background_thread_info[i]; |
324 | malloc_mutex_lock(tsdn, &info->mtx); |
325 | /* |
326 | * In case of the background_thread_paused state because of |
327 | * arena reset, delay the creation. |
328 | */ |
329 | bool create = (info->state == background_thread_started); |
330 | malloc_mutex_unlock(tsdn, &info->mtx); |
331 | if (!create) { |
332 | continue; |
333 | } |
334 | |
335 | pre_reentrancy(tsd, NULL); |
336 | int err = background_thread_create_signals_masked(&info->thread, |
337 | NULL, background_thread_entry, (void *)(uintptr_t)i); |
338 | post_reentrancy(tsd); |
339 | |
340 | if (err == 0) { |
341 | (*n_created)++; |
342 | created_threads[i] = true; |
343 | } else { |
344 | malloc_printf("<jemalloc>: background thread " |
345 | "creation failed (%d)\n" , err); |
346 | if (opt_abort) { |
347 | abort(); |
348 | } |
349 | } |
350 | /* Return to restart the loop since we unlocked. */ |
351 | ret = true; |
352 | break; |
353 | } |
354 | malloc_mutex_lock(tsdn, &background_thread_info[0].mtx); |
355 | |
356 | return ret; |
357 | } |
358 | |
359 | static void |
360 | background_thread0_work(tsd_t *tsd) { |
361 | /* Thread0 is also responsible for launching / terminating threads. */ |
362 | VARIABLE_ARRAY(bool, created_threads, max_background_threads); |
363 | unsigned i; |
364 | for (i = 1; i < max_background_threads; i++) { |
365 | created_threads[i] = false; |
366 | } |
367 | /* Start working, and create more threads when asked. */ |
368 | unsigned n_created = 1; |
369 | while (background_thread_info[0].state != background_thread_stopped) { |
370 | if (background_thread_pause_check(tsd_tsdn(tsd), |
371 | &background_thread_info[0])) { |
372 | continue; |
373 | } |
374 | if (check_background_thread_creation(tsd, &n_created, |
375 | (bool *)&created_threads)) { |
376 | continue; |
377 | } |
378 | background_work_sleep_once(tsd_tsdn(tsd), |
379 | &background_thread_info[0], 0); |
380 | } |
381 | |
382 | /* |
383 | * Shut down other threads at exit. Note that the ctl thread is holding |
384 | * the global background_thread mutex (and is waiting) for us. |
385 | */ |
386 | assert(!background_thread_enabled()); |
387 | for (i = 1; i < max_background_threads; i++) { |
388 | background_thread_info_t *info = &background_thread_info[i]; |
389 | assert(info->state != background_thread_paused); |
390 | if (created_threads[i]) { |
391 | background_threads_disable_single(tsd, info); |
392 | } else { |
393 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
394 | if (info->state != background_thread_stopped) { |
395 | /* The thread was not created. */ |
396 | assert(info->state == |
397 | background_thread_started); |
398 | n_background_threads--; |
399 | info->state = background_thread_stopped; |
400 | } |
401 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
402 | } |
403 | } |
404 | background_thread_info[0].state = background_thread_stopped; |
405 | assert(n_background_threads == 1); |
406 | } |
407 | |
408 | static void |
409 | background_work(tsd_t *tsd, unsigned ind) { |
410 | background_thread_info_t *info = &background_thread_info[ind]; |
411 | |
412 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
413 | background_thread_wakeup_time_set(tsd_tsdn(tsd), info, |
414 | BACKGROUND_THREAD_INDEFINITE_SLEEP); |
415 | if (ind == 0) { |
416 | background_thread0_work(tsd); |
417 | } else { |
418 | while (info->state != background_thread_stopped) { |
419 | if (background_thread_pause_check(tsd_tsdn(tsd), |
420 | info)) { |
421 | continue; |
422 | } |
423 | background_work_sleep_once(tsd_tsdn(tsd), info, ind); |
424 | } |
425 | } |
426 | assert(info->state == background_thread_stopped); |
427 | background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0); |
428 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
429 | } |
430 | |
431 | static void * |
432 | background_thread_entry(void *ind_arg) { |
433 | unsigned thread_ind = (unsigned)(uintptr_t)ind_arg; |
434 | assert(thread_ind < max_background_threads); |
435 | #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP |
436 | pthread_setname_np(pthread_self(), "jemalloc_bg_thd" ); |
437 | #elif defined(__FreeBSD__) || defined(__DragonFly__) |
438 | pthread_set_name_np(pthread_self(), "jemalloc_bg_thd" ); |
439 | #endif |
440 | if (opt_percpu_arena != percpu_arena_disabled) { |
441 | set_current_thread_affinity((int)thread_ind); |
442 | } |
443 | /* |
444 | * Start periodic background work. We use internal tsd which avoids |
445 | * side effects, for example triggering new arena creation (which in |
446 | * turn triggers another background thread creation). |
447 | */ |
448 | background_work(tsd_internal_fetch(), thread_ind); |
449 | assert(pthread_equal(pthread_self(), |
450 | background_thread_info[thread_ind].thread)); |
451 | |
452 | return NULL; |
453 | } |
454 | |
455 | static void |
456 | background_thread_init(tsd_t *tsd, background_thread_info_t *info) { |
457 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); |
458 | info->state = background_thread_started; |
459 | background_thread_info_init(tsd_tsdn(tsd), info); |
460 | n_background_threads++; |
461 | } |
462 | |
463 | static bool |
464 | background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { |
465 | assert(have_background_thread); |
466 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); |
467 | |
468 | /* We create at most NCPUs threads. */ |
469 | size_t thread_ind = arena_ind % max_background_threads; |
470 | background_thread_info_t *info = &background_thread_info[thread_ind]; |
471 | |
472 | bool need_new_thread; |
473 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
474 | need_new_thread = background_thread_enabled() && |
475 | (info->state == background_thread_stopped); |
476 | if (need_new_thread) { |
477 | background_thread_init(tsd, info); |
478 | } |
479 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
480 | if (!need_new_thread) { |
481 | return false; |
482 | } |
483 | if (arena_ind != 0) { |
484 | /* Threads are created asynchronously by Thread 0. */ |
485 | background_thread_info_t *t0 = &background_thread_info[0]; |
486 | malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx); |
487 | assert(t0->state == background_thread_started); |
488 | pthread_cond_signal(&t0->cond); |
489 | malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx); |
490 | |
491 | return false; |
492 | } |
493 | |
494 | pre_reentrancy(tsd, NULL); |
495 | /* |
496 | * To avoid complications (besides reentrancy), create internal |
497 | * background threads with the underlying pthread_create. |
498 | */ |
499 | int err = background_thread_create_signals_masked(&info->thread, NULL, |
500 | background_thread_entry, (void *)thread_ind); |
501 | post_reentrancy(tsd); |
502 | |
503 | if (err != 0) { |
504 | malloc_printf("<jemalloc>: arena 0 background thread creation " |
505 | "failed (%d)\n" , err); |
506 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
507 | info->state = background_thread_stopped; |
508 | n_background_threads--; |
509 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
510 | |
511 | return true; |
512 | } |
513 | |
514 | return false; |
515 | } |
516 | |
517 | /* Create a new background thread if needed. */ |
518 | bool |
519 | background_thread_create(tsd_t *tsd, unsigned arena_ind) { |
520 | assert(have_background_thread); |
521 | |
522 | bool ret; |
523 | malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); |
524 | ret = background_thread_create_locked(tsd, arena_ind); |
525 | malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); |
526 | |
527 | return ret; |
528 | } |
529 | |
530 | bool |
531 | background_threads_enable(tsd_t *tsd) { |
532 | assert(n_background_threads == 0); |
533 | assert(background_thread_enabled()); |
534 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); |
535 | |
536 | VARIABLE_ARRAY(bool, marked, max_background_threads); |
537 | unsigned nmarked; |
538 | for (unsigned i = 0; i < max_background_threads; i++) { |
539 | marked[i] = false; |
540 | } |
541 | nmarked = 0; |
542 | /* Thread 0 is required and created at the end. */ |
543 | marked[0] = true; |
544 | /* Mark the threads we need to create for thread 0. */ |
545 | unsigned narenas = narenas_total_get(); |
546 | for (unsigned i = 1; i < narenas; i++) { |
547 | if (marked[i % max_background_threads] || |
548 | arena_get(tsd_tsdn(tsd), i, false) == NULL) { |
549 | continue; |
550 | } |
551 | background_thread_info_t *info = &background_thread_info[ |
552 | i % max_background_threads]; |
553 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); |
554 | assert(info->state == background_thread_stopped); |
555 | background_thread_init(tsd, info); |
556 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); |
557 | marked[i % max_background_threads] = true; |
558 | if (++nmarked == max_background_threads) { |
559 | break; |
560 | } |
561 | } |
562 | |
563 | bool err = background_thread_create_locked(tsd, 0); |
564 | if (err) { |
565 | return true; |
566 | } |
567 | for (unsigned i = 0; i < narenas; i++) { |
568 | arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); |
569 | if (arena != NULL) { |
570 | pa_shard_set_deferral_allowed(tsd_tsdn(tsd), |
571 | &arena->pa_shard, true); |
572 | } |
573 | } |
574 | return false; |
575 | } |
576 | |
577 | bool |
578 | background_threads_disable(tsd_t *tsd) { |
579 | assert(!background_thread_enabled()); |
580 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); |
581 | |
582 | /* Thread 0 will be responsible for terminating other threads. */ |
583 | if (background_threads_disable_single(tsd, |
584 | &background_thread_info[0])) { |
585 | return true; |
586 | } |
587 | assert(n_background_threads == 0); |
588 | unsigned narenas = narenas_total_get(); |
589 | for (unsigned i = 0; i < narenas; i++) { |
590 | arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); |
591 | if (arena != NULL) { |
592 | pa_shard_set_deferral_allowed(tsd_tsdn(tsd), |
593 | &arena->pa_shard, false); |
594 | } |
595 | } |
596 | |
597 | return false; |
598 | } |
599 | |
600 | bool |
601 | background_thread_is_started(background_thread_info_t *info) { |
602 | return info->state == background_thread_started; |
603 | } |
604 | |
605 | void |
606 | background_thread_wakeup_early(background_thread_info_t *info, |
607 | nstime_t *remaining_sleep) { |
608 | /* |
609 | * This is an optimization to increase batching. At this point |
610 | * we know that background thread wakes up soon, so the time to cache |
611 | * the just freed memory is bounded and low. |
612 | */ |
613 | if (remaining_sleep != NULL && nstime_ns(remaining_sleep) < |
614 | BACKGROUND_THREAD_MIN_INTERVAL_NS) { |
615 | return; |
616 | } |
617 | pthread_cond_signal(&info->cond); |
618 | } |
619 | |
620 | void |
621 | background_thread_prefork0(tsdn_t *tsdn) { |
622 | malloc_mutex_prefork(tsdn, &background_thread_lock); |
623 | background_thread_enabled_at_fork = background_thread_enabled(); |
624 | } |
625 | |
626 | void |
627 | background_thread_prefork1(tsdn_t *tsdn) { |
628 | for (unsigned i = 0; i < max_background_threads; i++) { |
629 | malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx); |
630 | } |
631 | } |
632 | |
633 | void |
634 | background_thread_postfork_parent(tsdn_t *tsdn) { |
635 | for (unsigned i = 0; i < max_background_threads; i++) { |
636 | malloc_mutex_postfork_parent(tsdn, |
637 | &background_thread_info[i].mtx); |
638 | } |
639 | malloc_mutex_postfork_parent(tsdn, &background_thread_lock); |
640 | } |
641 | |
642 | void |
643 | background_thread_postfork_child(tsdn_t *tsdn) { |
644 | for (unsigned i = 0; i < max_background_threads; i++) { |
645 | malloc_mutex_postfork_child(tsdn, |
646 | &background_thread_info[i].mtx); |
647 | } |
648 | malloc_mutex_postfork_child(tsdn, &background_thread_lock); |
649 | if (!background_thread_enabled_at_fork) { |
650 | return; |
651 | } |
652 | |
653 | /* Clear background_thread state (reset to disabled for child). */ |
654 | malloc_mutex_lock(tsdn, &background_thread_lock); |
655 | n_background_threads = 0; |
656 | background_thread_enabled_set(tsdn, false); |
657 | for (unsigned i = 0; i < max_background_threads; i++) { |
658 | background_thread_info_t *info = &background_thread_info[i]; |
659 | malloc_mutex_lock(tsdn, &info->mtx); |
660 | info->state = background_thread_stopped; |
661 | int ret = pthread_cond_init(&info->cond, NULL); |
662 | assert(ret == 0); |
663 | background_thread_info_init(tsdn, info); |
664 | malloc_mutex_unlock(tsdn, &info->mtx); |
665 | } |
666 | malloc_mutex_unlock(tsdn, &background_thread_lock); |
667 | } |
668 | |
669 | bool |
670 | background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { |
671 | assert(config_stats); |
672 | malloc_mutex_lock(tsdn, &background_thread_lock); |
673 | if (!background_thread_enabled()) { |
674 | malloc_mutex_unlock(tsdn, &background_thread_lock); |
675 | return true; |
676 | } |
677 | |
678 | nstime_init_zero(&stats->run_interval); |
679 | memset(&stats->max_counter_per_bg_thd, 0, sizeof(mutex_prof_data_t)); |
680 | |
681 | uint64_t num_runs = 0; |
682 | stats->num_threads = n_background_threads; |
683 | for (unsigned i = 0; i < max_background_threads; i++) { |
684 | background_thread_info_t *info = &background_thread_info[i]; |
685 | if (malloc_mutex_trylock(tsdn, &info->mtx)) { |
686 | /* |
687 | * Each background thread run may take a long time; |
688 | * avoid waiting on the stats if the thread is active. |
689 | */ |
690 | continue; |
691 | } |
692 | if (info->state != background_thread_stopped) { |
693 | num_runs += info->tot_n_runs; |
694 | nstime_add(&stats->run_interval, &info->tot_sleep_time); |
695 | malloc_mutex_prof_max_update(tsdn, |
696 | &stats->max_counter_per_bg_thd, &info->mtx); |
697 | } |
698 | malloc_mutex_unlock(tsdn, &info->mtx); |
699 | } |
700 | stats->num_runs = num_runs; |
701 | if (num_runs > 0) { |
702 | nstime_idivide(&stats->run_interval, num_runs); |
703 | } |
704 | malloc_mutex_unlock(tsdn, &background_thread_lock); |
705 | |
706 | return false; |
707 | } |
708 | |
709 | #undef BACKGROUND_THREAD_NPAGES_THRESHOLD |
710 | #undef BILLION |
711 | #undef BACKGROUND_THREAD_MIN_INTERVAL_NS |
712 | |
713 | #ifdef JEMALLOC_HAVE_DLSYM |
714 | #include <dlfcn.h> |
715 | #endif |
716 | |
717 | static bool |
718 | pthread_create_fptr_init(void) { |
719 | if (pthread_create_fptr != NULL) { |
720 | return false; |
721 | } |
722 | /* |
723 | * Try the next symbol first, because 1) when use lazy_lock we have a |
724 | * wrapper for pthread_create; and 2) application may define its own |
725 | * wrapper as well (and can call malloc within the wrapper). |
726 | */ |
727 | #ifdef JEMALLOC_HAVE_DLSYM |
728 | pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create" ); |
729 | #else |
730 | pthread_create_fptr = NULL; |
731 | #endif |
732 | if (pthread_create_fptr == NULL) { |
733 | if (config_lazy_lock) { |
734 | malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " |
735 | "\"pthread_create\")\n" ); |
736 | abort(); |
737 | } else { |
738 | /* Fall back to the default symbol. */ |
739 | pthread_create_fptr = pthread_create; |
740 | } |
741 | } |
742 | |
743 | return false; |
744 | } |
745 | |
746 | /* |
747 | * When lazy lock is enabled, we need to make sure setting isthreaded before |
748 | * taking any background_thread locks. This is called early in ctl (instead of |
749 | * wait for the pthread_create calls to trigger) because the mutex is required |
750 | * before creating background threads. |
751 | */ |
752 | void |
753 | background_thread_ctl_init(tsdn_t *tsdn) { |
754 | malloc_mutex_assert_not_owner(tsdn, &background_thread_lock); |
755 | #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER |
756 | pthread_create_fptr_init(); |
757 | pthread_create_wrapper_init(); |
758 | #endif |
759 | } |
760 | |
761 | #endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ |
762 | |
763 | bool |
764 | background_thread_boot0(void) { |
765 | if (!have_background_thread && opt_background_thread) { |
766 | malloc_printf("<jemalloc>: option background_thread currently " |
767 | "supports pthread only\n" ); |
768 | return true; |
769 | } |
770 | #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER |
771 | if ((config_lazy_lock || opt_background_thread) && |
772 | pthread_create_fptr_init()) { |
773 | return true; |
774 | } |
775 | #endif |
776 | return false; |
777 | } |
778 | |
779 | bool |
780 | background_thread_boot1(tsdn_t *tsdn, base_t *base) { |
781 | #ifdef JEMALLOC_BACKGROUND_THREAD |
782 | assert(have_background_thread); |
783 | assert(narenas_total_get() > 0); |
784 | |
785 | if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { |
786 | opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; |
787 | } |
788 | max_background_threads = opt_max_background_threads; |
789 | |
790 | background_thread_enabled_set(tsdn, opt_background_thread); |
791 | if (malloc_mutex_init(&background_thread_lock, |
792 | "background_thread_global" , |
793 | WITNESS_RANK_BACKGROUND_THREAD_GLOBAL, |
794 | malloc_mutex_rank_exclusive)) { |
795 | return true; |
796 | } |
797 | |
798 | background_thread_info = (background_thread_info_t *)base_alloc(tsdn, |
799 | base, opt_max_background_threads * |
800 | sizeof(background_thread_info_t), CACHELINE); |
801 | if (background_thread_info == NULL) { |
802 | return true; |
803 | } |
804 | |
805 | for (unsigned i = 0; i < max_background_threads; i++) { |
806 | background_thread_info_t *info = &background_thread_info[i]; |
807 | /* Thread mutex is rank_inclusive because of thread0. */ |
808 | if (malloc_mutex_init(&info->mtx, "background_thread" , |
809 | WITNESS_RANK_BACKGROUND_THREAD, |
810 | malloc_mutex_address_ordered)) { |
811 | return true; |
812 | } |
813 | if (pthread_cond_init(&info->cond, NULL)) { |
814 | return true; |
815 | } |
816 | malloc_mutex_lock(tsdn, &info->mtx); |
817 | info->state = background_thread_stopped; |
818 | background_thread_info_init(tsdn, info); |
819 | malloc_mutex_unlock(tsdn, &info->mtx); |
820 | } |
821 | #endif |
822 | |
823 | return false; |
824 | } |
825 | |