1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/assert.h"
5
6JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
7
8/******************************************************************************/
9/* Data. */
10
11/* This option should be opt-in only. */
12#define BACKGROUND_THREAD_DEFAULT false
13/* Read-only after initialization. */
14bool opt_background_thread = BACKGROUND_THREAD_DEFAULT;
15size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1;
16
17/* Used for thread creation, termination and stats. */
18malloc_mutex_t background_thread_lock;
19/* Indicates global state. Atomic because decay reads this w/o locking. */
20atomic_b_t background_thread_enabled_state;
21size_t n_background_threads;
22size_t max_background_threads;
23/* Thread info per-index. */
24background_thread_info_t *background_thread_info;
25
26/******************************************************************************/
27
28#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
29
30static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
31 void *(*)(void *), void *__restrict);
32
33static void
34pthread_create_wrapper_init(void) {
35#ifdef JEMALLOC_LAZY_LOCK
36 if (!isthreaded) {
37 isthreaded = true;
38 }
39#endif
40}
41
42int
43pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr,
44 void *(*start_routine)(void *), void *__restrict arg) {
45 pthread_create_wrapper_init();
46
47 return pthread_create_fptr(thread, attr, start_routine, arg);
48}
49#endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */
50
51#ifndef JEMALLOC_BACKGROUND_THREAD
52#define NOT_REACHED { not_reached(); }
53bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED
54bool background_threads_enable(tsd_t *tsd) NOT_REACHED
55bool background_threads_disable(tsd_t *tsd) NOT_REACHED
56bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED
57void background_thread_wakeup_early(background_thread_info_t *info,
58 nstime_t *remaining_sleep) NOT_REACHED
59void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED
60void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED
61void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED
62void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED
63bool background_thread_stats_read(tsdn_t *tsdn,
64 background_thread_stats_t *stats) NOT_REACHED
65void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED
66#undef NOT_REACHED
67#else
68
69static bool background_thread_enabled_at_fork;
70
71static void
72background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) {
73 background_thread_wakeup_time_set(tsdn, info, 0);
74 info->npages_to_purge_new = 0;
75 if (config_stats) {
76 info->tot_n_runs = 0;
77 nstime_init_zero(&info->tot_sleep_time);
78 }
79}
80
81static inline bool
82set_current_thread_affinity(int cpu) {
83#ifdef __OpenBSD__
84 return false;
85#else
86#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
87 cpu_set_t cpuset;
88#else
89# ifndef __NetBSD__
90 cpuset_t cpuset;
91# else
92 cpuset_t *cpuset;
93# endif
94#endif
95
96#ifndef __NetBSD__
97 CPU_ZERO(&cpuset);
98 CPU_SET(cpu, &cpuset);
99#else
100 cpuset = cpuset_create();
101#endif
102
103#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
104 return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0);
105#else
106# ifndef __NetBSD__
107 int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t),
108 &cpuset);
109# else
110 int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset),
111 cpuset);
112 cpuset_destroy(cpuset);
113# endif
114 return ret != 0;
115#endif
116#endif
117}
118
119#define BILLION UINT64_C(1000000000)
120/* Minimal sleep interval 100 ms. */
121#define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10)
122
123static void
124background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
125 uint64_t interval) {
126 if (config_stats) {
127 info->tot_n_runs++;
128 }
129 info->npages_to_purge_new = 0;
130
131 struct timeval tv;
132 /* Specific clock required by timedwait. */
133 gettimeofday(&tv, NULL);
134 nstime_t before_sleep;
135 nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
136
137 int ret;
138 if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
139 background_thread_wakeup_time_set(tsdn, info,
140 BACKGROUND_THREAD_INDEFINITE_SLEEP);
141 ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
142 assert(ret == 0);
143 } else {
144 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS &&
145 interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP);
146 /* We need malloc clock (can be different from tv). */
147 nstime_t next_wakeup;
148 nstime_init_update(&next_wakeup);
149 nstime_iadd(&next_wakeup, interval);
150 assert(nstime_ns(&next_wakeup) <
151 BACKGROUND_THREAD_INDEFINITE_SLEEP);
152 background_thread_wakeup_time_set(tsdn, info,
153 nstime_ns(&next_wakeup));
154
155 nstime_t ts_wakeup;
156 nstime_copy(&ts_wakeup, &before_sleep);
157 nstime_iadd(&ts_wakeup, interval);
158 struct timespec ts;
159 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
160 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
161
162 assert(!background_thread_indefinite_sleep(info));
163 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts);
164 assert(ret == ETIMEDOUT || ret == 0);
165 }
166 if (config_stats) {
167 gettimeofday(&tv, NULL);
168 nstime_t after_sleep;
169 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
170 if (nstime_compare(&after_sleep, &before_sleep) > 0) {
171 nstime_subtract(&after_sleep, &before_sleep);
172 nstime_add(&info->tot_sleep_time, &after_sleep);
173 }
174 }
175}
176
177static bool
178background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
179 if (unlikely(info->state == background_thread_paused)) {
180 malloc_mutex_unlock(tsdn, &info->mtx);
181 /* Wait on global lock to update status. */
182 malloc_mutex_lock(tsdn, &background_thread_lock);
183 malloc_mutex_unlock(tsdn, &background_thread_lock);
184 malloc_mutex_lock(tsdn, &info->mtx);
185 return true;
186 }
187
188 return false;
189}
190
191static inline void
192background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info,
193 unsigned ind) {
194 uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
195 unsigned narenas = narenas_total_get();
196 bool slept_indefinitely = background_thread_indefinite_sleep(info);
197
198 for (unsigned i = ind; i < narenas; i += max_background_threads) {
199 arena_t *arena = arena_get(tsdn, i, false);
200 if (!arena) {
201 continue;
202 }
203 /*
204 * If thread was woken up from the indefinite sleep, don't
205 * do the work instantly, but rather check when the deferred
206 * work that caused this thread to wake up is scheduled for.
207 */
208 if (!slept_indefinitely) {
209 arena_do_deferred_work(tsdn, arena);
210 }
211 if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
212 /* Min interval will be used. */
213 continue;
214 }
215 uint64_t ns_arena_deferred = pa_shard_time_until_deferred_work(
216 tsdn, &arena->pa_shard);
217 if (ns_arena_deferred < ns_until_deferred) {
218 ns_until_deferred = ns_arena_deferred;
219 }
220 }
221
222 uint64_t sleep_ns;
223 if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
224 sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
225 } else {
226 sleep_ns =
227 (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS)
228 ? BACKGROUND_THREAD_MIN_INTERVAL_NS
229 : ns_until_deferred;
230
231 }
232
233 background_thread_sleep(tsdn, info, sleep_ns);
234}
235
236static bool
237background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
238 if (info == &background_thread_info[0]) {
239 malloc_mutex_assert_owner(tsd_tsdn(tsd),
240 &background_thread_lock);
241 } else {
242 malloc_mutex_assert_not_owner(tsd_tsdn(tsd),
243 &background_thread_lock);
244 }
245
246 pre_reentrancy(tsd, NULL);
247 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
248 bool has_thread;
249 assert(info->state != background_thread_paused);
250 if (info->state == background_thread_started) {
251 has_thread = true;
252 info->state = background_thread_stopped;
253 pthread_cond_signal(&info->cond);
254 } else {
255 has_thread = false;
256 }
257 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
258
259 if (!has_thread) {
260 post_reentrancy(tsd);
261 return false;
262 }
263 void *ret;
264 if (pthread_join(info->thread, &ret)) {
265 post_reentrancy(tsd);
266 return true;
267 }
268 assert(ret == NULL);
269 n_background_threads--;
270 post_reentrancy(tsd);
271
272 return false;
273}
274
275static void *background_thread_entry(void *ind_arg);
276
277static int
278background_thread_create_signals_masked(pthread_t *thread,
279 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) {
280 /*
281 * Mask signals during thread creation so that the thread inherits
282 * an empty signal set.
283 */
284 sigset_t set;
285 sigfillset(&set);
286 sigset_t oldset;
287 int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
288 if (mask_err != 0) {
289 return mask_err;
290 }
291 int create_err = pthread_create_wrapper(thread, attr, start_routine,
292 arg);
293 /*
294 * Restore the signal mask. Failure to restore the signal mask here
295 * changes program behavior.
296 */
297 int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
298 if (restore_err != 0) {
299 malloc_printf("<jemalloc>: background thread creation "
300 "failed (%d), and signal mask restoration failed "
301 "(%d)\n", create_err, restore_err);
302 if (opt_abort) {
303 abort();
304 }
305 }
306 return create_err;
307}
308
309static bool
310check_background_thread_creation(tsd_t *tsd, unsigned *n_created,
311 bool *created_threads) {
312 bool ret = false;
313 if (likely(*n_created == n_background_threads)) {
314 return ret;
315 }
316
317 tsdn_t *tsdn = tsd_tsdn(tsd);
318 malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx);
319 for (unsigned i = 1; i < max_background_threads; i++) {
320 if (created_threads[i]) {
321 continue;
322 }
323 background_thread_info_t *info = &background_thread_info[i];
324 malloc_mutex_lock(tsdn, &info->mtx);
325 /*
326 * In case of the background_thread_paused state because of
327 * arena reset, delay the creation.
328 */
329 bool create = (info->state == background_thread_started);
330 malloc_mutex_unlock(tsdn, &info->mtx);
331 if (!create) {
332 continue;
333 }
334
335 pre_reentrancy(tsd, NULL);
336 int err = background_thread_create_signals_masked(&info->thread,
337 NULL, background_thread_entry, (void *)(uintptr_t)i);
338 post_reentrancy(tsd);
339
340 if (err == 0) {
341 (*n_created)++;
342 created_threads[i] = true;
343 } else {
344 malloc_printf("<jemalloc>: background thread "
345 "creation failed (%d)\n", err);
346 if (opt_abort) {
347 abort();
348 }
349 }
350 /* Return to restart the loop since we unlocked. */
351 ret = true;
352 break;
353 }
354 malloc_mutex_lock(tsdn, &background_thread_info[0].mtx);
355
356 return ret;
357}
358
359static void
360background_thread0_work(tsd_t *tsd) {
361 /* Thread0 is also responsible for launching / terminating threads. */
362 VARIABLE_ARRAY(bool, created_threads, max_background_threads);
363 unsigned i;
364 for (i = 1; i < max_background_threads; i++) {
365 created_threads[i] = false;
366 }
367 /* Start working, and create more threads when asked. */
368 unsigned n_created = 1;
369 while (background_thread_info[0].state != background_thread_stopped) {
370 if (background_thread_pause_check(tsd_tsdn(tsd),
371 &background_thread_info[0])) {
372 continue;
373 }
374 if (check_background_thread_creation(tsd, &n_created,
375 (bool *)&created_threads)) {
376 continue;
377 }
378 background_work_sleep_once(tsd_tsdn(tsd),
379 &background_thread_info[0], 0);
380 }
381
382 /*
383 * Shut down other threads at exit. Note that the ctl thread is holding
384 * the global background_thread mutex (and is waiting) for us.
385 */
386 assert(!background_thread_enabled());
387 for (i = 1; i < max_background_threads; i++) {
388 background_thread_info_t *info = &background_thread_info[i];
389 assert(info->state != background_thread_paused);
390 if (created_threads[i]) {
391 background_threads_disable_single(tsd, info);
392 } else {
393 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
394 if (info->state != background_thread_stopped) {
395 /* The thread was not created. */
396 assert(info->state ==
397 background_thread_started);
398 n_background_threads--;
399 info->state = background_thread_stopped;
400 }
401 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
402 }
403 }
404 background_thread_info[0].state = background_thread_stopped;
405 assert(n_background_threads == 1);
406}
407
408static void
409background_work(tsd_t *tsd, unsigned ind) {
410 background_thread_info_t *info = &background_thread_info[ind];
411
412 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
413 background_thread_wakeup_time_set(tsd_tsdn(tsd), info,
414 BACKGROUND_THREAD_INDEFINITE_SLEEP);
415 if (ind == 0) {
416 background_thread0_work(tsd);
417 } else {
418 while (info->state != background_thread_stopped) {
419 if (background_thread_pause_check(tsd_tsdn(tsd),
420 info)) {
421 continue;
422 }
423 background_work_sleep_once(tsd_tsdn(tsd), info, ind);
424 }
425 }
426 assert(info->state == background_thread_stopped);
427 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0);
428 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
429}
430
431static void *
432background_thread_entry(void *ind_arg) {
433 unsigned thread_ind = (unsigned)(uintptr_t)ind_arg;
434 assert(thread_ind < max_background_threads);
435#ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP
436 pthread_setname_np(pthread_self(), "jemalloc_bg_thd");
437#elif defined(__FreeBSD__) || defined(__DragonFly__)
438 pthread_set_name_np(pthread_self(), "jemalloc_bg_thd");
439#endif
440 if (opt_percpu_arena != percpu_arena_disabled) {
441 set_current_thread_affinity((int)thread_ind);
442 }
443 /*
444 * Start periodic background work. We use internal tsd which avoids
445 * side effects, for example triggering new arena creation (which in
446 * turn triggers another background thread creation).
447 */
448 background_work(tsd_internal_fetch(), thread_ind);
449 assert(pthread_equal(pthread_self(),
450 background_thread_info[thread_ind].thread));
451
452 return NULL;
453}
454
455static void
456background_thread_init(tsd_t *tsd, background_thread_info_t *info) {
457 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
458 info->state = background_thread_started;
459 background_thread_info_init(tsd_tsdn(tsd), info);
460 n_background_threads++;
461}
462
463static bool
464background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
465 assert(have_background_thread);
466 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
467
468 /* We create at most NCPUs threads. */
469 size_t thread_ind = arena_ind % max_background_threads;
470 background_thread_info_t *info = &background_thread_info[thread_ind];
471
472 bool need_new_thread;
473 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
474 need_new_thread = background_thread_enabled() &&
475 (info->state == background_thread_stopped);
476 if (need_new_thread) {
477 background_thread_init(tsd, info);
478 }
479 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
480 if (!need_new_thread) {
481 return false;
482 }
483 if (arena_ind != 0) {
484 /* Threads are created asynchronously by Thread 0. */
485 background_thread_info_t *t0 = &background_thread_info[0];
486 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
487 assert(t0->state == background_thread_started);
488 pthread_cond_signal(&t0->cond);
489 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
490
491 return false;
492 }
493
494 pre_reentrancy(tsd, NULL);
495 /*
496 * To avoid complications (besides reentrancy), create internal
497 * background threads with the underlying pthread_create.
498 */
499 int err = background_thread_create_signals_masked(&info->thread, NULL,
500 background_thread_entry, (void *)thread_ind);
501 post_reentrancy(tsd);
502
503 if (err != 0) {
504 malloc_printf("<jemalloc>: arena 0 background thread creation "
505 "failed (%d)\n", err);
506 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
507 info->state = background_thread_stopped;
508 n_background_threads--;
509 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
510
511 return true;
512 }
513
514 return false;
515}
516
517/* Create a new background thread if needed. */
518bool
519background_thread_create(tsd_t *tsd, unsigned arena_ind) {
520 assert(have_background_thread);
521
522 bool ret;
523 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
524 ret = background_thread_create_locked(tsd, arena_ind);
525 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
526
527 return ret;
528}
529
530bool
531background_threads_enable(tsd_t *tsd) {
532 assert(n_background_threads == 0);
533 assert(background_thread_enabled());
534 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
535
536 VARIABLE_ARRAY(bool, marked, max_background_threads);
537 unsigned nmarked;
538 for (unsigned i = 0; i < max_background_threads; i++) {
539 marked[i] = false;
540 }
541 nmarked = 0;
542 /* Thread 0 is required and created at the end. */
543 marked[0] = true;
544 /* Mark the threads we need to create for thread 0. */
545 unsigned narenas = narenas_total_get();
546 for (unsigned i = 1; i < narenas; i++) {
547 if (marked[i % max_background_threads] ||
548 arena_get(tsd_tsdn(tsd), i, false) == NULL) {
549 continue;
550 }
551 background_thread_info_t *info = &background_thread_info[
552 i % max_background_threads];
553 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
554 assert(info->state == background_thread_stopped);
555 background_thread_init(tsd, info);
556 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
557 marked[i % max_background_threads] = true;
558 if (++nmarked == max_background_threads) {
559 break;
560 }
561 }
562
563 bool err = background_thread_create_locked(tsd, 0);
564 if (err) {
565 return true;
566 }
567 for (unsigned i = 0; i < narenas; i++) {
568 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
569 if (arena != NULL) {
570 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
571 &arena->pa_shard, true);
572 }
573 }
574 return false;
575}
576
577bool
578background_threads_disable(tsd_t *tsd) {
579 assert(!background_thread_enabled());
580 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
581
582 /* Thread 0 will be responsible for terminating other threads. */
583 if (background_threads_disable_single(tsd,
584 &background_thread_info[0])) {
585 return true;
586 }
587 assert(n_background_threads == 0);
588 unsigned narenas = narenas_total_get();
589 for (unsigned i = 0; i < narenas; i++) {
590 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false);
591 if (arena != NULL) {
592 pa_shard_set_deferral_allowed(tsd_tsdn(tsd),
593 &arena->pa_shard, false);
594 }
595 }
596
597 return false;
598}
599
600bool
601background_thread_is_started(background_thread_info_t *info) {
602 return info->state == background_thread_started;
603}
604
605void
606background_thread_wakeup_early(background_thread_info_t *info,
607 nstime_t *remaining_sleep) {
608 /*
609 * This is an optimization to increase batching. At this point
610 * we know that background thread wakes up soon, so the time to cache
611 * the just freed memory is bounded and low.
612 */
613 if (remaining_sleep != NULL && nstime_ns(remaining_sleep) <
614 BACKGROUND_THREAD_MIN_INTERVAL_NS) {
615 return;
616 }
617 pthread_cond_signal(&info->cond);
618}
619
620void
621background_thread_prefork0(tsdn_t *tsdn) {
622 malloc_mutex_prefork(tsdn, &background_thread_lock);
623 background_thread_enabled_at_fork = background_thread_enabled();
624}
625
626void
627background_thread_prefork1(tsdn_t *tsdn) {
628 for (unsigned i = 0; i < max_background_threads; i++) {
629 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx);
630 }
631}
632
633void
634background_thread_postfork_parent(tsdn_t *tsdn) {
635 for (unsigned i = 0; i < max_background_threads; i++) {
636 malloc_mutex_postfork_parent(tsdn,
637 &background_thread_info[i].mtx);
638 }
639 malloc_mutex_postfork_parent(tsdn, &background_thread_lock);
640}
641
642void
643background_thread_postfork_child(tsdn_t *tsdn) {
644 for (unsigned i = 0; i < max_background_threads; i++) {
645 malloc_mutex_postfork_child(tsdn,
646 &background_thread_info[i].mtx);
647 }
648 malloc_mutex_postfork_child(tsdn, &background_thread_lock);
649 if (!background_thread_enabled_at_fork) {
650 return;
651 }
652
653 /* Clear background_thread state (reset to disabled for child). */
654 malloc_mutex_lock(tsdn, &background_thread_lock);
655 n_background_threads = 0;
656 background_thread_enabled_set(tsdn, false);
657 for (unsigned i = 0; i < max_background_threads; i++) {
658 background_thread_info_t *info = &background_thread_info[i];
659 malloc_mutex_lock(tsdn, &info->mtx);
660 info->state = background_thread_stopped;
661 int ret = pthread_cond_init(&info->cond, NULL);
662 assert(ret == 0);
663 background_thread_info_init(tsdn, info);
664 malloc_mutex_unlock(tsdn, &info->mtx);
665 }
666 malloc_mutex_unlock(tsdn, &background_thread_lock);
667}
668
669bool
670background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
671 assert(config_stats);
672 malloc_mutex_lock(tsdn, &background_thread_lock);
673 if (!background_thread_enabled()) {
674 malloc_mutex_unlock(tsdn, &background_thread_lock);
675 return true;
676 }
677
678 nstime_init_zero(&stats->run_interval);
679 memset(&stats->max_counter_per_bg_thd, 0, sizeof(mutex_prof_data_t));
680
681 uint64_t num_runs = 0;
682 stats->num_threads = n_background_threads;
683 for (unsigned i = 0; i < max_background_threads; i++) {
684 background_thread_info_t *info = &background_thread_info[i];
685 if (malloc_mutex_trylock(tsdn, &info->mtx)) {
686 /*
687 * Each background thread run may take a long time;
688 * avoid waiting on the stats if the thread is active.
689 */
690 continue;
691 }
692 if (info->state != background_thread_stopped) {
693 num_runs += info->tot_n_runs;
694 nstime_add(&stats->run_interval, &info->tot_sleep_time);
695 malloc_mutex_prof_max_update(tsdn,
696 &stats->max_counter_per_bg_thd, &info->mtx);
697 }
698 malloc_mutex_unlock(tsdn, &info->mtx);
699 }
700 stats->num_runs = num_runs;
701 if (num_runs > 0) {
702 nstime_idivide(&stats->run_interval, num_runs);
703 }
704 malloc_mutex_unlock(tsdn, &background_thread_lock);
705
706 return false;
707}
708
709#undef BACKGROUND_THREAD_NPAGES_THRESHOLD
710#undef BILLION
711#undef BACKGROUND_THREAD_MIN_INTERVAL_NS
712
713#ifdef JEMALLOC_HAVE_DLSYM
714#include <dlfcn.h>
715#endif
716
717static bool
718pthread_create_fptr_init(void) {
719 if (pthread_create_fptr != NULL) {
720 return false;
721 }
722 /*
723 * Try the next symbol first, because 1) when use lazy_lock we have a
724 * wrapper for pthread_create; and 2) application may define its own
725 * wrapper as well (and can call malloc within the wrapper).
726 */
727#ifdef JEMALLOC_HAVE_DLSYM
728 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
729#else
730 pthread_create_fptr = NULL;
731#endif
732 if (pthread_create_fptr == NULL) {
733 if (config_lazy_lock) {
734 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
735 "\"pthread_create\")\n");
736 abort();
737 } else {
738 /* Fall back to the default symbol. */
739 pthread_create_fptr = pthread_create;
740 }
741 }
742
743 return false;
744}
745
746/*
747 * When lazy lock is enabled, we need to make sure setting isthreaded before
748 * taking any background_thread locks. This is called early in ctl (instead of
749 * wait for the pthread_create calls to trigger) because the mutex is required
750 * before creating background threads.
751 */
752void
753background_thread_ctl_init(tsdn_t *tsdn) {
754 malloc_mutex_assert_not_owner(tsdn, &background_thread_lock);
755#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
756 pthread_create_fptr_init();
757 pthread_create_wrapper_init();
758#endif
759}
760
761#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
762
763bool
764background_thread_boot0(void) {
765 if (!have_background_thread && opt_background_thread) {
766 malloc_printf("<jemalloc>: option background_thread currently "
767 "supports pthread only\n");
768 return true;
769 }
770#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
771 if ((config_lazy_lock || opt_background_thread) &&
772 pthread_create_fptr_init()) {
773 return true;
774 }
775#endif
776 return false;
777}
778
779bool
780background_thread_boot1(tsdn_t *tsdn, base_t *base) {
781#ifdef JEMALLOC_BACKGROUND_THREAD
782 assert(have_background_thread);
783 assert(narenas_total_get() > 0);
784
785 if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) {
786 opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD;
787 }
788 max_background_threads = opt_max_background_threads;
789
790 background_thread_enabled_set(tsdn, opt_background_thread);
791 if (malloc_mutex_init(&background_thread_lock,
792 "background_thread_global",
793 WITNESS_RANK_BACKGROUND_THREAD_GLOBAL,
794 malloc_mutex_rank_exclusive)) {
795 return true;
796 }
797
798 background_thread_info = (background_thread_info_t *)base_alloc(tsdn,
799 base, opt_max_background_threads *
800 sizeof(background_thread_info_t), CACHELINE);
801 if (background_thread_info == NULL) {
802 return true;
803 }
804
805 for (unsigned i = 0; i < max_background_threads; i++) {
806 background_thread_info_t *info = &background_thread_info[i];
807 /* Thread mutex is rank_inclusive because of thread0. */
808 if (malloc_mutex_init(&info->mtx, "background_thread",
809 WITNESS_RANK_BACKGROUND_THREAD,
810 malloc_mutex_address_ordered)) {
811 return true;
812 }
813 if (pthread_cond_init(&info->cond, NULL)) {
814 return true;
815 }
816 malloc_mutex_lock(tsdn, &info->mtx);
817 info->state = background_thread_stopped;
818 background_thread_info_init(tsdn, info);
819 malloc_mutex_unlock(tsdn, &info->mtx);
820 }
821#endif
822
823 return false;
824}
825