1#ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H
2#define JEMALLOC_INTERNAL_PROF_INLINES_A_H
3
4#include "jemalloc/internal/mutex.h"
5
6static inline bool
7prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum,
8 uint64_t accumbytes) {
9 cassert(config_prof);
10
11 bool overflow;
12 uint64_t a0, a1;
13
14 /*
15 * If the application allocates fast enough (and/or if idump is slow
16 * enough), extreme overflow here (a1 >= prof_interval * 2) can cause
17 * idump trigger coalescing. This is an intentional mechanism that
18 * avoids rate-limiting allocation.
19 */
20#ifdef JEMALLOC_ATOMIC_U64
21 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED);
22 do {
23 a1 = a0 + accumbytes;
24 assert(a1 >= a0);
25 overflow = (a1 >= prof_interval);
26 if (overflow) {
27 a1 %= prof_interval;
28 }
29 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0,
30 a1, ATOMIC_RELAXED, ATOMIC_RELAXED));
31#else
32 malloc_mutex_lock(tsdn, &prof_accum->mtx);
33 a0 = prof_accum->accumbytes;
34 a1 = a0 + accumbytes;
35 overflow = (a1 >= prof_interval);
36 if (overflow) {
37 a1 %= prof_interval;
38 }
39 prof_accum->accumbytes = a1;
40 malloc_mutex_unlock(tsdn, &prof_accum->mtx);
41#endif
42 return overflow;
43}
44
45static inline void
46prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum,
47 size_t usize) {
48 cassert(config_prof);
49
50 /*
51 * Cancel out as much of the excessive prof_accumbytes increase as
52 * possible without underflowing. Interval-triggered dumps occur
53 * slightly more often than intended as a result of incomplete
54 * canceling.
55 */
56 uint64_t a0, a1;
57#ifdef JEMALLOC_ATOMIC_U64
58 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED);
59 do {
60 a1 = (a0 >= SC_LARGE_MINCLASS - usize)
61 ? a0 - (SC_LARGE_MINCLASS - usize) : 0;
62 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0,
63 a1, ATOMIC_RELAXED, ATOMIC_RELAXED));
64#else
65 malloc_mutex_lock(tsdn, &prof_accum->mtx);
66 a0 = prof_accum->accumbytes;
67 a1 = (a0 >= SC_LARGE_MINCLASS - usize)
68 ? a0 - (SC_LARGE_MINCLASS - usize) : 0;
69 prof_accum->accumbytes = a1;
70 malloc_mutex_unlock(tsdn, &prof_accum->mtx);
71#endif
72}
73
74JEMALLOC_ALWAYS_INLINE bool
75prof_active_get_unlocked(void) {
76 /*
77 * Even if opt_prof is true, sampling can be temporarily disabled by
78 * setting prof_active to false. No locking is used when reading
79 * prof_active in the fast path, so there are no guarantees regarding
80 * how long it will take for all threads to notice state changes.
81 */
82 return prof_active;
83}
84
85#endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */
86