1 | #ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H |
2 | #define JEMALLOC_INTERNAL_PROF_INLINES_A_H |
3 | |
4 | #include "jemalloc/internal/mutex.h" |
5 | |
6 | static inline bool |
7 | prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, |
8 | uint64_t accumbytes) { |
9 | cassert(config_prof); |
10 | |
11 | bool overflow; |
12 | uint64_t a0, a1; |
13 | |
14 | /* |
15 | * If the application allocates fast enough (and/or if idump is slow |
16 | * enough), extreme overflow here (a1 >= prof_interval * 2) can cause |
17 | * idump trigger coalescing. This is an intentional mechanism that |
18 | * avoids rate-limiting allocation. |
19 | */ |
20 | #ifdef JEMALLOC_ATOMIC_U64 |
21 | a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); |
22 | do { |
23 | a1 = a0 + accumbytes; |
24 | assert(a1 >= a0); |
25 | overflow = (a1 >= prof_interval); |
26 | if (overflow) { |
27 | a1 %= prof_interval; |
28 | } |
29 | } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, |
30 | a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); |
31 | #else |
32 | malloc_mutex_lock(tsdn, &prof_accum->mtx); |
33 | a0 = prof_accum->accumbytes; |
34 | a1 = a0 + accumbytes; |
35 | overflow = (a1 >= prof_interval); |
36 | if (overflow) { |
37 | a1 %= prof_interval; |
38 | } |
39 | prof_accum->accumbytes = a1; |
40 | malloc_mutex_unlock(tsdn, &prof_accum->mtx); |
41 | #endif |
42 | return overflow; |
43 | } |
44 | |
45 | static inline void |
46 | prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, |
47 | size_t usize) { |
48 | cassert(config_prof); |
49 | |
50 | /* |
51 | * Cancel out as much of the excessive prof_accumbytes increase as |
52 | * possible without underflowing. Interval-triggered dumps occur |
53 | * slightly more often than intended as a result of incomplete |
54 | * canceling. |
55 | */ |
56 | uint64_t a0, a1; |
57 | #ifdef JEMALLOC_ATOMIC_U64 |
58 | a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); |
59 | do { |
60 | a1 = (a0 >= SC_LARGE_MINCLASS - usize) |
61 | ? a0 - (SC_LARGE_MINCLASS - usize) : 0; |
62 | } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, |
63 | a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); |
64 | #else |
65 | malloc_mutex_lock(tsdn, &prof_accum->mtx); |
66 | a0 = prof_accum->accumbytes; |
67 | a1 = (a0 >= SC_LARGE_MINCLASS - usize) |
68 | ? a0 - (SC_LARGE_MINCLASS - usize) : 0; |
69 | prof_accum->accumbytes = a1; |
70 | malloc_mutex_unlock(tsdn, &prof_accum->mtx); |
71 | #endif |
72 | } |
73 | |
74 | JEMALLOC_ALWAYS_INLINE bool |
75 | prof_active_get_unlocked(void) { |
76 | /* |
77 | * Even if opt_prof is true, sampling can be temporarily disabled by |
78 | * setting prof_active to false. No locking is used when reading |
79 | * prof_active in the fast path, so there are no guarantees regarding |
80 | * how long it will take for all threads to notice state changes. |
81 | */ |
82 | return prof_active; |
83 | } |
84 | |
85 | #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ |
86 | |