1#ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
2#define JEMALLOC_INTERNAL_ARENA_STATS_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/mutex.h"
6#include "jemalloc/internal/mutex_prof.h"
7#include "jemalloc/internal/sc.h"
8
9JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
10
11/*
12 * In those architectures that support 64-bit atomics, we use atomic updates for
13 * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize
14 * externally.
15 */
16#ifdef JEMALLOC_ATOMIC_U64
17typedef atomic_u64_t arena_stats_u64_t;
18#else
19/* Must hold the arena stats mutex while reading atomically. */
20typedef uint64_t arena_stats_u64_t;
21#endif
22
23typedef struct arena_stats_large_s arena_stats_large_t;
24struct arena_stats_large_s {
25 /*
26 * Total number of allocation/deallocation requests served directly by
27 * the arena.
28 */
29 arena_stats_u64_t nmalloc;
30 arena_stats_u64_t ndalloc;
31
32 /*
33 * Number of allocation requests that correspond to this size class.
34 * This includes requests served by tcache, though tcache only
35 * periodically merges into this counter.
36 */
37 arena_stats_u64_t nrequests; /* Partially derived. */
38 /*
39 * Number of tcache fills / flushes for large (similarly, periodically
40 * merged). Note that there is no large tcache batch-fill currently
41 * (i.e. only fill 1 at a time); however flush may be batched.
42 */
43 arena_stats_u64_t nfills; /* Partially derived. */
44 arena_stats_u64_t nflushes; /* Partially derived. */
45
46 /* Current number of allocations of this size class. */
47 size_t curlextents; /* Derived. */
48};
49
50typedef struct arena_stats_decay_s arena_stats_decay_t;
51struct arena_stats_decay_s {
52 /* Total number of purge sweeps. */
53 arena_stats_u64_t npurge;
54 /* Total number of madvise calls made. */
55 arena_stats_u64_t nmadvise;
56 /* Total number of pages purged. */
57 arena_stats_u64_t purged;
58};
59
60typedef struct arena_stats_extents_s arena_stats_extents_t;
61struct arena_stats_extents_s {
62 /*
63 * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t.
64 * We track both bytes and # of extents: two extents in the same bucket
65 * may have different sizes if adjacent size classes differ by more than
66 * a page, so bytes cannot always be derived from # of extents.
67 */
68 atomic_zu_t ndirty;
69 atomic_zu_t dirty_bytes;
70 atomic_zu_t nmuzzy;
71 atomic_zu_t muzzy_bytes;
72 atomic_zu_t nretained;
73 atomic_zu_t retained_bytes;
74};
75
76/*
77 * Arena stats. Note that fields marked "derived" are not directly maintained
78 * within the arena code; rather their values are derived during stats merge
79 * requests.
80 */
81typedef struct arena_stats_s arena_stats_t;
82struct arena_stats_s {
83#ifndef JEMALLOC_ATOMIC_U64
84 malloc_mutex_t mtx;
85#endif
86
87 /* Number of bytes currently mapped, excluding retained memory. */
88 atomic_zu_t mapped; /* Partially derived. */
89
90 /*
91 * Number of unused virtual memory bytes currently retained. Retained
92 * bytes are technically mapped (though always decommitted or purged),
93 * but they are excluded from the mapped statistic (above).
94 */
95 atomic_zu_t retained; /* Derived. */
96
97 /* Number of extent_t structs allocated by base, but not being used. */
98 atomic_zu_t extent_avail;
99
100 arena_stats_decay_t decay_dirty;
101 arena_stats_decay_t decay_muzzy;
102
103 atomic_zu_t base; /* Derived. */
104 atomic_zu_t internal;
105 atomic_zu_t resident; /* Derived. */
106 atomic_zu_t metadata_thp;
107
108 atomic_zu_t allocated_large; /* Derived. */
109 arena_stats_u64_t nmalloc_large; /* Derived. */
110 arena_stats_u64_t ndalloc_large; /* Derived. */
111 arena_stats_u64_t nfills_large; /* Derived. */
112 arena_stats_u64_t nflushes_large; /* Derived. */
113 arena_stats_u64_t nrequests_large; /* Derived. */
114
115 /* VM space had to be leaked (undocumented). Normally 0. */
116 atomic_zu_t abandoned_vm;
117
118 /* Number of bytes cached in tcache associated with this arena. */
119 atomic_zu_t tcache_bytes; /* Derived. */
120
121 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
122
123 /* One element for each large size class. */
124 arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
125
126 /* Arena uptime. */
127 nstime_t uptime;
128};
129
130static inline bool
131arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
132 if (config_debug) {
133 for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
134 assert(((char *)arena_stats)[i] == 0);
135 }
136 }
137#ifndef JEMALLOC_ATOMIC_U64
138 if (malloc_mutex_init(&arena_stats->mtx, "arena_stats",
139 WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
140 return true;
141 }
142#endif
143 /* Memory is zeroed, so there is no need to clear stats. */
144 return false;
145}
146
147static inline void
148arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
149#ifndef JEMALLOC_ATOMIC_U64
150 malloc_mutex_lock(tsdn, &arena_stats->mtx);
151#endif
152}
153
154static inline void
155arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
156#ifndef JEMALLOC_ATOMIC_U64
157 malloc_mutex_unlock(tsdn, &arena_stats->mtx);
158#endif
159}
160
161static inline uint64_t
162arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
163 arena_stats_u64_t *p) {
164#ifdef JEMALLOC_ATOMIC_U64
165 return atomic_load_u64(p, ATOMIC_RELAXED);
166#else
167 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
168 return *p;
169#endif
170}
171
172static inline void
173arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
174 arena_stats_u64_t *p, uint64_t x) {
175#ifdef JEMALLOC_ATOMIC_U64
176 atomic_fetch_add_u64(p, x, ATOMIC_RELAXED);
177#else
178 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
179 *p += x;
180#endif
181}
182
183static inline void
184arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
185 arena_stats_u64_t *p, uint64_t x) {
186#ifdef JEMALLOC_ATOMIC_U64
187 uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
188 assert(r - x <= r);
189#else
190 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
191 *p -= x;
192 assert(*p + x >= *p);
193#endif
194}
195
196/*
197 * Non-atomically sets *dst += src. *dst needs external synchronization.
198 * This lets us avoid the cost of a fetch_add when its unnecessary (note that
199 * the types here are atomic).
200 */
201static inline void
202arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
203#ifdef JEMALLOC_ATOMIC_U64
204 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
205 atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED);
206#else
207 *dst += src;
208#endif
209}
210
211static inline size_t
212arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
213 atomic_zu_t *p) {
214#ifdef JEMALLOC_ATOMIC_U64
215 return atomic_load_zu(p, ATOMIC_RELAXED);
216#else
217 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
218 return atomic_load_zu(p, ATOMIC_RELAXED);
219#endif
220}
221
222static inline void
223arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
224 atomic_zu_t *p, size_t x) {
225#ifdef JEMALLOC_ATOMIC_U64
226 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
227#else
228 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
229 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
230 atomic_store_zu(p, cur + x, ATOMIC_RELAXED);
231#endif
232}
233
234static inline void
235arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
236 atomic_zu_t *p, size_t x) {
237#ifdef JEMALLOC_ATOMIC_U64
238 size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
239 assert(r - x <= r);
240#else
241 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
242 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
243 atomic_store_zu(p, cur - x, ATOMIC_RELAXED);
244#endif
245}
246
247/* Like the _u64 variant, needs an externally synchronized *dst. */
248static inline void
249arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
250 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
251 atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED);
252}
253
254static inline void
255arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
256 szind_t szind, uint64_t nrequests) {
257 arena_stats_lock(tsdn, arena_stats);
258 arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
259 arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests);
260 arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1);
261 arena_stats_unlock(tsdn, arena_stats);
262}
263
264static inline void
265arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
266 arena_stats_lock(tsdn, arena_stats);
267 arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size);
268 arena_stats_unlock(tsdn, arena_stats);
269}
270
271#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */
272