1 | #ifndef JEMALLOC_INTERNAL_PAC_H |
2 | #define JEMALLOC_INTERNAL_PAC_H |
3 | |
4 | #include "jemalloc/internal/exp_grow.h" |
5 | #include "jemalloc/internal/pai.h" |
6 | #include "san_bump.h" |
7 | |
8 | |
9 | /* |
10 | * Page allocator classic; an implementation of the PAI interface that: |
11 | * - Can be used for arenas with custom extent hooks. |
12 | * - Can always satisfy any allocation request (including highly-fragmentary |
13 | * ones). |
14 | * - Can use efficient OS-level zeroing primitives for demand-filled pages. |
15 | */ |
16 | |
17 | /* How "eager" decay/purging should be. */ |
18 | enum pac_purge_eagerness_e { |
19 | PAC_PURGE_ALWAYS, |
20 | PAC_PURGE_NEVER, |
21 | PAC_PURGE_ON_EPOCH_ADVANCE |
22 | }; |
23 | typedef enum pac_purge_eagerness_e pac_purge_eagerness_t; |
24 | |
25 | typedef struct pac_decay_stats_s pac_decay_stats_t; |
26 | struct pac_decay_stats_s { |
27 | /* Total number of purge sweeps. */ |
28 | locked_u64_t npurge; |
29 | /* Total number of madvise calls made. */ |
30 | locked_u64_t nmadvise; |
31 | /* Total number of pages purged. */ |
32 | locked_u64_t purged; |
33 | }; |
34 | |
35 | typedef struct pac_estats_s pac_estats_t; |
36 | struct pac_estats_s { |
37 | /* |
38 | * Stats for a given index in the range [0, SC_NPSIZES] in the various |
39 | * ecache_ts. |
40 | * We track both bytes and # of extents: two extents in the same bucket |
41 | * may have different sizes if adjacent size classes differ by more than |
42 | * a page, so bytes cannot always be derived from # of extents. |
43 | */ |
44 | size_t ndirty; |
45 | size_t dirty_bytes; |
46 | size_t nmuzzy; |
47 | size_t muzzy_bytes; |
48 | size_t nretained; |
49 | size_t retained_bytes; |
50 | }; |
51 | |
52 | typedef struct pac_stats_s pac_stats_t; |
53 | struct pac_stats_s { |
54 | pac_decay_stats_t decay_dirty; |
55 | pac_decay_stats_t decay_muzzy; |
56 | |
57 | /* |
58 | * Number of unused virtual memory bytes currently retained. Retained |
59 | * bytes are technically mapped (though always decommitted or purged), |
60 | * but they are excluded from the mapped statistic (above). |
61 | */ |
62 | size_t retained; /* Derived. */ |
63 | |
64 | /* |
65 | * Number of bytes currently mapped, excluding retained memory (and any |
66 | * base-allocated memory, which is tracked by the arena stats). |
67 | * |
68 | * We name this "pac_mapped" to avoid confusion with the arena_stats |
69 | * "mapped". |
70 | */ |
71 | atomic_zu_t pac_mapped; |
72 | |
73 | /* VM space had to be leaked (undocumented). Normally 0. */ |
74 | atomic_zu_t abandoned_vm; |
75 | }; |
76 | |
77 | typedef struct pac_s pac_t; |
78 | struct pac_s { |
79 | /* |
80 | * Must be the first member (we convert it to a PAC given only a |
81 | * pointer). The handle to the allocation interface. |
82 | */ |
83 | pai_t pai; |
84 | /* |
85 | * Collections of extents that were previously allocated. These are |
86 | * used when allocating extents, in an attempt to re-use address space. |
87 | * |
88 | * Synchronization: internal. |
89 | */ |
90 | ecache_t ecache_dirty; |
91 | ecache_t ecache_muzzy; |
92 | ecache_t ecache_retained; |
93 | |
94 | base_t *base; |
95 | emap_t *emap; |
96 | edata_cache_t *edata_cache; |
97 | |
98 | /* The grow info for the retained ecache. */ |
99 | exp_grow_t exp_grow; |
100 | malloc_mutex_t grow_mtx; |
101 | |
102 | /* Special allocator for guarded frequently reused extents. */ |
103 | san_bump_alloc_t sba; |
104 | |
105 | /* How large extents should be before getting auto-purged. */ |
106 | atomic_zu_t oversize_threshold; |
107 | |
108 | /* |
109 | * Decay-based purging state, responsible for scheduling extent state |
110 | * transitions. |
111 | * |
112 | * Synchronization: via the internal mutex. |
113 | */ |
114 | decay_t decay_dirty; /* dirty --> muzzy */ |
115 | decay_t decay_muzzy; /* muzzy --> retained */ |
116 | |
117 | malloc_mutex_t *stats_mtx; |
118 | pac_stats_t *stats; |
119 | |
120 | /* Extent serial number generator state. */ |
121 | atomic_zu_t extent_sn_next; |
122 | }; |
123 | |
124 | bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, |
125 | edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold, |
126 | ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, |
127 | malloc_mutex_t *stats_mtx); |
128 | |
129 | static inline size_t |
130 | pac_mapped(pac_t *pac) { |
131 | return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED); |
132 | } |
133 | |
134 | static inline ehooks_t * |
135 | pac_ehooks_get(pac_t *pac) { |
136 | return base_ehooks_get(pac->base); |
137 | } |
138 | |
139 | /* |
140 | * All purging functions require holding decay->mtx. This is one of the few |
141 | * places external modules are allowed to peek inside pa_shard_t internals. |
142 | */ |
143 | |
144 | /* |
145 | * Decays the number of pages currently in the ecache. This might not leave the |
146 | * ecache empty if other threads are inserting dirty objects into it |
147 | * concurrently with the call. |
148 | */ |
149 | void pac_decay_all(tsdn_t *tsdn, pac_t *pac, decay_t *decay, |
150 | pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay); |
151 | /* |
152 | * Updates decay settings for the current time, and conditionally purges in |
153 | * response (depending on decay_purge_setting). Returns whether or not the |
154 | * epoch advanced. |
155 | */ |
156 | bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, |
157 | pac_decay_stats_t *decay_stats, ecache_t *ecache, |
158 | pac_purge_eagerness_t eagerness); |
159 | |
160 | /* |
161 | * Gets / sets the maximum amount that we'll grow an arena down the |
162 | * grow-retained pathways (unless forced to by an allocaction request). |
163 | * |
164 | * Set new_limit to NULL if it's just a query, or old_limit to NULL if you don't |
165 | * care about the previous value. |
166 | * |
167 | * Returns true on error (if the new limit is not valid). |
168 | */ |
169 | bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, |
170 | size_t *new_limit); |
171 | |
172 | bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, |
173 | ssize_t decay_ms, pac_purge_eagerness_t eagerness); |
174 | ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state); |
175 | |
176 | void pac_reset(tsdn_t *tsdn, pac_t *pac); |
177 | void pac_destroy(tsdn_t *tsdn, pac_t *pac); |
178 | |
179 | #endif /* JEMALLOC_INTERNAL_PAC_H */ |
180 | |