1 | #ifndef JEMALLOC_INTERNAL_PSSET_H |
2 | #define JEMALLOC_INTERNAL_PSSET_H |
3 | |
4 | #include "jemalloc/internal/hpdata.h" |
5 | |
6 | /* |
7 | * A page-slab set. What the eset is to PAC, the psset is to HPA. It maintains |
8 | * a collection of page-slabs (the intent being that they are backed by |
9 | * hugepages, or at least could be), and handles allocation and deallocation |
10 | * requests. |
11 | */ |
12 | |
13 | /* |
14 | * One more than the maximum pszind_t we will serve out of the HPA. |
15 | * Practically, we expect only the first few to be actually used. This |
16 | * corresponds to a maximum size of of 512MB on systems with 4k pages and |
17 | * SC_NGROUP == 4, which is already an unreasonably large maximum. Morally, you |
18 | * can think of this as being SC_NPSIZES, but there's no sense in wasting that |
19 | * much space in the arena, making bitmaps that much larger, etc. |
20 | */ |
21 | #define PSSET_NPSIZES 64 |
22 | |
23 | /* |
24 | * We keep two purge lists per page size class; one for hugified hpdatas (at |
25 | * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind + |
26 | * 1). This lets us implement a preference for purging non-hugified hpdatas |
27 | * among similarly-dirty ones. |
28 | * We reserve the last two indices for empty slabs, in that case purging |
29 | * hugified ones (which are definitionally all waste) before non-hugified ones |
30 | * (i.e. reversing the order). |
31 | */ |
32 | #define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES) |
33 | |
34 | typedef struct psset_bin_stats_s psset_bin_stats_t; |
35 | struct psset_bin_stats_s { |
36 | /* How many pageslabs are in this bin? */ |
37 | size_t npageslabs; |
38 | /* Of them, how many pages are active? */ |
39 | size_t nactive; |
40 | /* And how many are dirty? */ |
41 | size_t ndirty; |
42 | }; |
43 | |
44 | typedef struct psset_stats_s psset_stats_t; |
45 | struct psset_stats_s { |
46 | /* |
47 | * The second index is huge stats; nonfull_slabs[pszind][0] contains |
48 | * stats for the non-huge slabs in bucket pszind, while |
49 | * nonfull_slabs[pszind][1] contains stats for the huge slabs. |
50 | */ |
51 | psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2]; |
52 | |
53 | /* |
54 | * Full slabs don't live in any edata heap, but we still track their |
55 | * stats. |
56 | */ |
57 | psset_bin_stats_t full_slabs[2]; |
58 | |
59 | /* Empty slabs are similar. */ |
60 | psset_bin_stats_t empty_slabs[2]; |
61 | }; |
62 | |
63 | typedef struct psset_s psset_t; |
64 | struct psset_s { |
65 | /* |
66 | * The pageslabs, quantized by the size class of the largest contiguous |
67 | * free run of pages in a pageslab. |
68 | */ |
69 | hpdata_age_heap_t pageslabs[PSSET_NPSIZES]; |
70 | /* Bitmap for which set bits correspond to non-empty heaps. */ |
71 | fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)]; |
72 | /* |
73 | * The sum of all bin stats in stats. This lets us quickly answer |
74 | * queries for the number of dirty, active, and retained pages in the |
75 | * entire set. |
76 | */ |
77 | psset_bin_stats_t merged_stats; |
78 | psset_stats_t stats; |
79 | /* |
80 | * Slabs with no active allocations, but which are allowed to serve new |
81 | * allocations. |
82 | */ |
83 | hpdata_empty_list_t empty; |
84 | /* |
85 | * Slabs which are available to be purged, ordered by how much we want |
86 | * to purge them (with later indices indicating slabs we want to purge |
87 | * more). |
88 | */ |
89 | hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS]; |
90 | /* Bitmap for which set bits correspond to non-empty purge lists. */ |
91 | fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)]; |
92 | /* Slabs which are available to be hugified. */ |
93 | hpdata_hugify_list_t to_hugify; |
94 | }; |
95 | |
96 | void psset_init(psset_t *psset); |
97 | void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src); |
98 | |
99 | /* |
100 | * Begin or end updating the given pageslab's metadata. While the pageslab is |
101 | * being updated, it won't be returned from psset_fit calls. |
102 | */ |
103 | void psset_update_begin(psset_t *psset, hpdata_t *ps); |
104 | void psset_update_end(psset_t *psset, hpdata_t *ps); |
105 | |
106 | /* Analogous to the eset_fit; pick a hpdata to serve the request. */ |
107 | hpdata_t *psset_pick_alloc(psset_t *psset, size_t size); |
108 | /* Pick one to purge. */ |
109 | hpdata_t *psset_pick_purge(psset_t *psset); |
110 | /* Pick one to hugify. */ |
111 | hpdata_t *psset_pick_hugify(psset_t *psset); |
112 | |
113 | void psset_insert(psset_t *psset, hpdata_t *ps); |
114 | void psset_remove(psset_t *psset, hpdata_t *ps); |
115 | |
116 | static inline size_t |
117 | psset_npageslabs(psset_t *psset) { |
118 | return psset->merged_stats.npageslabs; |
119 | } |
120 | |
121 | static inline size_t |
122 | psset_nactive(psset_t *psset) { |
123 | return psset->merged_stats.nactive; |
124 | } |
125 | |
126 | static inline size_t |
127 | psset_ndirty(psset_t *psset) { |
128 | return psset->merged_stats.ndirty; |
129 | } |
130 | |
131 | #endif /* JEMALLOC_INTERNAL_PSSET_H */ |
132 | |