1#ifndef JEMALLOC_INTERNAL_PSSET_H
2#define JEMALLOC_INTERNAL_PSSET_H
3
4#include "jemalloc/internal/hpdata.h"
5
6/*
7 * A page-slab set. What the eset is to PAC, the psset is to HPA. It maintains
8 * a collection of page-slabs (the intent being that they are backed by
9 * hugepages, or at least could be), and handles allocation and deallocation
10 * requests.
11 */
12
13/*
14 * One more than the maximum pszind_t we will serve out of the HPA.
15 * Practically, we expect only the first few to be actually used. This
16 * corresponds to a maximum size of of 512MB on systems with 4k pages and
17 * SC_NGROUP == 4, which is already an unreasonably large maximum. Morally, you
18 * can think of this as being SC_NPSIZES, but there's no sense in wasting that
19 * much space in the arena, making bitmaps that much larger, etc.
20 */
21#define PSSET_NPSIZES 64
22
23/*
24 * We keep two purge lists per page size class; one for hugified hpdatas (at
25 * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
26 * 1). This lets us implement a preference for purging non-hugified hpdatas
27 * among similarly-dirty ones.
28 * We reserve the last two indices for empty slabs, in that case purging
29 * hugified ones (which are definitionally all waste) before non-hugified ones
30 * (i.e. reversing the order).
31 */
32#define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
33
34typedef struct psset_bin_stats_s psset_bin_stats_t;
35struct psset_bin_stats_s {
36 /* How many pageslabs are in this bin? */
37 size_t npageslabs;
38 /* Of them, how many pages are active? */
39 size_t nactive;
40 /* And how many are dirty? */
41 size_t ndirty;
42};
43
44typedef struct psset_stats_s psset_stats_t;
45struct psset_stats_s {
46 /*
47 * The second index is huge stats; nonfull_slabs[pszind][0] contains
48 * stats for the non-huge slabs in bucket pszind, while
49 * nonfull_slabs[pszind][1] contains stats for the huge slabs.
50 */
51 psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
52
53 /*
54 * Full slabs don't live in any edata heap, but we still track their
55 * stats.
56 */
57 psset_bin_stats_t full_slabs[2];
58
59 /* Empty slabs are similar. */
60 psset_bin_stats_t empty_slabs[2];
61};
62
63typedef struct psset_s psset_t;
64struct psset_s {
65 /*
66 * The pageslabs, quantized by the size class of the largest contiguous
67 * free run of pages in a pageslab.
68 */
69 hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
70 /* Bitmap for which set bits correspond to non-empty heaps. */
71 fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)];
72 /*
73 * The sum of all bin stats in stats. This lets us quickly answer
74 * queries for the number of dirty, active, and retained pages in the
75 * entire set.
76 */
77 psset_bin_stats_t merged_stats;
78 psset_stats_t stats;
79 /*
80 * Slabs with no active allocations, but which are allowed to serve new
81 * allocations.
82 */
83 hpdata_empty_list_t empty;
84 /*
85 * Slabs which are available to be purged, ordered by how much we want
86 * to purge them (with later indices indicating slabs we want to purge
87 * more).
88 */
89 hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
90 /* Bitmap for which set bits correspond to non-empty purge lists. */
91 fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];
92 /* Slabs which are available to be hugified. */
93 hpdata_hugify_list_t to_hugify;
94};
95
96void psset_init(psset_t *psset);
97void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
98
99/*
100 * Begin or end updating the given pageslab's metadata. While the pageslab is
101 * being updated, it won't be returned from psset_fit calls.
102 */
103void psset_update_begin(psset_t *psset, hpdata_t *ps);
104void psset_update_end(psset_t *psset, hpdata_t *ps);
105
106/* Analogous to the eset_fit; pick a hpdata to serve the request. */
107hpdata_t *psset_pick_alloc(psset_t *psset, size_t size);
108/* Pick one to purge. */
109hpdata_t *psset_pick_purge(psset_t *psset);
110/* Pick one to hugify. */
111hpdata_t *psset_pick_hugify(psset_t *psset);
112
113void psset_insert(psset_t *psset, hpdata_t *ps);
114void psset_remove(psset_t *psset, hpdata_t *ps);
115
116static inline size_t
117psset_npageslabs(psset_t *psset) {
118 return psset->merged_stats.npageslabs;
119}
120
121static inline size_t
122psset_nactive(psset_t *psset) {
123 return psset->merged_stats.nactive;
124}
125
126static inline size_t
127psset_ndirty(psset_t *psset) {
128 return psset->merged_stats.ndirty;
129}
130
131#endif /* JEMALLOC_INTERNAL_PSSET_H */
132