1#ifndef JEMALLOC_INTERNAL_SEC_H
2#define JEMALLOC_INTERNAL_SEC_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/pai.h"
6
7/*
8 * Small extent cache.
9 *
10 * This includes some utilities to cache small extents. We have a per-pszind
11 * bin with its own list of extents of that size. We don't try to do any
12 * coalescing of extents (since it would in general require cross-shard locks or
13 * knowledge of the underlying PAI implementation).
14 */
15
16/*
17 * For now, this is just one field; eventually, we'll probably want to get more
18 * fine-grained data out (like per-size class statistics).
19 */
20typedef struct sec_stats_s sec_stats_t;
21struct sec_stats_s {
22 /* Sum of bytes_cur across all shards. */
23 size_t bytes;
24};
25
26static inline void
27sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
28 dst->bytes += src->bytes;
29}
30
31/* A collections of free extents, all of the same size. */
32typedef struct sec_bin_s sec_bin_t;
33struct sec_bin_s {
34 /*
35 * When we fail to fulfill an allocation, we do a batch-alloc on the
36 * underlying allocator to fill extra items, as well. We drop the SEC
37 * lock while doing so, to allow operations on other bins to succeed.
38 * That introduces the possibility of other threads also trying to
39 * allocate out of this bin, failing, and also going to the backing
40 * allocator. To avoid a thundering herd problem in which lots of
41 * threads do batch allocs and overfill this bin as a result, we only
42 * allow one batch allocation at a time for a bin. This bool tracks
43 * whether or not some thread is already batch allocating.
44 *
45 * Eventually, the right answer may be a smarter sharding policy for the
46 * bins (e.g. a mutex per bin, which would also be more scalable
47 * generally; the batch-allocating thread could hold it while
48 * batch-allocating).
49 */
50 bool being_batch_filled;
51
52 /*
53 * Number of bytes in this particular bin (as opposed to the
54 * sec_shard_t's bytes_cur. This isn't user visible or reported in
55 * stats; rather, it allows us to quickly determine the change in the
56 * centralized counter when flushing.
57 */
58 size_t bytes_cur;
59 edata_list_active_t freelist;
60};
61
62typedef struct sec_shard_s sec_shard_t;
63struct sec_shard_s {
64 /*
65 * We don't keep per-bin mutexes, even though that would allow more
66 * sharding; this allows global cache-eviction, which in turn allows for
67 * better balancing across free lists.
68 */
69 malloc_mutex_t mtx;
70 /*
71 * A SEC may need to be shut down (i.e. flushed of its contents and
72 * prevented from further caching). To avoid tricky synchronization
73 * issues, we just track enabled-status in each shard, guarded by a
74 * mutex. In practice, this is only ever checked during brief races,
75 * since the arena-level atomic boolean tracking HPA enabled-ness means
76 * that we won't go down these pathways very often after custom extent
77 * hooks are installed.
78 */
79 bool enabled;
80 sec_bin_t *bins;
81 /* Number of bytes in all bins in the shard. */
82 size_t bytes_cur;
83 /* The next pszind to flush in the flush-some pathways. */
84 pszind_t to_flush_next;
85};
86
87typedef struct sec_s sec_t;
88struct sec_s {
89 pai_t pai;
90 pai_t *fallback;
91
92 sec_opts_t opts;
93 sec_shard_t *shards;
94 pszind_t npsizes;
95};
96
97bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
98 const sec_opts_t *opts);
99void sec_flush(tsdn_t *tsdn, sec_t *sec);
100void sec_disable(tsdn_t *tsdn, sec_t *sec);
101
102/*
103 * Morally, these two stats methods probably ought to be a single one (and the
104 * mutex_prof_data ought to live in the sec_stats_t. But splitting them apart
105 * lets them fit easily into the pa_shard stats framework (which also has this
106 * split), which simplifies the stats management.
107 */
108void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats);
109void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec,
110 mutex_prof_data_t *mutex_prof_data);
111
112/*
113 * We use the arena lock ordering; these are acquired in phase 2 of forking, but
114 * should be acquired before the underlying allocator mutexes.
115 */
116void sec_prefork2(tsdn_t *tsdn, sec_t *sec);
117void sec_postfork_parent(tsdn_t *tsdn, sec_t *sec);
118void sec_postfork_child(tsdn_t *tsdn, sec_t *sec);
119
120#endif /* JEMALLOC_INTERNAL_SEC_H */
121