1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4bool
5edata_cache_init(edata_cache_t *edata_cache, base_t *base) {
6 edata_avail_new(&edata_cache->avail);
7 /*
8 * This is not strictly necessary, since the edata_cache_t is only
9 * created inside an arena, which is zeroed on creation. But this is
10 * handy as a safety measure.
11 */
12 atomic_store_zu(&edata_cache->count, 0, ATOMIC_RELAXED);
13 if (malloc_mutex_init(&edata_cache->mtx, "edata_cache",
14 WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) {
15 return true;
16 }
17 edata_cache->base = base;
18 return false;
19}
20
21edata_t *
22edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache) {
23 malloc_mutex_lock(tsdn, &edata_cache->mtx);
24 edata_t *edata = edata_avail_first(&edata_cache->avail);
25 if (edata == NULL) {
26 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
27 return base_alloc_edata(tsdn, edata_cache->base);
28 }
29 edata_avail_remove(&edata_cache->avail, edata);
30 atomic_load_sub_store_zu(&edata_cache->count, 1);
31 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
32 return edata;
33}
34
35void
36edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) {
37 malloc_mutex_lock(tsdn, &edata_cache->mtx);
38 edata_avail_insert(&edata_cache->avail, edata);
39 atomic_load_add_store_zu(&edata_cache->count, 1);
40 malloc_mutex_unlock(tsdn, &edata_cache->mtx);
41}
42
43void
44edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) {
45 malloc_mutex_prefork(tsdn, &edata_cache->mtx);
46}
47
48void
49edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) {
50 malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx);
51}
52
53void
54edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
55 malloc_mutex_postfork_child(tsdn, &edata_cache->mtx);
56}
57
58void
59edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback) {
60 edata_list_inactive_init(&ecs->list);
61 ecs->fallback = fallback;
62 ecs->disabled = false;
63}
64
65static void
66edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn,
67 edata_cache_fast_t *ecs) {
68 edata_t *edata;
69 malloc_mutex_lock(tsdn, &ecs->fallback->mtx);
70 for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) {
71 edata = edata_avail_remove_first(&ecs->fallback->avail);
72 if (edata == NULL) {
73 break;
74 }
75 edata_list_inactive_append(&ecs->list, edata);
76 atomic_load_sub_store_zu(&ecs->fallback->count, 1);
77 }
78 malloc_mutex_unlock(tsdn, &ecs->fallback->mtx);
79}
80
81edata_t *
82edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
83 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
84 WITNESS_RANK_EDATA_CACHE, 0);
85
86 if (ecs->disabled) {
87 assert(edata_list_inactive_first(&ecs->list) == NULL);
88 return edata_cache_get(tsdn, ecs->fallback);
89 }
90
91 edata_t *edata = edata_list_inactive_first(&ecs->list);
92 if (edata != NULL) {
93 edata_list_inactive_remove(&ecs->list, edata);
94 return edata;
95 }
96 /* Slow path; requires synchronization. */
97 edata_cache_fast_try_fill_from_fallback(tsdn, ecs);
98 edata = edata_list_inactive_first(&ecs->list);
99 if (edata != NULL) {
100 edata_list_inactive_remove(&ecs->list, edata);
101 } else {
102 /*
103 * Slowest path (fallback was also empty); allocate something
104 * new.
105 */
106 edata = base_alloc_edata(tsdn, ecs->fallback->base);
107 }
108 return edata;
109}
110
111static void
112edata_cache_fast_flush_all(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
113 /*
114 * You could imagine smarter cache management policies (like
115 * only flushing down to some threshold in anticipation of
116 * future get requests). But just flushing everything provides
117 * a good opportunity to defrag too, and lets us share code between the
118 * flush and disable pathways.
119 */
120 edata_t *edata;
121 size_t nflushed = 0;
122 malloc_mutex_lock(tsdn, &ecs->fallback->mtx);
123 while ((edata = edata_list_inactive_first(&ecs->list)) != NULL) {
124 edata_list_inactive_remove(&ecs->list, edata);
125 edata_avail_insert(&ecs->fallback->avail, edata);
126 nflushed++;
127 }
128 atomic_load_add_store_zu(&ecs->fallback->count, nflushed);
129 malloc_mutex_unlock(tsdn, &ecs->fallback->mtx);
130}
131
132void
133edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata) {
134 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
135 WITNESS_RANK_EDATA_CACHE, 0);
136
137 if (ecs->disabled) {
138 assert(edata_list_inactive_first(&ecs->list) == NULL);
139 edata_cache_put(tsdn, ecs->fallback, edata);
140 return;
141 }
142
143 /*
144 * Prepend rather than append, to do LIFO ordering in the hopes of some
145 * cache locality.
146 */
147 edata_list_inactive_prepend(&ecs->list, edata);
148}
149
150void
151edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs) {
152 edata_cache_fast_flush_all(tsdn, ecs);
153 ecs->disabled = true;
154}
155