1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | bool |
5 | edata_cache_init(edata_cache_t *edata_cache, base_t *base) { |
6 | edata_avail_new(&edata_cache->avail); |
7 | /* |
8 | * This is not strictly necessary, since the edata_cache_t is only |
9 | * created inside an arena, which is zeroed on creation. But this is |
10 | * handy as a safety measure. |
11 | */ |
12 | atomic_store_zu(&edata_cache->count, 0, ATOMIC_RELAXED); |
13 | if (malloc_mutex_init(&edata_cache->mtx, "edata_cache" , |
14 | WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) { |
15 | return true; |
16 | } |
17 | edata_cache->base = base; |
18 | return false; |
19 | } |
20 | |
21 | edata_t * |
22 | edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache) { |
23 | malloc_mutex_lock(tsdn, &edata_cache->mtx); |
24 | edata_t *edata = edata_avail_first(&edata_cache->avail); |
25 | if (edata == NULL) { |
26 | malloc_mutex_unlock(tsdn, &edata_cache->mtx); |
27 | return base_alloc_edata(tsdn, edata_cache->base); |
28 | } |
29 | edata_avail_remove(&edata_cache->avail, edata); |
30 | atomic_load_sub_store_zu(&edata_cache->count, 1); |
31 | malloc_mutex_unlock(tsdn, &edata_cache->mtx); |
32 | return edata; |
33 | } |
34 | |
35 | void |
36 | edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) { |
37 | malloc_mutex_lock(tsdn, &edata_cache->mtx); |
38 | edata_avail_insert(&edata_cache->avail, edata); |
39 | atomic_load_add_store_zu(&edata_cache->count, 1); |
40 | malloc_mutex_unlock(tsdn, &edata_cache->mtx); |
41 | } |
42 | |
43 | void |
44 | edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) { |
45 | malloc_mutex_prefork(tsdn, &edata_cache->mtx); |
46 | } |
47 | |
48 | void |
49 | edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) { |
50 | malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx); |
51 | } |
52 | |
53 | void |
54 | edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) { |
55 | malloc_mutex_postfork_child(tsdn, &edata_cache->mtx); |
56 | } |
57 | |
58 | void |
59 | edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback) { |
60 | edata_list_inactive_init(&ecs->list); |
61 | ecs->fallback = fallback; |
62 | ecs->disabled = false; |
63 | } |
64 | |
65 | static void |
66 | edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn, |
67 | edata_cache_fast_t *ecs) { |
68 | edata_t *edata; |
69 | malloc_mutex_lock(tsdn, &ecs->fallback->mtx); |
70 | for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) { |
71 | edata = edata_avail_remove_first(&ecs->fallback->avail); |
72 | if (edata == NULL) { |
73 | break; |
74 | } |
75 | edata_list_inactive_append(&ecs->list, edata); |
76 | atomic_load_sub_store_zu(&ecs->fallback->count, 1); |
77 | } |
78 | malloc_mutex_unlock(tsdn, &ecs->fallback->mtx); |
79 | } |
80 | |
81 | edata_t * |
82 | edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs) { |
83 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
84 | WITNESS_RANK_EDATA_CACHE, 0); |
85 | |
86 | if (ecs->disabled) { |
87 | assert(edata_list_inactive_first(&ecs->list) == NULL); |
88 | return edata_cache_get(tsdn, ecs->fallback); |
89 | } |
90 | |
91 | edata_t *edata = edata_list_inactive_first(&ecs->list); |
92 | if (edata != NULL) { |
93 | edata_list_inactive_remove(&ecs->list, edata); |
94 | return edata; |
95 | } |
96 | /* Slow path; requires synchronization. */ |
97 | edata_cache_fast_try_fill_from_fallback(tsdn, ecs); |
98 | edata = edata_list_inactive_first(&ecs->list); |
99 | if (edata != NULL) { |
100 | edata_list_inactive_remove(&ecs->list, edata); |
101 | } else { |
102 | /* |
103 | * Slowest path (fallback was also empty); allocate something |
104 | * new. |
105 | */ |
106 | edata = base_alloc_edata(tsdn, ecs->fallback->base); |
107 | } |
108 | return edata; |
109 | } |
110 | |
111 | static void |
112 | edata_cache_fast_flush_all(tsdn_t *tsdn, edata_cache_fast_t *ecs) { |
113 | /* |
114 | * You could imagine smarter cache management policies (like |
115 | * only flushing down to some threshold in anticipation of |
116 | * future get requests). But just flushing everything provides |
117 | * a good opportunity to defrag too, and lets us share code between the |
118 | * flush and disable pathways. |
119 | */ |
120 | edata_t *edata; |
121 | size_t nflushed = 0; |
122 | malloc_mutex_lock(tsdn, &ecs->fallback->mtx); |
123 | while ((edata = edata_list_inactive_first(&ecs->list)) != NULL) { |
124 | edata_list_inactive_remove(&ecs->list, edata); |
125 | edata_avail_insert(&ecs->fallback->avail, edata); |
126 | nflushed++; |
127 | } |
128 | atomic_load_add_store_zu(&ecs->fallback->count, nflushed); |
129 | malloc_mutex_unlock(tsdn, &ecs->fallback->mtx); |
130 | } |
131 | |
132 | void |
133 | edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata) { |
134 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), |
135 | WITNESS_RANK_EDATA_CACHE, 0); |
136 | |
137 | if (ecs->disabled) { |
138 | assert(edata_list_inactive_first(&ecs->list) == NULL); |
139 | edata_cache_put(tsdn, ecs->fallback, edata); |
140 | return; |
141 | } |
142 | |
143 | /* |
144 | * Prepend rather than append, to do LIFO ordering in the hopes of some |
145 | * cache locality. |
146 | */ |
147 | edata_list_inactive_prepend(&ecs->list, edata); |
148 | } |
149 | |
150 | void |
151 | edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs) { |
152 | edata_cache_fast_flush_all(tsdn, ecs); |
153 | ecs->disabled = true; |
154 | } |
155 | |