1 | #include "jemalloc/internal/jemalloc_preamble.h" |
2 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
3 | |
4 | #include "jemalloc/internal/bit_util.h" |
5 | #include "jemalloc/internal/cache_bin.h" |
6 | #include "jemalloc/internal/safety_check.h" |
7 | |
8 | void |
9 | cache_bin_info_init(cache_bin_info_t *info, |
10 | cache_bin_sz_t ncached_max) { |
11 | assert(ncached_max <= CACHE_BIN_NCACHED_MAX); |
12 | size_t stack_size = (size_t)ncached_max * sizeof(void *); |
13 | assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); |
14 | info->ncached_max = (cache_bin_sz_t)ncached_max; |
15 | } |
16 | |
17 | void |
18 | cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, |
19 | size_t *size, size_t *alignment) { |
20 | /* For the total bin stack region (per tcache), reserve 2 more slots so |
21 | * that |
22 | * 1) the empty position can be safely read on the fast path before |
23 | * checking "is_empty"; and |
24 | * 2) the cur_ptr can go beyond the empty position by 1 step safely on |
25 | * the fast path (i.e. no overflow). |
26 | */ |
27 | *size = sizeof(void *) * 2; |
28 | for (szind_t i = 0; i < ninfos; i++) { |
29 | assert(infos[i].ncached_max > 0); |
30 | *size += infos[i].ncached_max * sizeof(void *); |
31 | } |
32 | |
33 | /* |
34 | * Align to at least PAGE, to minimize the # of TLBs needed by the |
35 | * smaller sizes; also helps if the larger sizes don't get used at all. |
36 | */ |
37 | *alignment = PAGE; |
38 | } |
39 | |
40 | void |
41 | cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, |
42 | size_t *cur_offset) { |
43 | if (config_debug) { |
44 | size_t computed_size; |
45 | size_t computed_alignment; |
46 | |
47 | /* Pointer should be as aligned as we asked for. */ |
48 | cache_bin_info_compute_alloc(infos, ninfos, &computed_size, |
49 | &computed_alignment); |
50 | assert(((uintptr_t)alloc & (computed_alignment - 1)) == 0); |
51 | } |
52 | |
53 | *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = |
54 | cache_bin_preceding_junk; |
55 | *cur_offset += sizeof(void *); |
56 | } |
57 | |
58 | void |
59 | cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, |
60 | size_t *cur_offset) { |
61 | *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = |
62 | cache_bin_trailing_junk; |
63 | *cur_offset += sizeof(void *); |
64 | } |
65 | |
66 | void |
67 | cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, |
68 | size_t *cur_offset) { |
69 | /* |
70 | * The full_position points to the lowest available space. Allocations |
71 | * will access the slots toward higher addresses (for the benefit of |
72 | * adjacent prefetch). |
73 | */ |
74 | void *stack_cur = (void *)((uintptr_t)alloc + *cur_offset); |
75 | void *full_position = stack_cur; |
76 | uint16_t bin_stack_size = info->ncached_max * sizeof(void *); |
77 | |
78 | *cur_offset += bin_stack_size; |
79 | void *empty_position = (void *)((uintptr_t)alloc + *cur_offset); |
80 | |
81 | /* Init to the empty position. */ |
82 | bin->stack_head = (void **)empty_position; |
83 | bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head; |
84 | bin->low_bits_full = (uint16_t)(uintptr_t)full_position; |
85 | bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position; |
86 | cache_bin_sz_t free_spots = cache_bin_diff(bin, |
87 | bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head, |
88 | /* racy */ false); |
89 | assert(free_spots == bin_stack_size); |
90 | assert(cache_bin_ncached_get_local(bin, info) == 0); |
91 | assert(cache_bin_empty_position_get(bin) == empty_position); |
92 | |
93 | assert(bin_stack_size > 0 || empty_position == full_position); |
94 | } |
95 | |
96 | bool |
97 | cache_bin_still_zero_initialized(cache_bin_t *bin) { |
98 | return bin->stack_head == NULL; |
99 | } |
100 | |