1#include "jemalloc/internal/jemalloc_preamble.h"
2#include "jemalloc/internal/jemalloc_internal_includes.h"
3
4#include "jemalloc/internal/bit_util.h"
5#include "jemalloc/internal/cache_bin.h"
6#include "jemalloc/internal/safety_check.h"
7
8void
9cache_bin_info_init(cache_bin_info_t *info,
10 cache_bin_sz_t ncached_max) {
11 assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
12 size_t stack_size = (size_t)ncached_max * sizeof(void *);
13 assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
14 info->ncached_max = (cache_bin_sz_t)ncached_max;
15}
16
17void
18cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
19 size_t *size, size_t *alignment) {
20 /* For the total bin stack region (per tcache), reserve 2 more slots so
21 * that
22 * 1) the empty position can be safely read on the fast path before
23 * checking "is_empty"; and
24 * 2) the cur_ptr can go beyond the empty position by 1 step safely on
25 * the fast path (i.e. no overflow).
26 */
27 *size = sizeof(void *) * 2;
28 for (szind_t i = 0; i < ninfos; i++) {
29 assert(infos[i].ncached_max > 0);
30 *size += infos[i].ncached_max * sizeof(void *);
31 }
32
33 /*
34 * Align to at least PAGE, to minimize the # of TLBs needed by the
35 * smaller sizes; also helps if the larger sizes don't get used at all.
36 */
37 *alignment = PAGE;
38}
39
40void
41cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc,
42 size_t *cur_offset) {
43 if (config_debug) {
44 size_t computed_size;
45 size_t computed_alignment;
46
47 /* Pointer should be as aligned as we asked for. */
48 cache_bin_info_compute_alloc(infos, ninfos, &computed_size,
49 &computed_alignment);
50 assert(((uintptr_t)alloc & (computed_alignment - 1)) == 0);
51 }
52
53 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) =
54 cache_bin_preceding_junk;
55 *cur_offset += sizeof(void *);
56}
57
58void
59cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc,
60 size_t *cur_offset) {
61 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) =
62 cache_bin_trailing_junk;
63 *cur_offset += sizeof(void *);
64}
65
66void
67cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
68 size_t *cur_offset) {
69 /*
70 * The full_position points to the lowest available space. Allocations
71 * will access the slots toward higher addresses (for the benefit of
72 * adjacent prefetch).
73 */
74 void *stack_cur = (void *)((uintptr_t)alloc + *cur_offset);
75 void *full_position = stack_cur;
76 uint16_t bin_stack_size = info->ncached_max * sizeof(void *);
77
78 *cur_offset += bin_stack_size;
79 void *empty_position = (void *)((uintptr_t)alloc + *cur_offset);
80
81 /* Init to the empty position. */
82 bin->stack_head = (void **)empty_position;
83 bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head;
84 bin->low_bits_full = (uint16_t)(uintptr_t)full_position;
85 bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position;
86 cache_bin_sz_t free_spots = cache_bin_diff(bin,
87 bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head,
88 /* racy */ false);
89 assert(free_spots == bin_stack_size);
90 assert(cache_bin_ncached_get_local(bin, info) == 0);
91 assert(cache_bin_empty_position_get(bin) == empty_position);
92
93 assert(bin_stack_size > 0 || empty_position == full_position);
94}
95
96bool
97cache_bin_still_zero_initialized(cache_bin_t *bin) {
98 return bin->stack_head == NULL;
99}
100