1/* ----------------------------------------------------------------------------
2Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#pragma once
8#ifndef MIMALLOC_TYPES_H
9#define MIMALLOC_TYPES_H
10
11#include <stddef.h> // ptrdiff_t
12#include <stdint.h> // uintptr_t, uint16_t, etc
13#include "mimalloc-atomic.h" // _Atomic
14
15#ifdef _MSC_VER
16#pragma warning(disable:4214) // bitfield is not int
17#endif
18
19// Minimal alignment necessary. On most platforms 16 bytes are needed
20// due to SSE registers for example. This must be at least `sizeof(void*)`
21#ifndef MI_MAX_ALIGN_SIZE
22#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
23#endif
24
25// ------------------------------------------------------
26// Variants
27// ------------------------------------------------------
28
29// Define NDEBUG in the release version to disable assertions.
30// #define NDEBUG
31
32// Define MI_VALGRIND to enable valgrind support
33// #define MI_VALGRIND 1
34
35// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
36// #define MI_STAT 1
37
38// Define MI_SECURE to enable security mitigations
39// #define MI_SECURE 1 // guard page around metadata
40// #define MI_SECURE 2 // guard page around each mimalloc page
41// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
42// #define MI_SECURE 4 // checks for double free. (may be more expensive)
43
44#if !defined(MI_SECURE)
45#define MI_SECURE 0
46#endif
47
48// Define MI_DEBUG for debug mode
49// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free.
50// #define MI_DEBUG 2 // + internal assertion checks
51// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
52#if !defined(MI_DEBUG)
53#if !defined(NDEBUG) || defined(_DEBUG)
54#define MI_DEBUG 2
55#else
56#define MI_DEBUG 0
57#endif
58#endif
59
60// Reserve extra padding at the end of each block to be more resilient against heap block overflows.
61// The padding can detect byte-precise buffer overflow on free.
62#if !defined(MI_PADDING) && (MI_DEBUG>=1 || MI_VALGRIND)
63#define MI_PADDING 1
64#endif
65
66
67// Encoded free lists allow detection of corrupted free lists
68// and can detect buffer overflows, modify after free, and double `free`s.
69#if (MI_SECURE>=3 || MI_DEBUG>=1)
70#define MI_ENCODE_FREELIST 1
71#endif
72
73
74// ------------------------------------------------------
75// Platform specific values
76// ------------------------------------------------------
77
78// ------------------------------------------------------
79// Size of a pointer.
80// We assume that `sizeof(void*)==sizeof(intptr_t)`
81// and it holds for all platforms we know of.
82//
83// However, the C standard only requires that:
84// p == (void*)((intptr_t)p))
85// but we also need:
86// i == (intptr_t)((void*)i)
87// or otherwise one might define an intptr_t type that is larger than a pointer...
88// ------------------------------------------------------
89
90#if INTPTR_MAX > INT64_MAX
91# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
92#elif INTPTR_MAX == INT64_MAX
93# define MI_INTPTR_SHIFT (3)
94#elif INTPTR_MAX == INT32_MAX
95# define MI_INTPTR_SHIFT (2)
96#else
97#error platform pointers must be 32, 64, or 128 bits
98#endif
99
100#if SIZE_MAX == UINT64_MAX
101# define MI_SIZE_SHIFT (3)
102typedef int64_t mi_ssize_t;
103#elif SIZE_MAX == UINT32_MAX
104# define MI_SIZE_SHIFT (2)
105typedef int32_t mi_ssize_t;
106#else
107#error platform objects must be 32 or 64 bits
108#endif
109
110#if (SIZE_MAX/2) > LONG_MAX
111# define MI_ZU(x) x##ULL
112# define MI_ZI(x) x##LL
113#else
114# define MI_ZU(x) x##UL
115# define MI_ZI(x) x##L
116#endif
117
118#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
119#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
120
121#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
122#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
123
124#define MI_KiB (MI_ZU(1024))
125#define MI_MiB (MI_KiB*MI_KiB)
126#define MI_GiB (MI_MiB*MI_KiB)
127
128
129// ------------------------------------------------------
130// Main internal data-structures
131// ------------------------------------------------------
132
133// Main tuning parameters for segment and page sizes
134// Sizes for 64-bit (usually divide by two for 32-bit)
135#define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB (32KiB on 32-bit)
136
137#if MI_INTPTR_SIZE > 4
138#define MI_SEGMENT_SHIFT (10 + MI_SEGMENT_SLICE_SHIFT) // 64MiB
139#else
140#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit
141#endif
142
143#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB
144#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB
145
146
147// Derived constants
148#define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT)
149#define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE
150#define MI_SEGMENT_MASK (MI_SEGMENT_SIZE - 1)
151#define MI_SEGMENT_SLICE_SIZE (MI_ZU(1)<< MI_SEGMENT_SLICE_SHIFT)
152#define MI_SLICES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_SEGMENT_SLICE_SIZE) // 1024
153
154#define MI_SMALL_PAGE_SIZE (MI_ZU(1)<<MI_SMALL_PAGE_SHIFT)
155#define MI_MEDIUM_PAGE_SIZE (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT)
156
157#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit
158#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit
159#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
160#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit
161#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
162
163// Maximum number of size classes. (spaced exponentially in 12.5% increments)
164#define MI_BIN_HUGE (73U)
165
166#if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360)
167#error "mimalloc internal: define more bins"
168#endif
169#if (MI_ALIGNMENT_MAX > MI_SEGMENT_SIZE/2)
170#error "mimalloc internal: the max aligned boundary is too large for the segment size"
171#endif
172#if (MI_ALIGNED_MAX % MI_SEGMENT_SLICE_SIZE != 0)
173#error "mimalloc internal: the max aligned boundary must be an integral multiple of the segment slice size"
174#endif
175
176// Maximum slice offset (15)
177#define MI_MAX_SLICE_OFFSET ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1)
178
179// Used as a special value to encode block sizes in 32 bits.
180#define MI_HUGE_BLOCK_SIZE ((uint32_t)(2*MI_GiB))
181
182// blocks up to this size are always allocated aligned
183#define MI_MAX_ALIGN_GUARANTEE (8*MI_MAX_ALIGN_SIZE)
184
185
186
187
188// ------------------------------------------------------
189// Mimalloc pages contain allocated blocks
190// ------------------------------------------------------
191
192// The free lists use encoded next fields
193// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
194typedef uintptr_t mi_encoded_t;
195
196// thread id's
197typedef size_t mi_threadid_t;
198
199// free lists contain blocks
200typedef struct mi_block_s {
201 mi_encoded_t next;
202} mi_block_t;
203
204
205// The delayed flags are used for efficient multi-threaded free-ing
206typedef enum mi_delayed_e {
207 MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
208 MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
209 MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
210 MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim
211} mi_delayed_t;
212
213
214// The `in_full` and `has_aligned` page flags are put in a union to efficiently
215// test if both are false (`full_aligned == 0`) in the `mi_free` routine.
216#if !MI_TSAN
217typedef union mi_page_flags_s {
218 uint8_t full_aligned;
219 struct {
220 uint8_t in_full : 1;
221 uint8_t has_aligned : 1;
222 } x;
223} mi_page_flags_t;
224#else
225// under thread sanitizer, use a byte for each flag to suppress warning, issue #130
226typedef union mi_page_flags_s {
227 uint16_t full_aligned;
228 struct {
229 uint8_t in_full;
230 uint8_t has_aligned;
231 } x;
232} mi_page_flags_t;
233#endif
234
235// Thread free list.
236// We use the bottom 2 bits of the pointer for mi_delayed_t flags
237typedef uintptr_t mi_thread_free_t;
238
239// A page contains blocks of one specific size (`block_size`).
240// Each page has three list of free blocks:
241// `free` for blocks that can be allocated,
242// `local_free` for freed blocks that are not yet available to `mi_malloc`
243// `thread_free` for freed blocks by other threads
244// The `local_free` and `thread_free` lists are migrated to the `free` list
245// when it is exhausted. The separate `local_free` list is necessary to
246// implement a monotonic heartbeat. The `thread_free` list is needed for
247// avoiding atomic operations in the common case.
248//
249//
250// `used - |thread_free|` == actual blocks that are in use (alive)
251// `used - |thread_free| + |free| + |local_free| == capacity`
252//
253// We don't count `freed` (as |free|) but use `used` to reduce
254// the number of memory accesses in the `mi_page_all_free` function(s).
255//
256// Notes:
257// - Access is optimized for `mi_free` and `mi_page_alloc` (in `alloc.c`)
258// - Using `uint16_t` does not seem to slow things down
259// - The size is 8 words on 64-bit which helps the page index calculations
260// (and 10 words on 32-bit, and encoded free lists add 2 words. Sizes 10
261// and 12 are still good for address calculation)
262// - To limit the structure size, the `xblock_size` is 32-bits only; for
263// blocks > MI_HUGE_BLOCK_SIZE the size is determined from the segment page size
264// - `thread_free` uses the bottom bits as a delayed-free flags to optimize
265// concurrent frees where only the first concurrent free adds to the owning
266// heap `thread_delayed_free` list (see `alloc.c:mi_free_block_mt`).
267// The invariant is that no-delayed-free is only set if there is
268// at least one block that will be added, or as already been added, to
269// the owning heap `thread_delayed_free` list. This guarantees that pages
270// will be freed correctly even if only other threads free blocks.
271typedef struct mi_page_s {
272 // "owned" by the segment
273 uint32_t slice_count; // slices in this page (0 if not a page)
274 uint32_t slice_offset; // distance from the actual page data slice (0 if a page)
275 uint8_t is_reset : 1; // `true` if the page memory was reset
276 uint8_t is_committed : 1; // `true` if the page virtual memory is committed
277 uint8_t is_zero_init : 1; // `true` if the page was zero initialized
278
279 // layout like this to optimize access in `mi_malloc` and `mi_free`
280 uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear`
281 uint16_t reserved; // number of blocks reserved in memory
282 mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits)
283 uint8_t is_zero : 1; // `true` if the blocks in the free list are zero initialized
284 uint8_t retire_expire : 7; // expiration count for retired blocks
285
286 mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
287 #ifdef MI_ENCODE_FREELIST
288 uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`)
289 #endif
290 uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`)
291 uint32_t xblock_size; // size available in each block (always `>0`)
292
293 mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
294 _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads
295 _Atomic(uintptr_t) xheap;
296
297 struct mi_page_s* next; // next page owned by this thread with the same `block_size`
298 struct mi_page_s* prev; // previous page owned by this thread with the same `block_size`
299
300 // 64-bit 9 words, 32-bit 12 words, (+2 for secure)
301 #if MI_INTPTR_SIZE==8
302 uintptr_t padding[1];
303 #endif
304} mi_page_t;
305
306
307
308typedef enum mi_page_kind_e {
309 MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment
310 MI_PAGE_MEDIUM, // medium blocks go into medium pages inside a segment
311 MI_PAGE_LARGE, // larger blocks go into a page of just one block
312 MI_PAGE_HUGE, // huge blocks (> 16 MiB) are put into a single page in a single segment.
313} mi_page_kind_t;
314
315typedef enum mi_segment_kind_e {
316 MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside.
317 MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside.
318} mi_segment_kind_t;
319
320// ------------------------------------------------------
321// A segment holds a commit mask where a bit is set if
322// the corresponding MI_COMMIT_SIZE area is committed.
323// The MI_COMMIT_SIZE must be a multiple of the slice
324// size. If it is equal we have the most fine grained
325// decommit (but setting it higher can be more efficient).
326// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will
327// be committed in one go which can be set higher than
328// MI_COMMIT_SIZE for efficiency (while the decommit mask
329// is still tracked in fine-grained MI_COMMIT_SIZE chunks)
330// ------------------------------------------------------
331
332#define MI_MINIMAL_COMMIT_SIZE (2*MI_MiB)
333#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB
334#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)
335#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS
336#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)
337
338#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS))
339#error "the segment size must be exactly divisible by the (commit size * size_t bits)"
340#endif
341
342typedef struct mi_commit_mask_s {
343 size_t mask[MI_COMMIT_MASK_FIELD_COUNT];
344} mi_commit_mask_t;
345
346typedef mi_page_t mi_slice_t;
347typedef int64_t mi_msecs_t;
348
349
350// Segments are large allocated memory blocks (8mb on 64 bit) from
351// the OS. Inside segments we allocated fixed size _pages_ that
352// contain blocks.
353typedef struct mi_segment_s {
354 size_t memid; // memory id for arena allocation
355 bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
356 bool mem_is_large; // in large/huge os pages?
357 bool mem_is_committed; // `true` if the whole segment is eagerly committed
358
359 bool allow_decommit;
360 mi_msecs_t decommit_expire;
361 mi_commit_mask_t decommit_mask;
362 mi_commit_mask_t commit_mask;
363
364 _Atomic(struct mi_segment_s*) abandoned_next;
365
366 // from here is zero initialized
367 struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`)
368
369 size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
370 size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long)
371 size_t used; // count of pages in use
372 uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie`
373
374 size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT`
375 size_t segment_info_slices; // initial slices we are using segment info and possible guard pages.
376
377 // layout like this to optimize access in `mi_free`
378 mi_segment_kind_t kind;
379 _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment
380 size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT`
381 mi_slice_t slices[MI_SLICES_PER_SEGMENT];
382} mi_segment_t;
383
384
385// ------------------------------------------------------
386// Heaps
387// Provide first-class heaps to allocate from.
388// A heap just owns a set of pages for allocation and
389// can only be allocate/reallocate from the thread that created it.
390// Freeing blocks can be done from any thread though.
391// Per thread, the segments are shared among its heaps.
392// Per thread, there is always a default heap that is
393// used for allocation; it is initialized to statically
394// point to an empty heap to avoid initialization checks
395// in the fast path.
396// ------------------------------------------------------
397
398// Thread local data
399typedef struct mi_tld_s mi_tld_t;
400
401// Pages of a certain block size are held in a queue.
402typedef struct mi_page_queue_s {
403 mi_page_t* first;
404 mi_page_t* last;
405 size_t block_size;
406} mi_page_queue_t;
407
408#define MI_BIN_FULL (MI_BIN_HUGE+1)
409
410// Random context
411typedef struct mi_random_cxt_s {
412 uint32_t input[16];
413 uint32_t output[16];
414 int output_available;
415} mi_random_ctx_t;
416
417
418// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
419#if (MI_PADDING)
420typedef struct mi_padding_s {
421 uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
422 uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
423} mi_padding_t;
424#define MI_PADDING_SIZE (sizeof(mi_padding_t))
425#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
426#else
427#define MI_PADDING_SIZE 0
428#define MI_PADDING_WSIZE 0
429#endif
430
431#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
432
433
434// A heap owns a set of pages.
435struct mi_heap_s {
436 mi_tld_t* tld;
437 mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
438 mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
439 _Atomic(mi_block_t*) thread_delayed_free;
440 mi_threadid_t thread_id; // thread this heap belongs too
441 mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0)
442 uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
443 uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list
444 mi_random_ctx_t random; // random number context used for secure allocation
445 size_t page_count; // total number of pages in the `pages` queues.
446 size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
447 size_t page_retired_max; // largest retired index into the `pages` array.
448 mi_heap_t* next; // list of heaps per thread
449 bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
450};
451
452
453
454// ------------------------------------------------------
455// Debug
456// ------------------------------------------------------
457
458#if !defined(MI_DEBUG_UNINIT)
459#define MI_DEBUG_UNINIT (0xD0)
460#endif
461#if !defined(MI_DEBUG_FREED)
462#define MI_DEBUG_FREED (0xDF)
463#endif
464#if !defined(MI_DEBUG_PADDING)
465#define MI_DEBUG_PADDING (0xDE)
466#endif
467
468#if (MI_DEBUG)
469// use our own assertion to print without memory allocation
470void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func );
471#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
472#else
473#define mi_assert(x)
474#endif
475
476#if (MI_DEBUG>1)
477#define mi_assert_internal mi_assert
478#else
479#define mi_assert_internal(x)
480#endif
481
482#if (MI_DEBUG>2)
483#define mi_assert_expensive mi_assert
484#else
485#define mi_assert_expensive(x)
486#endif
487
488// ------------------------------------------------------
489// Statistics
490// ------------------------------------------------------
491
492#ifndef MI_STAT
493#if (MI_DEBUG>0)
494#define MI_STAT 2
495#else
496#define MI_STAT 0
497#endif
498#endif
499
500typedef struct mi_stat_count_s {
501 int64_t allocated;
502 int64_t freed;
503 int64_t peak;
504 int64_t current;
505} mi_stat_count_t;
506
507typedef struct mi_stat_counter_s {
508 int64_t total;
509 int64_t count;
510} mi_stat_counter_t;
511
512typedef struct mi_stats_s {
513 mi_stat_count_t segments;
514 mi_stat_count_t pages;
515 mi_stat_count_t reserved;
516 mi_stat_count_t committed;
517 mi_stat_count_t reset;
518 mi_stat_count_t page_committed;
519 mi_stat_count_t segments_abandoned;
520 mi_stat_count_t pages_abandoned;
521 mi_stat_count_t threads;
522 mi_stat_count_t normal;
523 mi_stat_count_t huge;
524 mi_stat_count_t large;
525 mi_stat_count_t malloc;
526 mi_stat_count_t segments_cache;
527 mi_stat_counter_t pages_extended;
528 mi_stat_counter_t mmap_calls;
529 mi_stat_counter_t commit_calls;
530 mi_stat_counter_t page_no_retire;
531 mi_stat_counter_t searches;
532 mi_stat_counter_t normal_count;
533 mi_stat_counter_t huge_count;
534 mi_stat_counter_t large_count;
535#if MI_STAT>1
536 mi_stat_count_t normal_bins[MI_BIN_HUGE+1];
537#endif
538} mi_stats_t;
539
540
541void _mi_stat_increase(mi_stat_count_t* stat, size_t amount);
542void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
543void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
544
545#if (MI_STAT)
546#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount)
547#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount)
548#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount)
549#else
550#define mi_stat_increase(stat,amount) (void)0
551#define mi_stat_decrease(stat,amount) (void)0
552#define mi_stat_counter_increase(stat,amount) (void)0
553#endif
554
555#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount)
556#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)
557#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)
558
559// ------------------------------------------------------
560// Thread Local data
561// ------------------------------------------------------
562
563// A "span" is is an available range of slices. The span queues keep
564// track of slice spans of at most the given `slice_count` (but more than the previous size class).
565typedef struct mi_span_queue_s {
566 mi_slice_t* first;
567 mi_slice_t* last;
568 size_t slice_count;
569} mi_span_queue_t;
570
571#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT)
572
573// OS thread local data
574typedef struct mi_os_tld_s {
575 size_t region_idx; // start point for next allocation
576 mi_stats_t* stats; // points to tld stats
577} mi_os_tld_t;
578
579
580// Segments thread local data
581typedef struct mi_segments_tld_s {
582 mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments
583 size_t count; // current number of segments;
584 size_t peak_count; // peak number of segments
585 size_t current_size; // current size of all segments
586 size_t peak_size; // peak size of all segments
587 mi_stats_t* stats; // points to tld stats
588 mi_os_tld_t* os; // points to os stats
589} mi_segments_tld_t;
590
591// Thread local data
592struct mi_tld_s {
593 unsigned long long heartbeat; // monotonic heartbeat count
594 bool recurse; // true if deferred was called; used to prevent infinite recursion.
595 mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted)
596 mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates)
597 mi_segments_tld_t segments; // segment tld
598 mi_os_tld_t os; // os tld
599 mi_stats_t stats; // statistics
600};
601
602#endif
603