1 | /* ---------------------------------------------------------------------------- |
2 | Copyright (c) 2018-2021, Microsoft Research, Daan Leijen |
3 | This is free software; you can redistribute it and/or modify it under the |
4 | terms of the MIT license. A copy of the license can be found in the file |
5 | "LICENSE" at the root of this distribution. |
6 | -----------------------------------------------------------------------------*/ |
7 | #pragma once |
8 | #ifndef MIMALLOC_TYPES_H |
9 | #define MIMALLOC_TYPES_H |
10 | |
11 | #include <stddef.h> // ptrdiff_t |
12 | #include <stdint.h> // uintptr_t, uint16_t, etc |
13 | #include "mimalloc-atomic.h" // _Atomic |
14 | |
15 | #ifdef _MSC_VER |
16 | #pragma warning(disable:4214) // bitfield is not int |
17 | #endif |
18 | |
19 | // Minimal alignment necessary. On most platforms 16 bytes are needed |
20 | // due to SSE registers for example. This must be at least `sizeof(void*)` |
21 | #ifndef MI_MAX_ALIGN_SIZE |
22 | #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) |
23 | #endif |
24 | |
25 | // ------------------------------------------------------ |
26 | // Variants |
27 | // ------------------------------------------------------ |
28 | |
29 | // Define NDEBUG in the release version to disable assertions. |
30 | // #define NDEBUG |
31 | |
32 | // Define MI_VALGRIND to enable valgrind support |
33 | // #define MI_VALGRIND 1 |
34 | |
35 | // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). |
36 | // #define MI_STAT 1 |
37 | |
38 | // Define MI_SECURE to enable security mitigations |
39 | // #define MI_SECURE 1 // guard page around metadata |
40 | // #define MI_SECURE 2 // guard page around each mimalloc page |
41 | // #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) |
42 | // #define MI_SECURE 4 // checks for double free. (may be more expensive) |
43 | |
44 | #if !defined(MI_SECURE) |
45 | #define MI_SECURE 0 |
46 | #endif |
47 | |
48 | // Define MI_DEBUG for debug mode |
49 | // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. |
50 | // #define MI_DEBUG 2 // + internal assertion checks |
51 | // #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON) |
52 | #if !defined(MI_DEBUG) |
53 | #if !defined(NDEBUG) || defined(_DEBUG) |
54 | #define MI_DEBUG 2 |
55 | #else |
56 | #define MI_DEBUG 0 |
57 | #endif |
58 | #endif |
59 | |
60 | // Reserve extra padding at the end of each block to be more resilient against heap block overflows. |
61 | // The padding can detect byte-precise buffer overflow on free. |
62 | #if !defined(MI_PADDING) && (MI_DEBUG>=1 || MI_VALGRIND) |
63 | #define MI_PADDING 1 |
64 | #endif |
65 | |
66 | |
67 | // Encoded free lists allow detection of corrupted free lists |
68 | // and can detect buffer overflows, modify after free, and double `free`s. |
69 | #if (MI_SECURE>=3 || MI_DEBUG>=1) |
70 | #define MI_ENCODE_FREELIST 1 |
71 | #endif |
72 | |
73 | |
74 | // ------------------------------------------------------ |
75 | // Platform specific values |
76 | // ------------------------------------------------------ |
77 | |
78 | // ------------------------------------------------------ |
79 | // Size of a pointer. |
80 | // We assume that `sizeof(void*)==sizeof(intptr_t)` |
81 | // and it holds for all platforms we know of. |
82 | // |
83 | // However, the C standard only requires that: |
84 | // p == (void*)((intptr_t)p)) |
85 | // but we also need: |
86 | // i == (intptr_t)((void*)i) |
87 | // or otherwise one might define an intptr_t type that is larger than a pointer... |
88 | // ------------------------------------------------------ |
89 | |
90 | #if INTPTR_MAX > INT64_MAX |
91 | # define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) |
92 | #elif INTPTR_MAX == INT64_MAX |
93 | # define MI_INTPTR_SHIFT (3) |
94 | #elif INTPTR_MAX == INT32_MAX |
95 | # define MI_INTPTR_SHIFT (2) |
96 | #else |
97 | #error platform pointers must be 32, 64, or 128 bits |
98 | #endif |
99 | |
100 | #if SIZE_MAX == UINT64_MAX |
101 | # define MI_SIZE_SHIFT (3) |
102 | typedef int64_t mi_ssize_t; |
103 | #elif SIZE_MAX == UINT32_MAX |
104 | # define MI_SIZE_SHIFT (2) |
105 | typedef int32_t mi_ssize_t; |
106 | #else |
107 | #error platform objects must be 32 or 64 bits |
108 | #endif |
109 | |
110 | #if (SIZE_MAX/2) > LONG_MAX |
111 | # define MI_ZU(x) x##ULL |
112 | # define MI_ZI(x) x##LL |
113 | #else |
114 | # define MI_ZU(x) x##UL |
115 | # define MI_ZI(x) x##L |
116 | #endif |
117 | |
118 | #define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT) |
119 | #define MI_INTPTR_BITS (MI_INTPTR_SIZE*8) |
120 | |
121 | #define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT) |
122 | #define MI_SIZE_BITS (MI_SIZE_SIZE*8) |
123 | |
124 | #define MI_KiB (MI_ZU(1024)) |
125 | #define MI_MiB (MI_KiB*MI_KiB) |
126 | #define MI_GiB (MI_MiB*MI_KiB) |
127 | |
128 | |
129 | // ------------------------------------------------------ |
130 | // Main internal data-structures |
131 | // ------------------------------------------------------ |
132 | |
133 | // Main tuning parameters for segment and page sizes |
134 | // Sizes for 64-bit (usually divide by two for 32-bit) |
135 | #define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB (32KiB on 32-bit) |
136 | |
137 | #if MI_INTPTR_SIZE > 4 |
138 | #define MI_SEGMENT_SHIFT (10 + MI_SEGMENT_SLICE_SHIFT) // 64MiB |
139 | #else |
140 | #define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit |
141 | #endif |
142 | |
143 | #define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB |
144 | #define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB |
145 | |
146 | |
147 | // Derived constants |
148 | #define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT) |
149 | #define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE |
150 | #define MI_SEGMENT_MASK (MI_SEGMENT_SIZE - 1) |
151 | #define MI_SEGMENT_SLICE_SIZE (MI_ZU(1)<< MI_SEGMENT_SLICE_SHIFT) |
152 | #define MI_SLICES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_SEGMENT_SLICE_SIZE) // 1024 |
153 | |
154 | #define MI_SMALL_PAGE_SIZE (MI_ZU(1)<<MI_SMALL_PAGE_SHIFT) |
155 | #define MI_MEDIUM_PAGE_SIZE (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT) |
156 | |
157 | #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit |
158 | #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit |
159 | #define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) |
160 | #define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit |
161 | #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) |
162 | |
163 | // Maximum number of size classes. (spaced exponentially in 12.5% increments) |
164 | #define MI_BIN_HUGE (73U) |
165 | |
166 | #if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360) |
167 | #error "mimalloc internal: define more bins" |
168 | #endif |
169 | #if (MI_ALIGNMENT_MAX > MI_SEGMENT_SIZE/2) |
170 | #error "mimalloc internal: the max aligned boundary is too large for the segment size" |
171 | #endif |
172 | #if (MI_ALIGNED_MAX % MI_SEGMENT_SLICE_SIZE != 0) |
173 | #error "mimalloc internal: the max aligned boundary must be an integral multiple of the segment slice size" |
174 | #endif |
175 | |
176 | // Maximum slice offset (15) |
177 | #define MI_MAX_SLICE_OFFSET ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1) |
178 | |
179 | // Used as a special value to encode block sizes in 32 bits. |
180 | #define MI_HUGE_BLOCK_SIZE ((uint32_t)(2*MI_GiB)) |
181 | |
182 | // blocks up to this size are always allocated aligned |
183 | #define MI_MAX_ALIGN_GUARANTEE (8*MI_MAX_ALIGN_SIZE) |
184 | |
185 | |
186 | |
187 | |
188 | // ------------------------------------------------------ |
189 | // Mimalloc pages contain allocated blocks |
190 | // ------------------------------------------------------ |
191 | |
192 | // The free lists use encoded next fields |
193 | // (Only actually encodes when MI_ENCODED_FREELIST is defined.) |
194 | typedef uintptr_t mi_encoded_t; |
195 | |
196 | // thread id's |
197 | typedef size_t mi_threadid_t; |
198 | |
199 | // free lists contain blocks |
200 | typedef struct mi_block_s { |
201 | mi_encoded_t next; |
202 | } mi_block_t; |
203 | |
204 | |
205 | // The delayed flags are used for efficient multi-threaded free-ing |
206 | typedef enum mi_delayed_e { |
207 | MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list |
208 | MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap |
209 | MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list |
210 | MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim |
211 | } mi_delayed_t; |
212 | |
213 | |
214 | // The `in_full` and `has_aligned` page flags are put in a union to efficiently |
215 | // test if both are false (`full_aligned == 0`) in the `mi_free` routine. |
216 | #if !MI_TSAN |
217 | typedef union mi_page_flags_s { |
218 | uint8_t full_aligned; |
219 | struct { |
220 | uint8_t in_full : 1; |
221 | uint8_t has_aligned : 1; |
222 | } x; |
223 | } mi_page_flags_t; |
224 | #else |
225 | // under thread sanitizer, use a byte for each flag to suppress warning, issue #130 |
226 | typedef union mi_page_flags_s { |
227 | uint16_t full_aligned; |
228 | struct { |
229 | uint8_t in_full; |
230 | uint8_t has_aligned; |
231 | } x; |
232 | } mi_page_flags_t; |
233 | #endif |
234 | |
235 | // Thread free list. |
236 | // We use the bottom 2 bits of the pointer for mi_delayed_t flags |
237 | typedef uintptr_t mi_thread_free_t; |
238 | |
239 | // A page contains blocks of one specific size (`block_size`). |
240 | // Each page has three list of free blocks: |
241 | // `free` for blocks that can be allocated, |
242 | // `local_free` for freed blocks that are not yet available to `mi_malloc` |
243 | // `thread_free` for freed blocks by other threads |
244 | // The `local_free` and `thread_free` lists are migrated to the `free` list |
245 | // when it is exhausted. The separate `local_free` list is necessary to |
246 | // implement a monotonic heartbeat. The `thread_free` list is needed for |
247 | // avoiding atomic operations in the common case. |
248 | // |
249 | // |
250 | // `used - |thread_free|` == actual blocks that are in use (alive) |
251 | // `used - |thread_free| + |free| + |local_free| == capacity` |
252 | // |
253 | // We don't count `freed` (as |free|) but use `used` to reduce |
254 | // the number of memory accesses in the `mi_page_all_free` function(s). |
255 | // |
256 | // Notes: |
257 | // - Access is optimized for `mi_free` and `mi_page_alloc` (in `alloc.c`) |
258 | // - Using `uint16_t` does not seem to slow things down |
259 | // - The size is 8 words on 64-bit which helps the page index calculations |
260 | // (and 10 words on 32-bit, and encoded free lists add 2 words. Sizes 10 |
261 | // and 12 are still good for address calculation) |
262 | // - To limit the structure size, the `xblock_size` is 32-bits only; for |
263 | // blocks > MI_HUGE_BLOCK_SIZE the size is determined from the segment page size |
264 | // - `thread_free` uses the bottom bits as a delayed-free flags to optimize |
265 | // concurrent frees where only the first concurrent free adds to the owning |
266 | // heap `thread_delayed_free` list (see `alloc.c:mi_free_block_mt`). |
267 | // The invariant is that no-delayed-free is only set if there is |
268 | // at least one block that will be added, or as already been added, to |
269 | // the owning heap `thread_delayed_free` list. This guarantees that pages |
270 | // will be freed correctly even if only other threads free blocks. |
271 | typedef struct mi_page_s { |
272 | // "owned" by the segment |
273 | uint32_t slice_count; // slices in this page (0 if not a page) |
274 | uint32_t slice_offset; // distance from the actual page data slice (0 if a page) |
275 | uint8_t is_reset : 1; // `true` if the page memory was reset |
276 | uint8_t is_committed : 1; // `true` if the page virtual memory is committed |
277 | uint8_t is_zero_init : 1; // `true` if the page was zero initialized |
278 | |
279 | // layout like this to optimize access in `mi_malloc` and `mi_free` |
280 | uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` |
281 | uint16_t reserved; // number of blocks reserved in memory |
282 | mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) |
283 | uint8_t is_zero : 1; // `true` if the blocks in the free list are zero initialized |
284 | uint8_t retire_expire : 7; // expiration count for retired blocks |
285 | |
286 | mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) |
287 | #ifdef MI_ENCODE_FREELIST |
288 | uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) |
289 | #endif |
290 | uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) |
291 | uint32_t xblock_size; // size available in each block (always `>0`) |
292 | |
293 | mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) |
294 | _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads |
295 | _Atomic(uintptr_t) xheap; |
296 | |
297 | struct mi_page_s* next; // next page owned by this thread with the same `block_size` |
298 | struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` |
299 | |
300 | // 64-bit 9 words, 32-bit 12 words, (+2 for secure) |
301 | #if MI_INTPTR_SIZE==8 |
302 | uintptr_t padding[1]; |
303 | #endif |
304 | } mi_page_t; |
305 | |
306 | |
307 | |
308 | typedef enum mi_page_kind_e { |
309 | MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment |
310 | MI_PAGE_MEDIUM, // medium blocks go into medium pages inside a segment |
311 | MI_PAGE_LARGE, // larger blocks go into a page of just one block |
312 | MI_PAGE_HUGE, // huge blocks (> 16 MiB) are put into a single page in a single segment. |
313 | } mi_page_kind_t; |
314 | |
315 | typedef enum mi_segment_kind_e { |
316 | MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside. |
317 | MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside. |
318 | } mi_segment_kind_t; |
319 | |
320 | // ------------------------------------------------------ |
321 | // A segment holds a commit mask where a bit is set if |
322 | // the corresponding MI_COMMIT_SIZE area is committed. |
323 | // The MI_COMMIT_SIZE must be a multiple of the slice |
324 | // size. If it is equal we have the most fine grained |
325 | // decommit (but setting it higher can be more efficient). |
326 | // The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will |
327 | // be committed in one go which can be set higher than |
328 | // MI_COMMIT_SIZE for efficiency (while the decommit mask |
329 | // is still tracked in fine-grained MI_COMMIT_SIZE chunks) |
330 | // ------------------------------------------------------ |
331 | |
332 | #define MI_MINIMAL_COMMIT_SIZE (2*MI_MiB) |
333 | #define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB |
334 | #define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) |
335 | #define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS |
336 | #define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) |
337 | |
338 | #if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS)) |
339 | #error "the segment size must be exactly divisible by the (commit size * size_t bits)" |
340 | #endif |
341 | |
342 | typedef struct mi_commit_mask_s { |
343 | size_t mask[MI_COMMIT_MASK_FIELD_COUNT]; |
344 | } mi_commit_mask_t; |
345 | |
346 | typedef mi_page_t mi_slice_t; |
347 | typedef int64_t mi_msecs_t; |
348 | |
349 | |
350 | // Segments are large allocated memory blocks (8mb on 64 bit) from |
351 | // the OS. Inside segments we allocated fixed size _pages_ that |
352 | // contain blocks. |
353 | typedef struct mi_segment_s { |
354 | size_t memid; // memory id for arena allocation |
355 | bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) |
356 | bool mem_is_large; // in large/huge os pages? |
357 | bool mem_is_committed; // `true` if the whole segment is eagerly committed |
358 | |
359 | bool allow_decommit; |
360 | mi_msecs_t decommit_expire; |
361 | mi_commit_mask_t decommit_mask; |
362 | mi_commit_mask_t commit_mask; |
363 | |
364 | _Atomic(struct mi_segment_s*) abandoned_next; |
365 | |
366 | // from here is zero initialized |
367 | struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) |
368 | |
369 | size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) |
370 | size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) |
371 | size_t used; // count of pages in use |
372 | uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` |
373 | |
374 | size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT` |
375 | size_t segment_info_slices; // initial slices we are using segment info and possible guard pages. |
376 | |
377 | // layout like this to optimize access in `mi_free` |
378 | mi_segment_kind_t kind; |
379 | _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment |
380 | size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT` |
381 | mi_slice_t slices[MI_SLICES_PER_SEGMENT]; |
382 | } mi_segment_t; |
383 | |
384 | |
385 | // ------------------------------------------------------ |
386 | // Heaps |
387 | // Provide first-class heaps to allocate from. |
388 | // A heap just owns a set of pages for allocation and |
389 | // can only be allocate/reallocate from the thread that created it. |
390 | // Freeing blocks can be done from any thread though. |
391 | // Per thread, the segments are shared among its heaps. |
392 | // Per thread, there is always a default heap that is |
393 | // used for allocation; it is initialized to statically |
394 | // point to an empty heap to avoid initialization checks |
395 | // in the fast path. |
396 | // ------------------------------------------------------ |
397 | |
398 | // Thread local data |
399 | typedef struct mi_tld_s mi_tld_t; |
400 | |
401 | // Pages of a certain block size are held in a queue. |
402 | typedef struct mi_page_queue_s { |
403 | mi_page_t* first; |
404 | mi_page_t* last; |
405 | size_t block_size; |
406 | } mi_page_queue_t; |
407 | |
408 | #define MI_BIN_FULL (MI_BIN_HUGE+1) |
409 | |
410 | // Random context |
411 | typedef struct mi_random_cxt_s { |
412 | uint32_t input[16]; |
413 | uint32_t output[16]; |
414 | int output_available; |
415 | } mi_random_ctx_t; |
416 | |
417 | |
418 | // In debug mode there is a padding structure at the end of the blocks to check for buffer overflows |
419 | #if (MI_PADDING) |
420 | typedef struct mi_padding_s { |
421 | uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) |
422 | uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) |
423 | } mi_padding_t; |
424 | #define MI_PADDING_SIZE (sizeof(mi_padding_t)) |
425 | #define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) |
426 | #else |
427 | #define MI_PADDING_SIZE 0 |
428 | #define MI_PADDING_WSIZE 0 |
429 | #endif |
430 | |
431 | #define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) |
432 | |
433 | |
434 | // A heap owns a set of pages. |
435 | struct mi_heap_s { |
436 | mi_tld_t* tld; |
437 | mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. |
438 | mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") |
439 | _Atomic(mi_block_t*) thread_delayed_free; |
440 | mi_threadid_t thread_id; // thread this heap belongs too |
441 | mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0) |
442 | uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) |
443 | uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list |
444 | mi_random_ctx_t random; // random number context used for secure allocation |
445 | size_t page_count; // total number of pages in the `pages` queues. |
446 | size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) |
447 | size_t page_retired_max; // largest retired index into the `pages` array. |
448 | mi_heap_t* next; // list of heaps per thread |
449 | bool no_reclaim; // `true` if this heap should not reclaim abandoned pages |
450 | }; |
451 | |
452 | |
453 | |
454 | // ------------------------------------------------------ |
455 | // Debug |
456 | // ------------------------------------------------------ |
457 | |
458 | #if !defined(MI_DEBUG_UNINIT) |
459 | #define MI_DEBUG_UNINIT (0xD0) |
460 | #endif |
461 | #if !defined(MI_DEBUG_FREED) |
462 | #define MI_DEBUG_FREED (0xDF) |
463 | #endif |
464 | #if !defined(MI_DEBUG_PADDING) |
465 | #define MI_DEBUG_PADDING (0xDE) |
466 | #endif |
467 | |
468 | #if (MI_DEBUG) |
469 | // use our own assertion to print without memory allocation |
470 | void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); |
471 | #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) |
472 | #else |
473 | #define mi_assert(x) |
474 | #endif |
475 | |
476 | #if (MI_DEBUG>1) |
477 | #define mi_assert_internal mi_assert |
478 | #else |
479 | #define mi_assert_internal(x) |
480 | #endif |
481 | |
482 | #if (MI_DEBUG>2) |
483 | #define mi_assert_expensive mi_assert |
484 | #else |
485 | #define mi_assert_expensive(x) |
486 | #endif |
487 | |
488 | // ------------------------------------------------------ |
489 | // Statistics |
490 | // ------------------------------------------------------ |
491 | |
492 | #ifndef MI_STAT |
493 | #if (MI_DEBUG>0) |
494 | #define MI_STAT 2 |
495 | #else |
496 | #define MI_STAT 0 |
497 | #endif |
498 | #endif |
499 | |
500 | typedef struct mi_stat_count_s { |
501 | int64_t allocated; |
502 | int64_t freed; |
503 | int64_t peak; |
504 | int64_t current; |
505 | } mi_stat_count_t; |
506 | |
507 | typedef struct mi_stat_counter_s { |
508 | int64_t total; |
509 | int64_t count; |
510 | } mi_stat_counter_t; |
511 | |
512 | typedef struct mi_stats_s { |
513 | mi_stat_count_t segments; |
514 | mi_stat_count_t pages; |
515 | mi_stat_count_t reserved; |
516 | mi_stat_count_t committed; |
517 | mi_stat_count_t reset; |
518 | mi_stat_count_t page_committed; |
519 | mi_stat_count_t segments_abandoned; |
520 | mi_stat_count_t pages_abandoned; |
521 | mi_stat_count_t threads; |
522 | mi_stat_count_t normal; |
523 | mi_stat_count_t huge; |
524 | mi_stat_count_t large; |
525 | mi_stat_count_t malloc; |
526 | mi_stat_count_t segments_cache; |
527 | mi_stat_counter_t pages_extended; |
528 | mi_stat_counter_t mmap_calls; |
529 | mi_stat_counter_t commit_calls; |
530 | mi_stat_counter_t page_no_retire; |
531 | mi_stat_counter_t searches; |
532 | mi_stat_counter_t normal_count; |
533 | mi_stat_counter_t huge_count; |
534 | mi_stat_counter_t large_count; |
535 | #if MI_STAT>1 |
536 | mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; |
537 | #endif |
538 | } mi_stats_t; |
539 | |
540 | |
541 | void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); |
542 | void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); |
543 | void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); |
544 | |
545 | #if (MI_STAT) |
546 | #define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) |
547 | #define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) |
548 | #define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) |
549 | #else |
550 | #define mi_stat_increase(stat,amount) (void)0 |
551 | #define mi_stat_decrease(stat,amount) (void)0 |
552 | #define mi_stat_counter_increase(stat,amount) (void)0 |
553 | #endif |
554 | |
555 | #define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) |
556 | #define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) |
557 | #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) |
558 | |
559 | // ------------------------------------------------------ |
560 | // Thread Local data |
561 | // ------------------------------------------------------ |
562 | |
563 | // A "span" is is an available range of slices. The span queues keep |
564 | // track of slice spans of at most the given `slice_count` (but more than the previous size class). |
565 | typedef struct mi_span_queue_s { |
566 | mi_slice_t* first; |
567 | mi_slice_t* last; |
568 | size_t slice_count; |
569 | } mi_span_queue_t; |
570 | |
571 | #define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT) |
572 | |
573 | // OS thread local data |
574 | typedef struct mi_os_tld_s { |
575 | size_t region_idx; // start point for next allocation |
576 | mi_stats_t* stats; // points to tld stats |
577 | } mi_os_tld_t; |
578 | |
579 | |
580 | // Segments thread local data |
581 | typedef struct mi_segments_tld_s { |
582 | mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments |
583 | size_t count; // current number of segments; |
584 | size_t peak_count; // peak number of segments |
585 | size_t current_size; // current size of all segments |
586 | size_t peak_size; // peak size of all segments |
587 | mi_stats_t* stats; // points to tld stats |
588 | mi_os_tld_t* os; // points to os stats |
589 | } mi_segments_tld_t; |
590 | |
591 | // Thread local data |
592 | struct mi_tld_s { |
593 | unsigned long long heartbeat; // monotonic heartbeat count |
594 | bool recurse; // true if deferred was called; used to prevent infinite recursion. |
595 | mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) |
596 | mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) |
597 | mi_segments_tld_t segments; // segment tld |
598 | mi_os_tld_t os; // os tld |
599 | mi_stats_t stats; // statistics |
600 | }; |
601 | |
602 | #endif |
603 | |