1/* ----------------------------------------------------------------------------
2Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#pragma once
8#ifndef MIMALLOC_INTERNAL_H
9#define MIMALLOC_INTERNAL_H
10
11#include "mimalloc-types.h"
12#include "mimalloc-track.h"
13
14#if (MI_DEBUG>0)
15#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
16#else
17#define mi_trace_message(...)
18#endif
19
20#define MI_CACHE_LINE 64
21#if defined(_MSC_VER)
22#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
23#pragma warning(disable:26812) // unscoped enum warning
24#define mi_decl_noinline __declspec(noinline)
25#define mi_decl_thread __declspec(thread)
26#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
27#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
28#define mi_decl_noinline __attribute__((noinline))
29#define mi_decl_thread __thread
30#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
31#else
32#define mi_decl_noinline
33#define mi_decl_thread __thread // hope for the best :-)
34#define mi_decl_cache_align
35#endif
36
37#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
38#define __wasi__
39#endif
40
41#if defined(__cplusplus)
42#define mi_decl_externc extern "C"
43#else
44#define mi_decl_externc
45#endif
46
47#if !defined(_WIN32) && !defined(__wasi__)
48#define MI_USE_PTHREADS
49#include <pthread.h>
50#endif
51
52// "options.c"
53void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
54void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
55void _mi_warning_message(const char* fmt, ...);
56void _mi_verbose_message(const char* fmt, ...);
57void _mi_trace_message(const char* fmt, ...);
58void _mi_options_init(void);
59void _mi_error_message(int err, const char* fmt, ...);
60
61// random.c
62void _mi_random_init(mi_random_ctx_t* ctx);
63void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
64uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
65uintptr_t _mi_heap_random_next(mi_heap_t* heap);
66uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
67static inline uintptr_t _mi_random_shuffle(uintptr_t x);
68
69// init.c
70extern mi_decl_cache_align mi_stats_t _mi_stats_main;
71extern mi_decl_cache_align const mi_page_t _mi_page_empty;
72bool _mi_is_main_thread(void);
73size_t _mi_current_thread_count(void);
74bool _mi_preloading(void); // true while the C runtime is not ready
75
76// os.c
77size_t _mi_os_page_size(void);
78void _mi_os_init(void); // called from process init
79void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data
80void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data
81
82bool _mi_os_protect(void* addr, size_t size);
83bool _mi_os_unprotect(void* addr, size_t size);
84bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* stats);
85bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats);
86bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
87// bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
88size_t _mi_os_good_alloc_size(size_t size);
89bool _mi_os_has_overcommit(void);
90
91// arena.c
92void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld);
93void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld);
94void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, mi_os_tld_t* tld);
95mi_arena_id_t _mi_arena_id_none(void);
96bool _mi_arena_memid_is_suitable(size_t memid, mi_arena_id_t req_arena_id);
97
98// "segment-cache.c"
99void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld);
100bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
101void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld);
102void _mi_segment_map_allocated_at(const mi_segment_t* segment);
103void _mi_segment_map_freed_at(const mi_segment_t* segment);
104
105// "segment.c"
106mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
107void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
108void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
109bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld);
110void _mi_segment_thread_collect(mi_segments_tld_t* tld);
111void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
112
113uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page
114void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
115void _mi_abandoned_await_readers(void);
116void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld);
117
118
119
120// "page.c"
121void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept mi_attr_malloc;
122
123void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks
124void _mi_page_unfull(mi_page_t* page);
125void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
126void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
127void _mi_heap_delayed_free_all(mi_heap_t* heap);
128bool _mi_heap_delayed_free_partial(mi_heap_t* heap);
129void _mi_heap_collect_retired(mi_heap_t* heap, bool force);
130
131void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
132bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
133size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
134void _mi_deferred_free(mi_heap_t* heap, bool force);
135
136void _mi_page_free_collect(mi_page_t* page,bool force);
137void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
138
139size_t _mi_bin_size(uint8_t bin); // for stats
140uint8_t _mi_bin(size_t size); // for stats
141
142// "heap.c"
143void _mi_heap_destroy_pages(mi_heap_t* heap);
144void _mi_heap_collect_abandon(mi_heap_t* heap);
145void _mi_heap_set_default_direct(mi_heap_t* heap);
146bool _mi_heap_memid_is_suitable(mi_heap_t* heap, size_t memid);
147
148// "stats.c"
149void _mi_stats_done(mi_stats_t* stats);
150
151mi_msecs_t _mi_clock_now(void);
152mi_msecs_t _mi_clock_end(mi_msecs_t start);
153mi_msecs_t _mi_clock_start(void);
154
155// "alloc.c"
156void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic`
157void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
158void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept;
159mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p);
160bool _mi_free_delayed_block(mi_block_t* block);
161
162#if MI_DEBUG>1
163bool _mi_page_is_valid(mi_page_t* page);
164#endif
165
166
167// ------------------------------------------------------
168// Branches
169// ------------------------------------------------------
170
171#if defined(__GNUC__) || defined(__clang__)
172#define mi_unlikely(x) (__builtin_expect(!!(x),false))
173#define mi_likely(x) (__builtin_expect(!!(x),true))
174#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
175#define mi_unlikely(x) (x) [[unlikely]]
176#define mi_likely(x) (x) [[likely]]
177#else
178#define mi_unlikely(x) (x)
179#define mi_likely(x) (x)
180#endif
181
182#ifndef __has_builtin
183#define __has_builtin(x) 0
184#endif
185
186
187/* -----------------------------------------------------------
188 Error codes passed to `_mi_fatal_error`
189 All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
190 For portability define undefined error codes using common Unix codes:
191 <https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html>
192----------------------------------------------------------- */
193#include <errno.h>
194#ifndef EAGAIN // double free
195#define EAGAIN (11)
196#endif
197#ifndef ENOMEM // out of memory
198#define ENOMEM (12)
199#endif
200#ifndef EFAULT // corrupted free-list or meta-data
201#define EFAULT (14)
202#endif
203#ifndef EINVAL // trying to free an invalid pointer
204#define EINVAL (22)
205#endif
206#ifndef EOVERFLOW // count*size overflow
207#define EOVERFLOW (75)
208#endif
209
210
211/* -----------------------------------------------------------
212 Inlined definitions
213----------------------------------------------------------- */
214#define MI_UNUSED(x) (void)(x)
215#if (MI_DEBUG>0)
216#define MI_UNUSED_RELEASE(x)
217#else
218#define MI_UNUSED_RELEASE(x) MI_UNUSED(x)
219#endif
220
221#define MI_INIT4(x) x(),x(),x(),x()
222#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x)
223#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x)
224#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x)
225#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x)
226#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)
227#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)
228
229
230// Is `x` a power of two? (0 is considered a power of two)
231static inline bool _mi_is_power_of_two(uintptr_t x) {
232 return ((x & (x - 1)) == 0);
233}
234
235// Is a pointer aligned?
236static inline bool _mi_is_aligned(void* p, size_t alignment) {
237 mi_assert_internal(alignment != 0);
238 return (((uintptr_t)p % alignment) == 0);
239}
240
241// Align upwards
242static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
243 mi_assert_internal(alignment != 0);
244 uintptr_t mask = alignment - 1;
245 if ((alignment & mask) == 0) { // power of two?
246 return ((sz + mask) & ~mask);
247 }
248 else {
249 return (((sz + mask)/alignment)*alignment);
250 }
251}
252
253// Align downwards
254static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
255 mi_assert_internal(alignment != 0);
256 uintptr_t mask = alignment - 1;
257 if ((alignment & mask) == 0) { // power of two?
258 return (sz & ~mask);
259 }
260 else {
261 return ((sz / alignment) * alignment);
262 }
263}
264
265// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.
266static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {
267 mi_assert_internal(divider != 0);
268 return (divider == 0 ? size : ((size + divider - 1) / divider));
269}
270
271// Is memory zero initialized?
272static inline bool mi_mem_is_zero(void* p, size_t size) {
273 for (size_t i = 0; i < size; i++) {
274 if (((uint8_t*)p)[i] != 0) return false;
275 }
276 return true;
277}
278
279
280// Align a byte size to a size in _machine words_,
281// i.e. byte size == `wsize*sizeof(void*)`.
282static inline size_t _mi_wsize_from_size(size_t size) {
283 mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));
284 return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
285}
286
287// Overflow detecting multiply
288#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))
289#include <limits.h> // UINT_MAX, ULONG_MAX
290#if defined(_CLOCK_T) // for Illumos
291#undef _CLOCK_T
292#endif
293static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
294 #if (SIZE_MAX == ULONG_MAX)
295 return __builtin_umull_overflow(count, size, (unsigned long *)total);
296 #elif (SIZE_MAX == UINT_MAX)
297 return __builtin_umul_overflow(count, size, (unsigned int *)total);
298 #else
299 return __builtin_umulll_overflow(count, size, (unsigned long long *)total);
300 #endif
301}
302#else /* __builtin_umul_overflow is unavailable */
303static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
304 #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
305 *total = count * size;
306 // note: gcc/clang optimize this to directly check the overflow flag
307 return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);
308}
309#endif
310
311// Safe multiply `count*size` into `total`; return `true` on overflow.
312static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {
313 if (count==1) { // quick check for the case where count is one (common for C++ allocators)
314 *total = size;
315 return false;
316 }
317 else if mi_unlikely(mi_mul_overflow(count, size, total)) {
318 #if MI_DEBUG > 0
319 _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size);
320 #endif
321 *total = SIZE_MAX;
322 return true;
323 }
324 else return false;
325}
326
327
328/* ----------------------------------------------------------------------------------------
329The thread local default heap: `_mi_get_default_heap` returns the thread local heap.
330On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a
331__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures
332that the storage will always be available (allocated on the thread stacks).
333On some platforms though we cannot use that when overriding `malloc` since the underlying
334TLS implementation (or the loader) will call itself `malloc` on a first access and recurse.
335We try to circumvent this in an efficient way:
336- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the
337 loader itself calls `malloc` even before the modules are initialized.
338- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS).
339- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323)
340------------------------------------------------------------------------------------------- */
341
342extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
343extern bool _mi_process_is_initialized;
344mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
345
346#if defined(MI_MALLOC_OVERRIDE)
347#if defined(__APPLE__) // macOS
348#define MI_TLS_SLOT 89 // seems unused?
349// #define MI_TLS_RECURSE_GUARD 1
350// other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)
351// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
352#elif defined(__OpenBSD__)
353// use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16)
354// see <https://github.com/openbsd/src/blob/master/lib/libc/include/thread_private.h#L371>
355#define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24)
356// #elif defined(__DragonFly__)
357// #warning "mimalloc is not working correctly on DragonFly yet."
358// #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) <https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/lib/libthread_xu/thread/thr_private.h#L458>
359#elif defined(__ANDROID__)
360// See issue #381
361#define MI_TLS_PTHREAD
362#endif
363#endif
364
365#if defined(MI_TLS_SLOT)
366static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept; // forward declaration
367#elif defined(MI_TLS_PTHREAD_SLOT_OFS)
368static inline mi_heap_t** mi_tls_pthread_heap_slot(void) {
369 pthread_t self = pthread_self();
370 #if defined(__DragonFly__)
371 if (self==NULL) {
372 mi_heap_t* pheap_main = _mi_heap_main_get();
373 return &pheap_main;
374 }
375 #endif
376 return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS);
377}
378#elif defined(MI_TLS_PTHREAD)
379extern pthread_key_t _mi_heap_default_key;
380#endif
381
382// Default heap to allocate from (if not using TLS- or pthread slots).
383// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`).
384// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined.
385// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356).
386extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
387
388static inline mi_heap_t* mi_get_default_heap(void) {
389#if defined(MI_TLS_SLOT)
390 mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT);
391 if mi_unlikely(heap == NULL) {
392 #ifdef __GNUC__
393 __asm(""); // prevent conditional load of the address of _mi_heap_empty
394 #endif
395 heap = (mi_heap_t*)&_mi_heap_empty;
396 }
397 return heap;
398#elif defined(MI_TLS_PTHREAD_SLOT_OFS)
399 mi_heap_t* heap = *mi_tls_pthread_heap_slot();
400 return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
401#elif defined(MI_TLS_PTHREAD)
402 mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key));
403 return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
404#else
405 #if defined(MI_TLS_RECURSE_GUARD)
406 if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get();
407 #endif
408 return _mi_heap_default;
409#endif
410}
411
412static inline bool mi_heap_is_default(const mi_heap_t* heap) {
413 return (heap == mi_get_default_heap());
414}
415
416static inline bool mi_heap_is_backing(const mi_heap_t* heap) {
417 return (heap->tld->heap_backing == heap);
418}
419
420static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
421 mi_assert_internal(heap != NULL);
422 return (heap != &_mi_heap_empty);
423}
424
425static inline uintptr_t _mi_ptr_cookie(const void* p) {
426 extern mi_heap_t _mi_heap_main;
427 mi_assert_internal(_mi_heap_main.cookie != 0);
428 return ((uintptr_t)p ^ _mi_heap_main.cookie);
429}
430
431/* -----------------------------------------------------------
432 Pages
433----------------------------------------------------------- */
434
435static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {
436 mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
437 const size_t idx = _mi_wsize_from_size(size);
438 mi_assert_internal(idx < MI_PAGES_DIRECT);
439 return heap->pages_free_direct[idx];
440}
441
442// Get the page belonging to a certain size class
443static inline mi_page_t* _mi_get_free_small_page(size_t size) {
444 return _mi_heap_get_free_small_page(mi_get_default_heap(), size);
445}
446
447// Segment that contains the pointer
448static inline mi_segment_t* _mi_ptr_segment(const void* p) {
449 // mi_assert_internal(p != NULL);
450 return (mi_segment_t*)((uintptr_t)p & ~MI_SEGMENT_MASK);
451}
452
453static inline mi_page_t* mi_slice_to_page(mi_slice_t* s) {
454 mi_assert_internal(s->slice_offset== 0 && s->slice_count > 0);
455 return (mi_page_t*)(s);
456}
457
458static inline mi_slice_t* mi_page_to_slice(mi_page_t* p) {
459 mi_assert_internal(p->slice_offset== 0 && p->slice_count > 0);
460 return (mi_slice_t*)(p);
461}
462
463// Segment belonging to a page
464static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {
465 mi_segment_t* segment = _mi_ptr_segment(page);
466 mi_assert_internal(segment == NULL || ((mi_slice_t*)page >= segment->slices && (mi_slice_t*)page < segment->slices + segment->slice_entries));
467 return segment;
468}
469
470static inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) {
471 mi_slice_t* start = (mi_slice_t*)((uint8_t*)slice - slice->slice_offset);
472 mi_assert_internal(start >= _mi_ptr_segment(slice)->slices);
473 mi_assert_internal(start->slice_offset == 0);
474 mi_assert_internal(start + start->slice_count > slice);
475 return start;
476}
477
478// Get the page containing the pointer
479static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {
480 ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;
481 mi_assert_internal(diff >= 0 && diff < (ptrdiff_t)MI_SEGMENT_SIZE);
482 size_t idx = (size_t)diff >> MI_SEGMENT_SLICE_SHIFT;
483 mi_assert_internal(idx < segment->slice_entries);
484 mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx];
485 mi_slice_t* slice = mi_slice_first(slice0); // adjust to the block that holds the page data
486 mi_assert_internal(slice->slice_offset == 0);
487 mi_assert_internal(slice >= segment->slices && slice < segment->slices + segment->slice_entries);
488 return mi_slice_to_page(slice);
489}
490
491// Quick page start for initialized pages
492static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
493 return _mi_segment_page_start(segment, page, page_size);
494}
495
496// Get the page containing the pointer
497static inline mi_page_t* _mi_ptr_page(void* p) {
498 return _mi_segment_page_of(_mi_ptr_segment(p), p);
499}
500
501// Get the block size of a page (special case for huge objects)
502static inline size_t mi_page_block_size(const mi_page_t* page) {
503 const size_t bsize = page->xblock_size;
504 mi_assert_internal(bsize > 0);
505 if mi_likely(bsize < MI_HUGE_BLOCK_SIZE) {
506 return bsize;
507 }
508 else {
509 size_t psize;
510 _mi_segment_page_start(_mi_page_segment(page), page, &psize);
511 return psize;
512 }
513}
514
515// Get the usable block size of a page without fixed padding.
516// This may still include internal padding due to alignment and rounding up size classes.
517static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
518 return mi_page_block_size(page) - MI_PADDING_SIZE;
519}
520
521// size of a segment
522static inline size_t mi_segment_size(mi_segment_t* segment) {
523 return segment->segment_slices * MI_SEGMENT_SLICE_SIZE;
524}
525
526static inline uint8_t* mi_segment_end(mi_segment_t* segment) {
527 return (uint8_t*)segment + mi_segment_size(segment);
528}
529
530// Thread free access
531static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
532 return (mi_block_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & ~3);
533}
534
535static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) {
536 return (mi_delayed_t)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & 3);
537}
538
539// Heap access
540static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
541 return (mi_heap_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xheap));
542}
543
544static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
545 mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);
546 mi_atomic_store_release(&page->xheap,(uintptr_t)heap);
547}
548
549// Thread free flag helpers
550static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
551 return (mi_block_t*)(tf & ~0x03);
552}
553static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) {
554 return (mi_delayed_t)(tf & 0x03);
555}
556static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) {
557 return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed);
558}
559static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) {
560 return mi_tf_make(mi_tf_block(tf),delayed);
561}
562static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) {
563 return mi_tf_make(block, mi_tf_delayed(tf));
564}
565
566// are all blocks in a page freed?
567// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.
568static inline bool mi_page_all_free(const mi_page_t* page) {
569 mi_assert_internal(page != NULL);
570 return (page->used == 0);
571}
572
573// are there any available blocks?
574static inline bool mi_page_has_any_available(const mi_page_t* page) {
575 mi_assert_internal(page != NULL && page->reserved > 0);
576 return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));
577}
578
579// are there immediately available blocks, i.e. blocks available on the free list.
580static inline bool mi_page_immediate_available(const mi_page_t* page) {
581 mi_assert_internal(page != NULL);
582 return (page->free != NULL);
583}
584
585// is more than 7/8th of a page in use?
586static inline bool mi_page_mostly_used(const mi_page_t* page) {
587 if (page==NULL) return true;
588 uint16_t frac = page->reserved / 8U;
589 return (page->reserved - page->used <= frac);
590}
591
592static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {
593 return &((mi_heap_t*)heap)->pages[_mi_bin(size)];
594}
595
596
597
598//-----------------------------------------------------------
599// Page flags
600//-----------------------------------------------------------
601static inline bool mi_page_is_in_full(const mi_page_t* page) {
602 return page->flags.x.in_full;
603}
604
605static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
606 page->flags.x.in_full = in_full;
607}
608
609static inline bool mi_page_has_aligned(const mi_page_t* page) {
610 return page->flags.x.has_aligned;
611}
612
613static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) {
614 page->flags.x.has_aligned = has_aligned;
615}
616
617
618/* -------------------------------------------------------------------
619Encoding/Decoding the free list next pointers
620
621This is to protect against buffer overflow exploits where the
622free list is mutated. Many hardened allocators xor the next pointer `p`
623with a secret key `k1`, as `p^k1`. This prevents overwriting with known
624values but might be still too weak: if the attacker can guess
625the pointer `p` this can reveal `k1` (since `p^k1^p == k1`).
626Moreover, if multiple blocks can be read as well, the attacker can
627xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot
628about the pointers (and subsequently `k1`).
629
630Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<<k1)+k1`.
631Since these operations are not associative, the above approaches do not
632work so well any more even if the `p` can be guesstimated. For example,
633for the read case we can subtract two entries to discard the `+k1` term,
634but that leads to `((p1^k2)<<<k1) - ((p2^k2)<<<k1)` at best.
635We include the left-rotation since xor and addition are otherwise linear
636in the lowest bit. Finally, both keys are unique per page which reduces
637the re-use of keys by a large factor.
638
639We also pass a separate `null` value to be used as `NULL` or otherwise
640`(k2<<<k1)+k1` would appear (too) often as a sentinel value.
641------------------------------------------------------------------- */
642
643static inline bool mi_is_in_same_segment(const void* p, const void* q) {
644 return (_mi_ptr_segment(p) == _mi_ptr_segment(q));
645}
646
647static inline bool mi_is_in_same_page(const void* p, const void* q) {
648 mi_segment_t* segment = _mi_ptr_segment(p);
649 if (_mi_ptr_segment(q) != segment) return false;
650 // assume q may be invalid // return (_mi_segment_page_of(segment, p) == _mi_segment_page_of(segment, q));
651 mi_page_t* page = _mi_segment_page_of(segment, p);
652 size_t psize;
653 uint8_t* start = _mi_segment_page_start(segment, page, &psize);
654 return (start <= (uint8_t*)q && (uint8_t*)q < start + psize);
655}
656
657static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) {
658 shift %= MI_INTPTR_BITS;
659 return (shift==0 ? x : ((x << shift) | (x >> (MI_INTPTR_BITS - shift))));
660}
661static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) {
662 shift %= MI_INTPTR_BITS;
663 return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift))));
664}
665
666static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
667 void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
668 return (p==null ? NULL : p);
669}
670
671static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {
672 uintptr_t x = (uintptr_t)(p==NULL ? null : p);
673 return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
674}
675
676static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
677 mi_track_mem_defined(block,sizeof(mi_block_t));
678 mi_block_t* next;
679 #ifdef MI_ENCODE_FREELIST
680 next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);
681 #else
682 MI_UNUSED(keys); MI_UNUSED(null);
683 next = (mi_block_t*)block->next;
684 #endif
685 mi_track_mem_noaccess(block,sizeof(mi_block_t));
686 return next;
687}
688
689static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
690 mi_track_mem_undefined(block,sizeof(mi_block_t));
691 #ifdef MI_ENCODE_FREELIST
692 block->next = mi_ptr_encode(null, next, keys);
693 #else
694 MI_UNUSED(keys); MI_UNUSED(null);
695 block->next = (mi_encoded_t)next;
696 #endif
697 mi_track_mem_noaccess(block,sizeof(mi_block_t));
698}
699
700static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
701 #ifdef MI_ENCODE_FREELIST
702 mi_block_t* next = mi_block_nextx(page,block,page->keys);
703 // check for free list corruption: is `next` at least in the same page?
704 // TODO: check if `next` is `page->block_size` aligned?
705 if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {
706 _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
707 next = NULL;
708 }
709 return next;
710 #else
711 MI_UNUSED(page);
712 return mi_block_nextx(page,block,NULL);
713 #endif
714}
715
716static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {
717 #ifdef MI_ENCODE_FREELIST
718 mi_block_set_nextx(page,block,next, page->keys);
719 #else
720 MI_UNUSED(page);
721 mi_block_set_nextx(page,block,next,NULL);
722 #endif
723}
724
725
726// -------------------------------------------------------------------
727// commit mask
728// -------------------------------------------------------------------
729
730static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) {
731 for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
732 cm->mask[i] = 0;
733 }
734}
735
736static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) {
737 for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
738 cm->mask[i] = ~((size_t)0);
739 }
740}
741
742static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) {
743 for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
744 if (cm->mask[i] != 0) return false;
745 }
746 return true;
747}
748
749static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) {
750 for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
751 if (cm->mask[i] != ~((size_t)0)) return false;
752 }
753 return true;
754}
755
756// defined in `segment.c`:
757size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total);
758size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx);
759
760#define mi_commit_mask_foreach(cm,idx,count) \
761 idx = 0; \
762 while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) {
763
764#define mi_commit_mask_foreach_end() \
765 idx += count; \
766 }
767
768
769
770
771// -------------------------------------------------------------------
772// Fast "random" shuffle
773// -------------------------------------------------------------------
774
775static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
776 if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros
777#if (MI_INTPTR_SIZE==8)
778 // by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>
779 x ^= x >> 30;
780 x *= 0xbf58476d1ce4e5b9UL;
781 x ^= x >> 27;
782 x *= 0x94d049bb133111ebUL;
783 x ^= x >> 31;
784#elif (MI_INTPTR_SIZE==4)
785 // by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/>
786 x ^= x >> 16;
787 x *= 0x7feb352dUL;
788 x ^= x >> 15;
789 x *= 0x846ca68bUL;
790 x ^= x >> 16;
791#endif
792 return x;
793}
794
795// -------------------------------------------------------------------
796// Optimize numa node access for the common case (= one node)
797// -------------------------------------------------------------------
798
799int _mi_os_numa_node_get(mi_os_tld_t* tld);
800size_t _mi_os_numa_node_count_get(void);
801
802extern _Atomic(size_t) _mi_numa_node_count;
803static inline int _mi_os_numa_node(mi_os_tld_t* tld) {
804 if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; }
805 else return _mi_os_numa_node_get(tld);
806}
807static inline size_t _mi_os_numa_node_count(void) {
808 const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count);
809 if mi_likely(count > 0) { return count; }
810 else return _mi_os_numa_node_count_get();
811}
812
813
814// -------------------------------------------------------------------
815// Getting the thread id should be performant as it is called in the
816// fast path of `_mi_free` and we specialize for various platforms.
817// We only require _mi_threadid() to return a unique id for each thread.
818// -------------------------------------------------------------------
819#if defined(_WIN32)
820
821#define WIN32_LEAN_AND_MEAN
822#include <windows.h>
823static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
824 // Windows: works on Intel and ARM in both 32- and 64-bit
825 return (uintptr_t)NtCurrentTeb();
826}
827
828// We use assembly for a fast thread id on the main platforms. The TLS layout depends on
829// both the OS and libc implementation so we use specific tests for each main platform.
830// If you test on another platform and it works please send a PR :-)
831// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
832#elif defined(__GNUC__) && ( \
833 (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
834 || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \
835 || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
836 || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
837 || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
838 )
839
840static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
841 void* res;
842 const size_t ofs = (slot*sizeof(void*));
843 #if defined(__i386__)
844 __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
845 #elif defined(__APPLE__) && defined(__x86_64__)
846 __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
847 #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
848 __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
849 #elif defined(__x86_64__)
850 __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
851 #elif defined(__arm__)
852 void** tcb; MI_UNUSED(ofs);
853 __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
854 res = tcb[slot];
855 #elif defined(__aarch64__)
856 void** tcb; MI_UNUSED(ofs);
857 #if defined(__APPLE__) // M1, issue #343
858 __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
859 #else
860 __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
861 #endif
862 res = tcb[slot];
863 #endif
864 return res;
865}
866
867// setting a tls slot is only used on macOS for now
868static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
869 const size_t ofs = (slot*sizeof(void*));
870 #if defined(__i386__)
871 __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
872 #elif defined(__APPLE__) && defined(__x86_64__)
873 __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
874 #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
875 __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
876 #elif defined(__x86_64__)
877 __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
878 #elif defined(__arm__)
879 void** tcb; MI_UNUSED(ofs);
880 __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
881 tcb[slot] = value;
882 #elif defined(__aarch64__)
883 void** tcb; MI_UNUSED(ofs);
884 #if defined(__APPLE__) // M1, issue #343
885 __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
886 #else
887 __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
888 #endif
889 tcb[slot] = value;
890 #endif
891}
892
893static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
894 #if defined(__BIONIC__)
895 // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
896 // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
897 return (uintptr_t)mi_tls_slot(1);
898 #else
899 // in all our other targets, slot 0 is the thread id
900 // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
901 // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
902 return (uintptr_t)mi_tls_slot(0);
903 #endif
904}
905
906#else
907
908// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
909static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
910 return (uintptr_t)&_mi_heap_default;
911}
912
913#endif
914
915
916// -----------------------------------------------------------------------
917// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero)
918// -----------------------------------------------------------------------
919
920#if defined(__GNUC__)
921
922#include <limits.h> // LONG_MAX
923#define MI_HAVE_FAST_BITSCAN
924static inline size_t mi_clz(uintptr_t x) {
925 if (x==0) return MI_INTPTR_BITS;
926#if (INTPTR_MAX == LONG_MAX)
927 return __builtin_clzl(x);
928#else
929 return __builtin_clzll(x);
930#endif
931}
932static inline size_t mi_ctz(uintptr_t x) {
933 if (x==0) return MI_INTPTR_BITS;
934#if (INTPTR_MAX == LONG_MAX)
935 return __builtin_ctzl(x);
936#else
937 return __builtin_ctzll(x);
938#endif
939}
940
941#elif defined(_MSC_VER)
942
943#include <limits.h> // LONG_MAX
944#define MI_HAVE_FAST_BITSCAN
945static inline size_t mi_clz(uintptr_t x) {
946 if (x==0) return MI_INTPTR_BITS;
947 unsigned long idx;
948#if (INTPTR_MAX == LONG_MAX)
949 _BitScanReverse(&idx, x);
950#else
951 _BitScanReverse64(&idx, x);
952#endif
953 return ((MI_INTPTR_BITS - 1) - idx);
954}
955static inline size_t mi_ctz(uintptr_t x) {
956 if (x==0) return MI_INTPTR_BITS;
957 unsigned long idx;
958#if (INTPTR_MAX == LONG_MAX)
959 _BitScanForward(&idx, x);
960#else
961 _BitScanForward64(&idx, x);
962#endif
963 return idx;
964}
965
966#else
967static inline size_t mi_ctz32(uint32_t x) {
968 // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
969 static const unsigned char debruijn[32] = {
970 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
971 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
972 };
973 if (x==0) return 32;
974 return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27];
975}
976static inline size_t mi_clz32(uint32_t x) {
977 // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
978 static const uint8_t debruijn[32] = {
979 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
980 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
981 };
982 if (x==0) return 32;
983 x |= x >> 1;
984 x |= x >> 2;
985 x |= x >> 4;
986 x |= x >> 8;
987 x |= x >> 16;
988 return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27];
989}
990
991static inline size_t mi_clz(uintptr_t x) {
992 if (x==0) return MI_INTPTR_BITS;
993#if (MI_INTPTR_BITS <= 32)
994 return mi_clz32((uint32_t)x);
995#else
996 size_t count = mi_clz32((uint32_t)(x >> 32));
997 if (count < 32) return count;
998 return (32 + mi_clz32((uint32_t)x));
999#endif
1000}
1001static inline size_t mi_ctz(uintptr_t x) {
1002 if (x==0) return MI_INTPTR_BITS;
1003#if (MI_INTPTR_BITS <= 32)
1004 return mi_ctz32((uint32_t)x);
1005#else
1006 size_t count = mi_ctz32((uint32_t)x);
1007 if (count < 32) return count;
1008 return (32 + mi_ctz32((uint32_t)(x>>32)));
1009#endif
1010}
1011
1012#endif
1013
1014// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero)
1015static inline size_t mi_bsr(uintptr_t x) {
1016 return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x));
1017}
1018
1019
1020// ---------------------------------------------------------------------------------
1021// Provide our own `_mi_memcpy` for potential performance optimizations.
1022//
1023// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if
1024// we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support
1025// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.
1026// ---------------------------------------------------------------------------------
1027
1028#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
1029#include <intrin.h>
1030#include <string.h>
1031extern bool _mi_cpu_has_fsrm;
1032static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
1033 if (_mi_cpu_has_fsrm) {
1034 __movsb((unsigned char*)dst, (const unsigned char*)src, n);
1035 }
1036 else {
1037 memcpy(dst, src, n);
1038 }
1039}
1040static inline void _mi_memzero(void* dst, size_t n) {
1041 if (_mi_cpu_has_fsrm) {
1042 __stosb((unsigned char*)dst, 0, n);
1043 }
1044 else {
1045 memset(dst, 0, n);
1046 }
1047}
1048#else
1049#include <string.h>
1050static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
1051 memcpy(dst, src, n);
1052}
1053static inline void _mi_memzero(void* dst, size_t n) {
1054 memset(dst, 0, n);
1055}
1056#endif
1057
1058
1059// -------------------------------------------------------------------------------
1060// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned
1061// This is used for example in `mi_realloc`.
1062// -------------------------------------------------------------------------------
1063
1064#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
1065// On GCC/CLang we provide a hint that the pointers are word aligned.
1066#include <string.h>
1067static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
1068 mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
1069 void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
1070 const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
1071 _mi_memcpy(adst, asrc, n);
1072}
1073
1074static inline void _mi_memzero_aligned(void* dst, size_t n) {
1075 mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
1076 void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
1077 _mi_memzero(adst, n);
1078}
1079#else
1080// Default fallback on `_mi_memcpy`
1081static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
1082 mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
1083 _mi_memcpy(dst, src, n);
1084}
1085
1086static inline void _mi_memzero_aligned(void* dst, size_t n) {
1087 mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
1088 _mi_memzero(dst, n);
1089}
1090#endif
1091
1092
1093#endif
1094