1/* This file implements atomic counters using c11 _Atomic, __atomic or __sync
2 * macros if available, otherwise we will throw an error when compile.
3 *
4 * The exported interface is composed of three macros:
5 *
6 * atomicIncr(var,count) -- Increment the atomic counter
7 * atomicGetIncr(var,oldvalue_var,count) -- Get and increment the atomic counter
8 * atomicDecr(var,count) -- Decrement the atomic counter
9 * atomicGet(var,dstvar) -- Fetch the atomic counter value
10 * atomicSet(var,value) -- Set the atomic counter value
11 * atomicGetWithSync(var,value) -- 'atomicGet' with inter-thread synchronization
12 * atomicSetWithSync(var,value) -- 'atomicSet' with inter-thread synchronization
13 *
14 * Never use return value from the macros, instead use the AtomicGetIncr()
15 * if you need to get the current value and increment it atomically, like
16 * in the following example:
17 *
18 * long oldvalue;
19 * atomicGetIncr(myvar,oldvalue,1);
20 * doSomethingWith(oldvalue);
21 *
22 * ----------------------------------------------------------------------------
23 *
24 * Copyright (c) 2015, Salvatore Sanfilippo <antirez at gmail dot com>
25 * All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions are met:
29 *
30 * * Redistributions of source code must retain the above copyright notice,
31 * this list of conditions and the following disclaimer.
32 * * Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * * Neither the name of Redis nor the names of its contributors may be used
36 * to endorse or promote products derived from this software without
37 * specific prior written permission.
38 *
39 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
40 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
42 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
43 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
44 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
45 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
46 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
47 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
49 * POSSIBILITY OF SUCH DAMAGE.
50 */
51
52#include <pthread.h>
53#include "config.h"
54
55#ifndef __ATOMIC_VAR_H
56#define __ATOMIC_VAR_H
57
58/* Define redisAtomic for atomic variable. */
59#define redisAtomic
60
61/* To test Redis with Helgrind (a Valgrind tool) it is useful to define
62 * the following macro, so that __sync macros are used: those can be detected
63 * by Helgrind (even if they are less efficient) so that no false positive
64 * is reported. */
65// #define __ATOMIC_VAR_FORCE_SYNC_MACROS
66
67/* There will be many false positives if we test Redis with Helgrind, since
68 * Helgrind can't understand we have imposed ordering on the program, so
69 * we use macros in helgrind.h to tell Helgrind inter-thread happens-before
70 * relationship explicitly for avoiding false positives.
71 *
72 * For more details, please see: valgrind/helgrind.h and
73 * https://www.valgrind.org/docs/manual/hg-manual.html#hg-manual.effective-use
74 *
75 * These macros take effect only when 'make helgrind', and you must first
76 * install Valgrind in the default path configuration. */
77#ifdef __ATOMIC_VAR_FORCE_SYNC_MACROS
78#include <valgrind/helgrind.h>
79#else
80#define ANNOTATE_HAPPENS_BEFORE(v) ((void) v)
81#define ANNOTATE_HAPPENS_AFTER(v) ((void) v)
82#endif
83
84#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__STDC_VERSION__) && \
85 (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__)
86/* Use '_Atomic' keyword if the compiler supports. */
87#undef redisAtomic
88#define redisAtomic _Atomic
89/* Implementation using _Atomic in C11. */
90
91#include <stdatomic.h>
92#define atomicIncr(var,count) atomic_fetch_add_explicit(&var,(count),memory_order_relaxed)
93#define atomicGetIncr(var,oldvalue_var,count) do { \
94 oldvalue_var = atomic_fetch_add_explicit(&var,(count),memory_order_relaxed); \
95} while(0)
96#define atomicDecr(var,count) atomic_fetch_sub_explicit(&var,(count),memory_order_relaxed)
97#define atomicGet(var,dstvar) do { \
98 dstvar = atomic_load_explicit(&var,memory_order_relaxed); \
99} while(0)
100#define atomicSet(var,value) atomic_store_explicit(&var,value,memory_order_relaxed)
101#define atomicGetWithSync(var,dstvar) do { \
102 dstvar = atomic_load_explicit(&var,memory_order_seq_cst); \
103} while(0)
104#define atomicSetWithSync(var,value) \
105 atomic_store_explicit(&var,value,memory_order_seq_cst)
106#define REDIS_ATOMIC_API "c11-builtin"
107
108#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && \
109 (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \
110 defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST)
111/* Implementation using __atomic macros. */
112
113#define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED)
114#define atomicGetIncr(var,oldvalue_var,count) do { \
115 oldvalue_var = __atomic_fetch_add(&var,(count),__ATOMIC_RELAXED); \
116} while(0)
117#define atomicDecr(var,count) __atomic_sub_fetch(&var,(count),__ATOMIC_RELAXED)
118#define atomicGet(var,dstvar) do { \
119 dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \
120} while(0)
121#define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED)
122#define atomicGetWithSync(var,dstvar) do { \
123 dstvar = __atomic_load_n(&var,__ATOMIC_SEQ_CST); \
124} while(0)
125#define atomicSetWithSync(var,value) \
126 __atomic_store_n(&var,value,__ATOMIC_SEQ_CST)
127#define REDIS_ATOMIC_API "atomic-builtin"
128
129#elif defined(HAVE_ATOMIC)
130/* Implementation using __sync macros. */
131
132#define atomicIncr(var,count) __sync_add_and_fetch(&var,(count))
133#define atomicGetIncr(var,oldvalue_var,count) do { \
134 oldvalue_var = __sync_fetch_and_add(&var,(count)); \
135} while(0)
136#define atomicDecr(var,count) __sync_sub_and_fetch(&var,(count))
137#define atomicGet(var,dstvar) do { \
138 dstvar = __sync_sub_and_fetch(&var,0); \
139} while(0)
140#define atomicSet(var,value) do { \
141 while(!__sync_bool_compare_and_swap(&var,var,value)); \
142} while(0)
143/* Actually the builtin issues a full memory barrier by default. */
144#define atomicGetWithSync(var,dstvar) { \
145 dstvar = __sync_sub_and_fetch(&var,0,__sync_synchronize); \
146 ANNOTATE_HAPPENS_AFTER(&var); \
147} while(0)
148#define atomicSetWithSync(var,value) do { \
149 ANNOTATE_HAPPENS_BEFORE(&var); \
150 while(!__sync_bool_compare_and_swap(&var,var,value,__sync_synchronize)); \
151} while(0)
152#define REDIS_ATOMIC_API "sync-builtin"
153
154#else
155#error "Unable to determine atomic operations for your platform"
156
157#endif
158#endif /* __ATOMIC_VAR_H */
159