1 | /* Copyright 2016 Google Inc. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. */ |
14 | |
15 | #include "nsync_cpp.h" |
16 | #include "platform.h" |
17 | #include "compiler.h" |
18 | #include "cputype.h" |
19 | #include "nsync.h" |
20 | #include "dll.h" |
21 | #include "sem.h" |
22 | #include "wait_internal.h" |
23 | #include "common.h" |
24 | #include "atomic.h" |
25 | |
26 | NSYNC_CPP_START_ |
27 | |
28 | /* An once_sync_s struct contains a lock, and a condition variable on which |
29 | threads may wait for an nsync_once to be initialized by another thread. |
30 | |
31 | A separate struct is used only to keep nsync_once small. |
32 | |
33 | A given nsync_once can be associated with any once_sync_s struct, but cannot |
34 | be associated with more than one. nsync_once instances are mapped to |
35 | once_sync_s instances by a trivial hashing scheme implemented by |
36 | NSYNC_ONCE_SYNC_(). |
37 | |
38 | The number of once_sync_s structs in the following array is greater than one |
39 | only to reduce the probability of contention if a great many distinct |
40 | nsync_once variables are initialized concurrently. */ |
41 | static struct once_sync_s { |
42 | nsync_mu once_mu; |
43 | nsync_cv once_cv; |
44 | } once_sync[64]; |
45 | |
46 | /* Return a pointer to the once_sync_s struct associated with the nsync_once *p. */ |
47 | #define NSYNC_ONCE_SYNC_(p) &once_sync[(((uintptr_t) (p)) / sizeof (*(p))) % \ |
48 | (sizeof (once_sync) / sizeof (once_sync[0]))] |
49 | |
50 | /* Implement nsync_run_once, nsync_run_once_arg, nsync_run_once_spin, or |
51 | nsync_run_once_arg_spin, chosen as described below. |
52 | |
53 | If s!=NULL, s is required to point to the once_sync_s associated with *once, |
54 | and the semantics of nsync_run_once or nsync_run_once_arg are provided. |
55 | If s==NULL, the semantics of nsync_run_once_spin, or nsync_run_once_arg_spin |
56 | are provided. |
57 | |
58 | If f!=NULL, the semantics of nsync_run_once or nsync_run_once_spin are |
59 | provided. Otherwise, farg is required to be non-NULL, and the semantics of |
60 | nsync_run_once_arg or nsync_run_once_arg_spin are provided. */ |
61 | static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s, |
62 | void (*f) (void), void (*farg) (void *arg), void *arg) { |
63 | uint32_t o = ATM_LOAD_ACQ (once); |
64 | if (o != 2) { |
65 | unsigned attempts = 0; |
66 | if (s != NULL) { |
67 | nsync_mu_lock (&s->once_mu); |
68 | } |
69 | while (o == 0 && !ATM_CAS_ACQ (once, 0, 1)) { |
70 | o = ATM_LOAD (once); |
71 | } |
72 | if (o == 0) { |
73 | if (s != NULL) { |
74 | nsync_mu_unlock (&s->once_mu); |
75 | } |
76 | if (f != NULL) { |
77 | (*f) (); |
78 | } else { |
79 | (*farg) (arg); |
80 | } |
81 | if (s != NULL) { |
82 | nsync_mu_lock (&s->once_mu); |
83 | nsync_cv_broadcast (&s->once_cv); |
84 | } |
85 | ATM_STORE_REL (once, 2); |
86 | } |
87 | while (ATM_LOAD_ACQ (once) != 2) { |
88 | if (s != NULL) { |
89 | nsync_time deadline; |
90 | if (attempts < 50) { |
91 | attempts += 10; |
92 | } |
93 | deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts)); |
94 | nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL); |
95 | } else { |
96 | attempts = nsync_spin_delay_ (attempts); |
97 | } |
98 | } |
99 | if (s != NULL) { |
100 | nsync_mu_unlock (&s->once_mu); |
101 | } |
102 | } |
103 | } |
104 | |
105 | void nsync_run_once (nsync_once *once, void (*f) (void)) { |
106 | uint32_t o; |
107 | IGNORE_RACES_START (); |
108 | o = ATM_LOAD_ACQ (once); |
109 | if (o != 2) { |
110 | struct once_sync_s *s = NSYNC_ONCE_SYNC_ (once); |
111 | nsync_run_once_impl (once, s, f, NULL, NULL); |
112 | } |
113 | IGNORE_RACES_END (); |
114 | } |
115 | |
116 | void nsync_run_once_arg (nsync_once *once, void (*farg) (void *arg), void *arg) { |
117 | uint32_t o; |
118 | IGNORE_RACES_START (); |
119 | o = ATM_LOAD_ACQ (once); |
120 | if (o != 2) { |
121 | struct once_sync_s *s = NSYNC_ONCE_SYNC_ (once); |
122 | nsync_run_once_impl (once, s, NULL, farg, arg); |
123 | } |
124 | IGNORE_RACES_END (); |
125 | } |
126 | |
127 | void nsync_run_once_spin (nsync_once *once, void (*f) (void)) { |
128 | uint32_t o; |
129 | IGNORE_RACES_START (); |
130 | o = ATM_LOAD_ACQ (once); |
131 | if (o != 2) { |
132 | nsync_run_once_impl (once, NULL, f, NULL, NULL); |
133 | } |
134 | IGNORE_RACES_END (); |
135 | } |
136 | |
137 | void nsync_run_once_arg_spin (nsync_once *once, void (*farg) (void *arg), void *arg) { |
138 | uint32_t o; |
139 | IGNORE_RACES_START (); |
140 | o = ATM_LOAD_ACQ (once); |
141 | if (o != 2) { |
142 | nsync_run_once_impl (once, NULL, NULL, farg, arg); |
143 | } |
144 | IGNORE_RACES_END (); |
145 | } |
146 | |
147 | NSYNC_CPP_END_ |
148 | |