Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * generic.h
4 : * Implement higher level operations based on some lower level atomic
5 : * operations.
6 : *
7 : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8 : * Portions Copyright (c) 1994, Regents of the University of California
9 : *
10 : * src/include/port/atomics/generic.h
11 : *
12 : *-------------------------------------------------------------------------
13 : */
14 :
15 : /* intentionally no include guards, should only be included by atomics.h */
16 : #ifndef INSIDE_ATOMICS_H
17 : # error "should be included via atomics.h"
18 : #endif
19 :
20 : /*
21 : * If read or write barriers are undefined, we upgrade them to full memory
22 : * barriers.
23 : */
24 : #if !defined(pg_read_barrier_impl)
25 : # define pg_read_barrier_impl pg_memory_barrier_impl
26 : #endif
27 : #if !defined(pg_write_barrier_impl)
28 : # define pg_write_barrier_impl pg_memory_barrier_impl
29 : #endif
30 :
31 : /* provide fallback */
32 : #if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT)
33 : #define PG_HAVE_ATOMIC_FLAG_SUPPORT
34 : typedef pg_atomic_uint32 pg_atomic_flag;
35 : #endif
36 :
37 : #ifndef PG_HAVE_ATOMIC_READ_U32
38 : #define PG_HAVE_ATOMIC_READ_U32
39 : static inline uint32
40 376944980 : pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr)
41 : {
42 376944980 : return ptr->value;
43 : }
44 : #endif
45 :
46 : #ifndef PG_HAVE_ATOMIC_WRITE_U32
47 : #define PG_HAVE_ATOMIC_WRITE_U32
48 : static inline void
49 77048 : pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
50 : {
51 77048 : ptr->value = val;
52 77048 : }
53 : #endif
54 :
55 : #ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
56 : #define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
57 : static inline void
58 : pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
59 : {
60 : ptr->value = val;
61 : }
62 : #endif
63 :
64 : /*
65 : * provide fallback for test_and_set using atomic_exchange if available
66 : */
67 : #if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32)
68 :
69 : #define PG_HAVE_ATOMIC_INIT_FLAG
70 : static inline void
71 : pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
72 : {
73 : pg_atomic_write_u32_impl(ptr, 0);
74 : }
75 :
76 : #define PG_HAVE_ATOMIC_TEST_SET_FLAG
77 : static inline bool
78 : pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
79 : {
80 : return pg_atomic_exchange_u32_impl(ptr, 1) == 0;
81 : }
82 :
83 : #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
84 : static inline bool
85 : pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
86 : {
87 : return pg_atomic_read_u32_impl(ptr) == 0;
88 : }
89 :
90 :
91 : #define PG_HAVE_ATOMIC_CLEAR_FLAG
92 : static inline void
93 : pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
94 : {
95 : /* XXX: release semantics suffice? */
96 : pg_memory_barrier_impl();
97 : pg_atomic_write_u32_impl(ptr, 0);
98 : }
99 :
100 : /*
101 : * provide fallback for test_and_set using atomic_compare_exchange if
102 : * available.
103 : */
104 : #elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
105 :
106 : #define PG_HAVE_ATOMIC_INIT_FLAG
107 : static inline void
108 : pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
109 : {
110 : pg_atomic_write_u32_impl(ptr, 0);
111 : }
112 :
113 : #define PG_HAVE_ATOMIC_TEST_SET_FLAG
114 : static inline bool
115 : pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
116 : {
117 : uint32 value = 0;
118 : return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1);
119 : }
120 :
121 : #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
122 : static inline bool
123 : pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
124 : {
125 : return pg_atomic_read_u32_impl(ptr) == 0;
126 : }
127 :
128 : #define PG_HAVE_ATOMIC_CLEAR_FLAG
129 : static inline void
130 : pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
131 : {
132 : /* XXX: release semantics suffice? */
133 : pg_memory_barrier_impl();
134 : pg_atomic_write_u32_impl(ptr, 0);
135 : }
136 :
137 : #elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG)
138 : # error "No pg_atomic_test_and_set provided"
139 : #endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */
140 :
141 :
142 : #ifndef PG_HAVE_ATOMIC_INIT_U32
143 : #define PG_HAVE_ATOMIC_INIT_U32
144 : static inline void
145 4036577 : pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_)
146 : {
147 4036577 : ptr->value = val_;
148 4036577 : }
149 : #endif
150 :
151 : #if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
152 : #define PG_HAVE_ATOMIC_EXCHANGE_U32
153 : static inline uint32
154 : pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_)
155 : {
156 : uint32 old;
157 : old = ptr->value; /* ok if read is not atomic */
158 : while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_))
159 : /* skip */;
160 : return old;
161 : }
162 : #endif
163 :
164 : #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
165 : #define PG_HAVE_ATOMIC_FETCH_ADD_U32
166 : static inline uint32
167 : pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
168 : {
169 : uint32 old;
170 : old = ptr->value; /* ok if read is not atomic */
171 : while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_))
172 : /* skip */;
173 : return old;
174 : }
175 : #endif
176 :
177 : #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
178 : #define PG_HAVE_ATOMIC_FETCH_SUB_U32
179 : static inline uint32
180 : pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
181 : {
182 : return pg_atomic_fetch_add_u32_impl(ptr, -sub_);
183 : }
184 : #endif
185 :
186 : #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
187 : #define PG_HAVE_ATOMIC_FETCH_AND_U32
188 : static inline uint32
189 : pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
190 : {
191 : uint32 old;
192 : old = ptr->value; /* ok if read is not atomic */
193 : while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_))
194 : /* skip */;
195 : return old;
196 : }
197 : #endif
198 :
199 : #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
200 : #define PG_HAVE_ATOMIC_FETCH_OR_U32
201 : static inline uint32
202 : pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
203 : {
204 : uint32 old;
205 : old = ptr->value; /* ok if read is not atomic */
206 : while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_))
207 : /* skip */;
208 : return old;
209 : }
210 : #endif
211 :
212 : #if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32)
213 : #define PG_HAVE_ATOMIC_ADD_FETCH_U32
214 : static inline uint32
215 897 : pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
216 : {
217 897 : return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_;
218 : }
219 : #endif
220 :
221 : #if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32)
222 : #define PG_HAVE_ATOMIC_SUB_FETCH_U32
223 : static inline uint32
224 347625019 : pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
225 : {
226 347625019 : return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_;
227 : }
228 : #endif
229 :
230 : #if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32)
231 : #define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32
232 : static inline uint32
233 19 : pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr)
234 : {
235 19 : return pg_atomic_fetch_add_u32_impl(ptr, 0);
236 : }
237 : #endif
238 :
239 : #if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32)
240 : #define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32
241 : static inline void
242 24748 : pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
243 : {
244 24748 : (void) pg_atomic_exchange_u32_impl(ptr, val);
245 24748 : }
246 : #endif
247 :
248 : #if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
249 : #define PG_HAVE_ATOMIC_EXCHANGE_U64
250 : static inline uint64
251 : pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_)
252 : {
253 : uint64 old;
254 : old = ptr->value; /* ok if read is not atomic */
255 : while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_))
256 : /* skip */;
257 : return old;
258 : }
259 : #endif
260 :
261 : #ifndef PG_HAVE_ATOMIC_WRITE_U64
262 : #define PG_HAVE_ATOMIC_WRITE_U64
263 :
264 : #if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
265 : !defined(PG_HAVE_ATOMIC_U64_SIMULATION)
266 :
267 : static inline void
268 21199021 : pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
269 : {
270 : /*
271 : * On this platform aligned 64bit writes are guaranteed to be atomic,
272 : * except if using the fallback implementation, where can't guarantee the
273 : * required alignment.
274 : */
275 : AssertPointerAlignment(ptr, 8);
276 21199021 : ptr->value = val;
277 21199021 : }
278 :
279 : #else
280 :
281 : static inline void
282 : pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
283 : {
284 : /*
285 : * 64 bit writes aren't safe on all platforms. In the generic
286 : * implementation implement them as an atomic exchange.
287 : */
288 : pg_atomic_exchange_u64_impl(ptr, val);
289 : }
290 :
291 : #endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
292 : #endif /* PG_HAVE_ATOMIC_WRITE_U64 */
293 :
294 : #ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64
295 : #define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64
296 : static inline void
297 5652574 : pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
298 : {
299 5652574 : ptr->value = val;
300 5652574 : }
301 : #endif
302 :
303 : #ifndef PG_HAVE_ATOMIC_READ_U64
304 : #define PG_HAVE_ATOMIC_READ_U64
305 :
306 : #if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
307 : !defined(PG_HAVE_ATOMIC_U64_SIMULATION)
308 :
309 : static inline uint64
310 574996472 : pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
311 : {
312 : /*
313 : * On this platform aligned 64-bit reads are guaranteed to be atomic.
314 : */
315 : AssertPointerAlignment(ptr, 8);
316 574996472 : return ptr->value;
317 : }
318 :
319 : #else
320 :
321 : static inline uint64
322 : pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
323 : {
324 : uint64 old = 0;
325 :
326 : /*
327 : * 64-bit reads aren't atomic on all platforms. In the generic
328 : * implementation implement them as a compare/exchange with 0. That'll
329 : * fail or succeed, but always return the old value. Possibly might store
330 : * a 0, but only if the previous value also was a 0 - i.e. harmless.
331 : */
332 : pg_atomic_compare_exchange_u64_impl(ptr, &old, 0);
333 :
334 : return old;
335 : }
336 : #endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
337 : #endif /* PG_HAVE_ATOMIC_READ_U64 */
338 :
339 : #ifndef PG_HAVE_ATOMIC_INIT_U64
340 : #define PG_HAVE_ATOMIC_INIT_U64
341 : static inline void
342 15630139 : pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)
343 : {
344 15630139 : ptr->value = val_;
345 15630139 : }
346 : #endif
347 :
348 : #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
349 : #define PG_HAVE_ATOMIC_FETCH_ADD_U64
350 : static inline uint64
351 : pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
352 : {
353 : uint64 old;
354 : old = ptr->value; /* ok if read is not atomic */
355 : while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_))
356 : /* skip */;
357 : return old;
358 : }
359 : #endif
360 :
361 : #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
362 : #define PG_HAVE_ATOMIC_FETCH_SUB_U64
363 : static inline uint64
364 : pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
365 : {
366 : return pg_atomic_fetch_add_u64_impl(ptr, -sub_);
367 : }
368 : #endif
369 :
370 : #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
371 : #define PG_HAVE_ATOMIC_FETCH_AND_U64
372 : static inline uint64
373 : pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
374 : {
375 : uint64 old;
376 : old = ptr->value; /* ok if read is not atomic */
377 : while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_))
378 : /* skip */;
379 : return old;
380 : }
381 : #endif
382 :
383 : #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
384 : #define PG_HAVE_ATOMIC_FETCH_OR_U64
385 : static inline uint64
386 : pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
387 : {
388 : uint64 old;
389 : old = ptr->value; /* ok if read is not atomic */
390 : while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_))
391 : /* skip */;
392 : return old;
393 : }
394 : #endif
395 :
396 : #if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64)
397 : #define PG_HAVE_ATOMIC_ADD_FETCH_U64
398 : static inline uint64
399 711 : pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
400 : {
401 711 : return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_;
402 : }
403 : #endif
404 :
405 : #if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64)
406 : #define PG_HAVE_ATOMIC_SUB_FETCH_U64
407 : static inline uint64
408 114589778 : pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
409 : {
410 114589778 : return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_;
411 : }
412 : #endif
413 :
414 : #if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64)
415 : #define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64
416 : static inline uint64
417 11730995 : pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr)
418 : {
419 11730995 : return pg_atomic_fetch_add_u64_impl(ptr, 0);
420 : }
421 : #endif
422 :
423 : #if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64)
424 : #define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64
425 : static inline void
426 126767 : pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
427 : {
428 126767 : (void) pg_atomic_exchange_u64_impl(ptr, val);
429 126767 : }
430 : #endif
|
