Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * atomics.h
4 : * Atomic operations.
5 : *
6 : * Hardware and compiler dependent functions for manipulating memory
7 : * atomically and dealing with cache coherency. Used to implement locking
8 : * facilities and lockless algorithms/data structures.
9 : *
10 : * To bring up postgres on a platform/compiler at the very least
11 : * implementations for the following operations should be provided:
12 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
13 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
14 : * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag()
15 : * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate.
16 : *
17 : * There exist generic, hardware independent, implementations for several
18 : * compilers which might be sufficient, although possibly not optimal, for a
19 : * new platform. If no such generic implementation is available spinlocks will
20 : * be used to implement the 64-bit parts of the API.
21 : *
22 : * Implement _u64 atomics if and only if your platform can use them
23 : * efficiently (and obviously correctly).
24 : *
25 : * Use higher level functionality (lwlocks, spinlocks, heavyweight locks)
26 : * whenever possible. Writing correct code using these facilities is hard.
27 : *
28 : * For an introduction to using memory barriers within the PostgreSQL backend,
29 : * see src/backend/storage/lmgr/README.barrier
30 : *
31 : * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
32 : * Portions Copyright (c) 1994, Regents of the University of California
33 : *
34 : * src/include/port/atomics.h
35 : *
36 : *-------------------------------------------------------------------------
37 : */
38 : #ifndef ATOMICS_H
39 : #define ATOMICS_H
40 :
41 : #ifdef FRONTEND
42 : #error "atomics.h may not be included from frontend code"
43 : #endif
44 :
45 : #define INSIDE_ATOMICS_H
46 :
47 : #include <limits.h>
48 :
49 : /*
50 : * First a set of architecture specific files is included.
51 : *
52 : * These files can provide the full set of atomics or can do pretty much
53 : * nothing if all the compilers commonly used on these platforms provide
54 : * usable generics.
55 : *
56 : * Don't add an inline assembly of the actual atomic operations if all the
57 : * common implementations of your platform provide intrinsics. Intrinsics are
58 : * much easier to understand and potentially support more architectures.
59 : *
60 : * It will often make sense to define memory barrier semantics here, since
61 : * e.g. generic compiler intrinsics for x86 memory barriers can't know that
62 : * postgres doesn't need x86 read/write barriers do anything more than a
63 : * compiler barrier.
64 : *
65 : */
66 : #if defined(__arm__) || defined(__arm) || defined(__aarch64__)
67 : #include "port/atomics/arch-arm.h"
68 : #elif defined(__i386__) || defined(__i386) || defined(__x86_64__)
69 : #include "port/atomics/arch-x86.h"
70 : #elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__)
71 : #include "port/atomics/arch-ppc.h"
72 : #endif
73 :
74 : /*
75 : * Compiler specific, but architecture independent implementations.
76 : *
77 : * Provide architecture independent implementations of the atomic
78 : * facilities. At the very least compiler barriers should be provided, but a
79 : * full implementation of
80 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
81 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
82 : * using compiler intrinsics are a good idea.
83 : */
84 : /*
85 : * gcc or compatible, including clang and icc.
86 : */
87 : #if defined(__GNUC__) || defined(__INTEL_COMPILER)
88 : #include "port/atomics/generic-gcc.h"
89 : #elif defined(_MSC_VER)
90 : #include "port/atomics/generic-msvc.h"
91 : #else
92 : /* Unknown compiler. */
93 : #endif
94 :
95 : /* Fail if we couldn't find implementations of required facilities. */
96 : #if !defined(PG_HAVE_ATOMIC_U32_SUPPORT)
97 : #error "could not find an implementation of pg_atomic_uint32"
98 : #endif
99 : #if !defined(pg_compiler_barrier_impl)
100 : #error "could not find an implementation of pg_compiler_barrier"
101 : #endif
102 : #if !defined(pg_memory_barrier_impl)
103 : #error "could not find an implementation of pg_memory_barrier_impl"
104 : #endif
105 :
106 :
107 : /*
108 : * Provide a spinlock-based implementation of the 64 bit variants, if
109 : * necessary.
110 : */
111 : #include "port/atomics/fallback.h"
112 :
113 : /*
114 : * Provide additional operations using supported infrastructure. These are
115 : * expected to be efficient if the underlying atomic operations are efficient.
116 : */
117 : #include "port/atomics/generic.h"
118 :
119 :
120 : /*
121 : * pg_compiler_barrier - prevent the compiler from moving code across
122 : *
123 : * A compiler barrier need not (and preferably should not) emit any actual
124 : * machine code, but must act as an optimization fence: the compiler must not
125 : * reorder loads or stores to main memory around the barrier. However, the
126 : * CPU may still reorder loads or stores at runtime, if the architecture's
127 : * memory model permits this.
128 : */
129 : #define pg_compiler_barrier() pg_compiler_barrier_impl()
130 :
131 : /*
132 : * pg_memory_barrier - prevent the CPU from reordering memory access
133 : *
134 : * A memory barrier must act as a compiler barrier, and in addition must
135 : * guarantee that all loads and stores issued prior to the barrier are
136 : * completed before any loads or stores issued after the barrier. Unless
137 : * loads and stores are totally ordered (which is not the case on most
138 : * architectures) this requires issuing some sort of memory fencing
139 : * instruction.
140 : */
141 : #define pg_memory_barrier() pg_memory_barrier_impl()
142 :
143 : /*
144 : * pg_(read|write)_barrier - prevent the CPU from reordering memory access
145 : *
146 : * A read barrier must act as a compiler barrier, and in addition must
147 : * guarantee that any loads issued prior to the barrier are completed before
148 : * any loads issued after the barrier. Similarly, a write barrier acts
149 : * as a compiler barrier, and also orders stores. Read and write barriers
150 : * are thus weaker than a full memory barrier, but stronger than a compiler
151 : * barrier. In practice, on machines with strong memory ordering, read and
152 : * write barriers may require nothing more than a compiler barrier.
153 : */
154 : #define pg_read_barrier() pg_read_barrier_impl()
155 : #define pg_write_barrier() pg_write_barrier_impl()
156 :
157 : /*
158 : * Spinloop delay - Allow CPU to relax in busy loops
159 : */
160 : #define pg_spin_delay() pg_spin_delay_impl()
161 :
162 : /*
163 : * pg_atomic_init_flag - initialize atomic flag.
164 : *
165 : * No barrier semantics.
166 : */
167 : static inline void
168 23888 : pg_atomic_init_flag(volatile pg_atomic_flag *ptr)
169 : {
170 23888 : pg_atomic_init_flag_impl(ptr);
171 23888 : }
172 :
173 : /*
174 : * pg_atomic_test_set_flag - TAS()
175 : *
176 : * Returns true if the flag has successfully been set, false otherwise.
177 : *
178 : * Acquire (including read barrier) semantics.
179 : */
180 : static inline bool
181 507954 : pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr)
182 : {
183 507954 : return pg_atomic_test_set_flag_impl(ptr);
184 : }
185 :
186 : /*
187 : * pg_atomic_unlocked_test_flag - Check if the lock is free
188 : *
189 : * Returns true if the flag currently is not set, false otherwise.
190 : *
191 : * No barrier semantics.
192 : */
193 : static inline bool
194 1088392 : pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr)
195 : {
196 1088392 : return pg_atomic_unlocked_test_flag_impl(ptr);
197 : }
198 :
199 : /*
200 : * pg_atomic_clear_flag - release lock set by TAS()
201 : *
202 : * Release (including write barrier) semantics.
203 : */
204 : static inline void
205 4728 : pg_atomic_clear_flag(volatile pg_atomic_flag *ptr)
206 : {
207 4728 : pg_atomic_clear_flag_impl(ptr);
208 4728 : }
209 :
210 :
211 : /*
212 : * pg_atomic_init_u32 - initialize atomic variable
213 : *
214 : * Has to be done before any concurrent usage..
215 : *
216 : * No barrier semantics.
217 : */
218 : static inline void
219 47551730 : pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
220 : {
221 : AssertPointerAlignment(ptr, 4);
222 :
223 47551730 : pg_atomic_init_u32_impl(ptr, val);
224 47551730 : }
225 :
226 : /*
227 : * pg_atomic_read_u32 - unlocked read from atomic variable.
228 : *
229 : * The read is guaranteed to return a value as it has been written by this or
230 : * another process at some point in the past. There's however no cache
231 : * coherency interaction guaranteeing the value hasn't since been written to
232 : * again.
233 : *
234 : * No barrier semantics.
235 : */
236 : static inline uint32
237 1130499530 : pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr)
238 : {
239 : AssertPointerAlignment(ptr, 4);
240 1130499530 : return pg_atomic_read_u32_impl(ptr);
241 : }
242 :
243 : /*
244 : * pg_atomic_read_membarrier_u32 - read with barrier semantics.
245 : *
246 : * This read is guaranteed to return the current value, provided that the value
247 : * is only ever updated via operations with barrier semantics, such as
248 : * pg_atomic_compare_exchange_u32() and pg_atomic_write_membarrier_u32().
249 : * While this may be less performant than pg_atomic_read_u32(), it may be
250 : * easier to reason about correctness with this function in less performance-
251 : * sensitive code.
252 : *
253 : * Full barrier semantics.
254 : */
255 : static inline uint32
256 : pg_atomic_read_membarrier_u32(volatile pg_atomic_uint32 *ptr)
257 : {
258 : AssertPointerAlignment(ptr, 4);
259 :
260 : return pg_atomic_read_membarrier_u32_impl(ptr);
261 : }
262 :
263 : /*
264 : * pg_atomic_write_u32 - write to atomic variable.
265 : *
266 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
267 : * observe a partial write for any reader. Note that this correctly interacts
268 : * with pg_atomic_compare_exchange_u32, in contrast to
269 : * pg_atomic_unlocked_write_u32().
270 : *
271 : * No barrier semantics.
272 : */
273 : static inline void
274 64259612 : pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
275 : {
276 : AssertPointerAlignment(ptr, 4);
277 :
278 64259612 : pg_atomic_write_u32_impl(ptr, val);
279 64259612 : }
280 :
281 : /*
282 : * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable.
283 : *
284 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
285 : * observe a partial write for any reader. But note that writing this way is
286 : * not guaranteed to correctly interact with read-modify-write operations like
287 : * pg_atomic_compare_exchange_u32. This should only be used in cases where
288 : * minor performance regressions due to atomics emulation are unacceptable.
289 : *
290 : * No barrier semantics.
291 : */
292 : static inline void
293 8732416 : pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
294 : {
295 : AssertPointerAlignment(ptr, 4);
296 :
297 8732416 : pg_atomic_unlocked_write_u32_impl(ptr, val);
298 8732416 : }
299 :
300 : /*
301 : * pg_atomic_write_membarrier_u32 - write with barrier semantics.
302 : *
303 : * The write is guaranteed to succeed as a whole, i.e., it's not possible to
304 : * observe a partial write for any reader. Note that this correctly interacts
305 : * with both pg_atomic_compare_exchange_u32() and
306 : * pg_atomic_read_membarrier_u32(). While this may be less performant than
307 : * pg_atomic_write_u32(), it may be easier to reason about correctness with
308 : * this function in less performance-sensitive code.
309 : *
310 : * Full barrier semantics.
311 : */
312 : static inline void
313 28 : pg_atomic_write_membarrier_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
314 : {
315 : AssertPointerAlignment(ptr, 4);
316 :
317 28 : pg_atomic_write_membarrier_u32_impl(ptr, val);
318 28 : }
319 :
320 : /*
321 : * pg_atomic_exchange_u32 - exchange newval with current value
322 : *
323 : * Returns the old value of 'ptr' before the swap.
324 : *
325 : * Full barrier semantics.
326 : */
327 : static inline uint32
328 30938 : pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval)
329 : {
330 : AssertPointerAlignment(ptr, 4);
331 :
332 30938 : return pg_atomic_exchange_u32_impl(ptr, newval);
333 : }
334 :
335 : /*
336 : * pg_atomic_compare_exchange_u32 - CAS operation
337 : *
338 : * Atomically compare the current value of ptr with *expected and store newval
339 : * iff ptr and *expected have the same value. The current value of *ptr will
340 : * always be stored in *expected.
341 : *
342 : * Return true if values have been exchanged, false otherwise.
343 : *
344 : * Full barrier semantics.
345 : */
346 : static inline bool
347 1052925110 : pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr,
348 : uint32 *expected, uint32 newval)
349 : {
350 : AssertPointerAlignment(ptr, 4);
351 : AssertPointerAlignment(expected, 4);
352 :
353 1052925110 : return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval);
354 : }
355 :
356 : /*
357 : * pg_atomic_fetch_add_u32 - atomically add to variable
358 : *
359 : * Returns the value of ptr before the arithmetic operation.
360 : *
361 : * Full barrier semantics.
362 : */
363 : static inline uint32
364 14956730 : pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
365 : {
366 : AssertPointerAlignment(ptr, 4);
367 14956730 : return pg_atomic_fetch_add_u32_impl(ptr, add_);
368 : }
369 :
370 : /*
371 : * pg_atomic_fetch_sub_u32 - atomically subtract from variable
372 : *
373 : * Returns the value of ptr before the arithmetic operation. Note that sub_
374 : * may not be INT_MIN due to platform limitations.
375 : *
376 : * Full barrier semantics.
377 : */
378 : static inline uint32
379 2249648 : pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
380 : {
381 : AssertPointerAlignment(ptr, 4);
382 : Assert(sub_ != INT_MIN);
383 2249648 : return pg_atomic_fetch_sub_u32_impl(ptr, sub_);
384 : }
385 :
386 : /*
387 : * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable
388 : *
389 : * Returns the value of ptr before the arithmetic operation.
390 : *
391 : * Full barrier semantics.
392 : */
393 : static inline uint32
394 9879170 : pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_)
395 : {
396 : AssertPointerAlignment(ptr, 4);
397 9879170 : return pg_atomic_fetch_and_u32_impl(ptr, and_);
398 : }
399 :
400 : /*
401 : * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable
402 : *
403 : * Returns the value of ptr before the arithmetic operation.
404 : *
405 : * Full barrier semantics.
406 : */
407 : static inline uint32
408 86011404 : pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_)
409 : {
410 : AssertPointerAlignment(ptr, 4);
411 86011404 : return pg_atomic_fetch_or_u32_impl(ptr, or_);
412 : }
413 :
414 : /*
415 : * pg_atomic_add_fetch_u32 - atomically add to variable
416 : *
417 : * Returns the value of ptr after the arithmetic operation.
418 : *
419 : * Full barrier semantics.
420 : */
421 : static inline uint32
422 794 : pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
423 : {
424 : AssertPointerAlignment(ptr, 4);
425 794 : return pg_atomic_add_fetch_u32_impl(ptr, add_);
426 : }
427 :
428 : /*
429 : * pg_atomic_sub_fetch_u32 - atomically subtract from variable
430 : *
431 : * Returns the value of ptr after the arithmetic operation. Note that sub_ may
432 : * not be INT_MIN due to platform limitations.
433 : *
434 : * Full barrier semantics.
435 : */
436 : static inline uint32
437 755242654 : pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
438 : {
439 : AssertPointerAlignment(ptr, 4);
440 : Assert(sub_ != INT_MIN);
441 755242654 : return pg_atomic_sub_fetch_u32_impl(ptr, sub_);
442 : }
443 :
444 : /* ----
445 : * The 64 bit operations have the same semantics as their 32bit counterparts
446 : * if they are available. Check the corresponding 32bit function for
447 : * documentation.
448 : * ----
449 : */
450 : static inline void
451 5744726 : pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
452 : {
453 : /*
454 : * Can't necessarily enforce alignment - and don't need it - when using
455 : * the spinlock based fallback implementation. Therefore only assert when
456 : * not using it.
457 : */
458 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
459 : AssertPointerAlignment(ptr, 8);
460 : #endif
461 5744726 : pg_atomic_init_u64_impl(ptr, val);
462 5744726 : }
463 :
464 : static inline uint64
465 570646038 : pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr)
466 : {
467 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
468 : AssertPointerAlignment(ptr, 8);
469 : #endif
470 570646038 : return pg_atomic_read_u64_impl(ptr);
471 : }
472 :
473 : static inline uint64
474 4849744 : pg_atomic_read_membarrier_u64(volatile pg_atomic_uint64 *ptr)
475 : {
476 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
477 : AssertPointerAlignment(ptr, 8);
478 : #endif
479 4849744 : return pg_atomic_read_membarrier_u64_impl(ptr);
480 : }
481 :
482 : static inline void
483 40162488 : pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
484 : {
485 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
486 : AssertPointerAlignment(ptr, 8);
487 : #endif
488 40162488 : pg_atomic_write_u64_impl(ptr, val);
489 40162488 : }
490 :
491 : static inline void
492 1896 : pg_atomic_write_membarrier_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
493 : {
494 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
495 : AssertPointerAlignment(ptr, 8);
496 : #endif
497 1896 : pg_atomic_write_membarrier_u64_impl(ptr, val);
498 1896 : }
499 :
500 : static inline uint64
501 34656714 : pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval)
502 : {
503 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
504 : AssertPointerAlignment(ptr, 8);
505 : #endif
506 34656714 : return pg_atomic_exchange_u64_impl(ptr, newval);
507 : }
508 :
509 : static inline bool
510 3699642 : pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr,
511 : uint64 *expected, uint64 newval)
512 : {
513 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
514 : AssertPointerAlignment(ptr, 8);
515 : #endif
516 3699642 : return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval);
517 : }
518 :
519 : static inline uint64
520 192502 : pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
521 : {
522 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
523 : AssertPointerAlignment(ptr, 8);
524 : #endif
525 192502 : return pg_atomic_fetch_add_u64_impl(ptr, add_);
526 : }
527 :
528 : static inline uint64
529 6 : pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
530 : {
531 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
532 : AssertPointerAlignment(ptr, 8);
533 : #endif
534 : Assert(sub_ != PG_INT64_MIN);
535 6 : return pg_atomic_fetch_sub_u64_impl(ptr, sub_);
536 : }
537 :
538 : static inline uint64
539 18 : pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_)
540 : {
541 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
542 : AssertPointerAlignment(ptr, 8);
543 : #endif
544 18 : return pg_atomic_fetch_and_u64_impl(ptr, and_);
545 : }
546 :
547 : static inline uint64
548 12 : pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_)
549 : {
550 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
551 : AssertPointerAlignment(ptr, 8);
552 : #endif
553 12 : return pg_atomic_fetch_or_u64_impl(ptr, or_);
554 : }
555 :
556 : static inline uint64
557 200 : pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
558 : {
559 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
560 : AssertPointerAlignment(ptr, 8);
561 : #endif
562 200 : return pg_atomic_add_fetch_u64_impl(ptr, add_);
563 : }
564 :
565 : static inline uint64
566 8 : pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
567 : {
568 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
569 : AssertPointerAlignment(ptr, 8);
570 : #endif
571 : Assert(sub_ != PG_INT64_MIN);
572 8 : return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
573 : }
574 :
575 : /*
576 : * Monotonically advance the given variable using only atomic operations until
577 : * it's at least the target value. Returns the latest value observed, which
578 : * may or may not be the target value.
579 : *
580 : * Full barrier semantics (even when value is unchanged).
581 : */
582 : static inline uint64
583 880500 : pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target)
584 : {
585 : uint64 currval;
586 :
587 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
588 : AssertPointerAlignment(ptr, 8);
589 : #endif
590 :
591 880500 : currval = pg_atomic_read_u64_impl(ptr);
592 880500 : if (currval >= target)
593 : {
594 130642 : pg_memory_barrier();
595 130642 : return currval;
596 : }
597 :
598 752574 : while (currval < target)
599 : {
600 749902 : if (pg_atomic_compare_exchange_u64(ptr, &currval, target))
601 747186 : return target;
602 : }
603 :
604 2672 : return currval;
605 : }
606 :
607 : #undef INSIDE_ATOMICS_H
608 :
609 : #endif /* ATOMICS_H */
|
