Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * atomics.h
4 : : * Atomic operations.
5 : : *
6 : : * Hardware and compiler dependent functions for manipulating memory
7 : : * atomically and dealing with cache coherency. Used to implement locking
8 : : * facilities and lockless algorithms/data structures.
9 : : *
10 : : * To bring up postgres on a platform/compiler at the very least
11 : : * implementations for the following operations should be provided:
12 : : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
13 : : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
14 : : * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag()
15 : : * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate.
16 : : *
17 : : * There exist generic, hardware independent, implementations for several
18 : : * compilers which might be sufficient, although possibly not optimal, for a
19 : : * new platform. If no such generic implementation is available spinlocks will
20 : : * be used to implement the 64-bit parts of the API.
21 : : *
22 : : * Implement _u64 atomics if and only if your platform can use them
23 : : * efficiently (and obviously correctly).
24 : : *
25 : : * Use higher level functionality (lwlocks, spinlocks, heavyweight locks)
26 : : * whenever possible. Writing correct code using these facilities is hard.
27 : : *
28 : : * For an introduction to using memory barriers within the PostgreSQL backend,
29 : : * see src/backend/storage/lmgr/README.barrier
30 : : *
31 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
32 : : * Portions Copyright (c) 1994, Regents of the University of California
33 : : *
34 : : * src/include/port/atomics.h
35 : : *
36 : : *-------------------------------------------------------------------------
37 : : */
38 : : #ifndef ATOMICS_H
39 : : #define ATOMICS_H
40 : :
41 : : #ifdef FRONTEND
42 : : #error "atomics.h may not be included from frontend code"
43 : : #endif
44 : :
45 : : #define INSIDE_ATOMICS_H
46 : :
47 : : #include <limits.h>
48 : :
49 : : /*
50 : : * First a set of architecture specific files is included.
51 : : *
52 : : * These files can provide the full set of atomics or can do pretty much
53 : : * nothing if all the compilers commonly used on these platforms provide
54 : : * usable generics.
55 : : *
56 : : * Don't add an inline assembly of the actual atomic operations if all the
57 : : * common implementations of your platform provide intrinsics. Intrinsics are
58 : : * much easier to understand and potentially support more architectures.
59 : : *
60 : : * It will often make sense to define memory barrier semantics here, since
61 : : * e.g. generic compiler intrinsics for x86 memory barriers can't know that
62 : : * postgres doesn't need x86 read/write barriers do anything more than a
63 : : * compiler barrier.
64 : : *
65 : : */
66 : : #if defined(__arm__) || defined(__aarch64__)
67 : : #include "port/atomics/arch-arm.h"
68 : : #elif defined(__i386__) || defined(__x86_64__)
69 : : #include "port/atomics/arch-x86.h"
70 : : #elif defined(__powerpc__) || defined(__powerpc64__)
71 : : #include "port/atomics/arch-ppc.h"
72 : : #endif
73 : :
74 : : /*
75 : : * Compiler specific, but architecture independent implementations.
76 : : *
77 : : * Provide architecture independent implementations of the atomic
78 : : * facilities. At the very least compiler barriers should be provided, but a
79 : : * full implementation of
80 : : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
81 : : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
82 : : * using compiler intrinsics are a good idea.
83 : : */
84 : : /*
85 : : * gcc or compatible, including clang and icc.
86 : : */
87 : : #if defined(__GNUC__) || defined(__INTEL_COMPILER)
88 : : #include "port/atomics/generic-gcc.h"
89 : : #elif defined(_MSC_VER)
90 : : #include "port/atomics/generic-msvc.h"
91 : : #else
92 : : /* Unknown compiler. */
93 : : #endif
94 : :
95 : : /* Fail if we couldn't find implementations of required facilities. */
96 : : #if !defined(PG_HAVE_ATOMIC_U32_SUPPORT)
97 : : #error "could not find an implementation of pg_atomic_uint32"
98 : : #endif
99 : : #if !defined(pg_compiler_barrier_impl)
100 : : #error "could not find an implementation of pg_compiler_barrier"
101 : : #endif
102 : : #if !defined(pg_memory_barrier_impl)
103 : : #error "could not find an implementation of pg_memory_barrier_impl"
104 : : #endif
105 : :
106 : :
107 : : /*
108 : : * Provide a spinlock-based implementation of the 64 bit variants, if
109 : : * necessary.
110 : : */
111 : : #include "port/atomics/fallback.h"
112 : :
113 : : /*
114 : : * Provide additional operations using supported infrastructure. These are
115 : : * expected to be efficient if the underlying atomic operations are efficient.
116 : : */
117 : : #include "port/atomics/generic.h"
118 : :
119 : :
120 : : /*
121 : : * pg_compiler_barrier - prevent the compiler from moving code across
122 : : *
123 : : * A compiler barrier need not (and preferably should not) emit any actual
124 : : * machine code, but must act as an optimization fence: the compiler must not
125 : : * reorder loads or stores to main memory around the barrier. However, the
126 : : * CPU may still reorder loads or stores at runtime, if the architecture's
127 : : * memory model permits this.
128 : : */
129 : : #define pg_compiler_barrier() pg_compiler_barrier_impl()
130 : :
131 : : /*
132 : : * pg_memory_barrier - prevent the CPU from reordering memory access
133 : : *
134 : : * A memory barrier must act as a compiler barrier, and in addition must
135 : : * guarantee that all loads and stores issued prior to the barrier are
136 : : * completed before any loads or stores issued after the barrier. Unless
137 : : * loads and stores are totally ordered (which is not the case on most
138 : : * architectures) this requires issuing some sort of memory fencing
139 : : * instruction.
140 : : */
141 : : #define pg_memory_barrier() pg_memory_barrier_impl()
142 : :
143 : : /*
144 : : * pg_(read|write)_barrier - prevent the CPU from reordering memory access
145 : : *
146 : : * A read barrier must act as a compiler barrier, and in addition must
147 : : * guarantee that any loads issued prior to the barrier are completed before
148 : : * any loads issued after the barrier. Similarly, a write barrier acts
149 : : * as a compiler barrier, and also orders stores. Read and write barriers
150 : : * are thus weaker than a full memory barrier, but stronger than a compiler
151 : : * barrier. In practice, on machines with strong memory ordering, read and
152 : : * write barriers may require nothing more than a compiler barrier.
153 : : */
154 : : #define pg_read_barrier() pg_read_barrier_impl()
155 : : #define pg_write_barrier() pg_write_barrier_impl()
156 : :
157 : : /*
158 : : * pg_atomic_init_flag - initialize atomic flag.
159 : : *
160 : : * No barrier semantics.
161 : : */
162 : : static inline void
4321 andres@anarazel.de 163 :CBC 13267 : pg_atomic_init_flag(volatile pg_atomic_flag *ptr)
164 : : {
165 : 13267 : pg_atomic_init_flag_impl(ptr);
166 : 13267 : }
167 : :
168 : : /*
169 : : * pg_atomic_test_set_flag - TAS()
170 : : *
171 : : * Returns true if the flag has successfully been set, false otherwise.
172 : : *
173 : : * Acquire (including read barrier) semantics.
174 : : */
175 : : static inline bool
176 : 1592 : pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr)
177 : : {
178 : 1592 : return pg_atomic_test_set_flag_impl(ptr);
179 : : }
180 : :
181 : : /*
182 : : * pg_atomic_unlocked_test_flag - Check if the lock is free
183 : : *
184 : : * Returns true if the flag currently is not set, false otherwise.
185 : : *
186 : : * No barrier semantics.
187 : : */
188 : : static inline bool
189 : 6155 : pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr)
190 : : {
191 : 6155 : return pg_atomic_unlocked_test_flag_impl(ptr);
192 : : }
193 : :
194 : : /*
195 : : * pg_atomic_clear_flag - release lock set by TAS()
196 : : *
197 : : * Release (including write barrier) semantics.
198 : : */
199 : : static inline void
200 : 157 : pg_atomic_clear_flag(volatile pg_atomic_flag *ptr)
201 : : {
202 : 157 : pg_atomic_clear_flag_impl(ptr);
203 : 157 : }
204 : :
205 : :
206 : : /*
207 : : * pg_atomic_init_u32 - initialize atomic variable
208 : : *
209 : : * Has to be done before any concurrent usage..
210 : : *
211 : : * No barrier semantics.
212 : : */
213 : : static inline void
214 : 3925550 : pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
215 : : {
216 [ - + ]: 3925550 : AssertPointerAlignment(ptr, 4);
217 : :
218 : 3925550 : pg_atomic_init_u32_impl(ptr, val);
219 : 3925550 : }
220 : :
221 : : /*
222 : : * pg_atomic_read_u32 - unlocked read from atomic variable.
223 : : *
224 : : * The read is guaranteed to return a value as it has been written by this or
225 : : * another process at some point in the past. There's however no cache
226 : : * coherency interaction guaranteeing the value hasn't since been written to
227 : : * again.
228 : : *
229 : : * No barrier semantics.
230 : : */
231 : : static inline uint32
232 : 243863876 : pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr)
233 : : {
234 [ - + ]: 243863876 : AssertPointerAlignment(ptr, 4);
235 : 243863876 : return pg_atomic_read_u32_impl(ptr);
236 : : }
237 : :
238 : : /*
239 : : * pg_atomic_read_membarrier_u32 - read with barrier semantics.
240 : : *
241 : : * This read is guaranteed to return the current value, provided that the value
242 : : * is only ever updated via operations with barrier semantics, such as
243 : : * pg_atomic_compare_exchange_u32() and pg_atomic_write_membarrier_u32().
244 : : * While this may be less performant than pg_atomic_read_u32(), it may be
245 : : * easier to reason about correctness with this function in less performance-
246 : : * sensitive code.
247 : : *
248 : : * Full barrier semantics.
249 : : */
250 : : static inline uint32
877 nathan@postgresql.or 251 : 19 : pg_atomic_read_membarrier_u32(volatile pg_atomic_uint32 *ptr)
252 : : {
253 [ - + ]: 19 : AssertPointerAlignment(ptr, 4);
254 : :
255 : 19 : return pg_atomic_read_membarrier_u32_impl(ptr);
256 : : }
257 : :
258 : : /*
259 : : * pg_atomic_write_u32 - write to atomic variable.
260 : : *
261 : : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
262 : : * observe a partial write for any reader. Note that this correctly interacts
263 : : * with pg_atomic_compare_exchange_u32, in contrast to
264 : : * pg_atomic_unlocked_write_u32().
265 : : *
266 : : * No barrier semantics.
267 : : */
268 : : static inline void
4321 andres@anarazel.de 269 : 72675 : pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
270 : : {
271 [ - + ]: 72675 : AssertPointerAlignment(ptr, 4);
272 : :
273 : 72675 : pg_atomic_write_u32_impl(ptr, val);
274 : 72675 : }
275 : :
276 : : /*
277 : : * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable.
278 : : *
279 : : * Write to an atomic variable, without atomicity guarantees. I.e. it is not
280 : : * guaranteed that a concurrent reader will not see a torn value, nor is this
281 : : * guaranteed to correctly interact with concurrent read-modify-write
282 : : * operations like pg_atomic_compare_exchange_u32. This should only be used
283 : : * in cases where minor performance regressions due to atomic operations are
284 : : * unacceptable and where exclusive access is guaranteed via some external
285 : : * means.
286 : : *
287 : : * No barrier semantics.
288 : : */
289 : : static inline void
290 : : pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
291 : : {
292 : : AssertPointerAlignment(ptr, 4);
293 : :
294 : : pg_atomic_unlocked_write_u32_impl(ptr, val);
295 : : }
296 : :
297 : : /*
298 : : * pg_atomic_write_membarrier_u32 - write with barrier semantics.
299 : : *
300 : : * The write is guaranteed to succeed as a whole, i.e., it's not possible to
301 : : * observe a partial write for any reader. Note that this correctly interacts
302 : : * with both pg_atomic_compare_exchange_u32() and
303 : : * pg_atomic_read_membarrier_u32(). While this may be less performant than
304 : : * pg_atomic_write_u32(), it may be easier to reason about correctness with
305 : : * this function in less performance-sensitive code.
306 : : *
307 : : * Full barrier semantics.
308 : : */
309 : : static inline void
877 nathan@postgresql.or 310 : 22793 : pg_atomic_write_membarrier_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
311 : : {
312 [ - + ]: 22793 : AssertPointerAlignment(ptr, 4);
313 : :
314 : 22793 : pg_atomic_write_membarrier_u32_impl(ptr, val);
315 : 22793 : }
316 : :
317 : : /*
318 : : * pg_atomic_exchange_u32 - exchange newval with current value
319 : : *
320 : : * Returns the old value of 'ptr' before the swap.
321 : : *
322 : : * Full barrier semantics.
323 : : */
324 : : static inline uint32
4321 andres@anarazel.de 325 : 26348 : pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval)
326 : : {
327 [ - + ]: 26348 : AssertPointerAlignment(ptr, 4);
328 : :
329 : 26348 : return pg_atomic_exchange_u32_impl(ptr, newval);
330 : : }
331 : :
332 : : /*
333 : : * pg_atomic_compare_exchange_u32 - CAS operation
334 : : *
335 : : * Atomically compare the current value of ptr with *expected and store newval
336 : : * iff ptr and *expected have the same value. The current value of *ptr will
337 : : * always be stored in *expected.
338 : : *
339 : : * Return true if values have been exchanged, false otherwise.
340 : : *
341 : : * Full barrier semantics.
342 : : */
343 : : static inline bool
344 : 223306196 : pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr,
345 : : uint32 *expected, uint32 newval)
346 : : {
347 [ - + ]: 223306196 : AssertPointerAlignment(ptr, 4);
348 [ - + ]: 223306196 : AssertPointerAlignment(expected, 4);
349 : :
350 : 223306196 : return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval);
351 : : }
352 : :
353 : : /*
354 : : * pg_atomic_fetch_add_u32 - atomically add to variable
355 : : *
356 : : * Returns the value of ptr before the arithmetic operation.
357 : : *
358 : : * Full barrier semantics.
359 : : */
360 : : static inline uint32
361 : 8227205 : pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
362 : : {
363 [ - + ]: 8227205 : AssertPointerAlignment(ptr, 4);
364 : 8227205 : return pg_atomic_fetch_add_u32_impl(ptr, add_);
365 : : }
366 : :
367 : : /*
368 : : * pg_atomic_fetch_sub_u32 - atomically subtract from variable
369 : : *
370 : : * Returns the value of ptr before the arithmetic operation. Note that sub_
371 : : * may not be INT_MIN due to platform limitations.
372 : : *
373 : : * Full barrier semantics.
374 : : */
375 : : static inline uint32
376 : 861381 : pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
377 : : {
378 [ - + ]: 861381 : AssertPointerAlignment(ptr, 4);
379 [ - + ]: 861381 : Assert(sub_ != INT_MIN);
380 : 861381 : return pg_atomic_fetch_sub_u32_impl(ptr, sub_);
381 : : }
382 : :
383 : : /*
384 : : * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable
385 : : *
386 : : * Returns the value of ptr before the arithmetic operation.
387 : : *
388 : : * Full barrier semantics.
389 : : */
390 : : static inline uint32
391 : 3358357 : pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_)
392 : : {
393 [ - + ]: 3358357 : AssertPointerAlignment(ptr, 4);
394 : 3358357 : return pg_atomic_fetch_and_u32_impl(ptr, and_);
395 : : }
396 : :
397 : : /*
398 : : * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable
399 : : *
400 : : * Returns the value of ptr before the arithmetic operation.
401 : : *
402 : : * Full barrier semantics.
403 : : */
404 : : static inline uint32
405 : 3702495 : pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_)
406 : : {
407 [ - + ]: 3702495 : AssertPointerAlignment(ptr, 4);
408 : 3702495 : return pg_atomic_fetch_or_u32_impl(ptr, or_);
409 : : }
410 : :
411 : : /*
412 : : * pg_atomic_add_fetch_u32 - atomically add to variable
413 : : *
414 : : * Returns the value of ptr after the arithmetic operation.
415 : : *
416 : : * Full barrier semantics.
417 : : */
418 : : static inline uint32
419 : 814 : pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
420 : : {
421 [ - + ]: 814 : AssertPointerAlignment(ptr, 4);
422 : 814 : return pg_atomic_add_fetch_u32_impl(ptr, add_);
423 : : }
424 : :
425 : : /*
426 : : * pg_atomic_sub_fetch_u32 - atomically subtract from variable
427 : : *
428 : : * Returns the value of ptr after the arithmetic operation. Note that sub_ may
429 : : * not be INT_MIN due to platform limitations.
430 : : *
431 : : * Full barrier semantics.
432 : : */
433 : : static inline uint32
434 : 222827473 : pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
435 : : {
436 [ - + ]: 222827473 : AssertPointerAlignment(ptr, 4);
437 [ - + ]: 222827473 : Assert(sub_ != INT_MIN);
438 : 222827473 : return pg_atomic_sub_fetch_u32_impl(ptr, sub_);
439 : : }
440 : :
441 : : /* ----
442 : : * The 64 bit operations have the same semantics as their 32bit counterparts
443 : : * if they are available. Check the corresponding 32bit function for
444 : : * documentation.
445 : : * ----
446 : : */
447 : : static inline void
448 : 15060313 : pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
449 : : {
450 : : /*
451 : : * Can't necessarily enforce alignment - and don't need it - when using
452 : : * the spinlock based fallback implementation. Therefore only assert when
453 : : * not using it.
454 : : */
455 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
456 [ - + ]: 15060313 : AssertPointerAlignment(ptr, 8);
457 : : #endif
458 : 15060313 : pg_atomic_init_u64_impl(ptr, val);
459 : 15060313 : }
460 : :
461 : : static inline uint64
462 : 549611675 : pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr)
463 : : {
464 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
465 [ - + ]: 549611675 : AssertPointerAlignment(ptr, 8);
466 : : #endif
467 : 549611675 : return pg_atomic_read_u64_impl(ptr);
468 : : }
469 : :
470 : : static inline uint64
877 nathan@postgresql.or 471 : 11819759 : pg_atomic_read_membarrier_u64(volatile pg_atomic_uint64 *ptr)
472 : : {
473 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
474 [ - + ]: 11819759 : AssertPointerAlignment(ptr, 8);
475 : : #endif
476 : 11819759 : return pg_atomic_read_membarrier_u64_impl(ptr);
477 : : }
478 : :
479 : : static inline void
4321 andres@anarazel.de 480 : 14066872 : pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
481 : : {
482 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
483 [ - + ]: 14066872 : AssertPointerAlignment(ptr, 8);
484 : : #endif
485 : 14066872 : pg_atomic_write_u64_impl(ptr, val);
486 : 14066872 : }
487 : :
488 : : static inline void
234 489 : 5653099 : pg_atomic_unlocked_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
490 : : {
491 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
492 [ - + ]: 5653099 : AssertPointerAlignment(ptr, 8);
493 : : #endif
494 : :
495 : 5653099 : pg_atomic_unlocked_write_u64_impl(ptr, val);
496 : 5653099 : }
497 : :
498 : : static inline void
877 nathan@postgresql.or 499 : 116032 : pg_atomic_write_membarrier_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
500 : : {
501 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
502 [ - + ]: 116032 : AssertPointerAlignment(ptr, 8);
503 : : #endif
504 : 116032 : pg_atomic_write_membarrier_u64_impl(ptr, val);
505 : 116032 : }
506 : :
507 : : static inline uint64
4321 andres@anarazel.de 508 : 27341942 : pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval)
509 : : {
510 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
511 [ - + ]: 27341942 : AssertPointerAlignment(ptr, 8);
512 : : #endif
513 : 27341942 : return pg_atomic_exchange_u64_impl(ptr, newval);
514 : : }
515 : :
516 : : static inline bool
517 : 254855119 : pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr,
518 : : uint64 *expected, uint64 newval)
519 : : {
520 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
521 [ - + ]: 254855119 : AssertPointerAlignment(ptr, 8);
522 : : #endif
523 : 254855119 : return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval);
524 : : }
525 : :
526 : : static inline uint64
527 : 333173 : pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
528 : : {
529 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
530 [ - + ]: 333173 : AssertPointerAlignment(ptr, 8);
531 : : #endif
532 : 333173 : return pg_atomic_fetch_add_u64_impl(ptr, add_);
533 : : }
534 : :
535 : : static inline uint64
536 : 31810101 : pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
537 : : {
538 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
539 [ - + ]: 31810101 : AssertPointerAlignment(ptr, 8);
540 : : #endif
4132 541 [ - + ]: 31810101 : Assert(sub_ != PG_INT64_MIN);
4321 542 : 31810101 : return pg_atomic_fetch_sub_u64_impl(ptr, sub_);
543 : : }
544 : :
545 : : static inline uint64
546 : 43437 : pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_)
547 : : {
548 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
549 [ - + ]: 43437 : AssertPointerAlignment(ptr, 8);
550 : : #endif
551 : 43437 : return pg_atomic_fetch_and_u64_impl(ptr, and_);
552 : : }
553 : :
554 : : static inline uint64
555 : 28015899 : pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_)
556 : : {
557 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
558 [ - + ]: 28015899 : AssertPointerAlignment(ptr, 8);
559 : : #endif
560 : 28015899 : return pg_atomic_fetch_or_u64_impl(ptr, or_);
561 : : }
562 : :
563 : : static inline uint64
564 : 1079 : pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
565 : : {
566 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
567 [ - + ]: 1079 : AssertPointerAlignment(ptr, 8);
568 : : #endif
569 : 1079 : return pg_atomic_add_fetch_u64_impl(ptr, add_);
570 : : }
571 : :
572 : : static inline uint64
573 : 112219798 : pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
574 : : {
575 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
576 [ - + ]: 112219798 : AssertPointerAlignment(ptr, 8);
577 : : #endif
4132 578 [ - + ]: 112219798 : Assert(sub_ != PG_INT64_MIN);
4321 579 : 112219798 : return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
580 : : }
581 : :
582 : : /*
583 : : * Monotonically advance the given variable using only atomic operations until
584 : : * it's at least the target value. Returns the latest value observed, which
585 : : * may or may not be the target value.
586 : : *
587 : : * Full barrier semantics (even when value is unchanged).
588 : : */
589 : : static inline uint64
751 alvherre@alvh.no-ip. 590 : 548940 : pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target)
591 : : {
592 : : uint64 currval;
593 : :
594 : : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
839 595 [ - + ]: 548940 : AssertPointerAlignment(ptr, 8);
596 : : #endif
597 : :
598 : 548940 : currval = pg_atomic_read_u64_impl(ptr);
751 599 [ + + ]: 548940 : if (currval >= target)
600 : : {
839 601 : 96540 : pg_memory_barrier();
602 : 96540 : return currval;
603 : : }
604 : :
751 605 [ + + ]: 452551 : while (currval < target)
606 : : {
607 [ + + ]: 452405 : if (pg_atomic_compare_exchange_u64(ptr, &currval, target))
608 : 452254 : return target;
609 : : }
610 : :
611 : 146 : return currval;
612 : : }
613 : :
614 : : #undef INSIDE_ATOMICS_H
615 : :
616 : : #endif /* ATOMICS_H */
|
