LCOV - code coverage report
Current view: top level - src/include/port/atomics - generic-gcc.h (source / functions) Hit Total Coverage
Test: PostgreSQL 16beta1 Lines: 17 17 100.0 %
Date: 2023-05-31 03:12:18 Functions: 8 8 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * generic-gcc.h
       4             :  *    Atomic operations, implemented using gcc (or compatible) intrinsics.
       5             :  *
       6             :  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  * NOTES:
      10             :  *
      11             :  * Documentation:
      12             :  * * Legacy __sync Built-in Functions for Atomic Memory Access
      13             :  *   https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html
      14             :  * * Built-in functions for memory model aware atomic operations
      15             :  *   https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html
      16             :  *
      17             :  * src/include/port/atomics/generic-gcc.h
      18             :  *
      19             :  *-------------------------------------------------------------------------
      20             :  */
      21             : 
      22             : /* intentionally no include guards, should only be included by atomics.h */
      23             : #ifndef INSIDE_ATOMICS_H
      24             : #error "should be included via atomics.h"
      25             : #endif
      26             : 
      27             : /*
      28             :  * An empty asm block should be a sufficient compiler barrier.
      29             :  */
      30             : #define pg_compiler_barrier_impl()  __asm__ __volatile__("" ::: "memory")
      31             : 
      32             : /*
      33             :  * If we're on GCC 4.1.0 or higher, we should be able to get a memory barrier
      34             :  * out of this compiler built-in.  But we prefer to rely on platform specific
      35             :  * definitions where possible, and use this only as a fallback.
      36             :  */
      37             : #if !defined(pg_memory_barrier_impl)
      38             : #   if defined(HAVE_GCC__ATOMIC_INT32_CAS)
      39             : #       define pg_memory_barrier_impl()     __atomic_thread_fence(__ATOMIC_SEQ_CST)
      40             : #   elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))
      41             : #       define pg_memory_barrier_impl()     __sync_synchronize()
      42             : #   endif
      43             : #endif /* !defined(pg_memory_barrier_impl) */
      44             : 
      45             : #if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
      46             : /* acquire semantics include read barrier semantics */
      47             : #       define pg_read_barrier_impl()       __atomic_thread_fence(__ATOMIC_ACQUIRE)
      48             : #endif
      49             : 
      50             : #if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
      51             : /* release semantics include write barrier semantics */
      52             : #       define pg_write_barrier_impl()      __atomic_thread_fence(__ATOMIC_RELEASE)
      53             : #endif
      54             : 
      55             : 
      56             : #ifdef HAVE_ATOMICS
      57             : 
      58             : /* generic gcc based atomic flag implementation */
      59             : #if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \
      60             :     && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS))
      61             : 
      62             : #define PG_HAVE_ATOMIC_FLAG_SUPPORT
      63             : typedef struct pg_atomic_flag
      64             : {
      65             :     /*
      66             :      * If we have a choice, use int-width TAS, because that is more efficient
      67             :      * and/or more reliably implemented on most non-Intel platforms.  (Note
      68             :      * that this code isn't used on x86[_64]; see arch-x86.h for that.)
      69             :      */
      70             : #ifdef HAVE_GCC__SYNC_INT32_TAS
      71             :     volatile int value;
      72             : #else
      73             :     volatile char value;
      74             : #endif
      75             : } pg_atomic_flag;
      76             : 
      77             : #endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */
      78             : 
      79             : /* generic gcc based atomic uint32 implementation */
      80             : #if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \
      81             :     && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS))
      82             : 
      83             : #define PG_HAVE_ATOMIC_U32_SUPPORT
      84             : typedef struct pg_atomic_uint32
      85             : {
      86             :     volatile uint32 value;
      87             : } pg_atomic_uint32;
      88             : 
      89             : #endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */
      90             : 
      91             : /* generic gcc based atomic uint64 implementation */
      92             : #if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \
      93             :     && !defined(PG_DISABLE_64_BIT_ATOMICS) \
      94             :     && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS))
      95             : 
      96             : #define PG_HAVE_ATOMIC_U64_SUPPORT
      97             : 
      98             : typedef struct pg_atomic_uint64
      99             : {
     100             :     volatile uint64 value pg_attribute_aligned(8);
     101             : } pg_atomic_uint64;
     102             : 
     103             : #endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */
     104             : 
     105             : #ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT
     106             : 
     107             : #if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS)
     108             : 
     109             : #ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG
     110             : #define PG_HAVE_ATOMIC_TEST_SET_FLAG
     111             : static inline bool
     112             : pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
     113             : {
     114             :     /* NB: only an acquire barrier, not a full one */
     115             :     /* some platform only support a 1 here */
     116             :     return __sync_lock_test_and_set(&ptr->value, 1) == 0;
     117             : }
     118             : #endif
     119             : 
     120             : #endif /* defined(HAVE_GCC__SYNC_*_TAS) */
     121             : 
     122             : #ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
     123             : #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
     124             : static inline bool
     125        1754 : pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
     126             : {
     127        1754 :     return ptr->value == 0;
     128             : }
     129             : #endif
     130             : 
     131             : #ifndef PG_HAVE_ATOMIC_CLEAR_FLAG
     132             : #define PG_HAVE_ATOMIC_CLEAR_FLAG
     133             : static inline void
     134             : pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
     135             : {
     136             :     __sync_lock_release(&ptr->value);
     137             : }
     138             : #endif
     139             : 
     140             : #ifndef PG_HAVE_ATOMIC_INIT_FLAG
     141             : #define PG_HAVE_ATOMIC_INIT_FLAG
     142             : static inline void
     143       10914 : pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
     144             : {
     145       10914 :     pg_atomic_clear_flag_impl(ptr);
     146       10914 : }
     147             : #endif
     148             : 
     149             : #endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */
     150             : 
     151             : /* prefer __atomic, it has a better API */
     152             : #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
     153             : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
     154             : static inline bool
     155             : pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
     156             :                                     uint32 *expected, uint32 newval)
     157             : {
     158             :     /* FIXME: we can probably use a lower consistency model */
     159             :     return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
     160             :                                        __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
     161             : }
     162             : #endif
     163             : 
     164             : #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
     165             : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
     166             : static inline bool
     167             : pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
     168             :                                     uint32 *expected, uint32 newval)
     169             : {
     170             :     bool    ret;
     171             :     uint32  current;
     172             :     current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
     173             :     ret = current == *expected;
     174             :     *expected = current;
     175             :     return ret;
     176             : }
     177             : #endif
     178             : 
     179             : /* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */
     180             : 
     181             : #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
     182             : #define PG_HAVE_ATOMIC_FETCH_ADD_U32
     183             : static inline uint32
     184             : pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
     185             : {
     186             :     return __sync_fetch_and_add(&ptr->value, add_);
     187             : }
     188             : #endif
     189             : 
     190             : #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
     191             : #define PG_HAVE_ATOMIC_FETCH_SUB_U32
     192             : static inline uint32
     193   572414712 : pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
     194             : {
     195   572414712 :     return __sync_fetch_and_sub(&ptr->value, sub_);
     196             : }
     197             : #endif
     198             : 
     199             : #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
     200             : #define PG_HAVE_ATOMIC_FETCH_AND_U32
     201             : static inline uint32
     202    44848838 : pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
     203             : {
     204    44848838 :     return __sync_fetch_and_and(&ptr->value, and_);
     205             : }
     206             : #endif
     207             : 
     208             : #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
     209             : #define PG_HAVE_ATOMIC_FETCH_OR_U32
     210             : static inline uint32
     211   146789746 : pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
     212             : {
     213   146789746 :     return __sync_fetch_and_or(&ptr->value, or_);
     214             : }
     215             : #endif
     216             : 
     217             : 
     218             : #if !defined(PG_DISABLE_64_BIT_ATOMICS)
     219             : 
     220             : #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
     221             : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
     222             : static inline bool
     223             : pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
     224             :                                     uint64 *expected, uint64 newval)
     225             : {
     226             :     return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
     227             :                                        __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
     228             : }
     229             : #endif
     230             : 
     231             : #if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
     232             : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
     233             : static inline bool
     234             : pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
     235             :                                     uint64 *expected, uint64 newval)
     236             : {
     237             :     bool    ret;
     238             :     uint64  current;
     239             :     current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
     240             :     ret = current == *expected;
     241             :     *expected = current;
     242             :     return ret;
     243             : }
     244             : #endif
     245             : 
     246             : /* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */
     247             : 
     248             : #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
     249             : #define PG_HAVE_ATOMIC_FETCH_ADD_U64
     250             : static inline uint64
     251             : pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
     252             : {
     253             :     return __sync_fetch_and_add(&ptr->value, add_);
     254             : }
     255             : #endif
     256             : 
     257             : #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
     258             : #define PG_HAVE_ATOMIC_FETCH_SUB_U64
     259             : static inline uint64
     260        1326 : pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
     261             : {
     262        1326 :     return __sync_fetch_and_sub(&ptr->value, sub_);
     263             : }
     264             : #endif
     265             : 
     266             : #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
     267             : #define PG_HAVE_ATOMIC_FETCH_AND_U64
     268             : static inline uint64
     269          18 : pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
     270             : {
     271          18 :     return __sync_fetch_and_and(&ptr->value, and_);
     272             : }
     273             : #endif
     274             : 
     275             : #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
     276             : #define PG_HAVE_ATOMIC_FETCH_OR_U64
     277             : static inline uint64
     278          12 : pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
     279             : {
     280          12 :     return __sync_fetch_and_or(&ptr->value, or_);
     281             : }
     282             : #endif
     283             : 
     284             : #endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */
     285             : 
     286             : #endif /* defined(HAVE_ATOMICS) */

Generated by: LCOV version 1.14