Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * arch-x86.h
4 : * Atomic operations considerations specific to intel x86
5 : *
6 : * Note that we actually require a 486 upwards because the 386 doesn't have
7 : * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere
8 : * anymore that's not much of a restriction luckily.
9 : *
10 : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
11 : * Portions Copyright (c) 1994, Regents of the University of California
12 : *
13 : * NOTES:
14 : *
15 : * src/include/port/atomics/arch-x86.h
16 : *
17 : *-------------------------------------------------------------------------
18 : */
19 :
20 : /*
21 : * Both 32 and 64 bit x86 do not allow loads to be reordered with other loads,
22 : * or stores to be reordered with other stores, but a load can be performed
23 : * before a subsequent store.
24 : *
25 : * Technically, some x86-ish chips support uncached memory access and/or
26 : * special instructions that are weakly ordered. In those cases we'd need
27 : * the read and write barriers to be lfence and sfence. But since we don't
28 : * do those things, a compiler barrier should be enough.
29 : *
30 : * "lock; addl" has worked for longer than "mfence". It's also rumored to be
31 : * faster in many scenarios.
32 : */
33 :
34 : #if defined(__GNUC__) || defined(__INTEL_COMPILER)
35 : #if defined(__i386__)
36 : #define pg_memory_barrier_impl() \
37 : __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory", "cc")
38 : #elif defined(__x86_64__)
39 : #define pg_memory_barrier_impl() \
40 : __asm__ __volatile__ ("lock; addl $0,0(%%rsp)" : : : "memory", "cc")
41 : #endif
42 : #endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */
43 :
44 : #define pg_read_barrier_impl() pg_compiler_barrier_impl()
45 : #define pg_write_barrier_impl() pg_compiler_barrier_impl()
46 :
47 : /*
48 : * Provide implementation for atomics using inline assembly on x86 gcc. It's
49 : * nice to support older gcc's and the compare/exchange implementation here is
50 : * actually more efficient than the * __sync variant.
51 : */
52 : #if defined(__GNUC__) || defined(__INTEL_COMPILER)
53 :
54 : #define PG_HAVE_ATOMIC_FLAG_SUPPORT
55 : typedef struct pg_atomic_flag
56 : {
57 : volatile char value;
58 : } pg_atomic_flag;
59 :
60 : #define PG_HAVE_ATOMIC_U32_SUPPORT
61 : typedef struct pg_atomic_uint32
62 : {
63 : volatile uint32 value;
64 : } pg_atomic_uint32;
65 :
66 : /*
67 : * It's too complicated to write inline asm for 64bit types on 32bit and the
68 : * 486 can't do it anyway.
69 : */
70 : #ifdef __x86_64__
71 : #define PG_HAVE_ATOMIC_U64_SUPPORT
72 : typedef struct pg_atomic_uint64
73 : {
74 : /* alignment guaranteed due to being on a 64bit platform */
75 : volatile uint64 value;
76 : } pg_atomic_uint64;
77 : #endif /* __x86_64__ */
78 :
79 : #define PG_HAVE_ATOMIC_TEST_SET_FLAG
80 : static inline bool
81 223038 : pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
82 : {
83 223038 : char _res = 1;
84 :
85 223038 : __asm__ __volatile__(
86 : " lock \n"
87 : " xchgb %0,%1 \n"
88 : : "+q"(_res), "+m"(ptr->value)
89 : :
90 : : "memory");
91 223038 : return _res == 0;
92 : }
93 :
94 : #define PG_HAVE_ATOMIC_CLEAR_FLAG
95 : static inline void
96 15404 : pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
97 : {
98 : /*
99 : * On a TSO architecture like x86 it's sufficient to use a compiler
100 : * barrier to achieve release semantics.
101 : */
102 15404 : __asm__ __volatile__("" ::: "memory");
103 15404 : ptr->value = 0;
104 15404 : }
105 :
106 : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
107 : static inline bool
108 373295453 : pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
109 : uint32 *expected, uint32 newval)
110 : {
111 : char ret;
112 :
113 : /*
114 : * Perform cmpxchg and use the zero flag which it implicitly sets when
115 : * equal to measure the success.
116 : */
117 373295453 : __asm__ __volatile__(
118 : " lock \n"
119 : " cmpxchgl %4,%5 \n"
120 : " setz %2 \n"
121 : : "=a" (*expected), "=m"(ptr->value), "=q" (ret)
122 373295453 : : "a" (*expected), "r" (newval), "m"(ptr->value)
123 : : "memory", "cc");
124 373295453 : return (bool) ret;
125 : }
126 :
127 : #define PG_HAVE_ATOMIC_FETCH_ADD_U32
128 : static inline uint32
129 8357710 : pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
130 : {
131 : uint32 res;
132 8357710 : __asm__ __volatile__(
133 : " lock \n"
134 : " xaddl %0,%1 \n"
135 : : "=q"(res), "=m"(ptr->value)
136 : : "0" (add_), "m"(ptr->value)
137 : : "memory", "cc");
138 8357710 : return res;
139 : }
140 :
141 : #ifdef __x86_64__
142 :
143 : #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
144 : static inline bool
145 265203152 : pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
146 : uint64 *expected, uint64 newval)
147 : {
148 : char ret;
149 :
150 : AssertPointerAlignment(expected, 8);
151 :
152 : /*
153 : * Perform cmpxchg and use the zero flag which it implicitly sets when
154 : * equal to measure the success.
155 : */
156 265203152 : __asm__ __volatile__(
157 : " lock \n"
158 : " cmpxchgq %4,%5 \n"
159 : " setz %2 \n"
160 : : "=a" (*expected), "=m"(ptr->value), "=q" (ret)
161 265203152 : : "a" (*expected), "r" (newval), "m"(ptr->value)
162 : : "memory", "cc");
163 265203152 : return (bool) ret;
164 : }
165 :
166 : #define PG_HAVE_ATOMIC_FETCH_ADD_U64
167 : static inline uint64
168 12141717 : pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
169 : {
170 : uint64 res;
171 12141717 : __asm__ __volatile__(
172 : " lock \n"
173 : " xaddq %0,%1 \n"
174 : : "=q"(res), "=m"(ptr->value)
175 : : "0" (add_), "m"(ptr->value)
176 : : "memory", "cc");
177 12141717 : return res;
178 : }
179 :
180 : #endif /* __x86_64__ */
181 :
182 : #endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */
183 :
184 : /*
185 : * 8 byte reads / writes have single-copy atomicity on all x86-64 cpus.
186 : */
187 : #if defined(__x86_64__)
188 : #define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY
189 : #endif /* 8 byte single-copy atomicity */
|
