1/* Atomic operations. X86 version.
2 Copyright (C) 2018-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#ifndef _X86_ATOMIC_MACHINE_H
20#define _X86_ATOMIC_MACHINE_H 1
21
22#include <stdint.h>
23#include <tls.h> /* For tcbhead_t. */
24#include <libc-pointer-arith.h> /* For cast_to_integer. */
25
26typedef int8_t atomic8_t;
27typedef uint8_t uatomic8_t;
28typedef int_fast8_t atomic_fast8_t;
29typedef uint_fast8_t uatomic_fast8_t;
30
31typedef int16_t atomic16_t;
32typedef uint16_t uatomic16_t;
33typedef int_fast16_t atomic_fast16_t;
34typedef uint_fast16_t uatomic_fast16_t;
35
36typedef int32_t atomic32_t;
37typedef uint32_t uatomic32_t;
38typedef int_fast32_t atomic_fast32_t;
39typedef uint_fast32_t uatomic_fast32_t;
40
41typedef int64_t atomic64_t;
42typedef uint64_t uatomic64_t;
43typedef int_fast64_t atomic_fast64_t;
44typedef uint_fast64_t uatomic_fast64_t;
45
46typedef intptr_t atomicptr_t;
47typedef uintptr_t uatomicptr_t;
48typedef intmax_t atomic_max_t;
49typedef uintmax_t uatomic_max_t;
50
51
52#define LOCK_PREFIX "lock;"
53
54#define USE_ATOMIC_COMPILER_BUILTINS 1
55
56#ifdef __x86_64__
57# define __HAVE_64B_ATOMICS 1
58# define SP_REG "rsp"
59# define SEG_REG "fs"
60# define BR_CONSTRAINT "q"
61# define IBR_CONSTRAINT "iq"
62#else
63# define __HAVE_64B_ATOMICS 0
64# define SP_REG "esp"
65# define SEG_REG "gs"
66# define BR_CONSTRAINT "r"
67# define IBR_CONSTRAINT "ir"
68#endif
69#define ATOMIC_EXCHANGE_USES_CAS 0
70
71#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
72 __sync_val_compare_and_swap (mem, oldval, newval)
73#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
74 (! __sync_bool_compare_and_swap (mem, oldval, newval))
75
76
77#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \
78 ({ __typeof (*mem) ret; \
79 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
80 "je 0f\n\t" \
81 "lock\n" \
82 "0:\tcmpxchgb %b2, %1" \
83 : "=a" (ret), "=m" (*mem) \
84 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
85 "i" (offsetof (tcbhead_t, multiple_threads))); \
86 ret; })
87
88#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \
89 ({ __typeof (*mem) ret; \
90 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
91 "je 0f\n\t" \
92 "lock\n" \
93 "0:\tcmpxchgw %w2, %1" \
94 : "=a" (ret), "=m" (*mem) \
95 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
96 "i" (offsetof (tcbhead_t, multiple_threads))); \
97 ret; })
98
99#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \
100 ({ __typeof (*mem) ret; \
101 __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \
102 "je 0f\n\t" \
103 "lock\n" \
104 "0:\tcmpxchgl %2, %1" \
105 : "=a" (ret), "=m" (*mem) \
106 : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \
107 "i" (offsetof (tcbhead_t, multiple_threads))); \
108 ret; })
109
110#ifdef __x86_64__
111# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
112 ({ __typeof (*mem) ret; \
113 __asm __volatile ("cmpl $0, %%fs:%P5\n\t" \
114 "je 0f\n\t" \
115 "lock\n" \
116 "0:\tcmpxchgq %q2, %1" \
117 : "=a" (ret), "=m" (*mem) \
118 : "q" ((atomic64_t) cast_to_integer (newval)), \
119 "m" (*mem), \
120 "0" ((atomic64_t) cast_to_integer (oldval)), \
121 "i" (offsetof (tcbhead_t, multiple_threads))); \
122 ret; })
123# define do_exchange_and_add_val_64_acq(pfx, mem, value) 0
124# define do_add_val_64_acq(pfx, mem, value) do { } while (0)
125#else
126/* XXX We do not really need 64-bit compare-and-exchange. At least
127 not in the moment. Using it would mean causing portability
128 problems since not many other 32-bit architectures have support for
129 such an operation. So don't define any code for now. If it is
130 really going to be used the code below can be used on Intel Pentium
131 and later, but NOT on i486. */
132# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \
133 ({ __typeof (*mem) ret = *(mem); \
134 __atomic_link_error (); \
135 ret = (newval); \
136 ret = (oldval); \
137 ret; })
138
139# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \
140 ({ __typeof (*mem) ret = *(mem); \
141 __atomic_link_error (); \
142 ret = (newval); \
143 ret = (oldval); \
144 ret; })
145
146# define do_exchange_and_add_val_64_acq(pfx, mem, value) \
147 ({ __typeof (value) __addval = (value); \
148 __typeof (*mem) __result; \
149 __typeof (mem) __memp = (mem); \
150 __typeof (*mem) __tmpval; \
151 __result = *__memp; \
152 do \
153 __tmpval = __result; \
154 while ((__result = pfx##_compare_and_exchange_val_64_acq \
155 (__memp, __result + __addval, __result)) == __tmpval); \
156 __result; })
157
158# define do_add_val_64_acq(pfx, mem, value) \
159 { \
160 __typeof (value) __addval = (value); \
161 __typeof (mem) __memp = (mem); \
162 __typeof (*mem) __oldval = *__memp; \
163 __typeof (*mem) __tmpval; \
164 do \
165 __tmpval = __oldval; \
166 while ((__oldval = pfx##_compare_and_exchange_val_64_acq \
167 (__memp, __oldval + __addval, __oldval)) == __tmpval); \
168 }
169#endif
170
171
172/* Note that we need no lock prefix. */
173#define atomic_exchange_acq(mem, newvalue) \
174 ({ __typeof (*mem) result; \
175 if (sizeof (*mem) == 1) \
176 __asm __volatile ("xchgb %b0, %1" \
177 : "=q" (result), "=m" (*mem) \
178 : "0" (newvalue), "m" (*mem)); \
179 else if (sizeof (*mem) == 2) \
180 __asm __volatile ("xchgw %w0, %1" \
181 : "=r" (result), "=m" (*mem) \
182 : "0" (newvalue), "m" (*mem)); \
183 else if (sizeof (*mem) == 4) \
184 __asm __volatile ("xchgl %0, %1" \
185 : "=r" (result), "=m" (*mem) \
186 : "0" (newvalue), "m" (*mem)); \
187 else if (__HAVE_64B_ATOMICS) \
188 __asm __volatile ("xchgq %q0, %1" \
189 : "=r" (result), "=m" (*mem) \
190 : "0" ((atomic64_t) cast_to_integer (newvalue)), \
191 "m" (*mem)); \
192 else \
193 { \
194 result = 0; \
195 __atomic_link_error (); \
196 } \
197 result; })
198
199
200#define __arch_exchange_and_add_body(lock, pfx, mem, value) \
201 ({ __typeof (*mem) __result; \
202 __typeof (value) __addval = (value); \
203 if (sizeof (*mem) == 1) \
204 __asm __volatile (lock "xaddb %b0, %1" \
205 : "=q" (__result), "=m" (*mem) \
206 : "0" (__addval), "m" (*mem), \
207 "i" (offsetof (tcbhead_t, multiple_threads))); \
208 else if (sizeof (*mem) == 2) \
209 __asm __volatile (lock "xaddw %w0, %1" \
210 : "=r" (__result), "=m" (*mem) \
211 : "0" (__addval), "m" (*mem), \
212 "i" (offsetof (tcbhead_t, multiple_threads))); \
213 else if (sizeof (*mem) == 4) \
214 __asm __volatile (lock "xaddl %0, %1" \
215 : "=r" (__result), "=m" (*mem) \
216 : "0" (__addval), "m" (*mem), \
217 "i" (offsetof (tcbhead_t, multiple_threads))); \
218 else if (__HAVE_64B_ATOMICS) \
219 __asm __volatile (lock "xaddq %q0, %1" \
220 : "=r" (__result), "=m" (*mem) \
221 : "0" ((atomic64_t) cast_to_integer (__addval)), \
222 "m" (*mem), \
223 "i" (offsetof (tcbhead_t, multiple_threads))); \
224 else \
225 __result = do_exchange_and_add_val_64_acq (pfx, (mem), __addval); \
226 __result; })
227
228#define atomic_exchange_and_add(mem, value) \
229 __sync_fetch_and_add (mem, value)
230
231#define __arch_exchange_and_add_cprefix \
232 "cmpl $0, %%" SEG_REG ":%P4\n\tje 0f\n\tlock\n0:\t"
233
234#define catomic_exchange_and_add(mem, value) \
235 __arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, __arch_c, \
236 mem, value)
237
238
239#define __arch_add_body(lock, pfx, apfx, mem, value) \
240 do { \
241 if (__builtin_constant_p (value) && (value) == 1) \
242 pfx##_increment (mem); \
243 else if (__builtin_constant_p (value) && (value) == -1) \
244 pfx##_decrement (mem); \
245 else if (sizeof (*mem) == 1) \
246 __asm __volatile (lock "addb %b1, %0" \
247 : "=m" (*mem) \
248 : IBR_CONSTRAINT (value), "m" (*mem), \
249 "i" (offsetof (tcbhead_t, multiple_threads))); \
250 else if (sizeof (*mem) == 2) \
251 __asm __volatile (lock "addw %w1, %0" \
252 : "=m" (*mem) \
253 : "ir" (value), "m" (*mem), \
254 "i" (offsetof (tcbhead_t, multiple_threads))); \
255 else if (sizeof (*mem) == 4) \
256 __asm __volatile (lock "addl %1, %0" \
257 : "=m" (*mem) \
258 : "ir" (value), "m" (*mem), \
259 "i" (offsetof (tcbhead_t, multiple_threads))); \
260 else if (__HAVE_64B_ATOMICS) \
261 __asm __volatile (lock "addq %q1, %0" \
262 : "=m" (*mem) \
263 : "ir" ((atomic64_t) cast_to_integer (value)), \
264 "m" (*mem), \
265 "i" (offsetof (tcbhead_t, multiple_threads))); \
266 else \
267 do_add_val_64_acq (apfx, (mem), (value)); \
268 } while (0)
269
270# define atomic_add(mem, value) \
271 __arch_add_body (LOCK_PREFIX, atomic, __arch, mem, value)
272
273#define __arch_add_cprefix \
274 "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t"
275
276#define catomic_add(mem, value) \
277 __arch_add_body (__arch_add_cprefix, atomic, __arch_c, mem, value)
278
279
280#define atomic_add_negative(mem, value) \
281 ({ unsigned char __result; \
282 if (sizeof (*mem) == 1) \
283 __asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1" \
284 : "=m" (*mem), "=qm" (__result) \
285 : IBR_CONSTRAINT (value), "m" (*mem)); \
286 else if (sizeof (*mem) == 2) \
287 __asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1" \
288 : "=m" (*mem), "=qm" (__result) \
289 : "ir" (value), "m" (*mem)); \
290 else if (sizeof (*mem) == 4) \
291 __asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1" \
292 : "=m" (*mem), "=qm" (__result) \
293 : "ir" (value), "m" (*mem)); \
294 else if (__HAVE_64B_ATOMICS) \
295 __asm __volatile (LOCK_PREFIX "addq %q2, %0; sets %1" \
296 : "=m" (*mem), "=qm" (__result) \
297 : "ir" ((atomic64_t) cast_to_integer (value)), \
298 "m" (*mem)); \
299 else \
300 __atomic_link_error (); \
301 __result; })
302
303
304#define atomic_add_zero(mem, value) \
305 ({ unsigned char __result; \
306 if (sizeof (*mem) == 1) \
307 __asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1" \
308 : "=m" (*mem), "=qm" (__result) \
309 : IBR_CONSTRAINT (value), "m" (*mem)); \
310 else if (sizeof (*mem) == 2) \
311 __asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1" \
312 : "=m" (*mem), "=qm" (__result) \
313 : "ir" (value), "m" (*mem)); \
314 else if (sizeof (*mem) == 4) \
315 __asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1" \
316 : "=m" (*mem), "=qm" (__result) \
317 : "ir" (value), "m" (*mem)); \
318 else if (__HAVE_64B_ATOMICS) \
319 __asm __volatile (LOCK_PREFIX "addq %q2, %0; setz %1" \
320 : "=m" (*mem), "=qm" (__result) \
321 : "ir" ((atomic64_t) cast_to_integer (value)), \
322 "m" (*mem)); \
323 else \
324 __atomic_link_error (); \
325 __result; })
326
327
328#define __arch_increment_body(lock, pfx, mem) \
329 do { \
330 if (sizeof (*mem) == 1) \
331 __asm __volatile (lock "incb %b0" \
332 : "=m" (*mem) \
333 : "m" (*mem), \
334 "i" (offsetof (tcbhead_t, multiple_threads))); \
335 else if (sizeof (*mem) == 2) \
336 __asm __volatile (lock "incw %w0" \
337 : "=m" (*mem) \
338 : "m" (*mem), \
339 "i" (offsetof (tcbhead_t, multiple_threads))); \
340 else if (sizeof (*mem) == 4) \
341 __asm __volatile (lock "incl %0" \
342 : "=m" (*mem) \
343 : "m" (*mem), \
344 "i" (offsetof (tcbhead_t, multiple_threads))); \
345 else if (__HAVE_64B_ATOMICS) \
346 __asm __volatile (lock "incq %q0" \
347 : "=m" (*mem) \
348 : "m" (*mem), \
349 "i" (offsetof (tcbhead_t, multiple_threads))); \
350 else \
351 do_add_val_64_acq (pfx, mem, 1); \
352 } while (0)
353
354#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, __arch, mem)
355
356#define __arch_increment_cprefix \
357 "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t"
358
359#define catomic_increment(mem) \
360 __arch_increment_body (__arch_increment_cprefix, __arch_c, mem)
361
362
363#define atomic_increment_and_test(mem) \
364 ({ unsigned char __result; \
365 if (sizeof (*mem) == 1) \
366 __asm __volatile (LOCK_PREFIX "incb %b0; sete %b1" \
367 : "=m" (*mem), "=qm" (__result) \
368 : "m" (*mem)); \
369 else if (sizeof (*mem) == 2) \
370 __asm __volatile (LOCK_PREFIX "incw %w0; sete %w1" \
371 : "=m" (*mem), "=qm" (__result) \
372 : "m" (*mem)); \
373 else if (sizeof (*mem) == 4) \
374 __asm __volatile (LOCK_PREFIX "incl %0; sete %1" \
375 : "=m" (*mem), "=qm" (__result) \
376 : "m" (*mem)); \
377 else if (__HAVE_64B_ATOMICS) \
378 __asm __volatile (LOCK_PREFIX "incq %q0; sete %1" \
379 : "=m" (*mem), "=qm" (__result) \
380 : "m" (*mem)); \
381 else \
382 __atomic_link_error (); \
383 __result; })
384
385
386#define __arch_decrement_body(lock, pfx, mem) \
387 do { \
388 if (sizeof (*mem) == 1) \
389 __asm __volatile (lock "decb %b0" \
390 : "=m" (*mem) \
391 : "m" (*mem), \
392 "i" (offsetof (tcbhead_t, multiple_threads))); \
393 else if (sizeof (*mem) == 2) \
394 __asm __volatile (lock "decw %w0" \
395 : "=m" (*mem) \
396 : "m" (*mem), \
397 "i" (offsetof (tcbhead_t, multiple_threads))); \
398 else if (sizeof (*mem) == 4) \
399 __asm __volatile (lock "decl %0" \
400 : "=m" (*mem) \
401 : "m" (*mem), \
402 "i" (offsetof (tcbhead_t, multiple_threads))); \
403 else if (__HAVE_64B_ATOMICS) \
404 __asm __volatile (lock "decq %q0" \
405 : "=m" (*mem) \
406 : "m" (*mem), \
407 "i" (offsetof (tcbhead_t, multiple_threads))); \
408 else \
409 do_add_val_64_acq (pfx, mem, -1); \
410 } while (0)
411
412#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, __arch, mem)
413
414#define __arch_decrement_cprefix \
415 "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t"
416
417#define catomic_decrement(mem) \
418 __arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem)
419
420
421#define atomic_decrement_and_test(mem) \
422 ({ unsigned char __result; \
423 if (sizeof (*mem) == 1) \
424 __asm __volatile (LOCK_PREFIX "decb %b0; sete %1" \
425 : "=m" (*mem), "=qm" (__result) \
426 : "m" (*mem)); \
427 else if (sizeof (*mem) == 2) \
428 __asm __volatile (LOCK_PREFIX "decw %w0; sete %1" \
429 : "=m" (*mem), "=qm" (__result) \
430 : "m" (*mem)); \
431 else if (sizeof (*mem) == 4) \
432 __asm __volatile (LOCK_PREFIX "decl %0; sete %1" \
433 : "=m" (*mem), "=qm" (__result) \
434 : "m" (*mem)); \
435 else \
436 __asm __volatile (LOCK_PREFIX "decq %q0; sete %1" \
437 : "=m" (*mem), "=qm" (__result) \
438 : "m" (*mem)); \
439 __result; })
440
441
442#define atomic_bit_set(mem, bit) \
443 do { \
444 if (sizeof (*mem) == 1) \
445 __asm __volatile (LOCK_PREFIX "orb %b2, %0" \
446 : "=m" (*mem) \
447 : "m" (*mem), IBR_CONSTRAINT (1L << (bit))); \
448 else if (sizeof (*mem) == 2) \
449 __asm __volatile (LOCK_PREFIX "orw %w2, %0" \
450 : "=m" (*mem) \
451 : "m" (*mem), "ir" (1L << (bit))); \
452 else if (sizeof (*mem) == 4) \
453 __asm __volatile (LOCK_PREFIX "orl %2, %0" \
454 : "=m" (*mem) \
455 : "m" (*mem), "ir" (1L << (bit))); \
456 else if (__builtin_constant_p (bit) && (bit) < 32) \
457 __asm __volatile (LOCK_PREFIX "orq %2, %0" \
458 : "=m" (*mem) \
459 : "m" (*mem), "i" (1L << (bit))); \
460 else if (__HAVE_64B_ATOMICS) \
461 __asm __volatile (LOCK_PREFIX "orq %q2, %0" \
462 : "=m" (*mem) \
463 : "m" (*mem), "r" (1UL << (bit))); \
464 else \
465 __atomic_link_error (); \
466 } while (0)
467
468
469#define atomic_bit_test_set(mem, bit) \
470 ({ unsigned char __result; \
471 if (sizeof (*mem) == 1) \
472 __asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0" \
473 : "=q" (__result), "=m" (*mem) \
474 : "m" (*mem), IBR_CONSTRAINT (bit)); \
475 else if (sizeof (*mem) == 2) \
476 __asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0" \
477 : "=q" (__result), "=m" (*mem) \
478 : "m" (*mem), "ir" (bit)); \
479 else if (sizeof (*mem) == 4) \
480 __asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0" \
481 : "=q" (__result), "=m" (*mem) \
482 : "m" (*mem), "ir" (bit)); \
483 else if (__HAVE_64B_ATOMICS) \
484 __asm __volatile (LOCK_PREFIX "btsq %3, %1; setc %0" \
485 : "=q" (__result), "=m" (*mem) \
486 : "m" (*mem), "ir" (bit)); \
487 else \
488 __atomic_link_error (); \
489 __result; })
490
491
492#define __arch_and_body(lock, mem, mask) \
493 do { \
494 if (sizeof (*mem) == 1) \
495 __asm __volatile (lock "andb %b1, %0" \
496 : "=m" (*mem) \
497 : IBR_CONSTRAINT (mask), "m" (*mem), \
498 "i" (offsetof (tcbhead_t, multiple_threads))); \
499 else if (sizeof (*mem) == 2) \
500 __asm __volatile (lock "andw %w1, %0" \
501 : "=m" (*mem) \
502 : "ir" (mask), "m" (*mem), \
503 "i" (offsetof (tcbhead_t, multiple_threads))); \
504 else if (sizeof (*mem) == 4) \
505 __asm __volatile (lock "andl %1, %0" \
506 : "=m" (*mem) \
507 : "ir" (mask), "m" (*mem), \
508 "i" (offsetof (tcbhead_t, multiple_threads))); \
509 else if (__HAVE_64B_ATOMICS) \
510 __asm __volatile (lock "andq %q1, %0" \
511 : "=m" (*mem) \
512 : "ir" (mask), "m" (*mem), \
513 "i" (offsetof (tcbhead_t, multiple_threads))); \
514 else \
515 __atomic_link_error (); \
516 } while (0)
517
518#define __arch_cprefix \
519 "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t"
520
521#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask)
522
523#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask)
524
525
526#define __arch_or_body(lock, mem, mask) \
527 do { \
528 if (sizeof (*mem) == 1) \
529 __asm __volatile (lock "orb %b1, %0" \
530 : "=m" (*mem) \
531 : IBR_CONSTRAINT (mask), "m" (*mem), \
532 "i" (offsetof (tcbhead_t, multiple_threads))); \
533 else if (sizeof (*mem) == 2) \
534 __asm __volatile (lock "orw %w1, %0" \
535 : "=m" (*mem) \
536 : "ir" (mask), "m" (*mem), \
537 "i" (offsetof (tcbhead_t, multiple_threads))); \
538 else if (sizeof (*mem) == 4) \
539 __asm __volatile (lock "orl %1, %0" \
540 : "=m" (*mem) \
541 : "ir" (mask), "m" (*mem), \
542 "i" (offsetof (tcbhead_t, multiple_threads))); \
543 else if (__HAVE_64B_ATOMICS) \
544 __asm __volatile (lock "orq %q1, %0" \
545 : "=m" (*mem) \
546 : "ir" (mask), "m" (*mem), \
547 "i" (offsetof (tcbhead_t, multiple_threads))); \
548 else \
549 __atomic_link_error (); \
550 } while (0)
551
552#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask)
553
554#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask)
555
556/* We don't use mfence because it is supposedly slower due to having to
557 provide stronger guarantees (e.g., regarding self-modifying code). */
558#define atomic_full_barrier() \
559 __asm __volatile (LOCK_PREFIX "orl $0, (%%" SP_REG ")" ::: "memory")
560#define atomic_read_barrier() __asm ("" ::: "memory")
561#define atomic_write_barrier() __asm ("" ::: "memory")
562
563#define atomic_spin_nop() __asm ("pause")
564
565#endif /* atomic-machine.h */
566