1/*
2 * Copyright (c) 2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _MACHINE_ATOMIC_H
30#define _MACHINE_ATOMIC_H
31
32#include <stdatomic.h>
33
34#define _os_atomic_c11_atomic(p) \
35 ((typeof(*(p)) _Atomic *)(p))
36
37#define _os_atomic_basetypeof(p) \
38 typeof(atomic_load(((typeof(*(p)) _Atomic *)(p))))
39
40#define _os_atomic_c11_op_orig(p, v, m, o) \
41 atomic_##o##_explicit(_os_atomic_c11_atomic(p), v, \
42 memory_order_##m)
43
44#define _os_atomic_c11_op(p, v, m, o, op) \
45 ({ typeof(v) _v = (v); _os_atomic_c11_op_orig(p, v, m, o) op _v; })
46
47#define os_atomic_thread_fence(m) atomic_thread_fence(memory_order_##m)
48
49#define os_atomic_load(p, m) \
50 atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m)
51#define os_atomic_store(p, v, m) _os_atomic_c11_op_orig(p, v, m, store)
52
53#define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add)
54#define os_atomic_add(p, v, m) _os_atomic_c11_op(p, v, m, fetch_add, +)
55
56#define os_atomic_inc_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_add)
57#define os_atomic_inc(p, m) _os_atomic_c11_op(p, 1, m, fetch_add, +)
58
59#define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub)
60#define os_atomic_sub(p, v, m) _os_atomic_c11_op(p, v, m, fetch_sub, -)
61
62#define os_atomic_dec_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_sub)
63#define os_atomic_dec(p, m) _os_atomic_c11_op(p, 1, m, fetch_sub, -)
64
65#define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and)
66#define os_atomic_and(p, v, m) _os_atomic_c11_op(p, v, m, fetch_and, &)
67
68#define os_atomic_or_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_or)
69#define os_atomic_or(p, v, m) _os_atomic_c11_op(p, v, m, fetch_or, |)
70
71#define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor)
72#define os_atomic_xor(p, v, m) _os_atomic_c11_op(p, v, m, fetch_xor, ^)
73
74#define os_atomic_xchg(p, v, m) _os_atomic_c11_op_orig(p, v, m, exchange)
75
76#define os_atomic_cmpxchg(p, e, v, m) \
77 ({ _os_atomic_basetypeof(p) _r = (e); \
78 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
79 &_r, v, memory_order_##m, memory_order_relaxed); })
80#define os_atomic_cmpxchgv(p, e, v, g, m) \
81 ({ _os_atomic_basetypeof(p) _r = (e); int _b = \
82 atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
83 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
84#define os_atomic_cmpxchgvw(p, e, v, g, m) \
85 ({ _os_atomic_basetypeof(p) _r = (e); int _b = \
86 atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \
87 &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; })
88
89#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
90 bool _result = false; \
91 typeof(p) _p = (p); \
92 ov = os_atomic_load(_p, relaxed); \
93 do { \
94 __VA_ARGS__; \
95 _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \
96 } while (!_result); \
97 _result; \
98 })
99
100#define os_atomic_rmw_loop_give_up_with_fence(m, expr) \
101 ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); })
102#define os_atomic_rmw_loop_give_up(expr) \
103 os_atomic_rmw_loop_give_up_with_fence(relaxed, expr)
104
105#define os_atomic_force_dependency_on(p, e) (p)
106#define os_atomic_load_with_dependency_on(p, e) \
107 os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed)
108
109#if defined (__x86_64__)
110#include "i386/atomic.h"
111#elif defined (__arm__) || defined (__arm64__)
112#include "arm/atomic.h"
113#else
114#error architecture not supported
115#endif
116
117#endif /* _MACHINE_ATOMIC_H */
118