1/*
2 * Copyright (c) 2000-2015 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <libkern/OSAtomic.h>
30#include <kern/debug.h>
31#include <machine/atomic.h>
32
33enum {
34 false = 0,
35 true = 1
36};
37
38#ifndef NULL
39#define NULL ((void *)0)
40#endif
41
42#define ATOMIC_DEBUG DEBUG
43
44#if ATOMIC_DEBUG
45#define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
46#else
47#define ALIGN_TEST(p,t) do{}while(0)
48#endif
49
50// 19831745 - start of big hammer!
51#pragma clang diagnostic push
52#pragma clang diagnostic ignored "-Wcast-qual"
53
54/*
55 * atomic operations
56 * These are _the_ atomic operations, now implemented via compiler built-ins.
57 * It is expected that this C implementation is a candidate for Link-Time-
58 * Optimization inlining, whereas the assembler implementations they replace
59 * were not.
60 */
61
62#undef OSCompareAndSwap8
63Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
64{
65 return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
66 memory_order_acq_rel_smp, memory_order_relaxed);
67}
68
69#undef OSCompareAndSwap16
70Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
71{
72 return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
73 memory_order_acq_rel_smp, memory_order_relaxed);
74}
75
76#undef OSCompareAndSwap
77Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
78{
79 ALIGN_TEST(address, UInt32);
80 return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
81 memory_order_acq_rel_smp, memory_order_relaxed);
82}
83
84#undef OSCompareAndSwap64
85Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
86{
87 /*
88 * _Atomic uint64 requires 8-byte alignment on all architectures.
89 * This silences the compiler cast warning. ALIGN_TEST() verifies
90 * that the cast was legal, if defined.
91 */
92 _Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
93
94 ALIGN_TEST(address, UInt64);
95 return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
96 memory_order_acq_rel_smp, memory_order_relaxed);
97}
98
99#undef OSCompareAndSwapPtr
100Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
101{
102#if __LP64__
103 return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
104#else
105 return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
106#endif
107}
108
109SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
110{
111 return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
112}
113
114SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
115{
116 return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
117}
118
119#undef OSAddAtomic
120SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
121{
122 ALIGN_TEST(address, UInt32);
123 return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
124}
125
126#undef OSAddAtomic64
127SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
128{
129 _Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
130
131 ALIGN_TEST(address, SInt64);
132 return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
133}
134
135#undef OSAddAtomicLong
136long
137OSAddAtomicLong(long theAmount, volatile long *address)
138{
139#ifdef __LP64__
140 return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
141#else
142 return (long)OSAddAtomic((SInt32)theAmount, address);
143#endif
144}
145
146#undef OSIncrementAtomic
147SInt32 OSIncrementAtomic(volatile SInt32 * value)
148{
149 return OSAddAtomic(1, value);
150}
151
152#undef OSDecrementAtomic
153SInt32 OSDecrementAtomic(volatile SInt32 * value)
154{
155 return OSAddAtomic(-1, value);
156}
157
158#undef OSBitAndAtomic
159UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
160{
161 return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
162}
163
164#undef OSBitOrAtomic
165UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
166{
167 return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
168}
169
170#undef OSBitXorAtomic
171UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
172{
173 return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
174}
175
176static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
177{
178 UInt8 mask = 1;
179 UInt8 oldValue;
180 UInt8 wantValue;
181
182 startAddress += (bit / 8);
183 mask <<= (7 - (bit % 8));
184 wantValue = wantSet ? mask : 0;
185
186 do {
187 oldValue = *startAddress;
188 if ((oldValue & mask) == wantValue) {
189 break;
190 }
191 } while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
192 &oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
193
194 return (oldValue & mask) == wantValue;
195}
196
197Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
198{
199 return OSTestAndSetClear(bit, true, startAddress);
200}
201
202Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
203{
204 return OSTestAndSetClear(bit, false, startAddress);
205}
206
207/*
208 * silly unaligned versions
209 */
210
211SInt8 OSIncrementAtomic8(volatile SInt8 * value)
212{
213 return OSAddAtomic8(1, value);
214}
215
216SInt8 OSDecrementAtomic8(volatile SInt8 * value)
217{
218 return OSAddAtomic8(-1, value);
219}
220
221UInt8 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
222{
223 return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
224}
225
226UInt8 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
227{
228 return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
229}
230
231UInt8 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
232{
233 return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
234}
235
236SInt16 OSIncrementAtomic16(volatile SInt16 * value)
237{
238 return OSAddAtomic16(1, value);
239}
240
241SInt16 OSDecrementAtomic16(volatile SInt16 * value)
242{
243 return OSAddAtomic16(-1, value);
244}
245
246UInt16 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
247{
248 return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
249}
250
251UInt16 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
252{
253 return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
254}
255
256UInt16 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
257{
258 return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
259}
260
261// 19831745 - end of big hammer!
262#pragma clang diagnostic pop
263
264