1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include "pthreadP.h"
23#include <lowlevellock.h>
24#include <stap-probe.h>
25
26#ifndef lll_unlock_elision
27#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
28#endif
29
30static int
31internal_function
32__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
33 __attribute_noinline__;
34
35int
36internal_function attribute_hidden
37__pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
38{
39 int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
40 if (__builtin_expect (type &
41 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
42 return __pthread_mutex_unlock_full (mutex, decr);
43
44 if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
45 == PTHREAD_MUTEX_TIMED_NP)
46 {
47 /* Always reset the owner field. */
48 normal:
49 mutex->__data.__owner = 0;
50 if (decr)
51 /* One less user. */
52 --mutex->__data.__nusers;
53
54 /* Unlock. */
55 lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
56
57 LIBC_PROBE (mutex_release, 1, mutex);
58
59 return 0;
60 }
61 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
62 {
63 /* Don't reset the owner/users fields for elision. */
64 return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
65 PTHREAD_MUTEX_PSHARED (mutex));
66 }
67 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
68 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
69 {
70 /* Recursive mutex. */
71 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
72 return EPERM;
73
74 if (--mutex->__data.__count != 0)
75 /* We still hold the mutex. */
76 return 0;
77 goto normal;
78 }
79 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
80 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
81 goto normal;
82 else
83 {
84 /* Error checking mutex. */
85 assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
86 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
87 || ! lll_islocked (mutex->__data.__lock))
88 return EPERM;
89 goto normal;
90 }
91}
92
93
94static int
95internal_function
96__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
97{
98 int newowner = 0;
99
100 switch (PTHREAD_MUTEX_TYPE (mutex))
101 {
102 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
103 /* Recursive mutex. */
104 if ((mutex->__data.__lock & FUTEX_TID_MASK)
105 == THREAD_GETMEM (THREAD_SELF, tid)
106 && __builtin_expect (mutex->__data.__owner
107 == PTHREAD_MUTEX_INCONSISTENT, 0))
108 {
109 if (--mutex->__data.__count != 0)
110 /* We still hold the mutex. */
111 return ENOTRECOVERABLE;
112
113 goto notrecoverable;
114 }
115
116 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
117 return EPERM;
118
119 if (--mutex->__data.__count != 0)
120 /* We still hold the mutex. */
121 return 0;
122
123 goto robust;
124
125 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
126 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
127 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
128 if ((mutex->__data.__lock & FUTEX_TID_MASK)
129 != THREAD_GETMEM (THREAD_SELF, tid)
130 || ! lll_islocked (mutex->__data.__lock))
131 return EPERM;
132
133 /* If the previous owner died and the caller did not succeed in
134 making the state consistent, mark the mutex as unrecoverable
135 and make all waiters. */
136 if (__builtin_expect (mutex->__data.__owner
137 == PTHREAD_MUTEX_INCONSISTENT, 0))
138 notrecoverable:
139 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
140
141 robust:
142 /* Remove mutex from the list. */
143 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
144 &mutex->__data.__list.__next);
145 DEQUEUE_MUTEX (mutex);
146
147 mutex->__data.__owner = newowner;
148 if (decr)
149 /* One less user. */
150 --mutex->__data.__nusers;
151
152 /* Unlock. */
153 lll_robust_unlock (mutex->__data.__lock,
154 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
155
156 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
157 break;
158
159 /* The PI support requires the Linux futex system call. If that's not
160 available, pthread_mutex_init should never have allowed the type to
161 be set. So it will get the default case for an invalid type. */
162#ifdef __NR_futex
163 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
164 /* Recursive mutex. */
165 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
166 return EPERM;
167
168 if (--mutex->__data.__count != 0)
169 /* We still hold the mutex. */
170 return 0;
171 goto continue_pi_non_robust;
172
173 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
174 /* Recursive mutex. */
175 if ((mutex->__data.__lock & FUTEX_TID_MASK)
176 == THREAD_GETMEM (THREAD_SELF, tid)
177 && __builtin_expect (mutex->__data.__owner
178 == PTHREAD_MUTEX_INCONSISTENT, 0))
179 {
180 if (--mutex->__data.__count != 0)
181 /* We still hold the mutex. */
182 return ENOTRECOVERABLE;
183
184 goto pi_notrecoverable;
185 }
186
187 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
188 return EPERM;
189
190 if (--mutex->__data.__count != 0)
191 /* We still hold the mutex. */
192 return 0;
193
194 goto continue_pi_robust;
195
196 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
197 case PTHREAD_MUTEX_PI_NORMAL_NP:
198 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
199 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
200 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
201 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
202 if ((mutex->__data.__lock & FUTEX_TID_MASK)
203 != THREAD_GETMEM (THREAD_SELF, tid)
204 || ! lll_islocked (mutex->__data.__lock))
205 return EPERM;
206
207 /* If the previous owner died and the caller did not succeed in
208 making the state consistent, mark the mutex as unrecoverable
209 and make all waiters. */
210 if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
211 && __builtin_expect (mutex->__data.__owner
212 == PTHREAD_MUTEX_INCONSISTENT, 0))
213 pi_notrecoverable:
214 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
215
216 if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
217 {
218 continue_pi_robust:
219 /* Remove mutex from the list.
220 Note: robust PI futexes are signaled by setting bit 0. */
221 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
222 (void *) (((uintptr_t) &mutex->__data.__list.__next)
223 | 1));
224 DEQUEUE_MUTEX (mutex);
225 }
226
227 continue_pi_non_robust:
228 mutex->__data.__owner = newowner;
229 if (decr)
230 /* One less user. */
231 --mutex->__data.__nusers;
232
233 /* Unlock. Load all necessary mutex data before releasing the mutex
234 to not violate the mutex destruction requirements (see
235 lll_unlock). */
236 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
237 int private = (robust
238 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
239 : PTHREAD_MUTEX_PSHARED (mutex));
240 if ((mutex->__data.__lock & FUTEX_WAITERS) != 0
241 || atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock, 0,
242 THREAD_GETMEM (THREAD_SELF,
243 tid)))
244 {
245 INTERNAL_SYSCALL_DECL (__err);
246 INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
247 __lll_private_flag (FUTEX_UNLOCK_PI, private));
248 }
249
250 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
251 break;
252#endif /* __NR_futex. */
253
254 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
255 /* Recursive mutex. */
256 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
257 return EPERM;
258
259 if (--mutex->__data.__count != 0)
260 /* We still hold the mutex. */
261 return 0;
262 goto pp;
263
264 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
265 /* Error checking mutex. */
266 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
267 || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
268 return EPERM;
269 /* FALLTHROUGH */
270
271 case PTHREAD_MUTEX_PP_NORMAL_NP:
272 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
273 /* Always reset the owner field. */
274 pp:
275 mutex->__data.__owner = 0;
276
277 if (decr)
278 /* One less user. */
279 --mutex->__data.__nusers;
280
281 /* Unlock. */
282 int newval, oldval;
283 do
284 {
285 oldval = mutex->__data.__lock;
286 newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
287 }
288 while (atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock,
289 newval, oldval));
290
291 if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
292 lll_futex_wake (&mutex->__data.__lock, 1,
293 PTHREAD_MUTEX_PSHARED (mutex));
294
295 int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
296
297 LIBC_PROBE (mutex_release, 1, mutex);
298
299 return __pthread_tpp_change_priority (oldprio, -1);
300
301 default:
302 /* Correct code cannot set any other type. */
303 return EINVAL;
304 }
305
306 LIBC_PROBE (mutex_release, 1, mutex);
307 return 0;
308}
309
310
311int
312__pthread_mutex_unlock (pthread_mutex_t *mutex)
313{
314 return __pthread_mutex_unlock_usercnt (mutex, 1);
315}
316strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
317hidden_def (__pthread_mutex_unlock)
318