1/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include <unistd.h>
23#include <sys/param.h>
24#include <not-cancel.h>
25#include "pthreadP.h"
26#include <atomic.h>
27#include <lowlevellock.h>
28#include <stap-probe.h>
29
30#ifndef lll_lock_elision
31#define lll_lock_elision(lock, try_lock, private) ({ \
32 lll_lock (lock, private); 0; })
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39/* Some of the following definitions differ when pthread_mutex_cond_lock.c
40 includes this file. */
41#ifndef LLL_MUTEX_LOCK
42# define LLL_MUTEX_LOCK(mutex) \
43 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
44# define LLL_MUTEX_TRYLOCK(mutex) \
45 lll_trylock ((mutex)->__data.__lock)
46# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
47# define LLL_MUTEX_LOCK_ELISION(mutex) \
48 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
49 PTHREAD_MUTEX_PSHARED (mutex))
50# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
51 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
52 PTHREAD_MUTEX_PSHARED (mutex))
53#endif
54
55#ifndef FORCE_ELISION
56#define FORCE_ELISION(m, s)
57#endif
58
59static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
60 __attribute_noinline__;
61
62int
63__pthread_mutex_lock (pthread_mutex_t *mutex)
64{
65 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
66
67 LIBC_PROBE (mutex_entry, 1, mutex);
68
69 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
70 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
71 return __pthread_mutex_lock_full (mutex);
72
73 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
74 {
75 FORCE_ELISION (mutex, goto elision);
76 simple:
77 /* Normal mutex. */
78 LLL_MUTEX_LOCK (mutex);
79 assert (mutex->__data.__owner == 0);
80 }
81#ifdef HAVE_ELISION
82 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
83 {
84 elision: __attribute__((unused))
85 /* This case can never happen on a system without elision,
86 as the mutex type initialization functions will not
87 allow to set the elision flags. */
88 /* Don't record owner or users for elision case. This is a
89 tail call. */
90 return LLL_MUTEX_LOCK_ELISION (mutex);
91 }
92#endif
93 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
94 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
95 {
96 /* Recursive mutex. */
97 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
98
99 /* Check whether we already hold the mutex. */
100 if (mutex->__data.__owner == id)
101 {
102 /* Just bump the counter. */
103 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
104 /* Overflow of the counter. */
105 return EAGAIN;
106
107 ++mutex->__data.__count;
108
109 return 0;
110 }
111
112 /* We have to get the mutex. */
113 LLL_MUTEX_LOCK (mutex);
114
115 assert (mutex->__data.__owner == 0);
116 mutex->__data.__count = 1;
117 }
118 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
119 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
120 {
121 if (! __is_smp)
122 goto simple;
123
124 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
125 {
126 int cnt = 0;
127 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
128 mutex->__data.__spins * 2 + 10);
129 do
130 {
131 if (cnt++ >= max_cnt)
132 {
133 LLL_MUTEX_LOCK (mutex);
134 break;
135 }
136 atomic_spin_nop ();
137 }
138 while (LLL_MUTEX_TRYLOCK (mutex) != 0);
139
140 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
141 }
142 assert (mutex->__data.__owner == 0);
143 }
144 else
145 {
146 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
147 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
148 /* Check whether we already hold the mutex. */
149 if (__glibc_unlikely (mutex->__data.__owner == id))
150 return EDEADLK;
151 goto simple;
152 }
153
154 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
155
156 /* Record the ownership. */
157 mutex->__data.__owner = id;
158#ifndef NO_INCR
159 ++mutex->__data.__nusers;
160#endif
161
162 LIBC_PROBE (mutex_acquired, 1, mutex);
163
164 return 0;
165}
166
167static int
168__pthread_mutex_lock_full (pthread_mutex_t *mutex)
169{
170 int oldval;
171 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
172
173 switch (PTHREAD_MUTEX_TYPE (mutex))
174 {
175 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
176 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
177 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
178 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
179 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
180 &mutex->__data.__list.__next);
181 /* We need to set op_pending before starting the operation. Also
182 see comments at ENQUEUE_MUTEX. */
183 __asm ("" ::: "memory");
184
185 oldval = mutex->__data.__lock;
186 /* This is set to FUTEX_WAITERS iff we might have shared the
187 FUTEX_WAITERS flag with other threads, and therefore need to keep it
188 set to avoid lost wake-ups. We have the same requirement in the
189 simple mutex algorithm.
190 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
191 are building the special case mutexes for use from within condition
192 variables. */
193 unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
194 while (1)
195 {
196 /* Try to acquire the lock through a CAS from 0 (not acquired) to
197 our TID | assume_other_futex_waiters. */
198 if (__glibc_likely (oldval == 0))
199 {
200 oldval
201 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
202 id | assume_other_futex_waiters, 0);
203 if (__glibc_likely (oldval == 0))
204 break;
205 }
206
207 if ((oldval & FUTEX_OWNER_DIED) != 0)
208 {
209 /* The previous owner died. Try locking the mutex. */
210 int newval = id;
211#ifdef NO_INCR
212 /* We are not taking assume_other_futex_waiters into accoount
213 here simply because we'll set FUTEX_WAITERS anyway. */
214 newval |= FUTEX_WAITERS;
215#else
216 newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
217#endif
218
219 newval
220 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
221 newval, oldval);
222
223 if (newval != oldval)
224 {
225 oldval = newval;
226 continue;
227 }
228
229 /* We got the mutex. */
230 mutex->__data.__count = 1;
231 /* But it is inconsistent unless marked otherwise. */
232 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
233
234 /* We must not enqueue the mutex before we have acquired it.
235 Also see comments at ENQUEUE_MUTEX. */
236 __asm ("" ::: "memory");
237 ENQUEUE_MUTEX (mutex);
238 /* We need to clear op_pending after we enqueue the mutex. */
239 __asm ("" ::: "memory");
240 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
241
242 /* Note that we deliberately exit here. If we fall
243 through to the end of the function __nusers would be
244 incremented which is not correct because the old
245 owner has to be discounted. If we are not supposed
246 to increment __nusers we actually have to decrement
247 it here. */
248#ifdef NO_INCR
249 --mutex->__data.__nusers;
250#endif
251
252 return EOWNERDEAD;
253 }
254
255 /* Check whether we already hold the mutex. */
256 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
257 {
258 int kind = PTHREAD_MUTEX_TYPE (mutex);
259 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
260 {
261 /* We do not need to ensure ordering wrt another memory
262 access. Also see comments at ENQUEUE_MUTEX. */
263 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
264 NULL);
265 return EDEADLK;
266 }
267
268 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
269 {
270 /* We do not need to ensure ordering wrt another memory
271 access. */
272 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
273 NULL);
274
275 /* Just bump the counter. */
276 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
277 /* Overflow of the counter. */
278 return EAGAIN;
279
280 ++mutex->__data.__count;
281
282 return 0;
283 }
284 }
285
286 /* We cannot acquire the mutex nor has its owner died. Thus, try
287 to block using futexes. Set FUTEX_WAITERS if necessary so that
288 other threads are aware that there are potentially threads
289 blocked on the futex. Restart if oldval changed in the
290 meantime. */
291 if ((oldval & FUTEX_WAITERS) == 0)
292 {
293 if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
294 oldval | FUTEX_WAITERS,
295 oldval)
296 != 0)
297 {
298 oldval = mutex->__data.__lock;
299 continue;
300 }
301 oldval |= FUTEX_WAITERS;
302 }
303
304 /* It is now possible that we share the FUTEX_WAITERS flag with
305 another thread; therefore, update assume_other_futex_waiters so
306 that we do not forget about this when handling other cases
307 above and thus do not cause lost wake-ups. */
308 assume_other_futex_waiters |= FUTEX_WAITERS;
309
310 /* Block using the futex and reload current lock value. */
311 lll_futex_wait (&mutex->__data.__lock, oldval,
312 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
313 oldval = mutex->__data.__lock;
314 }
315
316 /* We have acquired the mutex; check if it is still consistent. */
317 if (__builtin_expect (mutex->__data.__owner
318 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
319 {
320 /* This mutex is now not recoverable. */
321 mutex->__data.__count = 0;
322 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
323 lll_unlock (mutex->__data.__lock, private);
324 /* FIXME This violates the mutex destruction requirements. See
325 __pthread_mutex_unlock_full. */
326 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
327 return ENOTRECOVERABLE;
328 }
329
330 mutex->__data.__count = 1;
331 /* We must not enqueue the mutex before we have acquired it.
332 Also see comments at ENQUEUE_MUTEX. */
333 __asm ("" ::: "memory");
334 ENQUEUE_MUTEX (mutex);
335 /* We need to clear op_pending after we enqueue the mutex. */
336 __asm ("" ::: "memory");
337 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
338 break;
339
340 /* The PI support requires the Linux futex system call. If that's not
341 available, pthread_mutex_init should never have allowed the type to
342 be set. So it will get the default case for an invalid type. */
343#ifdef __NR_futex
344 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
345 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
346 case PTHREAD_MUTEX_PI_NORMAL_NP:
347 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
348 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
349 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
350 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
351 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
352 {
353 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
354 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
355
356 if (robust)
357 {
358 /* Note: robust PI futexes are signaled by setting bit 0. */
359 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
360 (void *) (((uintptr_t) &mutex->__data.__list.__next)
361 | 1));
362 /* We need to set op_pending before starting the operation. Also
363 see comments at ENQUEUE_MUTEX. */
364 __asm ("" ::: "memory");
365 }
366
367 oldval = mutex->__data.__lock;
368
369 /* Check whether we already hold the mutex. */
370 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
371 {
372 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
373 {
374 /* We do not need to ensure ordering wrt another memory
375 access. */
376 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
377 return EDEADLK;
378 }
379
380 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
381 {
382 /* We do not need to ensure ordering wrt another memory
383 access. */
384 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
385
386 /* Just bump the counter. */
387 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
388 /* Overflow of the counter. */
389 return EAGAIN;
390
391 ++mutex->__data.__count;
392
393 return 0;
394 }
395 }
396
397 int newval = id;
398# ifdef NO_INCR
399 newval |= FUTEX_WAITERS;
400# endif
401 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
402 newval, 0);
403
404 if (oldval != 0)
405 {
406 /* The mutex is locked. The kernel will now take care of
407 everything. */
408 int private = (robust
409 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
410 : PTHREAD_MUTEX_PSHARED (mutex));
411 INTERNAL_SYSCALL_DECL (__err);
412 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
413 __lll_private_flag (FUTEX_LOCK_PI,
414 private), 1, 0);
415
416 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
417 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
418 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
419 {
420 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
421 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
422 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
423 /* ESRCH can happen only for non-robust PI mutexes where
424 the owner of the lock died. */
425 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
426
427 /* Delay the thread indefinitely. */
428 while (1)
429 __pause_nocancel ();
430 }
431
432 oldval = mutex->__data.__lock;
433
434 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
435 }
436
437 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
438 {
439 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
440
441 /* We got the mutex. */
442 mutex->__data.__count = 1;
443 /* But it is inconsistent unless marked otherwise. */
444 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
445
446 /* We must not enqueue the mutex before we have acquired it.
447 Also see comments at ENQUEUE_MUTEX. */
448 __asm ("" ::: "memory");
449 ENQUEUE_MUTEX_PI (mutex);
450 /* We need to clear op_pending after we enqueue the mutex. */
451 __asm ("" ::: "memory");
452 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
453
454 /* Note that we deliberately exit here. If we fall
455 through to the end of the function __nusers would be
456 incremented which is not correct because the old owner
457 has to be discounted. If we are not supposed to
458 increment __nusers we actually have to decrement it here. */
459# ifdef NO_INCR
460 --mutex->__data.__nusers;
461# endif
462
463 return EOWNERDEAD;
464 }
465
466 if (robust
467 && __builtin_expect (mutex->__data.__owner
468 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
469 {
470 /* This mutex is now not recoverable. */
471 mutex->__data.__count = 0;
472
473 INTERNAL_SYSCALL_DECL (__err);
474 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
475 __lll_private_flag (FUTEX_UNLOCK_PI,
476 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
477 0, 0);
478
479 /* To the kernel, this will be visible after the kernel has
480 acquired the mutex in the syscall. */
481 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
482 return ENOTRECOVERABLE;
483 }
484
485 mutex->__data.__count = 1;
486 if (robust)
487 {
488 /* We must not enqueue the mutex before we have acquired it.
489 Also see comments at ENQUEUE_MUTEX. */
490 __asm ("" ::: "memory");
491 ENQUEUE_MUTEX_PI (mutex);
492 /* We need to clear op_pending after we enqueue the mutex. */
493 __asm ("" ::: "memory");
494 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
495 }
496 }
497 break;
498#endif /* __NR_futex. */
499
500 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
501 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
502 case PTHREAD_MUTEX_PP_NORMAL_NP:
503 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
504 {
505 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
506
507 oldval = mutex->__data.__lock;
508
509 /* Check whether we already hold the mutex. */
510 if (mutex->__data.__owner == id)
511 {
512 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
513 return EDEADLK;
514
515 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
516 {
517 /* Just bump the counter. */
518 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
519 /* Overflow of the counter. */
520 return EAGAIN;
521
522 ++mutex->__data.__count;
523
524 return 0;
525 }
526 }
527
528 int oldprio = -1, ceilval;
529 do
530 {
531 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
532 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
533
534 if (__pthread_current_priority () > ceiling)
535 {
536 if (oldprio != -1)
537 __pthread_tpp_change_priority (oldprio, -1);
538 return EINVAL;
539 }
540
541 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
542 if (retval)
543 return retval;
544
545 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
546 oldprio = ceiling;
547
548 oldval
549 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
550#ifdef NO_INCR
551 ceilval | 2,
552#else
553 ceilval | 1,
554#endif
555 ceilval);
556
557 if (oldval == ceilval)
558 break;
559
560 do
561 {
562 oldval
563 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
564 ceilval | 2,
565 ceilval | 1);
566
567 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
568 break;
569
570 if (oldval != ceilval)
571 lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
572 PTHREAD_MUTEX_PSHARED (mutex));
573 }
574 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
575 ceilval | 2, ceilval)
576 != ceilval);
577 }
578 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
579
580 assert (mutex->__data.__owner == 0);
581 mutex->__data.__count = 1;
582 }
583 break;
584
585 default:
586 /* Correct code cannot set any other type. */
587 return EINVAL;
588 }
589
590 /* Record the ownership. */
591 mutex->__data.__owner = id;
592#ifndef NO_INCR
593 ++mutex->__data.__nusers;
594#endif
595
596 LIBC_PROBE (mutex_acquired, 1, mutex);
597
598 return 0;
599}
600#ifndef __pthread_mutex_lock
601weak_alias (__pthread_mutex_lock, pthread_mutex_lock)
602hidden_def (__pthread_mutex_lock)
603#endif
604
605
606#ifdef NO_INCR
607void
608__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
609{
610 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
611 assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
612 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
613
614 /* Record the ownership. */
615 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
616 mutex->__data.__owner = id;
617
618 if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
619 ++mutex->__data.__count;
620}
621#endif
622