1/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include <unistd.h>
23#include <sys/param.h>
24#include <not-cancel.h>
25#include "pthreadP.h"
26#include <atomic.h>
27#include <lowlevellock.h>
28#include <stap-probe.h>
29
30#ifndef lll_lock_elision
31#define lll_lock_elision(lock, try_lock, private) ({ \
32 lll_lock (lock, private); 0; })
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39/* Some of the following definitions differ when pthread_mutex_cond_lock.c
40 includes this file. */
41#ifndef LLL_MUTEX_LOCK
42# define LLL_MUTEX_LOCK(mutex) \
43 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
44# define LLL_MUTEX_TRYLOCK(mutex) \
45 lll_trylock ((mutex)->__data.__lock)
46# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
47# define LLL_MUTEX_LOCK_ELISION(mutex) \
48 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
49 PTHREAD_MUTEX_PSHARED (mutex))
50# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
51 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
52 PTHREAD_MUTEX_PSHARED (mutex))
53#endif
54
55#ifndef FORCE_ELISION
56#define FORCE_ELISION(m, s)
57#endif
58
59static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
60 __attribute_noinline__;
61
62int
63__pthread_mutex_lock (pthread_mutex_t *mutex)
64{
65 assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
66
67 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
68
69 LIBC_PROBE (mutex_entry, 1, mutex);
70
71 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
72 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
73 return __pthread_mutex_lock_full (mutex);
74
75 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
76 {
77 FORCE_ELISION (mutex, goto elision);
78 simple:
79 /* Normal mutex. */
80 LLL_MUTEX_LOCK (mutex);
81 assert (mutex->__data.__owner == 0);
82 }
83#ifdef HAVE_ELISION
84 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
85 {
86 elision: __attribute__((unused))
87 /* This case can never happen on a system without elision,
88 as the mutex type initialization functions will not
89 allow to set the elision flags. */
90 /* Don't record owner or users for elision case. This is a
91 tail call. */
92 return LLL_MUTEX_LOCK_ELISION (mutex);
93 }
94#endif
95 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
96 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
97 {
98 /* Recursive mutex. */
99 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
100
101 /* Check whether we already hold the mutex. */
102 if (mutex->__data.__owner == id)
103 {
104 /* Just bump the counter. */
105 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
106 /* Overflow of the counter. */
107 return EAGAIN;
108
109 ++mutex->__data.__count;
110
111 return 0;
112 }
113
114 /* We have to get the mutex. */
115 LLL_MUTEX_LOCK (mutex);
116
117 assert (mutex->__data.__owner == 0);
118 mutex->__data.__count = 1;
119 }
120 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
121 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
122 {
123 if (! __is_smp)
124 goto simple;
125
126 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
127 {
128 int cnt = 0;
129 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
130 mutex->__data.__spins * 2 + 10);
131 do
132 {
133 if (cnt++ >= max_cnt)
134 {
135 LLL_MUTEX_LOCK (mutex);
136 break;
137 }
138 atomic_spin_nop ();
139 }
140 while (LLL_MUTEX_TRYLOCK (mutex) != 0);
141
142 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
143 }
144 assert (mutex->__data.__owner == 0);
145 }
146 else
147 {
148 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
149 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
150 /* Check whether we already hold the mutex. */
151 if (__glibc_unlikely (mutex->__data.__owner == id))
152 return EDEADLK;
153 goto simple;
154 }
155
156 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
157
158 /* Record the ownership. */
159 mutex->__data.__owner = id;
160#ifndef NO_INCR
161 ++mutex->__data.__nusers;
162#endif
163
164 LIBC_PROBE (mutex_acquired, 1, mutex);
165
166 return 0;
167}
168
169static int
170__pthread_mutex_lock_full (pthread_mutex_t *mutex)
171{
172 int oldval;
173 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
174
175 switch (PTHREAD_MUTEX_TYPE (mutex))
176 {
177 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
178 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
179 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
180 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
181 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
182 &mutex->__data.__list.__next);
183 /* We need to set op_pending before starting the operation. Also
184 see comments at ENQUEUE_MUTEX. */
185 __asm ("" ::: "memory");
186
187 oldval = mutex->__data.__lock;
188 /* This is set to FUTEX_WAITERS iff we might have shared the
189 FUTEX_WAITERS flag with other threads, and therefore need to keep it
190 set to avoid lost wake-ups. We have the same requirement in the
191 simple mutex algorithm.
192 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
193 are building the special case mutexes for use from within condition
194 variables. */
195 unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
196 while (1)
197 {
198 /* Try to acquire the lock through a CAS from 0 (not acquired) to
199 our TID | assume_other_futex_waiters. */
200 if (__glibc_likely ((oldval == 0)
201 && (atomic_compare_and_exchange_bool_acq
202 (&mutex->__data.__lock,
203 id | assume_other_futex_waiters, 0) == 0)))
204 break;
205
206 if ((oldval & FUTEX_OWNER_DIED) != 0)
207 {
208 /* The previous owner died. Try locking the mutex. */
209 int newval = id;
210#ifdef NO_INCR
211 /* We are not taking assume_other_futex_waiters into accoount
212 here simply because we'll set FUTEX_WAITERS anyway. */
213 newval |= FUTEX_WAITERS;
214#else
215 newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
216#endif
217
218 newval
219 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
220 newval, oldval);
221
222 if (newval != oldval)
223 {
224 oldval = newval;
225 continue;
226 }
227
228 /* We got the mutex. */
229 mutex->__data.__count = 1;
230 /* But it is inconsistent unless marked otherwise. */
231 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
232
233 /* We must not enqueue the mutex before we have acquired it.
234 Also see comments at ENQUEUE_MUTEX. */
235 __asm ("" ::: "memory");
236 ENQUEUE_MUTEX (mutex);
237 /* We need to clear op_pending after we enqueue the mutex. */
238 __asm ("" ::: "memory");
239 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
240
241 /* Note that we deliberately exit here. If we fall
242 through to the end of the function __nusers would be
243 incremented which is not correct because the old
244 owner has to be discounted. If we are not supposed
245 to increment __nusers we actually have to decrement
246 it here. */
247#ifdef NO_INCR
248 --mutex->__data.__nusers;
249#endif
250
251 return EOWNERDEAD;
252 }
253
254 /* Check whether we already hold the mutex. */
255 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
256 {
257 int kind = PTHREAD_MUTEX_TYPE (mutex);
258 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
259 {
260 /* We do not need to ensure ordering wrt another memory
261 access. Also see comments at ENQUEUE_MUTEX. */
262 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
263 NULL);
264 return EDEADLK;
265 }
266
267 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
268 {
269 /* We do not need to ensure ordering wrt another memory
270 access. */
271 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
272 NULL);
273
274 /* Just bump the counter. */
275 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
276 /* Overflow of the counter. */
277 return EAGAIN;
278
279 ++mutex->__data.__count;
280
281 return 0;
282 }
283 }
284
285 /* We cannot acquire the mutex nor has its owner died. Thus, try
286 to block using futexes. Set FUTEX_WAITERS if necessary so that
287 other threads are aware that there are potentially threads
288 blocked on the futex. Restart if oldval changed in the
289 meantime. */
290 if ((oldval & FUTEX_WAITERS) == 0)
291 {
292 if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
293 oldval | FUTEX_WAITERS,
294 oldval)
295 != 0)
296 {
297 oldval = mutex->__data.__lock;
298 continue;
299 }
300 oldval |= FUTEX_WAITERS;
301 }
302
303 /* It is now possible that we share the FUTEX_WAITERS flag with
304 another thread; therefore, update assume_other_futex_waiters so
305 that we do not forget about this when handling other cases
306 above and thus do not cause lost wake-ups. */
307 assume_other_futex_waiters |= FUTEX_WAITERS;
308
309 /* Block using the futex and reload current lock value. */
310 lll_futex_wait (&mutex->__data.__lock, oldval,
311 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
312 oldval = mutex->__data.__lock;
313 }
314
315 /* We have acquired the mutex; check if it is still consistent. */
316 if (__builtin_expect (mutex->__data.__owner
317 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
318 {
319 /* This mutex is now not recoverable. */
320 mutex->__data.__count = 0;
321 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
322 lll_unlock (mutex->__data.__lock, private);
323 /* FIXME This violates the mutex destruction requirements. See
324 __pthread_mutex_unlock_full. */
325 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
326 return ENOTRECOVERABLE;
327 }
328
329 mutex->__data.__count = 1;
330 /* We must not enqueue the mutex before we have acquired it.
331 Also see comments at ENQUEUE_MUTEX. */
332 __asm ("" ::: "memory");
333 ENQUEUE_MUTEX (mutex);
334 /* We need to clear op_pending after we enqueue the mutex. */
335 __asm ("" ::: "memory");
336 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
337 break;
338
339 /* The PI support requires the Linux futex system call. If that's not
340 available, pthread_mutex_init should never have allowed the type to
341 be set. So it will get the default case for an invalid type. */
342#ifdef __NR_futex
343 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
344 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
345 case PTHREAD_MUTEX_PI_NORMAL_NP:
346 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
347 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
348 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
349 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
350 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
351 {
352 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
353 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
354
355 if (robust)
356 {
357 /* Note: robust PI futexes are signaled by setting bit 0. */
358 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
359 (void *) (((uintptr_t) &mutex->__data.__list.__next)
360 | 1));
361 /* We need to set op_pending before starting the operation. Also
362 see comments at ENQUEUE_MUTEX. */
363 __asm ("" ::: "memory");
364 }
365
366 oldval = mutex->__data.__lock;
367
368 /* Check whether we already hold the mutex. */
369 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
370 {
371 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
372 {
373 /* We do not need to ensure ordering wrt another memory
374 access. */
375 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
376 return EDEADLK;
377 }
378
379 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
380 {
381 /* We do not need to ensure ordering wrt another memory
382 access. */
383 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
384
385 /* Just bump the counter. */
386 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
387 /* Overflow of the counter. */
388 return EAGAIN;
389
390 ++mutex->__data.__count;
391
392 return 0;
393 }
394 }
395
396 int newval = id;
397# ifdef NO_INCR
398 newval |= FUTEX_WAITERS;
399# endif
400 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
401 newval, 0);
402
403 if (oldval != 0)
404 {
405 /* The mutex is locked. The kernel will now take care of
406 everything. */
407 int private = (robust
408 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
409 : PTHREAD_MUTEX_PSHARED (mutex));
410 INTERNAL_SYSCALL_DECL (__err);
411 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
412 __lll_private_flag (FUTEX_LOCK_PI,
413 private), 1, 0);
414
415 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
416 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
417 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
418 {
419 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
420 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
421 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
422 /* ESRCH can happen only for non-robust PI mutexes where
423 the owner of the lock died. */
424 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
425
426 /* Delay the thread indefinitely. */
427 while (1)
428 pause_not_cancel ();
429 }
430
431 oldval = mutex->__data.__lock;
432
433 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
434 }
435
436 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
437 {
438 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
439
440 /* We got the mutex. */
441 mutex->__data.__count = 1;
442 /* But it is inconsistent unless marked otherwise. */
443 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
444
445 /* We must not enqueue the mutex before we have acquired it.
446 Also see comments at ENQUEUE_MUTEX. */
447 __asm ("" ::: "memory");
448 ENQUEUE_MUTEX_PI (mutex);
449 /* We need to clear op_pending after we enqueue the mutex. */
450 __asm ("" ::: "memory");
451 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
452
453 /* Note that we deliberately exit here. If we fall
454 through to the end of the function __nusers would be
455 incremented which is not correct because the old owner
456 has to be discounted. If we are not supposed to
457 increment __nusers we actually have to decrement it here. */
458# ifdef NO_INCR
459 --mutex->__data.__nusers;
460# endif
461
462 return EOWNERDEAD;
463 }
464
465 if (robust
466 && __builtin_expect (mutex->__data.__owner
467 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
468 {
469 /* This mutex is now not recoverable. */
470 mutex->__data.__count = 0;
471
472 INTERNAL_SYSCALL_DECL (__err);
473 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
474 __lll_private_flag (FUTEX_UNLOCK_PI,
475 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
476 0, 0);
477
478 /* To the kernel, this will be visible after the kernel has
479 acquired the mutex in the syscall. */
480 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
481 return ENOTRECOVERABLE;
482 }
483
484 mutex->__data.__count = 1;
485 if (robust)
486 {
487 /* We must not enqueue the mutex before we have acquired it.
488 Also see comments at ENQUEUE_MUTEX. */
489 __asm ("" ::: "memory");
490 ENQUEUE_MUTEX_PI (mutex);
491 /* We need to clear op_pending after we enqueue the mutex. */
492 __asm ("" ::: "memory");
493 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
494 }
495 }
496 break;
497#endif /* __NR_futex. */
498
499 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
500 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
501 case PTHREAD_MUTEX_PP_NORMAL_NP:
502 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
503 {
504 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
505
506 oldval = mutex->__data.__lock;
507
508 /* Check whether we already hold the mutex. */
509 if (mutex->__data.__owner == id)
510 {
511 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
512 return EDEADLK;
513
514 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
515 {
516 /* Just bump the counter. */
517 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
518 /* Overflow of the counter. */
519 return EAGAIN;
520
521 ++mutex->__data.__count;
522
523 return 0;
524 }
525 }
526
527 int oldprio = -1, ceilval;
528 do
529 {
530 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
531 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
532
533 if (__pthread_current_priority () > ceiling)
534 {
535 if (oldprio != -1)
536 __pthread_tpp_change_priority (oldprio, -1);
537 return EINVAL;
538 }
539
540 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
541 if (retval)
542 return retval;
543
544 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
545 oldprio = ceiling;
546
547 oldval
548 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
549#ifdef NO_INCR
550 ceilval | 2,
551#else
552 ceilval | 1,
553#endif
554 ceilval);
555
556 if (oldval == ceilval)
557 break;
558
559 do
560 {
561 oldval
562 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
563 ceilval | 2,
564 ceilval | 1);
565
566 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
567 break;
568
569 if (oldval != ceilval)
570 lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
571 PTHREAD_MUTEX_PSHARED (mutex));
572 }
573 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
574 ceilval | 2, ceilval)
575 != ceilval);
576 }
577 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
578
579 assert (mutex->__data.__owner == 0);
580 mutex->__data.__count = 1;
581 }
582 break;
583
584 default:
585 /* Correct code cannot set any other type. */
586 return EINVAL;
587 }
588
589 /* Record the ownership. */
590 mutex->__data.__owner = id;
591#ifndef NO_INCR
592 ++mutex->__data.__nusers;
593#endif
594
595 LIBC_PROBE (mutex_acquired, 1, mutex);
596
597 return 0;
598}
599#ifndef __pthread_mutex_lock
600strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
601hidden_def (__pthread_mutex_lock)
602#endif
603
604
605#ifdef NO_INCR
606void
607internal_function
608__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
609{
610 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
611 assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
612 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
613
614 /* Record the ownership. */
615 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
616 mutex->__data.__owner = id;
617
618 if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
619 ++mutex->__data.__count;
620}
621#endif
622