1/* Copyright (C) 2002-2022 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
8
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
17
18#include <assert.h>
19#include <errno.h>
20#include <stdlib.h>
21#include <unistd.h>
22#include <sys/param.h>
23#include <not-cancel.h>
24#include "pthreadP.h"
25#include <atomic.h>
26#include <futex-internal.h>
27#include <stap-probe.h>
28#include <shlib-compat.h>
29
30/* Some of the following definitions differ when pthread_mutex_cond_lock.c
31 includes this file. */
32#ifndef LLL_MUTEX_LOCK
33/* lll_lock with single-thread optimization. */
34static inline void
35lll_mutex_lock_optimized (pthread_mutex_t *mutex)
36{
37 /* The single-threaded optimization is only valid for private
38 mutexes. For process-shared mutexes, the mutex could be in a
39 shared mapping, so synchronization with another process is needed
40 even without any threads. If the lock is already marked as
41 acquired, POSIX requires that pthread_mutex_lock deadlocks for
42 normal mutexes, so skip the optimization in that case as
43 well. */
44 int private = PTHREAD_MUTEX_PSHARED (mutex);
45 if (private == LLL_PRIVATE && SINGLE_THREAD_P && mutex->__data.__lock == 0)
46 mutex->__data.__lock = 1;
47 else
48 lll_lock (mutex->__data.__lock, private);
49}
50
51# define LLL_MUTEX_LOCK(mutex) \
52 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
53# define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex)
54# define LLL_MUTEX_TRYLOCK(mutex) \
55 lll_trylock ((mutex)->__data.__lock)
56# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
57# define LLL_MUTEX_LOCK_ELISION(mutex) \
58 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
59 PTHREAD_MUTEX_PSHARED (mutex))
60# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
61 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
62 PTHREAD_MUTEX_PSHARED (mutex))
63# define PTHREAD_MUTEX_LOCK ___pthread_mutex_lock
64# define PTHREAD_MUTEX_VERSIONS 1
65#endif
66
67#ifndef LLL_MUTEX_READ_LOCK
68# define LLL_MUTEX_READ_LOCK(mutex) \
69 atomic_load_relaxed (&(mutex)->__data.__lock)
70#endif
71
72static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
73 __attribute_noinline__;
74
75int
76PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex)
77{
78 /* See concurrency notes regarding mutex type which is loaded from __kind
79 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
80 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
81
82 LIBC_PROBE (mutex_entry, 1, mutex);
83
84 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
85 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
86 return __pthread_mutex_lock_full (mutex);
87
88 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
89 {
90 FORCE_ELISION (mutex, goto elision);
91 simple:
92 /* Normal mutex. */
93 LLL_MUTEX_LOCK_OPTIMIZED (mutex);
94 assert (mutex->__data.__owner == 0);
95 }
96#if ENABLE_ELISION_SUPPORT
97 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
98 {
99 elision: __attribute__((unused))
100 /* This case can never happen on a system without elision,
101 as the mutex type initialization functions will not
102 allow to set the elision flags. */
103 /* Don't record owner or users for elision case. This is a
104 tail call. */
105 return LLL_MUTEX_LOCK_ELISION (mutex);
106 }
107#endif
108 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
109 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
110 {
111 /* Recursive mutex. */
112 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
113
114 /* Check whether we already hold the mutex. */
115 if (mutex->__data.__owner == id)
116 {
117 /* Just bump the counter. */
118 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
119 /* Overflow of the counter. */
120 return EAGAIN;
121
122 ++mutex->__data.__count;
123
124 return 0;
125 }
126
127 /* We have to get the mutex. */
128 LLL_MUTEX_LOCK_OPTIMIZED (mutex);
129
130 assert (mutex->__data.__owner == 0);
131 mutex->__data.__count = 1;
132 }
133 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
134 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
135 {
136 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
137 {
138 int cnt = 0;
139 int max_cnt = MIN (max_adaptive_count (),
140 mutex->__data.__spins * 2 + 10);
141 do
142 {
143 if (cnt++ >= max_cnt)
144 {
145 LLL_MUTEX_LOCK (mutex);
146 break;
147 }
148 atomic_spin_nop ();
149 }
150 while (LLL_MUTEX_READ_LOCK (mutex) != 0
151 || LLL_MUTEX_TRYLOCK (mutex) != 0);
152
153 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
154 }
155 assert (mutex->__data.__owner == 0);
156 }
157 else
158 {
159 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
160 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
161 /* Check whether we already hold the mutex. */
162 if (__glibc_unlikely (mutex->__data.__owner == id))
163 return EDEADLK;
164 goto simple;
165 }
166
167 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
168
169 /* Record the ownership. */
170 mutex->__data.__owner = id;
171#ifndef NO_INCR
172 ++mutex->__data.__nusers;
173#endif
174
175 LIBC_PROBE (mutex_acquired, 1, mutex);
176
177 return 0;
178}
179
180static int
181__pthread_mutex_lock_full (pthread_mutex_t *mutex)
182{
183 int oldval;
184 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
185
186 switch (PTHREAD_MUTEX_TYPE (mutex))
187 {
188 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
189 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
190 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
191 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
192 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
193 &mutex->__data.__list.__next);
194 /* We need to set op_pending before starting the operation. Also
195 see comments at ENQUEUE_MUTEX. */
196 __asm ("" ::: "memory");
197
198 oldval = mutex->__data.__lock;
199 /* This is set to FUTEX_WAITERS iff we might have shared the
200 FUTEX_WAITERS flag with other threads, and therefore need to keep it
201 set to avoid lost wake-ups. We have the same requirement in the
202 simple mutex algorithm.
203 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
204 are building the special case mutexes for use from within condition
205 variables. */
206 unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
207 while (1)
208 {
209 /* Try to acquire the lock through a CAS from 0 (not acquired) to
210 our TID | assume_other_futex_waiters. */
211 if (__glibc_likely (oldval == 0))
212 {
213 oldval
214 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
215 id | assume_other_futex_waiters, 0);
216 if (__glibc_likely (oldval == 0))
217 break;
218 }
219
220 if ((oldval & FUTEX_OWNER_DIED) != 0)
221 {
222 /* The previous owner died. Try locking the mutex. */
223 int newval = id;
224#ifdef NO_INCR
225 /* We are not taking assume_other_futex_waiters into accoount
226 here simply because we'll set FUTEX_WAITERS anyway. */
227 newval |= FUTEX_WAITERS;
228#else
229 newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
230#endif
231
232 newval
233 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
234 newval, oldval);
235
236 if (newval != oldval)
237 {
238 oldval = newval;
239 continue;
240 }
241
242 /* We got the mutex. */
243 mutex->__data.__count = 1;
244 /* But it is inconsistent unless marked otherwise. */
245 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
246
247 /* We must not enqueue the mutex before we have acquired it.
248 Also see comments at ENQUEUE_MUTEX. */
249 __asm ("" ::: "memory");
250 ENQUEUE_MUTEX (mutex);
251 /* We need to clear op_pending after we enqueue the mutex. */
252 __asm ("" ::: "memory");
253 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
254
255 /* Note that we deliberately exit here. If we fall
256 through to the end of the function __nusers would be
257 incremented which is not correct because the old
258 owner has to be discounted. If we are not supposed
259 to increment __nusers we actually have to decrement
260 it here. */
261#ifdef NO_INCR
262 --mutex->__data.__nusers;
263#endif
264
265 return EOWNERDEAD;
266 }
267
268 /* Check whether we already hold the mutex. */
269 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
270 {
271 int kind = PTHREAD_MUTEX_TYPE (mutex);
272 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
273 {
274 /* We do not need to ensure ordering wrt another memory
275 access. Also see comments at ENQUEUE_MUTEX. */
276 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
277 NULL);
278 return EDEADLK;
279 }
280
281 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
282 {
283 /* We do not need to ensure ordering wrt another memory
284 access. */
285 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
286 NULL);
287
288 /* Just bump the counter. */
289 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
290 /* Overflow of the counter. */
291 return EAGAIN;
292
293 ++mutex->__data.__count;
294
295 return 0;
296 }
297 }
298
299 /* We cannot acquire the mutex nor has its owner died. Thus, try
300 to block using futexes. Set FUTEX_WAITERS if necessary so that
301 other threads are aware that there are potentially threads
302 blocked on the futex. Restart if oldval changed in the
303 meantime. */
304 if ((oldval & FUTEX_WAITERS) == 0)
305 {
306 int val = atomic_compare_and_exchange_val_acq
307 (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval);
308 if (val != oldval)
309 {
310 oldval = val;
311 continue;
312 }
313 oldval |= FUTEX_WAITERS;
314 }
315
316 /* It is now possible that we share the FUTEX_WAITERS flag with
317 another thread; therefore, update assume_other_futex_waiters so
318 that we do not forget about this when handling other cases
319 above and thus do not cause lost wake-ups. */
320 assume_other_futex_waiters |= FUTEX_WAITERS;
321
322 /* Block using the futex and reload current lock value. */
323 futex_wait ((unsigned int *) &mutex->__data.__lock, oldval,
324 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
325 oldval = mutex->__data.__lock;
326 }
327
328 /* We have acquired the mutex; check if it is still consistent. */
329 if (__builtin_expect (mutex->__data.__owner
330 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
331 {
332 /* This mutex is now not recoverable. */
333 mutex->__data.__count = 0;
334 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
335 lll_unlock (mutex->__data.__lock, private);
336 /* FIXME This violates the mutex destruction requirements. See
337 __pthread_mutex_unlock_full. */
338 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
339 return ENOTRECOVERABLE;
340 }
341
342 mutex->__data.__count = 1;
343 /* We must not enqueue the mutex before we have acquired it.
344 Also see comments at ENQUEUE_MUTEX. */
345 __asm ("" ::: "memory");
346 ENQUEUE_MUTEX (mutex);
347 /* We need to clear op_pending after we enqueue the mutex. */
348 __asm ("" ::: "memory");
349 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
350 break;
351
352 /* The PI support requires the Linux futex system call. If that's not
353 available, pthread_mutex_init should never have allowed the type to
354 be set. So it will get the default case for an invalid type. */
355#ifdef __NR_futex
356 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
357 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
358 case PTHREAD_MUTEX_PI_NORMAL_NP:
359 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
360 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
361 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
362 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
363 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
364 {
365 int kind, robust;
366 {
367 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
368 in sysdeps/nptl/bits/thread-shared-types.h. */
369 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
370 kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
371 robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
372 }
373
374 if (robust)
375 {
376 /* Note: robust PI futexes are signaled by setting bit 0. */
377 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
378 (void *) (((uintptr_t) &mutex->__data.__list.__next)
379 | 1));
380 /* We need to set op_pending before starting the operation. Also
381 see comments at ENQUEUE_MUTEX. */
382 __asm ("" ::: "memory");
383 }
384
385 oldval = mutex->__data.__lock;
386
387 /* Check whether we already hold the mutex. */
388 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
389 {
390 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
391 {
392 /* We do not need to ensure ordering wrt another memory
393 access. */
394 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
395 return EDEADLK;
396 }
397
398 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
399 {
400 /* We do not need to ensure ordering wrt another memory
401 access. */
402 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
403
404 /* Just bump the counter. */
405 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
406 /* Overflow of the counter. */
407 return EAGAIN;
408
409 ++mutex->__data.__count;
410
411 return 0;
412 }
413 }
414
415 int newval = id;
416# ifdef NO_INCR
417 newval |= FUTEX_WAITERS;
418# endif
419 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
420 newval, 0);
421
422 if (oldval != 0)
423 {
424 /* The mutex is locked. The kernel will now take care of
425 everything. */
426 int private = (robust
427 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
428 : PTHREAD_MUTEX_PSHARED (mutex));
429 int e = __futex_lock_pi64 (&mutex->__data.__lock, 0 /* ununsed */,
430 NULL, private);
431 if (e == ESRCH || e == EDEADLK)
432 {
433 assert (e != EDEADLK
434 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
435 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
436 /* ESRCH can happen only for non-robust PI mutexes where
437 the owner of the lock died. */
438 assert (e != ESRCH || !robust);
439
440 /* Delay the thread indefinitely. */
441 while (1)
442 __futex_abstimed_wait64 (&(unsigned int){0}, 0,
443 0 /* ignored */, NULL, private);
444 }
445
446 oldval = mutex->__data.__lock;
447
448 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
449 }
450
451 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
452 {
453 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
454
455 /* We got the mutex. */
456 mutex->__data.__count = 1;
457 /* But it is inconsistent unless marked otherwise. */
458 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
459
460 /* We must not enqueue the mutex before we have acquired it.
461 Also see comments at ENQUEUE_MUTEX. */
462 __asm ("" ::: "memory");
463 ENQUEUE_MUTEX_PI (mutex);
464 /* We need to clear op_pending after we enqueue the mutex. */
465 __asm ("" ::: "memory");
466 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
467
468 /* Note that we deliberately exit here. If we fall
469 through to the end of the function __nusers would be
470 incremented which is not correct because the old owner
471 has to be discounted. If we are not supposed to
472 increment __nusers we actually have to decrement it here. */
473# ifdef NO_INCR
474 --mutex->__data.__nusers;
475# endif
476
477 return EOWNERDEAD;
478 }
479
480 if (robust
481 && __builtin_expect (mutex->__data.__owner
482 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
483 {
484 /* This mutex is now not recoverable. */
485 mutex->__data.__count = 0;
486
487 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
488 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
489
490 /* To the kernel, this will be visible after the kernel has
491 acquired the mutex in the syscall. */
492 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
493 return ENOTRECOVERABLE;
494 }
495
496 mutex->__data.__count = 1;
497 if (robust)
498 {
499 /* We must not enqueue the mutex before we have acquired it.
500 Also see comments at ENQUEUE_MUTEX. */
501 __asm ("" ::: "memory");
502 ENQUEUE_MUTEX_PI (mutex);
503 /* We need to clear op_pending after we enqueue the mutex. */
504 __asm ("" ::: "memory");
505 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
506 }
507 }
508 break;
509#endif /* __NR_futex. */
510
511 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
512 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
513 case PTHREAD_MUTEX_PP_NORMAL_NP:
514 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
515 {
516 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
517 in sysdeps/nptl/bits/thread-shared-types.h. */
518 int kind = atomic_load_relaxed (&(mutex->__data.__kind))
519 & PTHREAD_MUTEX_KIND_MASK_NP;
520
521 oldval = mutex->__data.__lock;
522
523 /* Check whether we already hold the mutex. */
524 if (mutex->__data.__owner == id)
525 {
526 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
527 return EDEADLK;
528
529 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
530 {
531 /* Just bump the counter. */
532 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
533 /* Overflow of the counter. */
534 return EAGAIN;
535
536 ++mutex->__data.__count;
537
538 return 0;
539 }
540 }
541
542 int oldprio = -1, ceilval;
543 do
544 {
545 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
546 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
547
548 if (__pthread_current_priority () > ceiling)
549 {
550 if (oldprio != -1)
551 __pthread_tpp_change_priority (oldprio, -1);
552 return EINVAL;
553 }
554
555 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
556 if (retval)
557 return retval;
558
559 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
560 oldprio = ceiling;
561
562 oldval
563 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
564#ifdef NO_INCR
565 ceilval | 2,
566#else
567 ceilval | 1,
568#endif
569 ceilval);
570
571 if (oldval == ceilval)
572 break;
573
574 do
575 {
576 oldval
577 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
578 ceilval | 2,
579 ceilval | 1);
580
581 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
582 break;
583
584 if (oldval != ceilval)
585 futex_wait ((unsigned int * ) &mutex->__data.__lock,
586 ceilval | 2,
587 PTHREAD_MUTEX_PSHARED (mutex));
588 }
589 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
590 ceilval | 2, ceilval)
591 != ceilval);
592 }
593 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
594
595 assert (mutex->__data.__owner == 0);
596 mutex->__data.__count = 1;
597 }
598 break;
599
600 default:
601 /* Correct code cannot set any other type. */
602 return EINVAL;
603 }
604
605 /* Record the ownership. */
606 mutex->__data.__owner = id;
607#ifndef NO_INCR
608 ++mutex->__data.__nusers;
609#endif
610
611 LIBC_PROBE (mutex_acquired, 1, mutex);
612
613 return 0;
614}
615
616#if PTHREAD_MUTEX_VERSIONS
617libc_hidden_ver (___pthread_mutex_lock, __pthread_mutex_lock)
618# ifndef SHARED
619strong_alias (___pthread_mutex_lock, __pthread_mutex_lock)
620# endif
621versioned_symbol (libpthread, ___pthread_mutex_lock, pthread_mutex_lock,
622 GLIBC_2_0);
623
624# if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
625compat_symbol (libpthread, ___pthread_mutex_lock, __pthread_mutex_lock,
626 GLIBC_2_0);
627# endif
628#endif /* PTHREAD_MUTEX_VERSIONS */
629
630
631#ifdef NO_INCR
632void
633__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
634{
635 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
636 in sysdeps/nptl/bits/thread-shared-types.h. */
637 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
638 assert ((mutex_kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
639 assert ((mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
640 assert ((mutex_kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
641
642 /* Record the ownership. */
643 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
644 mutex->__data.__owner = id;
645
646 if (mutex_kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
647 ++mutex->__data.__count;
648}
649#endif
650