1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <time.h>
22#include <sys/param.h>
23#include <sys/time.h>
24#include "pthreadP.h"
25#include <atomic.h>
26#include <lowlevellock.h>
27#include <not-cancel.h>
28
29#include <stap-probe.h>
30
31#ifndef lll_timedlock_elision
32#define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39#ifndef FORCE_ELISION
40#define FORCE_ELISION(m, s)
41#endif
42
43int
44pthread_mutex_timedlock (pthread_mutex_t *mutex,
45 const struct timespec *abstime)
46{
47 int oldval;
48 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
49 int result = 0;
50
51 LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime);
52
53 /* We must not check ABSTIME here. If the thread does not block
54 abstime must not be checked for a valid value. */
55
56 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
57 PTHREAD_MUTEX_TIMED_NP))
58 {
59 /* Recursive mutex. */
60 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
61 case PTHREAD_MUTEX_RECURSIVE_NP:
62 /* Check whether we already hold the mutex. */
63 if (mutex->__data.__owner == id)
64 {
65 /* Just bump the counter. */
66 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
67 /* Overflow of the counter. */
68 return EAGAIN;
69
70 ++mutex->__data.__count;
71
72 goto out;
73 }
74
75 /* We have to get the mutex. */
76 result = lll_timedlock (mutex->__data.__lock, abstime,
77 PTHREAD_MUTEX_PSHARED (mutex));
78
79 if (result != 0)
80 goto out;
81
82 /* Only locked once so far. */
83 mutex->__data.__count = 1;
84 break;
85
86 /* Error checking mutex. */
87 case PTHREAD_MUTEX_ERRORCHECK_NP:
88 /* Check whether we already hold the mutex. */
89 if (__glibc_unlikely (mutex->__data.__owner == id))
90 return EDEADLK;
91
92 /* Don't do lock elision on an error checking mutex. */
93 goto simple;
94
95 case PTHREAD_MUTEX_TIMED_NP:
96 FORCE_ELISION (mutex, goto elision);
97 simple:
98 /* Normal mutex. */
99 result = lll_timedlock (mutex->__data.__lock, abstime,
100 PTHREAD_MUTEX_PSHARED (mutex));
101 break;
102
103 case PTHREAD_MUTEX_TIMED_ELISION_NP:
104 elision: __attribute__((unused))
105 /* Don't record ownership */
106 return lll_timedlock_elision (mutex->__data.__lock,
107 mutex->__data.__spins,
108 abstime,
109 PTHREAD_MUTEX_PSHARED (mutex));
110
111
112 case PTHREAD_MUTEX_ADAPTIVE_NP:
113 if (! __is_smp)
114 goto simple;
115
116 if (lll_trylock (mutex->__data.__lock) != 0)
117 {
118 int cnt = 0;
119 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
120 mutex->__data.__spins * 2 + 10);
121 do
122 {
123 if (cnt++ >= max_cnt)
124 {
125 result = lll_timedlock (mutex->__data.__lock, abstime,
126 PTHREAD_MUTEX_PSHARED (mutex));
127 break;
128 }
129 atomic_spin_nop ();
130 }
131 while (lll_trylock (mutex->__data.__lock) != 0);
132
133 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
134 }
135 break;
136
137 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
138 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
139 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
140 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
141 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
142 &mutex->__data.__list.__next);
143
144 oldval = mutex->__data.__lock;
145 do
146 {
147 again:
148 if ((oldval & FUTEX_OWNER_DIED) != 0)
149 {
150 /* The previous owner died. Try locking the mutex. */
151 int newval = id | (oldval & FUTEX_WAITERS);
152
153 newval
154 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
155 newval, oldval);
156 if (newval != oldval)
157 {
158 oldval = newval;
159 goto again;
160 }
161
162 /* We got the mutex. */
163 mutex->__data.__count = 1;
164 /* But it is inconsistent unless marked otherwise. */
165 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
166
167 ENQUEUE_MUTEX (mutex);
168 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
169
170 /* Note that we deliberately exit here. If we fall
171 through to the end of the function __nusers would be
172 incremented which is not correct because the old
173 owner has to be discounted. */
174 return EOWNERDEAD;
175 }
176
177 /* Check whether we already hold the mutex. */
178 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
179 {
180 int kind = PTHREAD_MUTEX_TYPE (mutex);
181 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
182 {
183 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
184 NULL);
185 return EDEADLK;
186 }
187
188 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
189 {
190 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
191 NULL);
192
193 /* Just bump the counter. */
194 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
195 /* Overflow of the counter. */
196 return EAGAIN;
197
198 ++mutex->__data.__count;
199
200 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
201
202 return 0;
203 }
204 }
205
206 result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
207 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
208
209 if (__builtin_expect (mutex->__data.__owner
210 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
211 {
212 /* This mutex is now not recoverable. */
213 mutex->__data.__count = 0;
214 lll_unlock (mutex->__data.__lock,
215 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
216 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
217 return ENOTRECOVERABLE;
218 }
219
220 if (result == ETIMEDOUT || result == EINVAL)
221 goto out;
222
223 oldval = result;
224 }
225 while ((oldval & FUTEX_OWNER_DIED) != 0);
226
227 mutex->__data.__count = 1;
228 ENQUEUE_MUTEX (mutex);
229 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
230 break;
231
232 /* The PI support requires the Linux futex system call. If that's not
233 available, pthread_mutex_init should never have allowed the type to
234 be set. So it will get the default case for an invalid type. */
235#ifdef __NR_futex
236 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
237 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
238 case PTHREAD_MUTEX_PI_NORMAL_NP:
239 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
240 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
241 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
242 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
243 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
244 {
245 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
246 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
247
248 if (robust)
249 /* Note: robust PI futexes are signaled by setting bit 0. */
250 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
251 (void *) (((uintptr_t) &mutex->__data.__list.__next)
252 | 1));
253
254 oldval = mutex->__data.__lock;
255
256 /* Check whether we already hold the mutex. */
257 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
258 {
259 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
260 {
261 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
262 return EDEADLK;
263 }
264
265 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
266 {
267 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
268
269 /* Just bump the counter. */
270 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
271 /* Overflow of the counter. */
272 return EAGAIN;
273
274 ++mutex->__data.__count;
275
276 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
277
278 return 0;
279 }
280 }
281
282 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
283 id, 0);
284
285 if (oldval != 0)
286 {
287 /* The mutex is locked. The kernel will now take care of
288 everything. The timeout value must be a relative value.
289 Convert it. */
290 int private = (robust
291 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
292 : PTHREAD_MUTEX_PSHARED (mutex));
293 INTERNAL_SYSCALL_DECL (__err);
294
295 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
296 __lll_private_flag (FUTEX_LOCK_PI,
297 private), 1,
298 abstime);
299 if (INTERNAL_SYSCALL_ERROR_P (e, __err))
300 {
301 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
302 return ETIMEDOUT;
303
304 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
305 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
306 {
307 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
308 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
309 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
310 /* ESRCH can happen only for non-robust PI mutexes where
311 the owner of the lock died. */
312 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
313 || !robust);
314
315 /* Delay the thread until the timeout is reached.
316 Then return ETIMEDOUT. */
317 struct timespec reltime;
318 struct timespec now;
319
320 INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
321 &now);
322 reltime.tv_sec = abstime->tv_sec - now.tv_sec;
323 reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
324 if (reltime.tv_nsec < 0)
325 {
326 reltime.tv_nsec += 1000000000;
327 --reltime.tv_sec;
328 }
329 if (reltime.tv_sec >= 0)
330 while (nanosleep_not_cancel (&reltime, &reltime) != 0)
331 continue;
332
333 return ETIMEDOUT;
334 }
335
336 return INTERNAL_SYSCALL_ERRNO (e, __err);
337 }
338
339 oldval = mutex->__data.__lock;
340
341 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
342 }
343
344 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
345 {
346 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
347
348 /* We got the mutex. */
349 mutex->__data.__count = 1;
350 /* But it is inconsistent unless marked otherwise. */
351 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
352
353 ENQUEUE_MUTEX_PI (mutex);
354 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
355
356 /* Note that we deliberately exit here. If we fall
357 through to the end of the function __nusers would be
358 incremented which is not correct because the old owner
359 has to be discounted. */
360 return EOWNERDEAD;
361 }
362
363 if (robust
364 && __builtin_expect (mutex->__data.__owner
365 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
366 {
367 /* This mutex is now not recoverable. */
368 mutex->__data.__count = 0;
369
370 INTERNAL_SYSCALL_DECL (__err);
371 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
372 __lll_private_flag (FUTEX_UNLOCK_PI,
373 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
374 0, 0);
375
376 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
377 return ENOTRECOVERABLE;
378 }
379
380 mutex->__data.__count = 1;
381 if (robust)
382 {
383 ENQUEUE_MUTEX_PI (mutex);
384 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
385 }
386 }
387 break;
388#endif /* __NR_futex. */
389
390 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
391 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
392 case PTHREAD_MUTEX_PP_NORMAL_NP:
393 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
394 {
395 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
396
397 oldval = mutex->__data.__lock;
398
399 /* Check whether we already hold the mutex. */
400 if (mutex->__data.__owner == id)
401 {
402 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
403 return EDEADLK;
404
405 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
406 {
407 /* Just bump the counter. */
408 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
409 /* Overflow of the counter. */
410 return EAGAIN;
411
412 ++mutex->__data.__count;
413
414 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
415
416 return 0;
417 }
418 }
419
420 int oldprio = -1, ceilval;
421 do
422 {
423 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
424 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
425
426 if (__pthread_current_priority () > ceiling)
427 {
428 result = EINVAL;
429 failpp:
430 if (oldprio != -1)
431 __pthread_tpp_change_priority (oldprio, -1);
432 return result;
433 }
434
435 result = __pthread_tpp_change_priority (oldprio, ceiling);
436 if (result)
437 return result;
438
439 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
440 oldprio = ceiling;
441
442 oldval
443 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
444 ceilval | 1, ceilval);
445
446 if (oldval == ceilval)
447 break;
448
449 do
450 {
451 oldval
452 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
453 ceilval | 2,
454 ceilval | 1);
455
456 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
457 break;
458
459 if (oldval != ceilval)
460 {
461 /* Reject invalid timeouts. */
462 if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
463 {
464 result = EINVAL;
465 goto failpp;
466 }
467
468 struct timeval tv;
469 struct timespec rt;
470
471 /* Get the current time. */
472 (void) __gettimeofday (&tv, NULL);
473
474 /* Compute relative timeout. */
475 rt.tv_sec = abstime->tv_sec - tv.tv_sec;
476 rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
477 if (rt.tv_nsec < 0)
478 {
479 rt.tv_nsec += 1000000000;
480 --rt.tv_sec;
481 }
482
483 /* Already timed out? */
484 if (rt.tv_sec < 0)
485 {
486 result = ETIMEDOUT;
487 goto failpp;
488 }
489
490 lll_futex_timed_wait (&mutex->__data.__lock,
491 ceilval | 2, &rt,
492 PTHREAD_MUTEX_PSHARED (mutex));
493 }
494 }
495 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
496 ceilval | 2, ceilval)
497 != ceilval);
498 }
499 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
500
501 assert (mutex->__data.__owner == 0);
502 mutex->__data.__count = 1;
503 }
504 break;
505
506 default:
507 /* Correct code cannot set any other type. */
508 return EINVAL;
509 }
510
511 if (result == 0)
512 {
513 /* Record the ownership. */
514 mutex->__data.__owner = id;
515 ++mutex->__data.__nusers;
516
517 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
518 }
519
520 out:
521 return result;
522}
523