1/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include <pthread-errnos.h>
21#include <kernel-features.h>
22#include <lowlevellock.h>
23
24#include <stap-probe.h>
25
26 .text
27
28#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
29 movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
30#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
31 movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
32#define LOAD_FUTEX_WAIT(reg) \
33 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
34#define LOAD_FUTEX_WAIT_ABS(reg) \
35 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
36#define LOAD_FUTEX_WAKE(reg) \
37 xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
38
39
40 .globl __lll_lock_wait_private
41 .type __lll_lock_wait_private,@function
42 .hidden __lll_lock_wait_private
43 .align 16
44__lll_lock_wait_private:
45 cfi_startproc
46 pushq %r10
47 cfi_adjust_cfa_offset(8)
48 pushq %rdx
49 cfi_adjust_cfa_offset(8)
50 cfi_offset(%r10, -16)
51 cfi_offset(%rdx, -24)
52 xorq %r10, %r10 /* No timeout. */
53 movl $2, %edx
54 LOAD_PRIVATE_FUTEX_WAIT (%esi)
55
56 cmpl %edx, %eax /* NB: %edx == 2 */
57 jne 2f
58
591: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
60 movl $SYS_futex, %eax
61 syscall
62
632: movl %edx, %eax
64 xchgl %eax, (%rdi) /* NB: lock is implied */
65
66 testl %eax, %eax
67 jnz 1b
68
69 popq %rdx
70 cfi_adjust_cfa_offset(-8)
71 cfi_restore(%rdx)
72 popq %r10
73 cfi_adjust_cfa_offset(-8)
74 cfi_restore(%r10)
75 retq
76 cfi_endproc
77 .size __lll_lock_wait_private,.-__lll_lock_wait_private
78
79#if !IS_IN (libc)
80 .globl __lll_lock_wait
81 .type __lll_lock_wait,@function
82 .hidden __lll_lock_wait
83 .align 16
84__lll_lock_wait:
85 cfi_startproc
86 pushq %r10
87 cfi_adjust_cfa_offset(8)
88 pushq %rdx
89 cfi_adjust_cfa_offset(8)
90 cfi_offset(%r10, -16)
91 cfi_offset(%rdx, -24)
92 xorq %r10, %r10 /* No timeout. */
93 movl $2, %edx
94 LOAD_FUTEX_WAIT (%esi)
95
96 cmpl %edx, %eax /* NB: %edx == 2 */
97 jne 2f
98
991: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
100 movl $SYS_futex, %eax
101 syscall
102
1032: movl %edx, %eax
104 xchgl %eax, (%rdi) /* NB: lock is implied */
105
106 testl %eax, %eax
107 jnz 1b
108
109 popq %rdx
110 cfi_adjust_cfa_offset(-8)
111 cfi_restore(%rdx)
112 popq %r10
113 cfi_adjust_cfa_offset(-8)
114 cfi_restore(%r10)
115 retq
116 cfi_endproc
117 .size __lll_lock_wait,.-__lll_lock_wait
118
119 /* %rdi: futex
120 %rsi: flags
121 %rdx: timeout
122 %eax: futex value
123 */
124 .globl __lll_timedlock_wait
125 .type __lll_timedlock_wait,@function
126 .hidden __lll_timedlock_wait
127 .align 16
128__lll_timedlock_wait:
129 cfi_startproc
130# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
131# ifdef PIC
132 cmpl $0, __have_futex_clock_realtime(%rip)
133# else
134 cmpl $0, __have_futex_clock_realtime
135# endif
136 je .Lreltmo
137# endif
138
139 cmpq $0, (%rdx)
140 js 5f
141
142 pushq %r9
143 cfi_adjust_cfa_offset(8)
144 cfi_rel_offset(%r9, 0)
145
146 movq %rdx, %r10
147 movl $0xffffffff, %r9d
148 LOAD_FUTEX_WAIT_ABS (%esi)
149
150 movl $2, %edx
151 cmpl %edx, %eax
152 jne 2f
153
1541: movl $SYS_futex, %eax
155 movl $2, %edx
156 syscall
157
1582: xchgl %edx, (%rdi) /* NB: lock is implied */
159
160 testl %edx, %edx
161 jz 3f
162
163 cmpl $-ETIMEDOUT, %eax
164 je 4f
165 cmpl $-EINVAL, %eax
166 jne 1b
1674: movl %eax, %edx
168 negl %edx
169
1703: movl %edx, %eax
171 popq %r9
172 cfi_adjust_cfa_offset(-8)
173 cfi_restore(%r9)
174 retq
175
1765: movl $ETIMEDOUT, %eax
177 retq
178
179# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
180.Lreltmo:
181 /* Check for a valid timeout value. */
182 cmpq $1000000000, 8(%rdx)
183 jae 3f
184
185 pushq %r8
186 cfi_adjust_cfa_offset(8)
187 pushq %r9
188 cfi_adjust_cfa_offset(8)
189 pushq %r12
190 cfi_adjust_cfa_offset(8)
191 pushq %r13
192 cfi_adjust_cfa_offset(8)
193 pushq %r14
194 cfi_adjust_cfa_offset(8)
195 cfi_offset(%r8, -16)
196 cfi_offset(%r9, -24)
197 cfi_offset(%r12, -32)
198 cfi_offset(%r13, -40)
199 cfi_offset(%r14, -48)
200 pushq %rsi
201 cfi_adjust_cfa_offset(8)
202
203 /* Stack frame for the timespec and timeval structs. */
204 subq $24, %rsp
205 cfi_adjust_cfa_offset(24)
206
207 movq %rdi, %r12
208 movq %rdx, %r13
209
210 movl $2, %edx
211 xchgl %edx, (%r12)
212
213 testl %edx, %edx
214 je 6f
215
2161:
217 /* Get current time. */
218 movq %rsp, %rdi
219 xorl %esi, %esi
220 /* This call works because we directly jump to a system call entry
221 which preserves all the registers. */
222 call JUMPTARGET(__gettimeofday)
223
224 /* Compute relative timeout. */
225 movq 8(%rsp), %rax
226 movl $1000, %edi
227 mul %rdi /* Milli seconds to nano seconds. */
228 movq (%r13), %rdi
229 movq 8(%r13), %rsi
230 subq (%rsp), %rdi
231 subq %rax, %rsi
232 jns 4f
233 addq $1000000000, %rsi
234 decq %rdi
2354: testq %rdi, %rdi
236 js 2f /* Time is already up. */
237
238 /* Store relative timeout. */
239 movq %rdi, (%rsp)
240 movq %rsi, 8(%rsp)
241
242 /* Futex call. */
243 movl $2, %edx
244 movl $1, %eax
245 movq %rsp, %r10
246 movl 24(%rsp), %esi
247 LOAD_FUTEX_WAIT (%esi)
248 movq %r12, %rdi
249 movl $SYS_futex, %eax
250 syscall
251
252 /* NB: %edx == 2 */
253 xchgl %edx, (%r12)
254
255 testl %edx, %edx
256 je 6f
257
258 cmpl $-ETIMEDOUT, %eax
259 jne 1b
2602: movl $ETIMEDOUT, %edx
261
2626: addq $32, %rsp
263 cfi_adjust_cfa_offset(-32)
264 popq %r14
265 cfi_adjust_cfa_offset(-8)
266 cfi_restore(%r14)
267 popq %r13
268 cfi_adjust_cfa_offset(-8)
269 cfi_restore(%r13)
270 popq %r12
271 cfi_adjust_cfa_offset(-8)
272 cfi_restore(%r12)
273 popq %r9
274 cfi_adjust_cfa_offset(-8)
275 cfi_restore(%r9)
276 popq %r8
277 cfi_adjust_cfa_offset(-8)
278 cfi_restore(%r8)
279 movl %edx, %eax
280 retq
281
2823: movl $EINVAL, %eax
283 retq
284# endif
285 cfi_endproc
286 .size __lll_timedlock_wait,.-__lll_timedlock_wait
287#endif
288
289
290 .globl __lll_unlock_wake_private
291 .type __lll_unlock_wake_private,@function
292 .hidden __lll_unlock_wake_private
293 .align 16
294__lll_unlock_wake_private:
295 cfi_startproc
296 pushq %rsi
297 cfi_adjust_cfa_offset(8)
298 pushq %rdx
299 cfi_adjust_cfa_offset(8)
300 cfi_offset(%rsi, -16)
301 cfi_offset(%rdx, -24)
302
303 movl $0, (%rdi)
304 LOAD_PRIVATE_FUTEX_WAKE (%esi)
305 movl $1, %edx /* Wake one thread. */
306 movl $SYS_futex, %eax
307 syscall
308
309 popq %rdx
310 cfi_adjust_cfa_offset(-8)
311 cfi_restore(%rdx)
312 popq %rsi
313 cfi_adjust_cfa_offset(-8)
314 cfi_restore(%rsi)
315 retq
316 cfi_endproc
317 .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
318
319#if !IS_IN (libc)
320 .globl __lll_unlock_wake
321 .type __lll_unlock_wake,@function
322 .hidden __lll_unlock_wake
323 .align 16
324__lll_unlock_wake:
325 cfi_startproc
326 pushq %rsi
327 cfi_adjust_cfa_offset(8)
328 pushq %rdx
329 cfi_adjust_cfa_offset(8)
330 cfi_offset(%rsi, -16)
331 cfi_offset(%rdx, -24)
332
333 movl $0, (%rdi)
334 LOAD_FUTEX_WAKE (%esi)
335 movl $1, %edx /* Wake one thread. */
336 movl $SYS_futex, %eax
337 syscall
338
339 popq %rdx
340 cfi_adjust_cfa_offset(-8)
341 cfi_restore(%rdx)
342 popq %rsi
343 cfi_adjust_cfa_offset(-8)
344 cfi_restore(%rsi)
345 retq
346 cfi_endproc
347 .size __lll_unlock_wake,.-__lll_unlock_wake
348
349 .globl __lll_timedwait_tid
350 .type __lll_timedwait_tid,@function
351 .hidden __lll_timedwait_tid
352 .align 16
353__lll_timedwait_tid:
354 cfi_startproc
355 pushq %r12
356 cfi_adjust_cfa_offset(8)
357 pushq %r13
358 cfi_adjust_cfa_offset(8)
359 cfi_offset(%r12, -16)
360 cfi_offset(%r13, -24)
361
362 movq %rdi, %r12
363 movq %rsi, %r13
364
365 /* Align stack to 16 bytes when calling __gettimeofday. */
366 subq $24, %rsp
367 cfi_adjust_cfa_offset(24)
368
369 /* Get current time. */
3702: movq %rsp, %rdi
371 xorl %esi, %esi
372 /* This call works because we directly jump to a system call entry
373 which preserves all the registers. */
374 call JUMPTARGET(__gettimeofday)
375
376 /* Compute relative timeout. */
377 movq 8(%rsp), %rax
378 movl $1000, %edi
379 mul %rdi /* Milli seconds to nano seconds. */
380 movq (%r13), %rdi
381 movq 8(%r13), %rsi
382 subq (%rsp), %rdi
383 subq %rax, %rsi
384 jns 5f
385 addq $1000000000, %rsi
386 decq %rdi
3875: testq %rdi, %rdi
388 js 6f /* Time is already up. */
389
390 movq %rdi, (%rsp) /* Store relative timeout. */
391 movq %rsi, 8(%rsp)
392
393 movl (%r12), %edx
394 testl %edx, %edx
395 jz 4f
396
397 movq %rsp, %r10
398 /* XXX The kernel so far uses global futex for the wakeup at
399 all times. */
400#if FUTEX_WAIT == 0
401 xorl %esi, %esi
402#else
403 movl $FUTEX_WAIT, %esi
404#endif
405 movq %r12, %rdi
406 movl $SYS_futex, %eax
407 syscall
408
409 cmpl $0, (%rdi)
410 jne 1f
4114: xorl %eax, %eax
412
4138: addq $24, %rsp
414 cfi_adjust_cfa_offset(-24)
415 popq %r13
416 cfi_adjust_cfa_offset(-8)
417 cfi_restore(%r13)
418 popq %r12
419 cfi_adjust_cfa_offset(-8)
420 cfi_restore(%r12)
421 retq
422
423 cfi_adjust_cfa_offset(32)
4241: cmpq $-ETIMEDOUT, %rax
425 jne 2b
426
4276: movl $ETIMEDOUT, %eax
428 jmp 8b
429 cfi_endproc
430 .size __lll_timedwait_tid,.-__lll_timedwait_tid
431#endif
432