1/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <signal.h>
22#include <stdint.h>
23#include <string.h>
24#include <unistd.h>
25#include <sys/mman.h>
26#include <sys/param.h>
27#include <dl-sysdep.h>
28#include <dl-tls.h>
29#include <tls.h>
30#include <list.h>
31#include <lowlevellock.h>
32#include <futex-internal.h>
33#include <kernel-features.h>
34#include <stack-aliasing.h>
35
36
37#ifndef NEED_SEPARATE_REGISTER_STACK
38
39/* Most architectures have exactly one stack pointer. Some have more. */
40# define STACK_VARIABLES void *stackaddr = NULL
41
42/* How to pass the values to the 'create_thread' function. */
43# define STACK_VARIABLES_ARGS stackaddr
44
45/* How to declare function which gets there parameters. */
46# define STACK_VARIABLES_PARMS void *stackaddr
47
48/* How to declare allocate_stack. */
49# define ALLOCATE_STACK_PARMS void **stack
50
51/* This is how the function is called. We do it this way to allow
52 other variants of the function to have more parameters. */
53# define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
54
55#else
56
57/* We need two stacks. The kernel will place them but we have to tell
58 the kernel about the size of the reserved address space. */
59# define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
60
61/* How to pass the values to the 'create_thread' function. */
62# define STACK_VARIABLES_ARGS stackaddr, stacksize
63
64/* How to declare function which gets there parameters. */
65# define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
66
67/* How to declare allocate_stack. */
68# define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
69
70/* This is how the function is called. We do it this way to allow
71 other variants of the function to have more parameters. */
72# define ALLOCATE_STACK(attr, pd) \
73 allocate_stack (attr, pd, &stackaddr, &stacksize)
74
75#endif
76
77
78/* Default alignment of stack. */
79#ifndef STACK_ALIGN
80# define STACK_ALIGN __alignof__ (long double)
81#endif
82
83/* Default value for minimal stack size after allocating thread
84 descriptor and guard. */
85#ifndef MINIMAL_REST_STACK
86# define MINIMAL_REST_STACK 4096
87#endif
88
89
90/* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
91 a stack. Use it when possible. */
92#ifndef MAP_STACK
93# define MAP_STACK 0
94#endif
95
96/* This yields the pointer that TLS support code calls the thread pointer. */
97#if TLS_TCB_AT_TP
98# define TLS_TPADJ(pd) (pd)
99#elif TLS_DTV_AT_TP
100# define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
101#endif
102
103/* Cache handling for not-yet free stacks. */
104
105/* Maximum size in kB of cache. */
106static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
107static size_t stack_cache_actsize;
108
109/* Mutex protecting this variable. */
110static int stack_cache_lock = LLL_LOCK_INITIALIZER;
111
112/* List of queued stack frames. */
113static LIST_HEAD (stack_cache);
114
115/* List of the stacks in use. */
116static LIST_HEAD (stack_used);
117
118/* We need to record what list operations we are going to do so that,
119 in case of an asynchronous interruption due to a fork() call, we
120 can correct for the work. */
121static uintptr_t in_flight_stack;
122
123/* List of the threads with user provided stacks in use. No need to
124 initialize this, since it's done in __pthread_initialize_minimal. */
125list_t __stack_user __attribute__ ((nocommon));
126hidden_data_def (__stack_user)
127
128
129/* Check whether the stack is still used or not. */
130#define FREE_P(descr) ((descr)->tid <= 0)
131
132
133static void
134stack_list_del (list_t *elem)
135{
136 in_flight_stack = (uintptr_t) elem;
137
138 atomic_write_barrier ();
139
140 list_del (elem);
141
142 atomic_write_barrier ();
143
144 in_flight_stack = 0;
145}
146
147
148static void
149stack_list_add (list_t *elem, list_t *list)
150{
151 in_flight_stack = (uintptr_t) elem | 1;
152
153 atomic_write_barrier ();
154
155 list_add (elem, list);
156
157 atomic_write_barrier ();
158
159 in_flight_stack = 0;
160}
161
162
163/* We create a double linked list of all cache entries. Double linked
164 because this allows removing entries from the end. */
165
166
167/* Get a stack frame from the cache. We have to match by size since
168 some blocks might be too small or far too large. */
169static struct pthread *
170get_cached_stack (size_t *sizep, void **memp)
171{
172 size_t size = *sizep;
173 struct pthread *result = NULL;
174 list_t *entry;
175
176 lll_lock (stack_cache_lock, LLL_PRIVATE);
177
178 /* Search the cache for a matching entry. We search for the
179 smallest stack which has at least the required size. Note that
180 in normal situations the size of all allocated stacks is the
181 same. As the very least there are only a few different sizes.
182 Therefore this loop will exit early most of the time with an
183 exact match. */
184 list_for_each (entry, &stack_cache)
185 {
186 struct pthread *curr;
187
188 curr = list_entry (entry, struct pthread, list);
189 if (FREE_P (curr) && curr->stackblock_size >= size)
190 {
191 if (curr->stackblock_size == size)
192 {
193 result = curr;
194 break;
195 }
196
197 if (result == NULL
198 || result->stackblock_size > curr->stackblock_size)
199 result = curr;
200 }
201 }
202
203 if (__builtin_expect (result == NULL, 0)
204 /* Make sure the size difference is not too excessive. In that
205 case we do not use the block. */
206 || __builtin_expect (result->stackblock_size > 4 * size, 0))
207 {
208 /* Release the lock. */
209 lll_unlock (stack_cache_lock, LLL_PRIVATE);
210
211 return NULL;
212 }
213
214 /* Don't allow setxid until cloned. */
215 result->setxid_futex = -1;
216
217 /* Dequeue the entry. */
218 stack_list_del (&result->list);
219
220 /* And add to the list of stacks in use. */
221 stack_list_add (&result->list, &stack_used);
222
223 /* And decrease the cache size. */
224 stack_cache_actsize -= result->stackblock_size;
225
226 /* Release the lock early. */
227 lll_unlock (stack_cache_lock, LLL_PRIVATE);
228
229 /* Report size and location of the stack to the caller. */
230 *sizep = result->stackblock_size;
231 *memp = result->stackblock;
232
233 /* Cancellation handling is back to the default. */
234 result->cancelhandling = 0;
235 result->cleanup = NULL;
236
237 /* No pending event. */
238 result->nextevent = NULL;
239
240 /* Clear the DTV. */
241 dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
242 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
243 free (dtv[1 + cnt].pointer.to_free);
244 memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
245
246 /* Re-initialize the TLS. */
247 _dl_allocate_tls_init (TLS_TPADJ (result));
248
249 return result;
250}
251
252
253/* Free stacks until cache size is lower than LIMIT. */
254void
255__free_stacks (size_t limit)
256{
257 /* We reduce the size of the cache. Remove the last entries until
258 the size is below the limit. */
259 list_t *entry;
260 list_t *prev;
261
262 /* Search from the end of the list. */
263 list_for_each_prev_safe (entry, prev, &stack_cache)
264 {
265 struct pthread *curr;
266
267 curr = list_entry (entry, struct pthread, list);
268 if (FREE_P (curr))
269 {
270 /* Unlink the block. */
271 stack_list_del (entry);
272
273 /* Account for the freed memory. */
274 stack_cache_actsize -= curr->stackblock_size;
275
276 /* Free the memory associated with the ELF TLS. */
277 _dl_deallocate_tls (TLS_TPADJ (curr), false);
278
279 /* Remove this block. This should never fail. If it does
280 something is really wrong. */
281 if (__munmap (curr->stackblock, curr->stackblock_size) != 0)
282 abort ();
283
284 /* Maybe we have freed enough. */
285 if (stack_cache_actsize <= limit)
286 break;
287 }
288 }
289}
290
291
292/* Add a stack frame which is not used anymore to the stack. Must be
293 called with the cache lock held. */
294static inline void
295__attribute ((always_inline))
296queue_stack (struct pthread *stack)
297{
298 /* We unconditionally add the stack to the list. The memory may
299 still be in use but it will not be reused until the kernel marks
300 the stack as not used anymore. */
301 stack_list_add (&stack->list, &stack_cache);
302
303 stack_cache_actsize += stack->stackblock_size;
304 if (__glibc_unlikely (stack_cache_actsize > stack_cache_maxsize))
305 __free_stacks (stack_cache_maxsize);
306}
307
308
309static int
310change_stack_perm (struct pthread *pd
311#ifdef NEED_SEPARATE_REGISTER_STACK
312 , size_t pagemask
313#endif
314 )
315{
316#ifdef NEED_SEPARATE_REGISTER_STACK
317 void *stack = (pd->stackblock
318 + (((((pd->stackblock_size - pd->guardsize) / 2)
319 & pagemask) + pd->guardsize) & pagemask));
320 size_t len = pd->stackblock + pd->stackblock_size - stack;
321#elif _STACK_GROWS_DOWN
322 void *stack = pd->stackblock + pd->guardsize;
323 size_t len = pd->stackblock_size - pd->guardsize;
324#elif _STACK_GROWS_UP
325 void *stack = pd->stackblock;
326 size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
327#else
328# error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
329#endif
330 if (__mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
331 return errno;
332
333 return 0;
334}
335
336/* Return the guard page position on allocated stack. */
337static inline char *
338__attribute ((always_inline))
339guard_position (void *mem, size_t size, size_t guardsize, struct pthread *pd,
340 size_t pagesize_m1)
341{
342#ifdef NEED_SEPARATE_REGISTER_STACK
343 return mem + (((size - guardsize) / 2) & ~pagesize_m1);
344#elif _STACK_GROWS_DOWN
345 return mem;
346#elif _STACK_GROWS_UP
347 return (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
348#endif
349}
350
351/* Based on stack allocated with PROT_NONE, setup the required portions with
352 'prot' flags based on the guard page position. */
353static inline int
354setup_stack_prot (char *mem, size_t size, char *guard, size_t guardsize,
355 const int prot)
356{
357 char *guardend = guard + guardsize;
358#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
359 /* As defined at guard_position, for architectures with downward stack
360 the guard page is always at start of the allocated area. */
361 if (__mprotect (guardend, size - guardsize, prot) != 0)
362 return errno;
363#else
364 size_t mprots1 = (uintptr_t) guard - (uintptr_t) mem;
365 if (__mprotect (mem, mprots1, prot) != 0)
366 return errno;
367 size_t mprots2 = ((uintptr_t) mem + size) - (uintptr_t) guardend;
368 if (__mprotect (guardend, mprots2, prot) != 0)
369 return errno;
370#endif
371 return 0;
372}
373
374/* Mark the memory of the stack as usable to the kernel. It frees everything
375 except for the space used for the TCB itself. */
376static inline void
377__always_inline
378advise_stack_range (void *mem, size_t size, uintptr_t pd, size_t guardsize)
379{
380 uintptr_t sp = (uintptr_t) CURRENT_STACK_FRAME;
381 size_t pagesize_m1 = __getpagesize () - 1;
382#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
383 size_t freesize = (sp - (uintptr_t) mem) & ~pagesize_m1;
384 assert (freesize < size);
385 if (freesize > PTHREAD_STACK_MIN)
386 __madvise (mem, freesize - PTHREAD_STACK_MIN, MADV_DONTNEED);
387#else
388 /* Page aligned start of memory to free (higher than or equal
389 to current sp plus the minimum stack size). */
390 uintptr_t freeblock = (sp + PTHREAD_STACK_MIN + pagesize_m1) & ~pagesize_m1;
391 uintptr_t free_end = (pd - guardsize) & ~pagesize_m1;
392 if (free_end > freeblock)
393 {
394 size_t freesize = free_end - freeblock;
395 assert (freesize < size);
396 __madvise ((void*) freeblock, freesize, MADV_DONTNEED);
397 }
398#endif
399}
400
401/* Returns a usable stack for a new thread either by allocating a
402 new stack or reusing a cached stack of sufficient size.
403 ATTR must be non-NULL and point to a valid pthread_attr.
404 PDP must be non-NULL. */
405static int
406allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
407 ALLOCATE_STACK_PARMS)
408{
409 struct pthread *pd;
410 size_t size;
411 size_t pagesize_m1 = __getpagesize () - 1;
412
413 assert (powerof2 (pagesize_m1 + 1));
414 assert (TCB_ALIGNMENT >= STACK_ALIGN);
415
416 /* Get the stack size from the attribute if it is set. Otherwise we
417 use the default we determined at start time. */
418 if (attr->stacksize != 0)
419 size = attr->stacksize;
420 else
421 {
422 lll_lock (__default_pthread_attr_lock, LLL_PRIVATE);
423 size = __default_pthread_attr.stacksize;
424 lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
425 }
426
427 /* Get memory for the stack. */
428 if (__glibc_unlikely (attr->flags & ATTR_FLAG_STACKADDR))
429 {
430 uintptr_t adj;
431 char *stackaddr = (char *) attr->stackaddr;
432
433 /* Assume the same layout as the _STACK_GROWS_DOWN case, with struct
434 pthread at the top of the stack block. Later we adjust the guard
435 location and stack address to match the _STACK_GROWS_UP case. */
436 if (_STACK_GROWS_UP)
437 stackaddr += attr->stacksize;
438
439 /* If the user also specified the size of the stack make sure it
440 is large enough. */
441 if (attr->stacksize != 0
442 && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
443 return EINVAL;
444
445 /* Adjust stack size for alignment of the TLS block. */
446#if TLS_TCB_AT_TP
447 adj = ((uintptr_t) stackaddr - TLS_TCB_SIZE)
448 & __static_tls_align_m1;
449 assert (size > adj + TLS_TCB_SIZE);
450#elif TLS_DTV_AT_TP
451 adj = ((uintptr_t) stackaddr - __static_tls_size)
452 & __static_tls_align_m1;
453 assert (size > adj);
454#endif
455
456 /* The user provided some memory. Let's hope it matches the
457 size... We do not allocate guard pages if the user provided
458 the stack. It is the user's responsibility to do this if it
459 is wanted. */
460#if TLS_TCB_AT_TP
461 pd = (struct pthread *) ((uintptr_t) stackaddr
462 - TLS_TCB_SIZE - adj);
463#elif TLS_DTV_AT_TP
464 pd = (struct pthread *) (((uintptr_t) stackaddr
465 - __static_tls_size - adj)
466 - TLS_PRE_TCB_SIZE);
467#endif
468
469 /* The user provided stack memory needs to be cleared. */
470 memset (pd, '\0', sizeof (struct pthread));
471
472 /* The first TSD block is included in the TCB. */
473 pd->specific[0] = pd->specific_1stblock;
474
475 /* Remember the stack-related values. */
476 pd->stackblock = (char *) stackaddr - size;
477 pd->stackblock_size = size;
478
479 /* This is a user-provided stack. It will not be queued in the
480 stack cache nor will the memory (except the TLS memory) be freed. */
481 pd->user_stack = true;
482
483 /* This is at least the second thread. */
484 pd->header.multiple_threads = 1;
485#ifndef TLS_MULTIPLE_THREADS_IN_TCB
486 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
487#endif
488
489#ifndef __ASSUME_PRIVATE_FUTEX
490 /* The thread must know when private futexes are supported. */
491 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
492 header.private_futex);
493#endif
494
495#ifdef NEED_DL_SYSINFO
496 SETUP_THREAD_SYSINFO (pd);
497#endif
498
499 /* Don't allow setxid until cloned. */
500 pd->setxid_futex = -1;
501
502 /* Allocate the DTV for this thread. */
503 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
504 {
505 /* Something went wrong. */
506 assert (errno == ENOMEM);
507 return errno;
508 }
509
510
511 /* Prepare to modify global data. */
512 lll_lock (stack_cache_lock, LLL_PRIVATE);
513
514 /* And add to the list of stacks in use. */
515 list_add (&pd->list, &__stack_user);
516
517 lll_unlock (stack_cache_lock, LLL_PRIVATE);
518 }
519 else
520 {
521 /* Allocate some anonymous memory. If possible use the cache. */
522 size_t guardsize;
523 size_t reqsize;
524 void *mem;
525 const int prot = (PROT_READ | PROT_WRITE
526 | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
527
528 /* Adjust the stack size for alignment. */
529 size &= ~__static_tls_align_m1;
530 assert (size != 0);
531
532 /* Make sure the size of the stack is enough for the guard and
533 eventually the thread descriptor. */
534 guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
535 if (guardsize < attr->guardsize || size + guardsize < guardsize)
536 /* Arithmetic overflow. */
537 return EINVAL;
538 size += guardsize;
539 if (__builtin_expect (size < ((guardsize + __static_tls_size
540 + MINIMAL_REST_STACK + pagesize_m1)
541 & ~pagesize_m1),
542 0))
543 /* The stack is too small (or the guard too large). */
544 return EINVAL;
545
546 /* Try to get a stack from the cache. */
547 reqsize = size;
548 pd = get_cached_stack (&size, &mem);
549 if (pd == NULL)
550 {
551 /* To avoid aliasing effects on a larger scale than pages we
552 adjust the allocated stack size if necessary. This way
553 allocations directly following each other will not have
554 aliasing problems. */
555#if MULTI_PAGE_ALIASING != 0
556 if ((size % MULTI_PAGE_ALIASING) == 0)
557 size += pagesize_m1 + 1;
558#endif
559
560 /* If a guard page is required, avoid committing memory by first
561 allocate with PROT_NONE and then reserve with required permission
562 excluding the guard page. */
563 mem = __mmap (NULL, size, (guardsize == 0) ? prot : PROT_NONE,
564 MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
565
566 if (__glibc_unlikely (mem == MAP_FAILED))
567 return errno;
568
569 /* SIZE is guaranteed to be greater than zero.
570 So we can never get a null pointer back from mmap. */
571 assert (mem != NULL);
572
573 /* Place the thread descriptor at the end of the stack. */
574#if TLS_TCB_AT_TP
575 pd = (struct pthread *) ((char *) mem + size) - 1;
576#elif TLS_DTV_AT_TP
577 pd = (struct pthread *) ((((uintptr_t) mem + size
578 - __static_tls_size)
579 & ~__static_tls_align_m1)
580 - TLS_PRE_TCB_SIZE);
581#endif
582
583 /* Now mprotect the required region excluding the guard area. */
584 if (__glibc_likely (guardsize > 0))
585 {
586 char *guard = guard_position (mem, size, guardsize, pd,
587 pagesize_m1);
588 if (setup_stack_prot (mem, size, guard, guardsize, prot) != 0)
589 {
590 __munmap (mem, size);
591 return errno;
592 }
593 }
594
595 /* Remember the stack-related values. */
596 pd->stackblock = mem;
597 pd->stackblock_size = size;
598 /* Update guardsize for newly allocated guardsize to avoid
599 an mprotect in guard resize below. */
600 pd->guardsize = guardsize;
601
602 /* We allocated the first block thread-specific data array.
603 This address will not change for the lifetime of this
604 descriptor. */
605 pd->specific[0] = pd->specific_1stblock;
606
607 /* This is at least the second thread. */
608 pd->header.multiple_threads = 1;
609#ifndef TLS_MULTIPLE_THREADS_IN_TCB
610 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
611#endif
612
613#ifndef __ASSUME_PRIVATE_FUTEX
614 /* The thread must know when private futexes are supported. */
615 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
616 header.private_futex);
617#endif
618
619#ifdef NEED_DL_SYSINFO
620 SETUP_THREAD_SYSINFO (pd);
621#endif
622
623 /* Don't allow setxid until cloned. */
624 pd->setxid_futex = -1;
625
626 /* Allocate the DTV for this thread. */
627 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
628 {
629 /* Something went wrong. */
630 assert (errno == ENOMEM);
631
632 /* Free the stack memory we just allocated. */
633 (void) __munmap (mem, size);
634
635 return errno;
636 }
637
638
639 /* Prepare to modify global data. */
640 lll_lock (stack_cache_lock, LLL_PRIVATE);
641
642 /* And add to the list of stacks in use. */
643 stack_list_add (&pd->list, &stack_used);
644
645 lll_unlock (stack_cache_lock, LLL_PRIVATE);
646
647
648 /* There might have been a race. Another thread might have
649 caused the stacks to get exec permission while this new
650 stack was prepared. Detect if this was possible and
651 change the permission if necessary. */
652 if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
653 && (prot & PROT_EXEC) == 0, 0))
654 {
655 int err = change_stack_perm (pd
656#ifdef NEED_SEPARATE_REGISTER_STACK
657 , ~pagesize_m1
658#endif
659 );
660 if (err != 0)
661 {
662 /* Free the stack memory we just allocated. */
663 (void) __munmap (mem, size);
664
665 return err;
666 }
667 }
668
669
670 /* Note that all of the stack and the thread descriptor is
671 zeroed. This means we do not have to initialize fields
672 with initial value zero. This is specifically true for
673 the 'tid' field which is always set back to zero once the
674 stack is not used anymore and for the 'guardsize' field
675 which will be read next. */
676 }
677
678 /* Create or resize the guard area if necessary. */
679 if (__glibc_unlikely (guardsize > pd->guardsize))
680 {
681 char *guard = guard_position (mem, size, guardsize, pd,
682 pagesize_m1);
683 if (__mprotect (guard, guardsize, PROT_NONE) != 0)
684 {
685 mprot_error:
686 lll_lock (stack_cache_lock, LLL_PRIVATE);
687
688 /* Remove the thread from the list. */
689 stack_list_del (&pd->list);
690
691 lll_unlock (stack_cache_lock, LLL_PRIVATE);
692
693 /* Get rid of the TLS block we allocated. */
694 _dl_deallocate_tls (TLS_TPADJ (pd), false);
695
696 /* Free the stack memory regardless of whether the size
697 of the cache is over the limit or not. If this piece
698 of memory caused problems we better do not use it
699 anymore. Uh, and we ignore possible errors. There
700 is nothing we could do. */
701 (void) __munmap (mem, size);
702
703 return errno;
704 }
705
706 pd->guardsize = guardsize;
707 }
708 else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
709 0))
710 {
711 /* The old guard area is too large. */
712
713#ifdef NEED_SEPARATE_REGISTER_STACK
714 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
715 char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
716
717 if (oldguard < guard
718 && __mprotect (oldguard, guard - oldguard, prot) != 0)
719 goto mprot_error;
720
721 if (__mprotect (guard + guardsize,
722 oldguard + pd->guardsize - guard - guardsize,
723 prot) != 0)
724 goto mprot_error;
725#elif _STACK_GROWS_DOWN
726 if (__mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
727 prot) != 0)
728 goto mprot_error;
729#elif _STACK_GROWS_UP
730 char *new_guard = (char *)(((uintptr_t) pd - guardsize)
731 & ~pagesize_m1);
732 char *old_guard = (char *)(((uintptr_t) pd - pd->guardsize)
733 & ~pagesize_m1);
734 /* The guard size difference might be > 0, but once rounded
735 to the nearest page the size difference might be zero. */
736 if (new_guard > old_guard
737 && mprotect (old_guard, new_guard - old_guard, prot) != 0)
738 goto mprot_error;
739#endif
740
741 pd->guardsize = guardsize;
742 }
743 /* The pthread_getattr_np() calls need to get passed the size
744 requested in the attribute, regardless of how large the
745 actually used guardsize is. */
746 pd->reported_guardsize = guardsize;
747 }
748
749 /* Initialize the lock. We have to do this unconditionally since the
750 stillborn thread could be canceled while the lock is taken. */
751 pd->lock = LLL_LOCK_INITIALIZER;
752
753 /* The robust mutex lists also need to be initialized
754 unconditionally because the cleanup for the previous stack owner
755 might have happened in the kernel. */
756 pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
757 - offsetof (pthread_mutex_t,
758 __data.__list.__next));
759 pd->robust_head.list_op_pending = NULL;
760#if __PTHREAD_MUTEX_HAVE_PREV
761 pd->robust_prev = &pd->robust_head;
762#endif
763 pd->robust_head.list = &pd->robust_head;
764
765 /* We place the thread descriptor at the end of the stack. */
766 *pdp = pd;
767
768#if _STACK_GROWS_DOWN
769 void *stacktop;
770
771# if TLS_TCB_AT_TP
772 /* The stack begins before the TCB and the static TLS block. */
773 stacktop = ((char *) (pd + 1) - __static_tls_size);
774# elif TLS_DTV_AT_TP
775 stacktop = (char *) (pd - 1);
776# endif
777
778# ifdef NEED_SEPARATE_REGISTER_STACK
779 *stack = pd->stackblock;
780 *stacksize = stacktop - *stack;
781# else
782 *stack = stacktop;
783# endif
784#else
785 *stack = pd->stackblock;
786#endif
787
788 return 0;
789}
790
791
792void
793__deallocate_stack (struct pthread *pd)
794{
795 lll_lock (stack_cache_lock, LLL_PRIVATE);
796
797 /* Remove the thread from the list of threads with user defined
798 stacks. */
799 stack_list_del (&pd->list);
800
801 /* Not much to do. Just free the mmap()ed memory. Note that we do
802 not reset the 'used' flag in the 'tid' field. This is done by
803 the kernel. If no thread has been created yet this field is
804 still zero. */
805 if (__glibc_likely (! pd->user_stack))
806 (void) queue_stack (pd);
807 else
808 /* Free the memory associated with the ELF TLS. */
809 _dl_deallocate_tls (TLS_TPADJ (pd), false);
810
811 lll_unlock (stack_cache_lock, LLL_PRIVATE);
812}
813
814
815int
816__make_stacks_executable (void **stack_endp)
817{
818 /* First the main thread's stack. */
819 int err = _dl_make_stack_executable (stack_endp);
820 if (err != 0)
821 return err;
822
823#ifdef NEED_SEPARATE_REGISTER_STACK
824 const size_t pagemask = ~(__getpagesize () - 1);
825#endif
826
827 lll_lock (stack_cache_lock, LLL_PRIVATE);
828
829 list_t *runp;
830 list_for_each (runp, &stack_used)
831 {
832 err = change_stack_perm (list_entry (runp, struct pthread, list)
833#ifdef NEED_SEPARATE_REGISTER_STACK
834 , pagemask
835#endif
836 );
837 if (err != 0)
838 break;
839 }
840
841 /* Also change the permission for the currently unused stacks. This
842 might be wasted time but better spend it here than adding a check
843 in the fast path. */
844 if (err == 0)
845 list_for_each (runp, &stack_cache)
846 {
847 err = change_stack_perm (list_entry (runp, struct pthread, list)
848#ifdef NEED_SEPARATE_REGISTER_STACK
849 , pagemask
850#endif
851 );
852 if (err != 0)
853 break;
854 }
855
856 lll_unlock (stack_cache_lock, LLL_PRIVATE);
857
858 return err;
859}
860
861
862/* In case of a fork() call the memory allocation in the child will be
863 the same but only one thread is running. All stacks except that of
864 the one running thread are not used anymore. We have to recycle
865 them. */
866void
867__reclaim_stacks (void)
868{
869 struct pthread *self = (struct pthread *) THREAD_SELF;
870
871 /* No locking necessary. The caller is the only stack in use. But
872 we have to be aware that we might have interrupted a list
873 operation. */
874
875 if (in_flight_stack != 0)
876 {
877 bool add_p = in_flight_stack & 1;
878 list_t *elem = (list_t *) (in_flight_stack & ~(uintptr_t) 1);
879
880 if (add_p)
881 {
882 /* We always add at the beginning of the list. So in this case we
883 only need to check the beginning of these lists to see if the
884 pointers at the head of the list are inconsistent. */
885 list_t *l = NULL;
886
887 if (stack_used.next->prev != &stack_used)
888 l = &stack_used;
889 else if (stack_cache.next->prev != &stack_cache)
890 l = &stack_cache;
891
892 if (l != NULL)
893 {
894 assert (l->next->prev == elem);
895 elem->next = l->next;
896 elem->prev = l;
897 l->next = elem;
898 }
899 }
900 else
901 {
902 /* We can simply always replay the delete operation. */
903 elem->next->prev = elem->prev;
904 elem->prev->next = elem->next;
905 }
906 }
907
908 /* Mark all stacks except the still running one as free. */
909 list_t *runp;
910 list_for_each (runp, &stack_used)
911 {
912 struct pthread *curp = list_entry (runp, struct pthread, list);
913 if (curp != self)
914 {
915 /* This marks the stack as free. */
916 curp->tid = 0;
917
918 /* Account for the size of the stack. */
919 stack_cache_actsize += curp->stackblock_size;
920
921 if (curp->specific_used)
922 {
923 /* Clear the thread-specific data. */
924 memset (curp->specific_1stblock, '\0',
925 sizeof (curp->specific_1stblock));
926
927 curp->specific_used = false;
928
929 for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
930 if (curp->specific[cnt] != NULL)
931 {
932 memset (curp->specific[cnt], '\0',
933 sizeof (curp->specific_1stblock));
934
935 /* We have allocated the block which we do not
936 free here so re-set the bit. */
937 curp->specific_used = true;
938 }
939 }
940 }
941 }
942
943 /* Add the stack of all running threads to the cache. */
944 list_splice (&stack_used, &stack_cache);
945
946 /* Remove the entry for the current thread to from the cache list
947 and add it to the list of running threads. Which of the two
948 lists is decided by the user_stack flag. */
949 stack_list_del (&self->list);
950
951 /* Re-initialize the lists for all the threads. */
952 INIT_LIST_HEAD (&stack_used);
953 INIT_LIST_HEAD (&__stack_user);
954
955 if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
956 list_add (&self->list, &__stack_user);
957 else
958 list_add (&self->list, &stack_used);
959
960 /* There is one thread running. */
961 __nptl_nthreads = 1;
962
963 in_flight_stack = 0;
964
965 /* Initialize locks. */
966 stack_cache_lock = LLL_LOCK_INITIALIZER;
967 __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
968}
969
970
971#if HP_TIMING_AVAIL
972# undef __find_thread_by_id
973/* Find a thread given the thread ID. */
974attribute_hidden
975struct pthread *
976__find_thread_by_id (pid_t tid)
977{
978 struct pthread *result = NULL;
979
980 lll_lock (stack_cache_lock, LLL_PRIVATE);
981
982 /* Iterate over the list with system-allocated threads first. */
983 list_t *runp;
984 list_for_each (runp, &stack_used)
985 {
986 struct pthread *curp;
987
988 curp = list_entry (runp, struct pthread, list);
989
990 if (curp->tid == tid)
991 {
992 result = curp;
993 goto out;
994 }
995 }
996
997 /* Now the list with threads using user-allocated stacks. */
998 list_for_each (runp, &__stack_user)
999 {
1000 struct pthread *curp;
1001
1002 curp = list_entry (runp, struct pthread, list);
1003
1004 if (curp->tid == tid)
1005 {
1006 result = curp;
1007 goto out;
1008 }
1009 }
1010
1011 out:
1012 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1013
1014 return result;
1015}
1016#endif
1017
1018
1019#ifdef SIGSETXID
1020static void
1021setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
1022{
1023 int ch;
1024
1025 /* Wait until this thread is cloned. */
1026 if (t->setxid_futex == -1
1027 && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1))
1028 do
1029 futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE);
1030 while (t->setxid_futex == -2);
1031
1032 /* Don't let the thread exit before the setxid handler runs. */
1033 t->setxid_futex = 0;
1034
1035 do
1036 {
1037 ch = t->cancelhandling;
1038
1039 /* If the thread is exiting right now, ignore it. */
1040 if ((ch & EXITING_BITMASK) != 0)
1041 {
1042 /* Release the futex if there is no other setxid in
1043 progress. */
1044 if ((ch & SETXID_BITMASK) == 0)
1045 {
1046 t->setxid_futex = 1;
1047 futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
1048 }
1049 return;
1050 }
1051 }
1052 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
1053 ch | SETXID_BITMASK, ch));
1054}
1055
1056
1057static void
1058setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t)
1059{
1060 int ch;
1061
1062 do
1063 {
1064 ch = t->cancelhandling;
1065 if ((ch & SETXID_BITMASK) == 0)
1066 return;
1067 }
1068 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
1069 ch & ~SETXID_BITMASK, ch));
1070
1071 /* Release the futex just in case. */
1072 t->setxid_futex = 1;
1073 futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
1074}
1075
1076
1077static int
1078setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
1079{
1080 if ((t->cancelhandling & SETXID_BITMASK) == 0)
1081 return 0;
1082
1083 int val;
1084 pid_t pid = __getpid ();
1085 INTERNAL_SYSCALL_DECL (err);
1086 val = INTERNAL_SYSCALL_CALL (tgkill, err, pid, t->tid, SIGSETXID);
1087
1088 /* If this failed, it must have had not started yet or else exited. */
1089 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
1090 {
1091 atomic_increment (&cmdp->cntr);
1092 return 1;
1093 }
1094 else
1095 return 0;
1096}
1097
1098/* Check for consistency across set*id system call results. The abort
1099 should not happen as long as all privileges changes happen through
1100 the glibc wrappers. ERROR must be 0 (no error) or an errno
1101 code. */
1102void
1103attribute_hidden
1104__nptl_setxid_error (struct xid_command *cmdp, int error)
1105{
1106 do
1107 {
1108 int olderror = cmdp->error;
1109 if (olderror == error)
1110 break;
1111 if (olderror != -1)
1112 {
1113 /* Mismatch between current and previous results. Save the
1114 error value to memory so that is not clobbered by the
1115 abort function and preserved in coredumps. */
1116 volatile int xid_err __attribute__((unused)) = error;
1117 abort ();
1118 }
1119 }
1120 while (atomic_compare_and_exchange_bool_acq (&cmdp->error, error, -1));
1121}
1122
1123int
1124attribute_hidden
1125__nptl_setxid (struct xid_command *cmdp)
1126{
1127 int signalled;
1128 int result;
1129 lll_lock (stack_cache_lock, LLL_PRIVATE);
1130
1131 __xidcmd = cmdp;
1132 cmdp->cntr = 0;
1133 cmdp->error = -1;
1134
1135 struct pthread *self = THREAD_SELF;
1136
1137 /* Iterate over the list with system-allocated threads first. */
1138 list_t *runp;
1139 list_for_each (runp, &stack_used)
1140 {
1141 struct pthread *t = list_entry (runp, struct pthread, list);
1142 if (t == self)
1143 continue;
1144
1145 setxid_mark_thread (cmdp, t);
1146 }
1147
1148 /* Now the list with threads using user-allocated stacks. */
1149 list_for_each (runp, &__stack_user)
1150 {
1151 struct pthread *t = list_entry (runp, struct pthread, list);
1152 if (t == self)
1153 continue;
1154
1155 setxid_mark_thread (cmdp, t);
1156 }
1157
1158 /* Iterate until we don't succeed in signalling anyone. That means
1159 we have gotten all running threads, and their children will be
1160 automatically correct once started. */
1161 do
1162 {
1163 signalled = 0;
1164
1165 list_for_each (runp, &stack_used)
1166 {
1167 struct pthread *t = list_entry (runp, struct pthread, list);
1168 if (t == self)
1169 continue;
1170
1171 signalled += setxid_signal_thread (cmdp, t);
1172 }
1173
1174 list_for_each (runp, &__stack_user)
1175 {
1176 struct pthread *t = list_entry (runp, struct pthread, list);
1177 if (t == self)
1178 continue;
1179
1180 signalled += setxid_signal_thread (cmdp, t);
1181 }
1182
1183 int cur = cmdp->cntr;
1184 while (cur != 0)
1185 {
1186 futex_wait_simple ((unsigned int *) &cmdp->cntr, cur,
1187 FUTEX_PRIVATE);
1188 cur = cmdp->cntr;
1189 }
1190 }
1191 while (signalled != 0);
1192
1193 /* Clean up flags, so that no thread blocks during exit waiting
1194 for a signal which will never come. */
1195 list_for_each (runp, &stack_used)
1196 {
1197 struct pthread *t = list_entry (runp, struct pthread, list);
1198 if (t == self)
1199 continue;
1200
1201 setxid_unmark_thread (cmdp, t);
1202 }
1203
1204 list_for_each (runp, &__stack_user)
1205 {
1206 struct pthread *t = list_entry (runp, struct pthread, list);
1207 if (t == self)
1208 continue;
1209
1210 setxid_unmark_thread (cmdp, t);
1211 }
1212
1213 /* This must be last, otherwise the current thread might not have
1214 permissions to send SIGSETXID syscall to the other threads. */
1215 INTERNAL_SYSCALL_DECL (err);
1216 result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
1217 cmdp->id[0], cmdp->id[1], cmdp->id[2]);
1218 int error = 0;
1219 if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, err)))
1220 {
1221 error = INTERNAL_SYSCALL_ERRNO (result, err);
1222 __set_errno (error);
1223 result = -1;
1224 }
1225 __nptl_setxid_error (cmdp, error);
1226
1227 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1228 return result;
1229}
1230#endif /* SIGSETXID. */
1231
1232
1233static inline void __attribute__((always_inline))
1234init_one_static_tls (struct pthread *curp, struct link_map *map)
1235{
1236# if TLS_TCB_AT_TP
1237 void *dest = (char *) curp - map->l_tls_offset;
1238# elif TLS_DTV_AT_TP
1239 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
1240# else
1241# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1242# endif
1243
1244 /* Initialize the memory. */
1245 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
1246 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
1247}
1248
1249void
1250attribute_hidden
1251__pthread_init_static_tls (struct link_map *map)
1252{
1253 lll_lock (stack_cache_lock, LLL_PRIVATE);
1254
1255 /* Iterate over the list with system-allocated threads first. */
1256 list_t *runp;
1257 list_for_each (runp, &stack_used)
1258 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1259
1260 /* Now the list with threads using user-allocated stacks. */
1261 list_for_each (runp, &__stack_user)
1262 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1263
1264 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1265}
1266
1267
1268void
1269attribute_hidden
1270__wait_lookup_done (void)
1271{
1272 lll_lock (stack_cache_lock, LLL_PRIVATE);
1273
1274 struct pthread *self = THREAD_SELF;
1275
1276 /* Iterate over the list with system-allocated threads first. */
1277 list_t *runp;
1278 list_for_each (runp, &stack_used)
1279 {
1280 struct pthread *t = list_entry (runp, struct pthread, list);
1281 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1282 continue;
1283
1284 int *const gscope_flagp = &t->header.gscope_flag;
1285
1286 /* We have to wait until this thread is done with the global
1287 scope. First tell the thread that we are waiting and
1288 possibly have to be woken. */
1289 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1290 THREAD_GSCOPE_FLAG_WAIT,
1291 THREAD_GSCOPE_FLAG_USED))
1292 continue;
1293
1294 do
1295 futex_wait_simple ((unsigned int *) gscope_flagp,
1296 THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
1297 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1298 }
1299
1300 /* Now the list with threads using user-allocated stacks. */
1301 list_for_each (runp, &__stack_user)
1302 {
1303 struct pthread *t = list_entry (runp, struct pthread, list);
1304 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1305 continue;
1306
1307 int *const gscope_flagp = &t->header.gscope_flag;
1308
1309 /* We have to wait until this thread is done with the global
1310 scope. First tell the thread that we are waiting and
1311 possibly have to be woken. */
1312 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1313 THREAD_GSCOPE_FLAG_WAIT,
1314 THREAD_GSCOPE_FLAG_USED))
1315 continue;
1316
1317 do
1318 futex_wait_simple ((unsigned int *) gscope_flagp,
1319 THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
1320 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1321 }
1322
1323 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1324}
1325