1/* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <libintl.h>
22#include <signal.h>
23#include <stdlib.h>
24#include <unistd.h>
25#include <sys/param.h>
26#include <atomic.h>
27
28#include <tls.h>
29#include <dl-tls.h>
30#include <ldsodefs.h>
31
32#if PTHREAD_IN_LIBC
33# include <list.h>
34#endif
35
36#define TUNABLE_NAMESPACE rtld
37#include <dl-tunables.h>
38
39/* Surplus static TLS, GLRO(dl_tls_static_surplus), is used for
40
41 - IE TLS in libc.so for all dlmopen namespaces except in the initial
42 one where libc.so is not loaded dynamically but at startup time,
43 - IE TLS in other libraries which may be dynamically loaded even in the
44 initial namespace,
45 - and optionally for optimizing dynamic TLS access.
46
47 The maximum number of namespaces is DL_NNS, but to support that many
48 namespaces correctly the static TLS allocation should be significantly
49 increased, which may cause problems with small thread stacks due to the
50 way static TLS is accounted (bug 11787).
51
52 So there is a rtld.nns tunable limit on the number of supported namespaces
53 that affects the size of the static TLS and by default it's small enough
54 not to cause problems with existing applications. The limit is not
55 enforced or checked: it is the user's responsibility to increase rtld.nns
56 if more dlmopen namespaces are used.
57
58 Audit modules use their own namespaces, they are not included in rtld.nns,
59 but come on top when computing the number of namespaces. */
60
61/* Size of initial-exec TLS in libc.so. This should be the maximum of
62 observed PT_GNU_TLS sizes across all architectures. Some
63 architectures have lower values due to differences in type sizes
64 and link editor capabilities. */
65#define LIBC_IE_TLS 144
66
67/* Size of initial-exec TLS in libraries other than libc.so.
68 This should be large enough to cover runtime libraries of the
69 compiler such as libgomp and libraries in libc other than libc.so. */
70#define OTHER_IE_TLS 144
71
72/* Default number of namespaces. */
73#define DEFAULT_NNS 4
74
75/* Default for dl_tls_static_optional. */
76#define OPTIONAL_TLS 512
77
78/* Compute the static TLS surplus based on the namespace count and the
79 TLS space that can be used for optimizations. */
80static inline int
81tls_static_surplus (int nns, int opt_tls)
82{
83 return (nns - 1) * LIBC_IE_TLS + nns * OTHER_IE_TLS + opt_tls;
84}
85
86/* This value is chosen so that with default values for the tunables,
87 the computation of dl_tls_static_surplus in
88 _dl_tls_static_surplus_init yields the historic value 1664, for
89 backwards compatibility. */
90#define LEGACY_TLS (1664 - tls_static_surplus (DEFAULT_NNS, OPTIONAL_TLS))
91
92/* Calculate the size of the static TLS surplus, when the given
93 number of audit modules are loaded. Must be called after the
94 number of audit modules is known and before static TLS allocation. */
95void
96_dl_tls_static_surplus_init (size_t naudit)
97{
98 size_t nns, opt_tls;
99
100#if HAVE_TUNABLES
101 nns = TUNABLE_GET (nns, size_t, NULL);
102 opt_tls = TUNABLE_GET (optional_static_tls, size_t, NULL);
103#else
104 /* Default values of the tunables. */
105 nns = DEFAULT_NNS;
106 opt_tls = OPTIONAL_TLS;
107#endif
108 if (nns > DL_NNS)
109 nns = DL_NNS;
110 if (DL_NNS - nns < naudit)
111 _dl_fatal_printf ("Failed loading %lu audit modules, %lu are supported.\n",
112 (unsigned long) naudit, (unsigned long) (DL_NNS - nns));
113 nns += naudit;
114
115 GL(dl_tls_static_optional) = opt_tls;
116 assert (LEGACY_TLS >= 0);
117 GLRO(dl_tls_static_surplus) = tls_static_surplus (nns, opt_tls) + LEGACY_TLS;
118}
119
120/* Out-of-memory handler. */
121static void
122__attribute__ ((__noreturn__))
123oom (void)
124{
125 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
126}
127
128
129void
130_dl_assign_tls_modid (struct link_map *l)
131{
132 size_t result;
133
134 if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
135 {
136 size_t disp = 0;
137 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
138
139 /* Note that this branch will never be executed during program
140 start since there are no gaps at that time. Therefore it
141 does not matter that the dl_tls_dtv_slotinfo is not allocated
142 yet when the function is called for the first times.
143
144 NB: the offset +1 is due to the fact that DTV[0] is used
145 for something else. */
146 result = GL(dl_tls_static_nelem) + 1;
147 if (result <= GL(dl_tls_max_dtv_idx))
148 do
149 {
150 while (result - disp < runp->len)
151 {
152 if (runp->slotinfo[result - disp].map == NULL)
153 break;
154
155 ++result;
156 assert (result <= GL(dl_tls_max_dtv_idx) + 1);
157 }
158
159 if (result - disp < runp->len)
160 {
161 /* Mark the entry as used, so any dependency see it. */
162 atomic_store_relaxed (&runp->slotinfo[result - disp].map, l);
163 break;
164 }
165
166 disp += runp->len;
167 }
168 while ((runp = runp->next) != NULL);
169
170 if (result > GL(dl_tls_max_dtv_idx))
171 {
172 /* The new index must indeed be exactly one higher than the
173 previous high. */
174 assert (result == GL(dl_tls_max_dtv_idx) + 1);
175 /* There is no gap anymore. */
176 GL(dl_tls_dtv_gaps) = false;
177
178 goto nogaps;
179 }
180 }
181 else
182 {
183 /* No gaps, allocate a new entry. */
184 nogaps:
185
186 result = GL(dl_tls_max_dtv_idx) + 1;
187 /* Can be read concurrently. */
188 atomic_store_relaxed (&GL(dl_tls_max_dtv_idx), result);
189 }
190
191 l->l_tls_modid = result;
192}
193
194
195size_t
196_dl_count_modids (void)
197{
198 /* The count is the max unless dlclose or failed dlopen created gaps. */
199 if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
200 return GL(dl_tls_max_dtv_idx);
201
202 /* We have gaps and are forced to count the non-NULL entries. */
203 size_t n = 0;
204 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
205 while (runp != NULL)
206 {
207 for (size_t i = 0; i < runp->len; ++i)
208 if (runp->slotinfo[i].map != NULL)
209 ++n;
210
211 runp = runp->next;
212 }
213
214 return n;
215}
216
217
218#ifdef SHARED
219void
220_dl_determine_tlsoffset (void)
221{
222 size_t max_align = TCB_ALIGNMENT;
223 size_t freetop = 0;
224 size_t freebottom = 0;
225
226 /* The first element of the dtv slot info list is allocated. */
227 assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
228 /* There is at this point only one element in the
229 dl_tls_dtv_slotinfo_list list. */
230 assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
231
232 struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
233
234 /* Determining the offset of the various parts of the static TLS
235 block has several dependencies. In addition we have to work
236 around bugs in some toolchains.
237
238 Each TLS block from the objects available at link time has a size
239 and an alignment requirement. The GNU ld computes the alignment
240 requirements for the data at the positions *in the file*, though.
241 I.e, it is not simply possible to allocate a block with the size
242 of the TLS program header entry. The data is layed out assuming
243 that the first byte of the TLS block fulfills
244
245 p_vaddr mod p_align == &TLS_BLOCK mod p_align
246
247 This means we have to add artificial padding at the beginning of
248 the TLS block. These bytes are never used for the TLS data in
249 this module but the first byte allocated must be aligned
250 according to mod p_align == 0 so that the first byte of the TLS
251 block is aligned according to p_vaddr mod p_align. This is ugly
252 and the linker can help by computing the offsets in the TLS block
253 assuming the first byte of the TLS block is aligned according to
254 p_align.
255
256 The extra space which might be allocated before the first byte of
257 the TLS block need not go unused. The code below tries to use
258 that memory for the next TLS block. This can work if the total
259 memory requirement for the next TLS block is smaller than the
260 gap. */
261
262#if TLS_TCB_AT_TP
263 /* We simply start with zero. */
264 size_t offset = 0;
265
266 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
267 {
268 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
269
270 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
271 & (slotinfo[cnt].map->l_tls_align - 1));
272 size_t off;
273 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
274
275 if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
276 {
277 off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
278 - firstbyte, slotinfo[cnt].map->l_tls_align)
279 + firstbyte;
280 if (off <= freebottom)
281 {
282 freetop = off;
283
284 /* XXX For some architectures we perhaps should store the
285 negative offset. */
286 slotinfo[cnt].map->l_tls_offset = off;
287 continue;
288 }
289 }
290
291 off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
292 slotinfo[cnt].map->l_tls_align) + firstbyte;
293 if (off > offset + slotinfo[cnt].map->l_tls_blocksize
294 + (freebottom - freetop))
295 {
296 freetop = offset;
297 freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
298 }
299 offset = off;
300
301 /* XXX For some architectures we perhaps should store the
302 negative offset. */
303 slotinfo[cnt].map->l_tls_offset = off;
304 }
305
306 GL(dl_tls_static_used) = offset;
307 GLRO (dl_tls_static_size) = (roundup (offset + GLRO(dl_tls_static_surplus),
308 max_align)
309 + TLS_TCB_SIZE);
310#elif TLS_DTV_AT_TP
311 /* The TLS blocks start right after the TCB. */
312 size_t offset = TLS_TCB_SIZE;
313
314 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
315 {
316 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
317
318 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
319 & (slotinfo[cnt].map->l_tls_align - 1));
320 size_t off;
321 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
322
323 if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
324 {
325 off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
326 if (off - freebottom < firstbyte)
327 off += slotinfo[cnt].map->l_tls_align;
328 if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
329 {
330 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
331 freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
332 - firstbyte);
333 continue;
334 }
335 }
336
337 off = roundup (offset, slotinfo[cnt].map->l_tls_align);
338 if (off - offset < firstbyte)
339 off += slotinfo[cnt].map->l_tls_align;
340
341 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
342 if (off - firstbyte - offset > freetop - freebottom)
343 {
344 freebottom = offset;
345 freetop = off - firstbyte;
346 }
347
348 offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
349 }
350
351 GL(dl_tls_static_used) = offset;
352 GLRO (dl_tls_static_size) = roundup (offset + GLRO(dl_tls_static_surplus),
353 TCB_ALIGNMENT);
354#else
355# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
356#endif
357
358 /* The alignment requirement for the static TLS block. */
359 GLRO (dl_tls_static_align) = max_align;
360}
361#endif /* SHARED */
362
363static void *
364allocate_dtv (void *result)
365{
366 dtv_t *dtv;
367 size_t dtv_length;
368
369 /* Relaxed MO, because the dtv size is later rechecked, not relied on. */
370 size_t max_modid = atomic_load_relaxed (&GL(dl_tls_max_dtv_idx));
371 /* We allocate a few more elements in the dtv than are needed for the
372 initial set of modules. This should avoid in most cases expansions
373 of the dtv. */
374 dtv_length = max_modid + DTV_SURPLUS;
375 dtv = calloc (dtv_length + 2, sizeof (dtv_t));
376 if (dtv != NULL)
377 {
378 /* This is the initial length of the dtv. */
379 dtv[0].counter = dtv_length;
380
381 /* The rest of the dtv (including the generation counter) is
382 Initialize with zero to indicate nothing there. */
383
384 /* Add the dtv to the thread data structures. */
385 INSTALL_DTV (result, dtv);
386 }
387 else
388 result = NULL;
389
390 return result;
391}
392
393/* Get size and alignment requirements of the static TLS block. This
394 function is no longer used by glibc itself, but the GCC sanitizers
395 use it despite the GLIBC_PRIVATE status. */
396void
397_dl_get_tls_static_info (size_t *sizep, size_t *alignp)
398{
399 *sizep = GLRO (dl_tls_static_size);
400 *alignp = GLRO (dl_tls_static_align);
401}
402
403/* Derive the location of the pointer to the start of the original
404 allocation (before alignment) from the pointer to the TCB. */
405static inline void **
406tcb_to_pointer_to_free_location (void *tcb)
407{
408#if TLS_TCB_AT_TP
409 /* The TCB follows the TLS blocks, and the pointer to the front
410 follows the TCB. */
411 void **original_pointer_location = tcb + TLS_TCB_SIZE;
412#elif TLS_DTV_AT_TP
413 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
414 before that. */
415 void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *);
416#endif
417 return original_pointer_location;
418}
419
420void *
421_dl_allocate_tls_storage (void)
422{
423 void *result;
424 size_t size = GLRO (dl_tls_static_size);
425
426#if TLS_DTV_AT_TP
427 /* Memory layout is:
428 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
429 ^ This should be returned. */
430 size += TLS_PRE_TCB_SIZE;
431#endif
432
433 /* Perform the allocation. Reserve space for the required alignment
434 and the pointer to the original allocation. */
435 size_t alignment = GLRO (dl_tls_static_align);
436 void *allocated = malloc (size + alignment + sizeof (void *));
437 if (__glibc_unlikely (allocated == NULL))
438 return NULL;
439
440 /* Perform alignment and allocate the DTV. */
441#if TLS_TCB_AT_TP
442 /* The TCB follows the TLS blocks, which determine the alignment.
443 (TCB alignment requirements have been taken into account when
444 calculating GLRO (dl_tls_static_align).) */
445 void *aligned = (void *) roundup ((uintptr_t) allocated, alignment);
446 result = aligned + size - TLS_TCB_SIZE;
447
448 /* Clear the TCB data structure. We can't ask the caller (i.e.
449 libpthread) to do it, because we will initialize the DTV et al. */
450 memset (result, '\0', TLS_TCB_SIZE);
451#elif TLS_DTV_AT_TP
452 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
453 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
454 TLS block alignment, and not just the TLS blocks after it. This
455 can leave an unused alignment gap between the TCB and the TLS
456 blocks. */
457 result = (void *) roundup
458 (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated,
459 alignment);
460
461 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
462 it. We can't ask the caller (i.e. libpthread) to do it, because
463 we will initialize the DTV et al. */
464 memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
465#endif
466
467 /* Record the value of the original pointer for later
468 deallocation. */
469 *tcb_to_pointer_to_free_location (result) = allocated;
470
471 result = allocate_dtv (result);
472 if (result == NULL)
473 free (allocated);
474 return result;
475}
476
477
478#ifndef SHARED
479extern dtv_t _dl_static_dtv[];
480# define _dl_initial_dtv (&_dl_static_dtv[1])
481#endif
482
483static dtv_t *
484_dl_resize_dtv (dtv_t *dtv, size_t max_modid)
485{
486 /* Resize the dtv. */
487 dtv_t *newp;
488 size_t newsize = max_modid + DTV_SURPLUS;
489 size_t oldsize = dtv[-1].counter;
490
491 if (dtv == GL(dl_initial_dtv))
492 {
493 /* This is the initial dtv that was either statically allocated in
494 __libc_setup_tls or allocated during rtld startup using the
495 dl-minimal.c malloc instead of the real malloc. We can't free
496 it, we have to abandon the old storage. */
497
498 newp = malloc ((2 + newsize) * sizeof (dtv_t));
499 if (newp == NULL)
500 oom ();
501 memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
502 }
503 else
504 {
505 newp = realloc (&dtv[-1],
506 (2 + newsize) * sizeof (dtv_t));
507 if (newp == NULL)
508 oom ();
509 }
510
511 newp[0].counter = newsize;
512
513 /* Clear the newly allocated part. */
514 memset (newp + 2 + oldsize, '\0',
515 (newsize - oldsize) * sizeof (dtv_t));
516
517 /* Return the generation counter. */
518 return &newp[1];
519}
520
521
522/* Allocate initial TLS. RESULT should be a non-NULL pointer to storage
523 for the TLS space. The DTV may be resized, and so this function may
524 call malloc to allocate that space. The loader's GL(dl_load_tls_lock)
525 is taken when manipulating global TLS-related data in the loader. */
526void *
527_dl_allocate_tls_init (void *result, bool init_tls)
528{
529 if (result == NULL)
530 /* The memory allocation failed. */
531 return NULL;
532
533 dtv_t *dtv = GET_DTV (result);
534 struct dtv_slotinfo_list *listp;
535 size_t total = 0;
536 size_t maxgen = 0;
537
538 /* Protects global dynamic TLS related state. */
539 __rtld_lock_lock_recursive (GL(dl_load_tls_lock));
540
541 /* Check if the current dtv is big enough. */
542 if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
543 {
544 /* Resize the dtv. */
545 dtv = _dl_resize_dtv (dtv, GL(dl_tls_max_dtv_idx));
546
547 /* Install this new dtv in the thread data structures. */
548 INSTALL_DTV (result, &dtv[-1]);
549 }
550
551 /* We have to prepare the dtv for all currently loaded modules using
552 TLS. For those which are dynamically loaded we add the values
553 indicating deferred allocation. */
554 listp = GL(dl_tls_dtv_slotinfo_list);
555 while (1)
556 {
557 size_t cnt;
558
559 for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
560 {
561 struct link_map *map;
562 void *dest;
563
564 /* Check for the total number of used slots. */
565 if (total + cnt > GL(dl_tls_max_dtv_idx))
566 break;
567
568 map = listp->slotinfo[cnt].map;
569 if (map == NULL)
570 /* Unused entry. */
571 continue;
572
573 /* Keep track of the maximum generation number. This might
574 not be the generation counter. */
575 assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
576 maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
577
578 dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
579 dtv[map->l_tls_modid].pointer.to_free = NULL;
580
581 if (map->l_tls_offset == NO_TLS_OFFSET
582 || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
583 continue;
584
585 assert (map->l_tls_modid == total + cnt);
586 assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
587#if TLS_TCB_AT_TP
588 assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
589 dest = (char *) result - map->l_tls_offset;
590#elif TLS_DTV_AT_TP
591 dest = (char *) result + map->l_tls_offset;
592#else
593# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
594#endif
595
596 /* Set up the DTV entry. The simplified __tls_get_addr that
597 some platforms use in static programs requires it. */
598 dtv[map->l_tls_modid].pointer.val = dest;
599
600 /* Copy the initialization image and clear the BSS part. For
601 audit modules or dependencies with initial-exec TLS, we can not
602 set the initial TLS image on default loader initialization
603 because it would already be set by the audit setup. However,
604 subsequent thread creation would need to follow the default
605 behaviour. */
606 if (map->l_ns != LM_ID_BASE && !init_tls)
607 continue;
608 memset (__mempcpy (dest, map->l_tls_initimage,
609 map->l_tls_initimage_size), '\0',
610 map->l_tls_blocksize - map->l_tls_initimage_size);
611 }
612
613 total += cnt;
614 if (total > GL(dl_tls_max_dtv_idx))
615 break;
616
617 listp = listp->next;
618 assert (listp != NULL);
619 }
620 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock));
621
622 /* The DTV version is up-to-date now. */
623 dtv[0].counter = maxgen;
624
625 return result;
626}
627rtld_hidden_def (_dl_allocate_tls_init)
628
629void *
630_dl_allocate_tls (void *mem)
631{
632 return _dl_allocate_tls_init (mem == NULL
633 ? _dl_allocate_tls_storage ()
634 : allocate_dtv (mem), true);
635}
636rtld_hidden_def (_dl_allocate_tls)
637
638
639void
640_dl_deallocate_tls (void *tcb, bool dealloc_tcb)
641{
642 dtv_t *dtv = GET_DTV (tcb);
643
644 /* We need to free the memory allocated for non-static TLS. */
645 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
646 free (dtv[1 + cnt].pointer.to_free);
647
648 /* The array starts with dtv[-1]. */
649 if (dtv != GL(dl_initial_dtv))
650 free (dtv - 1);
651
652 if (dealloc_tcb)
653 free (*tcb_to_pointer_to_free_location (tcb));
654}
655rtld_hidden_def (_dl_deallocate_tls)
656
657
658#ifdef SHARED
659/* The __tls_get_addr function has two basic forms which differ in the
660 arguments. The IA-64 form takes two parameters, the module ID and
661 offset. The form used, among others, on IA-32 takes a reference to
662 a special structure which contain the same information. The second
663 form seems to be more often used (in the moment) so we default to
664 it. Users of the IA-64 form have to provide adequate definitions
665 of the following macros. */
666# ifndef GET_ADDR_ARGS
667# define GET_ADDR_ARGS tls_index *ti
668# define GET_ADDR_PARAM ti
669# endif
670# ifndef GET_ADDR_MODULE
671# define GET_ADDR_MODULE ti->ti_module
672# endif
673# ifndef GET_ADDR_OFFSET
674# define GET_ADDR_OFFSET ti->ti_offset
675# endif
676
677/* Allocate one DTV entry. */
678static struct dtv_pointer
679allocate_dtv_entry (size_t alignment, size_t size)
680{
681 if (powerof2 (alignment) && alignment <= _Alignof (max_align_t))
682 {
683 /* The alignment is supported by malloc. */
684 void *ptr = malloc (size);
685 return (struct dtv_pointer) { ptr, ptr };
686 }
687
688 /* Emulate memalign to by manually aligning a pointer returned by
689 malloc. First compute the size with an overflow check. */
690 size_t alloc_size = size + alignment;
691 if (alloc_size < size)
692 return (struct dtv_pointer) {};
693
694 /* Perform the allocation. This is the pointer we need to free
695 later. */
696 void *start = malloc (alloc_size);
697 if (start == NULL)
698 return (struct dtv_pointer) {};
699
700 /* Find the aligned position within the larger allocation. */
701 void *aligned = (void *) roundup ((uintptr_t) start, alignment);
702
703 return (struct dtv_pointer) { .val = aligned, .to_free = start };
704}
705
706static struct dtv_pointer
707allocate_and_init (struct link_map *map)
708{
709 struct dtv_pointer result = allocate_dtv_entry
710 (map->l_tls_align, map->l_tls_blocksize);
711 if (result.val == NULL)
712 oom ();
713
714 /* Initialize the memory. */
715 memset (__mempcpy (result.val, map->l_tls_initimage,
716 map->l_tls_initimage_size),
717 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
718
719 return result;
720}
721
722
723struct link_map *
724_dl_update_slotinfo (unsigned long int req_modid)
725{
726 struct link_map *the_map = NULL;
727 dtv_t *dtv = THREAD_DTV ();
728
729 /* The global dl_tls_dtv_slotinfo array contains for each module
730 index the generation counter current when the entry was created.
731 This array never shrinks so that all module indices which were
732 valid at some time can be used to access it. Before the first
733 use of a new module index in this function the array was extended
734 appropriately. Access also does not have to be guarded against
735 modifications of the array. It is assumed that pointer-size
736 values can be read atomically even in SMP environments. It is
737 possible that other threads at the same time dynamically load
738 code and therefore add to the slotinfo list. This is a problem
739 since we must not pick up any information about incomplete work.
740 The solution to this is to ignore all dtv slots which were
741 created after the one we are currently interested. We know that
742 dynamic loading for this module is completed and this is the last
743 load operation we know finished. */
744 unsigned long int idx = req_modid;
745 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
746
747 while (idx >= listp->len)
748 {
749 idx -= listp->len;
750 listp = listp->next;
751 }
752
753 if (dtv[0].counter < listp->slotinfo[idx].gen)
754 {
755 /* CONCURRENCY NOTES:
756
757 Here the dtv needs to be updated to new_gen generation count.
758
759 This code may be called during TLS access when GL(dl_load_tls_lock)
760 is not held. In that case the user code has to synchronize with
761 dlopen and dlclose calls of relevant modules. A module m is
762 relevant if the generation of m <= new_gen and dlclose of m is
763 synchronized: a memory access here happens after the dlopen and
764 before the dlclose of relevant modules. The dtv entries for
765 relevant modules need to be updated, other entries can be
766 arbitrary.
767
768 This e.g. means that the first part of the slotinfo list can be
769 accessed race free, but the tail may be concurrently extended.
770 Similarly relevant slotinfo entries can be read race free, but
771 other entries are racy. However updating a non-relevant dtv
772 entry does not affect correctness. For a relevant module m,
773 max_modid >= modid of m. */
774 size_t new_gen = listp->slotinfo[idx].gen;
775 size_t total = 0;
776 size_t max_modid = atomic_load_relaxed (&GL(dl_tls_max_dtv_idx));
777 assert (max_modid >= req_modid);
778
779 /* We have to look through the entire dtv slotinfo list. */
780 listp = GL(dl_tls_dtv_slotinfo_list);
781 do
782 {
783 for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
784 {
785 size_t modid = total + cnt;
786
787 /* Later entries are not relevant. */
788 if (modid > max_modid)
789 break;
790
791 size_t gen = atomic_load_relaxed (&listp->slotinfo[cnt].gen);
792
793 if (gen > new_gen)
794 /* Not relevant. */
795 continue;
796
797 /* If the entry is older than the current dtv layout we
798 know we don't have to handle it. */
799 if (gen <= dtv[0].counter)
800 continue;
801
802 /* If there is no map this means the entry is empty. */
803 struct link_map *map
804 = atomic_load_relaxed (&listp->slotinfo[cnt].map);
805 /* Check whether the current dtv array is large enough. */
806 if (dtv[-1].counter < modid)
807 {
808 if (map == NULL)
809 continue;
810
811 /* Resize the dtv. */
812 dtv = _dl_resize_dtv (dtv, max_modid);
813
814 assert (modid <= dtv[-1].counter);
815
816 /* Install this new dtv in the thread data
817 structures. */
818 INSTALL_NEW_DTV (dtv);
819 }
820
821 /* If there is currently memory allocate for this
822 dtv entry free it. */
823 /* XXX Ideally we will at some point create a memory
824 pool. */
825 free (dtv[modid].pointer.to_free);
826 dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
827 dtv[modid].pointer.to_free = NULL;
828
829 if (modid == req_modid)
830 the_map = map;
831 }
832
833 total += listp->len;
834 if (total > max_modid)
835 break;
836
837 /* Synchronize with _dl_add_to_slotinfo. Ideally this would
838 be consume MO since we only need to order the accesses to
839 the next node after the read of the address and on most
840 hardware (other than alpha) a normal load would do that
841 because of the address dependency. */
842 listp = atomic_load_acquire (&listp->next);
843 }
844 while (listp != NULL);
845
846 /* This will be the new maximum generation counter. */
847 dtv[0].counter = new_gen;
848 }
849
850 return the_map;
851}
852
853
854static void *
855__attribute_noinline__
856tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
857{
858 /* The allocation was deferred. Do it now. */
859 if (the_map == NULL)
860 {
861 /* Find the link map for this module. */
862 size_t idx = GET_ADDR_MODULE;
863 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
864
865 while (idx >= listp->len)
866 {
867 idx -= listp->len;
868 listp = listp->next;
869 }
870
871 the_map = listp->slotinfo[idx].map;
872 }
873
874 /* Make sure that, if a dlopen running in parallel forces the
875 variable into static storage, we'll wait until the address in the
876 static TLS block is set up, and use that. If we're undecided
877 yet, make sure we make the decision holding the lock as well. */
878 if (__glibc_unlikely (the_map->l_tls_offset
879 != FORCED_DYNAMIC_TLS_OFFSET))
880 {
881 __rtld_lock_lock_recursive (GL(dl_load_tls_lock));
882 if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
883 {
884 the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
885 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock));
886 }
887 else if (__glibc_likely (the_map->l_tls_offset
888 != FORCED_DYNAMIC_TLS_OFFSET))
889 {
890#if TLS_TCB_AT_TP
891 void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
892#elif TLS_DTV_AT_TP
893 void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
894#else
895# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
896#endif
897 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock));
898
899 dtv[GET_ADDR_MODULE].pointer.to_free = NULL;
900 dtv[GET_ADDR_MODULE].pointer.val = p;
901
902 return (char *) p + GET_ADDR_OFFSET;
903 }
904 else
905 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock));
906 }
907 struct dtv_pointer result = allocate_and_init (the_map);
908 dtv[GET_ADDR_MODULE].pointer = result;
909 assert (result.to_free != NULL);
910
911 return (char *) result.val + GET_ADDR_OFFSET;
912}
913
914
915static struct link_map *
916__attribute_noinline__
917update_get_addr (GET_ADDR_ARGS)
918{
919 struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
920 dtv_t *dtv = THREAD_DTV ();
921
922 void *p = dtv[GET_ADDR_MODULE].pointer.val;
923
924 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
925 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
926
927 return (void *) p + GET_ADDR_OFFSET;
928}
929
930/* For all machines that have a non-macro version of __tls_get_addr, we
931 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
932 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
933 in ld.so for __tls_get_addr. */
934
935#ifndef __tls_get_addr
936extern void * __tls_get_addr (GET_ADDR_ARGS);
937rtld_hidden_proto (__tls_get_addr)
938rtld_hidden_def (__tls_get_addr)
939#endif
940
941/* The generic dynamic and local dynamic model cannot be used in
942 statically linked applications. */
943void *
944__tls_get_addr (GET_ADDR_ARGS)
945{
946 dtv_t *dtv = THREAD_DTV ();
947
948 /* Update is needed if dtv[0].counter < the generation of the accessed
949 module. The global generation counter is used here as it is easier
950 to check. Synchronization for the relaxed MO access is guaranteed
951 by user code, see CONCURRENCY NOTES in _dl_update_slotinfo. */
952 size_t gen = atomic_load_relaxed (&GL(dl_tls_generation));
953 if (__glibc_unlikely (dtv[0].counter != gen))
954 return update_get_addr (GET_ADDR_PARAM);
955
956 void *p = dtv[GET_ADDR_MODULE].pointer.val;
957
958 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
959 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
960
961 return (char *) p + GET_ADDR_OFFSET;
962}
963#endif
964
965
966/* Look up the module's TLS block as for __tls_get_addr,
967 but never touch anything. Return null if it's not allocated yet. */
968void *
969_dl_tls_get_addr_soft (struct link_map *l)
970{
971 if (__glibc_unlikely (l->l_tls_modid == 0))
972 /* This module has no TLS segment. */
973 return NULL;
974
975 dtv_t *dtv = THREAD_DTV ();
976 /* This may be called without holding the GL(dl_load_tls_lock). Reading
977 arbitrary gen value is fine since this is best effort code. */
978 size_t gen = atomic_load_relaxed (&GL(dl_tls_generation));
979 if (__glibc_unlikely (dtv[0].counter != gen))
980 {
981 /* This thread's DTV is not completely current,
982 but it might already cover this module. */
983
984 if (l->l_tls_modid >= dtv[-1].counter)
985 /* Nope. */
986 return NULL;
987
988 size_t idx = l->l_tls_modid;
989 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
990 while (idx >= listp->len)
991 {
992 idx -= listp->len;
993 listp = listp->next;
994 }
995
996 /* We've reached the slot for this module.
997 If its generation counter is higher than the DTV's,
998 this thread does not know about this module yet. */
999 if (dtv[0].counter < listp->slotinfo[idx].gen)
1000 return NULL;
1001 }
1002
1003 void *data = dtv[l->l_tls_modid].pointer.val;
1004 if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
1005 /* The DTV is current, but this thread has not yet needed
1006 to allocate this module's segment. */
1007 data = NULL;
1008
1009 return data;
1010}
1011
1012
1013void
1014_dl_add_to_slotinfo (struct link_map *l, bool do_add)
1015{
1016 /* Now that we know the object is loaded successfully add
1017 modules containing TLS data to the dtv info table. We
1018 might have to increase its size. */
1019 struct dtv_slotinfo_list *listp;
1020 struct dtv_slotinfo_list *prevp;
1021 size_t idx = l->l_tls_modid;
1022
1023 /* Find the place in the dtv slotinfo list. */
1024 listp = GL(dl_tls_dtv_slotinfo_list);
1025 prevp = NULL; /* Needed to shut up gcc. */
1026 do
1027 {
1028 /* Does it fit in the array of this list element? */
1029 if (idx < listp->len)
1030 break;
1031 idx -= listp->len;
1032 prevp = listp;
1033 listp = listp->next;
1034 }
1035 while (listp != NULL);
1036
1037 if (listp == NULL)
1038 {
1039 /* When we come here it means we have to add a new element
1040 to the slotinfo list. And the new module must be in
1041 the first slot. */
1042 assert (idx == 0);
1043
1044 listp = (struct dtv_slotinfo_list *)
1045 malloc (sizeof (struct dtv_slotinfo_list)
1046 + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
1047 if (listp == NULL)
1048 {
1049 /* We ran out of memory while resizing the dtv slotinfo list. */
1050 _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
1051cannot create TLS data structures"));
1052 }
1053
1054 listp->len = TLS_SLOTINFO_SURPLUS;
1055 listp->next = NULL;
1056 memset (listp->slotinfo, '\0',
1057 TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
1058 /* Synchronize with _dl_update_slotinfo. */
1059 atomic_store_release (&prevp->next, listp);
1060 }
1061
1062 /* Add the information into the slotinfo data structure. */
1063 if (do_add)
1064 {
1065 /* Can be read concurrently. See _dl_update_slotinfo. */
1066 atomic_store_relaxed (&listp->slotinfo[idx].map, l);
1067 atomic_store_relaxed (&listp->slotinfo[idx].gen,
1068 GL(dl_tls_generation) + 1);
1069 }
1070}
1071
1072#if PTHREAD_IN_LIBC
1073static inline void __attribute__((always_inline))
1074init_one_static_tls (struct pthread *curp, struct link_map *map)
1075{
1076# if TLS_TCB_AT_TP
1077 void *dest = (char *) curp - map->l_tls_offset;
1078# elif TLS_DTV_AT_TP
1079 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
1080# else
1081# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1082# endif
1083
1084 /* Initialize the memory. */
1085 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
1086 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
1087}
1088
1089void
1090_dl_init_static_tls (struct link_map *map)
1091{
1092 lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
1093
1094 /* Iterate over the list with system-allocated threads first. */
1095 list_t *runp;
1096 list_for_each (runp, &GL (dl_stack_used))
1097 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1098
1099 /* Now the list with threads using user-allocated stacks. */
1100 list_for_each (runp, &GL (dl_stack_user))
1101 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1102
1103 lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
1104}
1105#endif /* PTHREAD_IN_LIBC */
1106