1/* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <libintl.h>
22#include <signal.h>
23#include <stdlib.h>
24#include <unistd.h>
25#include <sys/param.h>
26#include <atomic.h>
27
28#include <tls.h>
29#include <dl-tls.h>
30#include <ldsodefs.h>
31
32#define TUNABLE_NAMESPACE rtld
33#include <dl-tunables.h>
34
35/* Surplus static TLS, GLRO(dl_tls_static_surplus), is used for
36
37 - IE TLS in libc.so for all dlmopen namespaces except in the initial
38 one where libc.so is not loaded dynamically but at startup time,
39 - IE TLS in other libraries which may be dynamically loaded even in the
40 initial namespace,
41 - and optionally for optimizing dynamic TLS access.
42
43 The maximum number of namespaces is DL_NNS, but to support that many
44 namespaces correctly the static TLS allocation should be significantly
45 increased, which may cause problems with small thread stacks due to the
46 way static TLS is accounted (bug 11787).
47
48 So there is a rtld.nns tunable limit on the number of supported namespaces
49 that affects the size of the static TLS and by default it's small enough
50 not to cause problems with existing applications. The limit is not
51 enforced or checked: it is the user's responsibility to increase rtld.nns
52 if more dlmopen namespaces are used.
53
54 Audit modules use their own namespaces, they are not included in rtld.nns,
55 but come on top when computing the number of namespaces. */
56
57/* Size of initial-exec TLS in libc.so. This should be the maximum of
58 observed PT_GNU_TLS sizes across all architectures. Some
59 architectures have lower values due to differences in type sizes
60 and link editor capabilities. */
61#define LIBC_IE_TLS 144
62
63/* Size of initial-exec TLS in libraries other than libc.so.
64 This should be large enough to cover runtime libraries of the
65 compiler such as libgomp and libraries in libc other than libc.so. */
66#define OTHER_IE_TLS 144
67
68/* Default number of namespaces. */
69#define DEFAULT_NNS 4
70
71/* Default for dl_tls_static_optional. */
72#define OPTIONAL_TLS 512
73
74/* Compute the static TLS surplus based on the namespace count and the
75 TLS space that can be used for optimizations. */
76static inline int
77tls_static_surplus (int nns, int opt_tls)
78{
79 return (nns - 1) * LIBC_IE_TLS + nns * OTHER_IE_TLS + opt_tls;
80}
81
82/* This value is chosen so that with default values for the tunables,
83 the computation of dl_tls_static_surplus in
84 _dl_tls_static_surplus_init yields the historic value 1664, for
85 backwards compatibility. */
86#define LEGACY_TLS (1664 - tls_static_surplus (DEFAULT_NNS, OPTIONAL_TLS))
87
88/* Calculate the size of the static TLS surplus, when the given
89 number of audit modules are loaded. Must be called after the
90 number of audit modules is known and before static TLS allocation. */
91void
92_dl_tls_static_surplus_init (size_t naudit)
93{
94 size_t nns, opt_tls;
95
96#if HAVE_TUNABLES
97 nns = TUNABLE_GET (nns, size_t, NULL);
98 opt_tls = TUNABLE_GET (optional_static_tls, size_t, NULL);
99#else
100 /* Default values of the tunables. */
101 nns = DEFAULT_NNS;
102 opt_tls = OPTIONAL_TLS;
103#endif
104 if (nns > DL_NNS)
105 nns = DL_NNS;
106 if (DL_NNS - nns < naudit)
107 _dl_fatal_printf ("Failed loading %lu audit modules, %lu are supported.\n",
108 (unsigned long) naudit, (unsigned long) (DL_NNS - nns));
109 nns += naudit;
110
111 GL(dl_tls_static_optional) = opt_tls;
112 assert (LEGACY_TLS >= 0);
113 GLRO(dl_tls_static_surplus) = tls_static_surplus (nns, opt_tls) + LEGACY_TLS;
114}
115
116/* Out-of-memory handler. */
117static void
118__attribute__ ((__noreturn__))
119oom (void)
120{
121 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
122}
123
124
125size_t
126_dl_next_tls_modid (void)
127{
128 size_t result;
129
130 if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
131 {
132 size_t disp = 0;
133 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
134
135 /* Note that this branch will never be executed during program
136 start since there are no gaps at that time. Therefore it
137 does not matter that the dl_tls_dtv_slotinfo is not allocated
138 yet when the function is called for the first times.
139
140 NB: the offset +1 is due to the fact that DTV[0] is used
141 for something else. */
142 result = GL(dl_tls_static_nelem) + 1;
143 if (result <= GL(dl_tls_max_dtv_idx))
144 do
145 {
146 while (result - disp < runp->len)
147 {
148 if (runp->slotinfo[result - disp].map == NULL)
149 break;
150
151 ++result;
152 assert (result <= GL(dl_tls_max_dtv_idx) + 1);
153 }
154
155 if (result - disp < runp->len)
156 break;
157
158 disp += runp->len;
159 }
160 while ((runp = runp->next) != NULL);
161
162 if (result > GL(dl_tls_max_dtv_idx))
163 {
164 /* The new index must indeed be exactly one higher than the
165 previous high. */
166 assert (result == GL(dl_tls_max_dtv_idx) + 1);
167 /* There is no gap anymore. */
168 GL(dl_tls_dtv_gaps) = false;
169
170 goto nogaps;
171 }
172 }
173 else
174 {
175 /* No gaps, allocate a new entry. */
176 nogaps:
177
178 result = ++GL(dl_tls_max_dtv_idx);
179 }
180
181 return result;
182}
183
184
185size_t
186_dl_count_modids (void)
187{
188 /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
189 we fail to load a module and unload it leaving a gap. If we don't
190 have gaps then the number of modids is the current maximum so
191 return that. */
192 if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
193 return GL(dl_tls_max_dtv_idx);
194
195 /* We have gaps and are forced to count the non-NULL entries. */
196 size_t n = 0;
197 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
198 while (runp != NULL)
199 {
200 for (size_t i = 0; i < runp->len; ++i)
201 if (runp->slotinfo[i].map != NULL)
202 ++n;
203
204 runp = runp->next;
205 }
206
207 return n;
208}
209
210
211#ifdef SHARED
212void
213_dl_determine_tlsoffset (void)
214{
215 size_t max_align = TLS_TCB_ALIGN;
216 size_t freetop = 0;
217 size_t freebottom = 0;
218
219 /* The first element of the dtv slot info list is allocated. */
220 assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
221 /* There is at this point only one element in the
222 dl_tls_dtv_slotinfo_list list. */
223 assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
224
225 struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
226
227 /* Determining the offset of the various parts of the static TLS
228 block has several dependencies. In addition we have to work
229 around bugs in some toolchains.
230
231 Each TLS block from the objects available at link time has a size
232 and an alignment requirement. The GNU ld computes the alignment
233 requirements for the data at the positions *in the file*, though.
234 I.e, it is not simply possible to allocate a block with the size
235 of the TLS program header entry. The data is layed out assuming
236 that the first byte of the TLS block fulfills
237
238 p_vaddr mod p_align == &TLS_BLOCK mod p_align
239
240 This means we have to add artificial padding at the beginning of
241 the TLS block. These bytes are never used for the TLS data in
242 this module but the first byte allocated must be aligned
243 according to mod p_align == 0 so that the first byte of the TLS
244 block is aligned according to p_vaddr mod p_align. This is ugly
245 and the linker can help by computing the offsets in the TLS block
246 assuming the first byte of the TLS block is aligned according to
247 p_align.
248
249 The extra space which might be allocated before the first byte of
250 the TLS block need not go unused. The code below tries to use
251 that memory for the next TLS block. This can work if the total
252 memory requirement for the next TLS block is smaller than the
253 gap. */
254
255#if TLS_TCB_AT_TP
256 /* We simply start with zero. */
257 size_t offset = 0;
258
259 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
260 {
261 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
262
263 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
264 & (slotinfo[cnt].map->l_tls_align - 1));
265 size_t off;
266 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
267
268 if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
269 {
270 off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
271 - firstbyte, slotinfo[cnt].map->l_tls_align)
272 + firstbyte;
273 if (off <= freebottom)
274 {
275 freetop = off;
276
277 /* XXX For some architectures we perhaps should store the
278 negative offset. */
279 slotinfo[cnt].map->l_tls_offset = off;
280 continue;
281 }
282 }
283
284 off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
285 slotinfo[cnt].map->l_tls_align) + firstbyte;
286 if (off > offset + slotinfo[cnt].map->l_tls_blocksize
287 + (freebottom - freetop))
288 {
289 freetop = offset;
290 freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
291 }
292 offset = off;
293
294 /* XXX For some architectures we perhaps should store the
295 negative offset. */
296 slotinfo[cnt].map->l_tls_offset = off;
297 }
298
299 GL(dl_tls_static_used) = offset;
300 GL(dl_tls_static_size) = (roundup (offset + GLRO(dl_tls_static_surplus),
301 max_align)
302 + TLS_TCB_SIZE);
303#elif TLS_DTV_AT_TP
304 /* The TLS blocks start right after the TCB. */
305 size_t offset = TLS_TCB_SIZE;
306
307 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
308 {
309 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
310
311 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
312 & (slotinfo[cnt].map->l_tls_align - 1));
313 size_t off;
314 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
315
316 if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
317 {
318 off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
319 if (off - freebottom < firstbyte)
320 off += slotinfo[cnt].map->l_tls_align;
321 if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
322 {
323 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
324 freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
325 - firstbyte);
326 continue;
327 }
328 }
329
330 off = roundup (offset, slotinfo[cnt].map->l_tls_align);
331 if (off - offset < firstbyte)
332 off += slotinfo[cnt].map->l_tls_align;
333
334 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
335 if (off - firstbyte - offset > freetop - freebottom)
336 {
337 freebottom = offset;
338 freetop = off - firstbyte;
339 }
340
341 offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
342 }
343
344 GL(dl_tls_static_used) = offset;
345 GL(dl_tls_static_size) = roundup (offset + GLRO(dl_tls_static_surplus),
346 TLS_TCB_ALIGN);
347#else
348# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
349#endif
350
351 /* The alignment requirement for the static TLS block. */
352 GL(dl_tls_static_align) = max_align;
353}
354#endif /* SHARED */
355
356static void *
357allocate_dtv (void *result)
358{
359 dtv_t *dtv;
360 size_t dtv_length;
361
362 /* We allocate a few more elements in the dtv than are needed for the
363 initial set of modules. This should avoid in most cases expansions
364 of the dtv. */
365 dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
366 dtv = calloc (dtv_length + 2, sizeof (dtv_t));
367 if (dtv != NULL)
368 {
369 /* This is the initial length of the dtv. */
370 dtv[0].counter = dtv_length;
371
372 /* The rest of the dtv (including the generation counter) is
373 Initialize with zero to indicate nothing there. */
374
375 /* Add the dtv to the thread data structures. */
376 INSTALL_DTV (result, dtv);
377 }
378 else
379 result = NULL;
380
381 return result;
382}
383
384
385/* Get size and alignment requirements of the static TLS block. */
386void
387_dl_get_tls_static_info (size_t *sizep, size_t *alignp)
388{
389 *sizep = GL(dl_tls_static_size);
390 *alignp = GL(dl_tls_static_align);
391}
392
393/* Derive the location of the pointer to the start of the original
394 allocation (before alignment) from the pointer to the TCB. */
395static inline void **
396tcb_to_pointer_to_free_location (void *tcb)
397{
398#if TLS_TCB_AT_TP
399 /* The TCB follows the TLS blocks, and the pointer to the front
400 follows the TCB. */
401 void **original_pointer_location = tcb + TLS_TCB_SIZE;
402#elif TLS_DTV_AT_TP
403 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
404 before that. */
405 void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *);
406#endif
407 return original_pointer_location;
408}
409
410void *
411_dl_allocate_tls_storage (void)
412{
413 void *result;
414 size_t size = GL(dl_tls_static_size);
415
416#if TLS_DTV_AT_TP
417 /* Memory layout is:
418 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
419 ^ This should be returned. */
420 size += TLS_PRE_TCB_SIZE;
421#endif
422
423 /* Perform the allocation. Reserve space for the required alignment
424 and the pointer to the original allocation. */
425 size_t alignment = GL(dl_tls_static_align);
426 void *allocated = malloc (size + alignment + sizeof (void *));
427 if (__glibc_unlikely (allocated == NULL))
428 return NULL;
429
430 /* Perform alignment and allocate the DTV. */
431#if TLS_TCB_AT_TP
432 /* The TCB follows the TLS blocks, which determine the alignment.
433 (TCB alignment requirements have been taken into account when
434 calculating GL(dl_tls_static_align).) */
435 void *aligned = (void *) roundup ((uintptr_t) allocated, alignment);
436 result = aligned + size - TLS_TCB_SIZE;
437
438 /* Clear the TCB data structure. We can't ask the caller (i.e.
439 libpthread) to do it, because we will initialize the DTV et al. */
440 memset (result, '\0', TLS_TCB_SIZE);
441#elif TLS_DTV_AT_TP
442 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
443 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
444 TLS block alignment, and not just the TLS blocks after it. This
445 can leave an unused alignment gap between the TCB and the TLS
446 blocks. */
447 result = (void *) roundup
448 (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated,
449 alignment);
450
451 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
452 it. We can't ask the caller (i.e. libpthread) to do it, because
453 we will initialize the DTV et al. */
454 memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
455#endif
456
457 /* Record the value of the original pointer for later
458 deallocation. */
459 *tcb_to_pointer_to_free_location (result) = allocated;
460
461 result = allocate_dtv (result);
462 if (result == NULL)
463 free (allocated);
464 return result;
465}
466
467
468#ifndef SHARED
469extern dtv_t _dl_static_dtv[];
470# define _dl_initial_dtv (&_dl_static_dtv[1])
471#endif
472
473static dtv_t *
474_dl_resize_dtv (dtv_t *dtv)
475{
476 /* Resize the dtv. */
477 dtv_t *newp;
478 /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
479 other threads concurrently. */
480 size_t newsize
481 = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS;
482 size_t oldsize = dtv[-1].counter;
483
484 if (dtv == GL(dl_initial_dtv))
485 {
486 /* This is the initial dtv that was either statically allocated in
487 __libc_setup_tls or allocated during rtld startup using the
488 dl-minimal.c malloc instead of the real malloc. We can't free
489 it, we have to abandon the old storage. */
490
491 newp = malloc ((2 + newsize) * sizeof (dtv_t));
492 if (newp == NULL)
493 oom ();
494 memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
495 }
496 else
497 {
498 newp = realloc (&dtv[-1],
499 (2 + newsize) * sizeof (dtv_t));
500 if (newp == NULL)
501 oom ();
502 }
503
504 newp[0].counter = newsize;
505
506 /* Clear the newly allocated part. */
507 memset (newp + 2 + oldsize, '\0',
508 (newsize - oldsize) * sizeof (dtv_t));
509
510 /* Return the generation counter. */
511 return &newp[1];
512}
513
514
515void *
516_dl_allocate_tls_init (void *result)
517{
518 if (result == NULL)
519 /* The memory allocation failed. */
520 return NULL;
521
522 dtv_t *dtv = GET_DTV (result);
523 struct dtv_slotinfo_list *listp;
524 size_t total = 0;
525 size_t maxgen = 0;
526
527 /* Check if the current dtv is big enough. */
528 if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
529 {
530 /* Resize the dtv. */
531 dtv = _dl_resize_dtv (dtv);
532
533 /* Install this new dtv in the thread data structures. */
534 INSTALL_DTV (result, &dtv[-1]);
535 }
536
537 /* We have to prepare the dtv for all currently loaded modules using
538 TLS. For those which are dynamically loaded we add the values
539 indicating deferred allocation. */
540 listp = GL(dl_tls_dtv_slotinfo_list);
541 while (1)
542 {
543 size_t cnt;
544
545 for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
546 {
547 struct link_map *map;
548 void *dest;
549
550 /* Check for the total number of used slots. */
551 if (total + cnt > GL(dl_tls_max_dtv_idx))
552 break;
553
554 map = listp->slotinfo[cnt].map;
555 if (map == NULL)
556 /* Unused entry. */
557 continue;
558
559 /* Keep track of the maximum generation number. This might
560 not be the generation counter. */
561 assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
562 maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
563
564 dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
565 dtv[map->l_tls_modid].pointer.to_free = NULL;
566
567 if (map->l_tls_offset == NO_TLS_OFFSET
568 || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
569 continue;
570
571 assert (map->l_tls_modid == total + cnt);
572 assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
573#if TLS_TCB_AT_TP
574 assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
575 dest = (char *) result - map->l_tls_offset;
576#elif TLS_DTV_AT_TP
577 dest = (char *) result + map->l_tls_offset;
578#else
579# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
580#endif
581
582 /* Set up the DTV entry. The simplified __tls_get_addr that
583 some platforms use in static programs requires it. */
584 dtv[map->l_tls_modid].pointer.val = dest;
585
586 /* Copy the initialization image and clear the BSS part. */
587 memset (__mempcpy (dest, map->l_tls_initimage,
588 map->l_tls_initimage_size), '\0',
589 map->l_tls_blocksize - map->l_tls_initimage_size);
590 }
591
592 total += cnt;
593 if (total >= GL(dl_tls_max_dtv_idx))
594 break;
595
596 listp = listp->next;
597 assert (listp != NULL);
598 }
599
600 /* The DTV version is up-to-date now. */
601 dtv[0].counter = maxgen;
602
603 return result;
604}
605rtld_hidden_def (_dl_allocate_tls_init)
606
607void *
608_dl_allocate_tls (void *mem)
609{
610 return _dl_allocate_tls_init (mem == NULL
611 ? _dl_allocate_tls_storage ()
612 : allocate_dtv (mem));
613}
614rtld_hidden_def (_dl_allocate_tls)
615
616
617void
618_dl_deallocate_tls (void *tcb, bool dealloc_tcb)
619{
620 dtv_t *dtv = GET_DTV (tcb);
621
622 /* We need to free the memory allocated for non-static TLS. */
623 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
624 free (dtv[1 + cnt].pointer.to_free);
625
626 /* The array starts with dtv[-1]. */
627 if (dtv != GL(dl_initial_dtv))
628 free (dtv - 1);
629
630 if (dealloc_tcb)
631 free (*tcb_to_pointer_to_free_location (tcb));
632}
633rtld_hidden_def (_dl_deallocate_tls)
634
635
636#ifdef SHARED
637/* The __tls_get_addr function has two basic forms which differ in the
638 arguments. The IA-64 form takes two parameters, the module ID and
639 offset. The form used, among others, on IA-32 takes a reference to
640 a special structure which contain the same information. The second
641 form seems to be more often used (in the moment) so we default to
642 it. Users of the IA-64 form have to provide adequate definitions
643 of the following macros. */
644# ifndef GET_ADDR_ARGS
645# define GET_ADDR_ARGS tls_index *ti
646# define GET_ADDR_PARAM ti
647# endif
648# ifndef GET_ADDR_MODULE
649# define GET_ADDR_MODULE ti->ti_module
650# endif
651# ifndef GET_ADDR_OFFSET
652# define GET_ADDR_OFFSET ti->ti_offset
653# endif
654
655/* Allocate one DTV entry. */
656static struct dtv_pointer
657allocate_dtv_entry (size_t alignment, size_t size)
658{
659 if (powerof2 (alignment) && alignment <= _Alignof (max_align_t))
660 {
661 /* The alignment is supported by malloc. */
662 void *ptr = malloc (size);
663 return (struct dtv_pointer) { ptr, ptr };
664 }
665
666 /* Emulate memalign to by manually aligning a pointer returned by
667 malloc. First compute the size with an overflow check. */
668 size_t alloc_size = size + alignment;
669 if (alloc_size < size)
670 return (struct dtv_pointer) {};
671
672 /* Perform the allocation. This is the pointer we need to free
673 later. */
674 void *start = malloc (alloc_size);
675 if (start == NULL)
676 return (struct dtv_pointer) {};
677
678 /* Find the aligned position within the larger allocation. */
679 void *aligned = (void *) roundup ((uintptr_t) start, alignment);
680
681 return (struct dtv_pointer) { .val = aligned, .to_free = start };
682}
683
684static struct dtv_pointer
685allocate_and_init (struct link_map *map)
686{
687 struct dtv_pointer result = allocate_dtv_entry
688 (map->l_tls_align, map->l_tls_blocksize);
689 if (result.val == NULL)
690 oom ();
691
692 /* Initialize the memory. */
693 memset (__mempcpy (result.val, map->l_tls_initimage,
694 map->l_tls_initimage_size),
695 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
696
697 return result;
698}
699
700
701struct link_map *
702_dl_update_slotinfo (unsigned long int req_modid)
703{
704 struct link_map *the_map = NULL;
705 dtv_t *dtv = THREAD_DTV ();
706
707 /* The global dl_tls_dtv_slotinfo array contains for each module
708 index the generation counter current when the entry was created.
709 This array never shrinks so that all module indices which were
710 valid at some time can be used to access it. Before the first
711 use of a new module index in this function the array was extended
712 appropriately. Access also does not have to be guarded against
713 modifications of the array. It is assumed that pointer-size
714 values can be read atomically even in SMP environments. It is
715 possible that other threads at the same time dynamically load
716 code and therefore add to the slotinfo list. This is a problem
717 since we must not pick up any information about incomplete work.
718 The solution to this is to ignore all dtv slots which were
719 created after the one we are currently interested. We know that
720 dynamic loading for this module is completed and this is the last
721 load operation we know finished. */
722 unsigned long int idx = req_modid;
723 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
724
725 while (idx >= listp->len)
726 {
727 idx -= listp->len;
728 listp = listp->next;
729 }
730
731 if (dtv[0].counter < listp->slotinfo[idx].gen)
732 {
733 /* The generation counter for the slot is higher than what the
734 current dtv implements. We have to update the whole dtv but
735 only those entries with a generation counter <= the one for
736 the entry we need. */
737 size_t new_gen = listp->slotinfo[idx].gen;
738 size_t total = 0;
739
740 /* We have to look through the entire dtv slotinfo list. */
741 listp = GL(dl_tls_dtv_slotinfo_list);
742 do
743 {
744 for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
745 {
746 size_t gen = listp->slotinfo[cnt].gen;
747
748 if (gen > new_gen)
749 /* This is a slot for a generation younger than the
750 one we are handling now. It might be incompletely
751 set up so ignore it. */
752 continue;
753
754 /* If the entry is older than the current dtv layout we
755 know we don't have to handle it. */
756 if (gen <= dtv[0].counter)
757 continue;
758
759 /* If there is no map this means the entry is empty. */
760 struct link_map *map = listp->slotinfo[cnt].map;
761 if (map == NULL)
762 {
763 if (dtv[-1].counter >= total + cnt)
764 {
765 /* If this modid was used at some point the memory
766 might still be allocated. */
767 free (dtv[total + cnt].pointer.to_free);
768 dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
769 dtv[total + cnt].pointer.to_free = NULL;
770 }
771
772 continue;
773 }
774
775 /* Check whether the current dtv array is large enough. */
776 size_t modid = map->l_tls_modid;
777 assert (total + cnt == modid);
778 if (dtv[-1].counter < modid)
779 {
780 /* Resize the dtv. */
781 dtv = _dl_resize_dtv (dtv);
782
783 assert (modid <= dtv[-1].counter);
784
785 /* Install this new dtv in the thread data
786 structures. */
787 INSTALL_NEW_DTV (dtv);
788 }
789
790 /* If there is currently memory allocate for this
791 dtv entry free it. */
792 /* XXX Ideally we will at some point create a memory
793 pool. */
794 free (dtv[modid].pointer.to_free);
795 dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
796 dtv[modid].pointer.to_free = NULL;
797
798 if (modid == req_modid)
799 the_map = map;
800 }
801
802 total += listp->len;
803 }
804 while ((listp = listp->next) != NULL);
805
806 /* This will be the new maximum generation counter. */
807 dtv[0].counter = new_gen;
808 }
809
810 return the_map;
811}
812
813
814static void *
815__attribute_noinline__
816tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
817{
818 /* The allocation was deferred. Do it now. */
819 if (the_map == NULL)
820 {
821 /* Find the link map for this module. */
822 size_t idx = GET_ADDR_MODULE;
823 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
824
825 while (idx >= listp->len)
826 {
827 idx -= listp->len;
828 listp = listp->next;
829 }
830
831 the_map = listp->slotinfo[idx].map;
832 }
833
834 /* Make sure that, if a dlopen running in parallel forces the
835 variable into static storage, we'll wait until the address in the
836 static TLS block is set up, and use that. If we're undecided
837 yet, make sure we make the decision holding the lock as well. */
838 if (__glibc_unlikely (the_map->l_tls_offset
839 != FORCED_DYNAMIC_TLS_OFFSET))
840 {
841 __rtld_lock_lock_recursive (GL(dl_load_lock));
842 if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
843 {
844 the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
845 __rtld_lock_unlock_recursive (GL(dl_load_lock));
846 }
847 else if (__glibc_likely (the_map->l_tls_offset
848 != FORCED_DYNAMIC_TLS_OFFSET))
849 {
850#if TLS_TCB_AT_TP
851 void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
852#elif TLS_DTV_AT_TP
853 void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
854#else
855# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
856#endif
857 __rtld_lock_unlock_recursive (GL(dl_load_lock));
858
859 dtv[GET_ADDR_MODULE].pointer.to_free = NULL;
860 dtv[GET_ADDR_MODULE].pointer.val = p;
861
862 return (char *) p + GET_ADDR_OFFSET;
863 }
864 else
865 __rtld_lock_unlock_recursive (GL(dl_load_lock));
866 }
867 struct dtv_pointer result = allocate_and_init (the_map);
868 dtv[GET_ADDR_MODULE].pointer = result;
869 assert (result.to_free != NULL);
870
871 return (char *) result.val + GET_ADDR_OFFSET;
872}
873
874
875static struct link_map *
876__attribute_noinline__
877update_get_addr (GET_ADDR_ARGS)
878{
879 struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
880 dtv_t *dtv = THREAD_DTV ();
881
882 void *p = dtv[GET_ADDR_MODULE].pointer.val;
883
884 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
885 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
886
887 return (void *) p + GET_ADDR_OFFSET;
888}
889
890/* For all machines that have a non-macro version of __tls_get_addr, we
891 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
892 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
893 in ld.so for __tls_get_addr. */
894
895#ifndef __tls_get_addr
896extern void * __tls_get_addr (GET_ADDR_ARGS);
897rtld_hidden_proto (__tls_get_addr)
898rtld_hidden_def (__tls_get_addr)
899#endif
900
901/* The generic dynamic and local dynamic model cannot be used in
902 statically linked applications. */
903void *
904__tls_get_addr (GET_ADDR_ARGS)
905{
906 dtv_t *dtv = THREAD_DTV ();
907
908 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
909 return update_get_addr (GET_ADDR_PARAM);
910
911 void *p = dtv[GET_ADDR_MODULE].pointer.val;
912
913 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
914 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
915
916 return (char *) p + GET_ADDR_OFFSET;
917}
918#endif
919
920
921/* Look up the module's TLS block as for __tls_get_addr,
922 but never touch anything. Return null if it's not allocated yet. */
923void *
924_dl_tls_get_addr_soft (struct link_map *l)
925{
926 if (__glibc_unlikely (l->l_tls_modid == 0))
927 /* This module has no TLS segment. */
928 return NULL;
929
930 dtv_t *dtv = THREAD_DTV ();
931 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
932 {
933 /* This thread's DTV is not completely current,
934 but it might already cover this module. */
935
936 if (l->l_tls_modid >= dtv[-1].counter)
937 /* Nope. */
938 return NULL;
939
940 size_t idx = l->l_tls_modid;
941 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
942 while (idx >= listp->len)
943 {
944 idx -= listp->len;
945 listp = listp->next;
946 }
947
948 /* We've reached the slot for this module.
949 If its generation counter is higher than the DTV's,
950 this thread does not know about this module yet. */
951 if (dtv[0].counter < listp->slotinfo[idx].gen)
952 return NULL;
953 }
954
955 void *data = dtv[l->l_tls_modid].pointer.val;
956 if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
957 /* The DTV is current, but this thread has not yet needed
958 to allocate this module's segment. */
959 data = NULL;
960
961 return data;
962}
963
964
965void
966_dl_add_to_slotinfo (struct link_map *l, bool do_add)
967{
968 /* Now that we know the object is loaded successfully add
969 modules containing TLS data to the dtv info table. We
970 might have to increase its size. */
971 struct dtv_slotinfo_list *listp;
972 struct dtv_slotinfo_list *prevp;
973 size_t idx = l->l_tls_modid;
974
975 /* Find the place in the dtv slotinfo list. */
976 listp = GL(dl_tls_dtv_slotinfo_list);
977 prevp = NULL; /* Needed to shut up gcc. */
978 do
979 {
980 /* Does it fit in the array of this list element? */
981 if (idx < listp->len)
982 break;
983 idx -= listp->len;
984 prevp = listp;
985 listp = listp->next;
986 }
987 while (listp != NULL);
988
989 if (listp == NULL)
990 {
991 /* When we come here it means we have to add a new element
992 to the slotinfo list. And the new module must be in
993 the first slot. */
994 assert (idx == 0);
995
996 listp = prevp->next = (struct dtv_slotinfo_list *)
997 malloc (sizeof (struct dtv_slotinfo_list)
998 + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
999 if (listp == NULL)
1000 {
1001 /* We ran out of memory. We will simply fail this
1002 call but don't undo anything we did so far. The
1003 application will crash or be terminated anyway very
1004 soon. */
1005
1006 /* We have to do this since some entries in the dtv
1007 slotinfo array might already point to this
1008 generation. */
1009 ++GL(dl_tls_generation);
1010
1011 _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
1012cannot create TLS data structures"));
1013 }
1014
1015 listp->len = TLS_SLOTINFO_SURPLUS;
1016 listp->next = NULL;
1017 memset (listp->slotinfo, '\0',
1018 TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
1019 }
1020
1021 /* Add the information into the slotinfo data structure. */
1022 if (do_add)
1023 {
1024 listp->slotinfo[idx].map = l;
1025 listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
1026 }
1027}
1028