1/* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2016 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <libintl.h>
22#include <signal.h>
23#include <stdlib.h>
24#include <unistd.h>
25#include <sys/param.h>
26#include <atomic.h>
27
28#include <tls.h>
29#include <dl-tls.h>
30#include <ldsodefs.h>
31
32/* Amount of excess space to allocate in the static TLS area
33 to allow dynamic loading of modules defining IE-model TLS data. */
34#define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
35
36
37/* Out-of-memory handler. */
38static void
39__attribute__ ((__noreturn__))
40oom (void)
41{
42 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
43}
44
45
46size_t
47internal_function
48_dl_next_tls_modid (void)
49{
50 size_t result;
51
52 if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
53 {
54 size_t disp = 0;
55 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
56
57 /* Note that this branch will never be executed during program
58 start since there are no gaps at that time. Therefore it
59 does not matter that the dl_tls_dtv_slotinfo is not allocated
60 yet when the function is called for the first times.
61
62 NB: the offset +1 is due to the fact that DTV[0] is used
63 for something else. */
64 result = GL(dl_tls_static_nelem) + 1;
65 if (result <= GL(dl_tls_max_dtv_idx))
66 do
67 {
68 while (result - disp < runp->len)
69 {
70 if (runp->slotinfo[result - disp].map == NULL)
71 break;
72
73 ++result;
74 assert (result <= GL(dl_tls_max_dtv_idx) + 1);
75 }
76
77 if (result - disp < runp->len)
78 break;
79
80 disp += runp->len;
81 }
82 while ((runp = runp->next) != NULL);
83
84 if (result > GL(dl_tls_max_dtv_idx))
85 {
86 /* The new index must indeed be exactly one higher than the
87 previous high. */
88 assert (result == GL(dl_tls_max_dtv_idx) + 1);
89 /* There is no gap anymore. */
90 GL(dl_tls_dtv_gaps) = false;
91
92 goto nogaps;
93 }
94 }
95 else
96 {
97 /* No gaps, allocate a new entry. */
98 nogaps:
99
100 result = ++GL(dl_tls_max_dtv_idx);
101 }
102
103 return result;
104}
105
106
107size_t
108internal_function
109_dl_count_modids (void)
110{
111 /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
112 we fail to load a module and unload it leaving a gap. If we don't
113 have gaps then the number of modids is the current maximum so
114 return that. */
115 if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
116 return GL(dl_tls_max_dtv_idx);
117
118 /* We have gaps and are forced to count the non-NULL entries. */
119 size_t n = 0;
120 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
121 while (runp != NULL)
122 {
123 for (size_t i = 0; i < runp->len; ++i)
124 if (runp->slotinfo[i].map != NULL)
125 ++n;
126
127 runp = runp->next;
128 }
129
130 return n;
131}
132
133
134#ifdef SHARED
135void
136internal_function
137_dl_determine_tlsoffset (void)
138{
139 size_t max_align = TLS_TCB_ALIGN;
140 size_t freetop = 0;
141 size_t freebottom = 0;
142
143 /* The first element of the dtv slot info list is allocated. */
144 assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
145 /* There is at this point only one element in the
146 dl_tls_dtv_slotinfo_list list. */
147 assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
148
149 struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
150
151 /* Determining the offset of the various parts of the static TLS
152 block has several dependencies. In addition we have to work
153 around bugs in some toolchains.
154
155 Each TLS block from the objects available at link time has a size
156 and an alignment requirement. The GNU ld computes the alignment
157 requirements for the data at the positions *in the file*, though.
158 I.e, it is not simply possible to allocate a block with the size
159 of the TLS program header entry. The data is layed out assuming
160 that the first byte of the TLS block fulfills
161
162 p_vaddr mod p_align == &TLS_BLOCK mod p_align
163
164 This means we have to add artificial padding at the beginning of
165 the TLS block. These bytes are never used for the TLS data in
166 this module but the first byte allocated must be aligned
167 according to mod p_align == 0 so that the first byte of the TLS
168 block is aligned according to p_vaddr mod p_align. This is ugly
169 and the linker can help by computing the offsets in the TLS block
170 assuming the first byte of the TLS block is aligned according to
171 p_align.
172
173 The extra space which might be allocated before the first byte of
174 the TLS block need not go unused. The code below tries to use
175 that memory for the next TLS block. This can work if the total
176 memory requirement for the next TLS block is smaller than the
177 gap. */
178
179#if TLS_TCB_AT_TP
180 /* We simply start with zero. */
181 size_t offset = 0;
182
183 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
184 {
185 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
186
187 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
188 & (slotinfo[cnt].map->l_tls_align - 1));
189 size_t off;
190 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
191
192 if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
193 {
194 off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
195 - firstbyte, slotinfo[cnt].map->l_tls_align)
196 + firstbyte;
197 if (off <= freebottom)
198 {
199 freetop = off;
200
201 /* XXX For some architectures we perhaps should store the
202 negative offset. */
203 slotinfo[cnt].map->l_tls_offset = off;
204 continue;
205 }
206 }
207
208 off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
209 slotinfo[cnt].map->l_tls_align) + firstbyte;
210 if (off > offset + slotinfo[cnt].map->l_tls_blocksize
211 + (freebottom - freetop))
212 {
213 freetop = offset;
214 freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
215 }
216 offset = off;
217
218 /* XXX For some architectures we perhaps should store the
219 negative offset. */
220 slotinfo[cnt].map->l_tls_offset = off;
221 }
222
223 GL(dl_tls_static_used) = offset;
224 GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
225 + TLS_TCB_SIZE);
226#elif TLS_DTV_AT_TP
227 /* The TLS blocks start right after the TCB. */
228 size_t offset = TLS_TCB_SIZE;
229
230 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
231 {
232 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
233
234 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
235 & (slotinfo[cnt].map->l_tls_align - 1));
236 size_t off;
237 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
238
239 if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
240 {
241 off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
242 if (off - freebottom < firstbyte)
243 off += slotinfo[cnt].map->l_tls_align;
244 if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
245 {
246 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
247 freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
248 - firstbyte);
249 continue;
250 }
251 }
252
253 off = roundup (offset, slotinfo[cnt].map->l_tls_align);
254 if (off - offset < firstbyte)
255 off += slotinfo[cnt].map->l_tls_align;
256
257 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
258 if (off - firstbyte - offset > freetop - freebottom)
259 {
260 freebottom = offset;
261 freetop = off - firstbyte;
262 }
263
264 offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
265 }
266
267 GL(dl_tls_static_used) = offset;
268 GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
269 TLS_TCB_ALIGN);
270#else
271# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
272#endif
273
274 /* The alignment requirement for the static TLS block. */
275 GL(dl_tls_static_align) = max_align;
276}
277
278
279/* This is called only when the data structure setup was skipped at startup,
280 when there was no need for it then. Now we have dynamically loaded
281 something needing TLS, or libpthread needs it. */
282int
283internal_function
284_dl_tls_setup (void)
285{
286 assert (GL(dl_tls_dtv_slotinfo_list) == NULL);
287 assert (GL(dl_tls_max_dtv_idx) == 0);
288
289 const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS;
290
291 GL(dl_tls_dtv_slotinfo_list)
292 = calloc (1, (sizeof (struct dtv_slotinfo_list)
293 + nelem * sizeof (struct dtv_slotinfo)));
294 if (GL(dl_tls_dtv_slotinfo_list) == NULL)
295 return -1;
296
297 GL(dl_tls_dtv_slotinfo_list)->len = nelem;
298
299 /* Number of elements in the static TLS block. It can't be zero
300 because of various assumptions. The one element is null. */
301 GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1;
302
303 /* This initializes more variables for us. */
304 _dl_determine_tlsoffset ();
305
306 return 0;
307}
308rtld_hidden_def (_dl_tls_setup)
309#endif
310
311static void *
312internal_function
313allocate_dtv (void *result)
314{
315 dtv_t *dtv;
316 size_t dtv_length;
317
318 /* We allocate a few more elements in the dtv than are needed for the
319 initial set of modules. This should avoid in most cases expansions
320 of the dtv. */
321 dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
322 dtv = calloc (dtv_length + 2, sizeof (dtv_t));
323 if (dtv != NULL)
324 {
325 /* This is the initial length of the dtv. */
326 dtv[0].counter = dtv_length;
327
328 /* The rest of the dtv (including the generation counter) is
329 Initialize with zero to indicate nothing there. */
330
331 /* Add the dtv to the thread data structures. */
332 INSTALL_DTV (result, dtv);
333 }
334 else
335 result = NULL;
336
337 return result;
338}
339
340
341/* Get size and alignment requirements of the static TLS block. */
342void
343internal_function
344_dl_get_tls_static_info (size_t *sizep, size_t *alignp)
345{
346 *sizep = GL(dl_tls_static_size);
347 *alignp = GL(dl_tls_static_align);
348}
349
350
351void *
352internal_function
353_dl_allocate_tls_storage (void)
354{
355 void *result;
356 size_t size = GL(dl_tls_static_size);
357
358#if TLS_DTV_AT_TP
359 /* Memory layout is:
360 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
361 ^ This should be returned. */
362 size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
363 & ~(GL(dl_tls_static_align) - 1);
364#endif
365
366 /* Allocate a correctly aligned chunk of memory. */
367 result = __libc_memalign (GL(dl_tls_static_align), size);
368 if (__builtin_expect (result != NULL, 1))
369 {
370 /* Allocate the DTV. */
371 void *allocated = result;
372
373#if TLS_TCB_AT_TP
374 /* The TCB follows the TLS blocks. */
375 result = (char *) result + size - TLS_TCB_SIZE;
376
377 /* Clear the TCB data structure. We can't ask the caller (i.e.
378 libpthread) to do it, because we will initialize the DTV et al. */
379 memset (result, '\0', TLS_TCB_SIZE);
380#elif TLS_DTV_AT_TP
381 result = (char *) result + size - GL(dl_tls_static_size);
382
383 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before it.
384 We can't ask the caller (i.e. libpthread) to do it, because we will
385 initialize the DTV et al. */
386 memset ((char *) result - TLS_PRE_TCB_SIZE, '\0',
387 TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
388#endif
389
390 result = allocate_dtv (result);
391 if (result == NULL)
392 free (allocated);
393 }
394
395 return result;
396}
397
398
399#ifndef SHARED
400extern dtv_t _dl_static_dtv[];
401# define _dl_initial_dtv (&_dl_static_dtv[1])
402#endif
403
404static dtv_t *
405_dl_resize_dtv (dtv_t *dtv)
406{
407 /* Resize the dtv. */
408 dtv_t *newp;
409 /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
410 other threads concurrently. */
411 size_t newsize
412 = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS;
413 size_t oldsize = dtv[-1].counter;
414
415 if (dtv == GL(dl_initial_dtv))
416 {
417 /* This is the initial dtv that was either statically allocated in
418 __libc_setup_tls or allocated during rtld startup using the
419 dl-minimal.c malloc instead of the real malloc. We can't free
420 it, we have to abandon the old storage. */
421
422 newp = malloc ((2 + newsize) * sizeof (dtv_t));
423 if (newp == NULL)
424 oom ();
425 memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
426 }
427 else
428 {
429 newp = realloc (&dtv[-1],
430 (2 + newsize) * sizeof (dtv_t));
431 if (newp == NULL)
432 oom ();
433 }
434
435 newp[0].counter = newsize;
436
437 /* Clear the newly allocated part. */
438 memset (newp + 2 + oldsize, '\0',
439 (newsize - oldsize) * sizeof (dtv_t));
440
441 /* Return the generation counter. */
442 return &newp[1];
443}
444
445
446void *
447internal_function
448_dl_allocate_tls_init (void *result)
449{
450 if (result == NULL)
451 /* The memory allocation failed. */
452 return NULL;
453
454 dtv_t *dtv = GET_DTV (result);
455 struct dtv_slotinfo_list *listp;
456 size_t total = 0;
457 size_t maxgen = 0;
458
459 /* Check if the current dtv is big enough. */
460 if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
461 {
462 /* Resize the dtv. */
463 dtv = _dl_resize_dtv (dtv);
464
465 /* Install this new dtv in the thread data structures. */
466 INSTALL_DTV (result, &dtv[-1]);
467 }
468
469 /* We have to prepare the dtv for all currently loaded modules using
470 TLS. For those which are dynamically loaded we add the values
471 indicating deferred allocation. */
472 listp = GL(dl_tls_dtv_slotinfo_list);
473 while (1)
474 {
475 size_t cnt;
476
477 for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
478 {
479 struct link_map *map;
480 void *dest;
481
482 /* Check for the total number of used slots. */
483 if (total + cnt > GL(dl_tls_max_dtv_idx))
484 break;
485
486 map = listp->slotinfo[cnt].map;
487 if (map == NULL)
488 /* Unused entry. */
489 continue;
490
491 /* Keep track of the maximum generation number. This might
492 not be the generation counter. */
493 assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
494 maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
495
496 dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
497 dtv[map->l_tls_modid].pointer.is_static = false;
498
499 if (map->l_tls_offset == NO_TLS_OFFSET
500 || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
501 continue;
502
503 assert (map->l_tls_modid == total + cnt);
504 assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
505#if TLS_TCB_AT_TP
506 assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
507 dest = (char *) result - map->l_tls_offset;
508#elif TLS_DTV_AT_TP
509 dest = (char *) result + map->l_tls_offset;
510#else
511# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
512#endif
513
514 /* Copy the initialization image and clear the BSS part. */
515 memset (__mempcpy (dest, map->l_tls_initimage,
516 map->l_tls_initimage_size), '\0',
517 map->l_tls_blocksize - map->l_tls_initimage_size);
518 }
519
520 total += cnt;
521 if (total >= GL(dl_tls_max_dtv_idx))
522 break;
523
524 listp = listp->next;
525 assert (listp != NULL);
526 }
527
528 /* The DTV version is up-to-date now. */
529 dtv[0].counter = maxgen;
530
531 return result;
532}
533rtld_hidden_def (_dl_allocate_tls_init)
534
535void *
536internal_function
537_dl_allocate_tls (void *mem)
538{
539 return _dl_allocate_tls_init (mem == NULL
540 ? _dl_allocate_tls_storage ()
541 : allocate_dtv (mem));
542}
543rtld_hidden_def (_dl_allocate_tls)
544
545
546void
547internal_function
548_dl_deallocate_tls (void *tcb, bool dealloc_tcb)
549{
550 dtv_t *dtv = GET_DTV (tcb);
551
552 /* We need to free the memory allocated for non-static TLS. */
553 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
554 if (! dtv[1 + cnt].pointer.is_static
555 && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
556 free (dtv[1 + cnt].pointer.val);
557
558 /* The array starts with dtv[-1]. */
559 if (dtv != GL(dl_initial_dtv))
560 free (dtv - 1);
561
562 if (dealloc_tcb)
563 {
564#if TLS_TCB_AT_TP
565 /* The TCB follows the TLS blocks. Back up to free the whole block. */
566 tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE;
567#elif TLS_DTV_AT_TP
568 /* Back up the TLS_PRE_TCB_SIZE bytes. */
569 tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
570 & ~(GL(dl_tls_static_align) - 1);
571#endif
572 free (tcb);
573 }
574}
575rtld_hidden_def (_dl_deallocate_tls)
576
577
578#ifdef SHARED
579/* The __tls_get_addr function has two basic forms which differ in the
580 arguments. The IA-64 form takes two parameters, the module ID and
581 offset. The form used, among others, on IA-32 takes a reference to
582 a special structure which contain the same information. The second
583 form seems to be more often used (in the moment) so we default to
584 it. Users of the IA-64 form have to provide adequate definitions
585 of the following macros. */
586# ifndef GET_ADDR_ARGS
587# define GET_ADDR_ARGS tls_index *ti
588# define GET_ADDR_PARAM ti
589# endif
590# ifndef GET_ADDR_MODULE
591# define GET_ADDR_MODULE ti->ti_module
592# endif
593# ifndef GET_ADDR_OFFSET
594# define GET_ADDR_OFFSET ti->ti_offset
595# endif
596
597
598static void *
599allocate_and_init (struct link_map *map)
600{
601 void *newp;
602
603 newp = __libc_memalign (map->l_tls_align, map->l_tls_blocksize);
604 if (newp == NULL)
605 oom ();
606
607 /* Initialize the memory. */
608 memset (__mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size),
609 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
610
611 return newp;
612}
613
614
615struct link_map *
616_dl_update_slotinfo (unsigned long int req_modid)
617{
618 struct link_map *the_map = NULL;
619 dtv_t *dtv = THREAD_DTV ();
620
621 /* The global dl_tls_dtv_slotinfo array contains for each module
622 index the generation counter current when the entry was created.
623 This array never shrinks so that all module indices which were
624 valid at some time can be used to access it. Before the first
625 use of a new module index in this function the array was extended
626 appropriately. Access also does not have to be guarded against
627 modifications of the array. It is assumed that pointer-size
628 values can be read atomically even in SMP environments. It is
629 possible that other threads at the same time dynamically load
630 code and therefore add to the slotinfo list. This is a problem
631 since we must not pick up any information about incomplete work.
632 The solution to this is to ignore all dtv slots which were
633 created after the one we are currently interested. We know that
634 dynamic loading for this module is completed and this is the last
635 load operation we know finished. */
636 unsigned long int idx = req_modid;
637 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
638
639 while (idx >= listp->len)
640 {
641 idx -= listp->len;
642 listp = listp->next;
643 }
644
645 if (dtv[0].counter < listp->slotinfo[idx].gen)
646 {
647 /* The generation counter for the slot is higher than what the
648 current dtv implements. We have to update the whole dtv but
649 only those entries with a generation counter <= the one for
650 the entry we need. */
651 size_t new_gen = listp->slotinfo[idx].gen;
652 size_t total = 0;
653
654 /* We have to look through the entire dtv slotinfo list. */
655 listp = GL(dl_tls_dtv_slotinfo_list);
656 do
657 {
658 for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
659 {
660 size_t gen = listp->slotinfo[cnt].gen;
661
662 if (gen > new_gen)
663 /* This is a slot for a generation younger than the
664 one we are handling now. It might be incompletely
665 set up so ignore it. */
666 continue;
667
668 /* If the entry is older than the current dtv layout we
669 know we don't have to handle it. */
670 if (gen <= dtv[0].counter)
671 continue;
672
673 /* If there is no map this means the entry is empty. */
674 struct link_map *map = listp->slotinfo[cnt].map;
675 if (map == NULL)
676 {
677 if (dtv[-1].counter >= total + cnt)
678 {
679 /* If this modid was used at some point the memory
680 might still be allocated. */
681 if (! dtv[total + cnt].pointer.is_static
682 && (dtv[total + cnt].pointer.val
683 != TLS_DTV_UNALLOCATED))
684 free (dtv[total + cnt].pointer.val);
685 dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
686 dtv[total + cnt].pointer.is_static = false;
687 }
688
689 continue;
690 }
691
692 /* Check whether the current dtv array is large enough. */
693 size_t modid = map->l_tls_modid;
694 assert (total + cnt == modid);
695 if (dtv[-1].counter < modid)
696 {
697 /* Resize the dtv. */
698 dtv = _dl_resize_dtv (dtv);
699
700 assert (modid <= dtv[-1].counter);
701
702 /* Install this new dtv in the thread data
703 structures. */
704 INSTALL_NEW_DTV (dtv);
705 }
706
707 /* If there is currently memory allocate for this
708 dtv entry free it. */
709 /* XXX Ideally we will at some point create a memory
710 pool. */
711 if (! dtv[modid].pointer.is_static
712 && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED)
713 /* Note that free is called for NULL is well. We
714 deallocate even if it is this dtv entry we are
715 supposed to load. The reason is that we call
716 memalign and not malloc. */
717 free (dtv[modid].pointer.val);
718
719 dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
720 dtv[modid].pointer.is_static = false;
721
722 if (modid == req_modid)
723 the_map = map;
724 }
725
726 total += listp->len;
727 }
728 while ((listp = listp->next) != NULL);
729
730 /* This will be the new maximum generation counter. */
731 dtv[0].counter = new_gen;
732 }
733
734 return the_map;
735}
736
737
738static void *
739__attribute_noinline__
740tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
741{
742 /* The allocation was deferred. Do it now. */
743 if (the_map == NULL)
744 {
745 /* Find the link map for this module. */
746 size_t idx = GET_ADDR_MODULE;
747 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
748
749 while (idx >= listp->len)
750 {
751 idx -= listp->len;
752 listp = listp->next;
753 }
754
755 the_map = listp->slotinfo[idx].map;
756 }
757
758 /* Make sure that, if a dlopen running in parallel forces the
759 variable into static storage, we'll wait until the address in the
760 static TLS block is set up, and use that. If we're undecided
761 yet, make sure we make the decision holding the lock as well. */
762 if (__glibc_unlikely (the_map->l_tls_offset
763 != FORCED_DYNAMIC_TLS_OFFSET))
764 {
765 __rtld_lock_lock_recursive (GL(dl_load_lock));
766 if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
767 {
768 the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
769 __rtld_lock_unlock_recursive (GL(dl_load_lock));
770 }
771 else if (__glibc_likely (the_map->l_tls_offset
772 != FORCED_DYNAMIC_TLS_OFFSET))
773 {
774#if TLS_TCB_AT_TP
775 void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
776#elif TLS_DTV_AT_TP
777 void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
778#else
779# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
780#endif
781 __rtld_lock_unlock_recursive (GL(dl_load_lock));
782
783 dtv[GET_ADDR_MODULE].pointer.is_static = true;
784 dtv[GET_ADDR_MODULE].pointer.val = p;
785
786 return (char *) p + GET_ADDR_OFFSET;
787 }
788 else
789 __rtld_lock_unlock_recursive (GL(dl_load_lock));
790 }
791 void *p = dtv[GET_ADDR_MODULE].pointer.val = allocate_and_init (the_map);
792 assert (!dtv[GET_ADDR_MODULE].pointer.is_static);
793
794 return (char *) p + GET_ADDR_OFFSET;
795}
796
797
798static struct link_map *
799__attribute_noinline__
800update_get_addr (GET_ADDR_ARGS)
801{
802 struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
803 dtv_t *dtv = THREAD_DTV ();
804
805 void *p = dtv[GET_ADDR_MODULE].pointer.val;
806
807 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
808 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
809
810 return (void *) p + GET_ADDR_OFFSET;
811}
812
813/* For all machines that have a non-macro version of __tls_get_addr, we
814 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
815 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
816 in ld.so for __tls_get_addr. */
817
818#ifndef __tls_get_addr
819extern void * __tls_get_addr (GET_ADDR_ARGS);
820rtld_hidden_proto (__tls_get_addr)
821rtld_hidden_def (__tls_get_addr)
822#endif
823
824/* The generic dynamic and local dynamic model cannot be used in
825 statically linked applications. */
826void *
827__tls_get_addr (GET_ADDR_ARGS)
828{
829 dtv_t *dtv = THREAD_DTV ();
830
831 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
832 return update_get_addr (GET_ADDR_PARAM);
833
834 void *p = dtv[GET_ADDR_MODULE].pointer.val;
835
836 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
837 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
838
839 return (char *) p + GET_ADDR_OFFSET;
840}
841#endif
842
843
844/* Look up the module's TLS block as for __tls_get_addr,
845 but never touch anything. Return null if it's not allocated yet. */
846void *
847_dl_tls_get_addr_soft (struct link_map *l)
848{
849 if (__glibc_unlikely (l->l_tls_modid == 0))
850 /* This module has no TLS segment. */
851 return NULL;
852
853 dtv_t *dtv = THREAD_DTV ();
854 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
855 {
856 /* This thread's DTV is not completely current,
857 but it might already cover this module. */
858
859 if (l->l_tls_modid >= dtv[-1].counter)
860 /* Nope. */
861 return NULL;
862
863 size_t idx = l->l_tls_modid;
864 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
865 while (idx >= listp->len)
866 {
867 idx -= listp->len;
868 listp = listp->next;
869 }
870
871 /* We've reached the slot for this module.
872 If its generation counter is higher than the DTV's,
873 this thread does not know about this module yet. */
874 if (dtv[0].counter < listp->slotinfo[idx].gen)
875 return NULL;
876 }
877
878 void *data = dtv[l->l_tls_modid].pointer.val;
879 if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
880 /* The DTV is current, but this thread has not yet needed
881 to allocate this module's segment. */
882 data = NULL;
883
884 return data;
885}
886
887
888void
889_dl_add_to_slotinfo (struct link_map *l)
890{
891 /* Now that we know the object is loaded successfully add
892 modules containing TLS data to the dtv info table. We
893 might have to increase its size. */
894 struct dtv_slotinfo_list *listp;
895 struct dtv_slotinfo_list *prevp;
896 size_t idx = l->l_tls_modid;
897
898 /* Find the place in the dtv slotinfo list. */
899 listp = GL(dl_tls_dtv_slotinfo_list);
900 prevp = NULL; /* Needed to shut up gcc. */
901 do
902 {
903 /* Does it fit in the array of this list element? */
904 if (idx < listp->len)
905 break;
906 idx -= listp->len;
907 prevp = listp;
908 listp = listp->next;
909 }
910 while (listp != NULL);
911
912 if (listp == NULL)
913 {
914 /* When we come here it means we have to add a new element
915 to the slotinfo list. And the new module must be in
916 the first slot. */
917 assert (idx == 0);
918
919 listp = prevp->next = (struct dtv_slotinfo_list *)
920 malloc (sizeof (struct dtv_slotinfo_list)
921 + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
922 if (listp == NULL)
923 {
924 /* We ran out of memory. We will simply fail this
925 call but don't undo anything we did so far. The
926 application will crash or be terminated anyway very
927 soon. */
928
929 /* We have to do this since some entries in the dtv
930 slotinfo array might already point to this
931 generation. */
932 ++GL(dl_tls_generation);
933
934 _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
935cannot create TLS data structures"));
936 }
937
938 listp->len = TLS_SLOTINFO_SURPLUS;
939 listp->next = NULL;
940 memset (listp->slotinfo, '\0',
941 TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
942 }
943
944 /* Add the information into the slotinfo data structure. */
945 listp->slotinfo[idx].map = l;
946 listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
947}
948