1/* Machine-dependent ELF dynamic relocation inline functions. x86-64 version.
2 Copyright (C) 2001-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Andreas Jaeger <aj@suse.de>.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
19
20#ifndef dl_machine_h
21#define dl_machine_h
22
23#define ELF_MACHINE_NAME "x86_64"
24
25#include <sys/param.h>
26#include <sysdep.h>
27#include <tls.h>
28#include <dl-tlsdesc.h>
29
30/* Return nonzero iff ELF header is compatible with the running host. */
31static inline int __attribute__ ((unused))
32elf_machine_matches_host (const ElfW(Ehdr) *ehdr)
33{
34 return ehdr->e_machine == EM_X86_64;
35}
36
37
38/* Return the link-time address of _DYNAMIC. Conveniently, this is the
39 first element of the GOT. This must be inlined in a function which
40 uses global data. */
41static inline ElfW(Addr) __attribute__ ((unused))
42elf_machine_dynamic (void)
43{
44 /* This produces an IP-relative reloc which is resolved at link time. */
45 extern const ElfW(Addr) _GLOBAL_OFFSET_TABLE_[] attribute_hidden;
46 return _GLOBAL_OFFSET_TABLE_[0];
47}
48
49
50/* Return the run-time load address of the shared object. */
51static inline ElfW(Addr) __attribute__ ((unused))
52elf_machine_load_address (void)
53{
54 /* Compute the difference between the runtime address of _DYNAMIC as seen
55 by an IP-relative reference, and the link-time address found in the
56 special unrelocated first GOT entry. */
57 extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
58 return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic ();
59}
60
61/* Set up the loaded object described by L so its unrelocated PLT
62 entries will jump to the on-demand fixup code in dl-runtime.c. */
63
64static inline int __attribute__ ((unused, always_inline))
65elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
66{
67 Elf64_Addr *got;
68 extern void _dl_runtime_resolve_fxsave (ElfW(Word)) attribute_hidden;
69 extern void _dl_runtime_resolve_xsave (ElfW(Word)) attribute_hidden;
70 extern void _dl_runtime_resolve_xsavec (ElfW(Word)) attribute_hidden;
71 extern void _dl_runtime_profile_sse (ElfW(Word)) attribute_hidden;
72 extern void _dl_runtime_profile_avx (ElfW(Word)) attribute_hidden;
73 extern void _dl_runtime_profile_avx512 (ElfW(Word)) attribute_hidden;
74
75 if (l->l_info[DT_JMPREL] && lazy)
76 {
77 /* The GOT entries for functions in the PLT have not yet been filled
78 in. Their initial contents will arrange when called to push an
79 offset into the .rel.plt section, push _GLOBAL_OFFSET_TABLE_[1],
80 and then jump to _GLOBAL_OFFSET_TABLE_[2]. */
81 got = (Elf64_Addr *) D_PTR (l, l_info[DT_PLTGOT]);
82 /* If a library is prelinked but we have to relocate anyway,
83 we have to be able to undo the prelinking of .got.plt.
84 The prelinker saved us here address of .plt + 0x16. */
85 if (got[1])
86 {
87 l->l_mach.plt = got[1] + l->l_addr;
88 l->l_mach.gotplt = (ElfW(Addr)) &got[3];
89 }
90 /* Identify this shared object. */
91 *(ElfW(Addr) *) (got + 1) = (ElfW(Addr)) l;
92
93 /* The got[2] entry contains the address of a function which gets
94 called to get the address of a so far unresolved function and
95 jump to it. The profiling extension of the dynamic linker allows
96 to intercept the calls to collect information. In this case we
97 don't store the address in the GOT so that all future calls also
98 end in this function. */
99 if (__glibc_unlikely (profile))
100 {
101 if (CPU_FEATURE_USABLE (AVX512F))
102 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx512;
103 else if (CPU_FEATURE_USABLE (AVX))
104 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx;
105 else
106 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_sse;
107
108 if (GLRO(dl_profile) != NULL
109 && _dl_name_match_p (GLRO(dl_profile), l))
110 /* This is the object we are looking for. Say that we really
111 want profiling and the timers are started. */
112 GL(dl_profile_map) = l;
113 }
114 else
115 {
116 /* This function will get called to fix up the GOT entry
117 indicated by the offset on the stack, and then jump to
118 the resolved address. */
119 if (GLRO(dl_x86_cpu_features).xsave_state_size != 0)
120 *(ElfW(Addr) *) (got + 2)
121 = (CPU_FEATURE_USABLE (XSAVEC)
122 ? (ElfW(Addr)) &_dl_runtime_resolve_xsavec
123 : (ElfW(Addr)) &_dl_runtime_resolve_xsave);
124 else
125 *(ElfW(Addr) *) (got + 2)
126 = (ElfW(Addr)) &_dl_runtime_resolve_fxsave;
127 }
128 }
129
130 if (l->l_info[ADDRIDX (DT_TLSDESC_GOT)] && lazy)
131 *(ElfW(Addr)*)(D_PTR (l, l_info[ADDRIDX (DT_TLSDESC_GOT)]) + l->l_addr)
132 = (ElfW(Addr)) &_dl_tlsdesc_resolve_rela;
133
134 return lazy;
135}
136
137/* Initial entry point code for the dynamic linker.
138 The C function `_dl_start' is the real entry point;
139 its return value is the user program's entry point. */
140#define RTLD_START asm ("\n\
141.text\n\
142 .align 16\n\
143.globl _start\n\
144.globl _dl_start_user\n\
145_start:\n\
146 movq %rsp, %rdi\n\
147 call _dl_start\n\
148_dl_start_user:\n\
149 # Save the user entry point address in %r12.\n\
150 movq %rax, %r12\n\
151 # See if we were run as a command with the executable file\n\
152 # name as an extra leading argument.\n\
153 movl _dl_skip_args(%rip), %eax\n\
154 # Pop the original argument count.\n\
155 popq %rdx\n\
156 # Adjust the stack pointer to skip _dl_skip_args words.\n\
157 leaq (%rsp,%rax,8), %rsp\n\
158 # Subtract _dl_skip_args from argc.\n\
159 subl %eax, %edx\n\
160 # Push argc back on the stack.\n\
161 pushq %rdx\n\
162 # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\
163 # argc -> rsi\n\
164 movq %rdx, %rsi\n\
165 # Save %rsp value in %r13.\n\
166 movq %rsp, %r13\n\
167 # And align stack for the _dl_init call. \n\
168 andq $-16, %rsp\n\
169 # _dl_loaded -> rdi\n\
170 movq _rtld_local(%rip), %rdi\n\
171 # env -> rcx\n\
172 leaq 16(%r13,%rdx,8), %rcx\n\
173 # argv -> rdx\n\
174 leaq 8(%r13), %rdx\n\
175 # Clear %rbp to mark outermost frame obviously even for constructors.\n\
176 xorl %ebp, %ebp\n\
177 # Call the function to run the initializers.\n\
178 call _dl_init\n\
179 # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\
180 leaq _dl_fini(%rip), %rdx\n\
181 # And make sure %rsp points to argc stored on the stack.\n\
182 movq %r13, %rsp\n\
183 # Jump to the user's entry point.\n\
184 jmp *%r12\n\
185.previous\n\
186");
187
188/* ELF_RTYPE_CLASS_PLT iff TYPE describes relocation of a PLT entry or
189 TLS variable, so undefined references should not be allowed to
190 define the value.
191 ELF_RTYPE_CLASS_COPY iff TYPE should not be allowed to resolve to one
192 of the main executable's symbols, as for a COPY reloc.
193 ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA iff TYPE describes relocation may
194 against protected data whose address be external due to copy relocation.
195 */
196#define elf_machine_type_class(type) \
197 ((((type) == R_X86_64_JUMP_SLOT \
198 || (type) == R_X86_64_DTPMOD64 \
199 || (type) == R_X86_64_DTPOFF64 \
200 || (type) == R_X86_64_TPOFF64 \
201 || (type) == R_X86_64_TLSDESC) \
202 * ELF_RTYPE_CLASS_PLT) \
203 | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY) \
204 | (((type) == R_X86_64_GLOB_DAT) * ELF_RTYPE_CLASS_EXTERN_PROTECTED_DATA))
205
206/* A reloc type used for ld.so cmdline arg lookups to reject PLT entries. */
207#define ELF_MACHINE_JMP_SLOT R_X86_64_JUMP_SLOT
208
209/* The relative ifunc relocation. */
210// XXX This is a work-around for a broken linker. Remove!
211#define ELF_MACHINE_IRELATIVE R_X86_64_IRELATIVE
212
213/* The x86-64 never uses Elf64_Rel/Elf32_Rel relocations. */
214#define ELF_MACHINE_NO_REL 1
215#define ELF_MACHINE_NO_RELA 0
216
217/* We define an initialization function. This is called very early in
218 _dl_sysdep_start. */
219#define DL_PLATFORM_INIT dl_platform_init ()
220
221static inline void __attribute__ ((unused))
222dl_platform_init (void)
223{
224#if IS_IN (rtld)
225 /* _dl_x86_init_cpu_features is a wrapper for init_cpu_features which
226 has been called early from __libc_start_main in static executable. */
227 _dl_x86_init_cpu_features ();
228#else
229 if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
230 /* Avoid an empty string which would disturb us. */
231 GLRO(dl_platform) = NULL;
232#endif
233}
234
235static inline ElfW(Addr)
236elf_machine_fixup_plt (struct link_map *map, lookup_t t,
237 const ElfW(Sym) *refsym, const ElfW(Sym) *sym,
238 const ElfW(Rela) *reloc,
239 ElfW(Addr) *reloc_addr, ElfW(Addr) value)
240{
241 return *reloc_addr = value;
242}
243
244/* Return the final value of a PLT relocation. On x86-64 the
245 JUMP_SLOT relocation ignores the addend. */
246static inline ElfW(Addr)
247elf_machine_plt_value (struct link_map *map, const ElfW(Rela) *reloc,
248 ElfW(Addr) value)
249{
250 return value;
251}
252
253
254/* Names of the architecture-specific auditing callback functions. */
255#define ARCH_LA_PLTENTER x86_64_gnu_pltenter
256#define ARCH_LA_PLTEXIT x86_64_gnu_pltexit
257
258#endif /* !dl_machine_h */
259
260#ifdef RESOLVE_MAP
261
262/* Perform the relocation specified by RELOC and SYM (which is fully resolved).
263 MAP is the object containing the reloc. */
264
265auto inline void
266__attribute__ ((always_inline))
267elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc,
268 const ElfW(Sym) *sym, const struct r_found_version *version,
269 void *const reloc_addr_arg, int skip_ifunc)
270{
271 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
272 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
273
274# if !defined RTLD_BOOTSTRAP || !defined HAVE_Z_COMBRELOC
275 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE))
276 {
277# if !defined RTLD_BOOTSTRAP && !defined HAVE_Z_COMBRELOC
278 /* This is defined in rtld.c, but nowhere in the static libc.a;
279 make the reference weak so static programs can still link.
280 This declaration cannot be done when compiling rtld.c
281 (i.e. #ifdef RTLD_BOOTSTRAP) because rtld.c contains the
282 common defn for _dl_rtld_map, which is incompatible with a
283 weak decl in the same file. */
284# ifndef SHARED
285 weak_extern (GL(dl_rtld_map));
286# endif
287 if (map != &GL(dl_rtld_map)) /* Already done in rtld itself. */
288# endif
289 *reloc_addr = map->l_addr + reloc->r_addend;
290 }
291 else
292# endif
293# if !defined RTLD_BOOTSTRAP
294 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
295 relocation updates the whole 64-bit entry. */
296 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE64))
297 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) map->l_addr + reloc->r_addend;
298 else
299# endif
300 if (__glibc_unlikely (r_type == R_X86_64_NONE))
301 return;
302 else
303 {
304# ifndef RTLD_BOOTSTRAP
305 const ElfW(Sym) *const refsym = sym;
306# endif
307 struct link_map *sym_map = RESOLVE_MAP (&sym, version, r_type);
308 ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);
309
310 if (sym != NULL
311 && __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
312 && __glibc_likely (sym->st_shndx != SHN_UNDEF)
313 && __glibc_likely (!skip_ifunc))
314 {
315# ifndef RTLD_BOOTSTRAP
316 if (sym_map != map
317 && !sym_map->l_relocated)
318 {
319 const char *strtab
320 = (const char *) D_PTR (map, l_info[DT_STRTAB]);
321 if (sym_map->l_type == lt_executable)
322 _dl_fatal_printf ("\
323%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \
324and creates an unsatisfiable circular dependency.\n",
325 RTLD_PROGNAME, strtab + refsym->st_name,
326 map->l_name);
327 else
328 _dl_error_printf ("\
329%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
330 RTLD_PROGNAME, map->l_name,
331 sym_map->l_name,
332 strtab + refsym->st_name);
333 }
334# endif
335 value = ((ElfW(Addr) (*) (void)) value) ();
336 }
337
338 switch (r_type)
339 {
340# ifndef RTLD_BOOTSTRAP
341# ifdef __ILP32__
342 case R_X86_64_SIZE64:
343 /* Set to symbol size plus addend. */
344 *(Elf64_Addr *) (uintptr_t) reloc_addr
345 = (Elf64_Addr) sym->st_size + reloc->r_addend;
346 break;
347
348 case R_X86_64_SIZE32:
349# else
350 case R_X86_64_SIZE64:
351# endif
352 /* Set to symbol size plus addend. */
353 value = sym->st_size;
354# endif
355 /* Fall through. */
356 case R_X86_64_GLOB_DAT:
357 case R_X86_64_JUMP_SLOT:
358 *reloc_addr = value + reloc->r_addend;
359 break;
360
361# ifndef RESOLVE_CONFLICT_FIND_MAP
362 case R_X86_64_DTPMOD64:
363# ifdef RTLD_BOOTSTRAP
364 /* During startup the dynamic linker is always the module
365 with index 1.
366 XXX If this relocation is necessary move before RESOLVE
367 call. */
368 *reloc_addr = 1;
369# else
370 /* Get the information from the link map returned by the
371 resolve function. */
372 if (sym_map != NULL)
373 *reloc_addr = sym_map->l_tls_modid;
374# endif
375 break;
376 case R_X86_64_DTPOFF64:
377# ifndef RTLD_BOOTSTRAP
378 /* During relocation all TLS symbols are defined and used.
379 Therefore the offset is already correct. */
380 if (sym != NULL)
381 {
382 value = sym->st_value + reloc->r_addend;
383# ifdef __ILP32__
384 /* This relocation type computes a signed offset that is
385 usually negative. The symbol and addend values are 32
386 bits but the GOT entry is 64 bits wide and the whole
387 64-bit entry is used as a signed quantity, so we need
388 to sign-extend the computed value to 64 bits. */
389 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
390# else
391 *reloc_addr = value;
392# endif
393 }
394# endif
395 break;
396 case R_X86_64_TLSDESC:
397 {
398 struct tlsdesc volatile *td =
399 (struct tlsdesc volatile *)reloc_addr;
400
401# ifndef RTLD_BOOTSTRAP
402 if (! sym)
403 {
404 td->arg = (void*)reloc->r_addend;
405 td->entry = _dl_tlsdesc_undefweak;
406 }
407 else
408# endif
409 {
410# ifndef RTLD_BOOTSTRAP
411# ifndef SHARED
412 CHECK_STATIC_TLS (map, sym_map);
413# else
414 if (!TRY_STATIC_TLS (map, sym_map))
415 {
416 td->arg = _dl_make_tlsdesc_dynamic
417 (sym_map, sym->st_value + reloc->r_addend);
418 td->entry = _dl_tlsdesc_dynamic;
419 }
420 else
421# endif
422# endif
423 {
424 td->arg = (void*)(sym->st_value - sym_map->l_tls_offset
425 + reloc->r_addend);
426 td->entry = _dl_tlsdesc_return;
427 }
428 }
429 break;
430 }
431 case R_X86_64_TPOFF64:
432 /* The offset is negative, forward from the thread pointer. */
433# ifndef RTLD_BOOTSTRAP
434 if (sym != NULL)
435# endif
436 {
437# ifndef RTLD_BOOTSTRAP
438 CHECK_STATIC_TLS (map, sym_map);
439# endif
440 /* We know the offset of the object the symbol is contained in.
441 It is a negative value which will be added to the
442 thread pointer. */
443 value = (sym->st_value + reloc->r_addend
444 - sym_map->l_tls_offset);
445# ifdef __ILP32__
446 /* The symbol and addend values are 32 bits but the GOT
447 entry is 64 bits wide and the whole 64-bit entry is used
448 as a signed quantity, so we need to sign-extend the
449 computed value to 64 bits. */
450 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
451# else
452 *reloc_addr = value;
453# endif
454 }
455 break;
456# endif
457
458# ifndef RTLD_BOOTSTRAP
459 case R_X86_64_64:
460 /* value + r_addend may be > 0xffffffff and R_X86_64_64
461 relocation updates the whole 64-bit entry. */
462 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend;
463 break;
464# ifndef __ILP32__
465 case R_X86_64_SIZE32:
466 /* Set to symbol size plus addend. */
467 value = sym->st_size;
468# endif
469 /* Fall through. */
470 case R_X86_64_32:
471 value += reloc->r_addend;
472 *(unsigned int *) reloc_addr = value;
473
474 const char *fmt;
475 if (__glibc_unlikely (value > UINT_MAX))
476 {
477 const char *strtab;
478
479 fmt = "\
480%s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n";
481# ifndef RESOLVE_CONFLICT_FIND_MAP
482 print_err:
483# endif
484 strtab = (const char *) D_PTR (map, l_info[DT_STRTAB]);
485
486 _dl_error_printf (fmt, RTLD_PROGNAME, strtab + refsym->st_name);
487 }
488 break;
489# ifndef RESOLVE_CONFLICT_FIND_MAP
490 /* Not needed for dl-conflict.c. */
491 case R_X86_64_PC32:
492 value += reloc->r_addend - (ElfW(Addr)) reloc_addr;
493 *(unsigned int *) reloc_addr = value;
494 if (__glibc_unlikely (value != (int) value))
495 {
496 fmt = "\
497%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
498 goto print_err;
499 }
500 break;
501 case R_X86_64_COPY:
502 if (sym == NULL)
503 /* This can happen in trace mode if an object could not be
504 found. */
505 break;
506 memcpy (reloc_addr_arg, (void *) value,
507 MIN (sym->st_size, refsym->st_size));
508 if (__glibc_unlikely (sym->st_size > refsym->st_size)
509 || (__glibc_unlikely (sym->st_size < refsym->st_size)
510 && GLRO(dl_verbose)))
511 {
512 fmt = "\
513%s: Symbol `%s' has different size in shared object, consider re-linking\n";
514 goto print_err;
515 }
516 break;
517# endif
518 case R_X86_64_IRELATIVE:
519 value = map->l_addr + reloc->r_addend;
520 if (__glibc_likely (!skip_ifunc))
521 value = ((ElfW(Addr) (*) (void)) value) ();
522 *reloc_addr = value;
523 break;
524 default:
525 _dl_reloc_bad_type (map, r_type, 0);
526 break;
527# endif
528 }
529 }
530}
531
532auto inline void
533__attribute ((always_inline))
534elf_machine_rela_relative (ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
535 void *const reloc_addr_arg)
536{
537 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
538#if !defined RTLD_BOOTSTRAP
539 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
540 relocation updates the whole 64-bit entry. */
541 if (__glibc_unlikely (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE64))
542 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) l_addr + reloc->r_addend;
543 else
544#endif
545 {
546 assert (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE);
547 *reloc_addr = l_addr + reloc->r_addend;
548 }
549}
550
551auto inline void
552__attribute ((always_inline))
553elf_machine_lazy_rel (struct link_map *map,
554 ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
555 int skip_ifunc)
556{
557 ElfW(Addr) *const reloc_addr = (void *) (l_addr + reloc->r_offset);
558 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
559
560 /* Check for unexpected PLT reloc type. */
561 if (__glibc_likely (r_type == R_X86_64_JUMP_SLOT))
562 {
563 /* Prelink has been deprecated. */
564 if (__glibc_likely (map->l_mach.plt == 0))
565 *reloc_addr += l_addr;
566 else
567 *reloc_addr =
568 map->l_mach.plt
569 + (((ElfW(Addr)) reloc_addr) - map->l_mach.gotplt) * 2;
570 }
571 else if (__glibc_likely (r_type == R_X86_64_TLSDESC))
572 {
573 struct tlsdesc volatile * __attribute__((__unused__)) td =
574 (struct tlsdesc volatile *)reloc_addr;
575
576 td->arg = (void*)reloc;
577 td->entry = (void*)(D_PTR (map, l_info[ADDRIDX (DT_TLSDESC_PLT)])
578 + map->l_addr);
579 }
580 else if (__glibc_unlikely (r_type == R_X86_64_IRELATIVE))
581 {
582 ElfW(Addr) value = map->l_addr + reloc->r_addend;
583 if (__glibc_likely (!skip_ifunc))
584 value = ((ElfW(Addr) (*) (void)) value) ();
585 *reloc_addr = value;
586 }
587 else
588 _dl_reloc_bad_type (map, r_type, 1);
589}
590
591#endif /* RESOLVE_MAP */
592