1/* Common definition for memcpy, mempcpy and memmove implementation.
2 All versions must be listed in ifunc-impl-list.c.
3 Copyright (C) 2017-2022 Free Software Foundation, Inc.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
19
20#include <init-arch.h>
21
22extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
23
24extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
25 attribute_hidden;
26extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
27 attribute_hidden;
28extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
29 attribute_hidden;
30
31extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
32 attribute_hidden;
33extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
34 attribute_hidden;
35
36extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
37extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
38 attribute_hidden;
39extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm)
40 attribute_hidden;
41extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm)
42 attribute_hidden;
43
44extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
45
46extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
47 attribute_hidden;
48extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
49 attribute_hidden;
50
51static inline void *
52IFUNC_SELECTOR (void)
53{
54 const struct cpu_features *cpu_features = __get_cpu_features ();
55
56 if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS)
57 || CPU_FEATURES_ARCH_P (cpu_features, Prefer_FSRM))
58 return OPTIMIZE (erms);
59
60 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
61 && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
62 {
63 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
64 {
65 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
66 return OPTIMIZE (avx512_unaligned_erms);
67
68 return OPTIMIZE (avx512_unaligned);
69 }
70
71 return OPTIMIZE (avx512_no_vzeroupper);
72 }
73
74 if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
75 AVX_Fast_Unaligned_Load, ))
76 {
77 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
78 {
79 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
80 return OPTIMIZE (evex_unaligned_erms);
81
82 return OPTIMIZE (evex_unaligned);
83 }
84
85 if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
86 {
87 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
88 return OPTIMIZE (avx_unaligned_erms_rtm);
89
90 return OPTIMIZE (avx_unaligned_rtm);
91 }
92
93 if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
94 Prefer_No_VZEROUPPER, !))
95 {
96 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
97 return OPTIMIZE (avx_unaligned_erms);
98
99 return OPTIMIZE (avx_unaligned);
100 }
101 }
102
103 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)
104 /* Leave this as runtime check. The SSSE3 is optimized almost
105 exclusively for avoiding unaligned memory access during the
106 copy and by and large is not better than the sse2
107 implementation as a general purpose memmove. */
108 && !CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
109 {
110 return OPTIMIZE (ssse3);
111 }
112
113 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
114 return OPTIMIZE (sse2_unaligned_erms);
115
116 return OPTIMIZE (sse2_unaligned);
117}
118