1/* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2023 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#if HAVE_TUNABLES
20# define TUNABLE_NAMESPACE cpu
21# include <stdbool.h>
22# include <stdint.h>
23# include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
24# include <elf/dl-tunables.h>
25# include <string.h>
26# include <cpu-features.h>
27# include <ldsodefs.h>
28
29/* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
30 since IFUNC must be set up by init_cpu_features. */
31# if defined USE_MULTIARCH && !defined SHARED
32# ifdef __x86_64__
33/* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h. */
34# include <sysdeps/x86_64/memcmp-isa-default-impl.h>
35# else
36# define DEFAULT_MEMCMP __memcmp_ia32
37# endif
38extern __typeof (memcmp) DEFAULT_MEMCMP;
39# else
40# define DEFAULT_MEMCMP memcmp
41# endif
42
43# define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
44 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
45 if (!DEFAULT_MEMCMP (f, #name, len)) \
46 { \
47 CPU_FEATURE_UNSET (cpu_features, name) \
48 break; \
49 }
50
51/* Disable a preferred feature NAME. We don't enable a preferred feature
52 which isn't available. */
53# define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
54 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
55 if (!DEFAULT_MEMCMP (f, #name, len)) \
56 { \
57 cpu_features->preferred[index_arch_##name] \
58 &= ~bit_arch_##name; \
59 break; \
60 }
61
62/* Enable/disable a preferred feature NAME. */
63# define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
64 disable, len) \
65 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
66 if (!DEFAULT_MEMCMP (f, #name, len)) \
67 { \
68 if (disable) \
69 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
70 else \
71 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
72 break; \
73 }
74
75/* Enable/disable a preferred feature NAME. Enable a preferred feature
76 only if the feature NEED is usable. */
77# define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
78 need, disable, len) \
79 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
80 if (!DEFAULT_MEMCMP (f, #name, len)) \
81 { \
82 if (disable) \
83 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
84 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
85 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
86 break; \
87 }
88
89attribute_hidden
90void
91TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
92{
93 /* The current IFUNC selection is based on microbenchmarks in glibc.
94 It should give the best performance for most workloads. But other
95 choices may have better performance for a particular workload or on
96 the hardware which wasn't available when the selection was made.
97 The environment variable:
98
99 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
100
101 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
102 yyy and zzz, where the feature name is case-sensitive and has to
103 match the ones in cpu-features.h. It can be used by glibc developers
104 to tune for a new processor or override the IFUNC selection to
105 improve performance for a particular workload.
106
107 NOTE: the IFUNC selection may change over time. Please check all
108 multiarch implementations when experimenting. */
109
110 const char *p = valp->strval;
111 struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
112 size_t len;
113
114 do
115 {
116 const char *c, *n;
117 bool disable;
118 size_t nl;
119
120 for (c = p; *c != ','; c++)
121 if (*c == '\0')
122 break;
123
124 len = c - p;
125 disable = *p == '-';
126 if (disable)
127 {
128 n = p + 1;
129 nl = len - 1;
130 }
131 else
132 {
133 n = p;
134 nl = len;
135 }
136 switch (nl)
137 {
138 default:
139 break;
140 case 3:
141 if (disable)
142 {
143 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
144 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
145 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
148 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
149 }
150 break;
151 case 4:
152 if (disable)
153 {
154 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
155 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
156 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
157 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
158 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
159 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
160 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
161 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
162 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
163 }
164 break;
165 case 5:
166 if (disable)
167 {
168 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
171 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
172 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
173 }
174 break;
175 case 6:
176 if (disable)
177 {
178 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
179 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
180 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
181 if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
182 {
183 /* Update xsave_state_size to XSAVE state size. */
184 cpu_features->xsave_state_size
185 = cpu_features->xsave_state_full_size;
186 CPU_FEATURE_UNSET (cpu_features, XSAVEC);
187 }
188 }
189 break;
190 case 7:
191 if (disable)
192 {
193 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
194 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
195 }
196 break;
197 case 8:
198 if (disable)
199 {
200 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
201 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
202 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
203 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
204 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
205 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
206 }
207 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
208 disable, 8);
209 break;
210 case 11:
211 {
212 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
213 Prefer_ERMS,
214 disable, 11);
215 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
216 Prefer_FSRM,
217 disable, 11);
218 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
219 Slow_SSE4_2,
220 SSE4_2,
221 disable, 11);
222 }
223 break;
224 case 15:
225 {
226 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
227 Fast_Rep_String,
228 disable, 15);
229 }
230 break;
231 case 16:
232 {
233 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
234 (n, cpu_features, Prefer_No_AVX512, AVX512F,
235 disable, 16);
236 }
237 break;
238 case 18:
239 {
240 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
241 Fast_Copy_Backward,
242 disable, 18);
243 }
244 break;
245 case 19:
246 {
247 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
248 Fast_Unaligned_Load,
249 disable, 19);
250 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
251 Fast_Unaligned_Copy,
252 disable, 19);
253 }
254 break;
255 case 20:
256 {
257 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
258 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
259 20);
260 }
261 break;
262 case 23:
263 {
264 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
265 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
266 disable, 23);
267 }
268 break;
269 case 24:
270 {
271 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
272 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
273 disable, 24);
274 }
275 break;
276 case 26:
277 {
278 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
279 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
280 disable, 26);
281 }
282 break;
283 }
284 p += len + 1;
285 }
286 while (*p != '\0');
287}
288
289# if CET_ENABLED
290
291attribute_hidden
292void
293TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
294{
295 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
296 GL(dl_x86_feature_control).ibt = cet_always_on;
297 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
298 GL(dl_x86_feature_control).ibt = cet_always_off;
299 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
300 sizeof ("permissive")) == 0)
301 GL(dl_x86_feature_control).ibt = cet_permissive;
302}
303
304attribute_hidden
305void
306TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
307{
308 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
309 GL(dl_x86_feature_control).shstk = cet_always_on;
310 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
311 GL(dl_x86_feature_control).shstk = cet_always_off;
312 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
313 sizeof ("permissive")) == 0)
314 GL(dl_x86_feature_control).shstk = cet_permissive;
315}
316# endif
317#endif
318