1/* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2021 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#if HAVE_TUNABLES
20# define TUNABLE_NAMESPACE cpu
21# include <stdbool.h>
22# include <stdint.h>
23# include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
24# include <elf/dl-tunables.h>
25# include <string.h>
26# include <cpu-features.h>
27# include <ldsodefs.h>
28
29/* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
30 since IFUNC must be set up by init_cpu_features. */
31# if defined USE_MULTIARCH && !defined SHARED
32# ifdef __x86_64__
33# define DEFAULT_MEMCMP __memcmp_sse2
34# else
35# define DEFAULT_MEMCMP __memcmp_ia32
36# endif
37extern __typeof (memcmp) DEFAULT_MEMCMP;
38# else
39# define DEFAULT_MEMCMP memcmp
40# endif
41
42# define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
43 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
44 if (!DEFAULT_MEMCMP (f, #name, len)) \
45 { \
46 CPU_FEATURE_UNSET (cpu_features, name) \
47 break; \
48 }
49
50/* Disable a preferred feature NAME. We don't enable a preferred feature
51 which isn't available. */
52# define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
53 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
54 if (!DEFAULT_MEMCMP (f, #name, len)) \
55 { \
56 cpu_features->preferred[index_arch_##name] \
57 &= ~bit_arch_##name; \
58 break; \
59 }
60
61/* Enable/disable a preferred feature NAME. */
62# define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
63 disable, len) \
64 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
65 if (!DEFAULT_MEMCMP (f, #name, len)) \
66 { \
67 if (disable) \
68 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
69 else \
70 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
71 break; \
72 }
73
74/* Enable/disable a preferred feature NAME. Enable a preferred feature
75 only if the feature NEED is usable. */
76# define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
77 need, disable, len) \
78 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
79 if (!DEFAULT_MEMCMP (f, #name, len)) \
80 { \
81 if (disable) \
82 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
83 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
84 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
85 break; \
86 }
87
88attribute_hidden
89void
90TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
91{
92 /* The current IFUNC selection is based on microbenchmarks in glibc.
93 It should give the best performance for most workloads. But other
94 choices may have better performance for a particular workload or on
95 the hardware which wasn't available when the selection was made.
96 The environment variable:
97
98 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
99
100 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
101 yyy and zzz, where the feature name is case-sensitive and has to
102 match the ones in cpu-features.h. It can be used by glibc developers
103 to tune for a new processor or override the IFUNC selection to
104 improve performance for a particular workload.
105
106 NOTE: the IFUNC selection may change over time. Please check all
107 multiarch implementations when experimenting. */
108
109 const char *p = valp->strval;
110 struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
111 size_t len;
112
113 do
114 {
115 const char *c, *n;
116 bool disable;
117 size_t nl;
118
119 for (c = p; *c != ','; c++)
120 if (*c == '\0')
121 break;
122
123 len = c - p;
124 disable = *p == '-';
125 if (disable)
126 {
127 n = p + 1;
128 nl = len - 1;
129 }
130 else
131 {
132 n = p;
133 nl = len;
134 }
135 switch (nl)
136 {
137 default:
138 break;
139 case 3:
140 if (disable)
141 {
142 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
143 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
144 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
145 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
148 }
149 break;
150 case 4:
151 if (disable)
152 {
153 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
154 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
155 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
156 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
157 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
158 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
159 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
160 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
161 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
162 }
163 break;
164 case 5:
165 if (disable)
166 {
167 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
168 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
171 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
172 }
173 break;
174 case 6:
175 if (disable)
176 {
177 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
178 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
179 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
180 if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
181 {
182 /* Update xsave_state_size to XSAVE state size. */
183 cpu_features->xsave_state_size
184 = cpu_features->xsave_state_full_size;
185 CPU_FEATURE_UNSET (cpu_features, XSAVEC);
186 }
187 }
188 break;
189 case 7:
190 if (disable)
191 {
192 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
193 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
194 }
195 break;
196 case 8:
197 if (disable)
198 {
199 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
200 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
201 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
202 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
203 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
204 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
205 }
206 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
207 disable, 8);
208 break;
209 case 11:
210 {
211 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
212 Prefer_ERMS,
213 disable, 11);
214 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
215 Prefer_FSRM,
216 disable, 11);
217 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
218 Slow_SSE4_2,
219 SSE4_2,
220 disable, 11);
221 }
222 break;
223 case 15:
224 {
225 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
226 Fast_Rep_String,
227 disable, 15);
228 }
229 break;
230 case 16:
231 {
232 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
233 (n, cpu_features, Prefer_No_AVX512, AVX512F,
234 disable, 16);
235 }
236 break;
237 case 18:
238 {
239 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
240 Fast_Copy_Backward,
241 disable, 18);
242 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
243 (n, cpu_features, Prefer_AVX2_STRCMP, AVX2, disable, 18);
244 }
245 break;
246 case 19:
247 {
248 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
249 Fast_Unaligned_Load,
250 disable, 19);
251 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
252 Fast_Unaligned_Copy,
253 disable, 19);
254 }
255 break;
256 case 20:
257 {
258 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
259 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
260 20);
261 }
262 break;
263 case 21:
264 {
265 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
266 Prefer_MAP_32BIT_EXEC,
267 disable, 21);
268 }
269 break;
270 case 23:
271 {
272 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
273 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
274 disable, 23);
275 }
276 break;
277 case 24:
278 {
279 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
280 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
281 disable, 24);
282 }
283 break;
284 case 26:
285 {
286 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
287 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
288 disable, 26);
289 }
290 break;
291 }
292 p += len + 1;
293 }
294 while (*p != '\0');
295}
296
297# if CET_ENABLED
298
299attribute_hidden
300void
301TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
302{
303 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
304 GL(dl_x86_feature_control).ibt = cet_always_on;
305 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
306 GL(dl_x86_feature_control).ibt = cet_always_off;
307 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
308 sizeof ("permissive")) == 0)
309 GL(dl_x86_feature_control).ibt = cet_permissive;
310}
311
312attribute_hidden
313void
314TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
315{
316 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
317 GL(dl_x86_feature_control).shstk = cet_always_on;
318 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
319 GL(dl_x86_feature_control).shstk = cet_always_off;
320 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
321 sizeof ("permissive")) == 0)
322 GL(dl_x86_feature_control).shstk = cet_permissive;
323}
324# endif
325#endif
326