1/* Hardware capability support for run-time dynamic loader.
2 Copyright (C) 2012-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <elf.h>
21#include <errno.h>
22#include <libintl.h>
23#include <unistd.h>
24#include <ldsodefs.h>
25
26#include <dl-procinfo.h>
27#include <dl-hwcaps.h>
28
29/* This is the result of counting the substrings in a colon-separated
30 hwcaps string. */
31struct hwcaps_counts
32{
33 /* Number of substrings. */
34 size_t count;
35
36 /* Sum of the individual substring lengths (without separators or
37 null terminators). */
38 size_t total_length;
39
40 /* Maximum length of an individual substring. */
41 size_t maximum_length;
42};
43
44/* Update *COUNTS according to the contents of HWCAPS. Skip over
45 entries whose bit is not set in MASK. */
46static void
47update_hwcaps_counts (struct hwcaps_counts *counts, const char *hwcaps,
48 uint32_t bitmask, const char *mask)
49{
50 struct dl_hwcaps_split_masked sp;
51 _dl_hwcaps_split_masked_init (&sp, hwcaps, bitmask, mask);
52 while (_dl_hwcaps_split_masked (&sp))
53 {
54 ++counts->count;
55 counts->total_length += sp.split.length;
56 if (sp.split.length > counts->maximum_length)
57 counts->maximum_length = sp.split.length;
58 }
59}
60
61/* State for copy_hwcaps. Must be initialized to point to
62 the storage areas for the array and the strings themselves. */
63struct copy_hwcaps
64{
65 struct r_strlenpair *next_pair;
66 char *next_string;
67};
68
69/* Copy HWCAPS into the string pairs and strings, advancing *TARGET.
70 Skip over entries whose bit is not set in MASK. */
71static void
72copy_hwcaps (struct copy_hwcaps *target, const char *hwcaps,
73 uint32_t bitmask, const char *mask)
74{
75 struct dl_hwcaps_split_masked sp;
76 _dl_hwcaps_split_masked_init (&sp, hwcaps, bitmask, mask);
77 while (_dl_hwcaps_split_masked (&sp))
78 {
79 target->next_pair->str = target->next_string;
80 char *slash = __mempcpy (__mempcpy (target->next_string,
81 GLIBC_HWCAPS_PREFIX,
82 strlen (GLIBC_HWCAPS_PREFIX)),
83 sp.split.segment, sp.split.length);
84 *slash = '/';
85 target->next_pair->len
86 = strlen (GLIBC_HWCAPS_PREFIX) + sp.split.length + 1;
87 ++target->next_pair;
88 target->next_string = slash + 1;
89 }
90}
91
92struct dl_hwcaps_priority *_dl_hwcaps_priorities;
93uint32_t _dl_hwcaps_priorities_length;
94
95/* Allocate _dl_hwcaps_priorities and fill it with data. */
96static void
97compute_priorities (size_t total_count, const char *prepend,
98 uint32_t bitmask, const char *mask)
99{
100 _dl_hwcaps_priorities = malloc (total_count
101 * sizeof (*_dl_hwcaps_priorities));
102 if (_dl_hwcaps_priorities == NULL)
103 _dl_signal_error (ENOMEM, NULL, NULL,
104 N_("cannot create HWCAP priorities"));
105 _dl_hwcaps_priorities_length = total_count;
106
107 /* First the prepended subdirectories. */
108 size_t i = 0;
109 {
110 struct dl_hwcaps_split sp;
111 _dl_hwcaps_split_init (&sp, prepend);
112 while (_dl_hwcaps_split (&sp))
113 {
114 _dl_hwcaps_priorities[i].name = sp.segment;
115 _dl_hwcaps_priorities[i].name_length = sp.length;
116 _dl_hwcaps_priorities[i].priority = i + 1;
117 ++i;
118 }
119 }
120
121 /* Then the built-in subdirectories that are actually active. */
122 {
123 struct dl_hwcaps_split_masked sp;
124 _dl_hwcaps_split_masked_init (&sp, _dl_hwcaps_subdirs, bitmask, mask);
125 while (_dl_hwcaps_split_masked (&sp))
126 {
127 _dl_hwcaps_priorities[i].name = sp.split.segment;
128 _dl_hwcaps_priorities[i].name_length = sp.split.length;
129 _dl_hwcaps_priorities[i].priority = i + 1;
130 ++i;
131 }
132 }
133 assert (i == total_count);
134}
135
136/* Sort the _dl_hwcaps_priorities array by name. */
137static void
138sort_priorities_by_name (void)
139{
140 /* Insertion sort. There is no need to link qsort into the dynamic
141 loader for such a short array. */
142 for (size_t i = 1; i < _dl_hwcaps_priorities_length; ++i)
143 for (size_t j = i; j > 0; --j)
144 {
145 struct dl_hwcaps_priority *previous = _dl_hwcaps_priorities + j - 1;
146 struct dl_hwcaps_priority *current = _dl_hwcaps_priorities + j;
147
148 /* Bail out if current is greater or equal to the previous
149 value. */
150 uint32_t to_compare;
151 if (current->name_length < previous->name_length)
152 to_compare = current->name_length;
153 else
154 to_compare = previous->name_length;
155 int cmp = memcmp (current->name, previous->name, to_compare);
156 if (cmp > 0
157 || (cmp == 0 && current->name_length >= previous->name_length))
158 break;
159
160 /* Swap *previous and *current. */
161 struct dl_hwcaps_priority tmp = *previous;
162 *previous = *current;
163 *current = tmp;
164 }
165}
166
167/* Return an array of useful/necessary hardware capability names. */
168const struct r_strlenpair *
169_dl_important_hwcaps (const char *glibc_hwcaps_prepend,
170 const char *glibc_hwcaps_mask,
171 size_t *sz, size_t *max_capstrlen)
172{
173 uint64_t hwcap_mask = GET_HWCAP_MASK();
174 /* Determine how many important bits are set. */
175 uint64_t masked = GLRO(dl_hwcap) & hwcap_mask;
176 size_t cnt = GLRO (dl_platform) != NULL;
177 size_t n, m;
178 struct r_strlenpair *result;
179 struct r_strlenpair *rp;
180 char *cp;
181
182 /* glibc-hwcaps subdirectories. These are exempted from the power
183 set construction below. */
184 uint32_t hwcaps_subdirs_active = _dl_hwcaps_subdirs_active ();
185 struct hwcaps_counts hwcaps_counts = { 0, };
186 update_hwcaps_counts (&hwcaps_counts, glibc_hwcaps_prepend, -1, NULL);
187 update_hwcaps_counts (&hwcaps_counts, _dl_hwcaps_subdirs,
188 hwcaps_subdirs_active, glibc_hwcaps_mask);
189 compute_priorities (hwcaps_counts.count, glibc_hwcaps_prepend,
190 hwcaps_subdirs_active, glibc_hwcaps_mask);
191 sort_priorities_by_name ();
192
193 /* Each hwcaps subdirectory has a GLIBC_HWCAPS_PREFIX string prefix
194 and a "/" suffix once stored in the result. */
195 hwcaps_counts.maximum_length += strlen (GLIBC_HWCAPS_PREFIX) + 1;
196 size_t total = (hwcaps_counts.count * (strlen (GLIBC_HWCAPS_PREFIX) + 1)
197 + hwcaps_counts.total_length);
198
199 /* Count the number of bits set in the masked value. */
200 for (n = 0; (~((1ULL << n) - 1) & masked) != 0; ++n)
201 if ((masked & (1ULL << n)) != 0)
202 ++cnt;
203
204 /* For TLS enabled builds always add 'tls'. */
205 ++cnt;
206
207 /* Create temporary data structure to generate result table. */
208 struct r_strlenpair temp[cnt];
209 m = 0;
210 for (n = 0; masked != 0; ++n)
211 if ((masked & (1ULL << n)) != 0)
212 {
213 temp[m].str = _dl_hwcap_string (n);
214 temp[m].len = strlen (temp[m].str);
215 masked ^= 1ULL << n;
216 ++m;
217 }
218 if (GLRO (dl_platform) != NULL)
219 {
220 temp[m].str = GLRO (dl_platform);
221 temp[m].len = GLRO (dl_platformlen);
222 ++m;
223 }
224
225 temp[m].str = "tls";
226 temp[m].len = 3;
227 ++m;
228
229 assert (m == cnt);
230
231 /* Determine the total size of all strings together. */
232 if (cnt == 1)
233 total += temp[0].len + 1;
234 else
235 {
236 total += temp[0].len + temp[cnt - 1].len + 2;
237 if (cnt > 2)
238 {
239 total <<= 1;
240 for (n = 1; n + 1 < cnt; ++n)
241 total += temp[n].len + 1;
242 if (cnt > 3
243 && (cnt >= sizeof (size_t) * 8
244 || total + (sizeof (*result) << 3)
245 >= (1UL << (sizeof (size_t) * 8 - cnt + 3))))
246 _dl_signal_error (ENOMEM, NULL, NULL,
247 N_("cannot create capability list"));
248
249 total <<= cnt - 3;
250 }
251 }
252
253 *sz = hwcaps_counts.count + (1 << cnt);
254
255 /* This is the overall result, including both glibc-hwcaps
256 subdirectories and the legacy hwcaps subdirectories using the
257 power set construction. */
258 struct r_strlenpair *overall_result
259 = malloc (*sz * sizeof (*result) + total);
260 if (overall_result == NULL)
261 _dl_signal_error (ENOMEM, NULL, NULL,
262 N_("cannot create capability list"));
263
264 /* Fill in the glibc-hwcaps subdirectories. */
265 {
266 struct copy_hwcaps target;
267 target.next_pair = overall_result;
268 target.next_string = (char *) (overall_result + *sz);
269 copy_hwcaps (&target, glibc_hwcaps_prepend, -1, NULL);
270 copy_hwcaps (&target, _dl_hwcaps_subdirs,
271 hwcaps_subdirs_active, glibc_hwcaps_mask);
272 /* Set up the write target for the power set construction. */
273 result = target.next_pair;
274 cp = target.next_string;
275 }
276
277
278 /* Power set construction begins here. We use a very compressed way
279 to store the various combinations of capability names. */
280
281 if (cnt == 1)
282 {
283 result[0].str = cp;
284 result[0].len = temp[0].len + 1;
285 result[1].str = cp;
286 result[1].len = 0;
287 cp = __mempcpy (cp, temp[0].str, temp[0].len);
288 *cp = '/';
289 if (result[0].len > hwcaps_counts.maximum_length)
290 *max_capstrlen = result[0].len;
291 else
292 *max_capstrlen = hwcaps_counts.maximum_length;
293
294 return overall_result;
295 }
296
297 /* Fill in the information. This follows the following scheme
298 (indices from TEMP for four strings):
299 entry #0: 0, 1, 2, 3 binary: 1111
300 #1: 0, 1, 3 1101
301 #2: 0, 2, 3 1011
302 #3: 0, 3 1001
303 This allows the representation of all possible combinations of
304 capability names in the string. First generate the strings. */
305 result[1].str = result[0].str = cp;
306#define add(idx) \
307 cp = __mempcpy (__mempcpy (cp, temp[idx].str, temp[idx].len), "/", 1);
308 if (cnt == 2)
309 {
310 add (1);
311 add (0);
312 }
313 else
314 {
315 n = 1 << (cnt - 1);
316 do
317 {
318 n -= 2;
319
320 /* We always add the last string. */
321 add (cnt - 1);
322
323 /* Add the strings which have the bit set in N. */
324 for (m = cnt - 2; m > 0; --m)
325 if ((n & (1 << m)) != 0)
326 add (m);
327
328 /* Always add the first string. */
329 add (0);
330 }
331 while (n != 0);
332 }
333#undef add
334
335 /* Now we are ready to install the string pointers and length. */
336 for (n = 0; n < (1UL << cnt); ++n)
337 result[n].len = 0;
338 n = cnt;
339 do
340 {
341 size_t mask = 1 << --n;
342
343 rp = result;
344 for (m = 1 << cnt; m > 0; ++rp)
345 if ((--m & mask) != 0)
346 rp->len += temp[n].len + 1;
347 }
348 while (n != 0);
349
350 /* The first half of the strings all include the first string. */
351 n = (1 << cnt) - 2;
352 rp = &result[2];
353 while (n != (1UL << (cnt - 1)))
354 {
355 if ((--n & 1) != 0)
356 rp[0].str = rp[-2].str + rp[-2].len;
357 else
358 rp[0].str = rp[-1].str;
359 ++rp;
360 }
361
362 /* The second half starts right after the first part of the string of
363 the corresponding entry in the first half. */
364 do
365 {
366 rp[0].str = rp[-(1 << (cnt - 1))].str + temp[cnt - 1].len + 1;
367 ++rp;
368 }
369 while (--n != 0);
370
371 /* The maximum string length. */
372 if (result[0].len > hwcaps_counts.maximum_length)
373 *max_capstrlen = result[0].len;
374 else
375 *max_capstrlen = hwcaps_counts.maximum_length;
376
377 return overall_result;
378}
379