1/* Huge Page support. Linux implementation.
2 Copyright (C) 2021-2022 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public License as
7 published by the Free Software Foundation; either version 2.1 of the
8 License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; see the file COPYING.LIB. If
17 not, see <https://www.gnu.org/licenses/>. */
18
19#include <intprops.h>
20#include <dirent.h>
21#include <malloc-hugepages.h>
22#include <not-cancel.h>
23#include <sys/mman.h>
24
25unsigned long int
26__malloc_default_thp_pagesize (void)
27{
28 int fd = __open64_nocancel (
29 "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
30 if (fd == -1)
31 return 0;
32
33 char str[INT_BUFSIZE_BOUND (unsigned long int)];
34 ssize_t s = __read_nocancel (fd, str, sizeof (str));
35 __close_nocancel (fd);
36 if (s < 0)
37 return 0;
38
39 unsigned long int r = 0;
40 for (ssize_t i = 0; i < s; i++)
41 {
42 if (str[i] == '\n')
43 break;
44 r *= 10;
45 r += str[i] - '0';
46 }
47 return r;
48}
49
50enum malloc_thp_mode_t
51__malloc_thp_mode (void)
52{
53 int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled",
54 O_RDONLY);
55 if (fd == -1)
56 return malloc_thp_mode_not_supported;
57
58 static const char mode_always[] = "[always] madvise never\n";
59 static const char mode_madvise[] = "always [madvise] never\n";
60 static const char mode_never[] = "always madvise [never]\n";
61
62 char str[sizeof(mode_always)];
63 ssize_t s = __read_nocancel (fd, str, sizeof (str));
64 __close_nocancel (fd);
65
66 if (s == sizeof (mode_always) - 1)
67 {
68 if (strcmp (str, mode_always) == 0)
69 return malloc_thp_mode_always;
70 else if (strcmp (str, mode_madvise) == 0)
71 return malloc_thp_mode_madvise;
72 else if (strcmp (str, mode_never) == 0)
73 return malloc_thp_mode_never;
74 }
75 return malloc_thp_mode_not_supported;
76}
77
78static size_t
79malloc_default_hugepage_size (void)
80{
81 int fd = __open64_nocancel ("/proc/meminfo", O_RDONLY);
82 if (fd == -1)
83 return 0;
84
85 size_t hpsize = 0;
86
87 char buf[512];
88 off64_t off = 0;
89 while (1)
90 {
91 ssize_t r = __pread64_nocancel (fd, buf, sizeof (buf) - 1, off);
92 if (r < 0)
93 break;
94 buf[r] = '\0';
95
96 /* If the tag is not found, read the last line again. */
97 const char *s = strstr (buf, "Hugepagesize:");
98 if (s == NULL)
99 {
100 char *nl = strrchr (buf, '\n');
101 if (nl == NULL)
102 break;
103 off += (nl + 1) - buf;
104 continue;
105 }
106
107 /* The default huge page size is in the form:
108 Hugepagesize: NUMBER kB */
109 s += sizeof ("Hugepagesize: ") - 1;
110 for (int i = 0; (s[i] >= '0' && s[i] <= '9') || s[i] == ' '; i++)
111 {
112 if (s[i] == ' ')
113 continue;
114 hpsize *= 10;
115 hpsize += s[i] - '0';
116 }
117 hpsize *= 1024;
118 break;
119 }
120
121 __close_nocancel (fd);
122
123 return hpsize;
124}
125
126static inline int
127hugepage_flags (size_t pagesize)
128{
129 return MAP_HUGETLB | (__builtin_ctzll (pagesize) << MAP_HUGE_SHIFT);
130}
131
132void
133__malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags)
134{
135 *pagesize = 0;
136 *flags = 0;
137
138 if (requested == 0)
139 {
140 *pagesize = malloc_default_hugepage_size ();
141 if (*pagesize != 0)
142 *flags = hugepage_flags (*pagesize);
143 return;
144 }
145
146 /* Each entry represents a supported huge page in the form of:
147 hugepages-<size>kB. */
148 int dirfd = __open64_nocancel ("/sys/kernel/mm/hugepages",
149 O_RDONLY | O_DIRECTORY, 0);
150 if (dirfd == -1)
151 return;
152
153 char buffer[1024];
154 while (true)
155 {
156#if !IS_IN(libc)
157# define __getdents64 getdents64
158#endif
159 ssize_t ret = __getdents64 (dirfd, buffer, sizeof (buffer));
160 if (ret == -1)
161 break;
162 else if (ret == 0)
163 break;
164
165 bool found = false;
166 char *begin = buffer, *end = buffer + ret;
167 while (begin != end)
168 {
169 unsigned short int d_reclen;
170 memcpy (&d_reclen, begin + offsetof (struct dirent64, d_reclen),
171 sizeof (d_reclen));
172 const char *dname = begin + offsetof (struct dirent64, d_name);
173 begin += d_reclen;
174
175 if (dname[0] == '.'
176 || strncmp (dname, "hugepages-", sizeof ("hugepages-") - 1) != 0)
177 continue;
178
179 size_t hpsize = 0;
180 const char *sizestr = dname + sizeof ("hugepages-") - 1;
181 for (int i = 0; sizestr[i] >= '0' && sizestr[i] <= '9'; i++)
182 {
183 hpsize *= 10;
184 hpsize += sizestr[i] - '0';
185 }
186 hpsize *= 1024;
187
188 if (hpsize == requested)
189 {
190 *pagesize = hpsize;
191 *flags = hugepage_flags (*pagesize);
192 found = true;
193 break;
194 }
195 }
196 if (found)
197 break;
198 }
199
200 __close_nocancel (dirfd);
201}
202