1/*
2 * Copyright (c) 2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/machine_routines.h>
30#include <sys/sysproto.h>
31#include <sys/malloc.h>
32#include <sys/systm.h>
33#include <sys/pgo.h>
34#include <sys/kauth.h>
35#include <security/mac_framework.h>
36#include <libkern/OSKextLib.h>
37
38
39#ifdef PROFILE
40
41/* These __llvm functions are defined in InstrProfiling.h in compiler_rt. That
42 * is a internal header, so we need to re-prototype them here. */
43
44uint64_t __llvm_profile_get_size_for_buffer(void);
45int __llvm_profile_write_buffer(char *Buffer);
46uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin,
47 const char *DataEnd,
48 const char *CountersBegin,
49 const char *CountersEnd ,
50 const char *NamesBegin,
51 const char *NamesEnd);
52int __llvm_profile_write_buffer_internal(char *Buffer,
53 const char *DataBegin,
54 const char *DataEnd,
55 const char *CountersBegin,
56 const char *CountersEnd ,
57 const char *NamesBegin,
58 const char *NamesEnd);
59
60extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data");
61extern char __pgo_hib_DataEnd __asm("section$end$__HIB$__llvm_prf_data");
62extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names");
63extern char __pgo_hib_NamesEnd __asm("section$end$__HIB$__llvm_prf_names");
64extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts");
65extern char __pgo_hib_CountersEnd __asm("section$end$__HIB$__llvm_prf_cnts");
66
67
68static uint64_t get_size_for_buffer(int flags)
69{
70 if (flags & PGO_HIB) {
71 return __llvm_profile_get_size_for_buffer_internal(
72 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
73 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
74 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
75 } else {
76 return __llvm_profile_get_size_for_buffer();
77 }
78}
79
80
81static int write_buffer(int flags, char *buffer)
82{
83 if (flags & PGO_HIB) {
84 return __llvm_profile_write_buffer_internal(
85 buffer,
86 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
87 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
88 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
89 } else {
90 return __llvm_profile_write_buffer(buffer);
91 }
92}
93
94
95#endif
96
97/* this variable is used to signal to the debugger that we'd like it to reset
98 * the counters */
99int kdp_pgo_reset_counters = 0;
100
101/* called in debugger context */
102kern_return_t do_pgo_reset_counters()
103{
104#ifdef PROFILE
105 memset(&__pgo_hib_CountersStart, 0,
106 ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart)));
107#endif
108 OSKextResetPgoCounters();
109 kdp_pgo_reset_counters = 0;
110 return KERN_SUCCESS;
111}
112
113static kern_return_t
114kextpgo_trap()
115{
116 return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, NULL, FALSE, 0);
117}
118
119static kern_return_t
120pgo_reset_counters()
121{
122 kern_return_t r;
123 boolean_t istate;
124
125 OSKextResetPgoCountersLock();
126
127 istate = ml_set_interrupts_enabled(FALSE);
128
129 kdp_pgo_reset_counters = 1;
130 r = kextpgo_trap();
131
132 ml_set_interrupts_enabled(istate);
133
134 OSKextResetPgoCountersUnlock();
135 return r;
136}
137
138
139/*
140 * returns:
141 * EPERM unless you are root
142 * EINVAL for invalid args.
143 * ENOSYS for not implemented
144 * ERANGE for integer overflow
145 * ENOENT if kext not found
146 * ENOTSUP kext does not support PGO
147 * EIO llvm returned an error. shouldn't ever happen.
148 */
149
150int grab_pgo_data(struct proc *p,
151 struct grab_pgo_data_args *uap,
152 register_t *retval)
153{
154 char *buffer = NULL;
155 int err = 0;
156
157 (void) p;
158
159 if (!kauth_cred_issuser(kauth_cred_get())) {
160 err = EPERM;
161 goto out;
162 }
163
164#if CONFIG_MACF
165 err = mac_system_check_info(kauth_cred_get(), "kern.profiling_data");
166 if (err) {
167 goto out;
168 }
169#endif
170
171 if ( uap->flags & ~PGO_ALL_FLAGS ||
172 uap->size < 0 ||
173 (uap->size > 0 && uap->buffer == 0))
174 {
175 err = EINVAL;
176 goto out;
177 }
178
179 if ( uap->flags & PGO_RESET_ALL ) {
180 if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size ) {
181 err = EINVAL;
182 } else {
183 kern_return_t r = pgo_reset_counters();
184 switch (r) {
185 case KERN_SUCCESS:
186 err = 0;
187 break;
188 case KERN_OPERATION_TIMED_OUT:
189 err = ETIMEDOUT;
190 break;
191 default:
192 err = EIO;
193 break;
194 }
195 }
196 goto out;
197 }
198
199 *retval = 0;
200
201 if (uap->uuid) {
202 uuid_t uuid;
203 err = copyin(uap->uuid, &uuid, sizeof(uuid));
204 if (err) {
205 goto out;
206 }
207
208 if (uap->buffer == 0 && uap->size == 0) {
209 uint64_t size64;
210
211 if (uap->flags & PGO_WAIT_FOR_UNLOAD) {
212 err = EINVAL;
213 goto out;
214 }
215
216 err = OSKextGrabPgoData(uuid, &size64, NULL, 0, 0, !!(uap->flags & PGO_METADATA));
217 if (size64 == 0 && err == 0) {
218 err = EIO;
219 }
220 if (err) {
221 goto out;
222 }
223
224 ssize_t size = size64;
225 if ( ((uint64_t) size) != size64 ||
226 size < 0 )
227 {
228 err = ERANGE;
229 goto out;
230 }
231
232 *retval = size;
233 err = 0;
234 goto out;
235
236 } else if (!uap->buffer || uap->size <= 0) {
237
238 err = EINVAL;
239 goto out;
240
241 } else {
242
243 uint64_t size64 = 0 ;
244
245 err = OSKextGrabPgoData(uuid, &size64, NULL, 0,
246 false,
247 !!(uap->flags & PGO_METADATA));
248
249 if (size64 == 0 && err == 0) {
250 err = EIO;
251 }
252 if (err) {
253 goto out;
254 }
255
256 if (uap->size < 0 || (uint64_t)uap->size < size64) {
257 err = EINVAL;
258 goto out;
259 }
260
261 MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK | M_ZERO);
262 if (!buffer) {
263 err = ENOMEM;
264 goto out;
265 }
266
267 err = OSKextGrabPgoData(uuid, &size64, buffer, size64,
268 !!(uap->flags & PGO_WAIT_FOR_UNLOAD),
269 !!(uap->flags & PGO_METADATA));
270 if (err) {
271 goto out;
272 }
273
274 ssize_t size = size64;
275 if ( ((uint64_t) size) != size64 ||
276 size < 0 )
277 {
278 err = ERANGE;
279 goto out;
280 }
281
282 err = copyout(buffer, uap->buffer, size);
283 if (err) {
284 goto out;
285 }
286
287 *retval = size;
288 goto out;
289 }
290 }
291
292
293#ifdef PROFILE
294
295 uint64_t size64 = get_size_for_buffer(uap->flags);
296 ssize_t size = size64;
297
298 if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) {
299 err = EINVAL;
300 goto out;
301 }
302
303 if ( ((uint64_t) size) != size64 ||
304 size < 0 )
305 {
306 err = ERANGE;
307 goto out;
308 }
309
310
311 if (uap->buffer == 0 && uap->size == 0) {
312 *retval = size;
313 err = 0;
314 goto out;
315 } else if (uap->size < size) {
316 err = EINVAL;
317 goto out;
318 } else {
319 MALLOC(buffer, char *, size, M_TEMP, M_WAITOK | M_ZERO);
320 if (!buffer) {
321 err = ENOMEM;
322 goto out;
323 }
324
325 err = write_buffer(uap->flags, buffer);
326 if (err)
327 {
328 err = EIO;
329 goto out;
330 }
331
332 err = copyout(buffer, uap->buffer, size);
333 if (err) {
334 goto out;
335 }
336
337 *retval = size;
338 goto out;
339 }
340
341#else
342
343 *retval = -1;
344 err = ENOSYS;
345 goto out;
346
347#endif
348
349out:
350 if (buffer) {
351 FREE(buffer, M_TEMP);
352 }
353 if (err) {
354 *retval = -1;
355 }
356 return err;
357}
358