1/*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <kern/debug.h>
30#include <kern/kalloc.h>
31#include <sys/param.h>
32#include <sys/mman.h>
33#include <sys/stat.h>
34#include <sys/sysctl.h>
35#include <libkern/libkern.h>
36#include <kern/assert.h>
37
38#include <kern/kpc.h>
39#include <sys/ktrace.h>
40
41#include <pexpert/pexpert.h>
42#include <kperf/kperf.h>
43
44/* Various sysctl requests */
45#define REQ_CLASSES (1)
46#define REQ_COUNTING (2)
47#define REQ_THREAD_COUNTING (3)
48#define REQ_CONFIG_COUNT (4)
49#define REQ_COUNTER_COUNT (5)
50#define REQ_THREAD_COUNTERS (6)
51#define REQ_COUNTERS (7)
52#define REQ_SHADOW_COUNTERS (8)
53#define REQ_CONFIG (9)
54#define REQ_PERIOD (10)
55#define REQ_ACTIONID (11)
56#define REQ_SW_INC (14)
57#define REQ_PMU_VERSION (15)
58
59/* Type-munging casts */
60typedef int (*getint_t)(void);
61typedef int (*setint_t)(int);
62
63static int kpc_initted = 0;
64
65static lck_grp_attr_t *sysctl_lckgrp_attr = NULL;
66static lck_grp_t *sysctl_lckgrp = NULL;
67static lck_mtx_t sysctl_lock;
68
69/*
70 * Another element is needed to hold the CPU number when getting counter values.
71 */
72#define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1)
73
74typedef int (*setget_func_t)(int);
75
76void
77kpc_init(void)
78{
79 sysctl_lckgrp_attr = lck_grp_attr_alloc_init();
80 sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr);
81 lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL);
82
83 kpc_arch_init();
84 kpc_common_init();
85 kpc_thread_init();
86
87 kpc_initted = 1;
88}
89
90static uint64_t *
91kpc_get_bigarray(uint32_t *size_out)
92{
93 static uint64_t *bigarray = NULL;
94
95 LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED);
96
97 uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t);
98 *size_out = size;
99
100 if (bigarray) {
101 return bigarray;
102 }
103
104 /*
105 * Another element is needed to hold the CPU number when getting counter
106 * values.
107 */
108 bigarray = kalloc_tag(size, VM_KERN_MEMORY_DIAG);
109 assert(bigarray != NULL);
110 return bigarray;
111}
112
113/* abstract sysctl handlers */
114static int
115sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
116 uint32_t value )
117{
118 int error = 0;
119
120 /* copy out the old value */
121 error = sysctl_handle_int(oidp, &value, 0, req);
122
123 return error;
124}
125
126static int
127sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
128{
129 int error = 0;
130 int value = 0;
131
132 error = SYSCTL_IN( req, &value, sizeof(value) );
133 if( error )
134 return error;
135
136 error = set_func( value );
137
138 return error;
139}
140
141static int
142sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
143 int (*get_func)(void), int (*set_func)(int) )
144{
145 int error = 0;
146 uint32_t value = 0;
147
148 /* get the old value and process it */
149 value = get_func();
150
151 /* copy out the old value, get the new value */
152 error = sysctl_handle_int(oidp, &value, 0, req);
153 if (error || !req->newptr)
154 return (error);
155
156 /* if that worked, and we're writing... */
157 error = set_func( value );
158
159 return error;
160}
161
162
163static int
164sysctl_setget_int( struct sysctl_req *req,
165 int (*setget_func)(int) )
166{
167 int error = 0;
168 int value = 0;
169
170 error = SYSCTL_IN( req, &value, sizeof(value) );
171 if( error )
172 return error;
173
174 value = setget_func(value);
175
176 error = SYSCTL_OUT( req, &value, sizeof(value) );
177
178 return error;
179}
180
181static int
182sysctl_kpc_get_counters(uint32_t counters,
183 uint32_t *size, void *buf)
184{
185 uint64_t *ctr_buf = (uint64_t*)buf;
186 int curcpu;
187 uint32_t count;
188
189 count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
190 counters,
191 &curcpu, &ctr_buf[1]);
192 if (!count)
193 return EINVAL;
194
195 ctr_buf[0] = curcpu;
196
197 *size = (count+1) * sizeof(uint64_t);
198
199 return 0;
200}
201
202static int
203sysctl_kpc_get_shadow_counters(uint32_t counters,
204 uint32_t *size, void *buf)
205{
206 uint64_t *ctr_buf = (uint64_t*)buf;
207 int curcpu;
208 uint32_t count;
209
210 count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
211 counters,
212 &curcpu, &ctr_buf[1]);
213
214 if (!count)
215 return EINVAL;
216
217 ctr_buf[0] = curcpu;
218
219 *size = (count+1) * sizeof(uint64_t);
220
221 return 0;
222}
223
224static int
225sysctl_kpc_get_thread_counters(uint32_t tid,
226 uint32_t *size, void *buf)
227{
228 uint32_t count = *size / sizeof(uint64_t);
229 int r;
230
231 if( tid != 0 )
232 return EINVAL;
233
234 r = kpc_get_curthread_counters(&count, buf);
235 if( !r )
236 *size = count * sizeof(uint64_t);
237
238 return r;
239}
240
241static int
242sysctl_kpc_get_config(uint32_t classes, void* buf)
243{
244 return kpc_get_config( classes, buf );
245}
246
247static int
248sysctl_kpc_set_config(uint32_t classes, void* buf)
249{
250 /* userspace cannot reconfigure the power class */
251 if (classes & KPC_CLASS_POWER_MASK)
252 return (EPERM);
253 return kpc_set_config( classes, buf);
254}
255
256static int
257sysctl_kpc_get_period(uint32_t classes, void* buf)
258{
259 return kpc_get_period( classes, buf );
260}
261
262static int
263sysctl_kpc_set_period(uint32_t classes, void* buf)
264{
265 /* userspace cannot reconfigure the power class */
266 if (classes & KPC_CLASS_POWER_MASK)
267 return (EPERM);
268 return kpc_set_period( classes, buf);
269}
270
271static int
272sysctl_kpc_get_actionid(uint32_t classes, void* buf)
273{
274 return kpc_get_actionid( classes, buf );
275}
276
277static int
278sysctl_kpc_set_actionid(uint32_t classes, void* buf)
279{
280 return kpc_set_actionid( classes, buf);
281}
282
283
284static int
285sysctl_get_bigarray(struct sysctl_req *req,
286 int (*get_fn)(uint32_t, uint32_t*, void*))
287{
288 uint32_t bufsize = 0;
289 uint64_t *buf = kpc_get_bigarray(&bufsize);
290 uint32_t arg = 0;
291
292 /* get the argument */
293 int error = SYSCTL_IN(req, &arg, sizeof(arg));
294 if (error) {
295 return error;
296 }
297
298 error = get_fn(arg, &bufsize, buf);
299 if (!error) {
300 error = SYSCTL_OUT(req, buf, bufsize);
301 }
302
303 return error;
304}
305
306/* given a config word, how many bytes does it take? */
307static int
308sysctl_config_size( uint32_t config )
309{
310 return kpc_get_config_count(config) * sizeof(kpc_config_t);
311}
312
313static int
314sysctl_counter_size( uint32_t classes )
315{
316 return kpc_get_counter_count(classes) * sizeof(uint64_t);
317}
318
319static int
320sysctl_actionid_size( uint32_t classes )
321{
322 return kpc_get_counter_count(classes) * sizeof(int32_t);
323}
324
325static int
326sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
327 int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
328{
329 int error = 0;
330 uint64_t arg;
331
332 uint32_t bufsize = 0;
333 uint64_t *buf = kpc_get_bigarray(&bufsize);
334
335 /* get the config word */
336 error = SYSCTL_IN(req, &arg, sizeof(arg));
337 if (error) {
338 return error;
339 }
340
341 /* Determine the size of registers to modify. */
342 uint32_t regsize = size_fn((uint32_t)arg);
343 if (regsize == 0 || regsize > bufsize) {
344 return EINVAL;
345 }
346
347 /* if writing */
348 if (req->newptr) {
349 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
350 error = SYSCTL_IN(req, buf, regsize);
351
352 /* SYSCTL_IN failure means only need to read */
353 if (!error) {
354 error = set_fn((uint32_t)arg, buf);
355 if (error) {
356 return error;
357 }
358 }
359 }
360
361 /* if reading */
362 if (req->oldptr) {
363 error = get_fn((uint32_t)arg, buf);
364 if (error) {
365 return error;
366 }
367
368 error = SYSCTL_OUT(req, buf, regsize);
369 }
370
371 return error;
372}
373
374static int
375kpc_sysctl SYSCTL_HANDLER_ARGS
376{
377 int ret;
378
379 // __unused struct sysctl_oid *unused_oidp = oidp;
380 (void)arg2;
381
382 if (!kpc_initted) {
383 panic("kpc_init not called");
384 }
385
386 if (!kpc_supported) {
387 return ENOTSUP;
388 }
389
390 ktrace_lock();
391
392 // Most sysctls require an access check, but a few are public.
393 switch( (uintptr_t) arg1 ) {
394 case REQ_CLASSES:
395 case REQ_CONFIG_COUNT:
396 case REQ_COUNTER_COUNT:
397 // These read-only sysctls are public.
398 break;
399
400 default:
401 // Require kperf access to read or write anything else.
402 // This is either root or the blessed pid.
403 if ((ret = ktrace_read_check())) {
404 ktrace_unlock();
405 return ret;
406 }
407 break;
408 }
409
410 ktrace_unlock();
411
412 lck_mtx_lock(&sysctl_lock);
413
414 /* which request */
415 switch( (uintptr_t) arg1 )
416 {
417 case REQ_CLASSES:
418 ret = sysctl_get_int( oidp, req,
419 kpc_get_classes() );
420 break;
421 case REQ_COUNTING:
422 ret = sysctl_getset_int( oidp, req,
423 (getint_t)kpc_get_running,
424 (setint_t)kpc_set_running );
425 break;
426 case REQ_THREAD_COUNTING:
427 ret = sysctl_getset_int( oidp, req,
428 (getint_t)kpc_get_thread_counting,
429 (setint_t)kpc_set_thread_counting );
430 break;
431
432 case REQ_CONFIG_COUNT:
433 ret = sysctl_setget_int( req,
434 (setget_func_t)kpc_get_config_count );
435 break;
436
437 case REQ_COUNTER_COUNT:
438 ret = sysctl_setget_int( req,
439 (setget_func_t)kpc_get_counter_count );
440 break;
441
442
443 case REQ_THREAD_COUNTERS:
444 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
445 break;
446
447 case REQ_COUNTERS:
448 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
449 break;
450
451 case REQ_SHADOW_COUNTERS:
452 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
453 break;
454
455 case REQ_CONFIG:
456 ret = sysctl_getset_bigarray( req,
457 sysctl_config_size,
458 sysctl_kpc_get_config,
459 sysctl_kpc_set_config );
460 break;
461
462 case REQ_PERIOD:
463 ret = sysctl_getset_bigarray( req,
464 sysctl_counter_size,
465 sysctl_kpc_get_period,
466 sysctl_kpc_set_period );
467 break;
468
469 case REQ_ACTIONID:
470 ret = sysctl_getset_bigarray( req,
471 sysctl_actionid_size,
472 sysctl_kpc_get_actionid,
473 sysctl_kpc_set_actionid );
474 break;
475
476
477 case REQ_SW_INC:
478 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
479 break;
480
481 case REQ_PMU_VERSION:
482 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
483 break;
484
485 default:
486 ret = ENOENT;
487 break;
488 }
489
490 lck_mtx_unlock(&sysctl_lock);
491
492 return ret;
493}
494
495
496/*** sysctl definitions ***/
497
498/* root kperf node */
499SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
500 "kpc");
501
502/* values */
503SYSCTL_PROC(_kpc, OID_AUTO, classes,
504 CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
505 (void*)REQ_CLASSES,
506 sizeof(int), kpc_sysctl, "I", "Available classes");
507
508SYSCTL_PROC(_kpc, OID_AUTO, counting,
509 CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
510 (void*)REQ_COUNTING,
511 sizeof(int), kpc_sysctl, "I", "PMCs counting");
512
513SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
514 CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
515 (void*)REQ_THREAD_COUNTING,
516 sizeof(int), kpc_sysctl, "I", "Thread accumulation");
517
518SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
519 CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
520 (void *)REQ_PMU_VERSION,
521 sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
522
523/* faux values */
524SYSCTL_PROC(_kpc, OID_AUTO, config_count,
525 CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
526 (void*)REQ_CONFIG_COUNT,
527 sizeof(int), kpc_sysctl, "S", "Config count");
528
529SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
530 CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
531 (void*)REQ_COUNTER_COUNT,
532 sizeof(int), kpc_sysctl, "S", "Counter count");
533
534SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
535 CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
536 (void*)REQ_SW_INC,
537 sizeof(int), kpc_sysctl, "S", "Software increment");
538
539/* arrays */
540SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
541 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
542 (void*)REQ_THREAD_COUNTERS,
543 sizeof(uint64_t), kpc_sysctl,
544 "QU", "Current thread counters");
545
546SYSCTL_PROC(_kpc, OID_AUTO, counters,
547 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
548 (void*)REQ_COUNTERS,
549 sizeof(uint64_t), kpc_sysctl,
550 "QU", "Current counters");
551
552SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
553 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
554 (void*)REQ_SHADOW_COUNTERS,
555 sizeof(uint64_t), kpc_sysctl,
556 "QU", "Current shadow counters");
557
558SYSCTL_PROC(_kpc, OID_AUTO, config,
559 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
560 (void*)REQ_CONFIG,
561 sizeof(uint64_t), kpc_sysctl,
562 "QU", "Set counter configs");
563
564SYSCTL_PROC(_kpc, OID_AUTO, period,
565 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
566 (void*)REQ_PERIOD,
567 sizeof(uint64_t), kpc_sysctl,
568 "QU", "Set counter periods");
569
570SYSCTL_PROC(_kpc, OID_AUTO, actionid,
571 CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED,
572 (void*)REQ_ACTIONID,
573 sizeof(uint32_t), kpc_sysctl,
574 "QU", "Set counter actionids");
575
576
577