1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/* #pragma ident "@(#)sdt.c 1.9 08/07/01 SMI" */
27
28#ifdef KERNEL
29#ifndef _KERNEL
30#define _KERNEL /* Solaris vs. Darwin */
31#endif
32#endif
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/ioctl.h>
39#include <sys/conf.h>
40#include <sys/fcntl.h>
41#include <miscfs/devfs/devfs.h>
42
43#if CONFIG_EMBEDDED
44#include <arm/caches_internal.h>
45#endif
46
47#include <sys/dtrace.h>
48#include <sys/dtrace_impl.h>
49
50#include <sys/dtrace_glue.h>
51
52#include <sys/sdt_impl.h>
53extern int dtrace_kernel_symbol_mode;
54
55/* #include <machine/trap.h */
56struct savearea_t; /* Used anonymously */
57
58#if defined(__arm__)
59typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
60extern perfCallback tempDTraceTrapHook;
61extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
62#define SDT_PATCHVAL 0xdefc
63#define SDT_AFRAMES 7
64#elif defined(__arm64__)
65typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
66extern perfCallback tempDTraceTrapHook;
67extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
68#define SDT_PATCHVAL 0xe7eeee7e
69#define SDT_AFRAMES 7
70#elif defined(__x86_64__)
71typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
72extern perfCallback tempDTraceTrapHook;
73extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);
74#define SDT_PATCHVAL 0xf0
75#define SDT_AFRAMES 6
76#else
77#error Unknown architecture
78#endif
79
80#define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */
81
82#define DTRACE_PROBE_PREFIX "_dtrace_probe$"
83
84static int sdt_verbose = 0;
85sdt_probe_t **sdt_probetab;
86int sdt_probetab_size;
87int sdt_probetab_mask;
88
89/*ARGSUSED*/
90static void
91__sdt_provide_module(void *arg, struct modctl *ctl)
92{
93#pragma unused(arg)
94 struct module *mp = (struct module *)ctl->mod_address;
95 char *modname = ctl->mod_modname;
96 sdt_probedesc_t *sdpd;
97 sdt_probe_t *sdp, *old;
98 sdt_provider_t *prov;
99 int len;
100
101 /*
102 * One for all, and all for one: if we haven't yet registered all of
103 * our providers, we'll refuse to provide anything.
104 */
105 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
106 if (prov->sdtp_id == DTRACE_PROVNONE)
107 return;
108 }
109
110 if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
111 return;
112
113 for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
114 const char *name = sdpd->sdpd_name, *func;
115 char *nname;
116 int i, j;
117 dtrace_id_t id;
118
119 for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
120 const char *prefpart, *prefix = prov->sdtp_prefix;
121
122 if ((prefpart = strstr(name, prefix))) {
123 name = prefpart + strlen(prefix);
124 break;
125 }
126 }
127
128 nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
129
130 for (i = 0, j = 0; name[j] != '\0'; i++) {
131 if (name[j] == '_' && name[j + 1] == '_') {
132 nname[i] = '-';
133 j += 2;
134 } else {
135 nname[i] = name[j++];
136 }
137 }
138
139 nname[i] = '\0';
140
141 sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
142 sdp->sdp_loadcnt = ctl->mod_loadcnt;
143 sdp->sdp_ctl = ctl;
144 sdp->sdp_name = nname;
145 sdp->sdp_namelen = len;
146 sdp->sdp_provider = prov;
147
148 func = sdpd->sdpd_func;
149
150 if (func == NULL)
151 func = "<unknown>";
152
153 /*
154 * We have our provider. Now create the probe.
155 */
156 if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
157 func, nname)) != DTRACE_IDNONE) {
158 old = dtrace_probe_arg(prov->sdtp_id, id);
159 ASSERT(old != NULL);
160
161 sdp->sdp_next = old->sdp_next;
162 sdp->sdp_id = id;
163 old->sdp_next = sdp;
164 } else {
165 sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
166 modname, func, nname, SDT_AFRAMES, sdp);
167
168 mp->sdt_nprobes++;
169 }
170
171#if 0
172 printf ("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n", sdpd, sdp, nname, sdp->sdp_id);
173#endif
174
175 sdp->sdp_hashnext =
176 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
177 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
178
179 sdp->sdp_patchval = SDT_PATCHVAL;
180 sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
181 sdp->sdp_savedval = *sdp->sdp_patchpoint;
182 }
183}
184
185/*ARGSUSED*/
186static void
187sdt_destroy(void *arg, dtrace_id_t id, void *parg)
188{
189#pragma unused(arg,id)
190 sdt_probe_t *sdp = parg, *old, *last, *hash;
191 int ndx;
192
193#if !defined(__APPLE__)
194 /*
195 * APPLE NOTE: sdt probes for kexts not yet implemented
196 */
197 struct modctl *ctl = sdp->sdp_ctl;
198
199 if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
200 if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
201 ctl->mod_loaded)) {
202 ((struct module *)(ctl->mod_mp))->sdt_nprobes--;
203 }
204 }
205#endif /* __APPLE__ */
206
207 while (sdp != NULL) {
208 old = sdp;
209
210 /*
211 * Now we need to remove this probe from the sdt_probetab.
212 */
213 ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
214 last = NULL;
215 hash = sdt_probetab[ndx];
216
217 while (hash != sdp) {
218 ASSERT(hash != NULL);
219 last = hash;
220 hash = hash->sdp_hashnext;
221 }
222
223 if (last != NULL) {
224 last->sdp_hashnext = sdp->sdp_hashnext;
225 } else {
226 sdt_probetab[ndx] = sdp->sdp_hashnext;
227 }
228
229 kmem_free(sdp->sdp_name, sdp->sdp_namelen);
230 sdp = sdp->sdp_next;
231 kmem_free(old, sizeof (sdt_probe_t));
232 }
233}
234
235/*ARGSUSED*/
236static int
237sdt_enable(void *arg, dtrace_id_t id, void *parg)
238{
239#pragma unused(arg,id)
240 sdt_probe_t *sdp = parg;
241 struct modctl *ctl = sdp->sdp_ctl;
242
243 ctl->mod_nenabled++;
244
245 /*
246 * If this module has disappeared since we discovered its probes,
247 * refuse to enable it.
248 */
249 if (!ctl->mod_loaded) {
250 if (sdt_verbose) {
251 cmn_err(CE_NOTE, "sdt is failing for probe %s "
252 "(module %s unloaded)",
253 sdp->sdp_name, ctl->mod_modname);
254 }
255 goto err;
256 }
257
258 /*
259 * Now check that our modctl has the expected load count. If it
260 * doesn't, this module must have been unloaded and reloaded -- and
261 * we're not going to touch it.
262 */
263 if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
264 if (sdt_verbose) {
265 cmn_err(CE_NOTE, "sdt is failing for probe %s "
266 "(module %s reloaded)",
267 sdp->sdp_name, ctl->mod_modname);
268 }
269 goto err;
270 }
271
272 dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
273 if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
274 if (sdt_verbose) {
275 cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
276 "in module %s: tempDTraceTrapHook already occupied.",
277 sdp->sdp_name, ctl->mod_modname);
278 }
279 return (0);
280 }
281
282 while (sdp != NULL) {
283 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
284 (vm_size_t)sizeof(sdp->sdp_patchval));
285
286 /*
287 * Make the patched instruction visible via a data + instruction
288 * cache fush on platforms that need it
289 */
290 flush_dcache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_patchval), 0);
291 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_patchval), 0);
292
293 sdp = sdp->sdp_next;
294 }
295
296err:
297 return (0);
298}
299
300/*ARGSUSED*/
301static void
302sdt_disable(void *arg, dtrace_id_t id, void *parg)
303{
304#pragma unused(arg,id)
305 sdt_probe_t *sdp = parg;
306 struct modctl *ctl = sdp->sdp_ctl;
307
308 ctl->mod_nenabled--;
309
310 if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
311 goto err;
312
313 while (sdp != NULL) {
314 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
315 (vm_size_t)sizeof(sdp->sdp_savedval));
316 /*
317 * Make the patched instruction visible via a data + instruction
318 * cache flush on platforms that need it
319 */
320 flush_dcache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_savedval), 0);
321 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_savedval), 0);
322 sdp = sdp->sdp_next;
323 }
324
325err:
326 ;
327}
328
329static dtrace_pops_t sdt_pops = {
330 .dtps_provide = NULL,
331 .dtps_provide_module = sdt_provide_module,
332 .dtps_enable = sdt_enable,
333 .dtps_disable = sdt_disable,
334 .dtps_suspend = NULL,
335 .dtps_resume = NULL,
336 .dtps_getargdesc = sdt_getargdesc,
337 .dtps_getargval = sdt_getarg,
338 .dtps_usermode = NULL,
339 .dtps_destroy = sdt_destroy,
340};
341
342/*ARGSUSED*/
343static int
344sdt_attach(dev_info_t *devi)
345{
346 sdt_provider_t *prov;
347
348 if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
349 0, DDI_PSEUDO, 0) == DDI_FAILURE) {
350 cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
351 ddi_remove_minor_node(devi, NULL);
352 return (DDI_FAILURE);
353 }
354
355 if (sdt_probetab_size == 0)
356 sdt_probetab_size = SDT_PROBETAB_SIZE;
357
358 sdt_probetab_mask = sdt_probetab_size - 1;
359 sdt_probetab =
360 kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
361 dtrace_invop_add(sdt_invop);
362
363 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
364 if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
365 DTRACE_PRIV_KERNEL, NULL,
366 &sdt_pops, prov, &prov->sdtp_id) != 0) {
367 cmn_err(CE_WARN, "failed to register sdt provider %s",
368 prov->sdtp_name);
369 }
370 }
371
372 return (DDI_SUCCESS);
373}
374
375/*
376 * APPLE NOTE: sdt_detach not implemented
377 */
378#if !defined(__APPLE__)
379/*ARGSUSED*/
380static int
381sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
382{
383 sdt_provider_t *prov;
384
385 switch (cmd) {
386 case DDI_DETACH:
387 break;
388
389 case DDI_SUSPEND:
390 return (DDI_SUCCESS);
391
392 default:
393 return (DDI_FAILURE);
394 }
395
396 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
397 if (prov->sdtp_id != DTRACE_PROVNONE) {
398 if (dtrace_unregister(prov->sdtp_id) != 0)
399 return (DDI_FAILURE);
400
401 prov->sdtp_id = DTRACE_PROVNONE;
402 }
403 }
404
405 dtrace_invop_remove(sdt_invop);
406 kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
407
408 return (DDI_SUCCESS);
409}
410#endif /* __APPLE__ */
411
412d_open_t _sdt_open;
413
414int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
415{
416#pragma unused(dev,flags,devtype,p)
417 return 0;
418}
419
420#define SDT_MAJOR -24 /* let the kernel pick the device number */
421
422/*
423 * A struct describing which functions will get invoked for certain
424 * actions.
425 */
426static struct cdevsw sdt_cdevsw =
427{
428 _sdt_open, /* open */
429 eno_opcl, /* close */
430 eno_rdwrt, /* read */
431 eno_rdwrt, /* write */
432 eno_ioctl, /* ioctl */
433 (stop_fcn_t *)nulldev, /* stop */
434 (reset_fcn_t *)nulldev, /* reset */
435 NULL, /* tty's */
436 eno_select, /* select */
437 eno_mmap, /* mmap */
438 eno_strat, /* strategy */
439 eno_getc, /* getc */
440 eno_putc, /* putc */
441 0 /* type */
442};
443
444static struct modctl g_sdt_kernctl;
445static struct module g_sdt_mach_module;
446
447#include <mach-o/nlist.h>
448#include <libkern/kernel_mach_header.h>
449
450void sdt_early_init( void )
451{
452 if (dtrace_sdt_probes_restricted()) {
453 return;
454 }
455 if (MH_MAGIC_KERNEL != _mh_execute_header.magic) {
456 g_sdt_kernctl.mod_address = (vm_address_t)NULL;
457 g_sdt_kernctl.mod_size = 0;
458 } else {
459 kernel_mach_header_t *mh;
460 struct load_command *cmd;
461 kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL;
462 struct symtab_command *orig_st = NULL;
463 kernel_nlist_t *sym = NULL;
464 char *strings;
465 unsigned int i;
466
467 g_sdt_mach_module.sdt_nprobes = 0;
468 g_sdt_mach_module.sdt_probes = NULL;
469
470 g_sdt_kernctl.mod_address = (vm_address_t)&g_sdt_mach_module;
471 g_sdt_kernctl.mod_size = 0;
472 strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
473
474 g_sdt_kernctl.mod_next = NULL;
475 g_sdt_kernctl.mod_stale = NULL;
476 g_sdt_kernctl.mod_id = 0;
477 g_sdt_kernctl.mod_loadcnt = 1;
478 g_sdt_kernctl.mod_loaded = 1;
479 g_sdt_kernctl.mod_flags = 0;
480 g_sdt_kernctl.mod_nenabled = 0;
481
482 mh = &_mh_execute_header;
483 cmd = (struct load_command*) &mh[1];
484 for (i = 0; i < mh->ncmds; i++) {
485 if (cmd->cmd == LC_SEGMENT_KERNEL) {
486 kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
487
488 if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT))
489 orig_ts = orig_sg;
490 else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT))
491 orig_le = orig_sg;
492 else if (LIT_STRNEQL(orig_sg->segname, ""))
493 orig_ts = orig_sg; /* kexts have a single unnamed segment */
494 }
495 else if (cmd->cmd == LC_SYMTAB)
496 orig_st = (struct symtab_command *) cmd;
497
498 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
499 }
500
501 if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
502 return;
503
504 sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
505 strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
506
507 for (i = 0; i < orig_st->nsyms; i++) {
508 uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
509 char *name = strings + sym[i].n_un.n_strx;
510 const char *prev_name;
511 unsigned long best;
512 unsigned int j;
513
514 /* Check that the symbol is a global and that it has a name. */
515 if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
516 continue;
517
518 if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
519 continue;
520
521 /* Lop off omnipresent leading underscore. */
522 if (*name == '_')
523 name += 1;
524
525 if (strncmp(name, DTRACE_PROBE_PREFIX, sizeof(DTRACE_PROBE_PREFIX) - 1) == 0) {
526 sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
527 int len = strlen(name) + 1;
528
529 sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
530 strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
531
532 prev_name = "<unknown>";
533 best = 0;
534
535 /*
536 * Find the symbol immediately preceding the sdt probe site just discovered,
537 * that symbol names the function containing the sdt probe.
538 */
539 for (j = 0; j < orig_st->nsyms; j++) {
540 uint8_t jn_type = sym[j].n_type & N_TYPE;
541 char *jname = strings + sym[j].n_un.n_strx;
542
543 if ((N_SECT != jn_type && N_ABS != jn_type))
544 continue;
545
546 if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
547 continue;
548
549 if (*jname == '_')
550 jname += 1;
551
552 if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
553 continue;
554
555 if ((unsigned long)sym[j].n_value > best) {
556 best = (unsigned long)sym[j].n_value;
557 prev_name = jname;
558 }
559 }
560
561 sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
562 strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
563
564 sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;
565#if defined(__arm__)
566 /* PR8353094 - mask off thumb-bit */
567 sdpd->sdpd_offset &= ~0x1U;
568#elif defined(__arm64__)
569 sdpd->sdpd_offset &= ~0x1LU;
570#endif /* __arm__ */
571
572#if 0
573 printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n",
574 sdpd->sdpd_offset, *(unsigned long *)sym[i].n_value, name);
575#endif
576
577 sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
578 g_sdt_mach_module.sdt_probes = sdpd;
579 } else {
580 prev_name = name;
581 }
582 }
583 }
584}
585
586void sdt_init( void )
587{
588 int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
589
590 if (majdevno < 0) {
591 printf("sdt_init: failed to allocate a major number!\n");
592 return;
593 }
594
595 if (dtrace_sdt_probes_restricted()) {
596 return;
597 }
598
599 sdt_attach((dev_info_t*)(uintptr_t)majdevno);
600}
601
602#undef SDT_MAJOR
603
604/*ARGSUSED*/
605void
606sdt_provide_module(void *arg, struct modctl *ctl)
607{
608#pragma unused(arg)
609 ASSERT(ctl != NULL);
610 ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
611 LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
612
613 if (MOD_SDT_DONE(ctl))
614 return;
615
616 if (MOD_IS_MACH_KERNEL(ctl)) {
617 __sdt_provide_module(arg, &g_sdt_kernctl);
618
619 sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
620 while (sdpd) {
621 sdt_probedesc_t *this_sdpd = sdpd;
622 kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
623 kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
624 sdpd = sdpd->sdpd_next;
625 kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
626 }
627 g_sdt_mach_module.sdt_probes = NULL;
628 } else {
629 /*
630 * APPLE NOTE: sdt probes for kexts not yet implemented
631 */
632 }
633
634 /* Need to mark this module as completed */
635 ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
636}
637