1/*
2 * Copyright (c) 2005-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
30#include <kern/thread.h>
31#include <mach/thread_status.h>
32
33typedef x86_saved_state_t savearea_t;
34
35#include <stdarg.h>
36#include <string.h>
37#include <sys/malloc.h>
38#include <sys/time.h>
39#include <sys/systm.h>
40#include <sys/proc.h>
41#include <sys/proc_internal.h>
42#include <sys/kauth.h>
43#include <sys/dtrace.h>
44#include <sys/dtrace_impl.h>
45#include <libkern/OSAtomic.h>
46#include <kern/thread_call.h>
47#include <kern/task.h>
48#include <kern/sched_prim.h>
49#include <miscfs/devfs/devfs.h>
50#include <mach/vm_param.h>
51#include <machine/pal_routines.h>
52#include <i386/mp.h>
53
54/*
55 * APPLE NOTE: The regmap is used to decode which 64bit uregs[] register
56 * is being accessed when passed the 32bit uregs[] constant (based on
57 * the reg.d translator file). The dtrace_getreg() is smart enough to handle
58 * the register mappings. The register set definitions are the same as
59 * those used by the fasttrap_getreg code.
60 */
61#include "fasttrap_regset.h"
62static const uint8_t regmap[19] = {
63 REG_GS, /* GS */
64 REG_FS, /* FS */
65 REG_ES, /* ES */
66 REG_DS, /* DS */
67 REG_RDI, /* EDI */
68 REG_RSI, /* ESI */
69 REG_RBP, /* EBP, REG_FP */
70 REG_RSP, /* ESP */
71 REG_RBX, /* EBX */
72 REG_RDX, /* EDX, REG_R1 */
73 REG_RCX, /* ECX */
74 REG_RAX, /* EAX, REG_R0 */
75 REG_TRAPNO, /* TRAPNO */
76 REG_ERR, /* ERR */
77 REG_RIP, /* EIP, REG_PC */
78 REG_CS, /* CS */
79 REG_RFL, /* EFL, REG_PS */
80 REG_RSP, /* UESP, REG_SP */
81 REG_SS /* SS */
82};
83
84extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */
85
86void
87dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
88 int fltoffs, int fault, uint64_t illval)
89{
90 /*
91 * For the case of the error probe firing lets
92 * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
93 */
94 state->dts_arg_error_illval = illval;
95 dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
96}
97
98/*
99 * Atomicity and synchronization
100 */
101void
102dtrace_membar_producer(void)
103{
104 __asm__ volatile("sfence");
105}
106
107void
108dtrace_membar_consumer(void)
109{
110 __asm__ volatile("lfence");
111}
112
113/*
114 * Interrupt manipulation
115 * XXX dtrace_getipl() can be called from probe context.
116 */
117int
118dtrace_getipl(void)
119{
120 /*
121 * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
122 * in osfmk/kern/cpu_data.h
123 */
124 /* return get_interrupt_level(); */
125 return (ml_at_interrupt_context() ? 1: 0);
126}
127
128/*
129 * MP coordination
130 */
131typedef struct xcArg {
132 processorid_t cpu;
133 dtrace_xcall_t f;
134 void *arg;
135} xcArg_t;
136
137static void
138xcRemote( void *foo )
139{
140 xcArg_t *pArg = (xcArg_t *)foo;
141
142 if ( pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL ) {
143 (pArg->f)(pArg->arg);
144 }
145}
146
147
148/*
149 * dtrace_xcall() is not called from probe context.
150 */
151void
152dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
153{
154 xcArg_t xcArg;
155
156 xcArg.cpu = cpu;
157 xcArg.f = f;
158 xcArg.arg = arg;
159
160 if (cpu == DTRACE_CPUALL) {
161 mp_cpus_call (CPUMASK_ALL, ASYNC, xcRemote, (void*)&xcArg);
162 }
163 else {
164 mp_cpus_call (cpu_to_cpumask((cpu_t)cpu), ASYNC, xcRemote, (void*)&xcArg);
165 }
166}
167
168/*
169 * Initialization
170 */
171void
172dtrace_isa_init(void)
173{
174 return;
175}
176
177/*
178 * Runtime and ABI
179 */
180uint64_t
181dtrace_getreg(struct regs *savearea, uint_t reg)
182{
183 boolean_t is64Bit = proc_is64bit(current_proc());
184 x86_saved_state_t *regs = (x86_saved_state_t *)savearea;
185
186 if (regs == NULL) {
187 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
188 return (0);
189 }
190
191 if (is64Bit) {
192 if (reg <= SS) {
193 reg = regmap[reg];
194 } else {
195 reg -= (SS + 1);
196 }
197
198 switch (reg) {
199 case REG_RDI:
200 return (uint64_t)(regs->ss_64.rdi);
201 case REG_RSI:
202 return (uint64_t)(regs->ss_64.rsi);
203 case REG_RDX:
204 return (uint64_t)(regs->ss_64.rdx);
205 case REG_RCX:
206 return (uint64_t)(regs->ss_64.rcx);
207 case REG_R8:
208 return (uint64_t)(regs->ss_64.r8);
209 case REG_R9:
210 return (uint64_t)(regs->ss_64.r9);
211 case REG_RAX:
212 return (uint64_t)(regs->ss_64.rax);
213 case REG_RBX:
214 return (uint64_t)(regs->ss_64.rbx);
215 case REG_RBP:
216 return (uint64_t)(regs->ss_64.rbp);
217 case REG_R10:
218 return (uint64_t)(regs->ss_64.r10);
219 case REG_R11:
220 return (uint64_t)(regs->ss_64.r11);
221 case REG_R12:
222 return (uint64_t)(regs->ss_64.r12);
223 case REG_R13:
224 return (uint64_t)(regs->ss_64.r13);
225 case REG_R14:
226 return (uint64_t)(regs->ss_64.r14);
227 case REG_R15:
228 return (uint64_t)(regs->ss_64.r15);
229 case REG_FS:
230 return (uint64_t)(regs->ss_64.fs);
231 case REG_GS:
232 return (uint64_t)(regs->ss_64.gs);
233 case REG_TRAPNO:
234 return (uint64_t)(regs->ss_64.isf.trapno);
235 case REG_ERR:
236 return (uint64_t)(regs->ss_64.isf.err);
237 case REG_RIP:
238 return (uint64_t)(regs->ss_64.isf.rip);
239 case REG_CS:
240 return (uint64_t)(regs->ss_64.isf.cs);
241 case REG_SS:
242 return (uint64_t)(regs->ss_64.isf.ss);
243 case REG_RFL:
244 return (uint64_t)(regs->ss_64.isf.rflags);
245 case REG_RSP:
246 return (uint64_t)(regs->ss_64.isf.rsp);
247 case REG_DS:
248 case REG_ES:
249 default:
250 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
251 return (0);
252 }
253
254 } else { /* is 32bit user */
255 /* beyond register SS */
256 if (reg > x86_SAVED_STATE32_COUNT - 1) {
257 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
258 return (0);
259 }
260 return (uint64_t)((unsigned int *)(&(regs->ss_32.gs)))[reg];
261 }
262}
263
264#define RETURN_OFFSET 4
265#define RETURN_OFFSET64 8
266
267static int
268dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, user_addr_t pc,
269 user_addr_t sp)
270{
271#if 0
272 volatile uint16_t *flags =
273 (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
274
275 uintptr_t oldcontext = lwp->lwp_oldcontext; /* XXX signal stack crawl */
276 size_t s1, s2;
277#endif
278 int ret = 0;
279 boolean_t is64Bit = proc_is64bit(current_proc());
280
281 ASSERT(pcstack == NULL || pcstack_limit > 0);
282
283#if 0 /* XXX signal stack crawl */
284 if (p->p_model == DATAMODEL_NATIVE) {
285 s1 = sizeof (struct frame) + 2 * sizeof (long);
286 s2 = s1 + sizeof (siginfo_t);
287 } else {
288 s1 = sizeof (struct frame32) + 3 * sizeof (int);
289 s2 = s1 + sizeof (siginfo32_t);
290 }
291#endif
292
293 while (pc != 0) {
294 ret++;
295 if (pcstack != NULL) {
296 *pcstack++ = (uint64_t)pc;
297 pcstack_limit--;
298 if (pcstack_limit <= 0)
299 break;
300 }
301
302 if (sp == 0)
303 break;
304
305#if 0 /* XXX signal stack crawl */
306 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
307 if (p->p_model == DATAMODEL_NATIVE) {
308 ucontext_t *ucp = (ucontext_t *)oldcontext;
309 greg_t *gregs = ucp->uc_mcontext.gregs;
310
311 sp = dtrace_fulword(&gregs[REG_FP]);
312 pc = dtrace_fulword(&gregs[REG_PC]);
313
314 oldcontext = dtrace_fulword(&ucp->uc_link);
315 } else {
316 ucontext32_t *ucp = (ucontext32_t *)oldcontext;
317 greg32_t *gregs = ucp->uc_mcontext.gregs;
318
319 sp = dtrace_fuword32(&gregs[EBP]);
320 pc = dtrace_fuword32(&gregs[EIP]);
321
322 oldcontext = dtrace_fuword32(&ucp->uc_link);
323 }
324 }
325 else
326#endif
327 {
328 if (is64Bit) {
329 pc = dtrace_fuword64((sp + RETURN_OFFSET64));
330 sp = dtrace_fuword64(sp);
331 } else {
332 pc = dtrace_fuword32((sp + RETURN_OFFSET));
333 sp = dtrace_fuword32(sp);
334 }
335 }
336
337#if 0 /* XXX */
338 /*
339 * This is totally bogus: if we faulted, we're going to clear
340 * the fault and break. This is to deal with the apparently
341 * broken Java stacks on x86.
342 */
343 if (*flags & CPU_DTRACE_FAULT) {
344 *flags &= ~CPU_DTRACE_FAULT;
345 break;
346 }
347#endif
348 }
349
350 return (ret);
351}
352
353
354/*
355 * The return value indicates if we've modified the stack.
356 */
357static int
358dtrace_adjust_stack(uint64_t **pcstack, int *pcstack_limit, user_addr_t *pc,
359 user_addr_t sp)
360{
361 int64_t missing_tos;
362 int rc = 0;
363 boolean_t is64Bit = proc_is64bit(current_proc());
364
365 ASSERT(pc != NULL);
366
367 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
368 /*
369 * If we found ourselves in an entry probe, the frame pointer has not
370 * yet been pushed (that happens in the
371 * function prologue). The best approach is to
372 * add the current pc as a missing top of stack,
373 * and back the pc up to the caller, which is stored at the
374 * current stack pointer address since the call
375 * instruction puts it there right before
376 * the branch.
377 */
378
379 missing_tos = *pc;
380
381 if (is64Bit)
382 *pc = dtrace_fuword64(sp);
383 else
384 *pc = dtrace_fuword32(sp);
385 } else {
386 /*
387 * We might have a top of stack override, in which case we just
388 * add that frame without question to the top. This
389 * happens in return probes where you have a valid
390 * frame pointer, but it's for the callers frame
391 * and you'd like to add the pc of the return site
392 * to the frame.
393 */
394 missing_tos = cpu_core[CPU->cpu_id].cpuc_missing_tos;
395 }
396
397 if (missing_tos != 0) {
398 if (pcstack != NULL && pcstack_limit != NULL) {
399 /*
400 * If the missing top of stack has been filled out, then
401 * we add it and adjust the size.
402 */
403 *(*pcstack)++ = missing_tos;
404 (*pcstack_limit)--;
405 }
406 /*
407 * return 1 because we would have changed the
408 * stack whether or not it was passed in. This
409 * ensures the stack count is correct
410 */
411 rc = 1;
412 }
413 return rc;
414}
415
416void
417dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
418{
419 thread_t thread = current_thread();
420 x86_saved_state_t *regs;
421 user_addr_t pc, sp, fp;
422 volatile uint16_t *flags =
423 (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
424 int n;
425 boolean_t is64Bit = proc_is64bit(current_proc());
426
427 if (*flags & CPU_DTRACE_FAULT)
428 return;
429
430 if (pcstack_limit <= 0)
431 return;
432
433 /*
434 * If there's no user context we still need to zero the stack.
435 */
436 if (thread == NULL)
437 goto zero;
438
439 pal_register_cache_state(thread, VALID);
440 regs = (x86_saved_state_t *)find_user_regs(thread);
441 if (regs == NULL)
442 goto zero;
443
444 *pcstack++ = (uint64_t)dtrace_proc_selfpid();
445 pcstack_limit--;
446
447 if (pcstack_limit <= 0)
448 return;
449
450 if (is64Bit) {
451 pc = regs->ss_64.isf.rip;
452 sp = regs->ss_64.isf.rsp;
453 fp = regs->ss_64.rbp;
454 } else {
455 pc = regs->ss_32.eip;
456 sp = regs->ss_32.uesp;
457 fp = regs->ss_32.ebp;
458 }
459
460 /*
461 * The return value indicates if we've modified the stack.
462 * Since there is nothing else to fix up in either case,
463 * we can safely ignore it here.
464 */
465 (void)dtrace_adjust_stack(&pcstack, &pcstack_limit, &pc, sp);
466
467 if(pcstack_limit <= 0)
468 return;
469
470 /*
471 * Note that unlike ppc, the x86 code does not use
472 * CPU_DTRACE_USTACK_FP. This is because x86 always
473 * traces from the fp, even in syscall/profile/fbt
474 * providers.
475 */
476 n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
477 ASSERT(n >= 0);
478 ASSERT(n <= pcstack_limit);
479
480 pcstack += n;
481 pcstack_limit -= n;
482
483zero:
484 while (pcstack_limit-- > 0)
485 *pcstack++ = 0;
486}
487
488int
489dtrace_getustackdepth(void)
490{
491 thread_t thread = current_thread();
492 x86_saved_state_t *regs;
493 user_addr_t pc, sp, fp;
494 int n = 0;
495 boolean_t is64Bit = proc_is64bit(current_proc());
496
497 if (thread == NULL)
498 return 0;
499
500 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
501 return (-1);
502
503 pal_register_cache_state(thread, VALID);
504 regs = (x86_saved_state_t *)find_user_regs(thread);
505 if (regs == NULL)
506 return 0;
507
508 if (is64Bit) {
509 pc = regs->ss_64.isf.rip;
510 sp = regs->ss_64.isf.rsp;
511 fp = regs->ss_64.rbp;
512 } else {
513 pc = regs->ss_32.eip;
514 sp = regs->ss_32.uesp;
515 fp = regs->ss_32.ebp;
516 }
517
518 if (dtrace_adjust_stack(NULL, NULL, &pc, sp) == 1) {
519 /*
520 * we would have adjusted the stack if we had
521 * supplied one (that is what rc == 1 means).
522 * Also, as a side effect, the pc might have
523 * been fixed up, which is good for calling
524 * in to dtrace_getustack_common.
525 */
526 n++;
527 }
528
529 /*
530 * Note that unlike ppc, the x86 code does not use
531 * CPU_DTRACE_USTACK_FP. This is because x86 always
532 * traces from the fp, even in syscall/profile/fbt
533 * providers.
534 */
535
536 n += dtrace_getustack_common(NULL, 0, pc, fp);
537
538 return (n);
539}
540
541void
542dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
543{
544 thread_t thread = current_thread();
545 savearea_t *regs;
546 user_addr_t pc, sp;
547 volatile uint16_t *flags =
548 (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
549#if 0
550 uintptr_t oldcontext;
551 size_t s1, s2;
552#endif
553 boolean_t is64Bit = proc_is64bit(current_proc());
554
555 if (*flags & CPU_DTRACE_FAULT)
556 return;
557
558 if (pcstack_limit <= 0)
559 return;
560
561 /*
562 * If there's no user context we still need to zero the stack.
563 */
564 if (thread == NULL)
565 goto zero;
566
567 regs = (savearea_t *)find_user_regs(thread);
568 if (regs == NULL)
569 goto zero;
570
571 *pcstack++ = (uint64_t)dtrace_proc_selfpid();
572 pcstack_limit--;
573
574 if (pcstack_limit <= 0)
575 return;
576
577 pc = regs->ss_32.eip;
578 sp = regs->ss_32.ebp;
579
580#if 0 /* XXX signal stack crawl */
581 oldcontext = lwp->lwp_oldcontext;
582
583 if (p->p_model == DATAMODEL_NATIVE) {
584 s1 = sizeof (struct frame) + 2 * sizeof (long);
585 s2 = s1 + sizeof (siginfo_t);
586 } else {
587 s1 = sizeof (struct frame32) + 3 * sizeof (int);
588 s2 = s1 + sizeof (siginfo32_t);
589 }
590#endif
591
592 if(dtrace_adjust_stack(&pcstack, &pcstack_limit, &pc, sp) == 1) {
593 /*
594 * we made a change.
595 */
596 *fpstack++ = 0;
597 if (pcstack_limit <= 0)
598 return;
599 }
600
601 while (pc != 0) {
602 *pcstack++ = (uint64_t)pc;
603 *fpstack++ = sp;
604 pcstack_limit--;
605 if (pcstack_limit <= 0)
606 break;
607
608 if (sp == 0)
609 break;
610
611#if 0 /* XXX signal stack crawl */
612 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
613 if (p->p_model == DATAMODEL_NATIVE) {
614 ucontext_t *ucp = (ucontext_t *)oldcontext;
615 greg_t *gregs = ucp->uc_mcontext.gregs;
616
617 sp = dtrace_fulword(&gregs[REG_FP]);
618 pc = dtrace_fulword(&gregs[REG_PC]);
619
620 oldcontext = dtrace_fulword(&ucp->uc_link);
621 } else {
622 ucontext_t *ucp = (ucontext_t *)oldcontext;
623 greg_t *gregs = ucp->uc_mcontext.gregs;
624
625 sp = dtrace_fuword32(&gregs[EBP]);
626 pc = dtrace_fuword32(&gregs[EIP]);
627
628 oldcontext = dtrace_fuword32(&ucp->uc_link);
629 }
630 }
631 else
632#endif
633 {
634 if (is64Bit) {
635 pc = dtrace_fuword64((sp + RETURN_OFFSET64));
636 sp = dtrace_fuword64(sp);
637 } else {
638 pc = dtrace_fuword32((sp + RETURN_OFFSET));
639 sp = dtrace_fuword32(sp);
640 }
641 }
642
643#if 0 /* XXX */
644 /*
645 * This is totally bogus: if we faulted, we're going to clear
646 * the fault and break. This is to deal with the apparently
647 * broken Java stacks on x86.
648 */
649 if (*flags & CPU_DTRACE_FAULT) {
650 *flags &= ~CPU_DTRACE_FAULT;
651 break;
652 }
653#endif
654 }
655
656zero:
657 while (pcstack_limit-- > 0)
658 *pcstack++ = 0;
659}
660
661void
662dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes,
663 uint32_t *intrpc)
664{
665 struct frame *fp = (struct frame *)__builtin_frame_address(0);
666 struct frame *nextfp, *minfp, *stacktop;
667 int depth = 0;
668 int last = 0;
669 uintptr_t pc;
670 uintptr_t caller = CPU->cpu_dtrace_caller;
671 int on_intr;
672
673 if ((on_intr = CPU_ON_INTR(CPU)) != 0)
674 stacktop = (struct frame *)dtrace_get_cpu_int_stack_top();
675 else
676 stacktop = (struct frame *)(dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
677
678 minfp = fp;
679
680 aframes++;
681
682 if (intrpc != NULL && depth < pcstack_limit)
683 pcstack[depth++] = (pc_t)intrpc;
684
685 while (depth < pcstack_limit) {
686 nextfp = *(struct frame **)fp;
687 pc = *(uintptr_t *)(((uintptr_t)fp) + RETURN_OFFSET64);
688
689 if (nextfp <= minfp || nextfp >= stacktop) {
690 if (on_intr) {
691 /*
692 * Hop from interrupt stack to thread stack.
693 */
694 vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
695
696 minfp = (struct frame *)kstack_base;
697 stacktop = (struct frame *)(kstack_base + kernel_stack_size);
698
699 on_intr = 0;
700 continue;
701 }
702 /*
703 * This is the last frame we can process; indicate
704 * that we should return after processing this frame.
705 */
706 last = 1;
707 }
708
709 if (aframes > 0) {
710 if (--aframes == 0 && caller != 0) {
711 /*
712 * We've just run out of artificial frames,
713 * and we have a valid caller -- fill it in
714 * now.
715 */
716 ASSERT(depth < pcstack_limit);
717 pcstack[depth++] = (pc_t)caller;
718 caller = 0;
719 }
720 } else {
721 if (depth < pcstack_limit)
722 pcstack[depth++] = (pc_t)pc;
723 }
724
725 if (last) {
726 while (depth < pcstack_limit)
727 pcstack[depth++] = 0;
728 return;
729 }
730
731 fp = nextfp;
732 minfp = fp;
733 }
734}
735
736struct frame {
737 struct frame *backchain;
738 uintptr_t retaddr;
739};
740
741uint64_t
742dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
743{
744 uint64_t val = 0;
745 struct frame *fp = (struct frame *)__builtin_frame_address(0);
746 uintptr_t *stack;
747 uintptr_t pc;
748 int i;
749
750
751 /*
752 * A total of 6 arguments are passed via registers; any argument with
753 * index of 5 or lower is therefore in a register.
754 */
755 int inreg = 5;
756
757 for (i = 1; i <= aframes; i++) {
758 fp = fp->backchain;
759 pc = fp->retaddr;
760
761 if (dtrace_invop_callsite_pre != NULL
762 && pc > (uintptr_t)dtrace_invop_callsite_pre
763 && pc <= (uintptr_t)dtrace_invop_callsite_post) {
764 /*
765 * In the case of x86_64, we will use the pointer to the
766 * save area structure that was pushed when we took the
767 * trap. To get this structure, we must increment
768 * beyond the frame structure. If the
769 * argument that we're seeking is passed on the stack,
770 * we'll pull the true stack pointer out of the saved
771 * registers and decrement our argument by the number
772 * of arguments passed in registers; if the argument
773 * we're seeking is passed in regsiters, we can just
774 * load it directly.
775 */
776
777 /* fp points to frame of dtrace_invop() activation. */
778 fp = fp->backchain; /* to fbt_perfcallback() activation. */
779 fp = fp->backchain; /* to kernel_trap() activation. */
780 fp = fp->backchain; /* to trap_from_kernel() activation. */
781
782 x86_saved_state_t *tagged_regs = (x86_saved_state_t *)&fp[1];
783 x86_saved_state64_t *saved_state = saved_state64(tagged_regs);
784
785 if (arg <= inreg) {
786 stack = (uintptr_t *)(void*)&saved_state->rdi;
787 } else {
788 fp = (struct frame *)(saved_state->isf.rsp);
789 stack = (uintptr_t *)&fp[1]; /* Find marshalled
790 arguments */
791 arg -= inreg + 1;
792 }
793 goto load;
794 }
795 }
796
797 /*
798 * We know that we did not come through a trap to get into
799 * dtrace_probe() -- We arrive here when the provider has
800 * called dtrace_probe() directly.
801 * The probe ID is the first argument to dtrace_probe().
802 * We must advance beyond that to get the argX.
803 */
804 arg++; /* Advance past probeID */
805
806 if (arg <= inreg) {
807 /*
808 * This shouldn't happen. If the argument is passed in a
809 * register then it should have been, well, passed in a
810 * register...
811 */
812 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
813 return (0);
814 }
815
816 arg -= (inreg + 1);
817 stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
818
819load:
820 if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
821 mstate, vstate)) {
822 /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
823 val = dtrace_load64((uint64_t)(stack + arg));
824 }
825
826 return (val);
827}
828
829/*
830 * Load/Store Safety
831 */
832void
833dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
834{
835 /*
836 * "base" is the smallest toxic address in the range, "limit" is the first
837 * VALID address greater than "base".
838 */
839 func(0x0, VM_MIN_KERNEL_AND_KEXT_ADDRESS);
840 if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0)
841 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
842}
843
844