1/*
2 * Copyright (c) 2013-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach/mach_types.h>
30#include <mach/vm_param.h>
31#include <mach/mach_vm.h>
32#include <mach/clock_types.h>
33#include <sys/errno.h>
34#include <sys/stackshot.h>
35#ifdef IMPORTANCE_INHERITANCE
36#include <ipc/ipc_importance.h>
37#endif
38#include <sys/appleapiopts.h>
39#include <kern/debug.h>
40#include <kern/block_hint.h>
41#include <uuid/uuid.h>
42
43#include <kdp/kdp_dyld.h>
44#include <kdp/kdp_en_debugger.h>
45
46#include <libsa/types.h>
47#include <libkern/version.h>
48#include <libkern/section_keywords.h>
49
50#include <string.h> /* bcopy */
51
52#include <kern/coalition.h>
53#include <kern/processor.h>
54#include <kern/thread.h>
55#include <kern/thread_group.h>
56#include <kern/task.h>
57#include <kern/telemetry.h>
58#include <kern/clock.h>
59#include <kern/policy_internal.h>
60#include <vm/vm_map.h>
61#include <vm/vm_kern.h>
62#include <vm/vm_pageout.h>
63#include <vm/vm_fault.h>
64#include <vm/vm_shared_region.h>
65#include <libkern/OSKextLibPrivate.h>
66
67#if defined(__x86_64__)
68#include <i386/mp.h>
69#include <i386/cpu_threads.h>
70#endif
71
72#if CONFIG_EMBEDDED
73#include <pexpert/pexpert.h> /* For gPanicBase/gPanicBase */
74#endif
75
76#if MONOTONIC
77#include <kern/monotonic.h>
78#endif /* MONOTONIC */
79
80#include <san/kasan.h>
81
82extern unsigned int not_in_kdp;
83
84
85/* indicate to the compiler that some accesses are unaligned */
86typedef uint64_t unaligned_u64 __attribute__((aligned(1)));
87
88extern addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
89
90int kdp_snapshot = 0;
91static kern_return_t stack_snapshot_ret = 0;
92static uint32_t stack_snapshot_bytes_traced = 0;
93
94static kcdata_descriptor_t stackshot_kcdata_p = NULL;
95static void *stack_snapshot_buf;
96static uint32_t stack_snapshot_bufsize;
97int stack_snapshot_pid;
98static uint32_t stack_snapshot_flags;
99static uint64_t stack_snapshot_delta_since_timestamp;
100static boolean_t panic_stackshot;
101
102static boolean_t stack_enable_faulting = FALSE;
103static struct stackshot_fault_stats fault_stats;
104
105static unaligned_u64 * stackshot_duration_outer;
106static uint64_t stackshot_microsecs;
107
108void * kernel_stackshot_buf = NULL; /* Pointer to buffer for stackshots triggered from the kernel and retrieved later */
109int kernel_stackshot_buf_size = 0;
110
111void * stackshot_snapbuf = NULL; /* Used by stack_snapshot2 (to be removed) */
112
113__private_extern__ void stackshot_init( void );
114static boolean_t memory_iszero(void *addr, size_t size);
115#if CONFIG_TELEMETRY
116kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
117#endif
118uint32_t get_stackshot_estsize(uint32_t prev_size_hint);
119kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config,
120 size_t stackshot_config_size, boolean_t stackshot_from_user);
121kern_return_t do_stackshot(void *);
122void kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags, kcdata_descriptor_t data_p, uint64_t since_timestamp);
123boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
124static int kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t *pBytesTraced);
125uint32_t kdp_stack_snapshot_bytes_traced(void);
126static void kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap);
127static boolean_t kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_result);
128static boolean_t kdp_copyin_word(task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results);
129static uint64_t proc_was_throttled_from_task(task_t task);
130static void stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t * waitinfo);
131static int stackshot_thread_has_valid_waitinfo(thread_t thread);
132
133#if CONFIG_COALITIONS
134static void stackshot_coalition_jetsam_count(void *arg, int i, coalition_t coal);
135static void stackshot_coalition_jetsam_snapshot(void *arg, int i, coalition_t coal);
136#endif /* CONFIG_COALITIONS */
137
138
139extern uint32_t workqueue_get_pwq_state_kdp(void *proc);
140
141extern int proc_pid(void *p);
142extern uint64_t proc_uniqueid(void *p);
143extern uint64_t proc_was_throttled(void *p);
144extern uint64_t proc_did_throttle(void *p);
145extern int proc_exiting(void *p);
146extern int proc_in_teardown(void *p);
147static uint64_t proc_did_throttle_from_task(task_t task);
148extern void proc_name_kdp(task_t task, char * buf, int size);
149extern int proc_threadname_kdp(void * uth, char * buf, size_t size);
150extern void proc_starttime_kdp(void * p, uint64_t * tv_sec, uint64_t * tv_usec, uint64_t * abstime);
151extern int memorystatus_get_pressure_status_kdp(void);
152extern void memorystatus_proc_flags_unsafe(void * v, boolean_t *is_dirty, boolean_t *is_dirty_tracked, boolean_t *allow_idle_exit);
153
154extern int count_busy_buffers(void); /* must track with declaration in bsd/sys/buf_internal.h */
155extern void bcopy_phys(addr64_t, addr64_t, vm_size_t);
156
157#if CONFIG_TELEMETRY
158extern kern_return_t stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval);
159#endif /* CONFIG_TELEMETRY */
160
161extern kern_return_t kern_stack_snapshot_with_reason(char* reason);
162extern kern_return_t kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user);
163
164/*
165 * Validates that the given address is both a valid page and has
166 * default caching attributes for the current map. Returns
167 * 0 if the address is invalid, and a kernel virtual address for
168 * the given address if it is valid.
169 */
170vm_offset_t machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags);
171
172#define KDP_FAULT_RESULT_PAGED_OUT 0x1 /* some data was unable to be retrieved */
173#define KDP_FAULT_RESULT_TRIED_FAULT 0x2 /* tried to fault in data */
174#define KDP_FAULT_RESULT_FAULTED_IN 0x4 /* successfully faulted in data */
175
176/*
177 * Looks up the physical translation for the given address in the target map, attempting
178 * to fault data in if requested and it is not resident. Populates thread_trace_flags if requested
179 * as well.
180 */
181vm_offset_t kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results);
182
183static size_t stackshot_strlcpy(char *dst, const char *src, size_t maxlen);
184static void stackshot_memcpy(void *dst, const void *src, size_t len);
185
186/* Clears caching information used by the above validation routine
187 * (in case the current map has been changed or cleared).
188 */
189void machine_trace_thread_clear_validation_cache(void);
190
191#define MAX_FRAMES 1000
192#define MAX_LOADINFOS 500
193#define TASK_IMP_WALK_LIMIT 20
194
195typedef struct thread_snapshot *thread_snapshot_t;
196typedef struct task_snapshot *task_snapshot_t;
197
198#if CONFIG_KDP_INTERACTIVE_DEBUGGING
199extern kdp_send_t kdp_en_send_pkt;
200#endif
201
202/*
203 * Globals to support machine_trace_thread_get_kva.
204 */
205static vm_offset_t prev_target_page = 0;
206static vm_offset_t prev_target_kva = 0;
207static boolean_t validate_next_addr = TRUE;
208
209/*
210 * Stackshot locking and other defines.
211 */
212static lck_grp_t *stackshot_subsys_lck_grp;
213static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
214static lck_attr_t *stackshot_subsys_lck_attr;
215static lck_mtx_t stackshot_subsys_mutex;
216
217#define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
218#define STACKSHOT_SUBSYS_TRY_LOCK() lck_mtx_try_lock(&stackshot_subsys_mutex)
219#define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
220
221#define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
222#define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
223
224SECURITY_READ_ONLY_LATE(static uint32_t) max_tracebuf_size = SANE_TRACEBUF_SIZE;
225
226/*
227 * We currently set a ceiling of 3 milliseconds spent in the kdp fault path
228 * for non-panic stackshots where faulting is requested.
229 */
230#define KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS (3 * NSEC_PER_MSEC)
231
232#define STACKSHOT_SUPP_SIZE (16 * 1024) /* Minimum stackshot size */
233#define TASK_UUID_AVG_SIZE (16 * sizeof(uuid_t)) /* Average space consumed by UUIDs/task */
234
235/*
236 * Initialize the mutex governing access to the stack snapshot subsystem
237 * and other stackshot related bits.
238 */
239__private_extern__ void
240stackshot_init( void )
241{
242 mach_timebase_info_data_t timebase;
243
244 stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
245
246 stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
247
248 stackshot_subsys_lck_attr = lck_attr_alloc_init();
249
250 lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
251
252 clock_timebase_info(&timebase);
253 fault_stats.sfs_system_max_fault_time = ((KDP_FAULT_PATH_MAX_TIME_PER_STACKSHOT_NSECS * timebase.denom)/ timebase.numer);
254
255 PE_parse_boot_argn("stackshot_maxsz", &max_tracebuf_size, sizeof(max_tracebuf_size));
256}
257
258/*
259 * Method for grabbing timer values safely, in the sense that no infinite loop will occur
260 * Certain flavors of the timer_grab function, which would seem to be the thing to use,
261 * can loop infinitely if called while the timer is in the process of being updated.
262 * Unfortunately, it is (rarely) possible to get inconsistent top and bottom halves of
263 * the timer using this method. This seems insoluble, since stackshot runs in a context
264 * where the timer might be half-updated, and has no way of yielding control just long
265 * enough to finish the update.
266 */
267
268static uint64_t safe_grab_timer_value(struct timer *t)
269{
270#if defined(__LP64__)
271 return t->all_bits;
272#else
273 uint64_t time = t->high_bits; /* endian independent grab */
274 time = (time << 32) | t->low_bits;
275 return time;
276#endif
277}
278
279/*
280 * Called with interrupts disabled after stackshot context has been
281 * initialized. Updates stack_snapshot_ret.
282 */
283static kern_return_t
284stackshot_trap()
285{
286 kern_return_t rv;
287
288#if defined(__x86_64__)
289 /*
290 * Since mp_rendezvous and stackshot both attempt to capture cpus then perform an
291 * operation, it's essential to apply mutual exclusion to the other when one
292 * mechanism is in operation, lest there be a deadlock as the mechanisms race to
293 * capture CPUs.
294 *
295 * Further, we assert that invoking stackshot from mp_rendezvous*() is not
296 * allowed, so we check to ensure there there is no rendezvous in progress before
297 * trying to grab the lock (if there is, a deadlock will occur when we try to
298 * grab the lock). This is accomplished by setting cpu_rendezvous_in_progress to
299 * TRUE in the mp rendezvous action function. If stackshot_trap() is called by
300 * a subordinate of the call chain within the mp rendezvous action, this flag will
301 * be set and can be used to detect the inevitable deadlock that would occur
302 * if this thread tried to grab the rendezvous lock.
303 */
304
305 if (current_cpu_datap()->cpu_rendezvous_in_progress == TRUE) {
306 panic("Calling stackshot from a rendezvous is not allowed!");
307 }
308
309 mp_rendezvous_lock();
310#endif
311
312 rv = DebuggerTrapWithState(DBOP_STACKSHOT, NULL, NULL, NULL, 0, NULL, FALSE, 0);
313
314#if defined(__x86_64__)
315 mp_rendezvous_unlock();
316#endif
317 return (rv);
318}
319
320
321kern_return_t
322stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, uint64_t delta_since_timestamp, unsigned *bytes_traced)
323{
324 kern_return_t error = KERN_SUCCESS;
325 boolean_t istate;
326
327#if DEVELOPMENT || DEBUG
328 if (kern_feature_override(KF_STACKSHOT_OVRD) == TRUE) {
329 error = KERN_NOT_SUPPORTED;
330 goto out;
331 }
332#endif
333 if ((buf == NULL) || (size <= 0) || (bytes_traced == NULL)) {
334 return KERN_INVALID_ARGUMENT;
335 }
336
337 /* cap in individual stackshot to max_tracebuf_size */
338 if (size > max_tracebuf_size) {
339 size = max_tracebuf_size;
340 }
341
342 /* Serialize tracing */
343 if (flags & STACKSHOT_TRYLOCK) {
344 if (!STACKSHOT_SUBSYS_TRY_LOCK()) {
345 return KERN_LOCK_OWNED;
346 }
347 } else {
348 STACKSHOT_SUBSYS_LOCK();
349 }
350
351 struct kcdata_descriptor kcdata;
352 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ?
353 KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
354
355 error = kcdata_memory_static_init(&kcdata, (mach_vm_address_t)buf, hdr_tag, size,
356 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
357 if (error) {
358 goto out;
359 }
360
361 istate = ml_set_interrupts_enabled(FALSE);
362
363 /* Preload trace parameters*/
364 kdp_snapshot_preflight(pid, buf, size, flags, &kcdata, delta_since_timestamp);
365
366 /*
367 * Trap to the debugger to obtain a coherent stack snapshot; this populates
368 * the trace buffer
369 */
370 error = stackshot_trap();
371
372 ml_set_interrupts_enabled(istate);
373
374 *bytes_traced = kdp_stack_snapshot_bytes_traced();
375
376out:
377 stackshot_kcdata_p = NULL;
378 STACKSHOT_SUBSYS_UNLOCK();
379 return error;
380}
381
382#if CONFIG_TELEMETRY
383kern_return_t
384stack_microstackshot(user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, int32_t *retval)
385{
386 int error = KERN_SUCCESS;
387 uint32_t bytes_traced = 0;
388
389 *retval = -1;
390
391 /*
392 * Control related operations
393 */
394 if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
395 telemetry_global_ctl(1);
396 *retval = 0;
397 goto exit;
398 } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
399 telemetry_global_ctl(0);
400 *retval = 0;
401 goto exit;
402 }
403
404 /*
405 * Data related operations
406 */
407 *retval = -1;
408
409 if ((((void*)tracebuf) == NULL) || (tracebuf_size == 0)) {
410 error = KERN_INVALID_ARGUMENT;
411 goto exit;
412 }
413
414 STACKSHOT_SUBSYS_LOCK();
415
416 if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
417 if (tracebuf_size > max_tracebuf_size) {
418 error = KERN_INVALID_ARGUMENT;
419 goto unlock_exit;
420 }
421
422 bytes_traced = tracebuf_size;
423 error = telemetry_gather(tracebuf, &bytes_traced,
424 (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
425 *retval = (int)bytes_traced;
426 goto unlock_exit;
427 }
428
429 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
430
431 if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
432 error = KERN_INVALID_ARGUMENT;
433 goto unlock_exit;
434 }
435
436 bytes_traced = tracebuf_size;
437 error = bootprofile_gather(tracebuf, &bytes_traced);
438 *retval = (int)bytes_traced;
439 }
440
441unlock_exit:
442 STACKSHOT_SUBSYS_UNLOCK();
443exit:
444 return error;
445}
446#endif /* CONFIG_TELEMETRY */
447
448/*
449 * Return the estimated size of a stackshot based on the
450 * number of currently running threads and tasks.
451 */
452uint32_t
453get_stackshot_estsize(uint32_t prev_size_hint)
454{
455 vm_size_t thread_total;
456 vm_size_t task_total;
457 uint32_t estimated_size;
458
459 thread_total = (threads_count * sizeof(struct thread_snapshot));
460 task_total = (tasks_count * (sizeof(struct task_snapshot) + TASK_UUID_AVG_SIZE));
461
462 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE((thread_total + task_total + STACKSHOT_SUPP_SIZE), PAGE_MASK);
463 if (estimated_size < prev_size_hint) {
464 estimated_size = (uint32_t) VM_MAP_ROUND_PAGE(prev_size_hint, PAGE_MASK);
465 }
466
467 return estimated_size;
468}
469
470/*
471 * stackshot_remap_buffer: Utility function to remap bytes_traced bytes starting at stackshotbuf
472 * into the current task's user space and subsequently copy out the address
473 * at which the buffer has been mapped in user space to out_buffer_addr.
474 *
475 * Inputs: stackshotbuf - pointer to the original buffer in the kernel's address space
476 * bytes_traced - length of the buffer to remap starting from stackshotbuf
477 * out_buffer_addr - pointer to placeholder where newly mapped buffer will be mapped.
478 * out_size_addr - pointer to be filled in with the size of the buffer
479 *
480 * Outputs: ENOSPC if there is not enough free space in the task's address space to remap the buffer
481 * EINVAL for all other errors returned by task_remap_buffer/mach_vm_remap
482 * an error from copyout
483 */
484static kern_return_t
485stackshot_remap_buffer(void *stackshotbuf, uint32_t bytes_traced, uint64_t out_buffer_addr, uint64_t out_size_addr)
486{
487 int error = 0;
488 mach_vm_offset_t stackshotbuf_user_addr = (mach_vm_offset_t)NULL;
489 vm_prot_t cur_prot, max_prot;
490
491 error = mach_vm_remap_kernel(get_task_map(current_task()), &stackshotbuf_user_addr, bytes_traced, 0,
492 VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, kernel_map, (mach_vm_offset_t)stackshotbuf, FALSE, &cur_prot, &max_prot, VM_INHERIT_DEFAULT);
493 /*
494 * If the call to mach_vm_remap fails, we return the appropriate converted error
495 */
496 if (error == KERN_SUCCESS) {
497 /*
498 * If we fail to copy out the address or size of the new buffer, we remove the buffer mapping that
499 * we just made in the task's user space.
500 */
501 error = copyout(CAST_DOWN(void *, &stackshotbuf_user_addr), (user_addr_t)out_buffer_addr, sizeof(stackshotbuf_user_addr));
502 if (error != KERN_SUCCESS) {
503 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
504 return error;
505 }
506 error = copyout(&bytes_traced, (user_addr_t)out_size_addr, sizeof(bytes_traced));
507 if (error != KERN_SUCCESS) {
508 mach_vm_deallocate(get_task_map(current_task()), stackshotbuf_user_addr, (mach_vm_size_t)bytes_traced);
509 return error;
510 }
511 }
512 return error;
513}
514
515kern_return_t
516kern_stack_snapshot_internal(int stackshot_config_version, void *stackshot_config, size_t stackshot_config_size, boolean_t stackshot_from_user)
517{
518 int error = 0;
519 boolean_t prev_interrupt_state;
520 uint32_t bytes_traced = 0;
521 uint32_t stackshotbuf_size = 0;
522 void * stackshotbuf = NULL;
523 kcdata_descriptor_t kcdata_p = NULL;
524
525 void * buf_to_free = NULL;
526 int size_to_free = 0;
527
528 /* Parsed arguments */
529 uint64_t out_buffer_addr;
530 uint64_t out_size_addr;
531 int pid = -1;
532 uint32_t flags;
533 uint64_t since_timestamp;
534 uint32_t size_hint = 0;
535
536 if(stackshot_config == NULL) {
537 return KERN_INVALID_ARGUMENT;
538 }
539#if DEVELOPMENT || DEBUG
540 /* TBD: ask stackshot clients to avoid issuing stackshots in this
541 * configuration in lieu of the kernel feature override.
542 */
543 if (kern_feature_override(KF_STACKSHOT_OVRD) == TRUE) {
544 return KERN_NOT_SUPPORTED;
545 }
546#endif
547
548 switch (stackshot_config_version) {
549 case STACKSHOT_CONFIG_TYPE:
550 if (stackshot_config_size != sizeof(stackshot_config_t)) {
551 return KERN_INVALID_ARGUMENT;
552 }
553 stackshot_config_t *config = (stackshot_config_t *) stackshot_config;
554 out_buffer_addr = config->sc_out_buffer_addr;
555 out_size_addr = config->sc_out_size_addr;
556 pid = config->sc_pid;
557 flags = config->sc_flags;
558 since_timestamp = config->sc_delta_timestamp;
559 if (config->sc_size <= max_tracebuf_size) {
560 size_hint = config->sc_size;
561 }
562 break;
563 default:
564 return KERN_NOT_SUPPORTED;
565 }
566
567 /*
568 * Currently saving a kernel buffer and trylock are only supported from the
569 * internal/KEXT API.
570 */
571 if (stackshot_from_user) {
572 if (flags & (STACKSHOT_TRYLOCK | STACKSHOT_SAVE_IN_KERNEL_BUFFER | STACKSHOT_FROM_PANIC)) {
573 return KERN_NO_ACCESS;
574 }
575 } else {
576 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
577 return KERN_NOT_SUPPORTED;
578 }
579 }
580
581 if (!((flags & STACKSHOT_KCDATA_FORMAT) || (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER))) {
582 return KERN_NOT_SUPPORTED;
583 }
584
585 /*
586 * If we're not saving the buffer in the kernel pointer, we need a place to copy into.
587 */
588 if ((!out_buffer_addr || !out_size_addr) && !(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
589 return KERN_INVALID_ARGUMENT;
590 }
591
592 if (since_timestamp != 0 && ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) == 0)) {
593 return KERN_INVALID_ARGUMENT;
594 }
595
596#if MONOTONIC
597 if (!mt_core_supported) {
598 flags &= ~STACKSHOT_INSTRS_CYCLES;
599 }
600#else /* MONOTONIC */
601 flags &= ~STACKSHOT_INSTRS_CYCLES;
602#endif /* !MONOTONIC */
603
604 STACKSHOT_SUBSYS_LOCK();
605
606 if (flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER) {
607 /*
608 * Don't overwrite an existing stackshot
609 */
610 if (kernel_stackshot_buf != NULL) {
611 error = KERN_MEMORY_PRESENT;
612 goto error_exit;
613 }
614 } else if (flags & STACKSHOT_RETRIEVE_EXISTING_BUFFER) {
615 if ((kernel_stackshot_buf == NULL) || (kernel_stackshot_buf_size <= 0)) {
616 error = KERN_NOT_IN_SET;
617 goto error_exit;
618 }
619 error = stackshot_remap_buffer(kernel_stackshot_buf, kernel_stackshot_buf_size,
620 out_buffer_addr, out_size_addr);
621 /*
622 * If we successfully remapped the buffer into the user's address space, we
623 * set buf_to_free and size_to_free so the prior kernel mapping will be removed
624 * and then clear the kernel stackshot pointer and associated size.
625 */
626 if (error == KERN_SUCCESS) {
627 buf_to_free = kernel_stackshot_buf;
628 size_to_free = (int) VM_MAP_ROUND_PAGE(kernel_stackshot_buf_size, PAGE_MASK);
629 kernel_stackshot_buf = NULL;
630 kernel_stackshot_buf_size = 0;
631 }
632
633 goto error_exit;
634 }
635
636 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
637 void *bootprofile = NULL;
638 uint32_t len = 0;
639#if CONFIG_TELEMETRY
640 bootprofile_get(&bootprofile, &len);
641#endif
642 if (!bootprofile || !len) {
643 error = KERN_NOT_IN_SET;
644 goto error_exit;
645 }
646 error = stackshot_remap_buffer(bootprofile, len, out_buffer_addr, out_size_addr);
647 goto error_exit;
648 }
649
650 stackshotbuf_size = get_stackshot_estsize(size_hint);
651
652 for (; stackshotbuf_size <= max_tracebuf_size; stackshotbuf_size <<= 1) {
653 if (kmem_alloc(kernel_map, (vm_offset_t *)&stackshotbuf, stackshotbuf_size, VM_KERN_MEMORY_DIAG) != KERN_SUCCESS) {
654 error = KERN_RESOURCE_SHORTAGE;
655 goto error_exit;
656 }
657
658
659 uint32_t hdr_tag = (flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) ? KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT : KCDATA_BUFFER_BEGIN_STACKSHOT;
660 kcdata_p = kcdata_memory_alloc_init((mach_vm_address_t)stackshotbuf, hdr_tag, stackshotbuf_size,
661 KCFLAG_USE_MEMCOPY | KCFLAG_NO_AUTO_ENDBUFFER);
662
663 stackshot_duration_outer = NULL;
664 uint64_t time_start = mach_absolute_time();
665
666 /*
667 * Disable interrupts and save the current interrupt state.
668 */
669 prev_interrupt_state = ml_set_interrupts_enabled(FALSE);
670
671 /*
672 * Load stackshot parameters.
673 */
674 kdp_snapshot_preflight(pid, stackshotbuf, stackshotbuf_size, flags, kcdata_p, since_timestamp);
675
676 error = stackshot_trap();
677
678 ml_set_interrupts_enabled(prev_interrupt_state);
679
680 /* record the duration that interupts were disabled */
681
682 uint64_t time_end = mach_absolute_time();
683 if (stackshot_duration_outer) {
684 *stackshot_duration_outer = time_end - time_start;
685 }
686
687 if (error != KERN_SUCCESS) {
688 if (kcdata_p != NULL) {
689 kcdata_memory_destroy(kcdata_p);
690 kcdata_p = NULL;
691 stackshot_kcdata_p = NULL;
692 }
693 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
694 stackshotbuf = NULL;
695 if (error == KERN_INSUFFICIENT_BUFFER_SIZE) {
696 /*
697 * If we didn't allocate a big enough buffer, deallocate and try again.
698 */
699 continue;
700 } else {
701 goto error_exit;
702 }
703 }
704
705 bytes_traced = kdp_stack_snapshot_bytes_traced();
706
707 if (bytes_traced <= 0) {
708 error = KERN_ABORTED;
709 goto error_exit;
710 }
711
712 assert(bytes_traced <= stackshotbuf_size);
713 if (!(flags & STACKSHOT_SAVE_IN_KERNEL_BUFFER)) {
714 error = stackshot_remap_buffer(stackshotbuf, bytes_traced, out_buffer_addr, out_size_addr);
715 goto error_exit;
716 }
717
718 /*
719 * Save the stackshot in the kernel buffer.
720 */
721 kernel_stackshot_buf = stackshotbuf;
722 kernel_stackshot_buf_size = bytes_traced;
723 /*
724 * Figure out if we didn't use all the pages in the buffer. If so, we set buf_to_free to the beginning of
725 * the next page after the end of the stackshot in the buffer so that the kmem_free clips the buffer and
726 * update size_to_free for kmem_free accordingly.
727 */
728 size_to_free = stackshotbuf_size - (int) VM_MAP_ROUND_PAGE(bytes_traced, PAGE_MASK);
729
730 assert(size_to_free >= 0);
731
732 if (size_to_free != 0) {
733 buf_to_free = (void *)((uint64_t)stackshotbuf + stackshotbuf_size - size_to_free);
734 }
735
736 stackshotbuf = NULL;
737 stackshotbuf_size = 0;
738 goto error_exit;
739 }
740
741 if (stackshotbuf_size > max_tracebuf_size) {
742 error = KERN_RESOURCE_SHORTAGE;
743 }
744
745error_exit:
746 if (kcdata_p != NULL) {
747 kcdata_memory_destroy(kcdata_p);
748 kcdata_p = NULL;
749 stackshot_kcdata_p = NULL;
750 }
751
752 if (stackshotbuf != NULL) {
753 kmem_free(kernel_map, (vm_offset_t)stackshotbuf, stackshotbuf_size);
754 }
755 if (buf_to_free != NULL) {
756 kmem_free(kernel_map, (vm_offset_t)buf_to_free, size_to_free);
757 }
758 STACKSHOT_SUBSYS_UNLOCK();
759 return error;
760}
761
762/*
763 * Cache stack snapshot parameters in preparation for a trace.
764 */
765void
766kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t flags,
767 kcdata_descriptor_t data_p, uint64_t since_timestamp)
768{
769 uint64_t microsecs = 0, secs = 0;
770 clock_get_calendar_microtime((clock_sec_t *)&secs, (clock_usec_t *)&microsecs);
771
772 stackshot_microsecs = microsecs + (secs * USEC_PER_SEC);
773 stack_snapshot_pid = pid;
774 stack_snapshot_buf = tracebuf;
775 stack_snapshot_bufsize = tracebuf_size;
776 stack_snapshot_flags = flags;
777 stack_snapshot_delta_since_timestamp = since_timestamp;
778
779 panic_stackshot = ((flags & STACKSHOT_FROM_PANIC) != 0);
780
781 assert(data_p != NULL);
782 assert(stackshot_kcdata_p == NULL);
783 stackshot_kcdata_p = data_p;
784
785 stack_snapshot_bytes_traced = 0;
786}
787
788void
789panic_stackshot_reset_state()
790{
791 stackshot_kcdata_p = NULL;
792}
793
794boolean_t
795stackshot_active()
796{
797 return (stackshot_kcdata_p != NULL);
798}
799
800uint32_t
801kdp_stack_snapshot_bytes_traced(void)
802{
803 return stack_snapshot_bytes_traced;
804}
805
806static boolean_t memory_iszero(void *addr, size_t size)
807{
808 char *data = (char *)addr;
809 for (size_t i = 0; i < size; i++){
810 if (data[i] != 0)
811 return FALSE;
812 }
813 return TRUE;
814}
815
816#define kcd_end_address(kcd) ((void *)((uint64_t)((kcd)->kcd_addr_begin) + kcdata_memory_get_used_bytes((kcd))))
817#define kcd_max_address(kcd) ((void *)((kcd)->kcd_addr_begin + (kcd)->kcd_length))
818/*
819 * Use of the kcd_exit_on_error(action) macro requires a local
820 * 'kern_return_t error' variable and 'error_exit' label.
821 */
822#define kcd_exit_on_error(action) \
823 do { \
824 if (KERN_SUCCESS != (error = (action))) { \
825 if (error == KERN_RESOURCE_SHORTAGE) { \
826 error = KERN_INSUFFICIENT_BUFFER_SIZE; \
827 } \
828 goto error_exit; \
829 } \
830 } while (0); /* end kcd_exit_on_error */
831
832static uint64_t
833kcdata_get_task_ss_flags(task_t task)
834{
835 uint64_t ss_flags = 0;
836 boolean_t task_64bit_addr = task_has_64Bit_addr(task);
837
838 if (task_64bit_addr)
839 ss_flags |= kUser64_p;
840 if (!task->active || task_is_a_corpse(task) || proc_exiting(task->bsd_info))
841 ss_flags |= kTerminatedSnapshot;
842 if (task->pidsuspended)
843 ss_flags |= kPidSuspended;
844 if (task->frozen)
845 ss_flags |= kFrozen;
846 if (task->effective_policy.tep_darwinbg == 1)
847 ss_flags |= kTaskDarwinBG;
848 if (task->requested_policy.trp_role == TASK_FOREGROUND_APPLICATION)
849 ss_flags |= kTaskIsForeground;
850 if (task->requested_policy.trp_boosted == 1)
851 ss_flags |= kTaskIsBoosted;
852 if (task->effective_policy.tep_sup_active == 1)
853 ss_flags |= kTaskIsSuppressed;
854#if CONFIG_MEMORYSTATUS
855
856 boolean_t dirty = FALSE, dirty_tracked = FALSE, allow_idle_exit = FALSE;
857 memorystatus_proc_flags_unsafe(task->bsd_info, &dirty, &dirty_tracked, &allow_idle_exit);
858 if (dirty)
859 ss_flags |= kTaskIsDirty;
860 if (dirty_tracked)
861 ss_flags |= kTaskIsDirtyTracked;
862 if (allow_idle_exit)
863 ss_flags |= kTaskAllowIdleExit;
864
865#endif
866 if (task->effective_policy.tep_tal_engaged)
867 ss_flags |= kTaskTALEngaged;
868
869 ss_flags |= (0x7 & workqueue_get_pwq_state_kdp(task->bsd_info)) << 17;
870
871#if IMPORTANCE_INHERITANCE
872 if (task->task_imp_base) {
873 if (task->task_imp_base->iit_donor)
874 ss_flags |= kTaskIsImpDonor;
875 if (task->task_imp_base->iit_live_donor)
876 ss_flags |= kTaskIsLiveImpDonor;
877 }
878#endif
879 return ss_flags;
880}
881
882static kern_return_t
883kcdata_record_shared_cache_info(kcdata_descriptor_t kcd, task_t task, unaligned_u64 *task_snap_ss_flags)
884{
885 kern_return_t error = KERN_SUCCESS;
886 mach_vm_address_t out_addr = 0;
887
888 uint64_t shared_cache_slide = 0;
889 uint64_t shared_cache_base_address = 0;
890 uint32_t kdp_fault_results = 0;
891
892 assert(task_snap_ss_flags != NULL);
893
894 if (task->shared_region && ml_validate_nofault((vm_offset_t)task->shared_region, sizeof(struct vm_shared_region))) {
895 struct vm_shared_region *sr = task->shared_region;
896 shared_cache_base_address = sr->sr_base_address + sr->sr_first_mapping;
897 } else {
898 *task_snap_ss_flags |= kTaskSharedRegionInfoUnavailable;
899 goto error_exit;
900 }
901
902 /* We haven't copied in the shared region UUID yet as part of setup */
903 if (!shared_cache_base_address || !task->shared_region->sr_uuid_copied) {
904 goto error_exit;
905 }
906
907 /*
908 * No refcounting here, but we are in debugger
909 * context, so that should be safe.
910 */
911 shared_cache_slide = task->shared_region->sr_slide_info.slide;
912
913 if (task->shared_region == init_task_shared_region) {
914 /* skip adding shared cache info -- it's the same as the system level one */
915 goto error_exit;
916 }
917
918 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64_v2), &out_addr));
919 struct dyld_uuid_info_64_v2 *shared_cache_data = (struct dyld_uuid_info_64_v2 *)out_addr;
920 shared_cache_data->imageLoadAddress = shared_cache_slide;
921 stackshot_memcpy(shared_cache_data->imageUUID, task->shared_region->sr_uuid, sizeof(task->shared_region->sr_uuid));
922 shared_cache_data->imageSlidBaseAddress = shared_cache_base_address;
923
924error_exit:
925 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
926 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
927 }
928
929 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
930 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
931 }
932
933 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
934 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
935 }
936
937 return error;
938}
939
940static kern_return_t
941kcdata_record_uuid_info(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 *task_snap_ss_flags)
942{
943 boolean_t save_loadinfo_p = ((trace_flags & STACKSHOT_SAVE_LOADINFO) != 0);
944 boolean_t save_kextloadinfo_p = ((trace_flags & STACKSHOT_SAVE_KEXT_LOADINFO) != 0);
945 boolean_t should_fault = (trace_flags & STACKSHOT_ENABLE_UUID_FAULTING);
946
947 kern_return_t error = KERN_SUCCESS;
948 mach_vm_address_t out_addr = 0;
949
950 uint32_t uuid_info_count = 0;
951 mach_vm_address_t uuid_info_addr = 0;
952 uint64_t uuid_info_timestamp = 0;
953 uint32_t kdp_fault_results = 0;
954
955 assert(task_snap_ss_flags != NULL);
956
957 int task_pid = pid_from_task(task);
958 boolean_t task_64bit_addr = task_has_64Bit_addr(task);
959
960 if (save_loadinfo_p && have_pmap && task->active && task_pid > 0) {
961 /* Read the dyld_all_image_infos struct from the task memory to get UUID array count and location */
962 if (task_64bit_addr) {
963 struct user64_dyld_all_image_infos task_image_infos;
964 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
965 sizeof(struct user64_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
966 uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
967 uuid_info_addr = task_image_infos.uuidArray;
968 if (task_image_infos.version >= DYLD_ALL_IMAGE_INFOS_TIMESTAMP_MINIMUM_VERSION) {
969 uuid_info_timestamp = task_image_infos.timestamp;
970 }
971 }
972 } else {
973 struct user32_dyld_all_image_infos task_image_infos;
974 if (kdp_copyin(task->map, task->all_image_info_addr, &task_image_infos,
975 sizeof(struct user32_dyld_all_image_infos), should_fault, &kdp_fault_results)) {
976 uuid_info_count = task_image_infos.uuidArrayCount;
977 uuid_info_addr = task_image_infos.uuidArray;
978 if (task_image_infos.version >= DYLD_ALL_IMAGE_INFOS_TIMESTAMP_MINIMUM_VERSION) {
979 uuid_info_timestamp = task_image_infos.timestamp;
980 }
981 }
982 }
983
984 /*
985 * If we get a NULL uuid_info_addr (which can happen when we catch dyld in the middle of updating
986 * this data structure), we zero the uuid_info_count so that we won't even try to save load info
987 * for this task.
988 */
989 if (!uuid_info_addr) {
990 uuid_info_count = 0;
991 }
992 }
993
994 if (have_pmap && task_pid == 0) {
995 if (save_kextloadinfo_p && ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader))) {
996 uuid_info_count = gLoadedKextSummaries->numSummaries + 1; /* include main kernel UUID */
997 } else {
998 uuid_info_count = 1; /* include kernelcache UUID (embedded) or kernel UUID (desktop) */
999 }
1000 }
1001
1002 if (task_pid > 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
1003 uint32_t uuid_info_size = (uint32_t)(task_64bit_addr ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
1004 uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
1005
1006 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, (task_64bit_addr ? KCDATA_TYPE_LIBRARY_LOADINFO64 : KCDATA_TYPE_LIBRARY_LOADINFO),
1007 uuid_info_size, uuid_info_count, &out_addr));
1008
1009 /* Copy in the UUID info array
1010 * It may be nonresident, in which case just fix up nloadinfos to 0 in the task_snap
1011 */
1012 if (have_pmap && !kdp_copyin(task->map, uuid_info_addr, (void *)out_addr, uuid_info_array_size, should_fault, &kdp_fault_results)) {
1013 bzero((void *)out_addr, uuid_info_array_size);
1014 }
1015
1016 } else if (task_pid == 0 && uuid_info_count > 0 && uuid_info_count < MAX_LOADINFOS) {
1017 uintptr_t image_load_address;
1018
1019 do {
1020
1021#if CONFIG_EMBEDDED
1022 if (kernelcache_uuid_valid && !save_kextloadinfo_p) {
1023 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO, sizeof(struct dyld_uuid_info_64), &out_addr));
1024 struct dyld_uuid_info_64 *kc_uuid = (struct dyld_uuid_info_64 *)out_addr;
1025 kc_uuid->imageLoadAddress = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
1026 stackshot_memcpy(&kc_uuid->imageUUID, &kernelcache_uuid, sizeof(uuid_t));
1027 break;
1028 }
1029#endif /* CONFIG_EMBEDDED */
1030
1031 if (!kernel_uuid || !ml_validate_nofault((vm_offset_t)kernel_uuid, sizeof(uuid_t))) {
1032 /* Kernel UUID not found or inaccessible */
1033 break;
1034 }
1035
1036 kcd_exit_on_error(kcdata_get_memory_addr_for_array(
1037 kcd, (sizeof(kernel_uuid_info) == sizeof(struct user64_dyld_uuid_info)) ? KCDATA_TYPE_LIBRARY_LOADINFO64
1038 : KCDATA_TYPE_LIBRARY_LOADINFO,
1039 sizeof(kernel_uuid_info), uuid_info_count, &out_addr));
1040 kernel_uuid_info *uuid_info_array = (kernel_uuid_info *)out_addr;
1041 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(vm_kernel_stext);
1042 uuid_info_array[0].imageLoadAddress = image_load_address;
1043 stackshot_memcpy(&uuid_info_array[0].imageUUID, kernel_uuid, sizeof(uuid_t));
1044
1045 if (save_kextloadinfo_p &&
1046 ml_validate_nofault((vm_offset_t)(gLoadedKextSummaries), sizeof(OSKextLoadedKextSummaryHeader)) &&
1047 ml_validate_nofault((vm_offset_t)(&gLoadedKextSummaries->summaries[0]),
1048 gLoadedKextSummaries->entry_size * gLoadedKextSummaries->numSummaries)) {
1049 uint32_t kexti;
1050 for (kexti=0 ; kexti < gLoadedKextSummaries->numSummaries; kexti++) {
1051 image_load_address = (uintptr_t)VM_KERNEL_UNSLIDE(gLoadedKextSummaries->summaries[kexti].address);
1052 uuid_info_array[kexti + 1].imageLoadAddress = image_load_address;
1053 stackshot_memcpy(&uuid_info_array[kexti + 1].imageUUID, &gLoadedKextSummaries->summaries[kexti].uuid, sizeof(uuid_t));
1054 }
1055 }
1056 } while(0);
1057 }
1058
1059error_exit:
1060 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
1061 *task_snap_ss_flags |= kTaskUUIDInfoMissing;
1062 }
1063
1064 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
1065 *task_snap_ss_flags |= kTaskUUIDInfoTriedFault;
1066 }
1067
1068 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
1069 *task_snap_ss_flags |= kTaskUUIDInfoFaultedIn;
1070 }
1071
1072 return error;
1073}
1074
1075static kern_return_t
1076kcdata_record_task_iostats(kcdata_descriptor_t kcd, task_t task)
1077{
1078 kern_return_t error = KERN_SUCCESS;
1079 mach_vm_address_t out_addr = 0;
1080
1081 /* I/O Statistics if any counters are non zero */
1082 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1083 if (task->task_io_stats && !memory_iszero(task->task_io_stats, sizeof(struct io_stat_info))) {
1084 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1085 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1086 _iostat->ss_disk_reads_count = task->task_io_stats->disk_reads.count;
1087 _iostat->ss_disk_reads_size = task->task_io_stats->disk_reads.size;
1088 _iostat->ss_disk_writes_count = (task->task_io_stats->total_io.count - task->task_io_stats->disk_reads.count);
1089 _iostat->ss_disk_writes_size = (task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size);
1090 _iostat->ss_paging_count = task->task_io_stats->paging.count;
1091 _iostat->ss_paging_size = task->task_io_stats->paging.size;
1092 _iostat->ss_non_paging_count = (task->task_io_stats->total_io.count - task->task_io_stats->paging.count);
1093 _iostat->ss_non_paging_size = (task->task_io_stats->total_io.size - task->task_io_stats->paging.size);
1094 _iostat->ss_metadata_count = task->task_io_stats->metadata.count;
1095 _iostat->ss_metadata_size = task->task_io_stats->metadata.size;
1096 _iostat->ss_data_count = (task->task_io_stats->total_io.count - task->task_io_stats->metadata.count);
1097 _iostat->ss_data_size = (task->task_io_stats->total_io.size - task->task_io_stats->metadata.size);
1098 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1099 _iostat->ss_io_priority_count[i] = task->task_io_stats->io_priority[i].count;
1100 _iostat->ss_io_priority_size[i] = task->task_io_stats->io_priority[i].size;
1101 }
1102 }
1103
1104error_exit:
1105 return error;
1106}
1107
1108#if MONOTONIC
1109static kern_return_t
1110kcdata_record_task_instrs_cycles(kcdata_descriptor_t kcd, task_t task)
1111{
1112 uint64_t instrs = 0, cycles = 0;
1113 mt_stackshot_task(task, &instrs, &cycles);
1114
1115 kern_return_t error = KERN_SUCCESS;
1116 mach_vm_address_t out_addr = 0;
1117 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_INSTRS_CYCLES, sizeof(struct instrs_cycles_snapshot), &out_addr));
1118 struct instrs_cycles_snapshot *instrs_cycles = (struct instrs_cycles_snapshot *)out_addr;
1119 instrs_cycles->ics_instructions = instrs;
1120 instrs_cycles->ics_cycles = cycles;
1121
1122error_exit:
1123 return error;
1124}
1125#endif /* MONOTONIC */
1126
1127static kern_return_t
1128kcdata_record_task_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
1129{
1130 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1131 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_NO_IO_STATS);
1132#if MONOTONIC
1133 boolean_t collect_instrs_cycles = ((trace_flags & STACKSHOT_INSTRS_CYCLES) != 0);
1134#endif /* MONOTONIC */
1135#if __arm__ || __arm64__
1136 boolean_t collect_asid = ((trace_flags & STACKSHOT_ASID) != 0);
1137#endif
1138 boolean_t collect_pagetables = ((trace_flags & STACKSHOT_PAGE_TABLES) != 0);
1139
1140
1141 kern_return_t error = KERN_SUCCESS;
1142 mach_vm_address_t out_addr = 0;
1143 struct task_snapshot_v2 * cur_tsnap = NULL;
1144
1145 assert(task_snap_ss_flags != NULL);
1146
1147 int task_pid = pid_from_task(task);
1148 uint64_t task_uniqueid = get_task_uniqueid(task);
1149 uint64_t proc_starttime_secs = 0;
1150
1151 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_SNAPSHOT, sizeof(struct task_snapshot_v2), &out_addr));
1152 cur_tsnap = (struct task_snapshot_v2 *)out_addr;
1153 bzero(cur_tsnap, sizeof(*cur_tsnap));
1154
1155 cur_tsnap->ts_unique_pid = task_uniqueid;
1156 cur_tsnap->ts_ss_flags = kcdata_get_task_ss_flags(task);
1157 *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->ts_ss_flags;
1158 cur_tsnap->ts_user_time_in_terminated_threads = task->total_user_time;
1159 cur_tsnap->ts_system_time_in_terminated_threads = task->total_system_time;
1160
1161 proc_starttime_kdp(task->bsd_info, &proc_starttime_secs, NULL, NULL);
1162 cur_tsnap->ts_p_start_sec = proc_starttime_secs;
1163 cur_tsnap->ts_task_size = have_pmap ? get_task_phys_footprint(task) : 0;
1164 cur_tsnap->ts_max_resident_size = get_task_resident_max(task);
1165 cur_tsnap->ts_was_throttled = (uint32_t) proc_was_throttled_from_task(task);
1166 cur_tsnap->ts_did_throttle = (uint32_t) proc_did_throttle_from_task(task);
1167
1168 cur_tsnap->ts_suspend_count = task->suspend_count;
1169 cur_tsnap->ts_faults = task->faults;
1170 cur_tsnap->ts_pageins = task->pageins;
1171 cur_tsnap->ts_cow_faults = task->cow_faults;
1172 cur_tsnap->ts_latency_qos = (task->effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED) ?
1173 LATENCY_QOS_TIER_UNSPECIFIED : ((0xFF << 16) | task->effective_policy.tep_latency_qos);
1174 cur_tsnap->ts_pid = task_pid;
1175
1176#if __arm__ || __arm64__
1177 if (collect_asid && have_pmap) {
1178 uint32_t asid = task->map->pmap->asid;
1179 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_ASID, sizeof(uint32_t), &out_addr));
1180 stackshot_memcpy((void*)out_addr, &asid, sizeof(asid));
1181 }
1182#endif
1183 if (collect_pagetables && have_pmap) {
1184#if INTERRUPT_MASKED_DEBUG
1185 // pagetable dumps can be large; reset the interrupt timeout to avoid a panic
1186 ml_spin_debug_clear_self();
1187#endif
1188 size_t bytes_dumped = pmap_dump_page_tables(task->map->pmap, kcd_end_address(kcd), kcd_max_address(kcd));
1189 if (bytes_dumped == 0) {
1190 error = KERN_INSUFFICIENT_BUFFER_SIZE;
1191 goto error_exit;
1192 } else if (bytes_dumped == (size_t)-1) {
1193 error = KERN_NOT_SUPPORTED;
1194 goto error_exit;
1195 } else {
1196 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, STACKSHOT_KCTYPE_PAGE_TABLES,
1197 sizeof(uint64_t), (uint32_t)(bytes_dumped / sizeof(uint64_t)), &out_addr));
1198 }
1199 }
1200
1201 /* Add the BSD process identifiers */
1202 if (task_pid != -1 && task->bsd_info != NULL) {
1203 proc_name_kdp(task, cur_tsnap->ts_p_comm, sizeof(cur_tsnap->ts_p_comm));
1204#if CONFIG_COALITIONS
1205 if ((trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) && (task->coalition[COALITION_TYPE_JETSAM] != NULL)) {
1206 uint64_t jetsam_coal_id = coalition_id(task->coalition[COALITION_TYPE_JETSAM]);
1207 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_JETSAM_COALITION, sizeof(jetsam_coal_id), &out_addr));
1208 stackshot_memcpy((void*)out_addr, &jetsam_coal_id, sizeof(jetsam_coal_id));
1209 }
1210#endif /* CONFIG_COALITIONS */
1211 }
1212 else {
1213 cur_tsnap->ts_p_comm[0] = '\0';
1214#if IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG)
1215 if (task->task_imp_base != NULL) {
1216 stackshot_strlcpy(cur_tsnap->ts_p_comm, &task->task_imp_base->iit_procname[0],
1217 MIN((int)sizeof(task->task_imp_base->iit_procname), (int)sizeof(cur_tsnap->ts_p_comm)));
1218 }
1219#endif /* IMPORTANCE_INHERITANCE && (DEVELOPMENT || DEBUG) */
1220 }
1221
1222 if (collect_iostats) {
1223 kcd_exit_on_error(kcdata_record_task_iostats(kcd, task));
1224 }
1225
1226#if MONOTONIC
1227 if (collect_instrs_cycles) {
1228 kcd_exit_on_error(kcdata_record_task_instrs_cycles(kcd, task));
1229 }
1230#endif /* MONOTONIC */
1231
1232error_exit:
1233 return error;
1234}
1235
1236static kern_return_t
1237kcdata_record_task_delta_snapshot(kcdata_descriptor_t kcd, task_t task, uint32_t trace_flags, boolean_t have_pmap, unaligned_u64 **task_snap_ss_flags)
1238{
1239#if !MONOTONIC
1240#pragma unused(trace_flags)
1241#endif /* !MONOTONIC */
1242 kern_return_t error = KERN_SUCCESS;
1243 struct task_delta_snapshot_v2 * cur_tsnap = NULL;
1244 mach_vm_address_t out_addr = 0;
1245 (void) trace_flags;
1246#if __arm__ || __arm64__
1247 boolean_t collect_asid = ((trace_flags & STACKSHOT_ASID) != 0);
1248#endif
1249#if MONOTONIC
1250 boolean_t collect_instrs_cycles = ((trace_flags & STACKSHOT_INSTRS_CYCLES) != 0);
1251#endif /* MONOTONIC */
1252
1253 uint64_t task_uniqueid = get_task_uniqueid(task);
1254 assert(task_snap_ss_flags != NULL);
1255
1256 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT, sizeof(struct task_delta_snapshot_v2), &out_addr));
1257
1258 cur_tsnap = (struct task_delta_snapshot_v2 *)out_addr;
1259
1260 cur_tsnap->tds_unique_pid = task_uniqueid;
1261 cur_tsnap->tds_ss_flags = kcdata_get_task_ss_flags(task);
1262 *task_snap_ss_flags = (unaligned_u64 *)&cur_tsnap->tds_ss_flags;
1263
1264 cur_tsnap->tds_user_time_in_terminated_threads = task->total_user_time;
1265 cur_tsnap->tds_system_time_in_terminated_threads = task->total_system_time;
1266
1267 cur_tsnap->tds_task_size = have_pmap ? get_task_phys_footprint(task) : 0;
1268
1269 cur_tsnap->tds_max_resident_size = get_task_resident_max(task);
1270 cur_tsnap->tds_suspend_count = task->suspend_count;
1271 cur_tsnap->tds_faults = task->faults;
1272 cur_tsnap->tds_pageins = task->pageins;
1273 cur_tsnap->tds_cow_faults = task->cow_faults;
1274 cur_tsnap->tds_was_throttled = (uint32_t)proc_was_throttled_from_task(task);
1275 cur_tsnap->tds_did_throttle = (uint32_t)proc_did_throttle_from_task(task);
1276 cur_tsnap->tds_latency_qos = (task-> effective_policy.tep_latency_qos == LATENCY_QOS_TIER_UNSPECIFIED)
1277 ? LATENCY_QOS_TIER_UNSPECIFIED
1278 : ((0xFF << 16) | task-> effective_policy.tep_latency_qos);
1279
1280#if __arm__ || __arm64__
1281 if (collect_asid && have_pmap) {
1282 uint32_t asid = task->map->pmap->asid;
1283 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_ASID, sizeof(uint32_t), &out_addr));
1284 stackshot_memcpy((void*)out_addr, &asid, sizeof(asid));
1285 }
1286#endif
1287
1288#if MONOTONIC
1289 if (collect_instrs_cycles) {
1290 kcd_exit_on_error(kcdata_record_task_instrs_cycles(kcd, task));
1291 }
1292#endif /* MONOTONIC */
1293
1294error_exit:
1295 return error;
1296}
1297
1298static kern_return_t
1299kcdata_record_thread_iostats(kcdata_descriptor_t kcd, thread_t thread)
1300{
1301 kern_return_t error = KERN_SUCCESS;
1302 mach_vm_address_t out_addr = 0;
1303
1304 /* I/O Statistics */
1305 assert(IO_NUM_PRIORITIES == STACKSHOT_IO_NUM_PRIORITIES);
1306 if (thread->thread_io_stats && !memory_iszero(thread->thread_io_stats, sizeof(struct io_stat_info))) {
1307 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_IOSTATS, sizeof(struct io_stats_snapshot), &out_addr));
1308 struct io_stats_snapshot *_iostat = (struct io_stats_snapshot *)out_addr;
1309 _iostat->ss_disk_reads_count = thread->thread_io_stats->disk_reads.count;
1310 _iostat->ss_disk_reads_size = thread->thread_io_stats->disk_reads.size;
1311 _iostat->ss_disk_writes_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->disk_reads.count);
1312 _iostat->ss_disk_writes_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->disk_reads.size);
1313 _iostat->ss_paging_count = thread->thread_io_stats->paging.count;
1314 _iostat->ss_paging_size = thread->thread_io_stats->paging.size;
1315 _iostat->ss_non_paging_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->paging.count);
1316 _iostat->ss_non_paging_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->paging.size);
1317 _iostat->ss_metadata_count = thread->thread_io_stats->metadata.count;
1318 _iostat->ss_metadata_size = thread->thread_io_stats->metadata.size;
1319 _iostat->ss_data_count = (thread->thread_io_stats->total_io.count - thread->thread_io_stats->metadata.count);
1320 _iostat->ss_data_size = (thread->thread_io_stats->total_io.size - thread->thread_io_stats->metadata.size);
1321 for(int i = 0; i < IO_NUM_PRIORITIES; i++) {
1322 _iostat->ss_io_priority_count[i] = thread->thread_io_stats->io_priority[i].count;
1323 _iostat->ss_io_priority_size[i] = thread->thread_io_stats->io_priority[i].size;
1324 }
1325 }
1326
1327error_exit:
1328 return error;
1329}
1330
1331static kern_return_t
1332kcdata_record_thread_snapshot(
1333 kcdata_descriptor_t kcd, thread_t thread, task_t task, uint32_t trace_flags, boolean_t have_pmap, boolean_t thread_on_core)
1334{
1335 boolean_t dispatch_p = ((trace_flags & STACKSHOT_GET_DQ) != 0);
1336 boolean_t active_kthreads_only_p = ((trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1337 boolean_t trace_fp_p = false;
1338 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1339 boolean_t collect_iostats = !collect_delta_stackshot && !(trace_flags & STACKSHOT_NO_IO_STATS);
1340#if MONOTONIC
1341 boolean_t collect_instrs_cycles = ((trace_flags & STACKSHOT_INSTRS_CYCLES) != 0);
1342#endif /* MONOTONIC */
1343
1344 kern_return_t error = KERN_SUCCESS;
1345 mach_vm_address_t out_addr = 0;
1346 int saved_count = 0;
1347
1348 struct thread_snapshot_v4 * cur_thread_snap = NULL;
1349 char cur_thread_name[STACKSHOT_MAX_THREAD_NAME_SIZE];
1350 uint64_t tval = 0;
1351 const boolean_t is_64bit_data = task_has_64Bit_data(task);
1352
1353 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_SNAPSHOT, sizeof(struct thread_snapshot_v4), &out_addr));
1354 cur_thread_snap = (struct thread_snapshot_v4 *)out_addr;
1355
1356 /* Populate the thread snapshot header */
1357 cur_thread_snap->ths_thread_id = thread_tid(thread);
1358 cur_thread_snap->ths_wait_event = VM_KERNEL_UNSLIDE_OR_PERM(thread->wait_event);
1359 cur_thread_snap->ths_continuation = VM_KERNEL_UNSLIDE(thread->continuation);
1360 cur_thread_snap->ths_total_syscalls = thread->syscalls_mach + thread->syscalls_unix;
1361
1362 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1363 cur_thread_snap->ths_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1364 else
1365 cur_thread_snap->ths_voucher_identifier = 0;
1366
1367 cur_thread_snap->ths_dqserialnum = 0;
1368 if (dispatch_p && (task != kernel_task) && (task->active) && have_pmap) {
1369 uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
1370 if (dqkeyaddr != 0) {
1371 uint64_t dqaddr = 0;
1372 boolean_t copyin_ok = kdp_copyin_word(task, dqkeyaddr, &dqaddr, FALSE, NULL);
1373 if (copyin_ok && dqaddr != 0) {
1374 uint64_t dqserialnumaddr = dqaddr + get_task_dispatchqueue_serialno_offset(task);
1375 uint64_t dqserialnum = 0;
1376 copyin_ok = kdp_copyin_word(task, dqserialnumaddr, &dqserialnum, FALSE, NULL);
1377 if (copyin_ok) {
1378 cur_thread_snap->ths_ss_flags |= kHasDispatchSerial;
1379 cur_thread_snap->ths_dqserialnum = dqserialnum;
1380 }
1381 }
1382 }
1383 }
1384
1385 tval = safe_grab_timer_value(&thread->user_timer);
1386 cur_thread_snap->ths_user_time = tval;
1387 tval = safe_grab_timer_value(&thread->system_timer);
1388
1389 if (thread->precise_user_kernel_time) {
1390 cur_thread_snap->ths_sys_time = tval;
1391 } else {
1392 cur_thread_snap->ths_user_time += tval;
1393 cur_thread_snap->ths_sys_time = 0;
1394 }
1395
1396 cur_thread_snap->ths_ss_flags = 0;
1397 if (thread->thread_tag & THREAD_TAG_MAINTHREAD)
1398 cur_thread_snap->ths_ss_flags |= kThreadMain;
1399 if (thread->effective_policy.thep_darwinbg)
1400 cur_thread_snap->ths_ss_flags |= kThreadDarwinBG;
1401 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1402 cur_thread_snap->ths_ss_flags |= kThreadIOPassive;
1403 if (thread->suspend_count > 0)
1404 cur_thread_snap->ths_ss_flags |= kThreadSuspended;
1405 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1406 cur_thread_snap->ths_ss_flags |= kGlobalForcedIdle;
1407 if (thread_on_core)
1408 cur_thread_snap->ths_ss_flags |= kThreadOnCore;
1409 if (stackshot_thread_is_idle_worker_unsafe(thread))
1410 cur_thread_snap->ths_ss_flags |= kThreadIdleWorker;
1411
1412 /* make sure state flags defined in kcdata.h still match internal flags */
1413 static_assert(SS_TH_WAIT == TH_WAIT);
1414 static_assert(SS_TH_SUSP == TH_SUSP);
1415 static_assert(SS_TH_RUN == TH_RUN);
1416 static_assert(SS_TH_UNINT == TH_UNINT);
1417 static_assert(SS_TH_TERMINATE == TH_TERMINATE);
1418 static_assert(SS_TH_TERMINATE2 == TH_TERMINATE2);
1419 static_assert(SS_TH_IDLE == TH_IDLE);
1420
1421 cur_thread_snap->ths_last_run_time = thread->last_run_time;
1422 cur_thread_snap->ths_last_made_runnable_time = thread->last_made_runnable_time;
1423 cur_thread_snap->ths_state = thread->state;
1424 cur_thread_snap->ths_sched_flags = thread->sched_flags;
1425 cur_thread_snap->ths_base_priority = thread->base_pri;
1426 cur_thread_snap->ths_sched_priority = thread->sched_pri;
1427 cur_thread_snap->ths_eqos = thread->effective_policy.thep_qos;
1428 cur_thread_snap->ths_rqos = thread->requested_policy.thrp_qos;
1429 cur_thread_snap->ths_rqos_override = MAX(thread->requested_policy.thrp_qos_override,
1430 thread->requested_policy.thrp_qos_workq_override);
1431 cur_thread_snap->ths_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1432 cur_thread_snap->ths_thread_t = VM_KERNEL_UNSLIDE_OR_PERM(thread);
1433
1434 static_assert(sizeof(thread->effective_policy) == sizeof(uint64_t));
1435 static_assert(sizeof(thread->requested_policy) == sizeof(uint64_t));
1436 cur_thread_snap->ths_requested_policy = *(unaligned_u64 *) &thread->requested_policy;
1437 cur_thread_snap->ths_effective_policy = *(unaligned_u64 *) &thread->effective_policy;
1438
1439 /* if there is thread name then add to buffer */
1440 cur_thread_name[0] = '\0';
1441 proc_threadname_kdp(thread->uthread, cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE);
1442 if (strnlen(cur_thread_name, STACKSHOT_MAX_THREAD_NAME_SIZE) > 0) {
1443 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_THREAD_NAME, sizeof(cur_thread_name), &out_addr));
1444 stackshot_memcpy((void *)out_addr, (void *)cur_thread_name, sizeof(cur_thread_name));
1445 }
1446
1447 /* record system, user, and runnable times */
1448 time_value_t user_time, system_time, runnable_time;
1449 thread_read_times(thread, &user_time, &system_time, &runnable_time);
1450 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_CPU_TIMES, sizeof(struct stackshot_cpu_times_v2), &out_addr));
1451 struct stackshot_cpu_times_v2 *stackshot_cpu_times = (struct stackshot_cpu_times_v2 *)out_addr;
1452 *stackshot_cpu_times = (struct stackshot_cpu_times_v2){
1453 .user_usec = (uint64_t)user_time.seconds * USEC_PER_SEC + user_time.microseconds,
1454 .system_usec = (uint64_t)system_time.seconds * USEC_PER_SEC + system_time.microseconds,
1455 .runnable_usec = (uint64_t)runnable_time.seconds * USEC_PER_SEC + runnable_time.microseconds,
1456 };
1457
1458 /* Trace user stack, if any */
1459 if (!active_kthreads_only_p && task->active && thread->task->map != kernel_map) {
1460 uint32_t thread_snapshot_flags = 0;
1461
1462 /* Uses 64-bit machine state? */
1463 if (is_64bit_data) {
1464 uint64_t sp = 0;
1465 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1466 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE,
1467 trace_fp_p, &thread_snapshot_flags, &sp);
1468 if (saved_count > 0) {
1469 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1470 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME64
1471 : STACKSHOT_KCTYPE_USER_STACKLR64,
1472 frame_size, saved_count / frame_size, &out_addr));
1473 cur_thread_snap->ths_ss_flags |= kUser64_p;
1474 }
1475#if __x86_64__
1476 if (sp) {
1477 // I'm using 8 here and not sizeof(stack_contents) because this
1478 // code would not work if you just made stack_contents bigger.
1479 vm_offset_t kern_virt_addr = machine_trace_thread_get_kva(sp, thread->task->map, &thread_snapshot_flags);
1480 if (kern_virt_addr && (kern_virt_addr % 8) == 0) {
1481 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_USER_STACKTOP, sizeof(struct stack_snapshot_stacktop), &out_addr));
1482 struct stack_snapshot_stacktop *stacktop = (struct stack_snapshot_stacktop *)out_addr;
1483 stacktop->sp = sp;
1484 memcpy(stacktop->stack_contents, (void*) kern_virt_addr, 8);
1485 }
1486 }
1487#endif
1488 } else {
1489 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1490 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, TRUE, trace_fp_p,
1491 &thread_snapshot_flags);
1492 if (saved_count > 0) {
1493 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1494 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_USER_STACKFRAME
1495 : STACKSHOT_KCTYPE_USER_STACKLR,
1496 frame_size, saved_count / frame_size, &out_addr));
1497 }
1498 }
1499
1500 if (thread_snapshot_flags != 0) {
1501 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1502 }
1503 }
1504
1505 /* Call through to the machine specific trace routines
1506 * Frames are added past the snapshot header.
1507 */
1508 if (thread->kernel_stack != 0) {
1509 uint32_t thread_snapshot_flags = 0;
1510#if defined(__LP64__)
1511 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1512 saved_count = machine_trace_thread64(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1513 &thread_snapshot_flags, NULL);
1514 if (saved_count > 0) {
1515 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame64) : sizeof(uint64_t);
1516 cur_thread_snap->ths_ss_flags |= kKernel64_p;
1517 kcd_exit_on_error(kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME64
1518 : STACKSHOT_KCTYPE_KERN_STACKLR64,
1519 frame_size, saved_count / frame_size, &out_addr));
1520 }
1521#else
1522 out_addr = (mach_vm_address_t)kcd_end_address(kcd);
1523 saved_count = machine_trace_thread(thread, (char *)out_addr, (char *)kcd_max_address(kcd), MAX_FRAMES, FALSE, trace_fp_p,
1524 &thread_snapshot_flags);
1525 if (saved_count > 0) {
1526 int frame_size = trace_fp_p ? sizeof(struct stack_snapshot_frame32) : sizeof(uint32_t);
1527 kcd_exit_on_error(
1528 kcdata_get_memory_addr_for_array(kcd, trace_fp_p ? STACKSHOT_KCTYPE_KERN_STACKFRAME : STACKSHOT_KCTYPE_KERN_STACKLR,
1529 frame_size, saved_count / frame_size, &out_addr));
1530 }
1531#endif
1532 if (thread_snapshot_flags != 0) {
1533 cur_thread_snap->ths_ss_flags |= thread_snapshot_flags;
1534 }
1535 }
1536
1537
1538 if (collect_iostats) {
1539 kcd_exit_on_error(kcdata_record_thread_iostats(kcd, thread));
1540 }
1541
1542#if MONOTONIC
1543 if (collect_instrs_cycles) {
1544 uint64_t instrs = 0, cycles = 0;
1545 mt_stackshot_thread(thread, &instrs, &cycles);
1546
1547 kcd_exit_on_error(kcdata_get_memory_addr(kcd, STACKSHOT_KCTYPE_INSTRS_CYCLES, sizeof(struct instrs_cycles_snapshot), &out_addr));
1548 struct instrs_cycles_snapshot *instrs_cycles = (struct instrs_cycles_snapshot *)out_addr;
1549 instrs_cycles->ics_instructions = instrs;
1550 instrs_cycles->ics_cycles = cycles;
1551 }
1552#endif /* MONOTONIC */
1553
1554error_exit:
1555 return error;
1556}
1557
1558static int
1559kcdata_record_thread_delta_snapshot(struct thread_delta_snapshot_v3 * cur_thread_snap, thread_t thread, boolean_t thread_on_core)
1560{
1561 cur_thread_snap->tds_thread_id = thread_tid(thread);
1562 if (IPC_VOUCHER_NULL != thread->ith_voucher)
1563 cur_thread_snap->tds_voucher_identifier = VM_KERNEL_ADDRPERM(thread->ith_voucher);
1564 else
1565 cur_thread_snap->tds_voucher_identifier = 0;
1566
1567 cur_thread_snap->tds_ss_flags = 0;
1568 if (thread->effective_policy.thep_darwinbg)
1569 cur_thread_snap->tds_ss_flags |= kThreadDarwinBG;
1570 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO))
1571 cur_thread_snap->tds_ss_flags |= kThreadIOPassive;
1572 if (thread->suspend_count > 0)
1573 cur_thread_snap->tds_ss_flags |= kThreadSuspended;
1574 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE)
1575 cur_thread_snap->tds_ss_flags |= kGlobalForcedIdle;
1576 if (thread_on_core)
1577 cur_thread_snap->tds_ss_flags |= kThreadOnCore;
1578 if (stackshot_thread_is_idle_worker_unsafe(thread))
1579 cur_thread_snap->tds_ss_flags |= kThreadIdleWorker;
1580
1581 cur_thread_snap->tds_last_made_runnable_time = thread->last_made_runnable_time;
1582 cur_thread_snap->tds_state = thread->state;
1583 cur_thread_snap->tds_sched_flags = thread->sched_flags;
1584 cur_thread_snap->tds_base_priority = thread->base_pri;
1585 cur_thread_snap->tds_sched_priority = thread->sched_pri;
1586 cur_thread_snap->tds_eqos = thread->effective_policy.thep_qos;
1587 cur_thread_snap->tds_rqos = thread->requested_policy.thrp_qos;
1588 cur_thread_snap->tds_rqos_override = MAX(thread->requested_policy.thrp_qos_override,
1589 thread->requested_policy.thrp_qos_workq_override);
1590 cur_thread_snap->tds_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
1591
1592 static_assert(sizeof(thread->effective_policy) == sizeof(uint64_t));
1593 static_assert(sizeof(thread->requested_policy) == sizeof(uint64_t));
1594 cur_thread_snap->tds_requested_policy = *(unaligned_u64 *) &thread->requested_policy;
1595 cur_thread_snap->tds_effective_policy = *(unaligned_u64 *) &thread->effective_policy;
1596
1597 return 0;
1598}
1599
1600/*
1601 * Why 12? 12 strikes a decent balance between allocating a large array on
1602 * the stack and having large kcdata item overheads for recording nonrunable
1603 * tasks.
1604 */
1605#define UNIQUEIDSPERFLUSH 12
1606
1607struct saved_uniqueids {
1608 uint64_t ids[UNIQUEIDSPERFLUSH];
1609 unsigned count;
1610};
1611
1612enum thread_classification {
1613 tc_full_snapshot, /* take a full snapshot */
1614 tc_delta_snapshot, /* take a delta snapshot */
1615};
1616
1617static enum thread_classification
1618classify_thread(thread_t thread, boolean_t * thread_on_core_p, uint32_t trace_flags)
1619{
1620 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1621
1622 processor_t last_processor = thread->last_processor;
1623
1624 boolean_t thread_on_core =
1625 (last_processor != PROCESSOR_NULL && last_processor->state == PROCESSOR_RUNNING && last_processor->active_thread == thread);
1626
1627 *thread_on_core_p = thread_on_core;
1628
1629 /* Capture the full thread snapshot if this is not a delta stackshot or if the thread has run subsequent to the
1630 * previous full stackshot */
1631 if (!collect_delta_stackshot || thread_on_core || (thread->last_run_time > stack_snapshot_delta_since_timestamp)) {
1632 return tc_full_snapshot;
1633 } else {
1634 return tc_delta_snapshot;
1635 }
1636}
1637
1638struct stackshot_context
1639{
1640 int pid;
1641 uint32_t trace_flags;
1642};
1643
1644static kern_return_t
1645kdp_stackshot_record_task(struct stackshot_context *ctx, task_t task)
1646{
1647 boolean_t active_kthreads_only_p = ((ctx->trace_flags & STACKSHOT_ACTIVE_KERNEL_THREADS_ONLY) != 0);
1648 boolean_t save_donating_pids_p = ((ctx->trace_flags & STACKSHOT_SAVE_IMP_DONATION_PIDS) != 0);
1649 boolean_t collect_delta_stackshot = ((ctx->trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1650 boolean_t save_owner_info = ((ctx->trace_flags & STACKSHOT_THREAD_WAITINFO) != 0);
1651
1652
1653 kern_return_t error = KERN_SUCCESS;
1654 mach_vm_address_t out_addr = 0;
1655 int saved_count = 0;
1656
1657 int task_pid = 0;
1658 uint64_t task_uniqueid = 0;
1659 int num_delta_thread_snapshots = 0;
1660 int num_nonrunnable_threads = 0;
1661 int num_waitinfo_threads = 0;
1662
1663 uint64_t task_start_abstime = 0;
1664 boolean_t task_delta_stackshot = FALSE;
1665 boolean_t have_map = FALSE, have_pmap = FALSE;
1666 boolean_t some_thread_ran = FALSE;
1667 unaligned_u64 *task_snap_ss_flags = NULL;
1668
1669 if ((task == NULL) || !ml_validate_nofault((vm_offset_t)task, sizeof(struct task))) {
1670 error = KERN_FAILURE;
1671 goto error_exit;
1672 }
1673
1674 have_map = (task->map != NULL) && (ml_validate_nofault((vm_offset_t)(task->map), sizeof(struct _vm_map)));
1675 have_pmap = have_map && (task->map->pmap != NULL) && (ml_validate_nofault((vm_offset_t)(task->map->pmap), sizeof(struct pmap)));
1676
1677 task_pid = pid_from_task(task);
1678 task_uniqueid = get_task_uniqueid(task);
1679
1680 if (!task->active || task_is_a_corpse(task)) {
1681 /*
1682 * Not interested in terminated tasks without threads, and
1683 * at the moment, stackshot can't handle a task without a name.
1684 */
1685 if (queue_empty(&task->threads) || task_pid == -1) {
1686 return KERN_SUCCESS;
1687 }
1688 }
1689
1690 if (collect_delta_stackshot) {
1691 proc_starttime_kdp(task->bsd_info, NULL, NULL, &task_start_abstime);
1692 }
1693
1694 /* Trace everything, unless a process was specified */
1695 if ((ctx->pid == -1) || (ctx->pid == task_pid)) {
1696
1697 /* add task snapshot marker */
1698 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1699 STACKSHOT_KCCONTAINER_TASK, task_uniqueid));
1700
1701 if (!collect_delta_stackshot || (task_start_abstime == 0) ||
1702 (task_start_abstime > stack_snapshot_delta_since_timestamp)) {
1703 kcd_exit_on_error(kcdata_record_task_snapshot(stackshot_kcdata_p, task, ctx->trace_flags, have_pmap, &task_snap_ss_flags));
1704 } else {
1705 task_delta_stackshot = TRUE;
1706 kcd_exit_on_error(kcdata_record_task_delta_snapshot(stackshot_kcdata_p, task, ctx->trace_flags, have_pmap, &task_snap_ss_flags));
1707 }
1708
1709 /* Iterate over task threads */
1710 thread_t thread = THREAD_NULL;
1711 queue_iterate(&task->threads, thread, thread_t, task_threads)
1712 {
1713 uint64_t thread_uniqueid;
1714
1715 if ((thread == NULL) || !ml_validate_nofault((vm_offset_t)thread, sizeof(struct thread))) {
1716 error = KERN_FAILURE;
1717 goto error_exit;
1718 }
1719
1720 if (active_kthreads_only_p && thread->kernel_stack == 0)
1721 continue;
1722
1723 thread_uniqueid = thread_tid(thread);
1724
1725 boolean_t thread_on_core;
1726 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, ctx->trace_flags);
1727
1728 switch (thread_classification) {
1729 case tc_full_snapshot:
1730 /* add thread marker */
1731 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN,
1732 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1733 kcd_exit_on_error(
1734 kcdata_record_thread_snapshot(stackshot_kcdata_p, thread, task, ctx->trace_flags, have_pmap, thread_on_core));
1735
1736 /* mark end of thread snapshot data */
1737 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END,
1738 STACKSHOT_KCCONTAINER_THREAD, thread_uniqueid));
1739
1740 some_thread_ran = TRUE;
1741 break;
1742
1743 case tc_delta_snapshot:
1744 num_delta_thread_snapshots++;
1745 break;
1746 }
1747
1748 /* We want to report owner information regardless of whether a thread
1749 * has changed since the last delta, whether it's a normal stackshot,
1750 * or whether it's nonrunnable */
1751 if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread))
1752 num_waitinfo_threads++;
1753 }
1754
1755 struct thread_delta_snapshot_v3 * delta_snapshots = NULL;
1756 int current_delta_snapshot_index = 0;
1757
1758 if (num_delta_thread_snapshots > 0) {
1759 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT,
1760 sizeof(struct thread_delta_snapshot_v3),
1761 num_delta_thread_snapshots, &out_addr));
1762 delta_snapshots = (struct thread_delta_snapshot_v3 *)out_addr;
1763 }
1764
1765 uint64_t * nonrunnable_tids = NULL;
1766
1767 if (num_nonrunnable_threads > 0) {
1768 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_NONRUNNABLE_TIDS,
1769 sizeof(uint64_t), num_nonrunnable_threads, &out_addr));
1770 nonrunnable_tids = (uint64_t *)out_addr;
1771 }
1772
1773 thread_waitinfo_t *thread_waitinfo = NULL;
1774 int current_waitinfo_index = 0;
1775
1776 if (num_waitinfo_threads > 0) {
1777 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_WAITINFO,
1778 sizeof(thread_waitinfo_t), num_waitinfo_threads, &out_addr));
1779 thread_waitinfo = (thread_waitinfo_t *)out_addr;
1780 }
1781
1782 if (num_delta_thread_snapshots > 0 || num_nonrunnable_threads > 0 || num_waitinfo_threads > 0) {
1783 queue_iterate(&task->threads, thread, thread_t, task_threads)
1784 {
1785 if (active_kthreads_only_p && thread->kernel_stack == 0)
1786 continue;
1787
1788 /* If we want owner info, we should capture it regardless of its classification */
1789 if (save_owner_info && stackshot_thread_has_valid_waitinfo(thread)) {
1790 stackshot_thread_wait_owner_info(
1791 thread,
1792 &thread_waitinfo[current_waitinfo_index++]);
1793 }
1794
1795 boolean_t thread_on_core;
1796 enum thread_classification thread_classification = classify_thread(thread, &thread_on_core, ctx->trace_flags);
1797
1798 switch (thread_classification) {
1799 case tc_full_snapshot:
1800 /* full thread snapshot captured above */
1801 continue;
1802
1803 case tc_delta_snapshot:
1804 kcd_exit_on_error(kcdata_record_thread_delta_snapshot(&delta_snapshots[current_delta_snapshot_index++],
1805 thread, thread_on_core));
1806 break;
1807 }
1808 }
1809
1810#if DEBUG || DEVELOPMENT
1811 if (current_delta_snapshot_index != num_delta_thread_snapshots) {
1812 panic("delta thread snapshot count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1813 num_delta_thread_snapshots, current_delta_snapshot_index);
1814 }
1815 if (current_waitinfo_index != num_waitinfo_threads) {
1816 panic("thread wait info count mismatch while capturing snapshots for task %p. expected %d, found %d", task,
1817 num_waitinfo_threads, current_waitinfo_index);
1818 }
1819#endif
1820 }
1821
1822#if IMPORTANCE_INHERITANCE
1823 if (save_donating_pids_p) {
1824 kcd_exit_on_error(
1825 ((((mach_vm_address_t)kcd_end_address(stackshot_kcdata_p) + (TASK_IMP_WALK_LIMIT * sizeof(int32_t))) <
1826 (mach_vm_address_t)kcd_max_address(stackshot_kcdata_p))
1827 ? KERN_SUCCESS
1828 : KERN_RESOURCE_SHORTAGE));
1829 saved_count = task_importance_list_pids(task, TASK_IMP_LIST_DONATING_PIDS,
1830 (void *)kcd_end_address(stackshot_kcdata_p), TASK_IMP_WALK_LIMIT);
1831 if (saved_count > 0)
1832 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_DONATING_PIDS,
1833 sizeof(int32_t), saved_count, &out_addr));
1834 }
1835#endif
1836
1837 if (!collect_delta_stackshot || (num_delta_thread_snapshots != task->thread_count) || !task_delta_stackshot) {
1838 /*
1839 * Collect shared cache info and UUID info in these scenarios
1840 * 1) a full stackshot
1841 * 2) a delta stackshot where the task started after the previous full stackshot OR
1842 * any thread from the task has run since the previous full stackshot
1843 */
1844
1845 kcd_exit_on_error(kcdata_record_shared_cache_info(stackshot_kcdata_p, task, task_snap_ss_flags));
1846 kcd_exit_on_error(kcdata_record_uuid_info(stackshot_kcdata_p, task, ctx->trace_flags, have_pmap, task_snap_ss_flags));
1847 }
1848 /* mark end of task snapshot data */
1849 kcd_exit_on_error(kcdata_add_container_marker(stackshot_kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK,
1850 task_uniqueid));
1851 }
1852
1853error_exit:
1854 return error;
1855}
1856
1857
1858static kern_return_t
1859kdp_stackshot_kcdata_format(int pid, uint32_t trace_flags, uint32_t * pBytesTraced)
1860{
1861 kern_return_t error = KERN_SUCCESS;
1862 mach_vm_address_t out_addr = 0;
1863 uint64_t abs_time = 0, abs_time_end = 0;
1864 uint64_t *abs_time_addr = NULL;
1865 uint64_t system_state_flags = 0;
1866 task_t task = TASK_NULL;
1867 mach_timebase_info_data_t timebase = {0, 0};
1868 uint32_t length_to_copy = 0, tmp32 = 0;
1869 abs_time = mach_absolute_time();
1870
1871 /* process the flags */
1872 boolean_t collect_delta_stackshot = ((trace_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) != 0);
1873 boolean_t use_fault_path = ((trace_flags & (STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_ENABLE_BT_FAULTING)) != 0);
1874 stack_enable_faulting = (trace_flags & (STACKSHOT_ENABLE_BT_FAULTING));
1875
1876#if CONFIG_EMBEDDED
1877 /* KEXTs can't be described by just a base address on embedded */
1878 trace_flags &= ~(STACKSHOT_SAVE_KEXT_LOADINFO);
1879#endif
1880
1881 struct stackshot_context ctx = {};
1882 ctx.trace_flags = trace_flags;
1883 ctx.pid = pid;
1884
1885 if (use_fault_path) {
1886 fault_stats.sfs_pages_faulted_in = 0;
1887 fault_stats.sfs_time_spent_faulting = 0;
1888 fault_stats.sfs_stopped_faulting = (uint8_t) FALSE;
1889 }
1890
1891 if (sizeof(void *) == 8)
1892 system_state_flags |= kKernel64_p;
1893
1894 if (stackshot_kcdata_p == NULL || pBytesTraced == NULL) {
1895 error = KERN_INVALID_ARGUMENT;
1896 goto error_exit;
1897 }
1898
1899 /* setup mach_absolute_time and timebase info -- copy out in some cases and needed to convert since_timestamp to seconds for proc start time */
1900 clock_timebase_info(&timebase);
1901
1902 /* begin saving data into the buffer */
1903 *pBytesTraced = 0;
1904 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_in_flags"));
1905 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, (uint32_t)pid, "stackshot_in_pid"));
1906 kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_kcdata_p, system_state_flags, "system_state_flags"));
1907
1908#if CONFIG_JETSAM
1909 tmp32 = memorystatus_get_pressure_status_kdp();
1910 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_LEVEL, sizeof(uint32_t), &out_addr));
1911 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1912#endif
1913
1914 if (!collect_delta_stackshot) {
1915 tmp32 = THREAD_POLICY_INTERNAL_STRUCT_VERSION;
1916 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_THREAD_POLICY_VERSION, sizeof(uint32_t), &out_addr));
1917 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1918
1919 tmp32 = PAGE_SIZE;
1920 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_KERN_PAGE_SIZE, sizeof(uint32_t), &out_addr));
1921 stackshot_memcpy((void *)out_addr, &tmp32, sizeof(tmp32));
1922
1923 /* save boot-args and osversion string */
1924 length_to_copy = MIN((uint32_t)(strlen(version) + 1), OSVERSIZE);
1925 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_OSVERSION, length_to_copy, &out_addr));
1926 stackshot_strlcpy((char*)out_addr, &version[0], length_to_copy);
1927
1928 length_to_copy = MIN((uint32_t)(strlen(PE_boot_args()) + 1), OSVERSIZE);
1929 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_BOOTARGS, length_to_copy, &out_addr));
1930 stackshot_strlcpy((char*)out_addr, PE_boot_args(), length_to_copy);
1931
1932 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_TIMEBASE, sizeof(timebase), &out_addr));
1933 stackshot_memcpy((void *)out_addr, &timebase, sizeof(timebase));
1934 } else {
1935 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP, sizeof(uint64_t), &out_addr));
1936 stackshot_memcpy((void*)out_addr, &stack_snapshot_delta_since_timestamp, sizeof(stack_snapshot_delta_since_timestamp));
1937 }
1938
1939 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_MACH_ABSOLUTE_TIME, sizeof(uint64_t), &out_addr));
1940 abs_time_addr = (uint64_t *)out_addr;
1941 stackshot_memcpy((void *)abs_time_addr, &abs_time, sizeof(uint64_t));
1942
1943 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &out_addr));
1944 stackshot_memcpy((void *)out_addr, &stackshot_microsecs, sizeof(uint64_t));
1945
1946 /* record system level shared cache load info (if available) */
1947 if (!collect_delta_stackshot && init_task_shared_region &&
1948 ml_validate_nofault((vm_offset_t)init_task_shared_region, sizeof(struct vm_shared_region))) {
1949 struct dyld_uuid_info_64_v2 *sys_shared_cache_info = NULL;
1950 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO,
1951 sizeof(struct dyld_uuid_info_64_v2), &out_addr));
1952 sys_shared_cache_info = (struct dyld_uuid_info_64_v2 *)out_addr;
1953
1954 stackshot_memcpy(sys_shared_cache_info->imageUUID, &init_task_shared_region->sr_uuid, sizeof(init_task_shared_region->sr_uuid));
1955 sys_shared_cache_info->imageLoadAddress = init_task_shared_region->sr_slide_info.slide;
1956 sys_shared_cache_info->imageSlidBaseAddress = init_task_shared_region->sr_slide_info.slide + init_task_shared_region->sr_base_address;
1957
1958 if (trace_flags & STACKSHOT_COLLECT_SHAREDCACHE_LAYOUT) {
1959 /*
1960 * Include a map of the system shared cache layout if it has been populated
1961 * (which is only when the system is using a custom shared cache).
1962 */
1963 if (init_task_shared_region->sr_images && ml_validate_nofault((vm_offset_t)init_task_shared_region->sr_images,
1964 (init_task_shared_region->sr_images_count * sizeof(struct dyld_uuid_info_64)))) {
1965 assert(init_task_shared_region->sr_images_count != 0);
1966 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_SYS_SHAREDCACHE_LAYOUT,
1967 sizeof(struct dyld_uuid_info_64),
1968 init_task_shared_region->sr_images_count, &out_addr));
1969 stackshot_memcpy((void*)out_addr, init_task_shared_region->sr_images,
1970 (init_task_shared_region->sr_images_count * sizeof(struct dyld_uuid_info_64)));
1971 }
1972 }
1973 }
1974
1975 /* Add requested information first */
1976 if (trace_flags & STACKSHOT_GET_GLOBAL_MEM_STATS) {
1977 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_GLOBAL_MEM_STATS, sizeof(struct mem_and_io_snapshot), &out_addr));
1978 kdp_mem_and_io_snapshot((struct mem_and_io_snapshot *)out_addr);
1979 }
1980
1981#if CONFIG_COALITIONS
1982 int num_coalitions = 0;
1983 struct jetsam_coalition_snapshot *coalitions = NULL;
1984 /* Iterate over coalitions */
1985 if (trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) {
1986 if (coalition_iterate_stackshot(stackshot_coalition_jetsam_count, &num_coalitions, COALITION_TYPE_JETSAM) != KERN_SUCCESS) {
1987 trace_flags &= ~(STACKSHOT_SAVE_JETSAM_COALITIONS);
1988 }
1989 }
1990 if (trace_flags & STACKSHOT_SAVE_JETSAM_COALITIONS) {
1991 if (num_coalitions > 0) {
1992 kcd_exit_on_error(kcdata_get_memory_addr_for_array(stackshot_kcdata_p, STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT, sizeof(struct jetsam_coalition_snapshot), num_coalitions, &out_addr));
1993 coalitions = (struct jetsam_coalition_snapshot*)out_addr;
1994 }
1995
1996 if (coalition_iterate_stackshot(stackshot_coalition_jetsam_snapshot, coalitions, COALITION_TYPE_JETSAM) != KERN_SUCCESS) {
1997 error = KERN_FAILURE;
1998 goto error_exit;
1999 }
2000
2001 }
2002#else
2003 trace_flags &= ~(STACKSHOT_SAVE_JETSAM_COALITIONS);
2004#endif /* CONFIG_COALITIONS */
2005
2006 trace_flags &= ~(STACKSHOT_THREAD_GROUP);
2007
2008
2009 /* Iterate over tasks */
2010 queue_iterate(&tasks, task, task_t, tasks)
2011 {
2012 error = kdp_stackshot_record_task(&ctx, task);
2013 if (error)
2014 goto error_exit;
2015 }
2016 /*
2017 * Iterate over the tasks in the terminated tasks list. We only inspect
2018 * tasks that have a valid bsd_info pointer where P_LPEXIT is NOT set.
2019 * We're only interested in tasks that have remaining threads (which
2020 * could be involved in a deadlock, etc), and the last thread that tears
2021 * itself down during exit sets P_LPEXIT during proc_exit().
2022 */
2023 queue_iterate(&terminated_tasks, task, task_t, tasks)
2024 {
2025 if (task->bsd_info && !proc_in_teardown(task->bsd_info)) {
2026 error = kdp_stackshot_record_task(&ctx, task);
2027 if (error)
2028 goto error_exit;
2029 }
2030 }
2031
2032 if (use_fault_path) {
2033 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS,
2034 sizeof(struct stackshot_fault_stats), &out_addr));
2035 stackshot_memcpy((void*)out_addr, &fault_stats, sizeof(struct stackshot_fault_stats));
2036 }
2037
2038 /* update timestamp of the stackshot */
2039 abs_time_end = mach_absolute_time();
2040#if DEVELOPMENT || DEBUG
2041 kcd_exit_on_error(kcdata_get_memory_addr(stackshot_kcdata_p, STACKSHOT_KCTYPE_STACKSHOT_DURATION,
2042 sizeof(struct stackshot_duration), &out_addr));
2043 struct stackshot_duration * stackshot_duration = (struct stackshot_duration *)out_addr;
2044 stackshot_duration->stackshot_duration = (abs_time_end - abs_time);
2045 stackshot_duration->stackshot_duration_outer = 0;
2046 stackshot_duration_outer = (unaligned_u64 *)&stackshot_duration->stackshot_duration_outer;
2047#endif
2048 stackshot_memcpy((void *)abs_time_addr, &abs_time_end, sizeof(uint64_t));
2049
2050 kcd_exit_on_error(kcdata_add_uint32_with_description(stackshot_kcdata_p, trace_flags, "stackshot_out_flags"));
2051
2052 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_kcdata_p));
2053
2054 /* === END of populating stackshot data === */
2055
2056 *pBytesTraced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_kcdata_p);
2057error_exit:
2058
2059#if INTERRUPT_MASKED_DEBUG
2060 if (!panic_stackshot) {
2061 /*
2062 * Try to catch instances where stackshot takes too long BEFORE returning from
2063 * the debugger
2064 */
2065 ml_check_interrupts_disabled_duration(current_thread());
2066 }
2067#endif
2068
2069 stack_enable_faulting = FALSE;
2070
2071 return error;
2072}
2073
2074static uint64_t
2075proc_was_throttled_from_task(task_t task)
2076{
2077 uint64_t was_throttled = 0;
2078
2079 if (task->bsd_info)
2080 was_throttled = proc_was_throttled(task->bsd_info);
2081
2082 return was_throttled;
2083}
2084
2085static uint64_t
2086proc_did_throttle_from_task(task_t task)
2087{
2088 uint64_t did_throttle = 0;
2089
2090 if (task->bsd_info)
2091 did_throttle = proc_did_throttle(task->bsd_info);
2092
2093 return did_throttle;
2094}
2095
2096static void
2097kdp_mem_and_io_snapshot(struct mem_and_io_snapshot *memio_snap)
2098{
2099 unsigned int pages_reclaimed;
2100 unsigned int pages_wanted;
2101 kern_return_t kErr;
2102
2103 processor_t processor;
2104 vm_statistics64_t stat;
2105 vm_statistics64_data_t host_vm_stat;
2106
2107 processor = processor_list;
2108 stat = &PROCESSOR_DATA(processor, vm_stat);
2109 host_vm_stat = *stat;
2110
2111 if (processor_count > 1) {
2112 /*
2113 * processor_list may be in the process of changing as we are
2114 * attempting a stackshot. Ordinarily it will be lock protected,
2115 * but it is not safe to lock in the context of the debugger.
2116 * Fortunately we never remove elements from the processor list,
2117 * and only add to to the end of the list, so we SHOULD be able
2118 * to walk it. If we ever want to truly tear down processors,
2119 * this will have to change.
2120 */
2121 while ((processor = processor->processor_list) != NULL) {
2122 stat = &PROCESSOR_DATA(processor, vm_stat);
2123 host_vm_stat.compressions += stat->compressions;
2124 host_vm_stat.decompressions += stat->decompressions;
2125 }
2126 }
2127
2128 memio_snap->snapshot_magic = STACKSHOT_MEM_AND_IO_SNAPSHOT_MAGIC;
2129 memio_snap->free_pages = vm_page_free_count;
2130 memio_snap->active_pages = vm_page_active_count;
2131 memio_snap->inactive_pages = vm_page_inactive_count;
2132 memio_snap->purgeable_pages = vm_page_purgeable_count;
2133 memio_snap->wired_pages = vm_page_wire_count;
2134 memio_snap->speculative_pages = vm_page_speculative_count;
2135 memio_snap->throttled_pages = vm_page_throttled_count;
2136 memio_snap->busy_buffer_count = count_busy_buffers();
2137 memio_snap->filebacked_pages = vm_page_pageable_external_count;
2138 memio_snap->compressions = (uint32_t)host_vm_stat.compressions;
2139 memio_snap->decompressions = (uint32_t)host_vm_stat.decompressions;
2140 memio_snap->compressor_size = VM_PAGE_COMPRESSOR_COUNT;
2141 kErr = mach_vm_pressure_monitor(FALSE, VM_PRESSURE_TIME_WINDOW, &pages_reclaimed, &pages_wanted);
2142
2143 if ( ! kErr ) {
2144 memio_snap->pages_wanted = (uint32_t)pages_wanted;
2145 memio_snap->pages_reclaimed = (uint32_t)pages_reclaimed;
2146 memio_snap->pages_wanted_reclaimed_valid = 1;
2147 } else {
2148 memio_snap->pages_wanted = 0;
2149 memio_snap->pages_reclaimed = 0;
2150 memio_snap->pages_wanted_reclaimed_valid = 0;
2151 }
2152}
2153
2154void
2155stackshot_memcpy(void *dst, const void *src, size_t len)
2156{
2157#if CONFIG_EMBEDDED
2158 if (panic_stackshot) {
2159 uint8_t *dest_bytes = (uint8_t *)dst;
2160 const uint8_t *src_bytes = (const uint8_t *)src;
2161 for (size_t i = 0; i < len; i++) {
2162 dest_bytes[i] = src_bytes[i];
2163 }
2164 } else
2165#endif
2166 memcpy(dst, src, len);
2167}
2168
2169size_t
2170stackshot_strlcpy(char *dst, const char *src, size_t maxlen)
2171{
2172 const size_t srclen = strlen(src);
2173
2174 if (srclen < maxlen) {
2175 stackshot_memcpy(dst, src, srclen+1);
2176 } else if (maxlen != 0) {
2177 stackshot_memcpy(dst, src, maxlen-1);
2178 dst[maxlen-1] = '\0';
2179 }
2180
2181 return srclen;
2182}
2183
2184
2185/*
2186 * Returns the physical address of the specified map:target address,
2187 * using the kdp fault path if requested and the page is not resident.
2188 */
2189vm_offset_t
2190kdp_find_phys(vm_map_t map, vm_offset_t target_addr, boolean_t try_fault, uint32_t *kdp_fault_results)
2191{
2192 vm_offset_t cur_phys_addr;
2193 unsigned cur_wimg_bits;
2194 uint64_t fault_start_time = 0;
2195
2196 if (map == VM_MAP_NULL) {
2197 return 0;
2198 }
2199
2200 cur_phys_addr = kdp_vtophys(map->pmap, target_addr);
2201 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
2202 if (!try_fault || fault_stats.sfs_stopped_faulting) {
2203 if (kdp_fault_results)
2204 *kdp_fault_results |= KDP_FAULT_RESULT_PAGED_OUT;
2205
2206 return 0;
2207 }
2208
2209 /*
2210 * The pmap doesn't have a valid page so we start at the top level
2211 * vm map and try a lightweight fault. Update fault path usage stats.
2212 */
2213 fault_start_time = mach_absolute_time();
2214 cur_phys_addr = kdp_lightweight_fault(map, (target_addr & ~PAGE_MASK));
2215 fault_stats.sfs_time_spent_faulting += (mach_absolute_time() - fault_start_time);
2216
2217 if ((fault_stats.sfs_time_spent_faulting >= fault_stats.sfs_system_max_fault_time) && !panic_stackshot) {
2218 fault_stats.sfs_stopped_faulting = (uint8_t) TRUE;
2219 }
2220
2221 cur_phys_addr += (target_addr & PAGE_MASK);
2222
2223 if (!pmap_valid_page((ppnum_t) atop(cur_phys_addr))) {
2224 if (kdp_fault_results)
2225 *kdp_fault_results |= (KDP_FAULT_RESULT_TRIED_FAULT | KDP_FAULT_RESULT_PAGED_OUT);
2226
2227 return 0;
2228 }
2229
2230 if (kdp_fault_results)
2231 *kdp_fault_results |= KDP_FAULT_RESULT_FAULTED_IN;
2232
2233 fault_stats.sfs_pages_faulted_in++;
2234 } else {
2235 /*
2236 * This check is done in kdp_lightweight_fault for the fault path.
2237 */
2238 cur_wimg_bits = pmap_cache_attributes((ppnum_t) atop(cur_phys_addr));
2239
2240 if ((cur_wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
2241 return 0;
2242 }
2243 }
2244
2245 return cur_phys_addr;
2246}
2247
2248boolean_t
2249kdp_copyin_word(
2250 task_t task, uint64_t addr, uint64_t *result, boolean_t try_fault, uint32_t *kdp_fault_results)
2251{
2252 if (task_has_64Bit_data(task)) {
2253 return kdp_copyin(task->map, addr, result, sizeof(uint64_t), try_fault, kdp_fault_results);
2254 } else {
2255 uint32_t buf;
2256 boolean_t r = kdp_copyin(task->map, addr, &buf, sizeof(uint32_t), try_fault, kdp_fault_results);
2257 *result = buf;
2258 return r;
2259 }
2260}
2261
2262boolean_t
2263kdp_copyin(vm_map_t map, uint64_t uaddr, void *dest, size_t size, boolean_t try_fault, uint32_t *kdp_fault_results)
2264{
2265 size_t rem = size;
2266 char *kvaddr = dest;
2267
2268#if CONFIG_EMBEDDED
2269 /* Identify if destination buffer is in panic storage area */
2270 if (panic_stackshot && ((vm_offset_t)dest >= gPanicBase) && ((vm_offset_t)dest < (gPanicBase + gPanicSize))) {
2271 if (((vm_offset_t)dest + size) > (gPanicBase + gPanicSize)) {
2272 return FALSE;
2273 }
2274 }
2275#endif
2276
2277 while (rem) {
2278 uint64_t phys_src = kdp_find_phys(map, uaddr, try_fault, kdp_fault_results);
2279 uint64_t phys_dest = kvtophys((vm_offset_t)kvaddr);
2280 uint64_t src_rem = PAGE_SIZE - (phys_src & PAGE_MASK);
2281 uint64_t dst_rem = PAGE_SIZE - (phys_dest & PAGE_MASK);
2282 size_t cur_size = (uint32_t) MIN(src_rem, dst_rem);
2283 cur_size = MIN(cur_size, rem);
2284
2285 if (phys_src && phys_dest) {
2286#if CONFIG_EMBEDDED
2287 /*
2288 * On embedded the panic buffer is mapped as device memory and doesn't allow
2289 * unaligned accesses. To prevent these, we copy over bytes individually here.
2290 */
2291 if (panic_stackshot)
2292 stackshot_memcpy(kvaddr, (const void *)phystokv(phys_src), cur_size);
2293 else
2294#endif /* CONFIG_EMBEDDED */
2295 bcopy_phys(phys_src, phys_dest, cur_size);
2296 } else {
2297 break;
2298 }
2299
2300 uaddr += cur_size;
2301 kvaddr += cur_size;
2302 rem -= cur_size;
2303 }
2304
2305 return (rem == 0);
2306}
2307
2308kern_return_t
2309do_stackshot(void *context)
2310{
2311#pragma unused(context)
2312 kdp_snapshot++;
2313
2314 stack_snapshot_ret = kdp_stackshot_kcdata_format(stack_snapshot_pid,
2315 stack_snapshot_flags,
2316 &stack_snapshot_bytes_traced);
2317
2318 kdp_snapshot--;
2319 return stack_snapshot_ret;
2320}
2321
2322/*
2323 * A fantastical routine that tries to be fast about returning
2324 * translations. Caches the last page we found a translation
2325 * for, so that we can be quick about multiple queries to the
2326 * same page. It turns out this is exactly the workflow
2327 * machine_trace_thread and its relatives tend to throw at us.
2328 *
2329 * Please zero the nasty global this uses after a bulk lookup;
2330 * this isn't safe across a switch of the map or changes
2331 * to a pmap.
2332 *
2333 * This also means that if zero is a valid KVA, we are
2334 * screwed. Sucks to be us. Fortunately, this should never
2335 * happen.
2336 */
2337vm_offset_t
2338machine_trace_thread_get_kva(vm_offset_t cur_target_addr, vm_map_t map, uint32_t *thread_trace_flags)
2339{
2340 vm_offset_t cur_target_page;
2341 vm_offset_t cur_phys_addr;
2342 vm_offset_t kern_virt_target_addr;
2343 uint32_t kdp_fault_results = 0;
2344
2345 cur_target_page = atop(cur_target_addr);
2346
2347 if ((cur_target_page != prev_target_page) || validate_next_addr) {
2348
2349 /*
2350 * Alright; it wasn't our previous page. So
2351 * we must validate that there is a page
2352 * table entry for this address under the
2353 * current pmap, and that it has default
2354 * cache attributes (otherwise it may not be
2355 * safe to access it).
2356 */
2357 cur_phys_addr = kdp_find_phys(map, cur_target_addr, stack_enable_faulting, &kdp_fault_results);
2358 if (thread_trace_flags) {
2359 if (kdp_fault_results & KDP_FAULT_RESULT_PAGED_OUT) {
2360 *thread_trace_flags |= kThreadTruncatedBT;
2361 }
2362
2363 if (kdp_fault_results & KDP_FAULT_RESULT_TRIED_FAULT) {
2364 *thread_trace_flags |= kThreadTriedFaultBT;
2365 }
2366
2367 if (kdp_fault_results & KDP_FAULT_RESULT_FAULTED_IN) {
2368 *thread_trace_flags |= kThreadFaultedBT;
2369 }
2370 }
2371
2372 if (cur_phys_addr == 0) {
2373 return 0;
2374 }
2375#if __x86_64__
2376 kern_virt_target_addr = (vm_offset_t) PHYSMAP_PTOV(cur_phys_addr);
2377#elif __arm__ || __arm64__
2378 kern_virt_target_addr = phystokv(cur_phys_addr);
2379#else
2380#error Oh come on... we should really unify the physical -> kernel virtual interface
2381#endif
2382 prev_target_page = cur_target_page;
2383 prev_target_kva = (kern_virt_target_addr & ~PAGE_MASK);
2384 validate_next_addr = FALSE;
2385 } else {
2386 /* We found a translation, so stash this page */
2387 kern_virt_target_addr = prev_target_kva + (cur_target_addr & PAGE_MASK);
2388 }
2389
2390#if KASAN
2391 kasan_notify_address(kern_virt_target_addr, sizeof(uint64_t));
2392#endif
2393 return kern_virt_target_addr;
2394}
2395
2396void
2397machine_trace_thread_clear_validation_cache(void)
2398{
2399 validate_next_addr = TRUE;
2400}
2401
2402boolean_t
2403stackshot_thread_is_idle_worker_unsafe(thread_t thread)
2404{
2405 /* When the pthread kext puts a worker thread to sleep, it will
2406 * set kThreadWaitParkedWorkQueue in the block_hint of the thread
2407 * struct. See parkit() in kern/kern_support.c in libpthread.
2408 */
2409 return (thread->state & TH_WAIT) &&
2410 (thread->block_hint == kThreadWaitParkedWorkQueue);
2411}
2412
2413#if CONFIG_COALITIONS
2414static void
2415stackshot_coalition_jetsam_count(void *arg, int i, coalition_t coal)
2416{
2417#pragma unused(i, coal)
2418 unsigned int *coalition_count = (unsigned int*)arg;
2419 (*coalition_count)++;
2420}
2421
2422static void
2423stackshot_coalition_jetsam_snapshot(void *arg, int i, coalition_t coal)
2424{
2425 if (coalition_type(coal) != COALITION_TYPE_JETSAM)
2426 return;
2427
2428 struct jetsam_coalition_snapshot *coalitions = (struct jetsam_coalition_snapshot*)arg;
2429 struct jetsam_coalition_snapshot *jcs = &coalitions[i];
2430 task_t leader = TASK_NULL;
2431 jcs->jcs_id = coalition_id(coal);
2432 jcs->jcs_flags = 0;
2433
2434 if (coalition_term_requested(coal))
2435 jcs->jcs_flags |= kCoalitionTermRequested;
2436 if (coalition_is_terminated(coal))
2437 jcs->jcs_flags |= kCoalitionTerminated;
2438 if (coalition_is_reaped(coal))
2439 jcs->jcs_flags |= kCoalitionReaped;
2440 if (coalition_is_privileged(coal))
2441 jcs->jcs_flags |= kCoalitionPrivileged;
2442
2443
2444 leader = kdp_coalition_get_leader(coal);
2445 if (leader)
2446 jcs->jcs_leader_task_uniqueid = get_task_uniqueid(leader);
2447 else
2448 jcs->jcs_leader_task_uniqueid = 0;
2449}
2450#endif /* CONFIG_COALITIONS */
2451
2452
2453/* Determine if a thread has waitinfo that stackshot can provide */
2454static int
2455stackshot_thread_has_valid_waitinfo(thread_t thread)
2456{
2457 if (!(thread->state & TH_WAIT))
2458 return 0;
2459
2460 switch (thread->block_hint) {
2461 // If set to None or is a parked work queue, ignore it
2462 case kThreadWaitParkedWorkQueue:
2463 case kThreadWaitNone:
2464 return 0;
2465 // There is a short window where the pthread kext removes a thread
2466 // from its ksyn wait queue before waking the thread up
2467 case kThreadWaitPThreadMutex:
2468 case kThreadWaitPThreadRWLockRead:
2469 case kThreadWaitPThreadRWLockWrite:
2470 case kThreadWaitPThreadCondVar:
2471 return (kdp_pthread_get_thread_kwq(thread) != NULL);
2472 // All other cases are valid block hints if in a wait state
2473 default:
2474 return 1;
2475 }
2476}
2477
2478static void
2479stackshot_thread_wait_owner_info(thread_t thread, thread_waitinfo_t *waitinfo)
2480{
2481 waitinfo->waiter = thread_tid(thread);
2482 waitinfo->wait_type = thread->block_hint;
2483 switch (waitinfo->wait_type) {
2484 case kThreadWaitKernelMutex:
2485 kdp_lck_mtx_find_owner(thread->waitq, thread->wait_event, waitinfo);
2486 break;
2487 case kThreadWaitPortReceive:
2488 kdp_mqueue_recv_find_owner(thread->waitq, thread->wait_event, waitinfo);
2489 break;
2490 case kThreadWaitPortSend:
2491 kdp_mqueue_send_find_owner(thread->waitq, thread->wait_event, waitinfo);
2492 break;
2493 case kThreadWaitSemaphore:
2494 kdp_sema_find_owner(thread->waitq, thread->wait_event, waitinfo);
2495 break;
2496 case kThreadWaitUserLock:
2497 kdp_ulock_find_owner(thread->waitq, thread->wait_event, waitinfo);
2498 break;
2499 case kThreadWaitKernelRWLockRead:
2500 case kThreadWaitKernelRWLockWrite:
2501 case kThreadWaitKernelRWLockUpgrade:
2502 kdp_rwlck_find_owner(thread->waitq, thread->wait_event, waitinfo);
2503 break;
2504 case kThreadWaitPThreadMutex:
2505 case kThreadWaitPThreadRWLockRead:
2506 case kThreadWaitPThreadRWLockWrite:
2507 case kThreadWaitPThreadCondVar:
2508 kdp_pthread_find_owner(thread, waitinfo);
2509 break;
2510 case kThreadWaitWorkloopSyncWait:
2511 kdp_workloop_sync_wait_find_owner(thread, thread->wait_event, waitinfo);
2512 break;
2513 case kThreadWaitOnProcess:
2514 kdp_wait4_find_process(thread, thread->wait_event, waitinfo);
2515 break;
2516 default:
2517 waitinfo->owner = 0;
2518 waitinfo->context = 0;
2519 break;
2520 }
2521}
2522
2523