1/*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/* Sample thread data */
30
31#include <kern/debug.h> /* panic */
32#include <kern/thread.h> /* thread_* */
33#include <kern/timer.h> /* timer_data_t */
34#include <kern/policy_internal.h> /* TASK_POLICY_* */
35#include <mach/mach_types.h>
36
37#include <kperf/kperf.h>
38#include <kperf/buffer.h>
39#include <kperf/context.h>
40#include <kperf/thread_samplers.h>
41#include <kperf/ast.h>
42
43#if MONOTONIC
44#include <kern/monotonic.h>
45#include <machine/monotonic.h>
46#endif /* MONOTONIC */
47
48extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
49
50/*
51 * XXX Deprecated, use thread scheduling sampler instead.
52 *
53 * Taken from AppleProfileGetRunModeOfThread and CHUD. Still here for
54 * backwards compatibility.
55 */
56
57#define KPERF_TI_RUNNING (1U << 0)
58#define KPERF_TI_RUNNABLE (1U << 1)
59#define KPERF_TI_WAIT (1U << 2)
60#define KPERF_TI_UNINT (1U << 3)
61#define KPERF_TI_SUSP (1U << 4)
62#define KPERF_TI_TERMINATE (1U << 5)
63#define KPERF_TI_IDLE (1U << 6)
64
65static uint32_t
66kperf_thread_info_runmode_legacy(thread_t thread)
67{
68 uint32_t kperf_state = 0;
69 int sched_state = thread->state;
70 processor_t last_processor = thread->last_processor;
71
72 if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
73 kperf_state |= KPERF_TI_RUNNING;
74 }
75 if (sched_state & TH_RUN) {
76 kperf_state |= KPERF_TI_RUNNABLE;
77 }
78 if (sched_state & TH_WAIT) {
79 kperf_state |= KPERF_TI_WAIT;
80 }
81 if (sched_state & TH_UNINT) {
82 kperf_state |= KPERF_TI_UNINT;
83 }
84 if (sched_state & TH_SUSP) {
85 kperf_state |= KPERF_TI_SUSP;
86 }
87 if (sched_state & TH_TERMINATE) {
88 kperf_state |= KPERF_TI_TERMINATE;
89 }
90 if (sched_state & TH_IDLE) {
91 kperf_state |= KPERF_TI_IDLE;
92 }
93
94#if !CONFIG_EMBEDDED
95 /* on desktop, if state is blank, leave not idle set */
96 if (kperf_state == 0) {
97 return (TH_IDLE << 16);
98 }
99#endif /* !CONFIG_EMBEDDED */
100
101 /* high two bytes are inverted mask, low two bytes are normal */
102 return (((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff));
103}
104
105void
106kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context)
107{
108 thread_t cur_thread = context->cur_thread;
109
110 BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));
111
112 ti->kpthi_pid = context->cur_pid;
113 ti->kpthi_tid = thread_tid(cur_thread);
114 ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
115 ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);
116
117 BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END);
118}
119
120void
121kperf_thread_info_log(struct kperf_thread_info *ti)
122{
123 BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
124 ti->kpthi_dq_addr, ti->kpthi_runmode);
125}
126
127/*
128 * Scheduling information reports inputs and outputs of the scheduler state for
129 * a thread.
130 */
131
132void
133kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
134 struct kperf_context *context)
135{
136 assert(thsc != NULL);
137 assert(context != NULL);
138
139 thread_t thread = context->cur_thread;
140
141 BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
142
143 thsc->kpthsc_user_time = timer_grab(&thread->user_timer);
144 uint64_t system_time = timer_grab(&thread->system_timer);
145
146 if (thread->precise_user_kernel_time) {
147 thsc->kpthsc_system_time = system_time;
148 } else {
149 thsc->kpthsc_user_time += system_time;
150 thsc->kpthsc_system_time = 0;
151 }
152
153 thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer);
154 thsc->kpthsc_state = thread->state;
155 thsc->kpthsc_base_priority = thread->base_pri;
156 thsc->kpthsc_sched_priority = thread->sched_pri;
157 thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
158 thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
159 thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override,
160 thread->requested_policy.thrp_qos_workq_override);
161 thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
162 thsc->kpthsc_requested_qos_ipc_override = thread->requested_policy.thrp_qos_ipc_override;
163 thsc->kpthsc_requested_qos_sync_ipc_override = thread->requested_policy.thrp_qos_sync_ipc_override;
164 thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;
165
166 BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END);
167}
168
169
170void
171kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
172{
173 assert(thsc != NULL);
174#if defined(__LP64__)
175 BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
176 thsc->kpthsc_system_time,
177 (((uint64_t)thsc->kpthsc_base_priority) << 48)
178 | ((uint64_t)thsc->kpthsc_sched_priority << 32)
179 | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
180 | (thsc->kpthsc_effective_qos << 6)
181 | (thsc->kpthsc_requested_qos << 3)
182 | thsc->kpthsc_requested_qos_override,
183 ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
184 | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
185 | ((uint64_t)thsc->kpthsc_requested_qos_ipc_override << 55)
186 | ((uint64_t)thsc->kpthsc_requested_qos_sync_ipc_override << 52)
187 );
188 BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time);
189#else
190 BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
191 LOWER_32(thsc->kpthsc_user_time),
192 UPPER_32(thsc->kpthsc_system_time),
193 LOWER_32(thsc->kpthsc_system_time)
194 );
195 BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
196 | thsc->kpthsc_sched_priority,
197 ((thsc->kpthsc_state & 0xff) << 24)
198 | (thsc->kpthsc_effective_qos << 6)
199 | (thsc->kpthsc_requested_qos << 3)
200 | thsc->kpthsc_requested_qos_override,
201 ((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
202 | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
203 | ((uint32_t)thsc->kpthsc_requested_qos_ipc_override << 23)
204 | ((uint32_t)thsc->kpthsc_requested_qos_sync_ipc_override << 20)
205 );
206 BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time),
207 LOWER_32(thsc->kpthsc_runnable_time));
208#endif /* defined(__LP64__) */
209}
210
211/*
212 * Snapshot information maintains parity with stackshot information for other,
213 * miscellaneous information about threads.
214 */
215
216#define KPERF_THREAD_SNAPSHOT_DARWIN_BG (1U << 0);
217#define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
218#define KPERF_THREAD_SNAPSHOT_GFI (1U << 2);
219#define KPERF_THREAD_SNAPSHOT_IDLE_WQ (1U << 3);
220/* max is 1U << 7 */
221
222void
223kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
224 struct kperf_context *context)
225{
226 assert(thsn != NULL);
227 assert(context != NULL);
228
229 thread_t thread = context->cur_thread;
230
231 BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
232
233 thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;
234
235 thsn->kpthsn_flags = 0;
236 if (thread->effective_policy.thep_darwinbg) {
237 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
238 }
239 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
240 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
241 }
242 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
243 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI
244 }
245 if (stackshot_thread_is_idle_worker_unsafe(thread)) {
246 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
247 }
248
249 thsn->kpthsn_suspend_count = thread->suspend_count;
250 thsn->kpthsn_io_tier = proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
251
252 BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END);
253}
254
255void
256kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
257{
258 assert(thsn != NULL);
259#if defined(__LP64__)
260 BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
261 | (thsn->kpthsn_io_tier << 24),
262 thsn->kpthsn_last_made_runnable_time);
263#else
264 BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
265 | (thsn->kpthsn_io_tier << 24),
266 UPPER_32(thsn->kpthsn_last_made_runnable_time),
267 LOWER_32(thsn->kpthsn_last_made_runnable_time));
268#endif /* defined(__LP64__) */
269}
270
271/*
272 * Dispatch information only contains the dispatch queue serial number from
273 * libdispatch.
274 *
275 * It's a separate sampler because queue data must be copied in from user space.
276 */
277
278void
279kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
280 struct kperf_context *context)
281{
282 assert(thdi != NULL);
283 assert(context != NULL);
284
285 thread_t thread = context->cur_thread;
286
287 BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
288
289 task_t task = thread->task;
290 boolean_t task_64 = task_has_64Bit_addr(task);
291 size_t user_addr_size = task_64 ? 8 : 4;
292
293 assert(thread->task != kernel_task);
294 uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
295 if (user_dq_key_addr == 0) {
296 goto error;
297 }
298
299 uint64_t user_dq_addr;
300 if ((copyin((user_addr_t)user_dq_key_addr,
301 (char *)&user_dq_addr,
302 user_addr_size) != 0) ||
303 (user_dq_addr == 0))
304 {
305 goto error;
306 }
307
308 uint64_t user_dq_serialno_addr =
309 user_dq_addr + get_task_dispatchqueue_serialno_offset(task);
310
311 if (copyin((user_addr_t)user_dq_serialno_addr,
312 (char *)&(thdi->kpthdi_dq_serialno),
313 user_addr_size) == 0)
314 {
315 goto out;
316 }
317
318error:
319 thdi->kpthdi_dq_serialno = 0;
320
321out:
322 BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END);
323}
324
325int
326kperf_thread_dispatch_pend(struct kperf_context *context)
327{
328 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH);
329}
330
331void
332kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
333{
334 assert(thdi != NULL);
335#if defined(__LP64__)
336 BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
337#else
338 BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
339 LOWER_32(thdi->kpthdi_dq_serialno));
340#endif /* defined(__LP64__) */
341}
342
343/*
344 * A bit different from other samplers -- since logging disables interrupts,
345 * it's a fine place to sample the thread counters.
346 */
347void
348kperf_thread_inscyc_log(struct kperf_context *context)
349{
350#if MONOTONIC
351 thread_t cur_thread = current_thread();
352
353 if (context->cur_thread != cur_thread) {
354 /* can't safely access another thread's counters */
355 return;
356 }
357
358 uint64_t counts[MT_CORE_NFIXED];
359
360 int ret = mt_fixed_thread_counts(cur_thread, counts);
361 if (ret) {
362 return;
363 }
364
365#if defined(__LP64__)
366 BUF_DATA(PERF_TI_INSCYCDATA, counts[MT_CORE_INSTRS], counts[MT_CORE_CYCLES]);
367#else /* defined(__LP64__) */
368 /* 32-bit platforms don't count instructions */
369 BUF_DATA(PERF_TI_INSCYCDATA_32, 0, 0, UPPER_32(counts[MT_CORE_CYCLES]),
370 LOWER_32(counts[MT_CORE_CYCLES]));
371#endif /* !defined(__LP64__) */
372
373#else
374#pragma unused(context)
375#endif /* MONOTONIC */
376
377}
378