1/*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58
59/*
60 * processor.h: Processor and processor-related definitions.
61 */
62
63#ifndef _KERN_PROCESSOR_H_
64#define _KERN_PROCESSOR_H_
65
66#include <mach/boolean.h>
67#include <mach/kern_return.h>
68#include <kern/kern_types.h>
69
70#include <sys/cdefs.h>
71
72#ifdef MACH_KERNEL_PRIVATE
73
74#include <mach/mach_types.h>
75#include <kern/ast.h>
76#include <kern/cpu_number.h>
77#include <kern/smp.h>
78#include <kern/simple_lock.h>
79#include <kern/locks.h>
80#include <kern/queue.h>
81#include <kern/sched.h>
82#include <mach/sfi_class.h>
83#include <kern/processor_data.h>
84#include <kern/cpu_quiesce.h>
85
86/*
87 * Processor state is accessed by locking the scheduling lock
88 * for the assigned processor set.
89 *
90 * -------------------- SHUTDOWN
91 * / ^ ^
92 * _/ | \
93 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
94 * \_________________^ ^ ^______/ /
95 * \__________________/
96 *
97 * Most of these state transitions are externally driven as a
98 * a directive (for instance telling an IDLE processor to start
99 * coming out of the idle state to run a thread). However these
100 * are typically paired with a handshake by the processor itself
101 * to indicate that it has completed a transition of indeterminate
102 * length (for example, the DISPATCHING->RUNNING or START->RUNNING
103 * transitions must occur on the processor itself).
104 *
105 * The boot processor has some special cases, and skips the START state,
106 * since it has already bootstrapped and is ready to context switch threads.
107 *
108 * When a processor is in DISPATCHING or RUNNING state, the current_pri,
109 * current_thmode, and deadline fields should be set, so that other
110 * processors can evaluate if it is an appropriate candidate for preemption.
111 */
112#if defined(CONFIG_SCHED_DEFERRED_AST)
113/*
114 * -------------------- SHUTDOWN
115 * / ^ ^
116 * _/ | \
117 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
118 * \_________________^ ^ ^______/ ^_____ / /
119 * \__________________/
120 *
121 * A DISPATCHING processor may be put back into IDLE, if another
122 * processor determines that the target processor will have nothing to do
123 * upon reaching the RUNNING state. This is racy, but if the target
124 * responds and becomes RUNNING, it will not break the processor state
125 * machine.
126 *
127 * This change allows us to cancel an outstanding signal/AST on a processor
128 * (if such an operation is supported through hardware or software), and
129 * push the processor back into the IDLE state as a power optimization.
130 */
131#endif
132
133#define PROCESSOR_OFF_LINE 0 /* Not available */
134#define PROCESSOR_SHUTDOWN 1 /* Going off-line */
135#define PROCESSOR_START 2 /* Being started */
136/* 3 Formerly Inactive (unavailable) */
137#define PROCESSOR_IDLE 4 /* Idle (available) */
138#define PROCESSOR_DISPATCHING 5 /* Dispatching (idle -> active) */
139#define PROCESSOR_RUNNING 6 /* Normal execution */
140#define PROCESSOR_STATE_LEN (PROCESSOR_RUNNING+1)
141
142typedef enum {
143 PSET_SMP,
144} pset_cluster_type_t;
145
146typedef bitmap_t cpumap_t;
147
148struct processor_set {
149 int online_processor_count;
150 int load_average;
151
152 int cpu_set_low, cpu_set_hi;
153 int cpu_set_count;
154 int last_chosen;
155 cpumap_t cpu_bitmask;
156 cpumap_t recommended_bitmask;
157 cpumap_t cpu_state_map[PROCESSOR_STATE_LEN];
158 cpumap_t primary_map;
159
160#if __SMP__
161 decl_simple_lock_data(,sched_lock) /* lock for above */
162#endif
163
164#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ)
165 struct run_queue pset_runq; /* runq for this processor set */
166#endif
167 struct rt_queue rt_runq; /* realtime runq for this processor set */
168
169#if defined(CONFIG_SCHED_TRADITIONAL)
170 int pset_runq_bound_count;
171 /* # of threads in runq bound to any processor in pset */
172#endif
173
174 /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */
175 cpumap_t pending_AST_cpu_mask;
176#if defined(CONFIG_SCHED_DEFERRED_AST)
177 /*
178 * A separate mask, for ASTs that we may be able to cancel. This is dependent on
179 * some level of support for requesting an AST on a processor, and then quashing
180 * that request later.
181 *
182 * The purpose of this field (and the associated codepaths) is to infer when we
183 * no longer need a processor that is DISPATCHING to come up, and to prevent it
184 * from coming out of IDLE if possible. This should serve to decrease the number
185 * of spurious ASTs in the system, and let processors spend longer periods in
186 * IDLE.
187 */
188 cpumap_t pending_deferred_AST_cpu_mask;
189#endif
190 cpumap_t pending_spill_cpu_mask;
191
192 struct ipc_port * pset_self; /* port for operations */
193 struct ipc_port * pset_name_self; /* port for information */
194
195 processor_set_t pset_list; /* chain of associated psets */
196 pset_node_t node;
197 uint32_t pset_cluster_id;
198 pset_cluster_type_t pset_cluster_type;
199};
200
201extern struct processor_set pset0;
202
203struct pset_node {
204 processor_set_t psets; /* list of associated psets */
205
206 pset_node_t nodes; /* list of associated subnodes */
207 pset_node_t node_list; /* chain of associated nodes */
208
209 pset_node_t parent;
210};
211
212extern struct pset_node pset_node0;
213
214extern queue_head_t tasks, terminated_tasks, threads, corpse_tasks; /* Terminated tasks are ONLY for stackshot */
215extern int tasks_count, terminated_tasks_count, threads_count;
216decl_lck_mtx_data(extern,tasks_threads_lock)
217decl_lck_mtx_data(extern,tasks_corpse_lock)
218
219struct processor {
220 int state; /* See above */
221 bool is_SMT;
222 bool is_recommended;
223 struct thread *active_thread; /* thread running on processor */
224 struct thread *next_thread; /* next thread when dispatched */
225 struct thread *idle_thread; /* this processor's idle thread. */
226
227 processor_set_t processor_set; /* assigned set */
228
229 int current_pri; /* priority of current thread */
230 sfi_class_id_t current_sfi_class; /* SFI class of current thread */
231 perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */
232 int starting_pri; /* priority of current thread as it was when scheduled */
233 pset_cluster_type_t current_recommended_pset_type; /* Cluster type recommended for current thread */
234 int cpu_id; /* platform numeric id */
235 cpu_quiescent_state_t cpu_quiesce_state;
236 uint64_t cpu_quiesce_last_checkin;
237
238 timer_call_data_t quantum_timer; /* timer for quantum expiration */
239 uint64_t quantum_end; /* time when current quantum ends */
240 uint64_t last_dispatch; /* time of last dispatch */
241
242 uint64_t kperf_last_sample_time; /* time of last kperf sample */
243
244 uint64_t deadline; /* current deadline */
245 bool first_timeslice; /* has the quantum expired since context switch */
246
247#if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ)
248 struct run_queue runq; /* runq for this processor */
249#endif
250
251#if defined(CONFIG_SCHED_TRADITIONAL)
252 int runq_bound_count; /* # of threads bound to this processor */
253#endif
254#if defined(CONFIG_SCHED_GRRR)
255 struct grrr_run_queue grrr_runq; /* Group Ratio Round-Robin runq */
256#endif
257
258 processor_t processor_primary; /* pointer to primary processor for
259 * secondary SMT processors, or a pointer
260 * to ourselves for primaries or non-SMT */
261 processor_t processor_secondary;
262 struct ipc_port * processor_self; /* port for operations */
263
264 processor_t processor_list; /* all existing processors */
265 processor_data_t processor_data; /* per-processor data */
266};
267
268extern processor_t processor_list;
269decl_simple_lock_data(extern,processor_list_lock)
270
271#define MAX_SCHED_CPUS 64 /* Maximum number of CPUs supported by the scheduler. bits.h:bitmap_*() macros need to be used to support greater than 64 */
272extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */
273
274extern uint32_t processor_avail_count;
275
276extern processor_t master_processor;
277
278extern boolean_t sched_stats_active;
279
280extern processor_t current_processor(void);
281
282/* Lock macros, always acquired and released with interrupts disabled (splsched()) */
283
284#if __SMP__
285#define pset_lock(p) simple_lock(&(p)->sched_lock)
286#define pset_unlock(p) simple_unlock(&(p)->sched_lock)
287#define pset_lock_init(p) simple_lock_init(&(p)->sched_lock, 0)
288#if defined(__arm__) || defined(__arm64__)
289#define pset_assert_locked(p) LCK_SPIN_ASSERT(&(p)->sched_lock, LCK_ASSERT_OWNED)
290#else
291/* See <rdar://problem/39630910> pset_lock() should be converted to use lck_spin_lock() instead of simple_lock() */
292#define pset_assert_locked(p) do { (void)p; } while(0)
293#endif
294
295#define rt_lock_lock(p) simple_lock(&SCHED(rt_runq)(p)->rt_lock)
296#define rt_lock_unlock(p) simple_unlock(&SCHED(rt_runq)(p)->rt_lock)
297#define rt_lock_init(p) simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0)
298#else
299#define pset_lock(p) do { (void)p; } while(0)
300#define pset_unlock(p) do { (void)p; } while(0)
301#define pset_lock_init(p) do { (void)p; } while(0)
302#define pset_assert_locked(p) do { (void)p; } while(0)
303
304#define rt_lock_lock(p) do { (void)p; } while(0)
305#define rt_lock_unlock(p) do { (void)p; } while(0)
306#define rt_lock_init(p) do { (void)p; } while(0)
307#endif
308
309extern void processor_bootstrap(void);
310
311extern void processor_init(
312 processor_t processor,
313 int cpu_id,
314 processor_set_t processor_set);
315
316extern void processor_set_primary(
317 processor_t processor,
318 processor_t primary);
319
320extern kern_return_t processor_shutdown(
321 processor_t processor);
322
323extern void processor_queue_shutdown(
324 processor_t processor);
325
326extern processor_set_t processor_pset(
327 processor_t processor);
328
329extern pset_node_t pset_node_root(void);
330
331extern processor_set_t pset_create(
332 pset_node_t node);
333
334extern void pset_init(
335 processor_set_t pset,
336 pset_node_t node);
337
338extern processor_set_t pset_find(
339 uint32_t cluster_id,
340 processor_set_t default_pset);
341
342extern kern_return_t processor_info_count(
343 processor_flavor_t flavor,
344 mach_msg_type_number_t *count);
345
346#define pset_deallocate(x)
347#define pset_reference(x)
348
349extern void machine_run_count(
350 uint32_t count);
351
352extern processor_t machine_choose_processor(
353 processor_set_t pset,
354 processor_t processor);
355
356#define next_pset(p) (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets)
357
358#define PSET_THING_TASK 0
359#define PSET_THING_THREAD 1
360
361extern kern_return_t processor_set_things(
362 processor_set_t pset,
363 void **thing_list,
364 mach_msg_type_number_t *count,
365 int type);
366
367extern pset_cluster_type_t recommended_pset_type(thread_t thread);
368
369inline static bool
370pset_is_recommended(processor_set_t pset)
371{
372 return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0);
373}
374
375extern void processor_state_update_idle(processor_t processor);
376extern void processor_state_update_from_thread(processor_t processor, thread_t thread);
377extern void processor_state_update_explicit(processor_t processor, int pri,
378 sfi_class_id_t sfi_class, pset_cluster_type_t pset_type,
379 perfcontrol_class_t perfctl_class);
380
381#define PSET_LOAD_NUMERATOR_SHIFT 16
382#define PSET_LOAD_FRACTIONAL_SHIFT 4
383
384inline static int
385sched_get_pset_load_average(processor_set_t pset)
386{
387 return pset->load_average >> (PSET_LOAD_NUMERATOR_SHIFT - PSET_LOAD_FRACTIONAL_SHIFT);
388}
389extern void sched_update_pset_load_average(processor_set_t pset);
390
391inline static void
392pset_update_processor_state(processor_set_t pset, processor_t processor, uint new_state)
393{
394 pset_assert_locked(pset);
395
396 uint old_state = processor->state;
397 uint cpuid = processor->cpu_id;
398
399 assert(processor->processor_set == pset);
400 assert(bit_test(pset->cpu_bitmask, cpuid));
401
402 assert(old_state < PROCESSOR_STATE_LEN);
403 assert(new_state < PROCESSOR_STATE_LEN);
404
405 processor->state = new_state;
406
407 bit_clear(pset->cpu_state_map[old_state], cpuid);
408 bit_set(pset->cpu_state_map[new_state], cpuid);
409
410 if ((old_state == PROCESSOR_RUNNING) || (new_state == PROCESSOR_RUNNING)) {
411 sched_update_pset_load_average(pset);
412 }
413}
414
415#else /* MACH_KERNEL_PRIVATE */
416
417__BEGIN_DECLS
418
419extern void pset_deallocate(
420 processor_set_t pset);
421
422extern void pset_reference(
423 processor_set_t pset);
424
425__END_DECLS
426
427#endif /* MACH_KERNEL_PRIVATE */
428
429#ifdef KERNEL_PRIVATE
430__BEGIN_DECLS
431extern unsigned int processor_count;
432extern processor_t cpu_to_processor(int cpu);
433__END_DECLS
434
435#endif /* KERNEL_PRIVATE */
436
437#endif /* _KERN_PROCESSOR_H_ */
438