1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57/*
58 */
59#ifdef KERNEL_PRIVATE
60
61#ifndef _I386_MP_H_
62#define _I386_MP_H_
63
64//#define MP_DEBUG 1
65
66#include <i386/apic.h>
67#include <i386/mp_events.h>
68
69#define MAX_CPUS 64 /* 8 * sizeof(cpumask_t) */
70
71#ifndef ASSEMBLER
72#include <stdint.h>
73#include <sys/cdefs.h>
74#include <mach/boolean.h>
75#include <mach/kern_return.h>
76#include <mach/i386/thread_status.h>
77#include <mach/vm_types.h>
78#include <kern/simple_lock.h>
79
80__BEGIN_DECLS
81
82extern kern_return_t intel_startCPU(int slot_num);
83extern kern_return_t intel_startCPU_fast(int slot_num);
84extern void i386_init_slave(void);
85extern void i386_init_slave_fast(void);
86extern void smp_init(void);
87
88extern void cpu_interrupt(int cpu);
89__END_DECLS
90
91extern unsigned int real_ncpus; /* real number of cpus */
92extern unsigned int max_ncpus; /* max number of cpus */
93decl_simple_lock_data(extern,kdb_lock) /* kdb lock */
94
95__BEGIN_DECLS
96
97extern void console_init(void);
98extern void *console_cpu_alloc(boolean_t boot_cpu);
99extern void console_cpu_free(void *console_buf);
100
101extern int kdb_cpu; /* current cpu running kdb */
102extern int kdb_debug;
103extern int kdb_active[];
104
105extern volatile boolean_t mp_kdp_trap;
106extern volatile boolean_t mp_kdp_is_NMI;
107extern volatile boolean_t force_immediate_debugger_NMI;
108extern volatile boolean_t pmap_tlb_flush_timeout;
109extern volatile usimple_lock_t spinlock_timed_out;
110extern volatile uint32_t spinlock_owner_cpu;
111extern uint32_t spinlock_timeout_NMI(uintptr_t thread_addr);
112
113extern uint64_t LastDebuggerEntryAllowance;
114
115extern void mp_kdp_enter(boolean_t proceed_on_failure);
116extern void mp_kdp_exit(void);
117extern boolean_t mp_kdp_all_cpus_halted(void);
118
119extern boolean_t mp_recent_debugger_activity(void);
120extern void kernel_spin(uint64_t spin_ns);
121
122/*
123 * All cpu rendezvous:
124 */
125extern void mp_rendezvous(
126 void (*setup_func)(void *),
127 void (*action_func)(void *),
128 void (*teardown_func)(void *),
129 void *arg);
130extern void mp_rendezvous_no_intrs(
131 void (*action_func)(void *),
132 void *arg);
133extern void mp_rendezvous_break_lock(void);
134extern void mp_rendezvous_lock(void);
135extern void mp_rendezvous_unlock(void);
136
137/*
138 * All cpu broadcast.
139 * Called from thread context, this blocks until all active cpus have
140 * run action_func:
141 */
142extern void mp_broadcast(
143 void (*action_func)(void *),
144 void *arg);
145#if MACH_KDP
146typedef long (*kdp_x86_xcpu_func_t) (void *arg0, void *arg1, uint16_t lcpu);
147
148extern long kdp_x86_xcpu_invoke(const uint16_t lcpu,
149 kdp_x86_xcpu_func_t func,
150 void *arg0, void *arg1);
151typedef enum {KDP_XCPU_NONE = 0xffff, KDP_CURRENT_LCPU = 0xfffe} kdp_cpu_t;
152#endif
153
154typedef uint32_t cpu_t;
155typedef volatile uint64_t cpumask_t;
156static inline cpumask_t
157cpu_to_cpumask(cpu_t cpu)
158{
159 return (cpu < MAX_CPUS) ? (1ULL << cpu) : 0;
160}
161#define CPUMASK_ALL 0xffffffffffffffffULL
162#define CPUMASK_SELF cpu_to_cpumask(cpu_number())
163#define CPUMASK_OTHERS (CPUMASK_ALL & ~CPUMASK_SELF)
164
165/* Initialation routing called at processor registration */
166extern void mp_cpus_call_cpu_init(int cpu);
167
168/*
169 * Invoke a function (possibly NULL) on a set of cpus specified by a mask.
170 * The mask may include the local cpu.
171 * If the mode is:
172 * - ASYNC: other cpus make their calls in parallel
173 * - SYNC: the calls are performed serially in logical cpu order
174 * - NOSYNC: the calls are queued
175 * Unless the mode is NOSYNC, mp_cpus_call() returns when the function has been
176 * called on all specified cpus.
177 * The return value is the number of cpus where the call was made or queued.
178 * The action function is called with interrupts disabled.
179 */
180extern cpu_t mp_cpus_call(
181 cpumask_t cpus,
182 mp_sync_t mode,
183 void (*action_func)(void *),
184 void *arg);
185extern cpu_t mp_cpus_call1(
186 cpumask_t cpus,
187 mp_sync_t mode,
188 void (*action_func)(void *, void*),
189 void *arg0,
190 void *arg1,
191 cpumask_t *cpus_calledp);
192
193typedef enum {
194 NONE = 0,
195 SPINLOCK_TIMEOUT,
196 TLB_FLUSH_TIMEOUT,
197 CROSSCALL_TIMEOUT,
198 INTERRUPT_WATCHDOG
199} NMI_reason_t;
200extern void NMIPI_panic(cpumask_t cpus, NMI_reason_t reason);
201
202/* Interrupt a set of cpus, forcing an exit out of non-root mode */
203extern void mp_cpus_kick(cpumask_t cpus);
204/*
205 * Power-management-specific SPI to:
206 * - register a callout function, and
207 * - request the callout (if registered) on a given cpu.
208 */
209extern void PM_interrupt_register(void (*fn)(void));
210extern void cpu_PM_interrupt(int cpu);
211
212__END_DECLS
213
214#if MP_DEBUG
215typedef struct {
216 uint64_t time;
217 int cpu;
218 mp_event_t event;
219} cpu_signal_event_t;
220
221#define LOG_NENTRIES 100
222typedef struct {
223 uint64_t count[MP_LAST];
224 int next_entry;
225 cpu_signal_event_t entry[LOG_NENTRIES];
226} cpu_signal_event_log_t;
227
228extern cpu_signal_event_log_t *cpu_signal[];
229extern cpu_signal_event_log_t *cpu_handle[];
230
231#define DBGLOG(log,_cpu,_event) { \
232 boolean_t spl = ml_set_interrupts_enabled(FALSE); \
233 cpu_signal_event_log_t *logp = log[cpu_number()]; \
234 int next = logp->next_entry; \
235 cpu_signal_event_t *eventp = &logp->entry[next]; \
236 \
237 logp->count[_event]++; \
238 \
239 eventp->time = rdtsc64(); \
240 eventp->cpu = _cpu; \
241 eventp->event = _event; \
242 if (next == (LOG_NENTRIES - 1)) \
243 logp->next_entry = 0; \
244 else \
245 logp->next_entry++; \
246 \
247 (void) ml_set_interrupts_enabled(spl); \
248}
249
250#define DBGLOG_CPU_INIT(cpu) { \
251 cpu_signal_event_log_t **sig_logpp = &cpu_signal[cpu]; \
252 cpu_signal_event_log_t **hdl_logpp = &cpu_handle[cpu]; \
253 \
254 if (*sig_logpp == NULL && \
255 kmem_alloc(kernel_map, \
256 (vm_offset_t *) sig_logpp, \
257 sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
258 panic("DBGLOG_CPU_INIT cpu_signal allocation failed\n");\
259 bzero(*sig_logpp, sizeof(cpu_signal_event_log_t)); \
260 if (*hdl_logpp == NULL && \
261 kmem_alloc(kernel_map, \
262 (vm_offset_t *) hdl_logpp, \
263 sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
264 panic("DBGLOG_CPU_INIT cpu_handle allocation failed\n");\
265 bzero(*hdl_logpp, sizeof(cpu_signal_event_log_t)); \
266}
267#else /* MP_DEBUG */
268#define DBGLOG(log,_cpu,_event)
269#define DBGLOG_CPU_INIT(cpu)
270#endif /* MP_DEBUG */
271
272#endif /* ASSEMBLER */
273
274#ifdef ASSEMBLER
275#define i_bit(bit, word) ((long)(*(word)) & (1L << (bit)))
276#else
277__attribute__((always_inline)) static inline long
278i_bit_impl(long word, long bit) {
279 long bitmask = 1L << bit;
280 return word & bitmask;
281}
282#define i_bit(bit, word) i_bit_impl((long)(*(word)), bit)
283#endif
284
285
286#endif /* _I386_MP_H_ */
287
288#endif /* KERNEL_PRIVATE */
289