1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57#include <mach_debug.h>
58#include <mach_ldebug.h>
59
60#include <sys/kdebug.h>
61
62#include <mach/kern_return.h>
63#include <mach/thread_status.h>
64#include <mach/vm_param.h>
65
66#include <kern/counters.h>
67#include <kern/kalloc.h>
68#include <kern/mach_param.h>
69#include <kern/processor.h>
70#include <kern/cpu_data.h>
71#include <kern/cpu_number.h>
72#include <kern/task.h>
73#include <kern/thread.h>
74#include <kern/sched_prim.h>
75#include <kern/misc_protos.h>
76#include <kern/assert.h>
77#include <kern/spl.h>
78#include <kern/machine.h>
79#include <ipc/ipc_port.h>
80#include <vm/vm_kern.h>
81#include <vm/vm_map.h>
82#include <vm/pmap.h>
83#include <vm/vm_protos.h>
84
85#include <i386/commpage/commpage.h>
86#include <i386/cpu_data.h>
87#include <i386/cpu_number.h>
88#include <i386/eflags.h>
89#include <i386/proc_reg.h>
90#include <i386/tss.h>
91#include <i386/user_ldt.h>
92#include <i386/fpu.h>
93#include <i386/mp_desc.h>
94#include <i386/misc_protos.h>
95#include <i386/thread.h>
96#include <i386/seg.h>
97#include <i386/machine_routines.h>
98
99#if HYPERVISOR
100#include <kern/hv_support.h>
101#endif
102
103#define ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(_type_) \
104extern char assert_is_16byte_multiple_sizeof_ ## _type_ \
105 [(sizeof(_type_) % 16) == 0 ? 1 : -1]
106
107/* Compile-time checks for vital save area sizing: */
108ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_64_intr_stack_frame_t);
109ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_saved_state_t);
110
111#define DIRECTION_FLAG_DEBUG (DEBUG | DEVELOPMENT)
112
113extern zone_t iss_zone; /* zone for saved_state area */
114extern zone_t ids_zone; /* zone for debug_state area */
115
116void
117act_machine_switch_pcb(__unused thread_t old, thread_t new)
118{
119 pcb_t pcb = THREAD_TO_PCB(new);
120 cpu_data_t *cdp = current_cpu_datap();
121 struct real_descriptor *ldtp;
122 mach_vm_offset_t pcb_stack_top;
123
124 assert(new->kernel_stack != 0);
125 assert(ml_get_interrupts_enabled() == FALSE);
126#ifdef DIRECTION_FLAG_DEBUG
127 if (x86_get_flags() & EFL_DF) {
128 panic("Direction flag detected: 0x%lx", x86_get_flags());
129 }
130#endif
131
132 /*
133 * Clear segment state
134 * unconditionally for DS/ES/FS but more carefully for GS whose
135 * cached state we track.
136 */
137 set_ds(NULL_SEG);
138 set_es(NULL_SEG);
139 set_fs(NULL_SEG);
140
141 if (get_gs() != NULL_SEG) {
142 swapgs(); /* switch to user's GS context */
143 set_gs(NULL_SEG);
144 swapgs(); /* and back to kernel */
145
146 /* record the active machine state lost */
147 cdp->cpu_uber.cu_user_gs_base = 0;
148 }
149
150 vm_offset_t isf;
151
152 /*
153 * Set pointer to PCB's interrupt stack frame in cpu data.
154 * Used by syscall and double-fault trap handlers.
155 */
156 isf = (vm_offset_t) &pcb->iss->ss_64.isf;
157 cdp->cpu_uber.cu_isf = isf;
158 pcb_stack_top = (vm_offset_t) (pcb->iss + 1);
159 /* require 16-byte alignment */
160 assert((pcb_stack_top & 0xF) == 0);
161
162 current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku;
163 /*
164 * Top of temporary sysenter stack points to pcb stack.
165 * Although this is not normally used by 64-bit users,
166 * it needs to be set in case a sysenter is attempted.
167 */
168 *current_sstk64() = pcb_stack_top;
169
170 cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku;
171
172 if (is_saved_state64(pcb->iss)) {
173
174 cdp->cpu_task_map = new->map->pmap->pm_task_map;
175
176 /*
177 * Enable the 64-bit user code segment, USER64_CS.
178 * Disable the 32-bit user code segment, USER_CS.
179 */
180 gdt_desc_p(USER64_CS)->access |= ACC_PL_U;
181 gdt_desc_p(USER_CS)->access &= ~ACC_PL_U;
182
183 /*
184 * Switch user's GS base if necessary
185 * by setting the Kernel's GS base MSR
186 * - this will become the user's on the swapgs when
187 * returning to user-space. Avoid this for
188 * kernel threads (no user TLS support required)
189 * and verify the memory shadow of the segment base
190 * in the event it was altered in user space.
191 */
192 if ((pcb->cthread_self != 0) || (new->task != kernel_task)) {
193 if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
194 (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) {
195 cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
196 wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
197 }
198 }
199 } else {
200
201 cdp->cpu_task_map = TASK_MAP_32BIT;
202
203 /*
204 * Disable USER64_CS
205 * Enable USER_CS
206 */
207
208 /* It's possible that writing to the GDT areas
209 * is expensive, if the processor intercepts those
210 * writes to invalidate its internal segment caches
211 * TODO: perhaps only do this if switching bitness
212 */
213 gdt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
214 gdt_desc_p(USER_CS)->access |= ACC_PL_U;
215
216 /*
217 * Set the thread`s cthread (a.k.a pthread)
218 * For 32-bit user this involves setting the USER_CTHREAD
219 * descriptor in the LDT to point to the cthread data.
220 * The involves copying in the pre-initialized descriptor.
221 */
222 ldtp = (struct real_descriptor *)current_ldt();
223 ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
224 if (pcb->uldt_selector != 0)
225 ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
226 cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
227
228 /*
229 * Set the thread`s LDT or LDT entry.
230 */
231 if (new->task == TASK_NULL || new->task->i386_ldt == 0) {
232 /*
233 * Use system LDT.
234 */
235 ml_cpu_set_ldt(KERNEL_LDT);
236 } else {
237 /*
238 * Task has its own LDT.
239 */
240 user_ldt_set(new);
241 }
242 }
243
244 /*
245 * Bump the scheduler generation count in the commpage.
246 * This can be read by user code to detect its preemption.
247 */
248 commpage_sched_gen_inc();
249}
250
251kern_return_t
252thread_set_wq_state32(thread_t thread, thread_state_t tstate)
253{
254 x86_thread_state32_t *state;
255 x86_saved_state32_t *saved_state;
256 thread_t curth = current_thread();
257 spl_t s=0;
258
259 pal_register_cache_state(thread, DIRTY);
260
261 saved_state = USER_REGS32(thread);
262
263 state = (x86_thread_state32_t *)tstate;
264
265 if (curth != thread) {
266 s = splsched();
267 thread_lock(thread);
268 }
269
270 saved_state->ebp = 0;
271 saved_state->eip = state->eip;
272 saved_state->eax = state->eax;
273 saved_state->ebx = state->ebx;
274 saved_state->ecx = state->ecx;
275 saved_state->edx = state->edx;
276 saved_state->edi = state->edi;
277 saved_state->esi = state->esi;
278 saved_state->uesp = state->esp;
279 saved_state->efl = EFL_USER_SET;
280
281 saved_state->cs = USER_CS;
282 saved_state->ss = USER_DS;
283 saved_state->ds = USER_DS;
284 saved_state->es = USER_DS;
285
286 if (curth != thread) {
287 thread_unlock(thread);
288 splx(s);
289 }
290
291 return KERN_SUCCESS;
292}
293
294
295kern_return_t
296thread_set_wq_state64(thread_t thread, thread_state_t tstate)
297{
298 x86_thread_state64_t *state;
299 x86_saved_state64_t *saved_state;
300 thread_t curth = current_thread();
301 spl_t s=0;
302
303 saved_state = USER_REGS64(thread);
304 state = (x86_thread_state64_t *)tstate;
305
306 /* Disallow setting non-canonical PC or stack */
307 if (!IS_USERADDR64_CANONICAL(state->rsp) ||
308 !IS_USERADDR64_CANONICAL(state->rip)) {
309 return KERN_FAILURE;
310 }
311
312 pal_register_cache_state(thread, DIRTY);
313
314 if (curth != thread) {
315 s = splsched();
316 thread_lock(thread);
317 }
318
319 saved_state->rbp = 0;
320 saved_state->rdi = state->rdi;
321 saved_state->rsi = state->rsi;
322 saved_state->rdx = state->rdx;
323 saved_state->rcx = state->rcx;
324 saved_state->r8 = state->r8;
325 saved_state->r9 = state->r9;
326
327 saved_state->isf.rip = state->rip;
328 saved_state->isf.rsp = state->rsp;
329 saved_state->isf.cs = USER64_CS;
330 saved_state->isf.rflags = EFL_USER_SET;
331
332 if (curth != thread) {
333 thread_unlock(thread);
334 splx(s);
335 }
336
337 return KERN_SUCCESS;
338}
339
340/*
341 * Initialize the machine-dependent state for a new thread.
342 */
343kern_return_t
344machine_thread_create(
345 thread_t thread,
346 task_t task)
347{
348 pcb_t pcb = THREAD_TO_PCB(thread);
349
350#if NCOPY_WINDOWS > 0
351 inval_copy_windows(thread);
352
353 thread->machine.physwindow_pte = 0;
354 thread->machine.physwindow_busy = 0;
355#endif
356
357 /*
358 * Allocate save frame only if required.
359 */
360 if (pcb->iss == NULL) {
361 assert((get_preemption_level() == 0));
362 pcb->iss = (x86_saved_state_t *) zalloc(iss_zone);
363 if (pcb->iss == NULL)
364 panic("iss_zone");
365 }
366
367 /*
368 * Ensure that the synthesized 32-bit state including
369 * the 64-bit interrupt state can be acommodated in the
370 * 64-bit state we allocate for both 32-bit and 64-bit threads.
371 */
372 assert(sizeof(pcb->iss->ss_32) + sizeof(pcb->iss->ss_64.isf) <=
373 sizeof(pcb->iss->ss_64));
374
375 bzero((char *)pcb->iss, sizeof(x86_saved_state_t));
376
377 if (task_has_64Bit_addr(task)) {
378 pcb->iss->flavor = x86_SAVED_STATE64;
379
380 pcb->iss->ss_64.isf.cs = USER64_CS;
381 pcb->iss->ss_64.isf.ss = USER_DS;
382 pcb->iss->ss_64.fs = USER_DS;
383 pcb->iss->ss_64.gs = USER_DS;
384 pcb->iss->ss_64.isf.rflags = EFL_USER_SET;
385 } else {
386 pcb->iss->flavor = x86_SAVED_STATE32;
387
388 pcb->iss->ss_32.cs = USER_CS;
389 pcb->iss->ss_32.ss = USER_DS;
390 pcb->iss->ss_32.ds = USER_DS;
391 pcb->iss->ss_32.es = USER_DS;
392 pcb->iss->ss_32.fs = USER_DS;
393 pcb->iss->ss_32.gs = USER_DS;
394 pcb->iss->ss_32.efl = EFL_USER_SET;
395 }
396
397 simple_lock_init(&pcb->lock, 0);
398
399 pcb->cthread_self = 0;
400 pcb->uldt_selector = 0;
401 pcb->thread_gpu_ns = 0;
402 /* Ensure that the "cthread" descriptor describes a valid
403 * segment.
404 */
405 if ((pcb->cthread_desc.access & ACC_P) == 0) {
406 pcb->cthread_desc = *gdt_desc_p(USER_DS);
407 }
408
409 return(KERN_SUCCESS);
410}
411
412/*
413 * Machine-dependent cleanup prior to destroying a thread
414 */
415void
416machine_thread_destroy(
417 thread_t thread)
418{
419 pcb_t pcb = THREAD_TO_PCB(thread);
420
421#if HYPERVISOR
422 if (thread->hv_thread_target) {
423 hv_callbacks.thread_destroy(thread->hv_thread_target);
424 thread->hv_thread_target = NULL;
425 }
426#endif
427
428 if (pcb->ifps != 0)
429 fpu_free(thread, pcb->ifps);
430 if (pcb->iss != 0) {
431 zfree(iss_zone, pcb->iss);
432 pcb->iss = 0;
433 }
434 if (pcb->ids) {
435 zfree(ids_zone, pcb->ids);
436 pcb->ids = NULL;
437 }
438}
439
440kern_return_t
441machine_thread_set_tsd_base(
442 thread_t thread,
443 mach_vm_offset_t tsd_base)
444{
445
446 if (thread->task == kernel_task) {
447 return KERN_INVALID_ARGUMENT;
448 }
449
450 if (thread_is_64bit_addr(thread)) {
451 /* check for canonical address, set 0 otherwise */
452 if (!IS_USERADDR64_CANONICAL(tsd_base))
453 tsd_base = 0ULL;
454 } else {
455 if (tsd_base > UINT32_MAX)
456 tsd_base = 0ULL;
457 }
458
459 pcb_t pcb = THREAD_TO_PCB(thread);
460 pcb->cthread_self = tsd_base;
461
462 if (!thread_is_64bit_addr(thread)) {
463 /* Set up descriptor for later use */
464 struct real_descriptor desc = {
465 .limit_low = 1,
466 .limit_high = 0,
467 .base_low = tsd_base & 0xffff,
468 .base_med = (tsd_base >> 16) & 0xff,
469 .base_high = (tsd_base >> 24) & 0xff,
470 .access = ACC_P|ACC_PL_U|ACC_DATA_W,
471 .granularity = SZ_32|SZ_G,
472 };
473
474 pcb->cthread_desc = desc;
475 saved_state32(pcb->iss)->gs = USER_CTHREAD;
476 }
477
478 /* For current thread, make the TSD base active immediately */
479 if (thread == current_thread()) {
480
481 if (thread_is_64bit_addr(thread)) {
482 cpu_data_t *cdp;
483
484 mp_disable_preemption();
485 cdp = current_cpu_datap();
486 if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
487 (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE)))
488 wrmsr64(MSR_IA32_KERNEL_GS_BASE, tsd_base);
489 cdp->cpu_uber.cu_user_gs_base = tsd_base;
490 mp_enable_preemption();
491 } else {
492
493 /* assign descriptor */
494 mp_disable_preemption();
495 *ldt_desc_p(USER_CTHREAD) = pcb->cthread_desc;
496 mp_enable_preemption();
497 }
498 }
499
500 return KERN_SUCCESS;
501}
502