1/*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 */
62/*
63 */
64
65/*
66 * Mach kernel startup.
67 */
68
69#include <debug.h>
70#include <xpr_debug.h>
71#include <mach_kdp.h>
72
73#include <mach/boolean.h>
74#include <mach/machine.h>
75#include <mach/thread_act.h>
76#include <mach/task_special_ports.h>
77#include <mach/vm_param.h>
78#include <ipc/ipc_init.h>
79#include <kern/assert.h>
80#include <kern/mach_param.h>
81#include <kern/misc_protos.h>
82#include <kern/clock.h>
83#include <kern/coalition.h>
84#include <kern/cpu_number.h>
85#include <kern/cpu_quiesce.h>
86#include <kern/ledger.h>
87#include <kern/machine.h>
88#include <kern/processor.h>
89#include <kern/sched_prim.h>
90#include <kern/turnstile.h>
91#if CONFIG_SCHED_SFI
92#include <kern/sfi.h>
93#endif
94#include <kern/startup.h>
95#include <kern/task.h>
96#include <kern/thread.h>
97#include <kern/timer.h>
98#if CONFIG_TELEMETRY
99#include <kern/telemetry.h>
100#endif
101#include <kern/xpr.h>
102#include <kern/zalloc.h>
103#include <kern/locks.h>
104#include <kern/debug.h>
105#include <corpses/task_corpse.h>
106#include <prng/random.h>
107#include <console/serial_protos.h>
108#include <vm/vm_kern.h>
109#include <vm/vm_init.h>
110#include <vm/vm_map.h>
111#include <vm/vm_object.h>
112#include <vm/vm_page.h>
113#include <vm/vm_pageout.h>
114#include <vm/vm_shared_region.h>
115#include <machine/pmap.h>
116#include <machine/commpage.h>
117#include <libkern/version.h>
118#include <sys/codesign.h>
119#include <sys/kdebug.h>
120#include <sys/random.h>
121#include <sys/ktrace.h>
122#include <libkern/section_keywords.h>
123
124#include <kern/ltable.h>
125#include <kern/waitq.h>
126#include <ipc/ipc_voucher.h>
127#include <voucher/ipc_pthread_priority_internal.h>
128#include <mach/host_info.h>
129#include <pthread/workqueue_internal.h>
130
131#if CONFIG_XNUPOST
132#include <tests/ktest.h>
133#include <tests/xnupost.h>
134#endif
135
136#if CONFIG_ATM
137#include <atm/atm_internal.h>
138#endif
139
140#if CONFIG_CSR
141#include <sys/csr.h>
142#endif
143
144#include <bank/bank_internal.h>
145
146#if ALTERNATE_DEBUGGER
147#include <arm64/alternate_debugger.h>
148#endif
149
150#if MACH_KDP
151#include <kdp/kdp.h>
152#endif
153
154#if CONFIG_MACF
155#include <security/mac_mach_internal.h>
156#if CONFIG_VNGUARD
157extern void vnguard_policy_init(void);
158#endif
159#endif
160
161#if KPC
162#include <kern/kpc.h>
163#endif
164
165#if HYPERVISOR
166#include <kern/hv_support.h>
167#endif
168
169#include <san/kasan.h>
170
171#if defined(__arm__) || defined(__arm64__)
172#include <arm/misc_protos.h> // for arm_vm_prot_finalize
173#endif
174
175#include <i386/pmCPU.h>
176static void kernel_bootstrap_thread(void);
177
178static void load_context(
179 thread_t thread);
180#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
181extern void cpu_userwindow_init(int);
182extern void cpu_physwindow_init(int);
183#endif
184
185#if CONFIG_ECC_LOGGING
186#include <kern/ecc.h>
187#endif
188
189#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX
190#include <i386/vmx/vmx_cpu.h>
191#endif
192
193#if CONFIG_DTRACE
194extern void dtrace_early_init(void);
195extern void sdt_early_init(void);
196#endif
197
198// libkern/OSKextLib.cpp
199extern void OSKextRemoveKextBootstrap(void);
200
201void scale_setup(void);
202extern void bsd_scale_setup(int);
203extern unsigned int semaphore_max;
204extern void stackshot_init(void);
205extern void ktrace_init(void);
206extern void oslog_init(void);
207
208/*
209 * Running in virtual memory, on the interrupt stack.
210 */
211
212extern int serverperfmode;
213
214/* size of kernel trace buffer, disabled by default */
215unsigned int new_nkdbufs = 0;
216unsigned int wake_nkdbufs = 0;
217unsigned int write_trace_on_panic = 0;
218static char trace_typefilter[64] = { 0 };
219unsigned int trace_wrap = 0;
220boolean_t trace_serial = FALSE;
221boolean_t early_boot_complete = FALSE;
222
223/* physically contiguous carveouts */
224SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout = 0;
225SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout_pa = 0;
226SECURITY_READ_ONLY_LATE(size_t) phys_carveout_size = 0;
227
228/* mach leak logging */
229int log_leaks = 0;
230
231static inline void
232kernel_bootstrap_log(const char *message)
233{
234// kprintf("kernel_bootstrap: %s\n", message);
235 kernel_debug_string_early(message);
236}
237
238static inline void
239kernel_bootstrap_thread_log(const char *message)
240{
241// kprintf("kernel_bootstrap_thread: %s\n", message);
242 kernel_debug_string_early(message);
243}
244
245void
246kernel_early_bootstrap(void)
247{
248 /* serverperfmode is needed by timer setup */
249 if (PE_parse_boot_argn("serverperfmode", &serverperfmode, sizeof (serverperfmode))) {
250 serverperfmode = 1;
251 }
252
253 lck_mod_init();
254
255 /*
256 * Initialize the timer callout world
257 */
258 timer_call_init();
259
260#if CONFIG_SCHED_SFI
261 /*
262 * Configure SFI classes
263 */
264 sfi_early_init();
265#endif
266}
267
268extern boolean_t IORamDiskBSDRoot(void);
269extern kern_return_t cpm_preallocate_early(void);
270
271void
272kernel_bootstrap(void)
273{
274 kern_return_t result;
275 thread_t thread;
276 char namep[16];
277
278 printf("%s\n", version); /* log kernel version */
279
280 if (PE_parse_boot_argn("-l", namep, sizeof (namep))) /* leaks logging */
281 log_leaks = 1;
282
283 PE_parse_boot_argn("trace", &new_nkdbufs, sizeof (new_nkdbufs));
284 PE_parse_boot_argn("trace_wake", &wake_nkdbufs, sizeof (wake_nkdbufs));
285 PE_parse_boot_argn("trace_panic", &write_trace_on_panic, sizeof(write_trace_on_panic));
286 PE_parse_boot_arg_str("trace_typefilter", trace_typefilter, sizeof(trace_typefilter));
287 PE_parse_boot_argn("trace_wrap", &trace_wrap, sizeof(trace_wrap));
288
289 scale_setup();
290
291 kernel_bootstrap_log("vm_mem_bootstrap");
292 vm_mem_bootstrap();
293
294 kernel_bootstrap_log("cs_init");
295 cs_init();
296
297 kernel_bootstrap_log("vm_mem_init");
298 vm_mem_init();
299
300 machine_info.memory_size = (uint32_t)mem_size;
301 machine_info.max_mem = max_mem;
302 machine_info.major_version = version_major;
303 machine_info.minor_version = version_minor;
304
305 oslog_init();
306
307#if KASAN
308 kernel_bootstrap_log("kasan_late_init");
309 kasan_late_init();
310#endif
311
312#if CONFIG_TELEMETRY
313 kernel_bootstrap_log("telemetry_init");
314 telemetry_init();
315#endif
316
317#if CONFIG_CSR
318 kernel_bootstrap_log("csr_init");
319 csr_init();
320#endif
321
322 if (PE_i_can_has_debugger(NULL)) {
323 if (PE_parse_boot_argn("-show_pointers", &namep, sizeof(namep))) {
324 doprnt_hide_pointers = FALSE;
325 }
326 if (PE_parse_boot_argn("-no_slto_panic", &namep, sizeof(namep))) {
327 extern boolean_t spinlock_timeout_panic;
328 spinlock_timeout_panic = FALSE;
329 }
330 }
331
332 kernel_bootstrap_log("console_init");
333 console_init();
334
335 kernel_bootstrap_log("stackshot_init");
336 stackshot_init();
337
338 kernel_bootstrap_log("sched_init");
339 sched_init();
340
341 kernel_bootstrap_log("ltable_bootstrap");
342 ltable_bootstrap();
343
344 kernel_bootstrap_log("waitq_bootstrap");
345 waitq_bootstrap();
346
347 kernel_bootstrap_log("ipc_bootstrap");
348 ipc_bootstrap();
349
350#if CONFIG_MACF
351 kernel_bootstrap_log("mac_policy_init");
352 mac_policy_init();
353#endif
354
355 kernel_bootstrap_log("ipc_init");
356 ipc_init();
357
358 /*
359 * As soon as the virtual memory system is up, we record
360 * that this CPU is using the kernel pmap.
361 */
362 kernel_bootstrap_log("PMAP_ACTIVATE_KERNEL");
363 PMAP_ACTIVATE_KERNEL(master_cpu);
364
365 kernel_bootstrap_log("mapping_free_prime");
366 mapping_free_prime(); /* Load up with temporary mapping blocks */
367
368 kernel_bootstrap_log("machine_init");
369 machine_init();
370
371 kernel_bootstrap_log("clock_init");
372 clock_init();
373
374 ledger_init();
375
376 /*
377 * Initialize the IPC, task, and thread subsystems.
378 */
379
380#if CONFIG_COALITIONS
381 kernel_bootstrap_log("coalitions_init");
382 coalitions_init();
383#endif
384
385 kernel_bootstrap_log("task_init");
386 task_init();
387
388 kernel_bootstrap_log("thread_init");
389 thread_init();
390
391 kernel_bootstrap_log("workq_init");
392 workq_init();
393
394 kernel_bootstrap_log("turnstiles_init");
395 turnstiles_init();
396
397#if CONFIG_ATM
398 /* Initialize the Activity Trace Resource Manager. */
399 kernel_bootstrap_log("atm_init");
400 atm_init();
401#endif
402 kernel_bootstrap_log("mach_init_activity_id");
403 mach_init_activity_id();
404
405 /* Initialize the BANK Manager. */
406 kernel_bootstrap_log("bank_init");
407 bank_init();
408
409 kernel_bootstrap_log("ipc_pthread_priority_init");
410 ipc_pthread_priority_init();
411
412 /* initialize the corpse config based on boot-args */
413 corpses_init();
414
415 /* initialize host_statistics */
416 host_statistics_init();
417
418 /*
419 * Create a kernel thread to execute the kernel bootstrap.
420 */
421 kernel_bootstrap_log("kernel_thread_create");
422 result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread);
423
424 if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n", result);
425
426 thread->state = TH_RUN;
427 thread->last_made_runnable_time = mach_absolute_time();
428 thread_deallocate(thread);
429
430 kernel_bootstrap_log("load_context - done");
431 load_context(thread);
432 /*NOTREACHED*/
433}
434
435int kth_started = 0;
436
437vm_offset_t vm_kernel_addrperm;
438vm_offset_t buf_kernel_addrperm;
439vm_offset_t vm_kernel_addrperm_ext;
440uint64_t vm_kernel_addrhash_salt;
441uint64_t vm_kernel_addrhash_salt_ext;
442
443/*
444 * Now running in a thread. Kick off other services,
445 * invoke user bootstrap, enter pageout loop.
446 */
447static void
448kernel_bootstrap_thread(void)
449{
450 processor_t processor = current_processor();
451
452#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
453 kernel_bootstrap_thread_log("idle_thread_create");
454 /*
455 * Create the idle processor thread.
456 */
457 idle_thread_create(processor);
458
459 /*
460 * N.B. Do not stick anything else
461 * before this point.
462 *
463 * Start up the scheduler services.
464 */
465 kernel_bootstrap_thread_log("sched_startup");
466 sched_startup();
467
468 /*
469 * Thread lifecycle maintenance (teardown, stack allocation)
470 */
471 kernel_bootstrap_thread_log("thread_daemon_init");
472 thread_daemon_init();
473
474 /* Create kernel map entry reserve */
475 vm_kernel_reserved_entry_init();
476
477 /*
478 * Thread callout service.
479 */
480 kernel_bootstrap_thread_log("thread_call_initialize");
481 thread_call_initialize();
482
483 /*
484 * Remain on current processor as
485 * additional processors come online.
486 */
487 kernel_bootstrap_thread_log("thread_bind");
488 thread_bind(processor);
489
490#if __arm64__
491 if (IORamDiskBSDRoot()) {
492 cpm_preallocate_early();
493 }
494#endif /* __arm64__ */
495
496 /*
497 * Initialize ipc thread call support.
498 */
499 kernel_bootstrap_thread_log("ipc_thread_call_init");
500 ipc_thread_call_init();
501
502 /*
503 * Kick off memory mapping adjustments.
504 */
505 kernel_bootstrap_thread_log("mapping_adjust");
506 mapping_adjust();
507
508 /*
509 * Create the clock service.
510 */
511 kernel_bootstrap_thread_log("clock_service_create");
512 clock_service_create();
513
514 /*
515 * Create the device service.
516 */
517 device_service_create();
518
519 kth_started = 1;
520
521#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
522 /*
523 * Create and initialize the physical copy window for processor 0
524 * This is required before starting kicking off IOKit.
525 */
526 cpu_physwindow_init(0);
527#endif
528
529 if (PE_i_can_has_debugger(NULL)) {
530 unsigned int phys_carveout_mb = 0;
531 if (PE_parse_boot_argn("phys_carveout_mb", &phys_carveout_mb,
532 sizeof(phys_carveout_mb)) && phys_carveout_mb > 0) {
533 phys_carveout_size = phys_carveout_mb * 1024 * 1024;
534 kern_return_t kr = kmem_alloc_contig(kernel_map,
535 (vm_offset_t *)&phys_carveout, phys_carveout_size,
536 VM_MAP_PAGE_MASK(kernel_map), 0, 0, KMA_NOPAGEWAIT,
537 VM_KERN_MEMORY_DIAG);
538 if (kr != KERN_SUCCESS) {
539 kprintf("failed to allocate %uMB for phys_carveout_mb: %u\n",
540 phys_carveout_mb, (unsigned int)kr);
541 } else {
542 phys_carveout_pa = kvtophys((vm_offset_t)phys_carveout);
543 }
544 }
545 }
546
547#if MACH_KDP
548 kernel_bootstrap_log("kdp_init");
549 kdp_init();
550#endif
551
552#if ALTERNATE_DEBUGGER
553 alternate_debugger_init();
554#endif
555
556#if KPC
557 kpc_init();
558#endif
559
560#if CONFIG_ECC_LOGGING
561 ecc_log_init();
562#endif
563
564#if HYPERVISOR
565 hv_support_init();
566#endif
567
568#if CONFIG_TELEMETRY
569 kernel_bootstrap_log("bootprofile_init");
570 bootprofile_init();
571#endif
572
573#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX
574 vmx_init();
575#endif
576
577 kernel_bootstrap_thread_log("ktrace_init");
578 ktrace_init();
579
580 kdebug_init(new_nkdbufs, trace_typefilter, trace_wrap);
581
582#ifdef MACH_BSD
583 kernel_bootstrap_log("bsd_early_init");
584 bsd_early_init();
585#endif
586
587#if defined(__arm64__)
588 ml_lockdown_init();
589#endif
590
591#ifdef IOKIT
592 kernel_bootstrap_log("PE_init_iokit");
593 PE_init_iokit();
594#endif
595
596 assert(ml_get_interrupts_enabled() == FALSE);
597
598 /*
599 * Past this point, kernel subsystems that expect to operate with
600 * interrupts or preemption enabled may begin enforcement.
601 */
602 early_boot_complete = TRUE;
603
604#if INTERRUPT_MASKED_DEBUG
605 // Reset interrupts masked timeout before we enable interrupts
606 ml_spin_debug_clear_self();
607#endif
608 (void) spllo(); /* Allow interruptions */
609
610#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
611 /*
612 * Create and initialize the copy window for processor 0
613 * This also allocates window space for all other processors.
614 * However, this is dependent on the number of processors - so this call
615 * must be after IOKit has been started because IOKit performs processor
616 * discovery.
617 */
618 cpu_userwindow_init(0);
619#endif
620
621 /*
622 * Initialize the shared region module.
623 */
624 vm_shared_region_init();
625 vm_commpage_init();
626 vm_commpage_text_init();
627
628#if CONFIG_MACF
629 kernel_bootstrap_log("mac_policy_initmach");
630 mac_policy_initmach();
631#if CONFIG_VNGUARD
632 vnguard_policy_init();
633#endif
634#endif
635
636#if CONFIG_DTRACE
637 dtrace_early_init();
638 sdt_early_init();
639#endif
640
641
642 /*
643 * Get rid of segments used to bootstrap kext loading. This removes
644 * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
645 * Must be done prior to lockdown so that we can free (and possibly relocate)
646 * the static KVA mappings used for the jettisoned bootstrap segments.
647 */
648 OSKextRemoveKextBootstrap();
649#if defined(__arm__) || defined(__arm64__)
650#if CONFIG_KERNEL_INTEGRITY
651 machine_lockdown_preflight();
652#endif
653 /*
654 * Finalize protections on statically mapped pages now that comm page mapping is established.
655 */
656 arm_vm_prot_finalize(PE_state.bootArgs);
657#endif
658
659#if CONFIG_SCHED_SFI
660 kernel_bootstrap_log("sfi_init");
661 sfi_init();
662#endif
663
664 /*
665 * Initialize the globals used for permuting kernel
666 * addresses that may be exported to userland as tokens
667 * using VM_KERNEL_ADDRPERM()/VM_KERNEL_ADDRPERM_EXTERNAL().
668 * Force the random number to be odd to avoid mapping a non-zero
669 * word-aligned address to zero via addition.
670 * Note: at this stage we can use the cryptographically secure PRNG
671 * rather than early_random().
672 */
673 read_random(&vm_kernel_addrperm, sizeof(vm_kernel_addrperm));
674 vm_kernel_addrperm |= 1;
675 read_random(&buf_kernel_addrperm, sizeof(buf_kernel_addrperm));
676 buf_kernel_addrperm |= 1;
677 read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext));
678 vm_kernel_addrperm_ext |= 1;
679 read_random(&vm_kernel_addrhash_salt, sizeof(vm_kernel_addrhash_salt));
680 read_random(&vm_kernel_addrhash_salt_ext, sizeof(vm_kernel_addrhash_salt_ext));
681
682 vm_set_restrictions();
683
684
685#ifdef CONFIG_XNUPOST
686 kern_return_t result = kernel_list_tests();
687 result = kernel_do_post();
688 if (result != KERN_SUCCESS) {
689 panic("kernel_do_post: Tests failed with result = 0x%08x\n", result);
690 }
691 kernel_bootstrap_log("kernel_do_post - done");
692#endif /* CONFIG_XNUPOST */
693
694
695 /*
696 * Start the user bootstrap.
697 */
698#ifdef MACH_BSD
699 bsd_init();
700#endif
701
702
703 /*
704 * Get rid of pages used for early boot tracing.
705 */
706 kdebug_free_early_buf();
707
708 serial_keyboard_init(); /* Start serial keyboard if wanted */
709
710 vm_page_init_local_q();
711
712 thread_bind(PROCESSOR_NULL);
713
714 /*
715 * Become the pageout daemon.
716 */
717 vm_pageout();
718 /*NOTREACHED*/
719}
720
721/*
722 * slave_main:
723 *
724 * Load the first thread to start a processor.
725 */
726void
727slave_main(void *machine_param)
728{
729 processor_t processor = current_processor();
730 thread_t thread;
731
732 /*
733 * Use the idle processor thread if there
734 * is no dedicated start up thread.
735 */
736 if (processor->next_thread == THREAD_NULL) {
737 thread = processor->idle_thread;
738 thread->continuation = (thread_continue_t)processor_start_thread;
739 thread->parameter = machine_param;
740 }
741 else {
742 thread = processor->next_thread;
743 processor->next_thread = THREAD_NULL;
744 }
745
746 load_context(thread);
747 /*NOTREACHED*/
748}
749
750/*
751 * processor_start_thread:
752 *
753 * First thread to execute on a started processor.
754 *
755 * Called at splsched.
756 */
757void
758processor_start_thread(void *machine_param)
759{
760 processor_t processor = current_processor();
761 thread_t self = current_thread();
762
763 slave_machine_init(machine_param);
764
765 /*
766 * If running the idle processor thread,
767 * reenter the idle loop, else terminate.
768 */
769 if (self == processor->idle_thread)
770 thread_block((thread_continue_t)idle_thread);
771
772 thread_terminate(self);
773 /*NOTREACHED*/
774}
775
776/*
777 * load_context:
778 *
779 * Start the first thread on a processor.
780 */
781static void __attribute__((noreturn))
782load_context(
783 thread_t thread)
784{
785 processor_t processor = current_processor();
786
787
788#define load_context_kprintf(x...) /* kprintf("load_context: " x) */
789
790 load_context_kprintf("machine_set_current_thread\n");
791 machine_set_current_thread(thread);
792
793 load_context_kprintf("processor_up\n");
794 processor_up(processor);
795
796 PMAP_ACTIVATE_KERNEL(processor->cpu_id);
797
798 /*
799 * Acquire a stack if none attached. The panic
800 * should never occur since the thread is expected
801 * to have reserved stack.
802 */
803 load_context_kprintf("thread %p, stack %lx, stackptr %lx\n", thread,
804 thread->kernel_stack, thread->machine.kstackptr);
805 if (!thread->kernel_stack) {
806 load_context_kprintf("stack_alloc_try\n");
807 if (!stack_alloc_try(thread))
808 panic("load_context");
809 }
810
811 /*
812 * The idle processor threads are not counted as
813 * running for load calculations.
814 */
815 if (!(thread->state & TH_IDLE))
816 sched_run_incr(thread);
817
818 processor->active_thread = thread;
819 processor_state_update_explicit(processor, thread->sched_pri,
820 SFI_CLASS_KERNEL, PSET_SMP, thread_get_perfcontrol_class(thread));
821 processor->starting_pri = thread->sched_pri;
822 processor->deadline = UINT64_MAX;
823 thread->last_processor = processor;
824
825 processor->last_dispatch = mach_absolute_time();
826 timer_start(&thread->system_timer, processor->last_dispatch);
827 PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
828
829 timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch);
830 PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
831
832
833 cpu_quiescent_counter_join(processor->last_dispatch);
834
835 PMAP_ACTIVATE_USER(thread, processor->cpu_id);
836
837 load_context_kprintf("machine_load_context\n");
838 machine_load_context(thread);
839 /*NOTREACHED*/
840}
841
842void
843scale_setup()
844{
845 int scale = 0;
846#if defined(__LP64__)
847 typeof(task_max) task_max_base = task_max;
848
849 /* Raise limits for servers with >= 16G */
850 if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 *1024ULL))) {
851 scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 *1024ULL));
852 /* limit to 128 G */
853 if (scale > 16)
854 scale = 16;
855 task_max_base = 2500;
856 /* Raise limits for machines with >= 3GB */
857 } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL)) {
858 if ((uint64_t)sane_size < (uint64_t)(8 * 1024 * 1024 *1024ULL)) {
859 scale = 2;
860 } else {
861 /* limit to 64GB */
862 scale = MIN(16, (int)((uint64_t)sane_size / (uint64_t)(4 * 1024 * 1024 *1024ULL)));
863 }
864 }
865
866 task_max = MAX(task_max, task_max_base * scale);
867
868 if (scale != 0) {
869 task_threadmax = task_max;
870 thread_max = task_max * 5;
871 }
872
873#endif
874
875 bsd_scale_setup(scale);
876
877 ipc_space_max = SPACE_MAX;
878 ipc_port_max = PORT_MAX;
879 ipc_pset_max = SET_MAX;
880 semaphore_max = SEMAPHORE_MAX;
881}
882
883