1/*
2 * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <vm/vm_kern.h>
29#include <kern/kalloc.h>
30#include <kern/timer_queue.h>
31#include <mach/machine.h>
32#include <i386/cpu_threads.h>
33#include <i386/cpuid.h>
34#include <i386/machine_cpu.h>
35#include <i386/pmCPU.h>
36#include <i386/bit_routines.h>
37
38#if MONOTONIC
39#include <kern/monotonic.h>
40#endif /* MONOTONIC */
41
42#define DIVISOR_GUARD(denom) \
43 if ((denom) == 0) { \
44 kprintf("%s: %d Zero divisor: " #denom, \
45 __FILE__, __LINE__); \
46 }
47
48static void debug_topology_print(void);
49
50boolean_t topo_dbg = FALSE;
51
52x86_pkg_t *x86_pkgs = NULL;
53uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 };
54
55static x86_pkg_t *free_pkgs = NULL;
56static x86_die_t *free_dies = NULL;
57static x86_core_t *free_cores = NULL;
58static uint32_t num_dies = 0;
59
60static x86_cpu_cache_t *x86_caches = NULL;
61static uint32_t num_caches = 0;
62
63static boolean_t topoParmsInited = FALSE;
64x86_topology_parameters_t topoParms;
65
66decl_simple_lock_data(, x86_topo_lock);
67
68static struct cpu_cache {
69 int level; int type;
70} cpu_caches [LCACHE_MAX] = {
71 [L1D] = { 1, CPU_CACHE_TYPE_DATA },
72 [L1I] = { 1, CPU_CACHE_TYPE_INST },
73 [L2U] = { 2, CPU_CACHE_TYPE_UNIF },
74 [L3U] = { 3, CPU_CACHE_TYPE_UNIF },
75};
76
77static boolean_t
78cpu_is_hyperthreaded(void)
79{
80 i386_cpu_info_t *cpuinfo;
81
82 cpuinfo = cpuid_info();
83 return(cpuinfo->thread_count > cpuinfo->core_count);
84}
85
86static x86_cpu_cache_t *
87x86_cache_alloc(void)
88{
89 x86_cpu_cache_t *cache;
90 int i;
91
92 if (x86_caches == NULL) {
93 cache = kalloc(sizeof(x86_cpu_cache_t) + (MAX_CPUS * sizeof(x86_lcpu_t *)));
94 if (cache == NULL)
95 return(NULL);
96 } else {
97 cache = x86_caches;
98 x86_caches = cache->next;
99 cache->next = NULL;
100 }
101
102 bzero(cache, sizeof(x86_cpu_cache_t));
103 cache->next = NULL;
104 cache->maxcpus = MAX_CPUS;
105 for (i = 0; i < cache->maxcpus; i += 1) {
106 cache->cpus[i] = NULL;
107 }
108
109 num_caches += 1;
110
111 return(cache);
112}
113
114static void
115x86_LLC_info(void)
116{
117 int cache_level = 0;
118 uint32_t nCPUsSharing = 1;
119 i386_cpu_info_t *cpuinfo;
120 struct cpu_cache *cachep;
121 int i;
122
123 cpuinfo = cpuid_info();
124
125 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
126
127 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0)
128 continue;
129
130 /*
131 * Only worry about it if it's a deeper level than
132 * what we've seen before.
133 */
134 if (cachep->level > cache_level) {
135 cache_level = cachep->level;
136
137 /*
138 * Save the number of CPUs sharing this cache.
139 */
140 nCPUsSharing = cpuinfo->cache_sharing[i];
141 }
142 }
143
144 /*
145 * Make the level of the LLC be 0 based.
146 */
147 topoParms.LLCDepth = cache_level - 1;
148
149 /*
150 * nCPUsSharing represents the *maximum* number of cores or
151 * logical CPUs sharing the cache.
152 */
153 topoParms.maxSharingLLC = nCPUsSharing;
154
155 topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count /
156 cpuinfo->core_count);
157 topoParms.nLCPUsSharingLLC = nCPUsSharing;
158
159 /*
160 * nCPUsSharing may not be the number of *active* cores or
161 * threads that are sharing the cache.
162 */
163 if (nCPUsSharing > cpuinfo->core_count)
164 topoParms.nCoresSharingLLC = cpuinfo->core_count;
165 if (nCPUsSharing > cpuinfo->thread_count)
166 topoParms.nLCPUsSharingLLC = cpuinfo->thread_count;
167}
168
169static void
170initTopoParms(void)
171{
172 i386_cpu_info_t *cpuinfo;
173
174 topoParms.stable = FALSE;
175
176 cpuinfo = cpuid_info();
177
178 PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg));
179
180 /*
181 * We need to start with getting the LLC information correct.
182 */
183 x86_LLC_info();
184
185 /*
186 * Compute the number of threads (logical CPUs) per core.
187 */
188 DIVISOR_GUARD(cpuinfo->core_count);
189 topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count;
190 DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package);
191 topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package;
192
193 /*
194 * Compute the number of dies per package.
195 */
196 DIVISOR_GUARD(topoParms.nCoresSharingLLC);
197 topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC;
198 DIVISOR_GUARD(topoParms.nPThreadsPerCore);
199 DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
200 topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
201
202
203 /*
204 * Compute the number of cores per die.
205 */
206 topoParms.nLCoresPerDie = topoParms.nCoresSharingLLC;
207 topoParms.nPCoresPerDie = (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
208
209 /*
210 * Compute the number of threads per die.
211 */
212 topoParms.nLThreadsPerDie = topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie;
213 topoParms.nPThreadsPerDie = topoParms.nPThreadsPerCore * topoParms.nPCoresPerDie;
214
215 /*
216 * Compute the number of cores per package.
217 */
218 topoParms.nLCoresPerPackage = topoParms.nLCoresPerDie * topoParms.nLDiesPerPackage;
219 topoParms.nPCoresPerPackage = topoParms.nPCoresPerDie * topoParms.nPDiesPerPackage;
220
221 /*
222 * Compute the number of threads per package.
223 */
224 topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage;
225 topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage;
226
227 TOPO_DBG("\nCache Topology Parameters:\n");
228 TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth);
229 TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC);
230 TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC);
231 TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC);
232
233 TOPO_DBG("\nLogical Topology Parameters:\n");
234 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
235 TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
236 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
237 TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
238 TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
239 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
240
241 TOPO_DBG("\nPhysical Topology Parameters:\n");
242 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
243 TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
244 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
245 TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
246 TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
247 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
248
249 topoParmsInited = TRUE;
250}
251
252static void
253x86_cache_free(x86_cpu_cache_t *cache)
254{
255 num_caches -= 1;
256 if (cache->level > 0 && cache->level <= MAX_CACHE_DEPTH)
257 num_Lx_caches[cache->level - 1] -= 1;
258 cache->next = x86_caches;
259 x86_caches = cache;
260}
261
262/*
263 * This returns a list of cache structures that represent the
264 * caches for a CPU. Some of the structures may have to be
265 * "freed" if they are actually shared between CPUs.
266 */
267static x86_cpu_cache_t *
268x86_cache_list(void)
269{
270 x86_cpu_cache_t *root = NULL;
271 x86_cpu_cache_t *cur = NULL;
272 x86_cpu_cache_t *last = NULL;
273 struct cpu_cache *cachep;
274 int i;
275
276 /*
277 * Cons up a list driven not by CPUID leaf 4 (deterministic cache params)
278 * but by the table above plus parameters already cracked from cpuid...
279 */
280 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
281
282 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0)
283 continue;
284
285 cur = x86_cache_alloc();
286 if (cur == NULL)
287 break;
288
289 cur->type = cachep->type;
290 cur->level = cachep->level;
291 cur->nlcpus = 0;
292 cur->maxcpus = cpuid_info()->cache_sharing[i];
293 cur->partitions = cpuid_info()->cache_partitions[i];
294 cur->cache_size = cpuid_info()->cache_size[i];
295 cur->line_size = cpuid_info()->cache_linesize;
296
297 if (last == NULL) {
298 root = cur;
299 last = cur;
300 } else {
301 last->next = cur;
302 last = cur;
303 }
304 num_Lx_caches[cur->level - 1] += 1;
305 }
306 return root;
307}
308
309
310static x86_cpu_cache_t *
311x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher)
312{
313 x86_cpu_cache_t *cur_cache;
314
315 cur_cache = list;
316 while (cur_cache != NULL) {
317 if (cur_cache->maxcpus == matcher->maxcpus
318 && cur_cache->type == matcher->type
319 && cur_cache->level == matcher->level
320 && cur_cache->partitions == matcher->partitions
321 && cur_cache->line_size == matcher->line_size
322 && cur_cache->cache_size == matcher->cache_size)
323 break;
324
325 cur_cache = cur_cache->next;
326 }
327
328 return(cur_cache);
329}
330
331static void
332x86_lcpu_init(int cpu)
333{
334 cpu_data_t *cpup;
335 x86_lcpu_t *lcpu;
336 int i;
337
338 cpup = cpu_datap(cpu);
339
340 lcpu = &cpup->lcpu;
341 lcpu->lcpu = lcpu;
342 lcpu->cpu = cpup;
343 lcpu->next_in_core = NULL;
344 lcpu->next_in_die = NULL;
345 lcpu->next_in_pkg = NULL;
346 lcpu->core = NULL;
347 lcpu->die = NULL;
348 lcpu->package = NULL;
349 lcpu->cpu_num = cpu;
350 lcpu->lnum = cpu;
351 lcpu->pnum = cpup->cpu_phys_number;
352 lcpu->state = LCPU_OFF;
353 for (i = 0; i < MAX_CACHE_DEPTH; i += 1)
354 lcpu->caches[i] = NULL;
355}
356
357static x86_core_t *
358x86_core_alloc(int cpu)
359{
360 x86_core_t *core;
361 cpu_data_t *cpup;
362
363 cpup = cpu_datap(cpu);
364
365 mp_safe_spin_lock(&x86_topo_lock);
366 if (free_cores != NULL) {
367 core = free_cores;
368 free_cores = core->next_in_die;
369 core->next_in_die = NULL;
370 simple_unlock(&x86_topo_lock);
371 } else {
372 simple_unlock(&x86_topo_lock);
373 core = kalloc(sizeof(x86_core_t));
374 if (core == NULL)
375 panic("x86_core_alloc() kalloc of x86_core_t failed!\n");
376 }
377
378 bzero((void *) core, sizeof(x86_core_t));
379
380 core->pcore_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
381 core->lcore_num = core->pcore_num % topoParms.nPCoresPerPackage;
382
383 core->flags = X86CORE_FL_PRESENT | X86CORE_FL_READY
384 | X86CORE_FL_HALTED | X86CORE_FL_IDLE;
385
386 return(core);
387}
388
389static void
390x86_core_free(x86_core_t *core)
391{
392 mp_safe_spin_lock(&x86_topo_lock);
393 core->next_in_die = free_cores;
394 free_cores = core;
395 simple_unlock(&x86_topo_lock);
396}
397
398static x86_pkg_t *
399x86_package_find(int cpu)
400{
401 x86_pkg_t *pkg;
402 cpu_data_t *cpup;
403 uint32_t pkg_num;
404
405 cpup = cpu_datap(cpu);
406
407 pkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
408
409 pkg = x86_pkgs;
410 while (pkg != NULL) {
411 if (pkg->ppkg_num == pkg_num)
412 break;
413 pkg = pkg->next;
414 }
415
416 return(pkg);
417}
418
419static x86_die_t *
420x86_die_find(int cpu)
421{
422 x86_die_t *die;
423 x86_pkg_t *pkg;
424 cpu_data_t *cpup;
425 uint32_t die_num;
426
427 cpup = cpu_datap(cpu);
428
429 die_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
430
431 pkg = x86_package_find(cpu);
432 if (pkg == NULL)
433 return(NULL);
434
435 die = pkg->dies;
436 while (die != NULL) {
437 if (die->pdie_num == die_num)
438 break;
439 die = die->next_in_pkg;
440 }
441
442 return(die);
443}
444
445static x86_core_t *
446x86_core_find(int cpu)
447{
448 x86_core_t *core;
449 x86_die_t *die;
450 cpu_data_t *cpup;
451 uint32_t core_num;
452
453 cpup = cpu_datap(cpu);
454
455 core_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
456
457 die = x86_die_find(cpu);
458 if (die == NULL)
459 return(NULL);
460
461 core = die->cores;
462 while (core != NULL) {
463 if (core->pcore_num == core_num)
464 break;
465 core = core->next_in_die;
466 }
467
468 return(core);
469}
470
471void
472x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum)
473{
474 x86_core_t *core = lcpu->core;
475 x86_die_t *die = lcpu->die;
476 x86_pkg_t *pkg = lcpu->package;
477
478 assert(core != NULL);
479 assert(die != NULL);
480 assert(pkg != NULL);
481
482 lcpu->cpu_num = lnum;
483 lcpu->pnum = pnum;
484 lcpu->master = (lnum == master_cpu);
485 lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0;
486
487 lcpu->lnum = lnum % topoParms.nLThreadsPerCore;
488
489 core->pcore_num = lnum / topoParms.nLThreadsPerCore;
490 core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie;
491
492 die->pdie_num = lnum / (topoParms.nLThreadsPerCore*topoParms.nLCoresPerDie);
493 die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage;
494
495 pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage;
496 pkg->lpkg_num = pkg->ppkg_num;
497
498}
499
500static x86_die_t *
501x86_die_alloc(int cpu)
502{
503 x86_die_t *die;
504 cpu_data_t *cpup;
505
506 cpup = cpu_datap(cpu);
507
508 mp_safe_spin_lock(&x86_topo_lock);
509 if (free_dies != NULL) {
510 die = free_dies;
511 free_dies = die->next_in_pkg;
512 die->next_in_pkg = NULL;
513 simple_unlock(&x86_topo_lock);
514 } else {
515 simple_unlock(&x86_topo_lock);
516 die = kalloc(sizeof(x86_die_t));
517 if (die == NULL)
518 panic("x86_die_alloc() kalloc of x86_die_t failed!\n");
519 }
520
521 bzero((void *) die, sizeof(x86_die_t));
522
523 die->pdie_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
524
525 die->ldie_num = num_dies;
526 atomic_incl((long *) &num_dies, 1);
527
528 die->flags = X86DIE_FL_PRESENT;
529 return(die);
530}
531
532static void
533x86_die_free(x86_die_t *die)
534{
535 mp_safe_spin_lock(&x86_topo_lock);
536 die->next_in_pkg = free_dies;
537 free_dies = die;
538 atomic_decl((long *) &num_dies, 1);
539 simple_unlock(&x86_topo_lock);
540}
541
542static x86_pkg_t *
543x86_package_alloc(int cpu)
544{
545 x86_pkg_t *pkg;
546 cpu_data_t *cpup;
547
548 cpup = cpu_datap(cpu);
549
550 mp_safe_spin_lock(&x86_topo_lock);
551 if (free_pkgs != NULL) {
552 pkg = free_pkgs;
553 free_pkgs = pkg->next;
554 pkg->next = NULL;
555 simple_unlock(&x86_topo_lock);
556 } else {
557 simple_unlock(&x86_topo_lock);
558 pkg = kalloc(sizeof(x86_pkg_t));
559 if (pkg == NULL)
560 panic("x86_package_alloc() kalloc of x86_pkg_t failed!\n");
561 }
562
563 bzero((void *) pkg, sizeof(x86_pkg_t));
564
565 pkg->ppkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
566
567 pkg->lpkg_num = topoParms.nPackages;
568 atomic_incl((long *) &topoParms.nPackages, 1);
569
570 pkg->flags = X86PKG_FL_PRESENT | X86PKG_FL_READY;
571 return(pkg);
572}
573
574static void
575x86_package_free(x86_pkg_t *pkg)
576{
577 mp_safe_spin_lock(&x86_topo_lock);
578 pkg->next = free_pkgs;
579 free_pkgs = pkg;
580 atomic_decl((long *) &topoParms.nPackages, 1);
581 simple_unlock(&x86_topo_lock);
582}
583
584static void
585x86_cache_add_lcpu(x86_cpu_cache_t *cache, x86_lcpu_t *lcpu)
586{
587 x86_cpu_cache_t *cur_cache;
588 int i;
589
590 /*
591 * Put the new CPU into the list of the cache.
592 */
593 cur_cache = lcpu->caches[cache->level - 1];
594 lcpu->caches[cache->level - 1] = cache;
595 cache->next = cur_cache;
596 cache->nlcpus += 1;
597 for (i = 0; i < cache->nlcpus; i += 1) {
598 if (cache->cpus[i] == NULL) {
599 cache->cpus[i] = lcpu;
600 break;
601 }
602 }
603}
604
605static void
606x86_lcpu_add_caches(x86_lcpu_t *lcpu)
607{
608 x86_cpu_cache_t *list;
609 x86_cpu_cache_t *cur;
610 x86_cpu_cache_t *match;
611 x86_die_t *die;
612 x86_core_t *core;
613 x86_lcpu_t *cur_lcpu;
614 uint32_t level;
615 boolean_t found = FALSE;
616
617 assert(lcpu != NULL);
618
619 /*
620 * Add the cache data to the topology.
621 */
622 list = x86_cache_list();
623
624 mp_safe_spin_lock(&x86_topo_lock);
625
626 while (list != NULL) {
627 /*
628 * Remove the cache from the front of the list.
629 */
630 cur = list;
631 list = cur->next;
632 cur->next = NULL;
633 level = cur->level - 1;
634
635 /*
636 * If the cache isn't shared then just put it where it
637 * belongs.
638 */
639 if (cur->maxcpus == 1) {
640 x86_cache_add_lcpu(cur, lcpu);
641 continue;
642 }
643
644 /*
645 * We'll assume that all of the caches at a particular level
646 * have the same sharing. So if we have a cache already at
647 * this level, we'll just skip looking for the match.
648 */
649 if (lcpu->caches[level] != NULL) {
650 x86_cache_free(cur);
651 continue;
652 }
653
654 /*
655 * This is a shared cache, so we have to figure out if
656 * this is the first time we've seen this cache. We do
657 * this by searching through the topology and seeing if
658 * this cache is already described.
659 *
660 * Assume that L{LLC-1} are all at the core level and that
661 * LLC is shared at the die level.
662 */
663 if (level < topoParms.LLCDepth) {
664 /*
665 * Shared at the core.
666 */
667 core = lcpu->core;
668 cur_lcpu = core->lcpus;
669 while (cur_lcpu != NULL) {
670 /*
671 * Skip ourselves.
672 */
673 if (cur_lcpu == lcpu) {
674 cur_lcpu = cur_lcpu->next_in_core;
675 continue;
676 }
677
678 /*
679 * If there's a cache on this logical CPU,
680 * then use that one.
681 */
682 match = x86_match_cache(cur_lcpu->caches[level], cur);
683 if (match != NULL) {
684 x86_cache_free(cur);
685 x86_cache_add_lcpu(match, lcpu);
686 found = TRUE;
687 break;
688 }
689
690 cur_lcpu = cur_lcpu->next_in_core;
691 }
692 } else {
693 /*
694 * Shared at the die.
695 */
696 die = lcpu->die;
697 cur_lcpu = die->lcpus;
698 while (cur_lcpu != NULL) {
699 /*
700 * Skip ourselves.
701 */
702 if (cur_lcpu == lcpu) {
703 cur_lcpu = cur_lcpu->next_in_die;
704 continue;
705 }
706
707 /*
708 * If there's a cache on this logical CPU,
709 * then use that one.
710 */
711 match = x86_match_cache(cur_lcpu->caches[level], cur);
712 if (match != NULL) {
713 x86_cache_free(cur);
714 x86_cache_add_lcpu(match, lcpu);
715 found = TRUE;
716 break;
717 }
718
719 cur_lcpu = cur_lcpu->next_in_die;
720 }
721 }
722
723 /*
724 * If a shared cache wasn't found, then this logical CPU must
725 * be the first one encountered.
726 */
727 if (!found) {
728 x86_cache_add_lcpu(cur, lcpu);
729 }
730 }
731
732 simple_unlock(&x86_topo_lock);
733}
734
735static void
736x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu)
737{
738 assert(core != NULL);
739 assert(lcpu != NULL);
740
741 mp_safe_spin_lock(&x86_topo_lock);
742
743 lcpu->next_in_core = core->lcpus;
744 lcpu->core = core;
745 core->lcpus = lcpu;
746 core->num_lcpus += 1;
747 simple_unlock(&x86_topo_lock);
748}
749
750static void
751x86_die_add_lcpu(x86_die_t *die, x86_lcpu_t *lcpu)
752{
753 assert(die != NULL);
754 assert(lcpu != NULL);
755
756 lcpu->next_in_die = die->lcpus;
757 lcpu->die = die;
758 die->lcpus = lcpu;
759}
760
761static void
762x86_die_add_core(x86_die_t *die, x86_core_t *core)
763{
764 assert(die != NULL);
765 assert(core != NULL);
766
767 core->next_in_die = die->cores;
768 core->die = die;
769 die->cores = core;
770 die->num_cores += 1;
771}
772
773 static void
774x86_package_add_lcpu(x86_pkg_t *pkg, x86_lcpu_t *lcpu)
775{
776 assert(pkg != NULL);
777 assert(lcpu != NULL);
778
779 lcpu->next_in_pkg = pkg->lcpus;
780 lcpu->package = pkg;
781 pkg->lcpus = lcpu;
782}
783
784static void
785x86_package_add_core(x86_pkg_t *pkg, x86_core_t *core)
786{
787 assert(pkg != NULL);
788 assert(core != NULL);
789
790 core->next_in_pkg = pkg->cores;
791 core->package = pkg;
792 pkg->cores = core;
793}
794
795static void
796x86_package_add_die(x86_pkg_t *pkg, x86_die_t *die)
797{
798 assert(pkg != NULL);
799 assert(die != NULL);
800
801 die->next_in_pkg = pkg->dies;
802 die->package = pkg;
803 pkg->dies = die;
804 pkg->num_dies += 1;
805}
806
807void *
808cpu_thread_alloc(int cpu)
809{
810 x86_core_t *core = NULL;
811 x86_die_t *die = NULL;
812 x86_pkg_t *pkg = NULL;
813 cpu_data_t *cpup;
814 uint32_t phys_cpu;
815
816 /*
817 * Only allow one to manipulate the topology at a time.
818 */
819 mp_safe_spin_lock(&x86_topo_lock);
820
821 /*
822 * Make sure all of the topology parameters have been initialized.
823 */
824 if (!topoParmsInited)
825 initTopoParms();
826
827 cpup = cpu_datap(cpu);
828
829 phys_cpu = cpup->cpu_phys_number;
830
831 x86_lcpu_init(cpu);
832
833 /*
834 * Assume that all cpus have the same features.
835 */
836 if (cpu_is_hyperthreaded()) {
837 cpup->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
838 } else {
839 cpup->cpu_threadtype = CPU_THREADTYPE_NONE;
840 }
841
842 /*
843 * Get the package that the logical CPU is in.
844 */
845 do {
846 pkg = x86_package_find(cpu);
847 if (pkg == NULL) {
848 /*
849 * Package structure hasn't been created yet, do it now.
850 */
851 simple_unlock(&x86_topo_lock);
852 pkg = x86_package_alloc(cpu);
853 mp_safe_spin_lock(&x86_topo_lock);
854 if (x86_package_find(cpu) != NULL) {
855 x86_package_free(pkg);
856 continue;
857 }
858
859 /*
860 * Add the new package to the global list of packages.
861 */
862 pkg->next = x86_pkgs;
863 x86_pkgs = pkg;
864 }
865 } while (pkg == NULL);
866
867 /*
868 * Get the die that the logical CPU is in.
869 */
870 do {
871 die = x86_die_find(cpu);
872 if (die == NULL) {
873 /*
874 * Die structure hasn't been created yet, do it now.
875 */
876 simple_unlock(&x86_topo_lock);
877 die = x86_die_alloc(cpu);
878 mp_safe_spin_lock(&x86_topo_lock);
879 if (x86_die_find(cpu) != NULL) {
880 x86_die_free(die);
881 continue;
882 }
883
884 /*
885 * Add the die to the package.
886 */
887 x86_package_add_die(pkg, die);
888 }
889 } while (die == NULL);
890
891 /*
892 * Get the core for this logical CPU.
893 */
894 do {
895 core = x86_core_find(cpu);
896 if (core == NULL) {
897 /*
898 * Allocate the core structure now.
899 */
900 simple_unlock(&x86_topo_lock);
901 core = x86_core_alloc(cpu);
902 mp_safe_spin_lock(&x86_topo_lock);
903 if (x86_core_find(cpu) != NULL) {
904 x86_core_free(core);
905 continue;
906 }
907
908 /*
909 * Add the core to the die & package.
910 */
911 x86_die_add_core(die, core);
912 x86_package_add_core(pkg, core);
913 machine_info.physical_cpu_max += 1;
914 }
915 } while (core == NULL);
916
917
918 /*
919 * Done manipulating the topology, so others can get in.
920 */
921 machine_info.logical_cpu_max += 1;
922 simple_unlock(&x86_topo_lock);
923
924 /*
925 * Add the logical CPU to the other topology structures.
926 */
927 x86_core_add_lcpu(core, &cpup->lcpu);
928 x86_die_add_lcpu(core->die, &cpup->lcpu);
929 x86_package_add_lcpu(core->package, &cpup->lcpu);
930 x86_lcpu_add_caches(&cpup->lcpu);
931
932 return (void *) core;
933}
934
935void
936cpu_thread_init(void)
937{
938 int my_cpu = get_cpu_number();
939 cpu_data_t *cpup = current_cpu_datap();
940 x86_core_t *core;
941 static int initialized = 0;
942
943 /*
944 * If we're the boot processor, we do all of the initialization of
945 * the CPU topology infrastructure.
946 */
947 if (my_cpu == master_cpu && !initialized) {
948 simple_lock_init(&x86_topo_lock, 0);
949
950 /*
951 * Put this logical CPU into the physical CPU topology.
952 */
953 cpup->lcpu.core = cpu_thread_alloc(my_cpu);
954
955 initialized = 1;
956 }
957
958 /*
959 * Do the CPU accounting.
960 */
961 core = cpup->lcpu.core;
962 mp_safe_spin_lock(&x86_topo_lock);
963 machine_info.logical_cpu += 1;
964 if (core->active_lcpus == 0)
965 machine_info.physical_cpu += 1;
966 core->active_lcpus += 1;
967 simple_unlock(&x86_topo_lock);
968
969 pmCPUMarkRunning(cpup);
970 timer_resync_deadlines();
971}
972
973/*
974 * Called for a cpu to halt permanently
975 * (as opposed to halting and expecting an interrupt to awaken it).
976 */
977__attribute__((noreturn))
978void
979cpu_thread_halt(void)
980{
981 x86_core_t *core;
982 cpu_data_t *cpup = current_cpu_datap();
983
984 mp_safe_spin_lock(&x86_topo_lock);
985 machine_info.logical_cpu -= 1;
986 core = cpup->lcpu.core;
987 core->active_lcpus -= 1;
988 if (core->active_lcpus == 0)
989 machine_info.physical_cpu -= 1;
990 simple_unlock(&x86_topo_lock);
991
992 /*
993 * Let the power management code determine the best way to "stop"
994 * the processor.
995 */
996 ml_set_interrupts_enabled(FALSE);
997 while (1) {
998 pmCPUHalt(PM_HALT_NORMAL);
999 }
1000 /* NOT REACHED */
1001}
1002
1003/*
1004 * Validates that the topology was built correctly. Must be called only
1005 * after the complete topology is built and no other changes are being made.
1006 */
1007void
1008x86_validate_topology(void)
1009{
1010 x86_pkg_t *pkg;
1011 x86_die_t *die;
1012 x86_core_t *core;
1013 x86_lcpu_t *lcpu;
1014 uint32_t nDies;
1015 uint32_t nCores;
1016 uint32_t nCPUs;
1017
1018 if (topo_dbg)
1019 debug_topology_print();
1020
1021 /*
1022 * Called after processors are registered but before non-boot processors
1023 * are started:
1024 * - real_ncpus: number of registered processors driven from MADT
1025 * - max_ncpus: max number of processors that will be started
1026 */
1027 nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage;
1028 if (nCPUs != real_ncpus)
1029 panic("x86_validate_topology() %d threads but %d registered from MADT",
1030 nCPUs, real_ncpus);
1031
1032 pkg = x86_pkgs;
1033 while (pkg != NULL) {
1034 /*
1035 * Make sure that the package has the correct number of dies.
1036 */
1037 nDies = 0;
1038 die = pkg->dies;
1039 while (die != NULL) {
1040 if (die->package == NULL)
1041 panic("Die(%d)->package is NULL",
1042 die->pdie_num);
1043 if (die->package != pkg)
1044 panic("Die %d points to package %d, should be %d",
1045 die->pdie_num, die->package->lpkg_num, pkg->lpkg_num);
1046
1047 TOPO_DBG("Die(%d)->package %d\n",
1048 die->pdie_num, pkg->lpkg_num);
1049
1050 /*
1051 * Make sure that the die has the correct number of cores.
1052 */
1053 TOPO_DBG("Die(%d)->cores: ", die->pdie_num);
1054 nCores = 0;
1055 core = die->cores;
1056 while (core != NULL) {
1057 if (core->die == NULL)
1058 panic("Core(%d)->die is NULL",
1059 core->pcore_num);
1060 if (core->die != die)
1061 panic("Core %d points to die %d, should be %d",
1062 core->pcore_num, core->die->pdie_num, die->pdie_num);
1063 nCores += 1;
1064 TOPO_DBG("%d ", core->pcore_num);
1065 core = core->next_in_die;
1066 }
1067 TOPO_DBG("\n");
1068
1069 if (nCores != topoParms.nLCoresPerDie)
1070 panic("Should have %d Cores, but only found %d for Die %d",
1071 topoParms.nLCoresPerDie, nCores, die->pdie_num);
1072
1073 /*
1074 * Make sure that the die has the correct number of CPUs.
1075 */
1076 TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num);
1077 nCPUs = 0;
1078 lcpu = die->lcpus;
1079 while (lcpu != NULL) {
1080 if (lcpu->die == NULL)
1081 panic("CPU(%d)->die is NULL",
1082 lcpu->cpu_num);
1083 if (lcpu->die != die)
1084 panic("CPU %d points to die %d, should be %d",
1085 lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num);
1086 nCPUs += 1;
1087 TOPO_DBG("%d ", lcpu->cpu_num);
1088 lcpu = lcpu->next_in_die;
1089 }
1090 TOPO_DBG("\n");
1091
1092 if (nCPUs != topoParms.nLThreadsPerDie)
1093 panic("Should have %d Threads, but only found %d for Die %d",
1094 topoParms.nLThreadsPerDie, nCPUs, die->pdie_num);
1095
1096 nDies += 1;
1097 die = die->next_in_pkg;
1098 }
1099
1100 if (nDies != topoParms.nLDiesPerPackage)
1101 panic("Should have %d Dies, but only found %d for package %d",
1102 topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num);
1103
1104 /*
1105 * Make sure that the package has the correct number of cores.
1106 */
1107 nCores = 0;
1108 core = pkg->cores;
1109 while (core != NULL) {
1110 if (core->package == NULL)
1111 panic("Core(%d)->package is NULL",
1112 core->pcore_num);
1113 if (core->package != pkg)
1114 panic("Core %d points to package %d, should be %d",
1115 core->pcore_num, core->package->lpkg_num, pkg->lpkg_num);
1116 TOPO_DBG("Core(%d)->package %d\n",
1117 core->pcore_num, pkg->lpkg_num);
1118
1119 /*
1120 * Make sure that the core has the correct number of CPUs.
1121 */
1122 nCPUs = 0;
1123 lcpu = core->lcpus;
1124 TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num);
1125 while (lcpu != NULL) {
1126 if (lcpu->core == NULL)
1127 panic("CPU(%d)->core is NULL",
1128 lcpu->cpu_num);
1129 if (lcpu->core != core)
1130 panic("CPU %d points to core %d, should be %d",
1131 lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num);
1132 TOPO_DBG("%d ", lcpu->cpu_num);
1133 nCPUs += 1;
1134 lcpu = lcpu->next_in_core;
1135 }
1136 TOPO_DBG("\n");
1137
1138 if (nCPUs != topoParms.nLThreadsPerCore)
1139 panic("Should have %d Threads, but only found %d for Core %d",
1140 topoParms.nLThreadsPerCore, nCPUs, core->pcore_num);
1141 nCores += 1;
1142 core = core->next_in_pkg;
1143 }
1144
1145 if (nCores != topoParms.nLCoresPerPackage)
1146 panic("Should have %d Cores, but only found %d for package %d",
1147 topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num);
1148
1149 /*
1150 * Make sure that the package has the correct number of CPUs.
1151 */
1152 nCPUs = 0;
1153 lcpu = pkg->lcpus;
1154 while (lcpu != NULL) {
1155 if (lcpu->package == NULL)
1156 panic("CPU(%d)->package is NULL",
1157 lcpu->cpu_num);
1158 if (lcpu->package != pkg)
1159 panic("CPU %d points to package %d, should be %d",
1160 lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num);
1161 TOPO_DBG("CPU(%d)->package %d\n",
1162 lcpu->cpu_num, pkg->lpkg_num);
1163 nCPUs += 1;
1164 lcpu = lcpu->next_in_pkg;
1165 }
1166
1167 if (nCPUs != topoParms.nLThreadsPerPackage)
1168 panic("Should have %d Threads, but only found %d for package %d",
1169 topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num);
1170
1171 pkg = pkg->next;
1172 }
1173}
1174
1175/*
1176 * Prints out the topology
1177 */
1178static void
1179debug_topology_print(void)
1180{
1181 x86_pkg_t *pkg;
1182 x86_die_t *die;
1183 x86_core_t *core;
1184 x86_lcpu_t *cpu;
1185
1186 pkg = x86_pkgs;
1187 while (pkg != NULL) {
1188 kprintf("Package:\n");
1189 kprintf(" Physical: %d\n", pkg->ppkg_num);
1190 kprintf(" Logical: %d\n", pkg->lpkg_num);
1191
1192 die = pkg->dies;
1193 while (die != NULL) {
1194 kprintf(" Die:\n");
1195 kprintf(" Physical: %d\n", die->pdie_num);
1196 kprintf(" Logical: %d\n", die->ldie_num);
1197
1198 core = die->cores;
1199 while (core != NULL) {
1200 kprintf(" Core:\n");
1201 kprintf(" Physical: %d\n", core->pcore_num);
1202 kprintf(" Logical: %d\n", core->lcore_num);
1203
1204 cpu = core->lcpus;
1205 while (cpu != NULL) {
1206 kprintf(" LCPU:\n");
1207 kprintf(" CPU #: %d\n", cpu->cpu_num);
1208 kprintf(" Physical: %d\n", cpu->pnum);
1209 kprintf(" Logical: %d\n", cpu->lnum);
1210 kprintf(" Flags: ");
1211 if (cpu->master)
1212 kprintf("MASTER ");
1213 if (cpu->primary)
1214 kprintf("PRIMARY");
1215 if (!cpu->master && !cpu->primary)
1216 kprintf("(NONE)");
1217 kprintf("\n");
1218
1219 cpu = cpu->next_in_core;
1220 }
1221
1222 core = core->next_in_die;
1223 }
1224
1225 die = die->next_in_pkg;
1226 }
1227
1228 pkg = pkg->next;
1229 }
1230}
1231