1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58#include <mach_assert.h>
59
60#include <string.h>
61#include <mach/boolean.h>
62#include <mach/i386/vm_types.h>
63#include <mach/i386/vm_param.h>
64#include <kern/kern_types.h>
65#include <kern/misc_protos.h>
66#include <sys/errno.h>
67#include <i386/param.h>
68#include <i386/misc_protos.h>
69#include <i386/cpu_data.h>
70#include <i386/machine_routines.h>
71#include <i386/cpuid.h>
72#include <i386/vmx.h>
73#include <vm/pmap.h>
74#include <vm/vm_map.h>
75#include <vm/vm_kern.h>
76#include <vm/vm_fault.h>
77
78#include <libkern/OSAtomic.h>
79#include <sys/kdebug.h>
80
81#if !MACH_KDP
82#include <kdp/kdp_callout.h>
83#endif /* !MACH_KDP */
84
85#include <libkern/OSDebug.h>
86#if CONFIG_DTRACE
87#include <mach/sdt.h>
88#endif
89
90#if 0
91
92#undef KERNEL_DEBUG
93#define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT
94#define KDEBUG 1
95
96#endif
97
98/* prevent infinite recursion when memmove calls bcopy; in string.h, bcopy is defined to call memmove */
99#undef bcopy
100
101/* XXX - should be gone from here */
102extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
103extern void flush_dcache64(addr64_t addr, unsigned count, int phys);
104extern boolean_t phys_page_exists(ppnum_t);
105extern void bcopy_no_overwrite(const char *from, char *to,vm_size_t bytes);
106extern void pmap_set_reference(ppnum_t pn);
107extern void mapping_set_mod(ppnum_t pa);
108extern void mapping_set_ref(ppnum_t pn);
109
110extern void ovbcopy(const char *from,
111 char *to,
112 vm_size_t nbytes);
113void machine_callstack(uintptr_t *buf, vm_size_t callstack_max);
114
115
116#define value_64bit(value) ((value) & 0xFFFFFFFF00000000ULL)
117#define low32(x) ((unsigned int)((x) & 0x00000000FFFFFFFFULL))
118
119#define INT_SIZE (BYTE_SIZE * sizeof (int))
120
121/*
122 * Set indicated bit in bit string.
123 */
124void
125setbit(int bitno, int *s)
126{
127 s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE);
128}
129
130/*
131 * Clear indicated bit in bit string.
132 */
133void
134clrbit(int bitno, int *s)
135{
136 s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE));
137}
138
139/*
140 * Test if indicated bit is set in bit string.
141 */
142int
143testbit(int bitno, int *s)
144{
145 return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE));
146}
147
148/*
149 * Find first bit set in bit string.
150 */
151int
152ffsbit(int *s)
153{
154 int offset;
155
156 for (offset = 0; !*s; offset += (int)INT_SIZE, ++s);
157 return offset + __builtin_ctz(*s);
158}
159
160int
161ffs(unsigned int mask)
162{
163 if (mask == 0)
164 return 0;
165
166 /*
167 * NOTE: cannot use __builtin_ffs because it generates a call to
168 * 'ffs'
169 */
170 return 1 + __builtin_ctz(mask);
171}
172
173int
174ffsll(unsigned long long mask)
175{
176 if (mask == 0)
177 return 0;
178
179 /*
180 * NOTE: cannot use __builtin_ffsll because it generates a call to
181 * 'ffsll'
182 */
183 return 1 + __builtin_ctzll(mask);
184}
185
186/*
187 * Find last bit set in bit string.
188 */
189int
190fls(unsigned int mask)
191{
192 if (mask == 0)
193 return 0;
194
195 return (sizeof (mask) << 3) - __builtin_clz(mask);
196}
197
198int
199flsll(unsigned long long mask)
200{
201 if (mask == 0)
202 return 0;
203
204 return (sizeof (mask) << 3) - __builtin_clzll(mask);
205}
206
207void
208bzero_phys_nc(
209 addr64_t src64,
210 uint32_t bytes)
211{
212 bzero_phys(src64,bytes);
213}
214
215void
216bzero_phys(
217 addr64_t src64,
218 uint32_t bytes)
219{
220 bzero(PHYSMAP_PTOV(src64), bytes);
221}
222
223
224/*
225 * bcopy_phys - like bcopy but copies from/to physical addresses.
226 */
227
228void
229bcopy_phys(
230 addr64_t src64,
231 addr64_t dst64,
232 vm_size_t bytes)
233{
234 /* Not necessary for K64 - but ensure we stay within a page */
235 if (((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) ||
236 ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
237 panic("bcopy_phys alignment");
238 }
239 bcopy(PHYSMAP_PTOV(src64), PHYSMAP_PTOV(dst64), bytes);
240}
241
242/*
243 * allow a function to get a quick virtual mapping of a physical page
244 */
245
246int
247apply_func_phys(
248 addr64_t dst64,
249 vm_size_t bytes,
250 int (*func)(void * buffer, vm_size_t bytes, void * arg),
251 void * arg)
252{
253 /* Not necessary for K64 - but ensure we stay within a page */
254 if (((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
255 panic("apply_func_phys alignment");
256 }
257
258 return func(PHYSMAP_PTOV(dst64), bytes, arg);
259}
260
261/*
262 * ovbcopy - like bcopy, but recognizes overlapping ranges and handles
263 * them correctly.
264 */
265
266void
267ovbcopy(
268 const char *from,
269 char *to,
270 vm_size_t bytes) /* num bytes to copy */
271{
272 /* Assume that bcopy copies left-to-right (low addr first). */
273 if (from + bytes <= to || to + bytes <= from || to == from)
274 bcopy_no_overwrite(from, to, bytes); /* non-overlapping or no-op*/
275 else if (from > to)
276 bcopy_no_overwrite(from, to, bytes); /* overlapping but OK */
277 else {
278 /* to > from: overlapping, and must copy right-to-left. */
279 from += bytes - 1;
280 to += bytes - 1;
281 while (bytes-- > 0)
282 *to-- = *from--;
283 }
284}
285
286
287/*
288 * Read data from a physical address. Memory should not be cache inhibited.
289 */
290
291uint64_t reportphyreaddelayabs;
292uint32_t reportphyreadosbt;
293
294#if DEVELOPMENT || DEBUG
295uint32_t phyreadpanic = 1;
296#else
297uint32_t phyreadpanic = 0;
298#endif
299
300__private_extern__ uint64_t
301ml_phys_read_data(pmap_paddr_t paddr, int size) {
302 uint64_t result = 0;
303 unsigned char s1;
304 unsigned short s2;
305 boolean_t istate = TRUE, timeread = FALSE;
306 uint64_t sabs = 0, eabs;
307
308 if (__improbable(!physmap_enclosed(paddr)))
309 panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
310
311 if (__improbable(reportphyreaddelayabs != 0)) {
312 istate = ml_set_interrupts_enabled(FALSE);
313 sabs = mach_absolute_time();
314 timeread = TRUE;
315 }
316
317 switch (size) {
318 case 1:
319 s1 = *(volatile unsigned char *)PHYSMAP_PTOV(paddr);
320 result = s1;
321 break;
322 case 2:
323 s2 = *(volatile unsigned short *)PHYSMAP_PTOV(paddr);
324 result = s2;
325 break;
326 case 4:
327 result = *(volatile unsigned int *)PHYSMAP_PTOV(paddr);
328 break;
329 case 8:
330 result = *(volatile unsigned long long *)PHYSMAP_PTOV(paddr);
331 break;
332 default:
333 panic("Invalid size %d for ml_phys_read_data\n", size);
334 break;
335 }
336
337 if (__improbable(timeread == TRUE)) {
338 eabs = mach_absolute_time();
339 (void)ml_set_interrupts_enabled(istate);
340
341 if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
342 if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
343 panic_io_port_read();
344 panic("Read from physical addr 0x%llx took %llu ns, result: 0x%llx (start: %llu, end: %llu), ceiling: %llu", paddr, (eabs - sabs), result, sabs, eabs, reportphyreaddelayabs);
345 }
346
347 if (reportphyreadosbt) {
348 OSReportWithBacktrace("ml_phys_read_data took %lluus\n", (eabs - sabs) / 1000);
349 }
350#if CONFIG_DTRACE
351 DTRACE_PHYSLAT3(physread, uint64_t, (eabs - sabs),
352 pmap_paddr_t, paddr, uint32_t, size);
353#endif
354 }
355 }
356
357 return result;
358}
359
360static unsigned long long
361ml_phys_read_long_long(pmap_paddr_t paddr) {
362 return ml_phys_read_data(paddr, 8);
363}
364
365unsigned int ml_phys_read( vm_offset_t paddr)
366{
367 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr, 4);
368}
369
370unsigned int ml_phys_read_word(vm_offset_t paddr) {
371
372 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr, 4);
373}
374
375unsigned int ml_phys_read_64(addr64_t paddr64)
376{
377 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr64, 4);
378}
379
380unsigned int ml_phys_read_word_64(addr64_t paddr64)
381{
382 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr64, 4);
383}
384
385unsigned int ml_phys_read_half(vm_offset_t paddr)
386{
387 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr, 2);
388}
389
390unsigned int ml_phys_read_half_64(addr64_t paddr64)
391{
392 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr64, 2);
393}
394
395unsigned int ml_phys_read_byte(vm_offset_t paddr)
396{
397 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr, 1);
398}
399
400unsigned int ml_phys_read_byte_64(addr64_t paddr64)
401{
402 return (unsigned int) ml_phys_read_data((pmap_paddr_t)paddr64, 1);
403}
404
405unsigned long long ml_phys_read_double(vm_offset_t paddr)
406{
407 return ml_phys_read_long_long((pmap_paddr_t)paddr);
408}
409
410unsigned long long ml_phys_read_double_64(addr64_t paddr64)
411{
412 return ml_phys_read_long_long((pmap_paddr_t)paddr64);
413}
414
415
416
417/*
418 * Write data to a physical address. Memory should not be cache inhibited.
419 */
420
421static inline void
422ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
423{
424 if (!physmap_enclosed(paddr))
425 panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
426
427 switch (size) {
428 case 1:
429 *(volatile unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data;
430 break;
431 case 2:
432 *(volatile unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data;
433 break;
434 case 4:
435 *(volatile unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data;
436 break;
437 default:
438 panic("Invalid size %d for ml_phys_write_data\n", size);
439 break;
440 }
441}
442
443static void
444ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
445{
446 if (!physmap_enclosed(paddr))
447 panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
448
449 *(volatile unsigned long long *)PHYSMAP_PTOV(paddr) = data;
450}
451
452void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
453{
454 ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
455}
456
457void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
458{
459 ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
460}
461
462void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
463{
464 ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
465}
466
467void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
468{
469 ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
470}
471
472void ml_phys_write(vm_offset_t paddr, unsigned int data)
473{
474 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
475}
476
477void ml_phys_write_64(addr64_t paddr64, unsigned int data)
478{
479 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
480}
481
482void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
483{
484 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
485}
486
487void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
488{
489 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
490}
491
492void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
493{
494 ml_phys_write_long_long((pmap_paddr_t)paddr, data);
495}
496
497void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
498{
499 ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
500}
501
502
503/* PCI config cycle probing
504 *
505 *
506 * Read the memory location at physical address paddr.
507 * *Does not* recover from machine checks, unlike the PowerPC implementation.
508 * Should probably be deprecated.
509 */
510
511boolean_t
512ml_probe_read(vm_offset_t paddr, unsigned int *val)
513{
514 if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4)
515 return FALSE;
516
517 *val = ml_phys_read((pmap_paddr_t)paddr);
518
519 return TRUE;
520}
521
522/*
523 * Read the memory location at physical address paddr.
524 * This is a part of a device probe, so there is a good chance we will
525 * have a machine check here. So we have to be able to handle that.
526 * We assume that machine checks are enabled both in MSR and HIDs
527 */
528boolean_t
529ml_probe_read_64(addr64_t paddr64, unsigned int *val)
530{
531 if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4)
532 return FALSE;
533
534 *val = ml_phys_read_64((pmap_paddr_t)paddr64);
535 return TRUE;
536}
537
538
539#undef bcmp
540int bcmp(
541 const void *pa,
542 const void *pb,
543 size_t len)
544{
545 const char *a = (const char *)pa;
546 const char *b = (const char *)pb;
547
548 if (len == 0)
549 return 0;
550
551 do
552 if (*a++ != *b++)
553 break;
554 while (--len);
555
556 return (int)len;
557}
558
559#undef memcmp
560int
561memcmp(const void *s1, const void *s2, size_t n)
562{
563 if (n != 0) {
564 const unsigned char *p1 = s1, *p2 = s2;
565
566 do {
567 if (*p1++ != *p2++)
568 return (*--p1 - *--p2);
569 } while (--n != 0);
570 }
571 return (0);
572}
573
574#undef memmove
575void *
576memmove(void *dst, const void *src, size_t ulen)
577{
578 bcopy(src, dst, ulen);
579 return dst;
580}
581
582/*
583 * Abstract:
584 * strlen returns the number of characters in "string" preceeding
585 * the terminating null character.
586 */
587
588#undef strlen
589size_t
590strlen(
591 const char *string)
592{
593 const char *ret = string;
594
595 while (*string++ != '\0')
596 continue;
597 return string - 1 - ret;
598}
599
600#if MACH_ASSERT
601
602/*
603 * Machine-dependent routine to fill in an array with up to callstack_max
604 * levels of return pc information.
605 */
606void machine_callstack(
607 __unused uintptr_t *buf,
608 __unused vm_size_t callstack_max)
609{
610}
611
612#endif /* MACH_ASSERT */
613
614void fillPage(ppnum_t pa, unsigned int fill)
615{
616 pmap_paddr_t src;
617 int i;
618 int cnt = PAGE_SIZE / sizeof(unsigned int);
619 unsigned int *addr;
620
621 src = i386_ptob(pa);
622 for (i = 0, addr = (unsigned int *)PHYSMAP_PTOV(src); i < cnt; i++)
623 *addr++ = fill;
624}
625
626static inline void __clflush(void *ptr)
627{
628 __asm__ volatile("clflush (%0)" : : "r" (ptr));
629}
630
631void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
632{
633 addr64_t linesize = cpuid_info()->cache_linesize;
634 addr64_t bound = (pa + count + linesize - 1) & ~(linesize - 1);
635
636 mfence();
637
638 while (pa < bound) {
639 __clflush(PHYSMAP_PTOV(pa));
640 pa += linesize;
641 }
642
643 mfence();
644}
645
646void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)
647{
648 return(dcache_incoherent_io_store64(pa,count));
649}
650
651void
652flush_dcache64(addr64_t addr, unsigned count, int phys)
653{
654 if (phys) {
655 dcache_incoherent_io_flush64(addr, count);
656 }
657 else {
658 uint64_t linesize = cpuid_info()->cache_linesize;
659 addr64_t bound = (addr + count + linesize -1) & ~(linesize - 1);
660 mfence();
661 while (addr < bound) {
662 __clflush((void *) (uintptr_t) addr);
663 addr += linesize;
664 }
665 mfence();
666 }
667}
668
669void
670invalidate_icache64(__unused addr64_t addr,
671 __unused unsigned count,
672 __unused int phys)
673{
674}
675
676
677addr64_t vm_last_addr;
678
679void
680mapping_set_mod(ppnum_t pn)
681{
682 pmap_set_modify(pn);
683}
684
685void
686mapping_set_ref(ppnum_t pn)
687{
688 pmap_set_reference(pn);
689}
690
691extern i386_cpu_info_t cpuid_cpu_info;
692void
693cache_flush_page_phys(ppnum_t pa)
694{
695 boolean_t istate;
696 unsigned char *cacheline_addr;
697 i386_cpu_info_t *cpuid_infop = cpuid_info();
698 int cacheline_size;
699 int cachelines_to_flush;
700
701 cacheline_size = cpuid_infop->cache_linesize;
702 if (cacheline_size == 0)
703 panic("cacheline_size=0 cpuid_infop=%p\n", cpuid_infop);
704 cachelines_to_flush = PAGE_SIZE/cacheline_size;
705
706 mfence();
707
708 istate = ml_set_interrupts_enabled(FALSE);
709
710 for (cacheline_addr = (unsigned char *)PHYSMAP_PTOV(i386_ptob(pa));
711 cachelines_to_flush > 0;
712 cachelines_to_flush--, cacheline_addr += cacheline_size) {
713 __clflush((void *) cacheline_addr);
714 }
715
716 (void) ml_set_interrupts_enabled(istate);
717
718 mfence();
719}
720
721
722#if !MACH_KDP
723void
724kdp_register_callout(kdp_callout_fn_t fn, void *arg)
725{
726#pragma unused(fn,arg)
727}
728#endif
729
730#if !CONFIG_VMX
731int host_vmxon(boolean_t exclusive __unused)
732{
733 return VMX_UNSUPPORTED;
734}
735
736void host_vmxoff(void)
737{
738 return;
739}
740#endif
741uint32_t xcpm_bios_mbox_cmd_read(uint32_t a); static uint32_t __xcpm_reg[64] = {};
742uint32_t xcpm_bios_mbox_cmd_read(uint32_t a) { return __xcpm_reg[a%64]; }
743void xcpm_bios_mbox_cmd_write(uint32_t a, uint32_t b);
744void xcpm_bios_mbox_cmd_write(uint32_t a, uint32_t b) { __xcpm_reg[a%64] = b; }
745boolean_t xcpm_is_hwp_enabled(void);
746boolean_t xcpm_is_hwp_enabled(void) { return 0; }
747void xcpm_mbox_lock(void);
748void xcpm_mbox_lock(void) {}
749