1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach_assert.h>
30
31#include <vm/pmap.h>
32#include <vm/vm_map.h>
33#include <vm/vm_kern.h>
34#include <kern/ledger.h>
35#include <i386/pmap_internal.h>
36
37void pmap_remove_range(
38 pmap_t pmap,
39 vm_map_offset_t va,
40 pt_entry_t *spte,
41 pt_entry_t *epte);
42
43void pmap_remove_range_options(
44 pmap_t pmap,
45 vm_map_offset_t va,
46 pt_entry_t *spte,
47 pt_entry_t *epte,
48 int options);
49
50void pmap_reusable_range(
51 pmap_t pmap,
52 vm_map_offset_t va,
53 pt_entry_t *spte,
54 pt_entry_t *epte,
55 boolean_t reusable);
56
57uint32_t pmap_update_clear_pte_count;
58
59/*
60 * The Intel platform can nest at the PDE level, so NBPDE (i.e. 2MB) at a time,
61 * on a NBPDE boundary.
62 */
63
64/* These symbols may be referenced directly by VM */
65uint64_t pmap_nesting_size_min = NBPDE;
66uint64_t pmap_nesting_size_max = 0 - (uint64_t)NBPDE;
67
68/*
69 * kern_return_t pmap_nest(grand, subord, va_start, size)
70 *
71 * grand = the pmap that we will nest subord into
72 * subord = the pmap that goes into the grand
73 * va_start = start of range in pmap to be inserted
74 * nstart = start of range in pmap nested pmap
75 * size = Size of nest area (up to 16TB)
76 *
77 * Inserts a pmap into another. This is used to implement shared segments.
78 *
79 * Note that we depend upon higher level VM locks to insure that things don't change while
80 * we are doing this. For example, VM should not be doing any pmap enters while it is nesting
81 * or do 2 nests at once.
82 */
83
84/*
85 * This routine can nest subtrees either at the PDPT level (1GiB) or at the
86 * PDE level (2MiB). We currently disallow disparate offsets for the "subord"
87 * container and the "grand" parent. A minor optimization to consider for the
88 * future: make the "subord" truly a container rather than a full-fledged
89 * pagetable hierarchy which can be unnecessarily sparse (DRK).
90 */
91
92kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t nstart, uint64_t size) {
93 vm_map_offset_t vaddr, nvaddr;
94 pd_entry_t *pde,*npde;
95 unsigned int i;
96 uint64_t num_pde;
97
98 assert(!is_ept_pmap(grand));
99 assert(!is_ept_pmap(subord));
100
101 if ((size & (pmap_nesting_size_min-1)) ||
102 (va_start & (pmap_nesting_size_min-1)) ||
103 (nstart & (pmap_nesting_size_min-1)) ||
104 ((size >> 28) > 65536)) /* Max size we can nest is 16TB */
105 return KERN_INVALID_VALUE;
106
107 if(size == 0) {
108 panic("pmap_nest: size is invalid - %016llX\n", size);
109 }
110
111 if (va_start != nstart)
112 panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);
113
114 PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
115 VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
116 VM_KERNEL_ADDRHIDE(va_start));
117
118 nvaddr = (vm_map_offset_t)nstart;
119 num_pde = size >> PDESHIFT;
120
121 PMAP_LOCK(subord);
122
123 subord->pm_shared = TRUE;
124
125 for (i = 0; i < num_pde;) {
126 if (((nvaddr & PDPTMASK) == 0) && (num_pde - i) >= NPDEPG && cpu_64bit) {
127
128 npde = pmap64_pdpt(subord, nvaddr);
129
130 while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
131 PMAP_UNLOCK(subord);
132 pmap_expand_pdpt(subord, nvaddr, PMAP_EXPAND_OPTIONS_NONE);
133 PMAP_LOCK(subord);
134 npde = pmap64_pdpt(subord, nvaddr);
135 }
136 *npde |= INTEL_PDPTE_NESTED;
137 nvaddr += NBPDPT;
138 i += (uint32_t)NPDEPG;
139 }
140 else {
141 npde = pmap_pde(subord, nvaddr);
142
143 while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
144 PMAP_UNLOCK(subord);
145 pmap_expand(subord, nvaddr, PMAP_EXPAND_OPTIONS_NONE);
146 PMAP_LOCK(subord);
147 npde = pmap_pde(subord, nvaddr);
148 }
149 nvaddr += NBPDE;
150 i++;
151 }
152 }
153
154 PMAP_UNLOCK(subord);
155
156 vaddr = (vm_map_offset_t)va_start;
157
158 PMAP_LOCK(grand);
159
160 for (i = 0;i < num_pde;) {
161 pd_entry_t tpde;
162
163 if (((vaddr & PDPTMASK) == 0) && ((num_pde - i) >= NPDEPG) && cpu_64bit) {
164 npde = pmap64_pdpt(subord, vaddr);
165 if (npde == 0)
166 panic("pmap_nest: no PDPT, subord %p nstart 0x%llx", subord, vaddr);
167 tpde = *npde;
168 pde = pmap64_pdpt(grand, vaddr);
169 if (0 == pde) {
170 PMAP_UNLOCK(grand);
171 pmap_expand_pml4(grand, vaddr, PMAP_EXPAND_OPTIONS_NONE);
172 PMAP_LOCK(grand);
173 pde = pmap64_pdpt(grand, vaddr);
174 }
175 if (pde == 0)
176 panic("pmap_nest: no PDPT, grand %p vaddr 0x%llx", grand, vaddr);
177 pmap_store_pte(pde, tpde);
178 vaddr += NBPDPT;
179 i += (uint32_t) NPDEPG;
180 }
181 else {
182 npde = pmap_pde(subord, vaddr);
183 if (npde == 0)
184 panic("pmap_nest: no npde, subord %p vaddr 0x%llx", subord, vaddr);
185 tpde = *npde;
186 pde = pmap_pde(grand, vaddr);
187 if ((0 == pde) && cpu_64bit) {
188 PMAP_UNLOCK(grand);
189 pmap_expand_pdpt(grand, vaddr, PMAP_EXPAND_OPTIONS_NONE);
190 PMAP_LOCK(grand);
191 pde = pmap_pde(grand, vaddr);
192 }
193
194 if (pde == 0)
195 panic("pmap_nest: no pde, grand %p vaddr 0x%llx", grand, vaddr);
196 vaddr += NBPDE;
197 pmap_store_pte(pde, tpde);
198 i++;
199 }
200 }
201
202 PMAP_UNLOCK(grand);
203
204 PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, KERN_SUCCESS);
205
206 return KERN_SUCCESS;
207}
208
209/*
210 * kern_return_t pmap_unnest(grand, vaddr)
211 *
212 * grand = the pmap that we will un-nest subord from
213 * vaddr = start of range in pmap to be unnested
214 *
215 * Removes a pmap from another. This is used to implement shared segments.
216 */
217
218kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
219 pd_entry_t *pde;
220 unsigned int i;
221 uint64_t num_pde;
222 addr64_t va_start, va_end;
223 uint64_t npdpt = PMAP_INVALID_PDPTNUM;
224
225 PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
226 VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));
227
228 if ((size & (pmap_nesting_size_min-1)) ||
229 (vaddr & (pmap_nesting_size_min-1))) {
230 panic("pmap_unnest(%p,0x%llx,0x%llx): unaligned...\n",
231 grand, vaddr, size);
232 }
233
234 assert(!is_ept_pmap(grand));
235
236 /* align everything to PDE boundaries */
237 va_start = vaddr & ~(NBPDE-1);
238 va_end = (vaddr + size + NBPDE - 1) & ~(NBPDE-1);
239 size = va_end - va_start;
240
241 PMAP_LOCK(grand);
242
243 num_pde = size >> PDESHIFT;
244 vaddr = va_start;
245
246 for (i = 0; i < num_pde; ) {
247 if ((pdptnum(grand, vaddr) != npdpt) && cpu_64bit) {
248 npdpt = pdptnum(grand, vaddr);
249 pde = pmap64_pdpt(grand, vaddr);
250 if (pde && (*pde & INTEL_PDPTE_NESTED)) {
251 pmap_store_pte(pde, (pd_entry_t)0);
252 i += (uint32_t) NPDEPG;
253 vaddr += NBPDPT;
254 continue;
255 }
256 }
257 pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
258 if (pde == 0)
259 panic("pmap_unnest: no pde, grand %p vaddr 0x%llx\n", grand, vaddr);
260 pmap_store_pte(pde, (pd_entry_t)0);
261 i++;
262 vaddr += NBPDE;
263 }
264
265 PMAP_UPDATE_TLBS(grand, va_start, va_end);
266
267 PMAP_UNLOCK(grand);
268
269 PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, KERN_SUCCESS);
270
271 return KERN_SUCCESS;
272}
273
274kern_return_t
275pmap_unnest_options(
276 pmap_t grand,
277 addr64_t vaddr,
278 __unused uint64_t size,
279 __unused unsigned int options) {
280 return pmap_unnest(grand, vaddr, size);
281}
282
283/* Invoked by the Mach VM to determine the platform specific unnest region */
284
285boolean_t pmap_adjust_unnest_parameters(pmap_t p, vm_map_offset_t *s, vm_map_offset_t *e) {
286 pd_entry_t *pdpte;
287 boolean_t rval = FALSE;
288
289 if (!cpu_64bit)
290 return rval;
291
292 PMAP_LOCK(p);
293
294 pdpte = pmap64_pdpt(p, *s);
295 if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
296 *s &= ~(NBPDPT -1);
297 rval = TRUE;
298 }
299
300 pdpte = pmap64_pdpt(p, *e);
301 if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
302 *e = ((*e + NBPDPT) & ~(NBPDPT -1));
303 rval = TRUE;
304 }
305
306 PMAP_UNLOCK(p);
307
308 return rval;
309}
310
311/*
312 * pmap_find_phys returns the (4K) physical page number containing a
313 * given virtual address in a given pmap.
314 * Note that pmap_pte may return a pde if this virtual address is
315 * mapped by a large page and this is taken into account in order
316 * to return the correct page number in this case.
317 */
318ppnum_t
319pmap_find_phys(pmap_t pmap, addr64_t va)
320{
321 pt_entry_t *ptp;
322 pd_entry_t *pdep;
323 ppnum_t ppn = 0;
324 pd_entry_t pde;
325 pt_entry_t pte;
326 boolean_t is_ept;
327
328 is_ept = is_ept_pmap(pmap);
329
330 mp_disable_preemption();
331
332 /* This refcount test is a band-aid--several infrastructural changes
333 * are necessary to eliminate invocation of this routine from arbitrary
334 * contexts.
335 */
336
337 if (!pmap->ref_count)
338 goto pfp_exit;
339
340 pdep = pmap_pde(pmap, va);
341
342 if ((pdep != PD_ENTRY_NULL) && ((pde = *pdep) & PTE_VALID_MASK(is_ept))) {
343 if (pde & PTE_PS) {
344 ppn = (ppnum_t) i386_btop(pte_to_pa(pde));
345 ppn += (ppnum_t) ptenum(va);
346 }
347 else {
348 ptp = pmap_pte(pmap, va);
349 if ((PT_ENTRY_NULL != ptp) && (((pte = *ptp) & PTE_VALID_MASK(is_ept)) != 0)) {
350 ppn = (ppnum_t) i386_btop(pte_to_pa(pte));
351 }
352 }
353 }
354pfp_exit:
355 mp_enable_preemption();
356
357 return ppn;
358}
359
360/*
361 * Update cache attributes for all extant managed mappings.
362 * Assumes PV for this page is locked, and that the page
363 * is managed. We assume that this physical page may be mapped in
364 * both EPT and normal Intel PTEs, so we convert the attributes
365 * to the corresponding format for each pmap.
366 *
367 * We assert that the passed set of attributes is a subset of the
368 * PHYS_CACHEABILITY_MASK.
369 */
370void
371pmap_update_cache_attributes_locked(ppnum_t pn, unsigned attributes) {
372 pv_rooted_entry_t pv_h, pv_e;
373 pv_hashed_entry_t pvh_e, nexth;
374 vm_map_offset_t vaddr;
375 pmap_t pmap;
376 pt_entry_t *ptep;
377 boolean_t is_ept;
378 unsigned ept_attributes;
379
380 assert(IS_MANAGED_PAGE(pn));
381 assert(((~PHYS_CACHEABILITY_MASK) & attributes) == 0);
382
383 /* We don't support the PTA bit for EPT PTEs */
384 if (attributes & INTEL_PTE_NCACHE)
385 ept_attributes = INTEL_EPT_NCACHE;
386 else
387 ept_attributes = INTEL_EPT_WB;
388
389 pv_h = pai_to_pvh(pn);
390 /* TODO: translate the PHYS_* bits to PTE bits, while they're
391 * currently identical, they may not remain so
392 * Potential optimization (here and in page_protect),
393 * parallel shootdowns, check for redundant
394 * attribute modifications.
395 */
396
397 /*
398 * Alter attributes on all mappings
399 */
400 if (pv_h->pmap != PMAP_NULL) {
401 pv_e = pv_h;
402 pvh_e = (pv_hashed_entry_t)pv_e;
403
404 do {
405 pmap = pv_e->pmap;
406 vaddr = PVE_VA(pv_e);
407 ptep = pmap_pte(pmap, vaddr);
408
409 if (0 == ptep)
410 panic("pmap_update_cache_attributes_locked: Missing PTE, pmap: %p, pn: 0x%x vaddr: 0x%llx kernel_pmap: %p", pmap, pn, vaddr, kernel_pmap);
411
412 is_ept = is_ept_pmap(pmap);
413
414 nexth = (pv_hashed_entry_t)queue_next(&pvh_e->qlink);
415 if (!is_ept) {
416 pmap_update_pte(ptep, PHYS_CACHEABILITY_MASK, attributes);
417 } else {
418 pmap_update_pte(ptep, INTEL_EPT_CACHE_MASK, ept_attributes);
419 }
420 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
421 pvh_e = nexth;
422 } while ((pv_e = (pv_rooted_entry_t)nexth) != pv_h);
423 }
424}
425
426void x86_filter_TLB_coherency_interrupts(boolean_t dofilter) {
427 assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
428
429 if (dofilter) {
430 CPU_CR3_MARK_INACTIVE();
431 } else {
432 CPU_CR3_MARK_ACTIVE();
433 mfence();
434 if (current_cpu_datap()->cpu_tlb_invalid)
435 process_pmap_updates();
436 }
437}
438
439
440/*
441 * Insert the given physical page (p) at
442 * the specified virtual address (v) in the
443 * target physical map with the protection requested.
444 *
445 * If specified, the page will be wired down, meaning
446 * that the related pte cannot be reclaimed.
447 *
448 * NB: This is the only routine which MAY NOT lazy-evaluate
449 * or lose information. That is, this routine must actually
450 * insert this page into the given map NOW.
451 */
452
453kern_return_t
454pmap_enter(
455 pmap_t pmap,
456 vm_map_offset_t vaddr,
457 ppnum_t pn,
458 vm_prot_t prot,
459 vm_prot_t fault_type,
460 unsigned int flags,
461 boolean_t wired)
462{
463 return pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL);
464}
465
466
467kern_return_t
468pmap_enter_options(
469 pmap_t pmap,
470 vm_map_offset_t vaddr,
471 ppnum_t pn,
472 vm_prot_t prot,
473 __unused vm_prot_t fault_type,
474 unsigned int flags,
475 boolean_t wired,
476 unsigned int options,
477 void *arg)
478{
479 pt_entry_t *pte;
480 pv_rooted_entry_t pv_h;
481 ppnum_t pai;
482 pv_hashed_entry_t pvh_e;
483 pv_hashed_entry_t pvh_new;
484 pt_entry_t template;
485 pmap_paddr_t old_pa;
486 pmap_paddr_t pa = (pmap_paddr_t) i386_ptob(pn);
487 boolean_t need_tlbflush = FALSE;
488 boolean_t set_NX;
489 char oattr;
490 boolean_t old_pa_locked;
491 /* 2MiB mappings are confined to x86_64 by VM */
492 boolean_t superpage = flags & VM_MEM_SUPERPAGE;
493 vm_object_t delpage_pm_obj = NULL;
494 uint64_t delpage_pde_index = 0;
495 pt_entry_t old_pte;
496 kern_return_t kr;
497 boolean_t is_ept;
498 boolean_t is_altacct;
499
500 kr = KERN_FAILURE;
501
502 pmap_intr_assert();
503
504 if (pmap == PMAP_NULL)
505 return KERN_INVALID_ARGUMENT;
506
507 is_ept = is_ept_pmap(pmap);
508
509 /* N.B. We can be supplied a zero page frame in the NOENTER case, it's an
510 * unused value for that scenario.
511 */
512 assert(pn != vm_page_fictitious_addr);
513
514 if (pn == vm_page_guard_addr)
515 return KERN_INVALID_ARGUMENT;
516
517 PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
518 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(vaddr), pn,
519 prot);
520
521 if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
522 set_NX = FALSE;
523 else
524 set_NX = TRUE;
525
526 if (__improbable(set_NX && (pmap == kernel_pmap) && ((pmap_disable_kstack_nx && (flags & VM_MEM_STACK)) || (pmap_disable_kheap_nx && !(flags & VM_MEM_STACK))))) {
527 set_NX = FALSE;
528 }
529
530 /*
531 * Must allocate a new pvlist entry while we're unlocked;
532 * zalloc may cause pageout (which will lock the pmap system).
533 * If we determine we need a pvlist entry, we will unlock
534 * and allocate one. Then we will retry, throughing away
535 * the allocated entry later (if we no longer need it).
536 */
537
538 pvh_new = PV_HASHED_ENTRY_NULL;
539Retry:
540 pvh_e = PV_HASHED_ENTRY_NULL;
541
542 PMAP_LOCK(pmap);
543
544 /*
545 * Expand pmap to include this pte. Assume that
546 * pmap is always expanded to include enough hardware
547 * pages to map one VM page.
548 */
549 if (superpage) {
550 while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) {
551 /* need room for another pde entry */
552 PMAP_UNLOCK(pmap);
553 kr = pmap_expand_pdpt(pmap, vaddr, options);
554 if (kr != KERN_SUCCESS)
555 goto done;
556 PMAP_LOCK(pmap);
557 }
558 } else {
559 while ((pte = pmap_pte(pmap, vaddr)) == PT_ENTRY_NULL) {
560 /*
561 * Must unlock to expand the pmap
562 * going to grow pde level page(s)
563 */
564 PMAP_UNLOCK(pmap);
565 kr = pmap_expand(pmap, vaddr, options);
566 if (kr != KERN_SUCCESS)
567 goto done;
568 PMAP_LOCK(pmap);
569 }
570 }
571 if (options & PMAP_EXPAND_OPTIONS_NOENTER) {
572 PMAP_UNLOCK(pmap);
573 kr = KERN_SUCCESS;
574 goto done;
575 }
576
577 if (superpage && *pte && !(*pte & PTE_PS)) {
578 /*
579 * There is still an empty page table mapped that
580 * was used for a previous base page mapping.
581 * Remember the PDE and the PDE index, so that we
582 * can free the page at the end of this function.
583 */
584 delpage_pde_index = pdeidx(pmap, vaddr);
585 delpage_pm_obj = pmap->pm_obj;
586 *pte = 0;
587 }
588
589 old_pa = pte_to_pa(*pte);
590 pai = pa_index(old_pa);
591 old_pa_locked = FALSE;
592
593 if (old_pa == 0 &&
594 PTE_IS_COMPRESSED(*pte)) {
595 /*
596 * "pmap" should be locked at this point, so this should
597 * not race with another pmap_enter() or pmap_remove_range().
598 */
599 assert(pmap != kernel_pmap);
600
601 /* one less "compressed" */
602 OSAddAtomic64(-1, &pmap->stats.compressed);
603 pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
604 PAGE_SIZE);
605 if (*pte & PTE_COMPRESSED_ALT) {
606 pmap_ledger_debit(
607 pmap,
608 task_ledgers.alternate_accounting_compressed,
609 PAGE_SIZE);
610 } else {
611 /* was part of the footprint */
612 pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
613 PAGE_SIZE);
614 }
615 /* marker will be cleared below */
616 }
617
618 /*
619 * if we have a previous managed page, lock the pv entry now. after
620 * we lock it, check to see if someone beat us to the lock and if so
621 * drop the lock
622 */
623 if ((0 != old_pa) && IS_MANAGED_PAGE(pai)) {
624 LOCK_PVH(pai);
625 old_pa_locked = TRUE;
626 old_pa = pte_to_pa(*pte);
627 if (0 == old_pa) {
628 UNLOCK_PVH(pai); /* another path beat us to it */
629 old_pa_locked = FALSE;
630 }
631 }
632
633 /*
634 * Special case if the incoming physical page is already mapped
635 * at this address.
636 */
637 if (old_pa == pa) {
638 pt_entry_t old_attributes =
639 *pte & ~(PTE_REF(is_ept) | PTE_MOD(is_ept));
640
641 /*
642 * May be changing its wired attribute or protection
643 */
644
645 template = pa_to_pte(pa);
646
647 /* ?: WORTH ASSERTING THAT AT LEAST ONE RWX (implicit valid) PASSED FOR EPT? */
648 if (!is_ept) {
649 template |= INTEL_PTE_VALID;
650 } else {
651 template |= INTEL_EPT_IPTA;
652 }
653
654 template |= pmap_get_cache_attributes(pa_index(pa), is_ept);
655
656 /*
657 * We don't support passing VM_MEM_NOT_CACHEABLE flags for EPT PTEs
658 */
659 if (!is_ept && (VM_MEM_NOT_CACHEABLE ==
660 (flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT)))) {
661 if (!(flags & VM_MEM_GUARDED))
662 template |= INTEL_PTE_PTA;
663 template |= INTEL_PTE_NCACHE;
664 }
665 if (pmap != kernel_pmap && !is_ept)
666 template |= INTEL_PTE_USER;
667
668 if (prot & VM_PROT_READ)
669 template |= PTE_READ(is_ept);
670
671 if (prot & VM_PROT_WRITE) {
672 template |= PTE_WRITE(is_ept);
673 if (is_ept && !pmap_ept_support_ad) {
674 template |= PTE_MOD(is_ept);
675 if (old_pa_locked) {
676 assert(IS_MANAGED_PAGE(pai));
677 pmap_phys_attributes[pai] |= PHYS_MODIFIED;
678 }
679 }
680 }
681 if (prot & VM_PROT_EXECUTE) {
682 assert(set_NX == 0);
683 template = pte_set_ex(template, is_ept);
684 }
685
686 if (set_NX)
687 template = pte_remove_ex(template, is_ept);
688
689 if (wired) {
690 template |= PTE_WIRED;
691 if (!iswired(old_attributes)) {
692 OSAddAtomic(+1, &pmap->stats.wired_count);
693 pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
694 }
695 } else {
696 if (iswired(old_attributes)) {
697 assert(pmap->stats.wired_count >= 1);
698 OSAddAtomic(-1, &pmap->stats.wired_count);
699 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
700 }
701 }
702
703 if (superpage) /* this path can not be used */
704 template |= PTE_PS; /* to change the page size! */
705
706 if (old_attributes == template)
707 goto dont_update_pte;
708
709 /* Determine delta, PV locked */
710 need_tlbflush =
711 ((old_attributes ^ template) != PTE_WIRED);
712
713 if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) {
714 if ((old_attributes ^ template) == PTE_WRITE(is_ept))
715 need_tlbflush = FALSE;
716 }
717
718 /* For hardware that doesn't have EPT AD support, we always set REFMOD for EPT PTEs */
719 if (is_ept && !pmap_ept_support_ad) {
720 template |= PTE_REF(is_ept);
721 if (old_pa_locked) {
722 assert(IS_MANAGED_PAGE(pai));
723 pmap_phys_attributes[pai] |= PHYS_REFERENCED;
724 }
725 }
726
727 /* store modified PTE and preserve RC bits */
728 pt_entry_t npte, opte;;
729 do {
730 opte = *pte;
731 npte = template | (opte & (PTE_REF(is_ept) | PTE_MOD(is_ept)));
732 } while (!pmap_cmpx_pte(pte, opte, npte));
733dont_update_pte:
734 if (old_pa_locked) {
735 UNLOCK_PVH(pai);
736 old_pa_locked = FALSE;
737 }
738 goto Done;
739 }
740
741 /*
742 * Outline of code from here:
743 * 1) If va was mapped, update TLBs, remove the mapping
744 * and remove old pvlist entry.
745 * 2) Add pvlist entry for new mapping
746 * 3) Enter new mapping.
747 *
748 * If the old physical page is not managed step 1) is skipped
749 * (except for updating the TLBs), and the mapping is
750 * overwritten at step 3). If the new physical page is not
751 * managed, step 2) is skipped.
752 */
753
754 if (old_pa != (pmap_paddr_t) 0) {
755 boolean_t was_altacct = FALSE;
756
757 /*
758 * Don't do anything to pages outside valid memory here.
759 * Instead convince the code that enters a new mapping
760 * to overwrite the old one.
761 */
762
763 /* invalidate the PTE */
764 pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0);
765 /* propagate invalidate everywhere */
766 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
767 /* remember reference and change */
768 old_pte = *pte;
769 oattr = (char) (old_pte & (PTE_MOD(is_ept) | PTE_REF(is_ept)));
770 /* completely invalidate the PTE */
771 pmap_store_pte(pte, 0);
772
773 if (IS_MANAGED_PAGE(pai)) {
774 /*
775 * Remove the mapping from the pvlist for
776 * this physical page.
777 * We'll end up with either a rooted pv or a
778 * hashed pv
779 */
780 pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte, &was_altacct);
781 }
782
783 if (IS_MANAGED_PAGE(pai)) {
784 pmap_assert(old_pa_locked == TRUE);
785 pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
786 assert(pmap->stats.resident_count >= 1);
787 OSAddAtomic(-1, &pmap->stats.resident_count);
788 if (pmap != kernel_pmap) {
789 /* update pmap stats */
790 if (IS_REUSABLE_PAGE(pai)) {
791 PMAP_STATS_ASSERTF(
792 (pmap->stats.reusable > 0,
793 "reusable %d",
794 pmap->stats.reusable));
795 OSAddAtomic(-1, &pmap->stats.reusable);
796 } else if (IS_INTERNAL_PAGE(pai)) {
797 PMAP_STATS_ASSERTF(
798 (pmap->stats.internal > 0,
799 "internal %d",
800 pmap->stats.internal));
801 OSAddAtomic(-1, &pmap->stats.internal);
802 } else {
803 PMAP_STATS_ASSERTF(
804 (pmap->stats.external > 0,
805 "external %d",
806 pmap->stats.external));
807 OSAddAtomic(-1, &pmap->stats.external);
808 }
809
810 /* update ledgers */
811 if (was_altacct) {
812 assert(IS_INTERNAL_PAGE(pai));
813 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
814 pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
815 } else if (IS_REUSABLE_PAGE(pai)) {
816 assert(!was_altacct);
817 assert(IS_INTERNAL_PAGE(pai));
818 /* was already not in phys_footprint */
819 } else if (IS_INTERNAL_PAGE(pai)) {
820 assert(!was_altacct);
821 assert(!IS_REUSABLE_PAGE(pai));
822 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
823 pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
824 } else {
825 /* not an internal page */
826 }
827 }
828 if (iswired(*pte)) {
829 assert(pmap->stats.wired_count >= 1);
830 OSAddAtomic(-1, &pmap->stats.wired_count);
831 pmap_ledger_debit(pmap, task_ledgers.wired_mem,
832 PAGE_SIZE);
833 }
834
835 if (!is_ept) {
836 pmap_phys_attributes[pai] |= oattr;
837 } else {
838 pmap_phys_attributes[pai] |= ept_refmod_to_physmap(oattr);
839 }
840
841 } else {
842
843 /*
844 * old_pa is not managed.
845 * Do removal part of accounting.
846 */
847
848 if (pmap != kernel_pmap) {
849#if 00
850 assert(pmap->stats.device > 0);
851 OSAddAtomic(-1, &pmap->stats.device);
852#endif
853 }
854 if (iswired(*pte)) {
855 assert(pmap->stats.wired_count >= 1);
856 OSAddAtomic(-1, &pmap->stats.wired_count);
857 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
858 }
859 }
860 }
861
862 /*
863 * if we had a previously managed paged locked, unlock it now
864 */
865 if (old_pa_locked) {
866 UNLOCK_PVH(pai);
867 old_pa_locked = FALSE;
868 }
869
870 pai = pa_index(pa); /* now working with new incoming phys page */
871 if (IS_MANAGED_PAGE(pai)) {
872
873 /*
874 * Step 2) Enter the mapping in the PV list for this
875 * physical page.
876 */
877 pv_h = pai_to_pvh(pai);
878
879 LOCK_PVH(pai);
880
881 if (pv_h->pmap == PMAP_NULL) {
882 /*
883 * No mappings yet, use rooted pv
884 */
885 pv_h->va_and_flags = vaddr;
886 pv_h->pmap = pmap;
887 queue_init(&pv_h->qlink);
888
889 if (options & PMAP_OPTIONS_INTERNAL) {
890 pmap_phys_attributes[pai] |= PHYS_INTERNAL;
891 } else {
892 pmap_phys_attributes[pai] &= ~PHYS_INTERNAL;
893 }
894 if (options & PMAP_OPTIONS_REUSABLE) {
895 pmap_phys_attributes[pai] |= PHYS_REUSABLE;
896 } else {
897 pmap_phys_attributes[pai] &= ~PHYS_REUSABLE;
898 }
899 if ((options & PMAP_OPTIONS_ALT_ACCT) &&
900 IS_INTERNAL_PAGE(pai)) {
901 pv_h->va_and_flags |= PVE_IS_ALTACCT;
902 is_altacct = TRUE;
903 } else {
904 pv_h->va_and_flags &= ~PVE_IS_ALTACCT;
905 is_altacct = FALSE;
906 }
907 } else {
908 /*
909 * Add new pv_hashed_entry after header.
910 */
911 if ((PV_HASHED_ENTRY_NULL == pvh_e) && pvh_new) {
912 pvh_e = pvh_new;
913 pvh_new = PV_HASHED_ENTRY_NULL;
914 } else if (PV_HASHED_ENTRY_NULL == pvh_e) {
915 PV_HASHED_ALLOC(&pvh_e);
916 if (PV_HASHED_ENTRY_NULL == pvh_e) {
917 /*
918 * the pv list is empty. if we are on
919 * the kernel pmap we'll use one of
920 * the special private kernel pv_e's,
921 * else, we need to unlock
922 * everything, zalloc a pv_e, and
923 * restart bringing in the pv_e with
924 * us.
925 */
926 if (kernel_pmap == pmap) {
927 PV_HASHED_KERN_ALLOC(&pvh_e);
928 } else {
929 UNLOCK_PVH(pai);
930 PMAP_UNLOCK(pmap);
931 pmap_pv_throttle(pmap);
932 pvh_new = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
933 goto Retry;
934 }
935 }
936 }
937
938 if (PV_HASHED_ENTRY_NULL == pvh_e)
939 panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings");
940
941 pvh_e->va_and_flags = vaddr;
942 pvh_e->pmap = pmap;
943 pvh_e->ppn = pn;
944 if ((options & PMAP_OPTIONS_ALT_ACCT) &&
945 IS_INTERNAL_PAGE(pai)) {
946 pvh_e->va_and_flags |= PVE_IS_ALTACCT;
947 is_altacct = TRUE;
948 } else {
949 pvh_e->va_and_flags &= ~PVE_IS_ALTACCT;
950 is_altacct = FALSE;
951 }
952 pv_hash_add(pvh_e, pv_h);
953
954 /*
955 * Remember that we used the pvlist entry.
956 */
957 pvh_e = PV_HASHED_ENTRY_NULL;
958 }
959
960 /*
961 * only count the mapping
962 * for 'managed memory'
963 */
964 pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
965 OSAddAtomic(+1, &pmap->stats.resident_count);
966 if (pmap->stats.resident_count > pmap->stats.resident_max) {
967 pmap->stats.resident_max = pmap->stats.resident_count;
968 }
969 if (pmap != kernel_pmap) {
970 /* update pmap stats */
971 if (IS_REUSABLE_PAGE(pai)) {
972 OSAddAtomic(+1, &pmap->stats.reusable);
973 PMAP_STATS_PEAK(pmap->stats.reusable);
974 } else if (IS_INTERNAL_PAGE(pai)) {
975 OSAddAtomic(+1, &pmap->stats.internal);
976 PMAP_STATS_PEAK(pmap->stats.internal);
977 } else {
978 OSAddAtomic(+1, &pmap->stats.external);
979 PMAP_STATS_PEAK(pmap->stats.external);
980 }
981
982 /* update ledgers */
983 if (is_altacct) {
984 /* internal but also alternate accounting */
985 assert(IS_INTERNAL_PAGE(pai));
986 pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
987 pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
988 /* alternate accounting, so not in footprint */
989 } else if (IS_REUSABLE_PAGE(pai)) {
990 assert(!is_altacct);
991 assert(IS_INTERNAL_PAGE(pai));
992 /* internal but reusable: not in footprint */
993 } else if (IS_INTERNAL_PAGE(pai)) {
994 assert(!is_altacct);
995 assert(!IS_REUSABLE_PAGE(pai));
996 /* internal: add to footprint */
997 pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
998 pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
999 } else {
1000 /* not internal: not in footprint */
1001 }
1002 }
1003 } else if (last_managed_page == 0) {
1004 /* Account for early mappings created before "managed pages"
1005 * are determined. Consider consulting the available DRAM map.
1006 */
1007 pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
1008 OSAddAtomic(+1, &pmap->stats.resident_count);
1009 if (pmap != kernel_pmap) {
1010#if 00
1011 OSAddAtomic(+1, &pmap->stats.device);
1012 PMAP_STATS_PEAK(pmap->stats.device);
1013#endif
1014 }
1015 }
1016 /*
1017 * Step 3) Enter the mapping.
1018 *
1019 * Build a template to speed up entering -
1020 * only the pfn changes.
1021 */
1022 template = pa_to_pte(pa);
1023
1024 if (!is_ept) {
1025 template |= INTEL_PTE_VALID;
1026 } else {
1027 template |= INTEL_EPT_IPTA;
1028 }
1029
1030
1031 /*
1032 * DRK: It may be worth asserting on cache attribute flags that diverge
1033 * from the existing physical page attributes.
1034 */
1035
1036 template |= pmap_get_cache_attributes(pa_index(pa), is_ept);
1037
1038 /*
1039 * We don't support passing VM_MEM_NOT_CACHEABLE flags for EPT PTEs
1040 */
1041 if (!is_ept && (flags & VM_MEM_NOT_CACHEABLE)) {
1042 if (!(flags & VM_MEM_GUARDED))
1043 template |= INTEL_PTE_PTA;
1044 template |= INTEL_PTE_NCACHE;
1045 }
1046 if (pmap != kernel_pmap && !is_ept)
1047 template |= INTEL_PTE_USER;
1048 if (prot & VM_PROT_READ)
1049 template |= PTE_READ(is_ept);
1050 if (prot & VM_PROT_WRITE) {
1051 template |= PTE_WRITE(is_ept);
1052 if (is_ept && !pmap_ept_support_ad) {
1053 template |= PTE_MOD(is_ept);
1054 if (IS_MANAGED_PAGE(pai))
1055 pmap_phys_attributes[pai] |= PHYS_MODIFIED;
1056 }
1057 }
1058 if (prot & VM_PROT_EXECUTE) {
1059 assert(set_NX == 0);
1060 template = pte_set_ex(template, is_ept);
1061 }
1062
1063 if (set_NX)
1064 template = pte_remove_ex(template, is_ept);
1065 if (wired) {
1066 template |= INTEL_PTE_WIRED;
1067 OSAddAtomic(+1, & pmap->stats.wired_count);
1068 pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
1069 }
1070 if (superpage)
1071 template |= INTEL_PTE_PS;
1072
1073 /* For hardware that doesn't have EPT AD support, we always set REFMOD for EPT PTEs */
1074 if (is_ept && !pmap_ept_support_ad) {
1075 template |= PTE_REF(is_ept);
1076 if (IS_MANAGED_PAGE(pai))
1077 pmap_phys_attributes[pai] |= PHYS_REFERENCED;
1078 }
1079
1080 pmap_store_pte(pte, template);
1081
1082 /*
1083 * if this was a managed page we delayed unlocking the pv until here
1084 * to prevent pmap_page_protect et al from finding it until the pte
1085 * has been stored
1086 */
1087 if (IS_MANAGED_PAGE(pai)) {
1088 UNLOCK_PVH(pai);
1089 }
1090Done:
1091 if (need_tlbflush == TRUE) {
1092 if (options & PMAP_OPTIONS_NOFLUSH)
1093 PMAP_UPDATE_TLBS_DELAYED(pmap, vaddr, vaddr + PAGE_SIZE, (pmap_flush_context *)arg);
1094 else
1095 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
1096 }
1097 if (pvh_e != PV_HASHED_ENTRY_NULL) {
1098 PV_HASHED_FREE_LIST(pvh_e, pvh_e, 1);
1099 }
1100 if (pvh_new != PV_HASHED_ENTRY_NULL) {
1101 PV_HASHED_KERN_FREE_LIST(pvh_new, pvh_new, 1);
1102 }
1103 PMAP_UNLOCK(pmap);
1104
1105 if (delpage_pm_obj) {
1106 vm_page_t m;
1107
1108 vm_object_lock(delpage_pm_obj);
1109 m = vm_page_lookup(delpage_pm_obj, (delpage_pde_index * PAGE_SIZE));
1110 if (m == VM_PAGE_NULL)
1111 panic("pmap_enter: pte page not in object");
1112 VM_PAGE_FREE(m);
1113 vm_object_unlock(delpage_pm_obj);
1114 OSAddAtomic(-1, &inuse_ptepages_count);
1115 PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
1116 }
1117
1118 kr = KERN_SUCCESS;
1119done:
1120 PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
1121 return kr;
1122}
1123
1124/*
1125 * Remove a range of hardware page-table entries.
1126 * The entries given are the first (inclusive)
1127 * and last (exclusive) entries for the VM pages.
1128 * The virtual address is the va for the first pte.
1129 *
1130 * The pmap must be locked.
1131 * If the pmap is not the kernel pmap, the range must lie
1132 * entirely within one pte-page. This is NOT checked.
1133 * Assumes that the pte-page exists.
1134 */
1135
1136void
1137pmap_remove_range(
1138 pmap_t pmap,
1139 vm_map_offset_t start_vaddr,
1140 pt_entry_t *spte,
1141 pt_entry_t *epte)
1142{
1143 pmap_remove_range_options(pmap, start_vaddr, spte, epte,
1144 PMAP_OPTIONS_REMOVE);
1145}
1146
1147void
1148pmap_remove_range_options(
1149 pmap_t pmap,
1150 vm_map_offset_t start_vaddr,
1151 pt_entry_t *spte,
1152 pt_entry_t *epte,
1153 int options)
1154{
1155 pt_entry_t *cpte;
1156 pv_hashed_entry_t pvh_et = PV_HASHED_ENTRY_NULL;
1157 pv_hashed_entry_t pvh_eh = PV_HASHED_ENTRY_NULL;
1158 pv_hashed_entry_t pvh_e;
1159 int pvh_cnt = 0;
1160 int num_removed, num_unwired, num_found, num_invalid;
1161 int stats_external, stats_internal, stats_reusable;
1162 uint64_t stats_compressed;
1163 int ledgers_internal, ledgers_alt_internal;
1164 uint64_t ledgers_compressed, ledgers_alt_compressed;
1165 ppnum_t pai;
1166 pmap_paddr_t pa;
1167 vm_map_offset_t vaddr;
1168 boolean_t is_ept = is_ept_pmap(pmap);
1169 boolean_t was_altacct;
1170
1171 num_removed = 0;
1172 num_unwired = 0;
1173 num_found = 0;
1174 num_invalid = 0;
1175 stats_external = 0;
1176 stats_internal = 0;
1177 stats_reusable = 0;
1178 stats_compressed = 0;
1179 ledgers_internal = 0;
1180 ledgers_compressed = 0;
1181 ledgers_alt_internal = 0;
1182 ledgers_alt_compressed = 0;
1183 /* invalidate the PTEs first to "freeze" them */
1184 for (cpte = spte, vaddr = start_vaddr;
1185 cpte < epte;
1186 cpte++, vaddr += PAGE_SIZE_64) {
1187 pt_entry_t p = *cpte;
1188
1189 pa = pte_to_pa(p);
1190 if (pa == 0) {
1191 if ((options & PMAP_OPTIONS_REMOVE) &&
1192 (PTE_IS_COMPRESSED(p))) {
1193 assert(pmap != kernel_pmap);
1194 /* one less "compressed"... */
1195 stats_compressed++;
1196 ledgers_compressed++;
1197 if (p & PTE_COMPRESSED_ALT) {
1198 /* ... but it used to be "ALTACCT" */
1199 ledgers_alt_compressed++;
1200 }
1201 /* clear marker(s) */
1202 /* XXX probably does not need to be atomic! */
1203 pmap_update_pte(cpte, INTEL_PTE_COMPRESSED_MASK, 0);
1204 }
1205 continue;
1206 }
1207 num_found++;
1208
1209 if (iswired(p))
1210 num_unwired++;
1211
1212 pai = pa_index(pa);
1213
1214 if (!IS_MANAGED_PAGE(pai)) {
1215 /*
1216 * Outside range of managed physical memory.
1217 * Just remove the mappings.
1218 */
1219 pmap_store_pte(cpte, 0);
1220 continue;
1221 }
1222
1223 if ((p & PTE_VALID_MASK(is_ept)) == 0)
1224 num_invalid++;
1225
1226 /* invalidate the PTE */
1227 pmap_update_pte(cpte, PTE_VALID_MASK(is_ept), 0);
1228 }
1229
1230 if (num_found == 0) {
1231 /* nothing was changed: we're done */
1232 goto update_counts;
1233 }
1234
1235 /* propagate the invalidates to other CPUs */
1236
1237 PMAP_UPDATE_TLBS(pmap, start_vaddr, vaddr);
1238
1239 for (cpte = spte, vaddr = start_vaddr;
1240 cpte < epte;
1241 cpte++, vaddr += PAGE_SIZE_64) {
1242
1243 pa = pte_to_pa(*cpte);
1244 if (pa == 0) {
1245 check_pte_for_compressed_marker:
1246 /*
1247 * This PTE could have been replaced with a
1248 * "compressed" marker after our first "freeze"
1249 * loop above, so check again.
1250 */
1251 if ((options & PMAP_OPTIONS_REMOVE) &&
1252 (PTE_IS_COMPRESSED(*cpte))) {
1253 assert(pmap != kernel_pmap);
1254 /* one less "compressed"... */
1255 stats_compressed++;
1256 ledgers_compressed++;
1257 if (*cpte & PTE_COMPRESSED_ALT) {
1258 /* ... but it used to be "ALTACCT" */
1259 ledgers_alt_compressed++;
1260 }
1261 pmap_store_pte(cpte, 0);
1262 }
1263 continue;
1264 }
1265
1266 pai = pa_index(pa);
1267
1268 LOCK_PVH(pai);
1269
1270 pa = pte_to_pa(*cpte);
1271 if (pa == 0) {
1272 UNLOCK_PVH(pai);
1273 goto check_pte_for_compressed_marker;
1274 }
1275
1276 /*
1277 * Remove the mapping from the pvlist for this physical page.
1278 */
1279 pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte, &was_altacct);
1280
1281 num_removed++;
1282 /* update pmap stats */
1283 if (IS_REUSABLE_PAGE(pai)) {
1284 stats_reusable++;
1285 } else if (IS_INTERNAL_PAGE(pai)) {
1286 stats_internal++;
1287 } else {
1288 stats_external++;
1289 }
1290 /* update ledgers */
1291 if (was_altacct) {
1292 /* internal and alternate accounting */
1293 assert(IS_INTERNAL_PAGE(pai));
1294 ledgers_internal++;
1295 ledgers_alt_internal++;
1296 } else if (IS_REUSABLE_PAGE(pai)) {
1297 /* internal but reusable */
1298 assert(!was_altacct);
1299 assert(IS_INTERNAL_PAGE(pai));
1300 } else if (IS_INTERNAL_PAGE(pai)) {
1301 /* internal */
1302 assert(!was_altacct);
1303 assert(!IS_REUSABLE_PAGE(pai));
1304 ledgers_internal++;
1305 } else {
1306 /* not internal */
1307 }
1308
1309 /*
1310 * Get the modify and reference bits, then
1311 * nuke the entry in the page table
1312 */
1313 /* remember reference and change */
1314 if (!is_ept) {
1315 pmap_phys_attributes[pai] |=
1316 *cpte & (PHYS_MODIFIED | PHYS_REFERENCED);
1317 } else {
1318 pmap_phys_attributes[pai] |=
1319 ept_refmod_to_physmap((*cpte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
1320 }
1321
1322 /* completely invalidate the PTE */
1323 pmap_store_pte(cpte, 0);
1324
1325 UNLOCK_PVH(pai);
1326
1327 if (pvh_e != PV_HASHED_ENTRY_NULL) {
1328 pvh_e->qlink.next = (queue_entry_t) pvh_eh;
1329 pvh_eh = pvh_e;
1330
1331 if (pvh_et == PV_HASHED_ENTRY_NULL) {
1332 pvh_et = pvh_e;
1333 }
1334 pvh_cnt++;
1335 }
1336 } /* for loop */
1337
1338 if (pvh_eh != PV_HASHED_ENTRY_NULL) {
1339 PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pvh_cnt);
1340 }
1341update_counts:
1342 /*
1343 * Update the counts
1344 */
1345#if TESTING
1346 if (pmap->stats.resident_count < num_removed)
1347 panic("pmap_remove_range: resident_count");
1348#endif
1349 pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));
1350 PMAP_STATS_ASSERTF((pmap->stats.resident_count >= num_removed,
1351 "pmap=%p num_removed=%d stats.resident_count=%d",
1352 pmap, num_removed, pmap->stats.resident_count));
1353 OSAddAtomic(-num_removed, &pmap->stats.resident_count);
1354
1355 if (pmap != kernel_pmap) {
1356 PMAP_STATS_ASSERTF((pmap->stats.external >= stats_external,
1357 "pmap=%p stats_external=%d stats.external=%d",
1358 pmap, stats_external, pmap->stats.external));
1359 PMAP_STATS_ASSERTF((pmap->stats.internal >= stats_internal,
1360 "pmap=%p stats_internal=%d stats.internal=%d",
1361 pmap, stats_internal, pmap->stats.internal));
1362 PMAP_STATS_ASSERTF((pmap->stats.reusable >= stats_reusable,
1363 "pmap=%p stats_reusable=%d stats.reusable=%d",
1364 pmap, stats_reusable, pmap->stats.reusable));
1365 PMAP_STATS_ASSERTF((pmap->stats.compressed >= stats_compressed,
1366 "pmap=%p stats_compressed=%lld, stats.compressed=%lld",
1367 pmap, stats_compressed, pmap->stats.compressed));
1368
1369 /* update pmap stats */
1370 if (stats_external) {
1371 OSAddAtomic(-stats_external, &pmap->stats.external);
1372 }
1373 if (stats_internal) {
1374 OSAddAtomic(-stats_internal, &pmap->stats.internal);
1375 }
1376 if (stats_reusable)
1377 OSAddAtomic(-stats_reusable, &pmap->stats.reusable);
1378 if (stats_compressed)
1379 OSAddAtomic64(-stats_compressed, &pmap->stats.compressed);
1380 /* update ledgers */
1381 if (ledgers_internal) {
1382 pmap_ledger_debit(pmap,
1383 task_ledgers.internal,
1384 machine_ptob(ledgers_internal));
1385 }
1386 if (ledgers_compressed) {
1387 pmap_ledger_debit(pmap,
1388 task_ledgers.internal_compressed,
1389 machine_ptob(ledgers_compressed));
1390 }
1391 if (ledgers_alt_internal) {
1392 pmap_ledger_debit(pmap,
1393 task_ledgers.alternate_accounting,
1394 machine_ptob(ledgers_alt_internal));
1395 }
1396 if (ledgers_alt_compressed) {
1397 pmap_ledger_debit(pmap,
1398 task_ledgers.alternate_accounting_compressed,
1399 machine_ptob(ledgers_alt_compressed));
1400 }
1401 pmap_ledger_debit(pmap,
1402 task_ledgers.phys_footprint,
1403 machine_ptob((ledgers_internal -
1404 ledgers_alt_internal) +
1405 (ledgers_compressed -
1406 ledgers_alt_compressed)));
1407 }
1408
1409#if TESTING
1410 if (pmap->stats.wired_count < num_unwired)
1411 panic("pmap_remove_range: wired_count");
1412#endif
1413 PMAP_STATS_ASSERTF((pmap->stats.wired_count >= num_unwired,
1414 "pmap=%p num_unwired=%d stats.wired_count=%d",
1415 pmap, num_unwired, pmap->stats.wired_count));
1416 OSAddAtomic(-num_unwired, &pmap->stats.wired_count);
1417 pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
1418
1419 return;
1420}
1421
1422
1423/*
1424 * Remove the given range of addresses
1425 * from the specified map.
1426 *
1427 * It is assumed that the start and end are properly
1428 * rounded to the hardware page size.
1429 */
1430void
1431pmap_remove(
1432 pmap_t map,
1433 addr64_t s64,
1434 addr64_t e64)
1435{
1436 pmap_remove_options(map, s64, e64, PMAP_OPTIONS_REMOVE);
1437}
1438
1439void
1440pmap_remove_options(
1441 pmap_t map,
1442 addr64_t s64,
1443 addr64_t e64,
1444 int options)
1445{
1446 pt_entry_t *pde;
1447 pt_entry_t *spte, *epte;
1448 addr64_t l64;
1449 uint64_t deadline;
1450 boolean_t is_ept;
1451
1452 pmap_intr_assert();
1453
1454 if (map == PMAP_NULL || s64 == e64)
1455 return;
1456
1457 is_ept = is_ept_pmap(map);
1458
1459 PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
1460 VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(s64),
1461 VM_KERNEL_ADDRHIDE(e64));
1462
1463 PMAP_LOCK(map);
1464
1465#if 0
1466 /*
1467 * Check that address range in the kernel does not overlap the stacks.
1468 * We initialize local static min/max variables once to avoid making
1469 * 2 function calls for every remove. Note also that these functions
1470 * both return 0 before kernel stacks have been initialized, and hence
1471 * the panic is not triggered in this case.
1472 */
1473 if (map == kernel_pmap) {
1474 static vm_offset_t kernel_stack_min = 0;
1475 static vm_offset_t kernel_stack_max = 0;
1476
1477 if (kernel_stack_min == 0) {
1478 kernel_stack_min = min_valid_stack_address();
1479 kernel_stack_max = max_valid_stack_address();
1480 }
1481 if ((kernel_stack_min <= s64 && s64 < kernel_stack_max) ||
1482 (kernel_stack_min < e64 && e64 <= kernel_stack_max))
1483 panic("pmap_remove() attempted in kernel stack");
1484 }
1485#else
1486
1487 /*
1488 * The values of kernel_stack_min and kernel_stack_max are no longer
1489 * relevant now that we allocate kernel stacks in the kernel map,
1490 * so the old code above no longer applies. If we wanted to check that
1491 * we weren't removing a mapping of a page in a kernel stack we'd
1492 * mark the PTE with an unused bit and check that here.
1493 */
1494
1495#endif
1496
1497 deadline = rdtsc64() + max_preemption_latency_tsc;
1498
1499 while (s64 < e64) {
1500 l64 = (s64 + pde_mapped_size) & ~(pde_mapped_size - 1);
1501 if (l64 > e64)
1502 l64 = e64;
1503 pde = pmap_pde(map, s64);
1504
1505 if (pde && (*pde & PTE_VALID_MASK(is_ept))) {
1506 if (*pde & PTE_PS) {
1507 /*
1508 * If we're removing a superpage, pmap_remove_range()
1509 * must work on level 2 instead of level 1; and we're
1510 * only passing a single level 2 entry instead of a
1511 * level 1 range.
1512 */
1513 spte = pde;
1514 epte = spte+1; /* excluded */
1515 } else {
1516 spte = pmap_pte(map, (s64 & ~(pde_mapped_size - 1)));
1517 spte = &spte[ptenum(s64)];
1518 epte = &spte[intel_btop(l64 - s64)];
1519 }
1520 pmap_remove_range_options(map, s64, spte, epte,
1521 options);
1522 }
1523 s64 = l64;
1524
1525 if (s64 < e64 && rdtsc64() >= deadline) {
1526 PMAP_UNLOCK(map)
1527 /* TODO: Rapid release/reacquisition can defeat
1528 * the "backoff" intent here; either consider a
1529 * fair spinlock, or a scheme whereby each lock
1530 * attempt marks the processor as within a spinlock
1531 * acquisition, and scan CPUs here to determine
1532 * if a backoff is necessary, to avoid sacrificing
1533 * performance in the common case.
1534 */
1535 PMAP_LOCK(map)
1536 deadline = rdtsc64() + max_preemption_latency_tsc;
1537 }
1538 }
1539
1540 PMAP_UNLOCK(map);
1541
1542 PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);
1543
1544}
1545
1546void
1547pmap_page_protect(
1548 ppnum_t pn,
1549 vm_prot_t prot)
1550{
1551 pmap_page_protect_options(pn, prot, 0, NULL);
1552}
1553
1554/*
1555 * Routine: pmap_page_protect_options
1556 *
1557 * Function:
1558 * Lower the permission for all mappings to a given
1559 * page.
1560 */
1561void
1562pmap_page_protect_options(
1563 ppnum_t pn,
1564 vm_prot_t prot,
1565 unsigned int options,
1566 void *arg)
1567{
1568 pv_hashed_entry_t pvh_eh = PV_HASHED_ENTRY_NULL;
1569 pv_hashed_entry_t pvh_et = PV_HASHED_ENTRY_NULL;
1570 pv_hashed_entry_t nexth;
1571 int pvh_cnt = 0;
1572 pv_rooted_entry_t pv_h;
1573 pv_rooted_entry_t pv_e;
1574 pv_hashed_entry_t pvh_e;
1575 pt_entry_t *pte;
1576 int pai;
1577 pmap_t pmap;
1578 boolean_t remove;
1579 pt_entry_t new_pte_value;
1580 boolean_t is_ept;
1581
1582 pmap_intr_assert();
1583 assert(pn != vm_page_fictitious_addr);
1584 if (pn == vm_page_guard_addr)
1585 return;
1586
1587 pai = ppn_to_pai(pn);
1588
1589 if (!IS_MANAGED_PAGE(pai)) {
1590 /*
1591 * Not a managed page.
1592 */
1593 return;
1594 }
1595
1596 PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, pn, prot);
1597
1598 /*
1599 * Determine the new protection.
1600 */
1601 switch (prot) {
1602 case VM_PROT_READ:
1603 case VM_PROT_READ | VM_PROT_EXECUTE:
1604 remove = FALSE;
1605 break;
1606 case VM_PROT_ALL:
1607 return; /* nothing to do */
1608 default:
1609 remove = TRUE;
1610 break;
1611 }
1612
1613 pv_h = pai_to_pvh(pai);
1614
1615 LOCK_PVH(pai);
1616
1617
1618 /*
1619 * Walk down PV list, if any, changing or removing all mappings.
1620 */
1621 if (pv_h->pmap == PMAP_NULL)
1622 goto done;
1623
1624 pv_e = pv_h;
1625 pvh_e = (pv_hashed_entry_t) pv_e; /* cheat */
1626
1627 do {
1628 vm_map_offset_t vaddr;
1629
1630 if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) &&
1631 (pmap_phys_attributes[pai] & PHYS_MODIFIED)) {
1632 /* page was modified, so it will be compressed */
1633 options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
1634 options |= PMAP_OPTIONS_COMPRESSOR;
1635 }
1636
1637 pmap = pv_e->pmap;
1638 is_ept = is_ept_pmap(pmap);
1639 vaddr = PVE_VA(pv_e);
1640 pte = pmap_pte(pmap, vaddr);
1641
1642 pmap_assert2((pa_index(pte_to_pa(*pte)) == pn),
1643 "pmap_page_protect: PTE mismatch, pn: 0x%x, pmap: %p, vaddr: 0x%llx, pte: 0x%llx", pn, pmap, vaddr, *pte);
1644
1645 if (0 == pte) {
1646 panic("pmap_page_protect() "
1647 "pmap=%p pn=0x%x vaddr=0x%llx\n",
1648 pmap, pn, vaddr);
1649 }
1650 nexth = (pv_hashed_entry_t) queue_next(&pvh_e->qlink);
1651
1652 /*
1653 * Remove the mapping if new protection is NONE
1654 */
1655 if (remove) {
1656
1657 /* Remove per-pmap wired count */
1658 if (iswired(*pte)) {
1659 OSAddAtomic(-1, &pmap->stats.wired_count);
1660 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
1661 }
1662
1663 if (pmap != kernel_pmap &&
1664 (options & PMAP_OPTIONS_COMPRESSOR) &&
1665 IS_INTERNAL_PAGE(pai)) {
1666 assert(!PTE_IS_COMPRESSED(*pte));
1667 /* mark this PTE as having been "compressed" */
1668 new_pte_value = PTE_COMPRESSED;
1669 if (IS_ALTACCT_PAGE(pai, pv_e)) {
1670 new_pte_value |= PTE_COMPRESSED_ALT;
1671 }
1672 } else {
1673 new_pte_value = 0;
1674 }
1675
1676 if (options & PMAP_OPTIONS_NOREFMOD) {
1677 pmap_store_pte(pte, new_pte_value);
1678
1679 if (options & PMAP_OPTIONS_NOFLUSH)
1680 PMAP_UPDATE_TLBS_DELAYED(pmap, vaddr, vaddr + PAGE_SIZE, (pmap_flush_context *)arg);
1681 else
1682 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
1683 } else {
1684 /*
1685 * Remove the mapping, collecting dirty bits.
1686 */
1687 pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0);
1688
1689 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE);
1690 if (!is_ept) {
1691 pmap_phys_attributes[pai] |=
1692 *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1693 } else {
1694 pmap_phys_attributes[pai] |=
1695 ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
1696 }
1697 if ((options &
1698 PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) &&
1699 IS_INTERNAL_PAGE(pai) &&
1700 (pmap_phys_attributes[pai] &
1701 PHYS_MODIFIED)) {
1702 /*
1703 * Page is actually "modified" and
1704 * will be compressed. Start
1705 * accounting for it as "compressed".
1706 */
1707 assert(!(options & PMAP_OPTIONS_COMPRESSOR));
1708 options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
1709 options |= PMAP_OPTIONS_COMPRESSOR;
1710 assert(new_pte_value == 0);
1711 if (pmap != kernel_pmap) {
1712 new_pte_value = PTE_COMPRESSED;
1713 if (IS_ALTACCT_PAGE(pai, pv_e)) {
1714 new_pte_value |= PTE_COMPRESSED_ALT;
1715 }
1716 }
1717 }
1718 pmap_store_pte(pte, new_pte_value);
1719 }
1720
1721#if TESTING
1722 if (pmap->stats.resident_count < 1)
1723 panic("pmap_page_protect: resident_count");
1724#endif
1725 pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
1726 assert(pmap->stats.resident_count >= 1);
1727 OSAddAtomic(-1, &pmap->stats.resident_count);
1728
1729 /*
1730 * We only ever compress internal pages.
1731 */
1732 if (options & PMAP_OPTIONS_COMPRESSOR) {
1733 assert(IS_INTERNAL_PAGE(pai));
1734 }
1735 if (pmap != kernel_pmap) {
1736 /* update pmap stats */
1737 if (IS_REUSABLE_PAGE(pai)) {
1738 assert(pmap->stats.reusable > 0);
1739 OSAddAtomic(-1, &pmap->stats.reusable);
1740 } else if (IS_INTERNAL_PAGE(pai)) {
1741 assert(pmap->stats.internal > 0);
1742 OSAddAtomic(-1, &pmap->stats.internal);
1743 } else {
1744 assert(pmap->stats.external > 0);
1745 OSAddAtomic(-1, &pmap->stats.external);
1746 }
1747 if ((options & PMAP_OPTIONS_COMPRESSOR) &&
1748 IS_INTERNAL_PAGE(pai)) {
1749 /* adjust "compressed" stats */
1750 OSAddAtomic64(+1, &pmap->stats.compressed);
1751 PMAP_STATS_PEAK(pmap->stats.compressed);
1752 pmap->stats.compressed_lifetime++;
1753 }
1754
1755 /* update ledgers */
1756 if (IS_ALTACCT_PAGE(pai, pv_e)) {
1757 assert(IS_INTERNAL_PAGE(pai));
1758 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
1759 pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
1760 if (options & PMAP_OPTIONS_COMPRESSOR) {
1761 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
1762 pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE);
1763 }
1764 } else if (IS_REUSABLE_PAGE(pai)) {
1765 assert(!IS_ALTACCT_PAGE(pai, pv_e));
1766 assert(IS_INTERNAL_PAGE(pai));
1767 if (options & PMAP_OPTIONS_COMPRESSOR) {
1768 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
1769 /* was not in footprint, but is now */
1770 pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
1771 }
1772 } else if (IS_INTERNAL_PAGE(pai)) {
1773 assert(!IS_ALTACCT_PAGE(pai, pv_e));
1774 assert(!IS_REUSABLE_PAGE(pai));
1775 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
1776 /*
1777 * Update all stats related to physical
1778 * footprint, which only deals with
1779 * internal pages.
1780 */
1781 if (options & PMAP_OPTIONS_COMPRESSOR) {
1782 /*
1783 * This removal is only being
1784 * done so we can send this page
1785 * to the compressor; therefore
1786 * it mustn't affect total task
1787 * footprint.
1788 */
1789 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
1790 } else {
1791 /*
1792 * This internal page isn't
1793 * going to the compressor,
1794 * so adjust stats to keep
1795 * phys_footprint up to date.
1796 */
1797 pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
1798 }
1799 }
1800 }
1801
1802 /*
1803 * Deal with the pv_rooted_entry.
1804 */
1805
1806 if (pv_e == pv_h) {
1807 /*
1808 * Fix up head later.
1809 */
1810 pv_h->pmap = PMAP_NULL;
1811 } else {
1812 /*
1813 * Delete this entry.
1814 */
1815 pv_hash_remove(pvh_e);
1816 pvh_e->qlink.next = (queue_entry_t) pvh_eh;
1817 pvh_eh = pvh_e;
1818
1819 if (pvh_et == PV_HASHED_ENTRY_NULL)
1820 pvh_et = pvh_e;
1821 pvh_cnt++;
1822 }
1823 } else {
1824 /*
1825 * Write-protect, after opportunistic refmod collect
1826 */
1827 if (!is_ept) {
1828 pmap_phys_attributes[pai] |=
1829 *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
1830 } else {
1831 pmap_phys_attributes[pai] |=
1832 ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
1833 }
1834 pmap_update_pte(pte, PTE_WRITE(is_ept), 0);
1835
1836 if (options & PMAP_OPTIONS_NOFLUSH)
1837 PMAP_UPDATE_TLBS_DELAYED(pmap, vaddr, vaddr + PAGE_SIZE, (pmap_flush_context *)arg);
1838 else
1839 PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE);
1840 }
1841 pvh_e = nexth;
1842 } while ((pv_e = (pv_rooted_entry_t) nexth) != pv_h);
1843
1844
1845 /*
1846 * If pv_head mapping was removed, fix it up.
1847 */
1848 if (pv_h->pmap == PMAP_NULL) {
1849 pvh_e = (pv_hashed_entry_t) queue_next(&pv_h->qlink);
1850
1851 if (pvh_e != (pv_hashed_entry_t) pv_h) {
1852 pv_hash_remove(pvh_e);
1853 pv_h->pmap = pvh_e->pmap;
1854 pv_h->va_and_flags = pvh_e->va_and_flags;
1855 pvh_e->qlink.next = (queue_entry_t) pvh_eh;
1856 pvh_eh = pvh_e;
1857
1858 if (pvh_et == PV_HASHED_ENTRY_NULL)
1859 pvh_et = pvh_e;
1860 pvh_cnt++;
1861 }
1862 }
1863 if (pvh_eh != PV_HASHED_ENTRY_NULL) {
1864 PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pvh_cnt);
1865 }
1866done:
1867 UNLOCK_PVH(pai);
1868
1869 PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);
1870}
1871
1872
1873/*
1874 * Clear specified attribute bits.
1875 */
1876void
1877phys_attribute_clear(
1878 ppnum_t pn,
1879 int bits,
1880 unsigned int options,
1881 void *arg)
1882{
1883 pv_rooted_entry_t pv_h;
1884 pv_hashed_entry_t pv_e;
1885 pt_entry_t *pte = NULL;
1886 int pai;
1887 pmap_t pmap;
1888 char attributes = 0;
1889 boolean_t is_internal, is_reusable, is_altacct, is_ept;
1890 int ept_bits_to_clear;
1891 boolean_t ept_keep_global_mod = FALSE;
1892
1893 if ((bits & PHYS_MODIFIED) &&
1894 (options & PMAP_OPTIONS_NOFLUSH) &&
1895 arg == NULL) {
1896 panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p): "
1897 "should not clear 'modified' without flushing TLBs\n",
1898 pn, bits, options, arg);
1899 }
1900
1901 /* We only support converting MOD and REF bits for EPT PTEs in this function */
1902 assert((bits & ~(PHYS_REFERENCED | PHYS_MODIFIED)) == 0);
1903
1904 ept_bits_to_clear = (unsigned)physmap_refmod_to_ept(bits & (PHYS_MODIFIED | PHYS_REFERENCED));
1905
1906 pmap_intr_assert();
1907 assert(pn != vm_page_fictitious_addr);
1908 if (pn == vm_page_guard_addr)
1909 return;
1910
1911 pai = ppn_to_pai(pn);
1912
1913 if (!IS_MANAGED_PAGE(pai)) {
1914 /*
1915 * Not a managed page.
1916 */
1917 return;
1918 }
1919
1920 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);
1921
1922 pv_h = pai_to_pvh(pai);
1923
1924 LOCK_PVH(pai);
1925
1926
1927 /*
1928 * Walk down PV list, clearing all modify or reference bits.
1929 * We do not have to lock the pv_list because we have
1930 * the per-pmap lock
1931 */
1932 if (pv_h->pmap != PMAP_NULL) {
1933 /*
1934 * There are some mappings.
1935 */
1936
1937 is_internal = IS_INTERNAL_PAGE(pai);
1938 is_reusable = IS_REUSABLE_PAGE(pai);
1939
1940 pv_e = (pv_hashed_entry_t)pv_h;
1941
1942 do {
1943 vm_map_offset_t va;
1944 char pte_bits;
1945
1946 pmap = pv_e->pmap;
1947 is_ept = is_ept_pmap(pmap);
1948 is_altacct = IS_ALTACCT_PAGE(pai, pv_e);
1949 va = PVE_VA(pv_e);
1950 pte_bits = 0;
1951
1952 if (bits) {
1953 pte = pmap_pte(pmap, va);
1954 /* grab ref/mod bits from this PTE */
1955 pte_bits = (*pte & (PTE_REF(is_ept) | PTE_MOD(is_ept)));
1956 /* propagate to page's global attributes */
1957 if (!is_ept) {
1958 attributes |= pte_bits;
1959 } else {
1960 attributes |= ept_refmod_to_physmap(pte_bits);
1961 if (!pmap_ept_support_ad && (pte_bits & INTEL_EPT_MOD)) {
1962 ept_keep_global_mod = TRUE;
1963 }
1964 }
1965 /* which bits to clear for this PTE? */
1966 if (!is_ept) {
1967 pte_bits &= bits;
1968 } else {
1969 pte_bits &= ept_bits_to_clear;
1970 }
1971 }
1972 if (options & PMAP_OPTIONS_CLEAR_WRITE)
1973 pte_bits |= PTE_WRITE(is_ept);
1974
1975 /*
1976 * Clear modify and/or reference bits.
1977 */
1978 if (pte_bits) {
1979 pmap_update_pte(pte, pte_bits, 0);
1980
1981 /* Ensure all processors using this translation
1982 * invalidate this TLB entry. The invalidation
1983 * *must* follow the PTE update, to ensure that
1984 * the TLB shadow of the 'D' bit (in particular)
1985 * is synchronized with the updated PTE.
1986 */
1987 if (! (options & PMAP_OPTIONS_NOFLUSH)) {
1988 /* flush TLBS now */
1989 PMAP_UPDATE_TLBS(pmap,
1990 va,
1991 va + PAGE_SIZE);
1992 } else if (arg) {
1993 /* delayed TLB flush: add "pmap" info */
1994 PMAP_UPDATE_TLBS_DELAYED(
1995 pmap,
1996 va,
1997 va + PAGE_SIZE,
1998 (pmap_flush_context *)arg);
1999 } else {
2000 /* no TLB flushing at all */
2001 }
2002 }
2003
2004 /* update pmap "reusable" stats */
2005 if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
2006 is_reusable &&
2007 pmap != kernel_pmap) {
2008 /* one less "reusable" */
2009 assert(pmap->stats.reusable > 0);
2010 OSAddAtomic(-1, &pmap->stats.reusable);
2011 if (is_internal) {
2012 /* one more "internal" */
2013 OSAddAtomic(+1, &pmap->stats.internal);
2014 PMAP_STATS_PEAK(pmap->stats.internal);
2015 assert(pmap->stats.internal > 0);
2016 if (is_altacct) {
2017 /* no impact on ledgers */
2018 } else {
2019 pmap_ledger_credit(pmap,
2020 task_ledgers.internal,
2021 PAGE_SIZE);
2022 pmap_ledger_credit(
2023 pmap,
2024 task_ledgers.phys_footprint,
2025 PAGE_SIZE);
2026 }
2027 } else {
2028 /* one more "external" */
2029 OSAddAtomic(+1, &pmap->stats.external);
2030 PMAP_STATS_PEAK(pmap->stats.external);
2031 assert(pmap->stats.external > 0);
2032 }
2033 } else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
2034 !is_reusable &&
2035 pmap != kernel_pmap) {
2036 /* one more "reusable" */
2037 OSAddAtomic(+1, &pmap->stats.reusable);
2038 PMAP_STATS_PEAK(pmap->stats.reusable);
2039 assert(pmap->stats.reusable > 0);
2040 if (is_internal) {
2041 /* one less "internal" */
2042 assert(pmap->stats.internal > 0);
2043 OSAddAtomic(-1, &pmap->stats.internal);
2044 if (is_altacct) {
2045 /* no impact on footprint */
2046 } else {
2047 pmap_ledger_debit(pmap,
2048 task_ledgers.internal,
2049 PAGE_SIZE);
2050 pmap_ledger_debit(
2051 pmap,
2052 task_ledgers.phys_footprint,
2053 PAGE_SIZE);
2054 }
2055 } else {
2056 /* one less "external" */
2057 assert(pmap->stats.external > 0);
2058 OSAddAtomic(-1, &pmap->stats.external);
2059 }
2060 }
2061
2062 pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
2063
2064 } while (pv_e != (pv_hashed_entry_t)pv_h);
2065 }
2066 /* Opportunistic refmod collection, annulled
2067 * if both REF and MOD are being cleared.
2068 */
2069
2070 pmap_phys_attributes[pai] |= attributes;
2071
2072 if (ept_keep_global_mod) {
2073 /*
2074 * If the hardware doesn't support AD bits for EPT PTEs and someone is
2075 * requesting that we clear the modified bit for a phys page, we need
2076 * to ensure that there are no EPT mappings for the page with the
2077 * modified bit set. If there are, we cannot clear the global modified bit.
2078 */
2079 bits &= ~PHYS_MODIFIED;
2080 }
2081 pmap_phys_attributes[pai] &= ~(bits);
2082
2083 /* update this page's "reusable" status */
2084 if (options & PMAP_OPTIONS_CLEAR_REUSABLE) {
2085 pmap_phys_attributes[pai] &= ~PHYS_REUSABLE;
2086 } else if (options & PMAP_OPTIONS_SET_REUSABLE) {
2087 pmap_phys_attributes[pai] |= PHYS_REUSABLE;
2088 }
2089
2090 UNLOCK_PVH(pai);
2091
2092 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);
2093}
2094
2095/*
2096 * Check specified attribute bits.
2097 */
2098int
2099phys_attribute_test(
2100 ppnum_t pn,
2101 int bits)
2102{
2103 pv_rooted_entry_t pv_h;
2104 pv_hashed_entry_t pv_e;
2105 pt_entry_t *pte;
2106 int pai;
2107 pmap_t pmap;
2108 int attributes = 0;
2109 boolean_t is_ept;
2110
2111 pmap_intr_assert();
2112 assert(pn != vm_page_fictitious_addr);
2113 assert((bits & ~(PHYS_MODIFIED | PHYS_REFERENCED)) == 0);
2114 if (pn == vm_page_guard_addr)
2115 return 0;
2116
2117 pai = ppn_to_pai(pn);
2118
2119 if (!IS_MANAGED_PAGE(pai)) {
2120 /*
2121 * Not a managed page.
2122 */
2123 return 0;
2124 }
2125
2126 /*
2127 * Fast check... if bits already collected
2128 * no need to take any locks...
2129 * if not set, we need to recheck after taking
2130 * the lock in case they got pulled in while
2131 * we were waiting for the lock
2132 */
2133 if ((pmap_phys_attributes[pai] & bits) == bits)
2134 return bits;
2135
2136 pv_h = pai_to_pvh(pai);
2137
2138 LOCK_PVH(pai);
2139
2140 attributes = pmap_phys_attributes[pai] & bits;
2141
2142
2143 /*
2144 * Walk down PV list, checking the mappings until we
2145 * reach the end or we've found the desired attributes.
2146 */
2147 if (attributes != bits &&
2148 pv_h->pmap != PMAP_NULL) {
2149 /*
2150 * There are some mappings.
2151 */
2152 pv_e = (pv_hashed_entry_t)pv_h;
2153 do {
2154 vm_map_offset_t va;
2155
2156 pmap = pv_e->pmap;
2157 is_ept = is_ept_pmap(pmap);
2158 va = PVE_VA(pv_e);
2159 /*
2160 * pick up modify and/or reference bits from mapping
2161 */
2162
2163 pte = pmap_pte(pmap, va);
2164 if (!is_ept) {
2165 attributes |= (int)(*pte & bits);
2166 } else {
2167 attributes |= (int)(ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED));
2168
2169 }
2170
2171 pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
2172
2173 } while ((attributes != bits) &&
2174 (pv_e != (pv_hashed_entry_t)pv_h));
2175 }
2176 pmap_phys_attributes[pai] |= attributes;
2177
2178 UNLOCK_PVH(pai);
2179 return (attributes);
2180}
2181
2182/*
2183 * Routine: pmap_change_wiring
2184 * Function: Change the wiring attribute for a map/virtual-address
2185 * pair.
2186 * In/out conditions:
2187 * The mapping must already exist in the pmap.
2188 */
2189void
2190pmap_change_wiring(
2191 pmap_t map,
2192 vm_map_offset_t vaddr,
2193 boolean_t wired)
2194{
2195 pt_entry_t *pte;
2196
2197 PMAP_LOCK(map);
2198
2199 if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
2200 panic("pmap_change_wiring(%p,0x%llx,%d): pte missing",
2201 map, vaddr, wired);
2202
2203 if (wired && !iswired(*pte)) {
2204 /*
2205 * wiring down mapping
2206 */
2207 pmap_ledger_credit(map, task_ledgers.wired_mem, PAGE_SIZE);
2208 OSAddAtomic(+1, &map->stats.wired_count);
2209 pmap_update_pte(pte, 0, PTE_WIRED);
2210 }
2211 else if (!wired && iswired(*pte)) {
2212 /*
2213 * unwiring mapping
2214 */
2215 assert(map->stats.wired_count >= 1);
2216 OSAddAtomic(-1, &map->stats.wired_count);
2217 pmap_ledger_debit(map, task_ledgers.wired_mem, PAGE_SIZE);
2218 pmap_update_pte(pte, PTE_WIRED, 0);
2219 }
2220
2221 PMAP_UNLOCK(map);
2222}
2223
2224/*
2225 * "Backdoor" direct map routine for early mappings.
2226 * Useful for mapping memory outside the range
2227 * Sets A, D and NC if requested
2228 */
2229
2230vm_offset_t
2231pmap_map_bd(
2232 vm_offset_t virt,
2233 vm_map_offset_t start_addr,
2234 vm_map_offset_t end_addr,
2235 vm_prot_t prot,
2236 unsigned int flags)
2237{
2238 pt_entry_t template;
2239 pt_entry_t *ptep;
2240
2241 vm_offset_t base = virt;
2242 boolean_t doflush = FALSE;
2243
2244 template = pa_to_pte(start_addr)
2245 | INTEL_PTE_REF
2246 | INTEL_PTE_MOD
2247 | INTEL_PTE_WIRED
2248 | INTEL_PTE_VALID;
2249
2250 if ((flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT)) == VM_MEM_NOT_CACHEABLE) {
2251 template |= INTEL_PTE_NCACHE;
2252 if (!(flags & (VM_MEM_GUARDED)))
2253 template |= INTEL_PTE_PTA;
2254 }
2255
2256 if ((prot & VM_PROT_EXECUTE) == 0)
2257 template |= INTEL_PTE_NX;
2258
2259 if (prot & VM_PROT_WRITE)
2260 template |= INTEL_PTE_WRITE;
2261
2262 while (start_addr < end_addr) {
2263 ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
2264 if (ptep == PT_ENTRY_NULL) {
2265 panic("pmap_map_bd: Invalid kernel address");
2266 }
2267 if (pte_to_pa(*ptep)) {
2268 doflush = TRUE;
2269 }
2270 pmap_store_pte(ptep, template);
2271 pte_increment_pa(template);
2272 virt += PAGE_SIZE;
2273 start_addr += PAGE_SIZE;
2274 }
2275 if (doflush) {
2276 flush_tlb_raw();
2277 PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
2278 }
2279 return(virt);
2280}
2281
2282/* Create a virtual alias beginning at 'ava' of the specified kernel virtual
2283 * range. The aliased pagetable range is expanded if
2284 * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization,
2285 * assumes caller has stabilized the source and destination ranges. Currently
2286 * used to populate sections of the trampoline "doublemap" at CPU startup.
2287 */
2288
2289void
2290pmap_alias(
2291 vm_offset_t ava,
2292 vm_map_offset_t start_addr,
2293 vm_map_offset_t end_addr,
2294 vm_prot_t prot,
2295 unsigned int eoptions)
2296{
2297 pt_entry_t prot_template, template;
2298 pt_entry_t *aptep, *sptep;
2299
2300 prot_template = INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID;
2301 if ((prot & VM_PROT_EXECUTE) == 0)
2302 prot_template |= INTEL_PTE_NX;
2303
2304 if (prot & VM_PROT_WRITE)
2305 prot_template |= INTEL_PTE_WRITE;
2306 assert(((start_addr | end_addr) & PAGE_MASK) == 0);
2307 while (start_addr < end_addr) {
2308 aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
2309 if (aptep == PT_ENTRY_NULL) {
2310 if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) {
2311 pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP);
2312 aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
2313 } else {
2314 panic("pmap_alias: Invalid alias address");
2315 }
2316 }
2317 /* The aliased range should not have any active mappings */
2318 assert(pte_to_pa(*aptep) == 0);
2319
2320 sptep = pmap_pte(kernel_pmap, start_addr);
2321 assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0));
2322 template = pa_to_pte(pte_to_pa(*sptep)) | prot_template;
2323 pmap_store_pte(aptep, template);
2324
2325 ava += PAGE_SIZE;
2326 start_addr += PAGE_SIZE;
2327 }
2328}
2329
2330mach_vm_size_t
2331pmap_query_resident(
2332 pmap_t pmap,
2333 addr64_t s64,
2334 addr64_t e64,
2335 mach_vm_size_t *compressed_bytes_p)
2336{
2337 pt_entry_t *pde;
2338 pt_entry_t *spte, *epte;
2339 addr64_t l64;
2340 uint64_t deadline;
2341 mach_vm_size_t resident_bytes;
2342 mach_vm_size_t compressed_bytes;
2343 boolean_t is_ept;
2344
2345 pmap_intr_assert();
2346
2347 if (pmap == PMAP_NULL || pmap == kernel_pmap || s64 == e64) {
2348 if (compressed_bytes_p) {
2349 *compressed_bytes_p = 0;
2350 }
2351 return 0;
2352 }
2353
2354 is_ept = is_ept_pmap(pmap);
2355
2356 PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
2357 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(s64),
2358 VM_KERNEL_ADDRHIDE(e64));
2359
2360 resident_bytes = 0;
2361 compressed_bytes = 0;
2362
2363 PMAP_LOCK(pmap);
2364
2365 deadline = rdtsc64() + max_preemption_latency_tsc;
2366
2367 while (s64 < e64) {
2368 l64 = (s64 + pde_mapped_size) & ~(pde_mapped_size - 1);
2369 if (l64 > e64)
2370 l64 = e64;
2371 pde = pmap_pde(pmap, s64);
2372
2373 if (pde && (*pde & PTE_VALID_MASK(is_ept))) {
2374 if (*pde & PTE_PS) {
2375 /* superpage: not supported */
2376 } else {
2377 spte = pmap_pte(pmap,
2378 (s64 & ~(pde_mapped_size - 1)));
2379 spte = &spte[ptenum(s64)];
2380 epte = &spte[intel_btop(l64 - s64)];
2381
2382 for (; spte < epte; spte++) {
2383 if (pte_to_pa(*spte) != 0) {
2384 resident_bytes += PAGE_SIZE;
2385 } else if (*spte & PTE_COMPRESSED) {
2386 compressed_bytes += PAGE_SIZE;
2387 }
2388 }
2389
2390 }
2391 }
2392 s64 = l64;
2393
2394 if (s64 < e64 && rdtsc64() >= deadline) {
2395 PMAP_UNLOCK(pmap);
2396 PMAP_LOCK(pmap);
2397 deadline = rdtsc64() + max_preemption_latency_tsc;
2398 }
2399 }
2400
2401 PMAP_UNLOCK(pmap);
2402
2403 PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
2404 resident_bytes);
2405
2406 if (compressed_bytes_p) {
2407 *compressed_bytes_p = compressed_bytes;
2408 }
2409 return resident_bytes;
2410}
2411
2412kern_return_t
2413pmap_query_page_info(
2414 pmap_t pmap,
2415 vm_map_offset_t va,
2416 int *disp_p)
2417{
2418 int disp;
2419 boolean_t is_ept;
2420 pmap_paddr_t pa;
2421 ppnum_t pai;
2422 pd_entry_t *pde;
2423 pt_entry_t *pte;
2424
2425 pmap_intr_assert();
2426 if (pmap == PMAP_NULL || pmap == kernel_pmap) {
2427 *disp_p = 0;
2428 return KERN_INVALID_ARGUMENT;
2429 }
2430
2431 disp = 0;
2432 is_ept = is_ept_pmap(pmap);
2433
2434 PMAP_LOCK(pmap);
2435
2436 pde = pmap_pde(pmap, va);
2437 if (!pde ||
2438 !(*pde & PTE_VALID_MASK(is_ept)) ||
2439 (*pde & PTE_PS)) {
2440 goto done;
2441 }
2442
2443 pte = pmap_pte(pmap, va);
2444 if (pte == PT_ENTRY_NULL) {
2445 goto done;
2446 }
2447
2448 pa = pte_to_pa(*pte);
2449 if (pa == 0) {
2450 if (PTE_IS_COMPRESSED(*pte)) {
2451 disp |= PMAP_QUERY_PAGE_COMPRESSED;
2452 if (*pte & PTE_COMPRESSED_ALT) {
2453 disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
2454 }
2455 }
2456 } else {
2457 disp |= PMAP_QUERY_PAGE_PRESENT;
2458 pai = pa_index(pa);
2459 if (!IS_MANAGED_PAGE(pai)) {
2460 } else if (pmap_pv_is_altacct(pmap, va, pai)) {
2461 assert(IS_INTERNAL_PAGE(pai));
2462 disp |= PMAP_QUERY_PAGE_INTERNAL;
2463 disp |= PMAP_QUERY_PAGE_ALTACCT;
2464 } else if (IS_REUSABLE_PAGE(pai)) {
2465 disp |= PMAP_QUERY_PAGE_REUSABLE;
2466 } else if (IS_INTERNAL_PAGE(pai)) {
2467 disp |= PMAP_QUERY_PAGE_INTERNAL;
2468 }
2469 }
2470
2471done:
2472 PMAP_UNLOCK(pmap);
2473 *disp_p = disp;
2474 return KERN_SUCCESS;
2475}
2476
2477void
2478pmap_set_jit_entitled(__unused pmap_t pmap)
2479{
2480 /* The x86 pmap layer does not care if a map has a JIT entry. */
2481 return;
2482}
2483
2484bool
2485pmap_has_prot_policy(__unused vm_prot_t prot)
2486{
2487 /*
2488 * The x86 pmap layer does not apply any policy to any protection
2489 * types.
2490 */
2491 return FALSE;
2492}
2493
2494uint64_t
2495pmap_release_pages_fast(void)
2496{
2497 return 0;
2498}
2499
2500void
2501pmap_trim(__unused pmap_t grand, __unused pmap_t subord, __unused addr64_t vstart, __unused addr64_t nstart, __unused uint64_t size)
2502{
2503 return;
2504}
2505
2506void pmap_ledger_alloc_init(size_t size)
2507{
2508 panic("%s: unsupported, "
2509 "size=%lu",
2510 __func__, size);
2511}
2512
2513ledger_t pmap_ledger_alloc(void)
2514{
2515 panic("%s: unsupported",
2516 __func__);
2517
2518 return NULL;
2519}
2520
2521void pmap_ledger_free(ledger_t ledger)
2522{
2523 panic("%s: unsupported, "
2524 "ledger=%p",
2525 __func__, ledger);
2526}
2527
2528size_t
2529pmap_dump_page_tables(pmap_t pmap __unused, void *bufp __unused, void *buf_end __unused)
2530{
2531 return (size_t)-1;
2532}
2533
2534