1/*
2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <vm/pmap.h>
29#include <kern/ledger.h>
30#include <i386/pmap_internal.h>
31
32
33/*
34 * Each entry in the pv_head_table is locked by a bit in the
35 * pv_lock_table. The lock bits are accessed by the physical
36 * address of the page they lock.
37 */
38
39char *pv_lock_table; /* pointer to array of bits */
40char *pv_hash_lock_table;
41
42pv_rooted_entry_t pv_head_table; /* array of entries, one per
43 * page */
44uint32_t pv_hashed_free_count = 0;
45uint32_t pv_hashed_kern_free_count = 0;
46
47pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
48uint32_t pmap_pagetable_corruption_incidents;
49uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
50uint64_t pmap_pagetable_corruption_interval_abstime;
51thread_call_t pmap_pagetable_corruption_log_call;
52static thread_call_data_t pmap_pagetable_corruption_log_call_data;
53boolean_t pmap_pagetable_corruption_timeout = FALSE;
54
55volatile uint32_t mappingrecurse = 0;
56
57uint32_t pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
58
59thread_t mapping_replenish_thread;
60event_t mapping_replenish_event, pmap_user_pv_throttle_event;
61
62uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
63
64int pmap_asserts_enabled = (DEBUG);
65int pmap_asserts_traced = 0;
66
67unsigned int pmap_cache_attributes(ppnum_t pn) {
68 if (pmap_get_cache_attributes(pn, FALSE) & INTEL_PTE_NCACHE)
69 return (VM_WIMG_IO);
70 else
71 return (VM_WIMG_COPYBACK);
72}
73
74void pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
75 unsigned int current, template = 0;
76 int pai;
77
78 if (cacheattr & VM_MEM_NOT_CACHEABLE) {
79 if(!(cacheattr & VM_MEM_GUARDED))
80 template |= PHYS_PTA;
81 template |= PHYS_NCACHE;
82 }
83
84 pmap_intr_assert();
85
86 assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
87
88 pai = ppn_to_pai(pn);
89
90 if (!IS_MANAGED_PAGE(pai)) {
91 return;
92 }
93
94 /* override cache attributes for this phys page
95 * Does not walk through existing mappings to adjust,
96 * assumes page is disconnected
97 */
98
99 LOCK_PVH(pai);
100
101 pmap_update_cache_attributes_locked(pn, template);
102
103 current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
104 pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
105 pmap_phys_attributes[pai] |= template;
106
107 UNLOCK_PVH(pai);
108
109 if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
110 pmap_sync_page_attributes_phys(pn);
111 }
112}
113
114unsigned pmap_get_cache_attributes(ppnum_t pn, boolean_t is_ept) {
115 if (last_managed_page == 0)
116 return 0;
117
118 if (!IS_MANAGED_PAGE(ppn_to_pai(pn)))
119 return PTE_NCACHE(is_ept);
120
121 /*
122 * The cache attributes are read locklessly for efficiency.
123 */
124 unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
125 unsigned int template = 0;
126
127 /*
128 * The PTA bit is currently unsupported for EPT PTEs.
129 */
130 if ((attr & PHYS_PTA) && !is_ept)
131 template |= INTEL_PTE_PTA;
132
133 /*
134 * If the page isn't marked as NCACHE, the default for EPT entries
135 * is WB.
136 */
137 if (attr & PHYS_NCACHE)
138 template |= PTE_NCACHE(is_ept);
139 else if (is_ept)
140 template |= INTEL_EPT_WB;
141
142 return template;
143}
144
145boolean_t
146pmap_has_managed_page(ppnum_t first, ppnum_t last)
147{
148 ppnum_t pn, kdata_start, kdata_end;
149 boolean_t result;
150 boot_args * args;
151
152 args = (boot_args *) PE_state.bootArgs;
153
154 // Allow pages that the booter added to the end of the kernel.
155 // We may miss reporting some pages in this range that were freed
156 // with ml_static_free()
157 kdata_start = atop_32(args->kaddr);
158 kdata_end = atop_32(args->kaddr + args->ksize);
159
160 assert(last_managed_page);
161 assert(first <= last);
162
163 for (result = FALSE, pn = first;
164 !result
165 && (pn <= last)
166 && (pn <= last_managed_page);
167 pn++)
168 {
169 if ((pn >= kdata_start) && (pn < kdata_end)) continue;
170 result = (0 != (pmap_phys_attributes[pn] & PHYS_MANAGED));
171 }
172
173 return (result);
174}
175
176boolean_t
177pmap_is_noencrypt(ppnum_t pn)
178{
179 int pai;
180
181 pai = ppn_to_pai(pn);
182
183 if (!IS_MANAGED_PAGE(pai))
184 return (FALSE);
185
186 if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
187 return (TRUE);
188
189 return (FALSE);
190}
191
192
193void
194pmap_set_noencrypt(ppnum_t pn)
195{
196 int pai;
197
198 pai = ppn_to_pai(pn);
199
200 if (IS_MANAGED_PAGE(pai)) {
201 LOCK_PVH(pai);
202
203 pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;
204
205 UNLOCK_PVH(pai);
206 }
207}
208
209
210void
211pmap_clear_noencrypt(ppnum_t pn)
212{
213 int pai;
214
215 pai = ppn_to_pai(pn);
216
217 if (IS_MANAGED_PAGE(pai)) {
218 /*
219 * synchronization at VM layer prevents PHYS_NOENCRYPT
220 * from changing state, so we don't need the lock to inspect
221 */
222 if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
223 LOCK_PVH(pai);
224
225 pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
226
227 UNLOCK_PVH(pai);
228 }
229 }
230}
231
232void
233compute_pmap_gc_throttle(void *arg __unused)
234{
235
236}
237
238
239void
240pmap_lock_phys_page(ppnum_t pn)
241{
242 int pai;
243
244 pai = ppn_to_pai(pn);
245
246 if (IS_MANAGED_PAGE(pai)) {
247 LOCK_PVH(pai);
248 } else
249 simple_lock(&phys_backup_lock);
250}
251
252
253void
254pmap_unlock_phys_page(ppnum_t pn)
255{
256 int pai;
257
258 pai = ppn_to_pai(pn);
259
260 if (IS_MANAGED_PAGE(pai)) {
261 UNLOCK_PVH(pai);
262 } else
263 simple_unlock(&phys_backup_lock);
264}
265
266
267
268__private_extern__ void
269pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
270 if (pmap_pagetable_corruption_incidents > 0) {
271 int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
272 (*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
273 for (i = 0; i < e; i++) {
274 (*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident, pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
275 }
276 }
277}
278
279static inline void
280pmap_pagetable_corruption_log_setup(void) {
281 if (pmap_pagetable_corruption_log_call == NULL) {
282 nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
283 thread_call_setup(&pmap_pagetable_corruption_log_call_data,
284 (thread_call_func_t) pmap_pagetable_corruption_msg_log,
285 (thread_call_param_t) &printf);
286 pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
287 }
288}
289
290void
291mapping_free_prime(void)
292{
293 unsigned i;
294 pv_hashed_entry_t pvh_e;
295 pv_hashed_entry_t pvh_eh;
296 pv_hashed_entry_t pvh_et;
297 int pv_cnt;
298
299 /* Scale based on DRAM size */
300 pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
301 pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
302 /* Alterable via sysctl */
303 pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
304 pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
305 pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
306 pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
307
308 pv_cnt = 0;
309 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
310
311 for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
312 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
313
314 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
315 pvh_eh = pvh_e;
316
317 if (pvh_et == PV_HASHED_ENTRY_NULL)
318 pvh_et = pvh_e;
319 pv_cnt++;
320 }
321 PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
322
323 pv_cnt = 0;
324 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
325 for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
326 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
327
328 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
329 pvh_eh = pvh_e;
330
331 if (pvh_et == PV_HASHED_ENTRY_NULL)
332 pvh_et = pvh_e;
333 pv_cnt++;
334 }
335 PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
336}
337
338void mapping_replenish(void);
339
340void mapping_adjust(void) {
341 kern_return_t mres;
342
343 pmap_pagetable_corruption_log_setup();
344
345 mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
346 if (mres != KERN_SUCCESS) {
347 panic("pmap: mapping_replenish_thread creation failed");
348 }
349 thread_deallocate(mapping_replenish_thread);
350}
351
352unsigned pmap_mapping_thread_wakeups;
353unsigned pmap_kernel_reserve_replenish_stat;
354unsigned pmap_user_reserve_replenish_stat;
355unsigned pmap_kern_reserve_alloc_stat;
356
357__attribute__((noreturn))
358void
359mapping_replenish(void)
360{
361 pv_hashed_entry_t pvh_e;
362 pv_hashed_entry_t pvh_eh;
363 pv_hashed_entry_t pvh_et;
364 int pv_cnt;
365 unsigned i;
366
367 /* We qualify for VM privileges...*/
368 current_thread()->options |= TH_OPT_VMPRIV;
369
370 for (;;) {
371
372 while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
373 pv_cnt = 0;
374 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
375
376 for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
377 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
378 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
379 pvh_eh = pvh_e;
380
381 if (pvh_et == PV_HASHED_ENTRY_NULL)
382 pvh_et = pvh_e;
383 pv_cnt++;
384 }
385 pmap_kernel_reserve_replenish_stat += pv_cnt;
386 PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
387 }
388
389 pv_cnt = 0;
390 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
391
392 if (pv_hashed_free_count < pv_hashed_low_water_mark) {
393 for (i = 0; i < pv_hashed_alloc_chunk; i++) {
394 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
395
396 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
397 pvh_eh = pvh_e;
398
399 if (pvh_et == PV_HASHED_ENTRY_NULL)
400 pvh_et = pvh_e;
401 pv_cnt++;
402 }
403 pmap_user_reserve_replenish_stat += pv_cnt;
404 PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
405 }
406/* Wake threads throttled while the kernel reserve was being replenished.
407 */
408 if (pmap_pv_throttled_waiters) {
409 pmap_pv_throttled_waiters = 0;
410 thread_wakeup(&pmap_user_pv_throttle_event);
411 }
412 /* Check if the kernel pool has been depleted since the
413 * first pass, to reduce refill latency.
414 */
415 if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
416 continue;
417 /* Block sans continuation to avoid yielding kernel stack */
418 assert_wait(&mapping_replenish_event, THREAD_UNINT);
419 mappingrecurse = 0;
420 thread_block(THREAD_CONTINUE_NULL);
421 pmap_mapping_thread_wakeups++;
422 }
423}
424
425/*
426 * Set specified attribute bits.
427 */
428
429void
430phys_attribute_set(
431 ppnum_t pn,
432 int bits)
433{
434 int pai;
435
436 pmap_intr_assert();
437 assert(pn != vm_page_fictitious_addr);
438 if (pn == vm_page_guard_addr)
439 return;
440
441 pai = ppn_to_pai(pn);
442
443 if (!IS_MANAGED_PAGE(pai)) {
444 /* Not a managed page. */
445 return;
446 }
447
448 LOCK_PVH(pai);
449 pmap_phys_attributes[pai] |= bits;
450 UNLOCK_PVH(pai);
451}
452
453/*
454 * Set the modify bit on the specified physical page.
455 */
456
457void
458pmap_set_modify(ppnum_t pn)
459{
460 phys_attribute_set(pn, PHYS_MODIFIED);
461}
462
463/*
464 * Clear the modify bits on the specified physical page.
465 */
466
467void
468pmap_clear_modify(ppnum_t pn)
469{
470 phys_attribute_clear(pn, PHYS_MODIFIED, 0, NULL);
471}
472
473/*
474 * pmap_is_modified:
475 *
476 * Return whether or not the specified physical page is modified
477 * by any physical maps.
478 */
479
480boolean_t
481pmap_is_modified(ppnum_t pn)
482{
483 if (phys_attribute_test(pn, PHYS_MODIFIED))
484 return TRUE;
485 return FALSE;
486}
487
488
489/*
490 * pmap_clear_reference:
491 *
492 * Clear the reference bit on the specified physical page.
493 */
494
495void
496pmap_clear_reference(ppnum_t pn)
497{
498 phys_attribute_clear(pn, PHYS_REFERENCED, 0, NULL);
499}
500
501void
502pmap_set_reference(ppnum_t pn)
503{
504 phys_attribute_set(pn, PHYS_REFERENCED);
505}
506
507/*
508 * pmap_is_referenced:
509 *
510 * Return whether or not the specified physical page is referenced
511 * by any physical maps.
512 */
513
514boolean_t
515pmap_is_referenced(ppnum_t pn)
516{
517 if (phys_attribute_test(pn, PHYS_REFERENCED))
518 return TRUE;
519 return FALSE;
520}
521
522
523/*
524 * pmap_get_refmod(phys)
525 * returns the referenced and modified bits of the specified
526 * physical page.
527 */
528unsigned int
529pmap_get_refmod(ppnum_t pn)
530{
531 int refmod;
532 unsigned int retval = 0;
533
534 refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);
535
536 if (refmod & PHYS_MODIFIED)
537 retval |= VM_MEM_MODIFIED;
538 if (refmod & PHYS_REFERENCED)
539 retval |= VM_MEM_REFERENCED;
540
541 return (retval);
542}
543
544
545void
546pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *arg)
547{
548 unsigned int x86Mask;
549
550 x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
551 | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
552
553 phys_attribute_clear(pn, x86Mask, options, arg);
554}
555
556/*
557 * pmap_clear_refmod(phys, mask)
558 * clears the referenced and modified bits as specified by the mask
559 * of the specified physical page.
560 */
561void
562pmap_clear_refmod(ppnum_t pn, unsigned int mask)
563{
564 unsigned int x86Mask;
565
566 x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
567 | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
568
569 phys_attribute_clear(pn, x86Mask, 0, NULL);
570}
571
572unsigned int
573pmap_disconnect(ppnum_t pa)
574{
575 return (pmap_disconnect_options(pa, 0, NULL));
576}
577
578/*
579 * Routine:
580 * pmap_disconnect_options
581 *
582 * Function:
583 * Disconnect all mappings for this page and return reference and change status
584 * in generic format.
585 *
586 */
587unsigned int
588pmap_disconnect_options(ppnum_t pa, unsigned int options, void *arg)
589{
590 unsigned refmod, vmrefmod = 0;
591
592 pmap_page_protect_options(pa, 0, options, arg); /* disconnect the page */
593
594 pmap_assert(pa != vm_page_fictitious_addr);
595 if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa) || (options & PMAP_OPTIONS_NOREFMOD))
596 return 0;
597 refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
598
599 if (refmod & PHYS_MODIFIED)
600 vmrefmod |= VM_MEM_MODIFIED;
601 if (refmod & PHYS_REFERENCED)
602 vmrefmod |= VM_MEM_REFERENCED;
603
604 return vmrefmod;
605}
606