1/*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_kern.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Kernel memory management.
64 */
65
66#include <mach/kern_return.h>
67#include <mach/vm_param.h>
68#include <kern/assert.h>
69#include <kern/thread.h>
70#include <vm/vm_kern.h>
71#include <vm/vm_map.h>
72#include <vm/vm_object.h>
73#include <vm/vm_page.h>
74#include <vm/vm_compressor.h>
75#include <vm/vm_pageout.h>
76#include <kern/misc_protos.h>
77#include <vm/cpm.h>
78
79#include <string.h>
80
81#include <libkern/OSDebug.h>
82#include <libkern/crypto/sha2.h>
83#include <sys/kdebug.h>
84
85#include <san/kasan.h>
86
87/*
88 * Variables exported by this module.
89 */
90
91vm_map_t kernel_map;
92vm_map_t kernel_pageable_map;
93
94extern boolean_t vm_kernel_ready;
95
96/*
97 * Forward declarations for internal functions.
98 */
99extern kern_return_t kmem_alloc_pages(
100 vm_object_t object,
101 vm_object_offset_t offset,
102 vm_object_size_t size);
103
104kern_return_t
105kmem_alloc_contig(
106 vm_map_t map,
107 vm_offset_t *addrp,
108 vm_size_t size,
109 vm_offset_t mask,
110 ppnum_t max_pnum,
111 ppnum_t pnum_mask,
112 int flags,
113 vm_tag_t tag)
114{
115 vm_object_t object;
116 vm_object_offset_t offset;
117 vm_map_offset_t map_addr;
118 vm_map_offset_t map_mask;
119 vm_map_size_t map_size, i;
120 vm_map_entry_t entry;
121 vm_page_t m, pages;
122 kern_return_t kr;
123
124 assert(VM_KERN_MEMORY_NONE != tag);
125
126 if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT)))
127 return KERN_INVALID_ARGUMENT;
128
129 map_size = vm_map_round_page(size,
130 VM_MAP_PAGE_MASK(map));
131 map_mask = (vm_map_offset_t)mask;
132
133 /* Check for zero allocation size (either directly or via overflow) */
134 if (map_size == 0) {
135 *addrp = 0;
136 return KERN_INVALID_ARGUMENT;
137 }
138
139 /*
140 * Allocate a new object (if necessary) and the reference we
141 * will be donating to the map entry. We must do this before
142 * locking the map, or risk deadlock with the default pager.
143 */
144 if ((flags & KMA_KOBJECT) != 0) {
145 object = kernel_object;
146 vm_object_reference(object);
147 } else {
148 object = vm_object_allocate(map_size);
149 }
150
151 kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0,
152 VM_MAP_KERNEL_FLAGS_NONE, tag, &entry);
153 if (KERN_SUCCESS != kr) {
154 vm_object_deallocate(object);
155 return kr;
156 }
157
158 if (object == kernel_object) {
159 offset = map_addr;
160 } else {
161 offset = 0;
162 }
163 VME_OBJECT_SET(entry, object);
164 VME_OFFSET_SET(entry, offset);
165
166 /* Take an extra object ref in case the map entry gets deleted */
167 vm_object_reference(object);
168 vm_map_unlock(map);
169
170 kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags);
171
172 if (kr != KERN_SUCCESS) {
173 vm_map_remove(map,
174 vm_map_trunc_page(map_addr,
175 VM_MAP_PAGE_MASK(map)),
176 vm_map_round_page(map_addr + map_size,
177 VM_MAP_PAGE_MASK(map)),
178 VM_MAP_REMOVE_NO_FLAGS);
179 vm_object_deallocate(object);
180 *addrp = 0;
181 return kr;
182 }
183
184 vm_object_lock(object);
185 for (i = 0; i < map_size; i += PAGE_SIZE) {
186 m = pages;
187 pages = NEXT_PAGE(m);
188 *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
189 m->vmp_busy = FALSE;
190 vm_page_insert(m, object, offset + i);
191 }
192 vm_object_unlock(object);
193
194 kr = vm_map_wire_kernel(map,
195 vm_map_trunc_page(map_addr,
196 VM_MAP_PAGE_MASK(map)),
197 vm_map_round_page(map_addr + map_size,
198 VM_MAP_PAGE_MASK(map)),
199 VM_PROT_DEFAULT, tag,
200 FALSE);
201
202 if (kr != KERN_SUCCESS) {
203 if (object == kernel_object) {
204 vm_object_lock(object);
205 vm_object_page_remove(object, offset, offset + map_size);
206 vm_object_unlock(object);
207 }
208 vm_map_remove(map,
209 vm_map_trunc_page(map_addr,
210 VM_MAP_PAGE_MASK(map)),
211 vm_map_round_page(map_addr + map_size,
212 VM_MAP_PAGE_MASK(map)),
213 VM_MAP_REMOVE_NO_FLAGS);
214 vm_object_deallocate(object);
215 return kr;
216 }
217 vm_object_deallocate(object);
218
219 if (object == kernel_object) {
220 vm_map_simplify(map, map_addr);
221 vm_tag_update_size(tag, map_size);
222 }
223 *addrp = (vm_offset_t) map_addr;
224 assert((vm_map_offset_t) *addrp == map_addr);
225
226 return KERN_SUCCESS;
227}
228
229/*
230 * Master entry point for allocating kernel memory.
231 * NOTE: this routine is _never_ interrupt safe.
232 *
233 * map : map to allocate into
234 * addrp : pointer to start address of new memory
235 * size : size of memory requested
236 * flags : options
237 * KMA_HERE *addrp is base address, else "anywhere"
238 * KMA_NOPAGEWAIT don't wait for pages if unavailable
239 * KMA_KOBJECT use kernel_object
240 * KMA_LOMEM support for 32 bit devices in a 64 bit world
241 * if set and a lomemory pool is available
242 * grab pages from it... this also implies
243 * KMA_NOPAGEWAIT
244 */
245
246kern_return_t
247kernel_memory_allocate(
248 vm_map_t map,
249 vm_offset_t *addrp,
250 vm_size_t size,
251 vm_offset_t mask,
252 int flags,
253 vm_tag_t tag)
254{
255 vm_object_t object;
256 vm_object_offset_t offset;
257 vm_object_offset_t pg_offset;
258 vm_map_entry_t entry = NULL;
259 vm_map_offset_t map_addr, fill_start;
260 vm_map_offset_t map_mask;
261 vm_map_size_t map_size, fill_size;
262 kern_return_t kr, pe_result;
263 vm_page_t mem;
264 vm_page_t guard_page_list = NULL;
265 vm_page_t wired_page_list = NULL;
266 int guard_page_count = 0;
267 int wired_page_count = 0;
268 int page_grab_count = 0;
269 int i;
270 int vm_alloc_flags;
271 vm_map_kernel_flags_t vmk_flags;
272 vm_prot_t kma_prot;
273
274 if (! vm_kernel_ready) {
275 panic("kernel_memory_allocate: VM is not ready");
276 }
277
278 map_size = vm_map_round_page(size,
279 VM_MAP_PAGE_MASK(map));
280 map_mask = (vm_map_offset_t) mask;
281
282 vm_alloc_flags = 0; //VM_MAKE_TAG(tag);
283 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
284
285 /* Check for zero allocation size (either directly or via overflow) */
286 if (map_size == 0) {
287 *addrp = 0;
288 return KERN_INVALID_ARGUMENT;
289 }
290
291 /*
292 * limit the size of a single extent of wired memory
293 * to try and limit the damage to the system if
294 * too many pages get wired down
295 * limit raised to 2GB with 128GB max physical limit,
296 * but scaled by installed memory above this
297 */
298 if (!(flags & (KMA_VAONLY | KMA_PAGEABLE)) &&
299 map_size > MAX(1ULL<<31, sane_size/64)) {
300 return KERN_RESOURCE_SHORTAGE;
301 }
302
303 /*
304 * Guard pages:
305 *
306 * Guard pages are implemented as ficticious pages. By placing guard pages
307 * on either end of a stack, they can help detect cases where a thread walks
308 * off either end of its stack. They are allocated and set up here and attempts
309 * to access those pages are trapped in vm_fault_page().
310 *
311 * The map_size we were passed may include extra space for
312 * guard pages. If those were requested, then back it out of fill_size
313 * since vm_map_find_space() takes just the actual size not including
314 * guard pages. Similarly, fill_start indicates where the actual pages
315 * will begin in the range.
316 */
317
318 fill_start = 0;
319 fill_size = map_size;
320
321 if (flags & KMA_GUARD_FIRST) {
322 vmk_flags.vmkf_guard_before = TRUE;
323 fill_start += PAGE_SIZE_64;
324 fill_size -= PAGE_SIZE_64;
325 if (map_size < fill_start + fill_size) {
326 /* no space for a guard page */
327 *addrp = 0;
328 return KERN_INVALID_ARGUMENT;
329 }
330 guard_page_count++;
331 }
332 if (flags & KMA_GUARD_LAST) {
333 vmk_flags.vmkf_guard_after = TRUE;
334 fill_size -= PAGE_SIZE_64;
335 if (map_size <= fill_start + fill_size) {
336 /* no space for a guard page */
337 *addrp = 0;
338 return KERN_INVALID_ARGUMENT;
339 }
340 guard_page_count++;
341 }
342 wired_page_count = (int) (fill_size / PAGE_SIZE_64);
343 assert(wired_page_count * PAGE_SIZE_64 == fill_size);
344
345#if DEBUG || DEVELOPMENT
346 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_START, size, 0, 0, 0);
347#endif
348
349 for (i = 0; i < guard_page_count; i++) {
350 for (;;) {
351 mem = vm_page_grab_guard();
352
353 if (mem != VM_PAGE_NULL)
354 break;
355 if (flags & KMA_NOPAGEWAIT) {
356 kr = KERN_RESOURCE_SHORTAGE;
357 goto out;
358 }
359 vm_page_more_fictitious();
360 }
361 mem->vmp_snext = guard_page_list;
362 guard_page_list = mem;
363 }
364
365 if (!(flags & (KMA_VAONLY | KMA_PAGEABLE))) {
366 for (i = 0; i < wired_page_count; i++) {
367 uint64_t unavailable;
368
369 for (;;) {
370 if (flags & KMA_LOMEM)
371 mem = vm_page_grablo();
372 else
373 mem = vm_page_grab();
374
375 if (mem != VM_PAGE_NULL)
376 break;
377
378 if (flags & KMA_NOPAGEWAIT) {
379 kr = KERN_RESOURCE_SHORTAGE;
380 goto out;
381 }
382 if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) {
383 kr = KERN_RESOURCE_SHORTAGE;
384 goto out;
385 }
386 unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE;
387
388 if (unavailable > max_mem || map_size > (max_mem - unavailable)) {
389 kr = KERN_RESOURCE_SHORTAGE;
390 goto out;
391 }
392 VM_PAGE_WAIT();
393 }
394 page_grab_count++;
395 if (KMA_ZERO & flags) vm_page_zero_fill(mem);
396 mem->vmp_snext = wired_page_list;
397 wired_page_list = mem;
398 }
399 }
400
401 /*
402 * Allocate a new object (if necessary). We must do this before
403 * locking the map, or risk deadlock with the default pager.
404 */
405 if ((flags & KMA_KOBJECT) != 0) {
406 object = kernel_object;
407 vm_object_reference(object);
408 } else if ((flags & KMA_COMPRESSOR) != 0) {
409 object = compressor_object;
410 vm_object_reference(object);
411 } else {
412 object = vm_object_allocate(map_size);
413 }
414
415 if (flags & KMA_ATOMIC)
416 vmk_flags.vmkf_atomic_entry = TRUE;
417
418 kr = vm_map_find_space(map, &map_addr,
419 fill_size, map_mask,
420 vm_alloc_flags, vmk_flags, tag, &entry);
421 if (KERN_SUCCESS != kr) {
422 vm_object_deallocate(object);
423 goto out;
424 }
425
426 if (object == kernel_object || object == compressor_object) {
427 offset = map_addr;
428 } else {
429 offset = 0;
430 }
431 VME_OBJECT_SET(entry, object);
432 VME_OFFSET_SET(entry, offset);
433
434 if (!(flags & (KMA_COMPRESSOR | KMA_PAGEABLE)))
435 entry->wired_count++;
436
437 if (flags & KMA_PERMANENT)
438 entry->permanent = TRUE;
439
440 if (object != kernel_object && object != compressor_object)
441 vm_object_reference(object);
442
443 vm_object_lock(object);
444 vm_map_unlock(map);
445
446 pg_offset = 0;
447
448 if (fill_start) {
449 if (guard_page_list == NULL)
450 panic("kernel_memory_allocate: guard_page_list == NULL");
451
452 mem = guard_page_list;
453 guard_page_list = mem->vmp_snext;
454 mem->vmp_snext = NULL;
455
456 vm_page_insert(mem, object, offset + pg_offset);
457
458 mem->vmp_busy = FALSE;
459 pg_offset += PAGE_SIZE_64;
460 }
461
462 kma_prot = VM_PROT_READ | VM_PROT_WRITE;
463
464#if KASAN
465 if (!(flags & KMA_VAONLY)) {
466 /* for VAONLY mappings we notify in populate only */
467 kasan_notify_address(map_addr, size);
468 }
469#endif
470
471 if (flags & (KMA_VAONLY | KMA_PAGEABLE)) {
472 pg_offset = fill_start + fill_size;
473 } else {
474 for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) {
475 if (wired_page_list == NULL)
476 panic("kernel_memory_allocate: wired_page_list == NULL");
477
478 mem = wired_page_list;
479 wired_page_list = mem->vmp_snext;
480 mem->vmp_snext = NULL;
481
482 assert(mem->vmp_wire_count == 0);
483 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
484
485 mem->vmp_q_state = VM_PAGE_IS_WIRED;
486 mem->vmp_wire_count++;
487 if (__improbable(mem->vmp_wire_count == 0)) {
488 panic("kernel_memory_allocate(%p): wire_count overflow",
489 mem);
490 }
491
492 vm_page_insert_wired(mem, object, offset + pg_offset, tag);
493
494 mem->vmp_busy = FALSE;
495 mem->vmp_pmapped = TRUE;
496 mem->vmp_wpmapped = TRUE;
497
498 PMAP_ENTER_OPTIONS(kernel_pmap, map_addr + pg_offset, mem,
499 kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE,
500 PMAP_OPTIONS_NOWAIT, pe_result);
501
502 if (pe_result == KERN_RESOURCE_SHORTAGE) {
503 vm_object_unlock(object);
504
505 PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem,
506 kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE,
507 pe_result);
508
509 vm_object_lock(object);
510 }
511
512 assert(pe_result == KERN_SUCCESS);
513
514 if (flags & KMA_NOENCRYPT) {
515 bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE);
516
517 pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
518 }
519 }
520 if (kernel_object == object) vm_tag_update_size(tag, fill_size);
521 }
522 if ((fill_start + fill_size) < map_size) {
523 if (guard_page_list == NULL)
524 panic("kernel_memory_allocate: guard_page_list == NULL");
525
526 mem = guard_page_list;
527 guard_page_list = mem->vmp_snext;
528 mem->vmp_snext = NULL;
529
530 vm_page_insert(mem, object, offset + pg_offset);
531
532 mem->vmp_busy = FALSE;
533 }
534 if (guard_page_list || wired_page_list)
535 panic("kernel_memory_allocate: non empty list\n");
536
537 if (!(flags & (KMA_VAONLY | KMA_PAGEABLE))) {
538 vm_page_lockspin_queues();
539 vm_page_wire_count += wired_page_count;
540 vm_page_unlock_queues();
541 }
542
543 vm_object_unlock(object);
544
545 /*
546 * now that the pages are wired, we no longer have to fear coalesce
547 */
548 if (object == kernel_object || object == compressor_object)
549 vm_map_simplify(map, map_addr);
550 else
551 vm_object_deallocate(object);
552
553#if DEBUG || DEVELOPMENT
554 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
555#endif
556
557 /*
558 * Return the memory, not zeroed.
559 */
560 *addrp = CAST_DOWN(vm_offset_t, map_addr);
561 return KERN_SUCCESS;
562
563out:
564 if (guard_page_list)
565 vm_page_free_list(guard_page_list, FALSE);
566
567 if (wired_page_list)
568 vm_page_free_list(wired_page_list, FALSE);
569
570#if DEBUG || DEVELOPMENT
571 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
572#endif
573
574 return kr;
575}
576
577kern_return_t
578kernel_memory_populate(
579 vm_map_t map,
580 vm_offset_t addr,
581 vm_size_t size,
582 int flags,
583 vm_tag_t tag)
584{
585 vm_object_t object;
586 vm_object_offset_t offset, pg_offset;
587 kern_return_t kr, pe_result;
588 vm_page_t mem;
589 vm_page_t page_list = NULL;
590 int page_count = 0;
591 int page_grab_count = 0;
592 int i;
593
594#if DEBUG || DEVELOPMENT
595 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_START, size, 0, 0, 0);
596#endif
597
598 page_count = (int) (size / PAGE_SIZE_64);
599
600 assert((flags & (KMA_COMPRESSOR|KMA_KOBJECT)) != (KMA_COMPRESSOR|KMA_KOBJECT));
601
602 if (flags & KMA_COMPRESSOR) {
603
604 pg_offset = page_count * PAGE_SIZE_64;
605
606 do {
607 for (;;) {
608 mem = vm_page_grab();
609
610 if (mem != VM_PAGE_NULL)
611 break;
612
613 VM_PAGE_WAIT();
614 }
615 page_grab_count++;
616 if (KMA_ZERO & flags) vm_page_zero_fill(mem);
617 mem->vmp_snext = page_list;
618 page_list = mem;
619
620 pg_offset -= PAGE_SIZE_64;
621
622 kr = pmap_enter_options(kernel_pmap,
623 addr + pg_offset, VM_PAGE_GET_PHYS_PAGE(mem),
624 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE,
625 PMAP_OPTIONS_INTERNAL, NULL);
626 assert(kr == KERN_SUCCESS);
627
628 } while (pg_offset);
629
630 offset = addr;
631 object = compressor_object;
632
633 vm_object_lock(object);
634
635 for (pg_offset = 0;
636 pg_offset < size;
637 pg_offset += PAGE_SIZE_64) {
638
639 mem = page_list;
640 page_list = mem->vmp_snext;
641 mem->vmp_snext = NULL;
642
643 vm_page_insert(mem, object, offset + pg_offset);
644 assert(mem->vmp_busy);
645
646 mem->vmp_busy = FALSE;
647 mem->vmp_pmapped = TRUE;
648 mem->vmp_wpmapped = TRUE;
649 mem->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR;
650 }
651 vm_object_unlock(object);
652
653#if KASAN
654 if (map == compressor_map) {
655 kasan_notify_address_nopoison(addr, size);
656 } else {
657 kasan_notify_address(addr, size);
658 }
659#endif
660
661#if DEBUG || DEVELOPMENT
662 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
663#endif
664 return KERN_SUCCESS;
665 }
666
667 for (i = 0; i < page_count; i++) {
668 for (;;) {
669 if (flags & KMA_LOMEM)
670 mem = vm_page_grablo();
671 else
672 mem = vm_page_grab();
673
674 if (mem != VM_PAGE_NULL)
675 break;
676
677 if (flags & KMA_NOPAGEWAIT) {
678 kr = KERN_RESOURCE_SHORTAGE;
679 goto out;
680 }
681 if ((flags & KMA_LOMEM) &&
682 (vm_lopage_needed == TRUE)) {
683 kr = KERN_RESOURCE_SHORTAGE;
684 goto out;
685 }
686 VM_PAGE_WAIT();
687 }
688 page_grab_count++;
689 if (KMA_ZERO & flags) vm_page_zero_fill(mem);
690 mem->vmp_snext = page_list;
691 page_list = mem;
692 }
693 if (flags & KMA_KOBJECT) {
694 offset = addr;
695 object = kernel_object;
696
697 vm_object_lock(object);
698 } else {
699 /*
700 * If it's not the kernel object, we need to:
701 * lock map;
702 * lookup entry;
703 * lock object;
704 * take reference on object;
705 * unlock map;
706 */
707 panic("kernel_memory_populate(%p,0x%llx,0x%llx,0x%x): "
708 "!KMA_KOBJECT",
709 map, (uint64_t) addr, (uint64_t) size, flags);
710 }
711
712 for (pg_offset = 0;
713 pg_offset < size;
714 pg_offset += PAGE_SIZE_64) {
715
716 if (page_list == NULL)
717 panic("kernel_memory_populate: page_list == NULL");
718
719 mem = page_list;
720 page_list = mem->vmp_snext;
721 mem->vmp_snext = NULL;
722
723 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
724 mem->vmp_q_state = VM_PAGE_IS_WIRED;
725 mem->vmp_wire_count++;
726 if (__improbable(mem->vmp_wire_count == 0)) {
727 panic("kernel_memory_populate(%p): wire_count overflow", mem);
728 }
729
730 vm_page_insert_wired(mem, object, offset + pg_offset, tag);
731
732 mem->vmp_busy = FALSE;
733 mem->vmp_pmapped = TRUE;
734 mem->vmp_wpmapped = TRUE;
735
736 PMAP_ENTER_OPTIONS(kernel_pmap, addr + pg_offset, mem,
737 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
738 ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE,
739 PMAP_OPTIONS_NOWAIT, pe_result);
740
741 if (pe_result == KERN_RESOURCE_SHORTAGE) {
742
743 vm_object_unlock(object);
744
745 PMAP_ENTER(kernel_pmap, addr + pg_offset, mem,
746 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
747 ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE,
748 pe_result);
749
750 vm_object_lock(object);
751 }
752
753 assert(pe_result == KERN_SUCCESS);
754
755 if (flags & KMA_NOENCRYPT) {
756 bzero(CAST_DOWN(void *, (addr + pg_offset)), PAGE_SIZE);
757 pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
758 }
759 }
760 vm_page_lockspin_queues();
761 vm_page_wire_count += page_count;
762 vm_page_unlock_queues();
763
764#if DEBUG || DEVELOPMENT
765 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
766#endif
767
768 if (kernel_object == object) vm_tag_update_size(tag, size);
769
770 vm_object_unlock(object);
771
772#if KASAN
773 if (map == compressor_map) {
774 kasan_notify_address_nopoison(addr, size);
775 } else {
776 kasan_notify_address(addr, size);
777 }
778#endif
779 return KERN_SUCCESS;
780
781out:
782 if (page_list)
783 vm_page_free_list(page_list, FALSE);
784
785#if DEBUG || DEVELOPMENT
786 VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
787#endif
788
789 return kr;
790}
791
792
793void
794kernel_memory_depopulate(
795 vm_map_t map,
796 vm_offset_t addr,
797 vm_size_t size,
798 int flags)
799{
800 vm_object_t object;
801 vm_object_offset_t offset, pg_offset;
802 vm_page_t mem;
803 vm_page_t local_freeq = NULL;
804
805 assert((flags & (KMA_COMPRESSOR|KMA_KOBJECT)) != (KMA_COMPRESSOR|KMA_KOBJECT));
806
807 if (flags & KMA_COMPRESSOR) {
808 offset = addr;
809 object = compressor_object;
810
811 vm_object_lock(object);
812 } else if (flags & KMA_KOBJECT) {
813 offset = addr;
814 object = kernel_object;
815 vm_object_lock(object);
816 } else {
817 offset = 0;
818 object = NULL;
819 /*
820 * If it's not the kernel object, we need to:
821 * lock map;
822 * lookup entry;
823 * lock object;
824 * unlock map;
825 */
826 panic("kernel_memory_depopulate(%p,0x%llx,0x%llx,0x%x): "
827 "!KMA_KOBJECT",
828 map, (uint64_t) addr, (uint64_t) size, flags);
829 }
830 pmap_protect(kernel_map->pmap, offset, offset + size, VM_PROT_NONE);
831
832 for (pg_offset = 0;
833 pg_offset < size;
834 pg_offset += PAGE_SIZE_64) {
835
836 mem = vm_page_lookup(object, offset + pg_offset);
837
838 assert(mem);
839
840 if (mem->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR)
841 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(mem));
842
843 mem->vmp_busy = TRUE;
844
845 assert(mem->vmp_tabled);
846 vm_page_remove(mem, TRUE);
847 assert(mem->vmp_busy);
848
849 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
850 assert((mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
851 (mem->vmp_q_state == VM_PAGE_NOT_ON_Q));
852
853 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
854 mem->vmp_snext = local_freeq;
855 local_freeq = mem;
856 }
857 vm_object_unlock(object);
858
859 if (local_freeq)
860 vm_page_free_list(local_freeq, TRUE);
861}
862
863/*
864 * kmem_alloc:
865 *
866 * Allocate wired-down memory in the kernel's address map
867 * or a submap. The memory is not zero-filled.
868 */
869
870kern_return_t
871kmem_alloc_external(
872 vm_map_t map,
873 vm_offset_t *addrp,
874 vm_size_t size)
875{
876 return (kmem_alloc(map, addrp, size, vm_tag_bt()));
877}
878
879
880kern_return_t
881kmem_alloc(
882 vm_map_t map,
883 vm_offset_t *addrp,
884 vm_size_t size,
885 vm_tag_t tag)
886{
887 return kmem_alloc_flags(map, addrp, size, tag, 0);
888}
889
890kern_return_t
891kmem_alloc_flags(
892 vm_map_t map,
893 vm_offset_t *addrp,
894 vm_size_t size,
895 vm_tag_t tag,
896 int flags)
897{
898 kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, flags, tag);
899 TRACE_MACHLEAKS(KMEM_ALLOC_CODE, KMEM_ALLOC_CODE_2, size, *addrp);
900 return kr;
901}
902
903/*
904 * kmem_realloc:
905 *
906 * Reallocate wired-down memory in the kernel's address map
907 * or a submap. Newly allocated pages are not zeroed.
908 * This can only be used on regions allocated with kmem_alloc.
909 *
910 * If successful, the pages in the old region are mapped twice.
911 * The old region is unchanged. Use kmem_free to get rid of it.
912 */
913kern_return_t
914kmem_realloc(
915 vm_map_t map,
916 vm_offset_t oldaddr,
917 vm_size_t oldsize,
918 vm_offset_t *newaddrp,
919 vm_size_t newsize,
920 vm_tag_t tag)
921{
922 vm_object_t object;
923 vm_object_offset_t offset;
924 vm_map_offset_t oldmapmin;
925 vm_map_offset_t oldmapmax;
926 vm_map_offset_t newmapaddr;
927 vm_map_size_t oldmapsize;
928 vm_map_size_t newmapsize;
929 vm_map_entry_t oldentry;
930 vm_map_entry_t newentry;
931 vm_page_t mem;
932 kern_return_t kr;
933
934 oldmapmin = vm_map_trunc_page(oldaddr,
935 VM_MAP_PAGE_MASK(map));
936 oldmapmax = vm_map_round_page(oldaddr + oldsize,
937 VM_MAP_PAGE_MASK(map));
938 oldmapsize = oldmapmax - oldmapmin;
939 newmapsize = vm_map_round_page(newsize,
940 VM_MAP_PAGE_MASK(map));
941 if (newmapsize < newsize) {
942 /* overflow */
943 *newaddrp = 0;
944 return KERN_INVALID_ARGUMENT;
945 }
946
947 /*
948 * Find the VM object backing the old region.
949 */
950
951 vm_map_lock(map);
952
953 if (!vm_map_lookup_entry(map, oldmapmin, &oldentry))
954 panic("kmem_realloc");
955 object = VME_OBJECT(oldentry);
956
957 /*
958 * Increase the size of the object and
959 * fill in the new region.
960 */
961
962 vm_object_reference(object);
963 /* by grabbing the object lock before unlocking the map */
964 /* we guarantee that we will panic if more than one */
965 /* attempt is made to realloc a kmem_alloc'd area */
966 vm_object_lock(object);
967 vm_map_unlock(map);
968 if (object->vo_size != oldmapsize)
969 panic("kmem_realloc");
970 object->vo_size = newmapsize;
971 vm_object_unlock(object);
972
973 /* allocate the new pages while expanded portion of the */
974 /* object is still not mapped */
975 kmem_alloc_pages(object, vm_object_round_page(oldmapsize),
976 vm_object_round_page(newmapsize-oldmapsize));
977
978 /*
979 * Find space for the new region.
980 */
981
982 kr = vm_map_find_space(map, &newmapaddr, newmapsize,
983 (vm_map_offset_t) 0, 0,
984 VM_MAP_KERNEL_FLAGS_NONE,
985 tag,
986 &newentry);
987 if (kr != KERN_SUCCESS) {
988 vm_object_lock(object);
989 for(offset = oldmapsize;
990 offset < newmapsize; offset += PAGE_SIZE) {
991 if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
992 VM_PAGE_FREE(mem);
993 }
994 }
995 object->vo_size = oldmapsize;
996 vm_object_unlock(object);
997 vm_object_deallocate(object);
998 return kr;
999 }
1000 VME_OBJECT_SET(newentry, object);
1001 VME_OFFSET_SET(newentry, 0);
1002 assert(newentry->wired_count == 0);
1003
1004
1005 /* add an extra reference in case we have someone doing an */
1006 /* unexpected deallocate */
1007 vm_object_reference(object);
1008 vm_map_unlock(map);
1009
1010 kr = vm_map_wire_kernel(map, newmapaddr, newmapaddr + newmapsize,
1011 VM_PROT_DEFAULT, tag, FALSE);
1012 if (KERN_SUCCESS != kr) {
1013 vm_map_remove(map, newmapaddr, newmapaddr + newmapsize, VM_MAP_REMOVE_NO_FLAGS);
1014 vm_object_lock(object);
1015 for(offset = oldsize; offset < newmapsize; offset += PAGE_SIZE) {
1016 if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
1017 VM_PAGE_FREE(mem);
1018 }
1019 }
1020 object->vo_size = oldmapsize;
1021 vm_object_unlock(object);
1022 vm_object_deallocate(object);
1023 return (kr);
1024 }
1025 vm_object_deallocate(object);
1026
1027 if (kernel_object == object) vm_tag_update_size(tag, newmapsize);
1028
1029 *newaddrp = CAST_DOWN(vm_offset_t, newmapaddr);
1030 return KERN_SUCCESS;
1031}
1032
1033/*
1034 * kmem_alloc_kobject:
1035 *
1036 * Allocate wired-down memory in the kernel's address map
1037 * or a submap. The memory is not zero-filled.
1038 *
1039 * The memory is allocated in the kernel_object.
1040 * It may not be copied with vm_map_copy, and
1041 * it may not be reallocated with kmem_realloc.
1042 */
1043
1044kern_return_t
1045kmem_alloc_kobject_external(
1046 vm_map_t map,
1047 vm_offset_t *addrp,
1048 vm_size_t size)
1049{
1050 return (kmem_alloc_kobject(map, addrp, size, vm_tag_bt()));
1051}
1052
1053kern_return_t
1054kmem_alloc_kobject(
1055 vm_map_t map,
1056 vm_offset_t *addrp,
1057 vm_size_t size,
1058 vm_tag_t tag)
1059{
1060 return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT, tag);
1061}
1062
1063/*
1064 * kmem_alloc_aligned:
1065 *
1066 * Like kmem_alloc_kobject, except that the memory is aligned.
1067 * The size should be a power-of-2.
1068 */
1069
1070kern_return_t
1071kmem_alloc_aligned(
1072 vm_map_t map,
1073 vm_offset_t *addrp,
1074 vm_size_t size,
1075 vm_tag_t tag)
1076{
1077 if ((size & (size - 1)) != 0)
1078 panic("kmem_alloc_aligned: size not aligned");
1079 return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT, tag);
1080}
1081
1082/*
1083 * kmem_alloc_pageable:
1084 *
1085 * Allocate pageable memory in the kernel's address map.
1086 */
1087
1088kern_return_t
1089kmem_alloc_pageable_external(
1090 vm_map_t map,
1091 vm_offset_t *addrp,
1092 vm_size_t size)
1093{
1094 return (kmem_alloc_pageable(map, addrp, size, vm_tag_bt()));
1095}
1096
1097kern_return_t
1098kmem_alloc_pageable(
1099 vm_map_t map,
1100 vm_offset_t *addrp,
1101 vm_size_t size,
1102 vm_tag_t tag)
1103{
1104 vm_map_offset_t map_addr;
1105 vm_map_size_t map_size;
1106 kern_return_t kr;
1107
1108#ifndef normal
1109 map_addr = (vm_map_min(map)) + PAGE_SIZE;
1110#else
1111 map_addr = vm_map_min(map);
1112#endif
1113 map_size = vm_map_round_page(size,
1114 VM_MAP_PAGE_MASK(map));
1115 if (map_size < size) {
1116 /* overflow */
1117 *addrp = 0;
1118 return KERN_INVALID_ARGUMENT;
1119 }
1120
1121 kr = vm_map_enter(map, &map_addr, map_size,
1122 (vm_map_offset_t) 0,
1123 VM_FLAGS_ANYWHERE,
1124 VM_MAP_KERNEL_FLAGS_NONE,
1125 tag,
1126 VM_OBJECT_NULL, (vm_object_offset_t) 0, FALSE,
1127 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
1128
1129 if (kr != KERN_SUCCESS)
1130 return kr;
1131
1132#if KASAN
1133 kasan_notify_address(map_addr, map_size);
1134#endif
1135 *addrp = CAST_DOWN(vm_offset_t, map_addr);
1136 return KERN_SUCCESS;
1137}
1138
1139/*
1140 * kmem_free:
1141 *
1142 * Release a region of kernel virtual memory allocated
1143 * with kmem_alloc, kmem_alloc_kobject, or kmem_alloc_pageable,
1144 * and return the physical pages associated with that region.
1145 */
1146
1147void
1148kmem_free(
1149 vm_map_t map,
1150 vm_offset_t addr,
1151 vm_size_t size)
1152{
1153 kern_return_t kr;
1154
1155 assert(addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS);
1156
1157 TRACE_MACHLEAKS(KMEM_FREE_CODE, KMEM_FREE_CODE_2, size, addr);
1158
1159 if(size == 0) {
1160#if MACH_ASSERT
1161 printf("kmem_free called with size==0 for map: %p with addr: 0x%llx\n",map,(uint64_t)addr);
1162#endif
1163 return;
1164 }
1165
1166 kr = vm_map_remove(map,
1167 vm_map_trunc_page(addr,
1168 VM_MAP_PAGE_MASK(map)),
1169 vm_map_round_page(addr + size,
1170 VM_MAP_PAGE_MASK(map)),
1171 VM_MAP_REMOVE_KUNWIRE);
1172 if (kr != KERN_SUCCESS)
1173 panic("kmem_free");
1174}
1175
1176/*
1177 * Allocate new pages in an object.
1178 */
1179
1180kern_return_t
1181kmem_alloc_pages(
1182 vm_object_t object,
1183 vm_object_offset_t offset,
1184 vm_object_size_t size)
1185{
1186 vm_object_size_t alloc_size;
1187
1188 alloc_size = vm_object_round_page(size);
1189 vm_object_lock(object);
1190 while (alloc_size) {
1191 vm_page_t mem;
1192
1193
1194 /*
1195 * Allocate a page
1196 */
1197 while (VM_PAGE_NULL ==
1198 (mem = vm_page_alloc(object, offset))) {
1199 vm_object_unlock(object);
1200 VM_PAGE_WAIT();
1201 vm_object_lock(object);
1202 }
1203 mem->vmp_busy = FALSE;
1204
1205 alloc_size -= PAGE_SIZE;
1206 offset += PAGE_SIZE;
1207 }
1208 vm_object_unlock(object);
1209 return KERN_SUCCESS;
1210}
1211
1212/*
1213 * kmem_suballoc:
1214 *
1215 * Allocates a map to manage a subrange
1216 * of the kernel virtual address space.
1217 *
1218 * Arguments are as follows:
1219 *
1220 * parent Map to take range from
1221 * addr Address of start of range (IN/OUT)
1222 * size Size of range to find
1223 * pageable Can region be paged
1224 * anywhere Can region be located anywhere in map
1225 * new_map Pointer to new submap
1226 */
1227kern_return_t
1228kmem_suballoc(
1229 vm_map_t parent,
1230 vm_offset_t *addr,
1231 vm_size_t size,
1232 boolean_t pageable,
1233 int flags,
1234 vm_map_kernel_flags_t vmk_flags,
1235 vm_tag_t tag,
1236 vm_map_t *new_map)
1237{
1238 vm_map_t map;
1239 vm_map_offset_t map_addr;
1240 vm_map_size_t map_size;
1241 kern_return_t kr;
1242
1243 map_size = vm_map_round_page(size,
1244 VM_MAP_PAGE_MASK(parent));
1245 if (map_size < size) {
1246 /* overflow */
1247 *addr = 0;
1248 return KERN_INVALID_ARGUMENT;
1249 }
1250
1251 /*
1252 * Need reference on submap object because it is internal
1253 * to the vm_system. vm_object_enter will never be called
1254 * on it (usual source of reference for vm_map_enter).
1255 */
1256 vm_object_reference(vm_submap_object);
1257
1258 map_addr = ((flags & VM_FLAGS_ANYWHERE)
1259 ? vm_map_min(parent)
1260 : vm_map_trunc_page(*addr,
1261 VM_MAP_PAGE_MASK(parent)));
1262
1263 kr = vm_map_enter(parent, &map_addr, map_size,
1264 (vm_map_offset_t) 0, flags, vmk_flags, tag,
1265 vm_submap_object, (vm_object_offset_t) 0, FALSE,
1266 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
1267 if (kr != KERN_SUCCESS) {
1268 vm_object_deallocate(vm_submap_object);
1269 return (kr);
1270 }
1271
1272 pmap_reference(vm_map_pmap(parent));
1273 map = vm_map_create(vm_map_pmap(parent), map_addr, map_addr + map_size, pageable);
1274 if (map == VM_MAP_NULL)
1275 panic("kmem_suballoc: vm_map_create failed"); /* "can't happen" */
1276 /* inherit the parent map's page size */
1277 vm_map_set_page_shift(map, VM_MAP_PAGE_SHIFT(parent));
1278
1279 kr = vm_map_submap(parent, map_addr, map_addr + map_size, map, map_addr, FALSE);
1280 if (kr != KERN_SUCCESS) {
1281 /*
1282 * See comment preceding vm_map_submap().
1283 */
1284 vm_map_remove(parent, map_addr, map_addr + map_size,
1285 VM_MAP_REMOVE_NO_FLAGS);
1286 vm_map_deallocate(map); /* also removes ref to pmap */
1287 vm_object_deallocate(vm_submap_object);
1288 return (kr);
1289 }
1290 *addr = CAST_DOWN(vm_offset_t, map_addr);
1291 *new_map = map;
1292 return (KERN_SUCCESS);
1293}
1294
1295/*
1296 * kmem_init:
1297 *
1298 * Initialize the kernel's virtual memory map, taking
1299 * into account all memory allocated up to this time.
1300 */
1301void
1302kmem_init(
1303 vm_offset_t start,
1304 vm_offset_t end)
1305{
1306 vm_map_offset_t map_start;
1307 vm_map_offset_t map_end;
1308 vm_map_kernel_flags_t vmk_flags;
1309
1310 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
1311 vmk_flags.vmkf_permanent = TRUE;
1312 vmk_flags.vmkf_no_pmap_check = TRUE;
1313
1314 map_start = vm_map_trunc_page(start,
1315 VM_MAP_PAGE_MASK(kernel_map));
1316 map_end = vm_map_round_page(end,
1317 VM_MAP_PAGE_MASK(kernel_map));
1318
1319#if defined(__arm__) || defined(__arm64__)
1320 kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS,
1321 VM_MAX_KERNEL_ADDRESS, FALSE);
1322 /*
1323 * Reserve virtual memory allocated up to this time.
1324 */
1325 {
1326 unsigned int region_select = 0;
1327 vm_map_offset_t region_start;
1328 vm_map_size_t region_size;
1329 vm_map_offset_t map_addr;
1330 kern_return_t kr;
1331
1332 while (pmap_virtual_region(region_select, &region_start, &region_size)) {
1333
1334 map_addr = region_start;
1335 kr = vm_map_enter(kernel_map, &map_addr,
1336 vm_map_round_page(region_size,
1337 VM_MAP_PAGE_MASK(kernel_map)),
1338 (vm_map_offset_t) 0,
1339 VM_FLAGS_FIXED,
1340 vmk_flags,
1341 VM_KERN_MEMORY_NONE,
1342 VM_OBJECT_NULL,
1343 (vm_object_offset_t) 0, FALSE, VM_PROT_NONE, VM_PROT_NONE,
1344 VM_INHERIT_DEFAULT);
1345
1346 if (kr != KERN_SUCCESS) {
1347 panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n",
1348 (uint64_t) start, (uint64_t) end, (uint64_t) region_start,
1349 (uint64_t) region_size, kr);
1350 }
1351
1352 region_select++;
1353 }
1354 }
1355#else
1356 kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS,
1357 map_end, FALSE);
1358 /*
1359 * Reserve virtual memory allocated up to this time.
1360 */
1361 if (start != VM_MIN_KERNEL_AND_KEXT_ADDRESS) {
1362 vm_map_offset_t map_addr;
1363 kern_return_t kr;
1364
1365 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
1366 vmk_flags.vmkf_no_pmap_check = TRUE;
1367
1368 map_addr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
1369 kr = vm_map_enter(kernel_map,
1370 &map_addr,
1371 (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS),
1372 (vm_map_offset_t) 0,
1373 VM_FLAGS_FIXED,
1374 vmk_flags,
1375 VM_KERN_MEMORY_NONE,
1376 VM_OBJECT_NULL,
1377 (vm_object_offset_t) 0, FALSE,
1378 VM_PROT_NONE, VM_PROT_NONE,
1379 VM_INHERIT_DEFAULT);
1380
1381 if (kr != KERN_SUCCESS) {
1382 panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n",
1383 (uint64_t) start, (uint64_t) end,
1384 (uint64_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS,
1385 (uint64_t) (map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS),
1386 kr);
1387 }
1388 }
1389#endif
1390
1391 /*
1392 * Set the default global user wire limit which limits the amount of
1393 * memory that can be locked via mlock(). We set this to the total
1394 * amount of memory that are potentially usable by a user app (max_mem)
1395 * minus a certain amount. This can be overridden via a sysctl.
1396 */
1397 vm_global_no_user_wire_amount = MIN(max_mem*20/100,
1398 VM_NOT_USER_WIREABLE);
1399 vm_global_user_wire_limit = max_mem - vm_global_no_user_wire_amount;
1400
1401 /* the default per user limit is the same as the global limit */
1402 vm_user_wire_limit = vm_global_user_wire_limit;
1403}
1404
1405
1406/*
1407 * Routine: copyinmap
1408 * Purpose:
1409 * Like copyin, except that fromaddr is an address
1410 * in the specified VM map. This implementation
1411 * is incomplete; it handles the current user map
1412 * and the kernel map/submaps.
1413 */
1414kern_return_t
1415copyinmap(
1416 vm_map_t map,
1417 vm_map_offset_t fromaddr,
1418 void *todata,
1419 vm_size_t length)
1420{
1421 kern_return_t kr = KERN_SUCCESS;
1422 vm_map_t oldmap;
1423
1424 if (vm_map_pmap(map) == pmap_kernel())
1425 {
1426 /* assume a correct copy */
1427 memcpy(todata, CAST_DOWN(void *, fromaddr), length);
1428 }
1429 else if (current_map() == map)
1430 {
1431 if (copyin(fromaddr, todata, length) != 0)
1432 kr = KERN_INVALID_ADDRESS;
1433 }
1434 else
1435 {
1436 vm_map_reference(map);
1437 oldmap = vm_map_switch(map);
1438 if (copyin(fromaddr, todata, length) != 0)
1439 kr = KERN_INVALID_ADDRESS;
1440 vm_map_switch(oldmap);
1441 vm_map_deallocate(map);
1442 }
1443 return kr;
1444}
1445
1446/*
1447 * Routine: copyoutmap
1448 * Purpose:
1449 * Like copyout, except that toaddr is an address
1450 * in the specified VM map. This implementation
1451 * is incomplete; it handles the current user map
1452 * and the kernel map/submaps.
1453 */
1454kern_return_t
1455copyoutmap(
1456 vm_map_t map,
1457 void *fromdata,
1458 vm_map_address_t toaddr,
1459 vm_size_t length)
1460{
1461 if (vm_map_pmap(map) == pmap_kernel()) {
1462 /* assume a correct copy */
1463 memcpy(CAST_DOWN(void *, toaddr), fromdata, length);
1464 return KERN_SUCCESS;
1465 }
1466
1467 if (current_map() != map)
1468 return KERN_NOT_SUPPORTED;
1469
1470 if (copyout(fromdata, toaddr, length) != 0)
1471 return KERN_INVALID_ADDRESS;
1472
1473 return KERN_SUCCESS;
1474}
1475
1476/*
1477 *
1478 * The following two functions are to be used when exposing kernel
1479 * addresses to userspace via any of the various debug or info
1480 * facilities that exist. These are basically the same as VM_KERNEL_ADDRPERM()
1481 * and VM_KERNEL_UNSLIDE_OR_PERM() except they use a different random seed and
1482 * are exported to KEXTs.
1483 *
1484 * NOTE: USE THE MACRO VERSIONS OF THESE FUNCTIONS (in vm_param.h) FROM WITHIN THE KERNEL
1485 */
1486
1487static void
1488vm_kernel_addrhash_internal(
1489 vm_offset_t addr,
1490 vm_offset_t *hash_addr,
1491 uint64_t salt)
1492{
1493 assert(salt != 0);
1494
1495 if (addr == 0) {
1496 *hash_addr = 0;
1497 return;
1498 }
1499
1500 if (VM_KERNEL_IS_SLID(addr)) {
1501 *hash_addr = VM_KERNEL_UNSLIDE(addr);
1502 return;
1503 }
1504
1505 vm_offset_t sha_digest[SHA256_DIGEST_LENGTH/sizeof(vm_offset_t)];
1506 SHA256_CTX sha_ctx;
1507
1508 SHA256_Init(&sha_ctx);
1509 SHA256_Update(&sha_ctx, &salt, sizeof(salt));
1510 SHA256_Update(&sha_ctx, &addr, sizeof(addr));
1511 SHA256_Final(sha_digest, &sha_ctx);
1512
1513 *hash_addr = sha_digest[0];
1514}
1515
1516void
1517vm_kernel_addrhash_external(
1518 vm_offset_t addr,
1519 vm_offset_t *hash_addr)
1520{
1521 return vm_kernel_addrhash_internal(addr, hash_addr, vm_kernel_addrhash_salt_ext);
1522}
1523
1524vm_offset_t
1525vm_kernel_addrhash(vm_offset_t addr)
1526{
1527 vm_offset_t hash_addr;
1528 vm_kernel_addrhash_internal(addr, &hash_addr, vm_kernel_addrhash_salt);
1529 return hash_addr;
1530}
1531
1532void
1533vm_kernel_addrhide(
1534 vm_offset_t addr,
1535 vm_offset_t *hide_addr)
1536{
1537 *hide_addr = VM_KERNEL_ADDRHIDE(addr);
1538}
1539
1540/*
1541 * vm_kernel_addrperm_external:
1542 * vm_kernel_unslide_or_perm_external:
1543 *
1544 * Use these macros when exposing an address to userspace that could come from
1545 * either kernel text/data *or* the heap.
1546 */
1547void
1548vm_kernel_addrperm_external(
1549 vm_offset_t addr,
1550 vm_offset_t *perm_addr)
1551{
1552 if (VM_KERNEL_IS_SLID(addr)) {
1553 *perm_addr = VM_KERNEL_UNSLIDE(addr);
1554 } else if (VM_KERNEL_ADDRESS(addr)) {
1555 *perm_addr = addr + vm_kernel_addrperm_ext;
1556 } else {
1557 *perm_addr = addr;
1558 }
1559}
1560
1561void
1562vm_kernel_unslide_or_perm_external(
1563 vm_offset_t addr,
1564 vm_offset_t *up_addr)
1565{
1566 vm_kernel_addrperm_external(addr, up_addr);
1567}
1568