1/*
2 * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifdef XNU_KERNEL_PRIVATE
30
31#ifndef _VM_VM_PROTOS_H_
32#define _VM_VM_PROTOS_H_
33
34#include <mach/mach_types.h>
35#include <kern/kern_types.h>
36
37#ifdef __cplusplus
38extern "C" {
39#endif
40
41/*
42 * This file contains various type definitions and routine prototypes
43 * that are needed to avoid compilation warnings for VM code (in osfmk,
44 * default_pager and bsd).
45 * Most of these should eventually go into more appropriate header files.
46 *
47 * Include it after all other header files since it doesn't include any
48 * type definitions and it works around some conflicts with other header
49 * files.
50 */
51
52/*
53 * iokit
54 */
55extern kern_return_t device_data_action(
56 uintptr_t device_handle,
57 ipc_port_t device_pager,
58 vm_prot_t protection,
59 vm_object_offset_t offset,
60 vm_size_t size);
61
62extern kern_return_t device_close(
63 uintptr_t device_handle);
64
65extern boolean_t vm_swap_files_pinned(void);
66
67/*
68 * osfmk
69 */
70#ifndef _IPC_IPC_PORT_H_
71extern mach_port_name_t ipc_port_copyout_send(
72 ipc_port_t sright,
73 ipc_space_t space);
74extern task_t port_name_to_task(
75 mach_port_name_t name);
76extern task_t port_name_to_task_inspect(
77 mach_port_name_t name);
78extern void ipc_port_release_send(
79 ipc_port_t port);
80#endif /* _IPC_IPC_PORT_H_ */
81
82extern ipc_space_t get_task_ipcspace(
83 task_t t);
84
85#if CONFIG_MEMORYSTATUS
86extern int max_task_footprint_mb; /* Per-task limit on physical memory consumption in megabytes */
87#endif /* CONFIG_MEMORYSTATUS */
88
89/* Some loose-ends VM stuff */
90
91extern vm_map_t kalloc_map;
92extern vm_size_t msg_ool_size_small;
93extern vm_map_t zone_map;
94
95extern void consider_machine_adjust(void);
96extern vm_map_offset_t get_map_min(vm_map_t);
97extern vm_map_offset_t get_map_max(vm_map_t);
98extern vm_map_size_t get_vmmap_size(vm_map_t);
99#if CONFIG_COREDUMP
100extern int get_vmmap_entries(vm_map_t);
101#endif
102extern int get_map_nentries(vm_map_t);
103
104extern vm_map_offset_t vm_map_page_mask(vm_map_t);
105
106extern kern_return_t vm_map_purgable_control(
107 vm_map_t map,
108 vm_map_offset_t address,
109 vm_purgable_t control,
110 int *state);
111
112extern kern_return_t
113vnode_pager_get_object_vnode(
114 memory_object_t mem_obj,
115 uintptr_t * vnodeaddr,
116 uint32_t * vid);
117
118#if CONFIG_COREDUMP
119extern boolean_t coredumpok(vm_map_t map, vm_offset_t va);
120#endif
121
122/*
123 * VM routines that used to be published to
124 * user space, and are now restricted to the kernel.
125 *
126 * They should eventually go away entirely -
127 * to be replaced with standard vm_map() and
128 * vm_deallocate() calls.
129 */
130
131extern kern_return_t vm_upl_map
132(
133 vm_map_t target_task,
134 upl_t upl,
135 vm_address_t *address
136);
137
138extern kern_return_t vm_upl_unmap
139(
140 vm_map_t target_task,
141 upl_t upl
142);
143
144extern kern_return_t vm_region_object_create
145(
146 vm_map_t target_task,
147 vm_size_t size,
148 ipc_port_t *object_handle
149);
150
151extern mach_vm_offset_t mach_get_vm_start(vm_map_t);
152extern mach_vm_offset_t mach_get_vm_end(vm_map_t);
153
154#if CONFIG_CODE_DECRYPTION
155#define VM_MAP_DEBUG_APPLE_PROTECT MACH_ASSERT
156#if VM_MAP_DEBUG_APPLE_PROTECT
157extern int vm_map_debug_apple_protect;
158#endif /* VM_MAP_DEBUG_APPLE_PROTECT */
159struct pager_crypt_info;
160extern kern_return_t vm_map_apple_protected(
161 vm_map_t map,
162 vm_map_offset_t start,
163 vm_map_offset_t end,
164 vm_object_offset_t crypto_backing_offset,
165 struct pager_crypt_info *crypt_info);
166extern void apple_protect_pager_bootstrap(void);
167extern memory_object_t apple_protect_pager_setup(
168 vm_object_t backing_object,
169 vm_object_offset_t backing_offset,
170 vm_object_offset_t crypto_backing_offset,
171 struct pager_crypt_info *crypt_info,
172 vm_object_offset_t crypto_start,
173 vm_object_offset_t crypto_end);
174#endif /* CONFIG_CODE_DECRYPTION */
175
176struct vm_shared_region_slide_info;
177extern kern_return_t vm_map_shared_region(
178 vm_map_t map,
179 vm_map_offset_t start,
180 vm_map_offset_t end,
181 vm_object_offset_t backing_offset,
182 struct vm_shared_region_slide_info *slide_info);
183extern void shared_region_pager_bootstrap(void);
184extern memory_object_t shared_region_pager_setup(
185 vm_object_t backing_object,
186 vm_object_offset_t backing_offset,
187 struct vm_shared_region_slide_info *slide_info);
188
189struct vnode;
190extern void swapfile_pager_bootstrap(void);
191extern memory_object_t swapfile_pager_setup(struct vnode *vp);
192extern memory_object_control_t swapfile_pager_control(memory_object_t mem_obj);
193
194#if __arm64__ || ((__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS))
195#define SIXTEENK_PAGE_SIZE 0x4000
196#define SIXTEENK_PAGE_MASK 0x3FFF
197#define SIXTEENK_PAGE_SHIFT 14
198#endif /* __arm64__ || ((__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS)) */
199
200#if __arm64__
201#define FOURK_PAGE_SIZE 0x1000
202#define FOURK_PAGE_MASK 0xFFF
203#define FOURK_PAGE_SHIFT 12
204
205extern unsigned int page_shift_user32;
206
207#define VM_MAP_DEBUG_FOURK MACH_ASSERT
208#if VM_MAP_DEBUG_FOURK
209extern int vm_map_debug_fourk;
210#endif /* VM_MAP_DEBUG_FOURK */
211extern void fourk_pager_bootstrap(void);
212extern memory_object_t fourk_pager_create(void);
213extern vm_object_t fourk_pager_to_vm_object(memory_object_t mem_obj);
214extern kern_return_t fourk_pager_populate(
215 memory_object_t mem_obj,
216 boolean_t overwrite,
217 int index,
218 vm_object_t new_backing_object,
219 vm_object_offset_t new_backing_offset,
220 vm_object_t *old_backing_object,
221 vm_object_offset_t *old_backing_offset);
222#endif /* __arm64__ */
223
224/*
225 * bsd
226 */
227struct vnode;
228extern void *upl_get_internal_page_list(
229 upl_t upl);
230
231extern void vnode_setswapmount(struct vnode *);
232extern int64_t vnode_getswappin_avail(struct vnode *);
233
234extern void vnode_pager_was_dirtied(
235 struct vnode *,
236 vm_object_offset_t,
237 vm_object_offset_t);
238
239typedef int pager_return_t;
240extern pager_return_t vnode_pagein(
241 struct vnode *, upl_t,
242 upl_offset_t, vm_object_offset_t,
243 upl_size_t, int, int *);
244extern pager_return_t vnode_pageout(
245 struct vnode *, upl_t,
246 upl_offset_t, vm_object_offset_t,
247 upl_size_t, int, int *);
248extern uint32_t vnode_trim (struct vnode *, int64_t offset, unsigned long len);
249extern memory_object_t vnode_pager_setup(
250 struct vnode *, memory_object_t);
251extern vm_object_offset_t vnode_pager_get_filesize(
252 struct vnode *);
253extern uint32_t vnode_pager_isinuse(
254 struct vnode *);
255extern boolean_t vnode_pager_isSSD(
256 struct vnode *);
257extern void vnode_pager_throttle(
258 void);
259extern uint32_t vnode_pager_return_throttle_io_limit(
260 struct vnode *,
261 uint32_t *);
262extern kern_return_t vnode_pager_get_name(
263 struct vnode *vp,
264 char *pathname,
265 vm_size_t pathname_len,
266 char *filename,
267 vm_size_t filename_len,
268 boolean_t *truncated_path_p);
269struct timespec;
270extern kern_return_t vnode_pager_get_mtime(
271 struct vnode *vp,
272 struct timespec *mtime,
273 struct timespec *cs_mtime);
274extern kern_return_t vnode_pager_get_cs_blobs(
275 struct vnode *vp,
276 void **blobs);
277
278#if CONFIG_IOSCHED
279void vnode_pager_issue_reprioritize_io(
280 struct vnode *devvp,
281 uint64_t blkno,
282 uint32_t len,
283 int priority);
284#endif
285
286#if CHECK_CS_VALIDATION_BITMAP
287/* used by the vnode_pager_cs_validation_bitmap routine*/
288#define CS_BITMAP_SET 1
289#define CS_BITMAP_CLEAR 2
290#define CS_BITMAP_CHECK 3
291
292#endif /* CHECK_CS_VALIDATION_BITMAP */
293
294extern void vnode_pager_bootstrap(void);
295extern kern_return_t
296vnode_pager_data_unlock(
297 memory_object_t mem_obj,
298 memory_object_offset_t offset,
299 memory_object_size_t size,
300 vm_prot_t desired_access);
301extern kern_return_t vnode_pager_init(
302 memory_object_t,
303 memory_object_control_t,
304 memory_object_cluster_size_t);
305extern kern_return_t vnode_pager_get_object_size(
306 memory_object_t,
307 memory_object_offset_t *);
308
309#if CONFIG_IOSCHED
310extern kern_return_t vnode_pager_get_object_devvp(
311 memory_object_t,
312 uintptr_t *);
313#endif
314
315extern void vnode_pager_dirtied(
316 memory_object_t,
317 vm_object_offset_t,
318 vm_object_offset_t);
319extern kern_return_t vnode_pager_get_isinuse(
320 memory_object_t,
321 uint32_t *);
322extern kern_return_t vnode_pager_get_isSSD(
323 memory_object_t,
324 boolean_t *);
325extern kern_return_t vnode_pager_get_throttle_io_limit(
326 memory_object_t,
327 uint32_t *);
328extern kern_return_t vnode_pager_get_object_name(
329 memory_object_t mem_obj,
330 char *pathname,
331 vm_size_t pathname_len,
332 char *filename,
333 vm_size_t filename_len,
334 boolean_t *truncated_path_p);
335extern kern_return_t vnode_pager_get_object_mtime(
336 memory_object_t mem_obj,
337 struct timespec *mtime,
338 struct timespec *cs_mtime);
339
340#if CHECK_CS_VALIDATION_BITMAP
341extern kern_return_t vnode_pager_cs_check_validation_bitmap(
342 memory_object_t mem_obj,
343 memory_object_offset_t offset,
344 int optype);
345#endif /*CHECK_CS_VALIDATION_BITMAP*/
346
347extern kern_return_t ubc_cs_check_validation_bitmap (
348 struct vnode *vp,
349 memory_object_offset_t offset,
350 int optype);
351
352extern kern_return_t vnode_pager_data_request(
353 memory_object_t,
354 memory_object_offset_t,
355 memory_object_cluster_size_t,
356 vm_prot_t,
357 memory_object_fault_info_t);
358extern kern_return_t vnode_pager_data_return(
359 memory_object_t,
360 memory_object_offset_t,
361 memory_object_cluster_size_t,
362 memory_object_offset_t *,
363 int *,
364 boolean_t,
365 boolean_t,
366 int);
367extern kern_return_t vnode_pager_data_initialize(
368 memory_object_t,
369 memory_object_offset_t,
370 memory_object_cluster_size_t);
371extern void vnode_pager_reference(
372 memory_object_t mem_obj);
373extern kern_return_t vnode_pager_synchronize(
374 memory_object_t mem_obj,
375 memory_object_offset_t offset,
376 memory_object_size_t length,
377 vm_sync_t sync_flags);
378extern kern_return_t vnode_pager_map(
379 memory_object_t mem_obj,
380 vm_prot_t prot);
381extern kern_return_t vnode_pager_last_unmap(
382 memory_object_t mem_obj);
383extern void vnode_pager_deallocate(
384 memory_object_t);
385extern kern_return_t vnode_pager_terminate(
386 memory_object_t);
387extern void vnode_pager_vrele(
388 struct vnode *vp);
389extern struct vnode *vnode_pager_lookup_vnode(
390 memory_object_t);
391
392extern int ubc_map(
393 struct vnode *vp,
394 int flags);
395extern void ubc_unmap(
396 struct vnode *vp);
397
398struct vm_map_entry;
399extern struct vm_object *find_vnode_object(struct vm_map_entry *entry);
400
401extern void device_pager_reference(memory_object_t);
402extern void device_pager_deallocate(memory_object_t);
403extern kern_return_t device_pager_init(memory_object_t,
404 memory_object_control_t,
405 memory_object_cluster_size_t);
406extern kern_return_t device_pager_terminate(memory_object_t);
407extern kern_return_t device_pager_data_request(memory_object_t,
408 memory_object_offset_t,
409 memory_object_cluster_size_t,
410 vm_prot_t,
411 memory_object_fault_info_t);
412extern kern_return_t device_pager_data_return(memory_object_t,
413 memory_object_offset_t,
414 memory_object_cluster_size_t,
415 memory_object_offset_t *,
416 int *,
417 boolean_t,
418 boolean_t,
419 int);
420extern kern_return_t device_pager_data_initialize(memory_object_t,
421 memory_object_offset_t,
422 memory_object_cluster_size_t);
423extern kern_return_t device_pager_data_unlock(memory_object_t,
424 memory_object_offset_t,
425 memory_object_size_t,
426 vm_prot_t);
427extern kern_return_t device_pager_synchronize(memory_object_t,
428 memory_object_offset_t,
429 memory_object_size_t,
430 vm_sync_t);
431extern kern_return_t device_pager_map(memory_object_t, vm_prot_t);
432extern kern_return_t device_pager_last_unmap(memory_object_t);
433extern kern_return_t device_pager_populate_object(
434 memory_object_t device,
435 memory_object_offset_t offset,
436 ppnum_t page_num,
437 vm_size_t size);
438extern memory_object_t device_pager_setup(
439 memory_object_t,
440 uintptr_t,
441 vm_size_t,
442 int);
443extern void device_pager_bootstrap(void);
444extern boolean_t is_device_pager_ops(const struct memory_object_pager_ops *pager_ops);
445
446extern kern_return_t pager_map_to_phys_contiguous(
447 memory_object_control_t object,
448 memory_object_offset_t offset,
449 addr64_t base_vaddr,
450 vm_size_t size);
451
452extern kern_return_t memory_object_create_named(
453 memory_object_t pager,
454 memory_object_offset_t size,
455 memory_object_control_t *control);
456
457struct macx_triggers_args;
458extern int mach_macx_triggers(
459 struct macx_triggers_args *args);
460
461extern int macx_swapinfo(
462 memory_object_size_t *total_p,
463 memory_object_size_t *avail_p,
464 vm_size_t *pagesize_p,
465 boolean_t *encrypted_p);
466
467extern void log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot);
468extern void log_unnest_badness(
469 vm_map_t map,
470 vm_map_offset_t start_unnest,
471 vm_map_offset_t end_unnest,
472 boolean_t is_nested_map,
473 vm_map_offset_t lowest_unnestable_addr);
474
475struct proc;
476extern int cs_allow_invalid(struct proc *p);
477extern int cs_invalid_page(addr64_t vaddr, boolean_t *cs_killed);
478
479#define CS_VALIDATE_TAINTED 0x00000001
480#define CS_VALIDATE_NX 0x00000002
481extern boolean_t cs_validate_range(struct vnode *vp,
482 memory_object_t pager,
483 memory_object_offset_t offset,
484 const void *data,
485 vm_size_t size,
486 unsigned *result);
487#if PMAP_CS
488extern kern_return_t cs_associate_blob_with_mapping(
489 void *pmap,
490 vm_map_offset_t start,
491 vm_map_size_t size,
492 vm_object_offset_t offset,
493 void *blobs_p);
494#endif /* PMAP_CS */
495
496extern kern_return_t memory_entry_purgeable_control_internal(
497 ipc_port_t entry_port,
498 vm_purgable_t control,
499 int *state);
500
501extern kern_return_t memory_entry_access_tracking_internal(
502 ipc_port_t entry_port,
503 int *access_tracking,
504 uint32_t *access_tracking_reads,
505 uint32_t *access_tracking_writes);
506
507extern kern_return_t mach_memory_entry_purgable_control(
508 ipc_port_t entry_port,
509 vm_purgable_t control,
510 int *state);
511
512extern kern_return_t mach_memory_entry_get_page_counts(
513 ipc_port_t entry_port,
514 unsigned int *resident_page_count,
515 unsigned int *dirty_page_count);
516
517extern kern_return_t mach_memory_entry_page_op(
518 ipc_port_t entry_port,
519 vm_object_offset_t offset,
520 int ops,
521 ppnum_t *phys_entry,
522 int *flags);
523
524extern kern_return_t mach_memory_entry_range_op(
525 ipc_port_t entry_port,
526 vm_object_offset_t offset_beg,
527 vm_object_offset_t offset_end,
528 int ops,
529 int *range);
530
531extern void mach_memory_entry_port_release(ipc_port_t port);
532extern void mach_destroy_memory_entry(ipc_port_t port);
533extern kern_return_t mach_memory_entry_allocate(
534 struct vm_named_entry **user_entry_p,
535 ipc_port_t *user_handle_p);
536
537extern void vm_paging_map_init(void);
538
539extern int macx_backing_store_compaction(int flags);
540extern unsigned int mach_vm_ctl_page_free_wanted(void);
541
542extern int no_paging_space_action(void);
543
544#define VM_TOGGLE_CLEAR 0
545#define VM_TOGGLE_SET 1
546#define VM_TOGGLE_GETVALUE 999
547int vm_toggle_entry_reuse(int, int*);
548
549#define SWAP_WRITE 0x00000000 /* Write buffer (pseudo flag). */
550#define SWAP_READ 0x00000001 /* Read buffer. */
551#define SWAP_ASYNC 0x00000002 /* Start I/O, do not wait. */
552
553extern void vm_compressor_pager_init(void);
554extern kern_return_t compressor_memory_object_create(
555 memory_object_size_t,
556 memory_object_t *);
557
558extern boolean_t vm_compressor_low_on_space(void);
559extern boolean_t vm_compressor_out_of_space(void);
560extern int vm_swap_low_on_space(void);
561void do_fastwake_warmup_all(void);
562#if CONFIG_JETSAM
563extern int proc_get_memstat_priority(struct proc*, boolean_t);
564#endif /* CONFIG_JETSAM */
565
566/* the object purger. purges the next eligible object from memory. */
567/* returns TRUE if an object was purged, otherwise FALSE. */
568boolean_t vm_purgeable_object_purge_one_unlocked(int force_purge_below_group);
569void vm_purgeable_disown(task_t task);
570void vm_purgeable_nonvolatile_owner_update(task_t owner,
571 int delta);
572void vm_purgeable_volatile_owner_update(task_t owner,
573 int delta);
574
575
576struct trim_list {
577 uint64_t tl_offset;
578 uint64_t tl_length;
579 struct trim_list *tl_next;
580};
581
582u_int32_t vnode_trim_list(struct vnode *vp, struct trim_list *tl, boolean_t route_only);
583
584#define MAX_SWAPFILENAME_LEN 1024
585#define SWAPFILENAME_INDEX_LEN 2 /* Doesn't include the terminating NULL character */
586
587extern char swapfilename[MAX_SWAPFILENAME_LEN + 1];
588
589struct vm_counters {
590 unsigned int do_collapse_compressor;
591 unsigned int do_collapse_compressor_pages;
592 unsigned int do_collapse_terminate;
593 unsigned int do_collapse_terminate_failure;
594 unsigned int should_cow_but_wired;
595 unsigned int create_upl_extra_cow;
596 unsigned int create_upl_extra_cow_pages;
597 unsigned int create_upl_lookup_failure_write;
598 unsigned int create_upl_lookup_failure_copy;
599};
600extern struct vm_counters vm_counters;
601
602#if CONFIG_SECLUDED_MEMORY
603struct vm_page_secluded_data {
604 int eligible_for_secluded;
605 int grab_success_free;
606 int grab_success_other;
607 int grab_failure_locked;
608 int grab_failure_state;
609 int grab_failure_dirty;
610 int grab_for_iokit;
611 int grab_for_iokit_success;
612};
613extern struct vm_page_secluded_data vm_page_secluded;
614
615extern int num_tasks_can_use_secluded_mem;
616
617/* boot-args */
618extern int secluded_for_apps;
619extern int secluded_for_iokit;
620extern int secluded_for_filecache;
621#if 11
622extern int secluded_for_fbdp;
623#endif
624
625extern void memory_object_mark_eligible_for_secluded(
626 memory_object_control_t control,
627 boolean_t eligible_for_secluded);
628
629#endif /* CONFIG_SECLUDED_MEMORY */
630
631#define MAX_PAGE_RANGE_QUERY (1ULL * 1024 * 1024 * 1024) /* 1 GB */
632
633extern kern_return_t mach_make_memory_entry_internal(
634 vm_map_t target_map,
635 memory_object_size_t *size,
636 memory_object_offset_t offset,
637 vm_prot_t permission,
638 ipc_port_t *object_handle,
639 ipc_port_t parent_handle);
640
641#define roundup(x, y) ((((x) % (y)) == 0) ? \
642 (x) : ((x) + ((y) - ((x) % (y)))))
643
644#ifdef __cplusplus
645}
646#endif
647
648/*
649 * Flags for the VM swapper/reclaimer.
650 * Used by vm_swap_consider_defragment()
651 * to force defrag/reclaim by the swap
652 * GC thread.
653 */
654#define VM_SWAP_FLAGS_NONE 0
655#define VM_SWAP_FLAGS_FORCE_DEFRAG 1
656#define VM_SWAP_FLAGS_FORCE_RECLAIM 2
657
658#endif /* _VM_VM_PROTOS_H_ */
659
660#endif /* XNU_KERNEL_PRIVATE */
661