1/*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_pageout.h
60 * Author: Avadis Tevanian, Jr.
61 * Date: 1986
62 *
63 * Declarations for the pageout daemon interface.
64 */
65
66#ifndef _VM_VM_PAGEOUT_H_
67#define _VM_VM_PAGEOUT_H_
68
69#ifdef KERNEL_PRIVATE
70
71#include <mach/mach_types.h>
72#include <mach/boolean.h>
73#include <mach/machine/vm_types.h>
74#include <mach/memory_object_types.h>
75
76#include <kern/kern_types.h>
77#include <kern/locks.h>
78
79#include <libkern/OSAtomic.h>
80
81
82#include <vm/vm_options.h>
83
84#ifdef MACH_KERNEL_PRIVATE
85#include <vm/vm_page.h>
86#endif
87
88#include <sys/kdebug.h>
89
90#define VM_PAGE_AVAILABLE_COUNT() ((unsigned int)(vm_page_cleaned_count))
91
92/* externally manipulated counters */
93extern unsigned int vm_pageout_cleaned_fault_reactivated;
94
95#if CONFIG_FREEZE
96extern boolean_t memorystatus_freeze_enabled;
97#endif
98
99#define VM_DYNAMIC_PAGING_ENABLED() (VM_CONFIG_COMPRESSOR_IS_ACTIVE)
100
101#if VM_PRESSURE_EVENTS
102extern boolean_t vm_pressure_events_enabled;
103#endif /* VM_PRESSURE_EVENTS */
104
105
106/*
107 * the following codes are used in the DBG_MACH_WORKINGSET subclass
108 * of the DBG_MACH class
109 */
110#define VM_DISCONNECT_ALL_PAGE_MAPPINGS 0x00
111#define VM_DISCONNECT_TASK_PAGE_MAPPINGS 0x01
112#define VM_REAL_FAULT_ADDR_INTERNAL 0x02
113#define VM_REAL_FAULT_ADDR_PURGABLE 0x03
114#define VM_REAL_FAULT_ADDR_EXTERNAL 0x04
115#define VM_REAL_FAULT_ADDR_SHAREDCACHE 0x05
116
117
118
119extern int vm_debug_events;
120
121#define VMF_CHECK_ZFDELAY 0x100
122#define VMF_COWDELAY 0x101
123#define VMF_ZFDELAY 0x102
124#define VMF_COMPRESSORDELAY 0x103
125
126#define VM_PAGEOUT_SCAN 0x104
127#define VM_PAGEOUT_BALANCE 0x105
128#define VM_PAGEOUT_FREELIST 0x106
129#define VM_PAGEOUT_PURGEONE 0x107
130#define VM_PAGEOUT_CACHE_EVICT 0x108
131#define VM_PAGEOUT_THREAD_BLOCK 0x109
132#define VM_PAGEOUT_JETSAM 0x10A
133#define VM_INFO1 0x10B
134#define VM_INFO2 0x10C
135#define VM_INFO3 0x10D
136#define VM_INFO4 0x10E
137#define VM_INFO5 0x10F
138#define VM_INFO6 0x110
139#define VM_INFO7 0x111
140#define VM_INFO8 0x112
141#define VM_INFO9 0x113
142
143#define VM_UPL_PAGE_WAIT 0x120
144#define VM_IOPL_PAGE_WAIT 0x121
145#define VM_PAGE_WAIT_BLOCK 0x122
146
147#if CONFIG_IOSCHED
148#define VM_PAGE_SLEEP 0x123
149#define VM_PAGE_EXPEDITE 0x124
150#define VM_PAGE_EXPEDITE_NO_MEMORY 0x125
151#endif
152
153#define VM_PAGE_GRAB 0x126
154#define VM_PAGE_RELEASE 0x127
155
156#define VM_PRESSURE_EVENT 0x130
157#define VM_EXECVE 0x131
158#define VM_WAKEUP_COMPACTOR_SWAPPER 0x132
159#define VM_UPL_REQUEST 0x133
160#define VM_IOPL_REQUEST 0x134
161#define VM_KERN_REQUEST 0x135
162
163#define VM_DATA_WRITE 0x140
164
165#define VM_PRESSURE_LEVEL_CHANGE 0x141
166
167#define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4) \
168 MACRO_BEGIN \
169 if (__improbable(vm_debug_events)) { \
170 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
171 } \
172 MACRO_END
173
174#define VM_DEBUG_CONSTANT_EVENT(name, event, control, arg1, arg2, arg3, arg4) \
175 MACRO_BEGIN \
176 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
177 MACRO_END
178
179extern void memoryshot(unsigned int event, unsigned int control);
180
181extern void update_vm_info(void);
182
183#if CONFIG_IOSCHED
184extern int upl_get_cached_tier(
185 upl_t upl);
186#endif
187
188extern void upl_set_iodone(upl_t, void *);
189extern void upl_set_iodone_error(upl_t, int);
190extern void upl_callout_iodone(upl_t);
191
192extern ppnum_t upl_get_highest_page(
193 upl_t upl);
194
195extern upl_size_t upl_get_size(
196 upl_t upl);
197
198extern upl_t upl_associated_upl(upl_t upl);
199extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl);
200
201#ifdef XNU_KERNEL_PRIVATE
202
203extern kern_return_t vm_map_create_upl(
204 vm_map_t map,
205 vm_map_address_t offset,
206 upl_size_t *upl_size,
207 upl_t *upl,
208 upl_page_info_array_t page_list,
209 unsigned int *count,
210 upl_control_flags_t *flags,
211 vm_tag_t tag);
212
213extern void iopl_valid_data(
214 upl_t upl_ptr,
215 vm_tag_t tag);
216
217#endif /* XNU_KERNEL_PRIVATE */
218
219extern struct vnode * upl_lookup_vnode(upl_t upl);
220
221#ifndef MACH_KERNEL_PRIVATE
222typedef struct vm_page *vm_page_t;
223#endif
224
225extern void vm_page_free_list(
226 vm_page_t mem,
227 boolean_t prepare_object);
228
229extern kern_return_t vm_page_alloc_list(
230 int page_count,
231 int flags,
232 vm_page_t * list);
233
234extern void vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
235extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
236extern ppnum_t vm_page_get_phys_page(vm_page_t page);
237extern vm_page_t vm_page_get_next(vm_page_t page);
238
239extern kern_return_t mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level);
240
241#if !CONFIG_EMBEDDED
242extern kern_return_t vm_pageout_wait(uint64_t deadline);
243#endif
244
245#ifdef MACH_KERNEL_PRIVATE
246
247#include <vm/vm_page.h>
248
249extern unsigned int vm_pageout_scan_event_counter;
250extern unsigned int vm_page_anonymous_count;
251
252
253/*
254 * must hold the page queues lock to
255 * manipulate this structure
256 */
257struct vm_pageout_queue {
258 vm_page_queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */
259 unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */
260 unsigned int pgo_maxlaundry;
261 uint64_t pgo_tid; /* thread ID of I/O thread that services this queue */
262 uint8_t pgo_lowpriority; /* iothread is set to use low priority I/O */
263
264 unsigned int pgo_idle:1, /* iothread is blocked waiting for work to do */
265 pgo_busy:1, /* iothread is currently processing request from pgo_pending */
266 pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
267 pgo_draining:1,
268 pgo_inited:1,
269 :0;
270};
271
272#define VM_PAGE_Q_THROTTLED(q) \
273 ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
274
275extern struct vm_pageout_queue vm_pageout_queue_internal;
276extern struct vm_pageout_queue vm_pageout_queue_external;
277
278
279/*
280 * Routines exported to Mach.
281 */
282extern void vm_pageout(void);
283
284extern kern_return_t vm_pageout_internal_start(void);
285
286extern void vm_pageout_object_terminate(
287 vm_object_t object);
288
289extern void vm_pageout_cluster(
290 vm_page_t m);
291
292extern void vm_pageout_initialize_page(
293 vm_page_t m);
294
295/* UPL exported routines and structures */
296
297#define upl_lock_init(object) lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
298#define upl_lock_destroy(object) lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
299#define upl_lock(object) lck_mtx_lock(&(object)->Lock)
300#define upl_unlock(object) lck_mtx_unlock(&(object)->Lock)
301#define upl_try_lock(object) lck_mtx_try_lock(&(object)->Lock)
302
303#define MAX_VECTOR_UPL_ELEMENTS 8
304
305struct _vector_upl_iostates{
306 upl_offset_t offset;
307 upl_size_t size;
308};
309
310typedef struct _vector_upl_iostates vector_upl_iostates_t;
311
312struct _vector_upl {
313 upl_size_t size;
314 uint32_t num_upls;
315 uint32_t invalid_upls;
316 uint32_t _reserved;
317 vm_map_t submap;
318 vm_offset_t submap_dst_addr;
319 vm_object_offset_t offset;
320 upl_t upl_elems[MAX_VECTOR_UPL_ELEMENTS];
321 upl_page_info_array_t pagelist;
322 vector_upl_iostates_t upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
323};
324
325typedef struct _vector_upl* vector_upl_t;
326
327/* universal page list structure */
328
329#if UPL_DEBUG
330#define UPL_DEBUG_STACK_FRAMES 16
331#define UPL_DEBUG_COMMIT_RECORDS 4
332
333struct ucd {
334 upl_offset_t c_beg;
335 upl_offset_t c_end;
336 int c_aborted;
337 void * c_retaddr[UPL_DEBUG_STACK_FRAMES];
338};
339#endif
340
341struct upl_io_completion {
342
343 void *io_context;
344 void (*io_done)(void *, int);
345
346 int io_error;
347};
348
349
350struct upl {
351 decl_lck_mtx_data(, Lock) /* Synchronization */
352 int ref_count;
353 int ext_ref_count;
354 int flags;
355 vm_object_offset_t offset;
356 upl_size_t size; /* size in bytes of the address space */
357 vm_offset_t kaddr; /* secondary mapping in kernel */
358 vm_object_t map_object;
359 ppnum_t highest_page;
360 void* vector_upl;
361 upl_t associated_upl;
362 struct upl_io_completion *upl_iodone;
363#if CONFIG_IOSCHED
364 int upl_priority;
365 uint64_t *upl_reprio_info;
366 void *decmp_io_upl;
367#endif
368#if CONFIG_IOSCHED || UPL_DEBUG
369 thread_t upl_creator;
370 queue_chain_t uplq; /* List of outstanding upls on an obj */
371#endif
372#if UPL_DEBUG
373 uintptr_t ubc_alias1;
374 uintptr_t ubc_alias2;
375
376 uint32_t upl_state;
377 uint32_t upl_commit_index;
378 void *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
379
380 struct ucd upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
381#endif /* UPL_DEBUG */
382};
383
384/* upl struct flags */
385#define UPL_PAGE_LIST_MAPPED 0x1
386#define UPL_KERNEL_MAPPED 0x2
387#define UPL_CLEAR_DIRTY 0x4
388#define UPL_COMPOSITE_LIST 0x8
389#define UPL_INTERNAL 0x10
390#define UPL_PAGE_SYNC_DONE 0x20
391#define UPL_DEVICE_MEMORY 0x40
392#define UPL_PAGEOUT 0x80
393#define UPL_LITE 0x100
394#define UPL_IO_WIRE 0x200
395#define UPL_ACCESS_BLOCKED 0x400
396#define UPL_SHADOWED 0x1000
397#define UPL_KERNEL_OBJECT 0x2000
398#define UPL_VECTOR 0x4000
399#define UPL_SET_DIRTY 0x8000
400#define UPL_HAS_BUSY 0x10000
401#define UPL_TRACKED_BY_OBJECT 0x20000
402#define UPL_EXPEDITE_SUPPORTED 0x40000
403#define UPL_DECMP_REQ 0x80000
404#define UPL_DECMP_REAL_IO 0x100000
405
406/* flags for upl_create flags parameter */
407#define UPL_CREATE_EXTERNAL 0
408#define UPL_CREATE_INTERNAL 0x1
409#define UPL_CREATE_LITE 0x2
410#define UPL_CREATE_IO_TRACKING 0x4
411#define UPL_CREATE_EXPEDITE_SUP 0x8
412
413extern upl_t vector_upl_create(vm_offset_t);
414extern void vector_upl_deallocate(upl_t);
415extern boolean_t vector_upl_is_valid(upl_t);
416extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
417extern void vector_upl_set_pagelist(upl_t);
418extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
419extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
420extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
421extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
422extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
423extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t);
424extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*);
425
426extern void vm_object_set_pmap_cache_attr(
427 vm_object_t object,
428 upl_page_info_array_t user_page_list,
429 unsigned int num_pages,
430 boolean_t batch_pmap_op);
431
432extern kern_return_t vm_object_iopl_request(
433 vm_object_t object,
434 vm_object_offset_t offset,
435 upl_size_t size,
436 upl_t *upl_ptr,
437 upl_page_info_array_t user_page_list,
438 unsigned int *page_list_count,
439 upl_control_flags_t cntrl_flags,
440 vm_tag_t tag);
441
442extern kern_return_t vm_object_super_upl_request(
443 vm_object_t object,
444 vm_object_offset_t offset,
445 upl_size_t size,
446 upl_size_t super_cluster,
447 upl_t *upl,
448 upl_page_info_t *user_page_list,
449 unsigned int *page_list_count,
450 upl_control_flags_t cntrl_flags,
451 vm_tag_t tag);
452
453/* should be just a regular vm_map_enter() */
454extern kern_return_t vm_map_enter_upl(
455 vm_map_t map,
456 upl_t upl,
457 vm_map_offset_t *dst_addr);
458
459/* should be just a regular vm_map_remove() */
460extern kern_return_t vm_map_remove_upl(
461 vm_map_t map,
462 upl_t upl);
463
464/* wired page list structure */
465typedef uint32_t *wpl_array_t;
466
467extern void vm_page_free_reserve(int pages);
468
469extern void vm_pageout_throttle_down(vm_page_t page);
470extern void vm_pageout_throttle_up(vm_page_t page);
471
472extern kern_return_t vm_paging_map_object(
473 vm_page_t page,
474 vm_object_t object,
475 vm_object_offset_t offset,
476 vm_prot_t protection,
477 boolean_t can_unlock_object,
478 vm_map_size_t *size, /* IN/OUT */
479 vm_map_offset_t *address, /* OUT */
480 boolean_t *need_unmap); /* OUT */
481extern void vm_paging_unmap_object(
482 vm_object_t object,
483 vm_map_offset_t start,
484 vm_map_offset_t end);
485decl_simple_lock_data(extern, vm_paging_lock)
486
487/*
488 * Backing store throttle when BS is exhausted
489 */
490extern unsigned int vm_backing_store_low;
491
492extern void vm_pageout_steal_laundry(
493 vm_page_t page,
494 boolean_t queues_locked);
495
496#endif /* MACH_KERNEL_PRIVATE */
497
498#if UPL_DEBUG
499extern kern_return_t upl_ubc_alias_set(
500 upl_t upl,
501 uintptr_t alias1,
502 uintptr_t alias2);
503extern int upl_ubc_alias_get(
504 upl_t upl,
505 uintptr_t * al,
506 uintptr_t * al2);
507#endif /* UPL_DEBUG */
508
509extern void vm_countdirtypages(void);
510
511extern void vm_backing_store_disable(
512 boolean_t suspend);
513
514extern kern_return_t upl_transpose(
515 upl_t upl1,
516 upl_t upl2);
517
518extern kern_return_t mach_vm_pressure_monitor(
519 boolean_t wait_for_pressure,
520 unsigned int nsecs_monitored,
521 unsigned int *pages_reclaimed_p,
522 unsigned int *pages_wanted_p);
523
524extern kern_return_t
525vm_set_buffer_cleanup_callout(
526 boolean_t (*func)(int));
527
528struct vm_page_stats_reusable {
529 SInt32 reusable_count;
530 uint64_t reusable;
531 uint64_t reused;
532 uint64_t reused_wire;
533 uint64_t reused_remove;
534 uint64_t all_reusable_calls;
535 uint64_t partial_reusable_calls;
536 uint64_t all_reuse_calls;
537 uint64_t partial_reuse_calls;
538 uint64_t reusable_pages_success;
539 uint64_t reusable_pages_failure;
540 uint64_t reusable_pages_shared;
541 uint64_t reuse_pages_success;
542 uint64_t reuse_pages_failure;
543 uint64_t can_reuse_success;
544 uint64_t can_reuse_failure;
545 uint64_t reusable_reclaimed;
546 uint64_t reusable_nonwritable;
547 uint64_t reusable_shared;
548 uint64_t free_shared;
549};
550extern struct vm_page_stats_reusable vm_page_stats_reusable;
551
552extern int hibernate_flush_memory(void);
553extern void hibernate_reset_stats(void);
554extern void hibernate_create_paddr_map(void);
555
556extern void vm_set_restrictions(void);
557
558extern int vm_compressor_mode;
559extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t);
560extern void vm_pageout_anonymous_pages(void);
561extern void vm_pageout_disconnect_all_pages(void);
562
563
564struct vm_config {
565 boolean_t compressor_is_present; /* compressor is initialized and can be used by the freezer, the sweep or the pager */
566 boolean_t compressor_is_active; /* pager can actively compress pages... 'compressor_is_present' must be set */
567 boolean_t swap_is_present; /* swap is initialized and can be used by the freezer, the sweep or the pager */
568 boolean_t swap_is_active; /* pager can actively swap out compressed segments... 'swap_is_present' must be set */
569 boolean_t freezer_swap_is_active; /* freezer can swap out frozen tasks... "compressor_is_present + swap_is_present" must be set */
570};
571
572extern struct vm_config vm_config;
573
574
575#define VM_PAGER_NOT_CONFIGURED 0x0 /* no compresser or swap configured */
576#define VM_PAGER_DEFAULT 0x1 /* Use default pager... DEPRECATED */
577#define VM_PAGER_COMPRESSOR_NO_SWAP 0x2 /* Active in-core compressor only. */
578#define VM_PAGER_COMPRESSOR_WITH_SWAP 0x4 /* Active in-core compressor + swap backend. */
579#define VM_PAGER_FREEZER_DEFAULT 0x8 /* Freezer backed by default pager... DEPRECATED */
580#define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP 0x10 /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/
581#define VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP 0x20 /* Active in-core compressor + Freezer backed by in-core compressor with swap support too.*/
582
583#define VM_PAGER_MAX_MODES 6 /* Total number of vm compressor modes supported */
584
585
586#define VM_CONFIG_COMPRESSOR_IS_PRESENT (vm_config.compressor_is_present == TRUE)
587#define VM_CONFIG_COMPRESSOR_IS_ACTIVE (vm_config.compressor_is_active == TRUE)
588#define VM_CONFIG_SWAP_IS_PRESENT (vm_config.swap_is_present == TRUE)
589#define VM_CONFIG_SWAP_IS_ACTIVE (vm_config.swap_is_active == TRUE)
590#define VM_CONFIG_FREEZER_SWAP_IS_ACTIVE (vm_config.freezer_swap_is_active == TRUE)
591
592#endif /* KERNEL_PRIVATE */
593
594#ifdef XNU_KERNEL_PRIVATE
595
596struct vm_pageout_state {
597 boolean_t vm_pressure_thread_running;
598 boolean_t vm_pressure_changed;
599 boolean_t vm_restricted_to_single_processor;
600 int vm_compressor_thread_count;
601
602 unsigned int vm_page_speculative_q_age_ms;
603 unsigned int vm_page_speculative_percentage;
604 unsigned int vm_page_speculative_target;
605
606 unsigned int vm_pageout_swap_wait;
607 unsigned int vm_pageout_idle_wait; /* milliseconds */
608 unsigned int vm_pageout_empty_wait; /* milliseconds */
609 unsigned int vm_pageout_burst_wait; /* milliseconds */
610 unsigned int vm_pageout_deadlock_wait; /* milliseconds */
611 unsigned int vm_pageout_deadlock_relief;
612 unsigned int vm_pageout_burst_inactive_throttle;
613
614 unsigned int vm_pageout_inactive;
615 unsigned int vm_pageout_inactive_used; /* debugging */
616 unsigned int vm_pageout_inactive_clean; /* debugging */
617
618 uint32_t vm_page_filecache_min;
619 uint32_t vm_page_filecache_min_divisor;
620 uint32_t vm_page_xpmapped_min;
621 uint32_t vm_page_xpmapped_min_divisor;
622 uint64_t vm_pageout_considered_page_last;
623
624 int vm_page_free_count_init;
625
626 unsigned int vm_memory_pressure;
627
628 int memorystatus_purge_on_critical;
629 int memorystatus_purge_on_warning;
630 int memorystatus_purge_on_urgent;
631
632 thread_t vm_pageout_external_iothread;
633 thread_t vm_pageout_internal_iothread;
634};
635
636extern struct vm_pageout_state vm_pageout_state;
637
638/*
639 * This structure is used to track the VM_INFO instrumentation
640 */
641struct vm_pageout_vminfo {
642 unsigned long vm_pageout_considered_page;
643 unsigned long vm_pageout_considered_bq_internal;
644 unsigned long vm_pageout_considered_bq_external;
645 unsigned long vm_pageout_skipped_external;
646
647 unsigned long vm_pageout_pages_evicted;
648 unsigned long vm_pageout_pages_purged;;
649 unsigned long vm_pageout_freed_cleaned;
650 unsigned long vm_pageout_freed_speculative;
651 unsigned long vm_pageout_freed_external;
652 unsigned long vm_pageout_freed_internal;
653 unsigned long vm_pageout_inactive_dirty_internal;
654 unsigned long vm_pageout_inactive_dirty_external;
655 unsigned long vm_pageout_inactive_referenced;
656 unsigned long vm_pageout_reactivation_limit_exceeded;
657 unsigned long vm_pageout_inactive_force_reclaim;
658 unsigned long vm_pageout_inactive_nolock;
659 unsigned long vm_pageout_filecache_min_reactivated;
660 unsigned long vm_pageout_scan_inactive_throttled_internal;
661 unsigned long vm_pageout_scan_inactive_throttled_external;
662
663 uint64_t vm_pageout_compressions;
664 uint64_t vm_compressor_pages_grabbed;
665 unsigned long vm_compressor_failed;
666
667 unsigned long vm_page_pages_freed;
668
669 unsigned long vm_phantom_cache_found_ghost;
670 unsigned long vm_phantom_cache_added_ghost;
671};
672
673extern struct vm_pageout_vminfo vm_pageout_vminfo;
674
675
676#if DEVELOPMENT || DEBUG
677
678/*
679 * This structure records the pageout daemon's actions:
680 * how many pages it looks at and what happens to those pages.
681 * No locking needed because only one thread modifies the fields.
682 */
683struct vm_pageout_debug {
684 uint32_t vm_pageout_balanced;
685 uint32_t vm_pageout_scan_event_counter;
686 uint32_t vm_pageout_speculative_dirty;
687
688 uint32_t vm_pageout_inactive_busy;
689 uint32_t vm_pageout_inactive_absent;
690 uint32_t vm_pageout_inactive_notalive;
691 uint32_t vm_pageout_inactive_error;
692 uint32_t vm_pageout_inactive_deactivated;
693
694 uint32_t vm_pageout_enqueued_cleaned;
695
696 uint32_t vm_pageout_cleaned_busy;
697 uint32_t vm_pageout_cleaned_nolock;
698 uint32_t vm_pageout_cleaned_reference_reactivated;
699 uint32_t vm_pageout_cleaned_volatile_reactivated;
700 uint32_t vm_pageout_cleaned_reactivated; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
701 uint32_t vm_pageout_cleaned_fault_reactivated;
702
703 uint32_t vm_pageout_dirty_no_pager;
704 uint32_t vm_pageout_purged_objects;
705
706 uint32_t vm_pageout_scan_throttle;
707 uint32_t vm_pageout_scan_reclaimed_throttled;
708 uint32_t vm_pageout_scan_burst_throttle;
709 uint32_t vm_pageout_scan_empty_throttle;
710 uint32_t vm_pageout_scan_swap_throttle;
711 uint32_t vm_pageout_scan_deadlock_detected;
712 uint32_t vm_pageout_scan_inactive_throttle_success;
713 uint32_t vm_pageout_scan_throttle_deferred;
714
715 uint32_t vm_pageout_inactive_external_forced_jetsam_count;
716
717 uint32_t vm_grab_anon_overrides;
718 uint32_t vm_grab_anon_nops;
719
720 uint32_t vm_pageout_no_victim;
721 unsigned long vm_pageout_throttle_up_count;
722 uint32_t vm_page_steal_pageout_page;
723
724 uint32_t vm_cs_validated_resets;
725 uint32_t vm_object_iopl_request_sleep_for_cleaning;
726 uint32_t vm_page_slide_counter;
727 uint32_t vm_page_slide_errors;
728 uint32_t vm_page_throttle_count;
729 /*
730 * Statistics about UPL enforcement of copy-on-write obligations.
731 */
732 unsigned long upl_cow;
733 unsigned long upl_cow_again;
734 unsigned long upl_cow_pages;
735 unsigned long upl_cow_again_pages;
736 unsigned long iopl_cow;
737 unsigned long iopl_cow_pages;
738};
739
740extern struct vm_pageout_debug vm_pageout_debug;
741
742#define VM_PAGEOUT_DEBUG(member, value) \
743 MACRO_BEGIN \
744 vm_pageout_debug.member += value; \
745 MACRO_END
746#else
747#define VM_PAGEOUT_DEBUG(member, value)
748#endif
749
750#define MAX_COMPRESSOR_THREAD_COUNT 8
751
752#if DEVELOPMENT || DEBUG
753typedef struct vmct_stats_s {
754 uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT];
755 uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT];
756 uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT];
757 uint64_t vmct_cthreads_total;
758 int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT];
759 int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT];
760} vmct_stats_t;
761#endif
762#endif
763#endif /* _VM_VM_PAGEOUT_H_ */
764