kern_memorystatus.c source code [codebrowser/bsd/kern/kern_memorystatus.c]

1	/*
2	* Copyright (c) 2006-2018 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*
28	*/
29
30	#include <kern/sched_prim.h>
31	#include <kern/kalloc.h>
32	#include <kern/assert.h>
33	#include <kern/debug.h>
34	#include <kern/locks.h>
35	#include <kern/task.h>
36	#include <kern/thread.h>
37	#include <kern/host.h>
38	#include <kern/policy_internal.h>
39	#include <kern/thread_group.h>
40
41	#include <IOKit/IOBSD.h>
42
43	#include <libkern/libkern.h>
44	#include <mach/coalition.h>
45	#include <mach/mach_time.h>
46	#include <mach/task.h>
47	#include <mach/host_priv.h>
48	#include <mach/mach_host.h>
49	#include <os/log.h>
50	#include <pexpert/pexpert.h>
51	#include <sys/coalition.h>
52	#include <sys/kern_event.h>
53	#include <sys/proc.h>
54	#include <sys/proc_info.h>
55	#include <sys/reason.h>
56	#include <sys/signal.h>
57	#include <sys/signalvar.h>
58	#include <sys/sysctl.h>
59	#include <sys/sysproto.h>
60	#include <sys/wait.h>
61	#include <sys/tree.h>
62	#include <sys/priv.h>
63	#include <vm/vm_pageout.h>
64	#include <vm/vm_protos.h>
65
66	#if CONFIG_FREEZE
67	#include <vm/vm_map.h>
68	#endif /* CONFIG_FREEZE */
69
70	#include <sys/kern_memorystatus.h>
71
72	#include <mach/machine/sdt.h>
73	#include <libkern/section_keywords.h>
74	#include <stdatomic.h>
75
76	/ For logging clarity /
77	static const char *memorystatus_kill_cause_name[] = {
78	"" , / kMemorystatusInvalid /
79	"jettisoned" , / kMemorystatusKilled /
80	"highwater" , / kMemorystatusKilledHiwat /
81	"vnode-limit" , / kMemorystatusKilledVnodes /
82	"vm-pageshortage" , / kMemorystatusKilledVMPageShortage /
83	"proc-thrashing" , / kMemorystatusKilledProcThrashing /
84	"fc-thrashing" , / kMemorystatusKilledFCThrashing /
85	"per-process-limit" , / kMemorystatusKilledPerProcessLimit /
86	"disk-space-shortage" , / kMemorystatusKilledDiskSpaceShortage /
87	"idle-exit" , / kMemorystatusKilledIdleExit /
88	"zone-map-exhaustion" , / kMemorystatusKilledZoneMapExhaustion /
89	"vm-compressor-thrashing" , / kMemorystatusKilledVMCompressorThrashing /
90	"vm-compressor-space-shortage" , / kMemorystatusKilledVMCompressorSpaceShortage /
91	};
92
93	static const char *
94	memorystatus_priority_band_name(int32_t priority)
95	{
96	switch (priority) {
97	case JETSAM_PRIORITY_FOREGROUND:
98	return "FOREGROUND";
99	case JETSAM_PRIORITY_AUDIO_AND_ACCESSORY:
100	return "AUDIO_AND_ACCESSORY";
101	case JETSAM_PRIORITY_CONDUCTOR:
102	return "CONDUCTOR";
103	case JETSAM_PRIORITY_HOME:
104	return "HOME";
105	case JETSAM_PRIORITY_EXECUTIVE:
106	return "EXECUTIVE";
107	case JETSAM_PRIORITY_IMPORTANT:
108	return "IMPORTANT";
109	case JETSAM_PRIORITY_CRITICAL:
110	return "CRITICAL";
111	}
112
113	return ("?");
114	}
115
116	/ Does cause indicate vm or fc thrashing? /
117	static boolean_t
118	is_reason_thrashing(unsigned cause)
119	{
120	switch (cause) {
121	case kMemorystatusKilledFCThrashing:
122	case kMemorystatusKilledVMCompressorThrashing:
123	case kMemorystatusKilledVMCompressorSpaceShortage:
124	return TRUE;
125	default:
126	return FALSE;
127	}
128	}
129
130	/ Is the zone map almost full? /
131	static boolean_t
132	is_reason_zone_map_exhaustion(unsigned cause)
133	{
134	if (cause == kMemorystatusKilledZoneMapExhaustion)
135	return TRUE;
136	return FALSE;
137	}
138
139	/*
140	* Returns the current zone map size and capacity to include in the jetsam snapshot.
141	* Defined in zalloc.c
142	*/
143	extern void get_zone_map_size(uint64_t current_size, uint64_t capacity);
144
145	/*
146	* Returns the name of the largest zone and its size to include in the jetsam snapshot.
147	* Defined in zalloc.c
148	*/
149	extern void get_largest_zone_info(char zone_name, size_t zone_name_len, uint64_t zone_size);
150
151	/ These are very verbose printfs(), enable with*
152	* MEMORYSTATUS_DEBUG_LOG
153	*/
154	#if MEMORYSTATUS_DEBUG_LOG
155	#define MEMORYSTATUS_DEBUG(cond, format, ...) \
156	do { \
157	if (cond) { printf(format, ##__VA_ARGS__); } \
158	} while(0)
159	#else
160	#define MEMORYSTATUS_DEBUG(cond, format, ...)
161	#endif
162
163	/*
164	* Active / Inactive limit support
165	* proc list must be locked
166	*
167	* The SET_*** macros are used to initialize a limit
168	* for the first time.
169	*
170	* The CACHE_*** macros are use to cache the limit that will
171	* soon be in effect down in the ledgers.
172	*/
173
174	#define SET_ACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
175	MACRO_BEGIN \
176	(p)->p_memstat_memlimit_active = (limit); \
177	if (is_fatal) { \
178	(p)->p_memstat_state \|= P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
179	} else { \
180	(p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
181	} \
182	MACRO_END
183
184	#define SET_INACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
185	MACRO_BEGIN \
186	(p)->p_memstat_memlimit_inactive = (limit); \
187	if (is_fatal) { \
188	(p)->p_memstat_state \|= P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
189	} else { \
190	(p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
191	} \
192	MACRO_END
193
194	#define CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal) \
195	MACRO_BEGIN \
196	(p)->p_memstat_memlimit = (p)->p_memstat_memlimit_active; \
197	if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) { \
198	(p)->p_memstat_state \|= P_MEMSTAT_FATAL_MEMLIMIT; \
199	is_fatal = TRUE; \
200	} else { \
201	(p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
202	is_fatal = FALSE; \
203	} \
204	MACRO_END
205
206	#define CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal) \
207	MACRO_BEGIN \
208	(p)->p_memstat_memlimit = (p)->p_memstat_memlimit_inactive; \
209	if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) { \
210	(p)->p_memstat_state \|= P_MEMSTAT_FATAL_MEMLIMIT; \
211	is_fatal = TRUE; \
212	} else { \
213	(p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
214	is_fatal = FALSE; \
215	} \
216	MACRO_END
217
218
219	/ General tunables /
220
221	unsigned long delta_percentage = `5`;
222	unsigned long critical_threshold_percentage = `5`;
223	unsigned long idle_offset_percentage = `5`;
224	unsigned long pressure_threshold_percentage = `15`;
225	unsigned long freeze_threshold_percentage = `50`;
226	unsigned long policy_more_free_offset_percentage = `5`;
227
228	/ General memorystatus stuff /
229
230	struct klist memorystatus_klist;
231	static lck_mtx_t memorystatus_klist_mutex;
232
233	static void memorystatus_klist_lock(void);
234	static void memorystatus_klist_unlock(void);
235
236	static uint64_t memorystatus_sysprocs_idle_delay_time = `0`;
237	static uint64_t memorystatus_apps_idle_delay_time = `0`;
238
239	/*
240	* Memorystatus kevents
241	*/
242
243	static int filt_memorystatusattach(struct knote kn, struct* kevent_internal_s *kev);
244	static void filt_memorystatusdetach(struct knote *kn);
245	static int filt_memorystatus(struct knote kn, long* hint);
246	static int filt_memorystatustouch(struct knote kn, struct* kevent_internal_s *kev);
247	static int filt_memorystatusprocess(struct knote kn, struct* filt_process_s data, struct* kevent_internal_s *kev);
248
249	SECURITY_READ_ONLY_EARLY(struct filterops) memorystatus_filtops = {
250	.f_attach = filt_memorystatusattach,
251	.f_detach = filt_memorystatusdetach,
252	.f_event = filt_memorystatus,
253	.f_touch = filt_memorystatustouch,
254	.f_process = filt_memorystatusprocess,
255	};
256
257	enum {
258	kMemorystatusNoPressure = `0x1`,
259	kMemorystatusPressure = `0x2`,
260	kMemorystatusLowSwap = `0x4`,
261	kMemorystatusProcLimitWarn = `0x8`,
262	kMemorystatusProcLimitCritical = `0x10`
263	};
264
265	/ Idle guard handling /
266
267	static int32_t memorystatus_scheduled_idle_demotions_sysprocs = `0`;
268	static int32_t memorystatus_scheduled_idle_demotions_apps = `0`;
269
270	static thread_call_t memorystatus_idle_demotion_call;
271
272	static void memorystatus_perform_idle_demotion(__unused void spare1, __unused void* *spare2);
273	static void memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state);
274	static void memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clean_state);
275	static void memorystatus_reschedule_idle_demotion_locked(void);
276
277	static void memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check);
278
279	int memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap);
280
281	vm_pressure_level_t convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
282
283	boolean_t is_knote_registered_modify_task_pressure_bits(struct knote, int*, task_t, vm_pressure_level_t, vm_pressure_level_t);
284	void memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear);
285	void memorystatus_send_low_swap_note(void);
286
287	unsigned int memorystatus_level = `0`;
288
289	static int memorystatus_list_count = `0`;
290
291
292	#define MEMSTAT_BUCKET_COUNT (JETSAM_PRIORITY_MAX + 1)
293
294	typedef struct memstat_bucket {
295	TAILQ_HEAD(, proc) list;
296	int count;
297	} memstat_bucket_t;
298
299	memstat_bucket_t memstat_bucket[MEMSTAT_BUCKET_COUNT];
300
301	int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index);
302
303	uint64_t memstat_idle_demotion_deadline = `0`;
304
305	int system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
306	int applications_aging_band = JETSAM_PRIORITY_IDLE;
307
308	#define isProcessInAgingBands(p) ((isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) \|\| (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)))
309
310	/*
311	* Checking the p_memstat_state almost always requires the proc_list_lock
312	* because the jetsam thread could be on the other core changing the state.
313	*
314	* App -- almost always managed by a system process. Always have dirty tracking OFF. Can include extensions too.
315	* System Processes -- not managed by anybody. Always have dirty tracking ON. Can include extensions (here) too.
316	*/
317	#define isApp(p) ((p->p_memstat_state & P_MEMSTAT_MANAGED) \|\| ! (p->p_memstat_dirty & P_DIRTY_TRACK))
318	#define isSysProc(p) ( ! (p->p_memstat_state & P_MEMSTAT_MANAGED) \|\| (p->p_memstat_dirty & P_DIRTY_TRACK))
319
320	#define kJetsamAgingPolicyNone (0)
321	#define kJetsamAgingPolicyLegacy (1)
322	#define kJetsamAgingPolicySysProcsReclaimedFirst (2)
323	#define kJetsamAgingPolicyAppsReclaimedFirst (3)
324	#define kJetsamAgingPolicyMax kJetsamAgingPolicyAppsReclaimedFirst
325
326	unsigned int jetsam_aging_policy = kJetsamAgingPolicyLegacy;
327
328	extern int corpse_for_fatal_memkill;
329	extern unsigned long total_corpses_count(void) __attribute__((pure));
330	extern void task_purge_all_corpses(void);
331	extern uint64_t vm_purgeable_purge_task_owned(task_t task);
332	boolean_t memorystatus_allowed_vm_map_fork(task_t);
333	#if DEVELOPMENT \|\| DEBUG
334	void memorystatus_abort_vm_map_fork(task_t);
335	#endif
336
337	#if 0
338
339	/ Keeping around for future use if we need a utility that can do this OR an app that needs a dynamic adjustment. /
340
341	static int
342	sysctl_set_jetsam_aging_policy SYSCTL_HANDLER_ARGS
343	{
344	#pragma unused(oidp, arg1, arg2)
345
346	int error = `0`, val = `0`;
347	memstat_bucket_t *old_bucket = `0`;
348	int old_system_procs_aging_band = `0`, new_system_procs_aging_band = `0`;
349	int old_applications_aging_band = `0`, new_applications_aging_band = `0`;
350	proc_t p = NULL, next_proc = NULL;
351
352
353	error = sysctl_io_number(req, jetsam_aging_policy, sizeof(int), &val, NULL);
354	if (error \|\| !req->newptr) {
355	return (error);
356	}
357
358	if ((val < `0`) \|\| (val > kJetsamAgingPolicyMax)) {
359	printf("jetsam: ordering policy sysctl has invalid value - %d\n", val);
360	return EINVAL;
361	}
362
363	/*
364	* We need to synchronize with any potential adding/removal from aging bands
365	* that might be in progress currently. We use the proc_list_lock() just for
366	* consistency with all the routines dealing with 'aging' processes. We need
367	* a lighterweight lock.
368	*/
369	proc_list_lock();
370
371	old_system_procs_aging_band = system_procs_aging_band;
372	old_applications_aging_band = applications_aging_band;
373
374	switch (val) {
375
376	case kJetsamAgingPolicyNone:
377	new_system_procs_aging_band = JETSAM_PRIORITY_IDLE;
378	new_applications_aging_band = JETSAM_PRIORITY_IDLE;
379	break;
380
381	case kJetsamAgingPolicyLegacy:
382	/*
383	* Legacy behavior where some daemons get a 10s protection once and only before the first clean->dirty->clean transition before going into IDLE band.
384	*/
385	new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
386	new_applications_aging_band = JETSAM_PRIORITY_IDLE;
387	break;
388
389	case kJetsamAgingPolicySysProcsReclaimedFirst:
390	new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
391	new_applications_aging_band = JETSAM_PRIORITY_AGING_BAND2;
392	break;
393
394	case kJetsamAgingPolicyAppsReclaimedFirst:
395	new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND2;
396	new_applications_aging_band = JETSAM_PRIORITY_AGING_BAND1;
397	break;
398
399	default:
400	break;
401	}
402
403	if (old_system_procs_aging_band && (old_system_procs_aging_band != new_system_procs_aging_band)) {
404
405	old_bucket = &memstat_bucket[old_system_procs_aging_band];
406	p = TAILQ_FIRST(&old_bucket->list);
407
408	while (p) {
409
410	next_proc = TAILQ_NEXT(p, p_memstat_list);
411
412	if (isSysProc(p)) {
413	if (new_system_procs_aging_band == JETSAM_PRIORITY_IDLE) {
414	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
415	}
416
417	memorystatus_update_priority_locked(p, new_system_procs_aging_band, false, true);
418	}
419
420	p = next_proc;
421	continue;
422	}
423	}
424
425	if (old_applications_aging_band && (old_applications_aging_band != new_applications_aging_band)) {
426
427	old_bucket = &memstat_bucket[old_applications_aging_band];
428	p = TAILQ_FIRST(&old_bucket->list);
429
430	while (p) {
431
432	next_proc = TAILQ_NEXT(p, p_memstat_list);
433
434	if (isApp(p)) {
435	if (new_applications_aging_band == JETSAM_PRIORITY_IDLE) {
436	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
437	}
438
439	memorystatus_update_priority_locked(p, new_applications_aging_band, false, true);
440	}
441
442	p = next_proc;
443	continue;
444	}
445	}
446
447	jetsam_aging_policy = val;
448	system_procs_aging_band = new_system_procs_aging_band;
449	applications_aging_band = new_applications_aging_band;
450
451	proc_list_unlock();
452
453	return (`0`);
454	}
455
456	SYSCTL_PROC(_kern, OID_AUTO, set_jetsam_aging_policy, CTLTYPE_INT\|CTLFLAG_RW,
457	`0`, `0`, sysctl_set_jetsam_aging_policy, "I", "Jetsam Aging Policy");
458	#endif /0/
459
460	static int
461	sysctl_jetsam_set_sysprocs_idle_delay_time SYSCTL_HANDLER_ARGS
462	{
463	#pragma unused(oidp, arg1, arg2)
464
465	int error = `0`, val = `0`, old_time_in_secs = `0`;
466	uint64_t old_time_in_ns = `0`;
467
468	absolutetime_to_nanoseconds(memorystatus_sysprocs_idle_delay_time, &old_time_in_ns);
469	old_time_in_secs = old_time_in_ns / NSEC_PER_SEC;
470
471	error = sysctl_io_number(req, old_time_in_secs, sizeof(int), &val, NULL);
472	if (error \|\| !req->newptr) {
473	return (error);
474	}
475
476	if ((val < `0`) \|\| (val > INT32_MAX)) {
477	printf("jetsam: new idle delay interval has invalid value.\n");
478	return EINVAL;
479	}
480
481	nanoseconds_to_absolutetime((uint64_t)val * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time);
482
483	return(`0`);
484	}
485
486	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_sysprocs_idle_delay_time, CTLTYPE_INT\|CTLFLAG_RW,
487	`0`, `0`, sysctl_jetsam_set_sysprocs_idle_delay_time, "I", "Aging window for system processes");
488
489
490	static int
491	sysctl_jetsam_set_apps_idle_delay_time SYSCTL_HANDLER_ARGS
492	{
493	#pragma unused(oidp, arg1, arg2)
494
495	int error = `0`, val = `0`, old_time_in_secs = `0`;
496	uint64_t old_time_in_ns = `0`;
497
498	absolutetime_to_nanoseconds(memorystatus_apps_idle_delay_time, &old_time_in_ns);
499	old_time_in_secs = old_time_in_ns / NSEC_PER_SEC;
500
501	error = sysctl_io_number(req, old_time_in_secs, sizeof(int), &val, NULL);
502	if (error \|\| !req->newptr) {
503	return (error);
504	}
505
506	if ((val < `0`) \|\| (val > INT32_MAX)) {
507	printf("jetsam: new idle delay interval has invalid value.\n");
508	return EINVAL;
509	}
510
511	nanoseconds_to_absolutetime((uint64_t)val * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time);
512
513	return(`0`);
514	}
515
516	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_apps_idle_delay_time, CTLTYPE_INT\|CTLFLAG_RW,
517	`0`, `0`, sysctl_jetsam_set_apps_idle_delay_time, "I", "Aging window for applications");
518
519	SYSCTL_INT(_kern, OID_AUTO, jetsam_aging_policy, CTLTYPE_INT\|CTLFLAG_RD, &jetsam_aging_policy, `0`, "");
520
521	static unsigned int memorystatus_dirty_count = `0`;
522
523	SYSCTL_INT(_kern, OID_AUTO, max_task_pmem, CTLFLAG_RD\|CTLFLAG_LOCKED\|CTLFLAG_MASKED, &max_task_footprint_mb, `0`, "");
524
525	#if CONFIG_EMBEDDED
526
527	SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_level, `0`, "");
528
529	#endif /* CONFIG_EMBEDDED */
530
531	int
532	memorystatus_get_level(__unused struct proc p, struct* memorystatus_get_level_args args, __unused int* *ret)
533	{
534	user_addr_t level = `0`;
535
536	level = args->level;
537
538	if (copyout(&memorystatus_level, level, sizeof(memorystatus_level)) != `0`) {
539	return EFAULT;
540	}
541
542	return `0`;
543	}
544
545	static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search);
546	static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search);
547
548	static void memorystatus_thread(void *param __unused, wait_result_t wr __unused);
549
550	/ Memory Limits /
551
552	static int memorystatus_highwater_enabled = `1`; / Update the cached memlimit data. /
553
554	static boolean_t proc_jetsam_state_is_active_locked(proc_t);
555	static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
556	static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
557
558
559	static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
560
561	static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry);
562
563	static int memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
564
565	static int memorystatus_cmd_get_memlimit_excess_np(pid_t pid, uint32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
566
567	int proc_get_memstat_priority(proc_t, boolean_t);
568
569	static boolean_t memorystatus_idle_snapshot = `0`;
570
571	unsigned int memorystatus_delta = `0`;
572
573	/ Jetsam Loop Detection /
574	static boolean_t memorystatus_jld_enabled = FALSE; / Enable jetsam loop detection /
575	static uint32_t memorystatus_jld_eval_period_msecs = `0`; / Init pass sets this based on device memory size /
576	static int memorystatus_jld_eval_aggressive_count = `3`; / Raise the priority max after 'n' aggressive loops /
577	static int memorystatus_jld_eval_aggressive_priority_band_max = `15`; / Kill aggressively up through this band /
578
579	/*
580	* A FG app can request that the aggressive jetsam mechanism display some leniency in the FG band. This 'lenient' mode is described as:
581	* --- if aggressive jetsam kills an app in the FG band and gets back >=AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD memory, it will stop the aggressive march further into and up the jetsam bands.
582	*
583	* RESTRICTIONS:
584	* - Such a request is respected/acknowledged only once while that 'requesting' app is in the FG band i.e. if aggressive jetsam was
585	* needed and the 'lenient' mode was deployed then that's it for this special mode while the app is in the FG band.
586	*
587	* - If the app is still in the FG band and aggressive jetsam is needed again, there will be no stop-and-check the next time around.
588	*
589	* - Also, the transition of the 'requesting' app away from the FG band will void this special behavior.
590	*/
591
592	#define AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD 25
593	boolean_t memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
594	boolean_t memorystatus_aggressive_jetsam_lenient = FALSE;
595
596	#if DEVELOPMENT \|\| DEBUG
597	/*
598	* Jetsam Loop Detection tunables.
599	*/
600
601	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_period_msecs, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_jld_eval_period_msecs, `0`, "");
602	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_count, `0`, "");
603	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, `0`, "");
604	#endif /* DEVELOPMENT \|\| DEBUG */
605
606	static uint32_t kill_under_pressure_cause = `0`;
607
608	/*
609	* default jetsam snapshot support
610	*/
611	static memorystatus_jetsam_snapshot_t *memorystatus_jetsam_snapshot;
612	static memorystatus_jetsam_snapshot_t *memorystatus_jetsam_snapshot_copy;
613	#define memorystatus_jetsam_snapshot_list memorystatus_jetsam_snapshot->entries
614	static unsigned int memorystatus_jetsam_snapshot_count = `0`;
615	static unsigned int memorystatus_jetsam_snapshot_copy_count = `0`;
616	static unsigned int memorystatus_jetsam_snapshot_max = `0`;
617	static unsigned int memorystatus_jetsam_snapshot_size = `0`;
618	static uint64_t memorystatus_jetsam_snapshot_last_timestamp = `0`;
619	static uint64_t memorystatus_jetsam_snapshot_timeout = `0`;
620	#define JETSAM_SNAPSHOT_TIMEOUT_SECS 30
621
622	/*
623	* snapshot support for memstats collected at boot.
624	*/
625	static memorystatus_jetsam_snapshot_t memorystatus_at_boot_snapshot;
626
627	static void memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count);
628	static boolean_t memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount);
629	static void memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause, uint64_t killtime);
630
631	static void memorystatus_clear_errors(void);
632	static void memorystatus_get_task_page_counts(task_t task, uint32_t footprint, uint32_t max_footprint_lifetime, uint32_t *purgeable_pages);
633	static void memorystatus_get_task_phys_footprint_page_counts(task_t task,
634	uint64_t internal_pages, uint64_t internal_compressed_pages,
635	uint64_t purgeable_nonvolatile_pages, uint64_t purgeable_nonvolatile_compressed_pages,
636	uint64_t alternate_accounting_pages, uint64_t alternate_accounting_compressed_pages,
637	uint64_t iokit_mapped_pages, uint64_t page_table_pages);
638
639	static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count);
640
641	static uint32_t memorystatus_build_state(proc_t p);
642	//static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured);
643
644	static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason, int32_t priority, uint32_t errors);
645	static boolean_t memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors);
646	static boolean_t memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, unsigned int band, int aggr_count, uint32_t *errors);
647	static boolean_t memorystatus_kill_hiwat_proc(uint32_t errors, boolean_t purged);
648
649	static boolean_t memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause);
650
651	/ Priority Band Sorting Routines /
652	static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order);
653	static int memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order);
654	static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index);
655	static int memorystatus_move_list_locked(unsigned int bucket_index, pid_t pid_list, int* list_sz);
656
657	/ qsort routines /
658	typedef int (cmpfunc_t)(const* void a, const* void *b);
659	extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
660	static int memstat_asc_cmp(const void a, const* void *b);
661
662	/ VM pressure /
663
664	extern unsigned int vm_page_free_count;
665	extern unsigned int vm_page_active_count;
666	extern unsigned int vm_page_inactive_count;
667	extern unsigned int vm_page_throttled_count;
668	extern unsigned int vm_page_purgeable_count;
669	extern unsigned int vm_page_wire_count;
670	#if CONFIG_SECLUDED_MEMORY
671	extern unsigned int vm_page_secluded_count;
672	#endif /* CONFIG_SECLUDED_MEMORY */
673
674	#if CONFIG_JETSAM
675	unsigned int memorystatus_available_pages = (unsigned int)-`1`;
676	unsigned int memorystatus_available_pages_pressure = `0`;
677	unsigned int memorystatus_available_pages_critical = `0`;
678	static unsigned int memorystatus_available_pages_critical_base = `0`;
679	static unsigned int memorystatus_available_pages_critical_idle_offset = `0`;
680
681	#if DEVELOPMENT \|\| DEBUG
682	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD \| CTLFLAG_LOCKED, &memorystatus_available_pages, `0`, "");
683	#else
684	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD \| CTLFLAG_MASKED \| CTLFLAG_LOCKED, &memorystatus_available_pages, `0`, "");
685	#endif /* DEVELOPMENT \|\| DEBUG */
686
687	static unsigned int memorystatus_jetsam_policy = kPolicyDefault;
688	unsigned int memorystatus_policy_more_free_offset_pages = `0`;
689	static void memorystatus_update_levels_locked(boolean_t critical_only);
690	static unsigned int memorystatus_thread_wasted_wakeup = `0`;
691
692	/ Callback into vm_compressor.c to signal that thrashing has been mitigated. /
693	extern void vm_thrashing_jetsam_done(void);
694	static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit);
695
696	int32_t max_kill_priority = JETSAM_PRIORITY_MAX;
697
698	#else /* CONFIG_JETSAM */
699
700	uint64_t memorystatus_available_pages = (uint64_t)-`1`;
701	uint64_t memorystatus_available_pages_pressure = (uint64_t)-`1`;
702	uint64_t memorystatus_available_pages_critical = (uint64_t)-`1`;
703
704	int32_t max_kill_priority = JETSAM_PRIORITY_IDLE;
705	#endif /* CONFIG_JETSAM */
706
707	unsigned int memorystatus_frozen_count = `0`;
708	unsigned int memorystatus_frozen_processes_max = `0`;
709	unsigned int memorystatus_frozen_shared_mb = `0`;
710	unsigned int memorystatus_frozen_shared_mb_max = `0`;
711	unsigned int memorystatus_freeze_shared_mb_per_process_max = `0`; / Max. MB allowed per process to be freezer-eligible. /
712	unsigned int memorystatus_freeze_private_shared_pages_ratio = `2`; / Ratio of private:shared pages for a process to be freezer-eligible. /
713	unsigned int memorystatus_suspended_count = `0`;
714	unsigned int memorystatus_thaw_count = `0`;
715	unsigned int memorystatus_refreeze_eligible_count = `0`; / # of processes currently thawed i.e. have state on disk & in-memory /
716
717	#if VM_PRESSURE_EVENTS
718
719	boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded);
720
721	vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal;
722
723	/*
724	* We use this flag to signal if we have any HWM offenders
725	* on the system. This way we can reduce the number of wakeups
726	* of the memorystatus_thread when the system is between the
727	* "pressure" and "critical" threshold.
728	*
729	* The (re-)setting of this variable is done without any locks
730	* or synchronization simply because it is not possible (currently)
731	* to keep track of HWM offenders that drop down below their memory
732	* limit and/or exit. So, we choose to burn a couple of wasted wakeups
733	* by allowing the unguarded modification of this variable.
734	*/
735	boolean_t memorystatus_hwm_candidates = `0`;
736
737	static int memorystatus_send_note(int event_code, void *data, size_t data_length);
738
739	/*
740	* This value is the threshold that a process must meet to be considered for scavenging.
741	*/
742	#if CONFIG_EMBEDDED
743	#define VM_PRESSURE_MINIMUM_RSIZE 6 /* MB */
744	#else /* CONFIG_EMBEDDED */
745	#define VM_PRESSURE_MINIMUM_RSIZE 10 /* MB */
746	#endif /* CONFIG_EMBEDDED */
747
748	uint32_t vm_pressure_task_footprint_min = VM_PRESSURE_MINIMUM_RSIZE;
749
750	#if DEVELOPMENT \|\| DEBUG
751	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_vm_pressure_task_footprint_min, CTLFLAG_RW\|CTLFLAG_LOCKED, &vm_pressure_task_footprint_min, `0`, "");
752	#endif /* DEVELOPMENT \|\| DEBUG */
753
754	#endif /* VM_PRESSURE_EVENTS */
755
756
757	#if DEVELOPMENT \|\| DEBUG
758
759	lck_grp_attr_t *disconnect_page_mappings_lck_grp_attr;
760	lck_grp_t *disconnect_page_mappings_lck_grp;
761	static lck_mtx_t disconnect_page_mappings_mutex;
762
763	extern boolean_t kill_on_no_paging_space;
764	#endif /* DEVELOPMENT \|\| DEBUG */
765
766
767	/*
768	* Table that expresses the probability of a process
769	* being used in the next hour.
770	*/
771	typedef struct memorystatus_internal_probabilities {
772	char proc_name[MAXCOMLEN + `1`];
773	int use_probability;
774	} memorystatus_internal_probabilities_t;
775
776	static memorystatus_internal_probabilities_t *memorystatus_global_probabilities_table = NULL;
777	static size_t memorystatus_global_probabilities_size = `0`;
778
779	/ Freeze /
780
781	#if CONFIG_FREEZE
782	boolean_t memorystatus_freeze_enabled = FALSE;
783	int memorystatus_freeze_wakeup = `0`;
784	int memorystatus_freeze_jetsam_band = `0`; / the jetsam band which will contain P_MEMSTAT_FROZEN processes /
785
786	lck_grp_attr_t *freezer_lck_grp_attr;
787	lck_grp_t *freezer_lck_grp;
788	static lck_mtx_t freezer_mutex;
789
790	static inline boolean_t memorystatus_can_freeze_processes(void);
791	static boolean_t memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low);
792	static boolean_t memorystatus_is_process_eligible_for_freeze(proc_t p);
793	static void memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused);
794	static boolean_t memorystatus_freeze_thread_should_run(void);
795
796	void memorystatus_disable_freeze(void);
797
798	/ Thresholds /
799	static unsigned int memorystatus_freeze_threshold = `0`;
800
801	static unsigned int memorystatus_freeze_pages_min = `0`;
802	static unsigned int memorystatus_freeze_pages_max = `0`;
803
804	static unsigned int memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
805
806	static unsigned int memorystatus_freeze_daily_mb_max = FREEZE_DAILY_MB_MAX_DEFAULT;
807	static uint64_t memorystatus_freeze_budget_pages_remaining = `0`; //remaining # of pages that can be frozen to disk
808	static boolean_t memorystatus_freeze_degradation = FALSE; //protected by the freezer mutex. Signals we are in a degraded freeze mode.
809
810	static unsigned int memorystatus_max_frozen_demotions_daily = `0`;
811	static unsigned int memorystatus_thaw_count_demotion_threshold = `0`;
812
813	/ Stats /
814	static uint64_t memorystatus_freeze_pageouts = `0`;
815
816	/ Throttling /
817	#define DEGRADED_WINDOW_MINS (30)
818	#define NORMAL_WINDOW_MINS (24 * 60)
819
820	static throttle_interval_t throttle_intervals[] = {
821	{ DEGRADED_WINDOW_MINS, `1`, `0`, `0`, { `0`, `0` }},
822	{ NORMAL_WINDOW_MINS, `1`, `0`, `0`, { `0`, `0` }},
823	};
824	throttle_interval_t *degraded_throttle_window = &throttle_intervals[`0`];
825	throttle_interval_t *normal_throttle_window = &throttle_intervals[`1`];
826
827	extern uint64_t vm_swap_get_free_space(void);
828	extern boolean_t vm_swap_max_budget(uint64_t *);
829
830	static void memorystatus_freeze_update_throttle(uint64_t *budget_pages_allowed);
831
832	static uint64_t memorystatus_freezer_thread_next_run_ts = `0`;
833
834	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_count, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_frozen_count, `0`, "");
835	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_thaw_count, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_thaw_count, `0`, "");
836	SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_pageouts, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_freeze_pageouts, "");
837	SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_budget_pages_remaining, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_freeze_budget_pages_remaining, "");
838
839	#endif /* CONFIG_FREEZE */
840
841	/ Debug /
842
843	extern struct knote vm_find_knote_from_pid(pid_t, struct* klist *);
844
845	#if DEVELOPMENT \|\| DEBUG
846
847	static unsigned int memorystatus_debug_dump_this_bucket = `0`;
848
849	static void
850	memorystatus_debug_dump_bucket_locked (unsigned int bucket_index)
851	{
852	proc_t p = NULL;
853	uint64_t bytes = `0`;
854	int ledger_limit = `0`;
855	unsigned int b = bucket_index;
856	boolean_t traverse_all_buckets = FALSE;
857
858	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
859	traverse_all_buckets = TRUE;
860	b = `0`;
861	} else {
862	traverse_all_buckets = FALSE;
863	b = bucket_index;
864	}
865
866	/*
867	* footprint reported in [pages / MB ]
868	* limits reported as:
869	* L-limit proc's Ledger limit
870	* C-limit proc's Cached limit, should match Ledger
871	* A-limit proc's Active limit
872	* IA-limit proc's Inactive limit
873	* F==Fatal, NF==NonFatal
874	*/
875
876	printf("memorystatus_debug_dump **START(PAGE_SIZE_64=%llu)**\n", PAGE_SIZE_64);
877	printf("bucket [pid] [pages / MB] [state] [EP / RP] dirty deadline [L-limit / C-limit / A-limit / IA-limit] name\n");
878	p = memorystatus_get_first_proc_locked(&b, traverse_all_buckets);
879	while (p) {
880	bytes = get_task_phys_footprint(p->task);
881	task_get_phys_footprint_limit(p->task, &ledger_limit);
882	printf("%2d [%5d] [%5lld /%3lldMB] 0x%-8x [%2d / %2d] 0x%-3x %10lld [%3d / %3d%s / %3d%s / %3d%s] %s\n",
883	b, p->p_pid,
884	(bytes / PAGE_SIZE_64), / task's footprint converted from bytes to pages /
885	(bytes / (`1024ULL` * `1024ULL`)), / task's footprint converted from bytes to MB /
886	p->p_memstat_state, p->p_memstat_effectivepriority, p->p_memstat_requestedpriority, p->p_memstat_dirty, p->p_memstat_idledeadline,
887	ledger_limit,
888	p->p_memstat_memlimit,
889	(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"),
890	p->p_memstat_memlimit_active,
891	(p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL ? "F " : "NF"),
892	p->p_memstat_memlimit_inactive,
893	(p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL ? "F " : "NF"),
894	(*p->p_name ? p->p_name : "unknown"));
895	p = memorystatus_get_next_proc_locked(&b, p, traverse_all_buckets);
896	}
897	printf("memorystatus_debug_dump *END*\n");
898	}
899
900	static int
901	sysctl_memorystatus_debug_dump_bucket SYSCTL_HANDLER_ARGS
902	{
903	#pragma unused(oidp, arg2)
904	int bucket_index = `0`;
905	int error;
906	error = SYSCTL_OUT(req, arg1, sizeof(int));
907	if (error \|\| !req->newptr) {
908	return (error);
909	}
910	error = SYSCTL_IN(req, &bucket_index, sizeof(int));
911	if (error \|\| !req->newptr) {
912	return (error);
913	}
914	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
915	/*
916	* All jetsam buckets will be dumped.
917	*/
918	} else {
919	/*
920	* Only a single bucket will be dumped.
921	*/
922	}
923
924	proc_list_lock();
925	memorystatus_debug_dump_bucket_locked(bucket_index);
926	proc_list_unlock();
927	memorystatus_debug_dump_this_bucket = bucket_index;
928	return (error);
929	}
930
931	/*
932	* Debug aid to look at jetsam buckets and proc jetsam fields.
933	* Use this sysctl to act on a particular jetsam bucket.
934	* Writing the sysctl triggers the dump.
935	* Usage: sysctl kern.memorystatus_debug_dump_this_bucket=<bucket_index>
936	*/
937
938	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_debug_dump_this_bucket, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_debug_dump_this_bucket, `0`, sysctl_memorystatus_debug_dump_bucket, "I", "");
939
940
941	/ Debug aid to aid determination of limit /
942
943	static int
944	sysctl_memorystatus_highwater_enable SYSCTL_HANDLER_ARGS
945	{
946	#pragma unused(oidp, arg2)
947	proc_t p;
948	unsigned int b = `0`;
949	int error, enable = `0`;
950	boolean_t use_active; / use the active limit and active limit attributes /
951	boolean_t is_fatal;
952
953	error = SYSCTL_OUT(req, arg1, sizeof(int));
954	if (error \|\| !req->newptr) {
955	return (error);
956	}
957
958	error = SYSCTL_IN(req, &enable, sizeof(int));
959	if (error \|\| !req->newptr) {
960	return (error);
961	}
962
963	if (!(enable == `0` \|\| enable == `1`)) {
964	return EINVAL;
965	}
966
967	proc_list_lock();
968
969	p = memorystatus_get_first_proc_locked(&b, TRUE);
970	while (p) {
971	use_active = proc_jetsam_state_is_active_locked(p);
972
973	if (enable) {
974
975	if (use_active == TRUE) {
976	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
977	} else {
978	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
979	}
980
981	} else {
982	/*
983	* Disabling limits does not touch the stored variants.
984	* Set the cached limit fields to system_wide defaults.
985	*/
986	p->p_memstat_memlimit = -`1`;
987	p->p_memstat_state \|= P_MEMSTAT_FATAL_MEMLIMIT;
988	is_fatal = TRUE;
989	}
990
991	/*
992	* Enforce the cached limit by writing to the ledger.
993	*/
994	task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > `0`) ? p->p_memstat_memlimit: -`1`, NULL, use_active, is_fatal);
995
996	p = memorystatus_get_next_proc_locked(&b, p, TRUE);
997	}
998
999	memorystatus_highwater_enabled = enable;
1000
1001	proc_list_unlock();
1002
1003	return `0`;
1004
1005	}
1006
1007	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_highwater_enabled, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_highwater_enabled, `0`, sysctl_memorystatus_highwater_enable, "I", "");
1008
1009	#if VM_PRESSURE_EVENTS
1010
1011	/*
1012	* This routine is used for targeted notifications regardless of system memory pressure
1013	* and regardless of whether or not the process has already been notified.
1014	* It bypasses and has no effect on the only-one-notification per soft-limit policy.
1015	*
1016	* "memnote" is the current user.
1017	*/
1018
1019	static int
1020	sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS
1021	{
1022	#pragma unused(arg1, arg2)
1023
1024	int error = `0`, pid = `0`;
1025	struct knote *kn = NULL;
1026	boolean_t found_knote = FALSE;
1027	int fflags = `0`; / filter flags for EVFILT_MEMORYSTATUS /
1028	uint64_t value = `0`;
1029
1030	error = sysctl_handle_quad(oidp, &value, `0`, req);
1031	if (error \|\| !req->newptr)
1032	return (error);
1033
1034	/*
1035	* Find the pid in the low 32 bits of value passed in.
1036	*/
1037	pid = (int)(value & `0xFFFFFFFF`);
1038
1039	/*
1040	* Find notification in the high 32 bits of the value passed in.
1041	*/
1042	fflags = (int)((value >> `32`) & `0xFFFFFFFF`);
1043
1044	/*
1045	* For backwards compatibility, when no notification is
1046	* passed in, default to the NOTE_MEMORYSTATUS_PRESSURE_WARN
1047	*/
1048	if (fflags == `0`) {
1049	fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
1050	// printf("memorystatus_vm_pressure_send: using default notification [0x%x]\n", fflags);
1051	}
1052
1053	/*
1054	* See event.h ... fflags for EVFILT_MEMORYSTATUS
1055	*/
1056	if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)\|\|
1057	(fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) \|\|
1058	(fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) \|\|
1059	(fflags == NOTE_MEMORYSTATUS_LOW_SWAP) \|\|
1060	(fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) \|\|
1061	(fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) \|\|
1062	(((fflags & NOTE_MEMORYSTATUS_MSL_STATUS) != `0` &&
1063	((fflags & ~NOTE_MEMORYSTATUS_MSL_STATUS) == `0`))))) {
1064
1065	printf("memorystatus_vm_pressure_send: notification [0x%x] not supported \n", fflags);
1066	error = `1`;
1067	return (error);
1068	}
1069
1070	/*
1071	* Forcibly send pid a memorystatus notification.
1072	*/
1073
1074	memorystatus_klist_lock();
1075
1076	SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
1077	proc_t knote_proc = knote_get_kq(kn)->kq_p;
1078	pid_t knote_pid = knote_proc->p_pid;
1079
1080	if (knote_pid == pid) {
1081	/*
1082	* Forcibly send this pid a memorystatus notification.
1083	*/
1084	kn->kn_fflags = fflags;
1085	found_knote = TRUE;
1086	}
1087	}
1088
1089	if (found_knote) {
1090	KNOTE(&memorystatus_klist, `0`);
1091	printf("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] sent to process [%d] \n", value, fflags, pid);
1092	error = `0`;
1093	} else {
1094	printf("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] not sent to process [%d] (none registered?)\n", value, fflags, pid);
1095	error = `1`;
1096	}
1097
1098	memorystatus_klist_unlock();
1099
1100	return (error);
1101	}
1102
1103	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_QUAD\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
1104	`0`, `0`, &sysctl_memorystatus_vm_pressure_send, "Q", "");
1105
1106	#endif /* VM_PRESSURE_EVENTS */
1107
1108	SYSCTL_INT(_kern, OID_AUTO, memorystatus_idle_snapshot, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_idle_snapshot, `0`, "");
1109
1110	#if CONFIG_JETSAM
1111	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_available_pages_critical, `0`, "");
1112	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_base, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_base, `0`, "");
1113	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_idle_offset, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_idle_offset, `0`, "");
1114	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_policy_more_free_offset_pages, CTLFLAG_RW, &memorystatus_policy_more_free_offset_pages, `0`, "");
1115
1116	static unsigned int memorystatus_jetsam_panic_debug = `0`;
1117	static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = `0`;
1118
1119	/ Diagnostic code /
1120
1121	enum {
1122	kJetsamDiagnosticModeNone = `0`,
1123	kJetsamDiagnosticModeAll = `1`,
1124	kJetsamDiagnosticModeStopAtFirstActive = `2`,
1125	kJetsamDiagnosticModeCount
1126	} jetsam_diagnostic_mode = kJetsamDiagnosticModeNone;
1127
1128	static int jetsam_diagnostic_suspended_one_active_proc = `0`;
1129
1130	static int
1131	sysctl_jetsam_diagnostic_mode SYSCTL_HANDLER_ARGS
1132	{
1133	#pragma unused(arg1, arg2)
1134
1135	const char *diagnosticStrings[] = {
1136	"jetsam: diagnostic mode: resetting critical level.",
1137	"jetsam: diagnostic mode: will examine all processes",
1138	"jetsam: diagnostic mode: will stop at first active process"
1139	};
1140
1141	int error, val = jetsam_diagnostic_mode;
1142	boolean_t changed = FALSE;
1143
1144	error = sysctl_handle_int(oidp, &val, `0`, req);
1145	if (error \|\| !req->newptr)
1146	return (error);
1147	if ((val < `0`) \|\| (val >= kJetsamDiagnosticModeCount)) {
1148	printf("jetsam: diagnostic mode: invalid value - %d\n", val);
1149	return EINVAL;
1150	}
1151
1152	proc_list_lock();
1153
1154	if ((unsigned int) val != jetsam_diagnostic_mode) {
1155	jetsam_diagnostic_mode = val;
1156
1157	memorystatus_jetsam_policy &= ~kPolicyDiagnoseActive;
1158
1159	switch (jetsam_diagnostic_mode) {
1160	case kJetsamDiagnosticModeNone:
1161	/ Already cleared /
1162	break;
1163	case kJetsamDiagnosticModeAll:
1164	memorystatus_jetsam_policy \|= kPolicyDiagnoseAll;
1165	break;
1166	case kJetsamDiagnosticModeStopAtFirstActive:
1167	memorystatus_jetsam_policy \|= kPolicyDiagnoseFirst;
1168	break;
1169	default:
1170	/ Already validated /
1171	break;
1172	}
1173
1174	memorystatus_update_levels_locked(FALSE);
1175	changed = TRUE;
1176	}
1177
1178	proc_list_unlock();
1179
1180	if (changed) {
1181	printf("%s\n", diagnosticStrings[val]);
1182	}
1183
1184	return (`0`);
1185	}
1186
1187	SYSCTL_PROC(_debug, OID_AUTO, jetsam_diagnostic_mode, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_LOCKED\|CTLFLAG_ANYBODY,
1188	&jetsam_diagnostic_mode, `0`, sysctl_jetsam_diagnostic_mode, "I", "Jetsam Diagnostic Mode");
1189
1190	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jetsam_policy_offset_pages_diagnostic, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_jetsam_policy_offset_pages_diagnostic, `0`, "");
1191
1192	#if VM_PRESSURE_EVENTS
1193
1194	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_pressure, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_available_pages_pressure, `0`, "");
1195
1196	#endif /* VM_PRESSURE_EVENTS */
1197
1198	#endif /* CONFIG_JETSAM */
1199
1200	#if CONFIG_FREEZE
1201
1202	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_jetsam_band, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_jetsam_band, `0`, "");
1203	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_daily_mb_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_daily_mb_max, `0`, "");
1204	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_degraded_mode, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_freeze_degradation, `0`, "");
1205
1206	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_threshold, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_threshold, `0`, "");
1207
1208	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_min, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_pages_min, `0`, "");
1209	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_pages_max, `0`, "");
1210
1211	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_refreeze_eligible_count, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_refreeze_eligible_count, `0`, "");
1212	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_processes_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_frozen_processes_max, `0`, "");
1213
1214	/*
1215	* Max. shared-anonymous memory in MB that can be held by frozen processes in the high jetsam band.
1216	* "0" means no limit.
1217	* Default is 10% of system-wide task limit.
1218	*/
1219
1220	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_frozen_shared_mb_max, `0`, "");
1221	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb, CTLFLAG_RD\|CTLFLAG_LOCKED, &memorystatus_frozen_shared_mb, `0`, "");
1222
1223	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb_per_process_max, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_shared_mb_per_process_max, `0`, "");
1224	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_private_shared_pages_ratio, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_private_shared_pages_ratio, `0`, "");
1225
1226	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_min_processes, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_suspended_threshold, `0`, "");
1227
1228	/*
1229	* max. # of frozen process demotions we will allow in our daily cycle.
1230	*/
1231	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_max_freeze_demotions_daily, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_max_frozen_demotions_daily, `0`, "");
1232	/*
1233	* min # of thaws needed by a process to protect it from getting demoted into the IDLE band.
1234	*/
1235	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_thaw_count_demotion_threshold, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_thaw_count_demotion_threshold, `0`, "");
1236
1237	boolean_t memorystatus_freeze_throttle_enabled = TRUE;
1238	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_throttle_enabled, CTLFLAG_RW\|CTLFLAG_LOCKED, &memorystatus_freeze_throttle_enabled, `0`, "");
1239
1240	#define VM_PAGES_FOR_ALL_PROCS (2)
1241	/*
1242	* Manual trigger of freeze and thaw for dev / debug kernels only.
1243	*/
1244	static int
1245	sysctl_memorystatus_freeze SYSCTL_HANDLER_ARGS
1246	{
1247	#pragma unused(arg1, arg2)
1248	int error, pid = `0`;
1249	proc_t p;
1250	int freezer_error_code = `0`;
1251
1252	if (memorystatus_freeze_enabled == FALSE) {
1253	printf("sysctl_freeze: Freeze is DISABLED\n");
1254	return ENOTSUP;
1255	}
1256
1257	error = sysctl_handle_int(oidp, &pid, `0`, req);
1258	if (error \|\| !req->newptr)
1259	return (error);
1260
1261	if (pid == VM_PAGES_FOR_ALL_PROCS) {
1262	vm_pageout_anonymous_pages();
1263
1264	return `0`;
1265	}
1266
1267	lck_mtx_lock(&freezer_mutex);
1268
1269	p = proc_find(pid);
1270	if (p != NULL) {
1271	uint32_t purgeable, wired, clean, dirty, shared;
1272	uint32_t max_pages = `0`, state = `0`;
1273
1274	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
1275	/*
1276	* Freezer backed by the compressor and swap file(s)
1277	* will hold compressed data.
1278	*
1279	* We don't care about the global freezer budget or the process's (min/max) budget here.
1280	* The freeze sysctl is meant to force-freeze a process.
1281	*
1282	* We also don't update any global or process stats on this path, so that the jetsam/ freeze
1283	* logic remains unaffected. The tasks we're performing here are: freeze the process, set the
1284	* P_MEMSTAT_FROZEN bit, and elevate the process to a higher band (if the freezer is active).
1285	*/
1286	max_pages = memorystatus_freeze_pages_max;
1287
1288	} else {
1289	/*
1290	* We only have the compressor without any swap.
1291	*/
1292	max_pages = UINT32_MAX - `1`;
1293	}
1294
1295	proc_list_lock();
1296	state = p->p_memstat_state;
1297	proc_list_unlock();
1298
1299	/*
1300	* The jetsam path also verifies that the process is a suspended App. We don't care about that here.
1301	* We simply ensure that jetsam is not already working on the process and that the process has not
1302	* explicitly disabled freezing.
1303	*/
1304	if (state & (P_MEMSTAT_TERMINATED \| P_MEMSTAT_LOCKED \| P_MEMSTAT_FREEZE_DISABLED)) {
1305	printf("sysctl_freeze: p_memstat_state check failed, process is%s%s%s\n",
1306	(state & P_MEMSTAT_TERMINATED) ? " terminated" : "",
1307	(state & P_MEMSTAT_LOCKED) ? " locked" : "",
1308	(state & P_MEMSTAT_FREEZE_DISABLED) ? " unfreezable" : "");
1309
1310	proc_rele(p);
1311	lck_mtx_unlock(&freezer_mutex);
1312	return EPERM;
1313	}
1314
1315	error = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, FALSE / eval only /);
1316
1317	if (error) {
1318	char reason[`128`];
1319	if (freezer_error_code == FREEZER_ERROR_EXCESS_SHARED_MEMORY) {
1320	strlcpy(reason, "too much shared memory", `128`);
1321	}
1322
1323	if (freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO) {
1324	strlcpy(reason, "low private-shared pages ratio", `128`);
1325	}
1326
1327	if (freezer_error_code == FREEZER_ERROR_NO_COMPRESSOR_SPACE) {
1328	strlcpy(reason, "no compressor space", `128`);
1329	}
1330
1331	if (freezer_error_code == FREEZER_ERROR_NO_SWAP_SPACE) {
1332	strlcpy(reason, "no swap space", `128`);
1333	}
1334
1335	printf("sysctl_freeze: task_freeze failed: %s\n", reason);
1336
1337	if (error == KERN_NO_SPACE) {
1338	/ Make it easy to distinguish between failures due to low compressor/ swap space and other failures. /
1339	error = ENOSPC;
1340	} else {
1341	error = EIO;
1342	}
1343	} else {
1344	proc_list_lock();
1345	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == `0`) {
1346	p->p_memstat_state \|= P_MEMSTAT_FROZEN;
1347	memorystatus_frozen_count++;
1348	}
1349	p->p_memstat_frozen_count++;
1350
1351
1352	proc_list_unlock();
1353
1354	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
1355	/*
1356	* We elevate only if we are going to swap out the data.
1357	*/
1358	error = memorystatus_update_inactive_jetsam_priority_band(pid, MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE,
1359	memorystatus_freeze_jetsam_band, TRUE);
1360
1361	if (error) {
1362	printf("sysctl_freeze: Elevating frozen process to higher jetsam band failed with %d\n", error);
1363	}
1364	}
1365	}
1366
1367	proc_rele(p);
1368
1369	lck_mtx_unlock(&freezer_mutex);
1370	return error;
1371	} else {
1372	printf("sysctl_freeze: Invalid process\n");
1373	}
1374
1375
1376	lck_mtx_unlock(&freezer_mutex);
1377	return EINVAL;
1378	}
1379
1380	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freeze, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
1381	`0`, `0`, &sysctl_memorystatus_freeze, "I", "");
1382
1383	static int
1384	sysctl_memorystatus_available_pages_thaw SYSCTL_HANDLER_ARGS
1385	{
1386	#pragma unused(arg1, arg2)
1387
1388	int error, pid = `0`;
1389	proc_t p;
1390
1391	if (memorystatus_freeze_enabled == FALSE) {
1392	return ENOTSUP;
1393	}
1394
1395	error = sysctl_handle_int(oidp, &pid, `0`, req);
1396	if (error \|\| !req->newptr)
1397	return (error);
1398
1399	if (pid == VM_PAGES_FOR_ALL_PROCS) {
1400	do_fastwake_warmup_all();
1401	return `0`;
1402	} else {
1403	p = proc_find(pid);
1404	if (p != NULL) {
1405	error = task_thaw(p->task);
1406
1407	if (error) {
1408	error = EIO;
1409	} else {
1410	/*
1411	* task_thaw() succeeded.
1412	*
1413	* We increment memorystatus_frozen_count on the sysctl freeze path.
1414	* And so we need the P_MEMSTAT_FROZEN to decrement the frozen count
1415	* when this process exits.
1416	*
1417	* proc_list_lock();
1418	* p->p_memstat_state &= ~P_MEMSTAT_FROZEN;
1419	* proc_list_unlock();
1420	*/
1421	}
1422	proc_rele(p);
1423	return error;
1424	}
1425	}
1426
1427	return EINVAL;
1428	}
1429
1430	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_thaw, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
1431	`0`, `0`, &sysctl_memorystatus_available_pages_thaw, "I", "");
1432
1433	typedef struct _global_freezable_status{
1434	boolean_t freeze_pages_threshold_crossed;
1435	boolean_t freeze_eligible_procs_available;
1436	boolean_t freeze_scheduled_in_future;
1437	}global_freezable_status_t;
1438
1439	typedef struct _proc_freezable_status{
1440	boolean_t freeze_has_memstat_state;
1441	boolean_t freeze_has_pages_min;
1442	int freeze_has_probability;
1443	boolean_t freeze_attempted;
1444	uint32_t p_memstat_state;
1445	uint32_t p_pages;
1446	int p_freeze_error_code;
1447	int p_pid;
1448	char p_name[MAXCOMLEN + `1`];
1449	}proc_freezable_status_t;
1450
1451	#define MAX_FREEZABLE_PROCESSES 100
1452
1453	static int
1454	memorystatus_freezer_get_status(user_addr_t buffer, size_t buffer_size, int32_t *retval)
1455	{
1456	uint32_t proc_count = `0`, i = `0`;
1457	global_freezable_status_t *list_head;
1458	proc_freezable_status_t *list_entry;
1459	size_t list_size = `0`;
1460	proc_t p;
1461	memstat_bucket_t *bucket;
1462	uint32_t state = `0`, pages = `0`, entry_count = `0`;
1463	boolean_t try_freeze = TRUE;
1464	int error = `0`, probability_of_use = `0`;
1465
1466
1467	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE == FALSE) {
1468	return ENOTSUP;
1469	}
1470
1471	list_size = sizeof(global_freezable_status_t) + (sizeof(proc_freezable_status_t) * MAX_FREEZABLE_PROCESSES);
1472
1473	if (buffer_size < list_size) {
1474	return EINVAL;
1475	}
1476
1477	list_head = (global_freezable_status_t*)kalloc(list_size);
1478	if (list_head == NULL) {
1479	return ENOMEM;
1480	}
1481
1482	memset(list_head, `0`, list_size);
1483
1484	list_size = sizeof(global_freezable_status_t);
1485
1486	proc_list_lock();
1487
1488	uint64_t curr_time = mach_absolute_time();
1489
1490	list_head->freeze_pages_threshold_crossed = (memorystatus_available_pages < memorystatus_freeze_threshold);
1491	list_head->freeze_eligible_procs_available = ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold);
1492	list_head->freeze_scheduled_in_future = (curr_time < memorystatus_freezer_thread_next_run_ts);
1493
1494	list_entry = (proc_freezable_status_t) ((uintptr_t)list_head + sizeof*(global_freezable_status_t));
1495
1496	bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
1497
1498	entry_count = (memorystatus_global_probabilities_size / sizeof(memorystatus_internal_probabilities_t));
1499
1500	p = memorystatus_get_first_proc_locked(&i, FALSE);
1501	proc_count++;
1502
1503	while ((proc_count <= MAX_FREEZABLE_PROCESSES) &&
1504	(p) &&
1505	(list_size < buffer_size)) {
1506
1507	if (isApp(p) == FALSE) {
1508	p = memorystatus_get_next_proc_locked(&i, p, FALSE);
1509	proc_count++;
1510	continue;
1511	}
1512
1513	strlcpy(list_entry->p_name, p->p_name, MAXCOMLEN + `1`);
1514
1515	list_entry->p_pid = p->p_pid;
1516
1517	state = p->p_memstat_state;
1518
1519	if ((state & (P_MEMSTAT_TERMINATED \| P_MEMSTAT_LOCKED \| P_MEMSTAT_FREEZE_DISABLED \| P_MEMSTAT_FREEZE_IGNORE)) \|\|
1520	!(state & P_MEMSTAT_SUSPENDED)) {
1521
1522	try_freeze = list_entry->freeze_has_memstat_state = FALSE;
1523	} else {
1524	try_freeze = list_entry->freeze_has_memstat_state = TRUE;
1525	}
1526
1527	list_entry->p_memstat_state = state;
1528
1529	memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
1530	if (pages < memorystatus_freeze_pages_min) {
1531	try_freeze = list_entry->freeze_has_pages_min = FALSE;
1532	} else {
1533	list_entry->freeze_has_pages_min = TRUE;
1534	if (try_freeze != FALSE) {
1535	try_freeze = TRUE;
1536	}
1537	}
1538
1539	list_entry->p_pages = pages;
1540
1541	if (entry_count) {
1542	uint32_t j = `0`;
1543	for (j = `0`; j < entry_count; j++ ) {
1544	if (strncmp(memorystatus_global_probabilities_table[j].proc_name,
1545	p->p_name,
1546	MAXCOMLEN + `1`) == `0`) {
1547
1548	probability_of_use = memorystatus_global_probabilities_table[j].use_probability;
1549	break;
1550	}
1551	}
1552
1553	list_entry->freeze_has_probability = probability_of_use;
1554
1555	if (probability_of_use && try_freeze != FALSE) {
1556	try_freeze = TRUE;
1557	} else {
1558	try_freeze = FALSE;
1559	}
1560	} else {
1561	if (try_freeze != FALSE) {
1562	try_freeze = TRUE;
1563	}
1564	list_entry->freeze_has_probability = -`1`;
1565	}
1566
1567	if (try_freeze) {
1568
1569	uint32_t purgeable, wired, clean, dirty, shared;
1570	uint32_t max_pages = `0`;
1571	int freezer_error_code = `0`;
1572
1573	error = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, TRUE / eval only /);
1574
1575	if (error) {
1576	list_entry->p_freeze_error_code = freezer_error_code;
1577	}
1578
1579	list_entry->freeze_attempted = TRUE;
1580	}
1581
1582	list_entry++;
1583
1584	list_size += sizeof(proc_freezable_status_t);
1585
1586	p = memorystatus_get_next_proc_locked(&i, p, FALSE);
1587	proc_count++;
1588	}
1589
1590	proc_list_unlock();
1591
1592	buffer_size = list_size;
1593
1594	error = copyout(list_head, buffer, buffer_size);
1595	if (error == `0`) {
1596	*retval = buffer_size;
1597	} else {
1598	*retval = `0`;
1599	}
1600
1601	list_size = sizeof(global_freezable_status_t) + (sizeof(proc_freezable_status_t) * MAX_FREEZABLE_PROCESSES);
1602	kfree(list_head, list_size);
1603
1604	MEMORYSTATUS_DEBUG(`1`, "memorystatus_freezer_get_status: returning %d (%lu - size)\n", error, (unsigned long)*list_size);
1605
1606	return error;
1607	}
1608
1609	static int
1610	memorystatus_freezer_control(int32_t flags, user_addr_t buffer, size_t buffer_size, int32_t *retval)
1611	{
1612	int err = ENOTSUP;
1613
1614	if (flags == FREEZER_CONTROL_GET_STATUS) {
1615	err = memorystatus_freezer_get_status(buffer, buffer_size, retval);
1616	}
1617
1618	return err;
1619	}
1620
1621	#endif /* CONFIG_FREEZE */
1622
1623	#endif /* DEVELOPMENT \|\| DEBUG */
1624
1625	extern kern_return_t kernel_thread_start_priority(thread_continue_t continuation,
1626	void *parameter,
1627	integer_t priority,
1628	thread_t *new_thread);
1629
1630	#if DEVELOPMENT \|\| DEBUG
1631
1632	static int
1633	sysctl_memorystatus_disconnect_page_mappings SYSCTL_HANDLER_ARGS
1634	{
1635	#pragma unused(arg1, arg2)
1636	int error = `0`, pid = `0`;
1637	proc_t p;
1638
1639	error = sysctl_handle_int(oidp, &pid, `0`, req);
1640	if (error \|\| !req->newptr)
1641	return (error);
1642
1643	lck_mtx_lock(&disconnect_page_mappings_mutex);
1644
1645	if (pid == -`1`) {
1646	vm_pageout_disconnect_all_pages();
1647	} else {
1648	p = proc_find(pid);
1649
1650	if (p != NULL) {
1651	error = task_disconnect_page_mappings(p->task);
1652
1653	proc_rele(p);
1654
1655	if (error)
1656	error = EIO;
1657	} else
1658	error = EINVAL;
1659	}
1660	lck_mtx_unlock(&disconnect_page_mappings_mutex);
1661
1662	return error;
1663	}
1664
1665	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_disconnect_page_mappings, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
1666	`0`, `0`, &sysctl_memorystatus_disconnect_page_mappings, "I", "");
1667
1668	#endif /* DEVELOPMENT \|\| DEBUG */
1669
1670
1671	/*
1672	* Picks the sorting routine for a given jetsam priority band.
1673	*
1674	* Input:
1675	* bucket_index - jetsam priority band to be sorted.
1676	* sort_order - JETSAM_SORT_xxx from kern_memorystatus.h
1677	* Currently sort_order is only meaningful when handling
1678	* coalitions.
1679	*
1680	* Return:
1681	* 0 on success
1682	* non-0 on failure
1683	*/
1684	static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order)
1685	{
1686	int coal_sort_order;
1687
1688	/*
1689	* Verify the jetsam priority
1690	*/
1691	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
1692	return(EINVAL);
1693	}
1694
1695	#if DEVELOPMENT \|\| DEBUG
1696	if (sort_order == JETSAM_SORT_DEFAULT) {
1697	coal_sort_order = COALITION_SORT_DEFAULT;
1698	} else {
1699	coal_sort_order = sort_order; / only used for testing scenarios /
1700	}
1701	#else
1702	/ Verify default /
1703	if (sort_order == JETSAM_SORT_DEFAULT) {
1704	coal_sort_order = COALITION_SORT_DEFAULT;
1705	} else {
1706	return(EINVAL);
1707	}
1708	#endif
1709
1710	proc_list_lock();
1711
1712	if (memstat_bucket[bucket_index].count == `0`) {
1713	proc_list_unlock();
1714	return (`0`);
1715	}
1716
1717	switch (bucket_index) {
1718	case JETSAM_PRIORITY_FOREGROUND:
1719	if (memorystatus_sort_by_largest_coalition_locked(bucket_index, coal_sort_order) == `0`) {
1720	/*
1721	* Fall back to per process sorting when zero coalitions are found.
1722	*/
1723	memorystatus_sort_by_largest_process_locked(bucket_index);
1724	}
1725	break;
1726	default:
1727	memorystatus_sort_by_largest_process_locked(bucket_index);
1728	break;
1729	}
1730	proc_list_unlock();
1731
1732	return(`0`);
1733	}
1734
1735	/*
1736	* Sort processes by size for a single jetsam bucket.
1737	*/
1738
1739	static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index)
1740	{
1741	proc_t p = NULL, insert_after_proc = NULL, max_proc = NULL;
1742	proc_t next_p = NULL, prev_max_proc = NULL;
1743	uint32_t pages = `0`, max_pages = `0`;
1744	memstat_bucket_t *current_bucket;
1745
1746	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
1747	return;
1748	}
1749
1750	current_bucket = &memstat_bucket[bucket_index];
1751
1752	p = TAILQ_FIRST(&current_bucket->list);
1753
1754	while (p) {
1755	memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
1756	max_pages = pages;
1757	max_proc = p;
1758	prev_max_proc = p;
1759
1760	while ((next_p = TAILQ_NEXT(p, p_memstat_list)) != NULL) {
1761	/ traversing list until we find next largest process /
1762	p=next_p;
1763	memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
1764	if (pages > max_pages) {
1765	max_pages = pages;
1766	max_proc = p;
1767	}
1768	}
1769
1770	if (prev_max_proc != max_proc) {
1771	/ found a larger process, place it in the list /
1772	TAILQ_REMOVE(&current_bucket->list, max_proc, p_memstat_list);
1773	if (insert_after_proc == NULL) {
1774	TAILQ_INSERT_HEAD(&current_bucket->list, max_proc, p_memstat_list);
1775	} else {
1776	TAILQ_INSERT_AFTER(&current_bucket->list, insert_after_proc, max_proc, p_memstat_list);
1777	}
1778	prev_max_proc = max_proc;
1779	}
1780
1781	insert_after_proc = max_proc;
1782
1783	p = TAILQ_NEXT(max_proc, p_memstat_list);
1784	}
1785	}
1786
1787	static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search) {
1788	memstat_bucket_t *current_bucket;
1789	proc_t next_p;
1790
1791	if ((*bucket_index) >= MEMSTAT_BUCKET_COUNT) {
1792	return NULL;
1793	}
1794
1795	current_bucket = &memstat_bucket[*bucket_index];
1796	next_p = TAILQ_FIRST(&current_bucket->list);
1797	if (!next_p && search) {
1798	while (!next_p && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1799	current_bucket = &memstat_bucket[*bucket_index];
1800	next_p = TAILQ_FIRST(&current_bucket->list);
1801	}
1802	}
1803
1804	return next_p;
1805	}
1806
1807	static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search) {
1808	memstat_bucket_t *current_bucket;
1809	proc_t next_p;
1810
1811	if (!p \|\| ((*bucket_index) >= MEMSTAT_BUCKET_COUNT)) {
1812	return NULL;
1813	}
1814
1815	next_p = TAILQ_NEXT(p, p_memstat_list);
1816	while (!next_p && search && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1817	current_bucket = &memstat_bucket[*bucket_index];
1818	next_p = TAILQ_FIRST(&current_bucket->list);
1819	}
1820
1821	return next_p;
1822	}
1823
1824	/*
1825	* Structure to hold state for a jetsam thread.
1826	* Typically there should be a single jetsam thread
1827	* unless parallel jetsam is enabled.
1828	*/
1829	struct jetsam_thread_state {
1830	boolean_t inited; / if the thread is initialized /
1831	int memorystatus_wakeup; / wake channel /
1832	int index; / jetsam thread index /
1833	thread_t thread; / jetsam thread pointer /
1834	} *jetsam_threads;
1835
1836	/ Maximum number of jetsam threads allowed /
1837	#define JETSAM_THREADS_LIMIT 3
1838
1839	/ Number of active jetsam threads /
1840	_Atomic int active_jetsam_threads = `1`;
1841
1842	/ Number of maximum jetsam threads configured /
1843	int max_jetsam_threads = JETSAM_THREADS_LIMIT;
1844
1845	/*
1846	* Global switch for enabling fast jetsam. Fast jetsam is
1847	* hooked up via the system_override() system call. It has the
1848	* following effects:
1849	* - Raise the jetsam threshold ("clear-the-deck")
1850	* - Enabled parallel jetsam on eligible devices
1851	*/
1852	int fast_jetsam_enabled = `0`;
1853
1854	/ Routine to find the jetsam state structure for the current jetsam thread /
1855	static inline struct jetsam_thread_state *
1856	jetsam_current_thread(void)
1857	{
1858	for (int thr_id = `0`; thr_id < max_jetsam_threads; thr_id++) {
1859	if (jetsam_threads[thr_id].thread == current_thread())
1860	return &(jetsam_threads[thr_id]);
1861	}
1862	panic("jetsam_current_thread() is being called from a non-jetsam thread\n");
1863	/ Contol should not reach here /
1864	return NULL;
1865	}
1866
1867
1868	__private_extern__ void
1869	memorystatus_init(void)
1870	{
1871	kern_return_t result;
1872	int i;
1873
1874	#if CONFIG_FREEZE
1875	memorystatus_freeze_jetsam_band = JETSAM_PRIORITY_UI_SUPPORT;
1876	memorystatus_frozen_processes_max = FREEZE_PROCESSES_MAX;
1877	memorystatus_frozen_shared_mb_max = ((MAX_FROZEN_SHARED_MB_PERCENT * max_task_footprint_mb) / `100`); / 10% of the system wide task limit /
1878	memorystatus_freeze_shared_mb_per_process_max = (memorystatus_frozen_shared_mb_max / `4`);
1879	memorystatus_freeze_pages_min = FREEZE_PAGES_MIN;
1880	memorystatus_freeze_pages_max = FREEZE_PAGES_MAX;
1881	memorystatus_max_frozen_demotions_daily = MAX_FROZEN_PROCESS_DEMOTIONS;
1882	memorystatus_thaw_count_demotion_threshold = MIN_THAW_DEMOTION_THRESHOLD;
1883	#endif
1884
1885	#if DEVELOPMENT \|\| DEBUG
1886	disconnect_page_mappings_lck_grp_attr = lck_grp_attr_alloc_init();
1887	disconnect_page_mappings_lck_grp = lck_grp_alloc_init("disconnect_page_mappings", disconnect_page_mappings_lck_grp_attr);
1888
1889	lck_mtx_init(&disconnect_page_mappings_mutex, disconnect_page_mappings_lck_grp, NULL);
1890
1891	if (kill_on_no_paging_space == TRUE) {
1892	max_kill_priority = JETSAM_PRIORITY_MAX;
1893	}
1894	#endif
1895
1896
1897	/ Init buckets /
1898	for (i = `0`; i < MEMSTAT_BUCKET_COUNT; i++) {
1899	TAILQ_INIT(&memstat_bucket[i].list);
1900	memstat_bucket[i].count = `0`;
1901	}
1902	memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL);
1903
1904	#if CONFIG_JETSAM
1905	nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time);
1906	nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time);
1907
1908	/ Apply overrides /
1909	PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage));
1910	if (delta_percentage == `0`) {
1911	delta_percentage = `5`;
1912	}
1913	assert(delta_percentage < `100`);
1914	PE_get_default("kern.jetsam_critical_threshold", &critical_threshold_percentage, sizeof(critical_threshold_percentage));
1915	assert(critical_threshold_percentage < `100`);
1916	PE_get_default("kern.jetsam_idle_offset", &idle_offset_percentage, sizeof(idle_offset_percentage));
1917	assert(idle_offset_percentage < `100`);
1918	PE_get_default("kern.jetsam_pressure_threshold", &pressure_threshold_percentage, sizeof(pressure_threshold_percentage));
1919	assert(pressure_threshold_percentage < `100`);
1920	PE_get_default("kern.jetsam_freeze_threshold", &freeze_threshold_percentage, sizeof(freeze_threshold_percentage));
1921	assert(freeze_threshold_percentage < `100`);
1922
1923	if (!PE_parse_boot_argn("jetsam_aging_policy", &jetsam_aging_policy,
1924	sizeof (jetsam_aging_policy))) {
1925
1926	if (!PE_get_default("kern.jetsam_aging_policy", &jetsam_aging_policy,
1927	sizeof(jetsam_aging_policy))) {
1928
1929	jetsam_aging_policy = kJetsamAgingPolicyLegacy;
1930	}
1931	}
1932
1933	if (jetsam_aging_policy > kJetsamAgingPolicyMax) {
1934	jetsam_aging_policy = kJetsamAgingPolicyLegacy;
1935	}
1936
1937	switch (jetsam_aging_policy) {
1938
1939	case kJetsamAgingPolicyNone:
1940	system_procs_aging_band = JETSAM_PRIORITY_IDLE;
1941	applications_aging_band = JETSAM_PRIORITY_IDLE;
1942	break;
1943
1944	case kJetsamAgingPolicyLegacy:
1945	/*
1946	* Legacy behavior where some daemons get a 10s protection once
1947	* AND only before the first clean->dirty->clean transition before
1948	* going into IDLE band.
1949	*/
1950	system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1951	applications_aging_band = JETSAM_PRIORITY_IDLE;
1952	break;
1953
1954	case kJetsamAgingPolicySysProcsReclaimedFirst:
1955	system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1956	applications_aging_band = JETSAM_PRIORITY_AGING_BAND2;
1957	break;
1958
1959	case kJetsamAgingPolicyAppsReclaimedFirst:
1960	system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND2;
1961	applications_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1962	break;
1963
1964	default:
1965	break;
1966	}
1967
1968	/*
1969	* The aging bands cannot overlap with the JETSAM_PRIORITY_ELEVATED_INACTIVE
1970	* band and must be below it in priority. This is so that we don't have to make
1971	* our 'aging' code worry about a mix of processes, some of which need to age
1972	* and some others that need to stay elevated in the jetsam bands.
1973	*/
1974	assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > system_procs_aging_band);
1975	assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > applications_aging_band);
1976
1977	/ Take snapshots for idle-exit kills by default? First check the boot-arg... /
1978	if (!PE_parse_boot_argn("jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof (memorystatus_idle_snapshot))) {
1979	/ ...no boot-arg, so check the device tree /
1980	PE_get_default("kern.jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot));
1981	}
1982
1983	memorystatus_delta = delta_percentage * atop_64(max_mem) / `100`;
1984	memorystatus_available_pages_critical_idle_offset = idle_offset_percentage * atop_64(max_mem) / `100`;
1985	memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta;
1986	memorystatus_policy_more_free_offset_pages = (policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta;
1987
1988	/ Jetsam Loop Detection /
1989	if (max_mem <= (`512` * `1024` * `1024`)) {
1990	/ 512 MB devices /
1991	memorystatus_jld_eval_period_msecs = `8000`; / 8000 msecs == 8 second window /
1992	} else {
1993	/ 1GB and larger devices /
1994	memorystatus_jld_eval_period_msecs = `6000`; / 6000 msecs == 6 second window /
1995	}
1996
1997	memorystatus_jld_enabled = TRUE;
1998
1999	/ No contention at this point /
2000	memorystatus_update_levels_locked(FALSE);
2001
2002	#endif /* CONFIG_JETSAM */
2003
2004	memorystatus_jetsam_snapshot_max = maxproc;
2005
2006	memorystatus_jetsam_snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
2007	(sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_max);
2008
2009	memorystatus_jetsam_snapshot =
2010	(memorystatus_jetsam_snapshot_t*)kalloc(memorystatus_jetsam_snapshot_size);
2011	if (!memorystatus_jetsam_snapshot) {
2012	panic("Could not allocate memorystatus_jetsam_snapshot");
2013	}
2014
2015	memorystatus_jetsam_snapshot_copy =
2016	(memorystatus_jetsam_snapshot_t*)kalloc(memorystatus_jetsam_snapshot_size);
2017	if (!memorystatus_jetsam_snapshot_copy) {
2018	panic("Could not allocate memorystatus_jetsam_snapshot_copy");
2019	}
2020
2021	nanoseconds_to_absolutetime((uint64_t)JETSAM_SNAPSHOT_TIMEOUT_SECS * NSEC_PER_SEC, &memorystatus_jetsam_snapshot_timeout);
2022
2023	memset(&memorystatus_at_boot_snapshot, `0`, sizeof(memorystatus_jetsam_snapshot_t));
2024
2025	#if CONFIG_FREEZE
2026	memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta;
2027	#endif
2028
2029	/ Check the boot-arg to see if fast jetsam is allowed /
2030	if (!PE_parse_boot_argn("fast_jetsam_enabled", &fast_jetsam_enabled, sizeof (fast_jetsam_enabled))) {
2031	fast_jetsam_enabled = `0`;
2032	}
2033
2034	/ Check the boot-arg to configure the maximum number of jetsam threads /
2035	if (!PE_parse_boot_argn("max_jetsam_threads", &max_jetsam_threads, sizeof (max_jetsam_threads))) {
2036	max_jetsam_threads = JETSAM_THREADS_LIMIT;
2037	}
2038
2039	/ Restrict the maximum number of jetsam threads to JETSAM_THREADS_LIMIT /
2040	if (max_jetsam_threads > JETSAM_THREADS_LIMIT) {
2041	max_jetsam_threads = JETSAM_THREADS_LIMIT;
2042	}
2043
2044	/ For low CPU systems disable fast jetsam mechanism /
2045	if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
2046	max_jetsam_threads = `1`;
2047	fast_jetsam_enabled = `0`;
2048	}
2049
2050	/ Initialize the jetsam_threads state array /
2051	jetsam_threads = kalloc(sizeof(struct jetsam_thread_state) * max_jetsam_threads);
2052
2053	/ Initialize all the jetsam threads /
2054	for (i = `0`; i < max_jetsam_threads; i++) {
2055
2056	result = kernel_thread_start_priority(memorystatus_thread, NULL, `95` / MAXPRI_KERNEL /, &jetsam_threads[i].thread);
2057	if (result == KERN_SUCCESS) {
2058	jetsam_threads[i].inited = FALSE;
2059	jetsam_threads[i].index = i;
2060	thread_deallocate(jetsam_threads[i].thread);
2061	} else {
2062	panic("Could not create memorystatus_thread %d", i);
2063	}
2064	}
2065	}
2066
2067	/ Centralised for the purposes of allowing panic-on-jetsam /
2068	extern void
2069	vm_run_compactor(void);
2070
2071	/*
2072	* The jetsam no frills kill call
2073	* Return: 0 on success
2074	* error code on failure (EINVAL...)
2075	*/
2076	static int
2077	jetsam_do_kill(proc_t p, int jetsam_flags, os_reason_t jetsam_reason) {
2078	int error = `0`;
2079	error = exit_with_reason(p, W_EXITCODE(`0`, SIGKILL), (int *)NULL, FALSE, FALSE, jetsam_flags, jetsam_reason);
2080	return(error);
2081	}
2082
2083	/*
2084	* Wrapper for processes exiting with memorystatus details
2085	*/
2086	static boolean_t
2087	memorystatus_do_kill(proc_t p, uint32_t cause, os_reason_t jetsam_reason) {
2088
2089	int error = `0`;
2090	__unused pid_t victim_pid = p->p_pid;
2091
2092	KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) \| DBG_FUNC_START,
2093	victim_pid, cause, vm_page_free_count, `0`, `0`);
2094
2095	DTRACE_MEMORYSTATUS3(memorystatus_do_kill, proc_t, p, os_reason_t, jetsam_reason, uint32_t, cause);
2096	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
2097	if (memorystatus_jetsam_panic_debug & (`1` << cause)) {
2098	panic("memorystatus_do_kill(): jetsam debug panic (cause: %d)", cause);
2099	}
2100	#else
2101	#pragma unused(cause)
2102	#endif
2103
2104	if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
2105	printf("memorystatus: killing process %d [%s] in high band %s (%d) - memorystatus_available_pages: %llu\n", p->p_pid,
2106	(*p->p_name ? p->p_name : "unknown"),
2107	memorystatus_priority_band_name(p->p_memstat_effectivepriority), p->p_memstat_effectivepriority,
2108	(uint64_t)memorystatus_available_pages);
2109	}
2110
2111	/*
2112	* The jetsam_reason (os_reason_t) has enough information about the kill cause.
2113	* We don't really need jetsam_flags anymore, so it's okay that not all possible kill causes have been mapped.
2114	*/
2115	int jetsam_flags = P_LTERM_JETSAM;
2116	switch (cause) {
2117	case kMemorystatusKilledHiwat: jetsam_flags \|= P_JETSAM_HIWAT; break;
2118	case kMemorystatusKilledVnodes: jetsam_flags \|= P_JETSAM_VNODE; break;
2119	case kMemorystatusKilledVMPageShortage: jetsam_flags \|= P_JETSAM_VMPAGESHORTAGE; break;
2120	case kMemorystatusKilledVMCompressorThrashing:
2121	case kMemorystatusKilledVMCompressorSpaceShortage: jetsam_flags \|= P_JETSAM_VMTHRASHING; break;
2122	case kMemorystatusKilledFCThrashing: jetsam_flags \|= P_JETSAM_FCTHRASHING; break;
2123	case kMemorystatusKilledPerProcessLimit: jetsam_flags \|= P_JETSAM_PID; break;
2124	case kMemorystatusKilledIdleExit: jetsam_flags \|= P_JETSAM_IDLEEXIT; break;
2125	}
2126	error = jetsam_do_kill(p, jetsam_flags, jetsam_reason);
2127
2128	KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) \| DBG_FUNC_END,
2129	victim_pid, cause, vm_page_free_count, error, `0`);
2130
2131	vm_run_compactor();
2132
2133	return (error == `0`);
2134	}
2135
2136	/*
2137	* Node manipulation
2138	*/
2139
2140	static void
2141	memorystatus_check_levels_locked(void) {
2142	#if CONFIG_JETSAM
2143	/ Update levels /
2144	memorystatus_update_levels_locked(TRUE);
2145	#else /* CONFIG_JETSAM */
2146	/*
2147	* Nothing to do here currently since we update
2148	* memorystatus_available_pages in vm_pressure_response.
2149	*/
2150	#endif /* CONFIG_JETSAM */
2151	}
2152
2153	/*
2154	* Pin a process to a particular jetsam band when it is in the background i.e. not doing active work.
2155	* For an application: that means no longer in the FG band
2156	* For a daemon: that means no longer in its 'requested' jetsam priority band
2157	*/
2158
2159	int
2160	memorystatus_update_inactive_jetsam_priority_band(pid_t pid, uint32_t op_flags, int jetsam_prio, boolean_t effective_now)
2161	{
2162	int error = `0`;
2163	boolean_t enable = FALSE;
2164	proc_t p = NULL;
2165
2166	if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE) {
2167	enable = TRUE;
2168	} else if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE) {
2169	enable = FALSE;
2170	} else {
2171	return EINVAL;
2172	}
2173
2174	p = proc_find(pid);
2175	if (p != NULL) {
2176
2177	if ((enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) \|\|
2178	(!enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == `0`))) {
2179	/*
2180	* No change in state.
2181	*/
2182
2183	} else {
2184
2185	proc_list_lock();
2186
2187	if (enable) {
2188	p->p_memstat_state \|= P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
2189	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2190
2191	if (effective_now) {
2192	if (p->p_memstat_effectivepriority < jetsam_prio) {
2193	if(memorystatus_highwater_enabled) {
2194	/*
2195	* Process is about to transition from
2196	* inactive --> active
2197	* assign active state
2198	*/
2199	boolean_t is_fatal;
2200	boolean_t use_active = TRUE;
2201	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2202	task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > `0`) ? p->p_memstat_memlimit : -`1`, NULL, use_active, is_fatal);
2203	}
2204	memorystatus_update_priority_locked(p, jetsam_prio, FALSE, FALSE);
2205	}
2206	} else {
2207	if (isProcessInAgingBands(p)) {
2208	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
2209	}
2210	}
2211	} else {
2212
2213	p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
2214	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2215
2216	if (effective_now) {
2217	if (p->p_memstat_effectivepriority == jetsam_prio) {
2218	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
2219	}
2220	} else {
2221	if (isProcessInAgingBands(p)) {
2222	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
2223	}
2224	}
2225	}
2226
2227	proc_list_unlock();
2228	}
2229	proc_rele(p);
2230	error = `0`;
2231
2232	} else {
2233	error = ESRCH;
2234	}
2235
2236	return error;
2237	}
2238
2239	static void
2240	memorystatus_perform_idle_demotion(__unused void spare1, __unused void* *spare2)
2241	{
2242	proc_t p;
2243	uint64_t current_time = `0`, idle_delay_time = `0`;
2244	int demote_prio_band = `0`;
2245	memstat_bucket_t *demotion_bucket;
2246
2247	MEMORYSTATUS_DEBUG(`1`, "memorystatus_perform_idle_demotion()\n");
2248
2249	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) \| DBG_FUNC_START, `0`, `0`, `0`, `0`, `0`);
2250
2251	current_time = mach_absolute_time();
2252
2253	proc_list_lock();
2254
2255	demote_prio_band = JETSAM_PRIORITY_IDLE + `1`;
2256
2257	for (; demote_prio_band < JETSAM_PRIORITY_MAX; demote_prio_band++) {
2258
2259	if (demote_prio_band != system_procs_aging_band && demote_prio_band != applications_aging_band)
2260	continue;
2261
2262	demotion_bucket = &memstat_bucket[demote_prio_band];
2263	p = TAILQ_FIRST(&demotion_bucket->list);
2264
2265	while (p) {
2266	MEMORYSTATUS_DEBUG(`1`, "memorystatus_perform_idle_demotion() found %d\n", p->p_pid);
2267
2268	assert(p->p_memstat_idledeadline);
2269
2270	assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
2271
2272	if (current_time >= p->p_memstat_idledeadline) {
2273
2274	if ((isSysProc(p) &&
2275	((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED\|P_DIRTY_IS_DIRTY)) != P_DIRTY_IDLE_EXIT_ENABLED)) \|\| / system proc marked dirty/
2276	task_has_assertions((struct task )(p->task))) { /* has outstanding assertions which might indicate outstanding work too /
2277	idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
2278
2279	p->p_memstat_idledeadline += idle_delay_time;
2280	p = TAILQ_NEXT(p, p_memstat_list);
2281
2282	} else {
2283
2284	proc_t next_proc = NULL;
2285
2286	next_proc = TAILQ_NEXT(p, p_memstat_list);
2287	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2288
2289	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, false, true);
2290
2291	p = next_proc;
2292	continue;
2293
2294	}
2295	} else {
2296	// No further candidates
2297	break;
2298	}
2299	}
2300
2301	}
2302
2303	memorystatus_reschedule_idle_demotion_locked();
2304
2305	proc_list_unlock();
2306
2307	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) \| DBG_FUNC_END, `0`, `0`, `0`, `0`, `0`);
2308	}
2309
2310	static void
2311	memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state)
2312	{
2313	boolean_t present_in_sysprocs_aging_bucket = FALSE;
2314	boolean_t present_in_apps_aging_bucket = FALSE;
2315	uint64_t idle_delay_time = `0`;
2316
2317	if (jetsam_aging_policy == kJetsamAgingPolicyNone) {
2318	return;
2319	}
2320
2321	if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
2322	/*
2323	* This process isn't going to be making the trip to the lower bands.
2324	*/
2325	return;
2326	}
2327
2328	if (isProcessInAgingBands(p)){
2329
2330	if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
2331	assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) != P_DIRTY_AGING_IN_PROGRESS);
2332	}
2333
2334	if (isSysProc(p) && system_procs_aging_band) {
2335	present_in_sysprocs_aging_bucket = TRUE;
2336
2337	} else if (isApp(p) && applications_aging_band) {
2338	present_in_apps_aging_bucket = TRUE;
2339	}
2340	}
2341
2342	assert(!present_in_sysprocs_aging_bucket);
2343	assert(!present_in_apps_aging_bucket);
2344
2345	MEMORYSTATUS_DEBUG(`1`, "memorystatus_schedule_idle_demotion_locked: scheduling demotion to idle band for pid %d (dirty:0x%x, set_state %d, demotions %d).\n",
2346	p->p_pid, p->p_memstat_dirty, set_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
2347
2348	if(isSysProc(p)) {
2349	assert((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED);
2350	}
2351
2352	idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
2353
2354	if (set_state) {
2355	p->p_memstat_dirty \|= P_DIRTY_AGING_IN_PROGRESS;
2356	p->p_memstat_idledeadline = mach_absolute_time() + idle_delay_time;
2357	}
2358
2359	assert(p->p_memstat_idledeadline);
2360
2361	if (isSysProc(p) && present_in_sysprocs_aging_bucket == FALSE) {
2362	memorystatus_scheduled_idle_demotions_sysprocs++;
2363
2364	} else if (isApp(p) && present_in_apps_aging_bucket == FALSE) {
2365	memorystatus_scheduled_idle_demotions_apps++;
2366	}
2367	}
2368
2369	static void
2370	memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clear_state)
2371	{
2372	boolean_t present_in_sysprocs_aging_bucket = FALSE;
2373	boolean_t present_in_apps_aging_bucket = FALSE;
2374
2375	if (!system_procs_aging_band && !applications_aging_band) {
2376	return;
2377	}
2378
2379	if ((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == `0`) {
2380	return;
2381	}
2382
2383	if (isProcessInAgingBands(p)) {
2384
2385	if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
2386	assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == P_DIRTY_AGING_IN_PROGRESS);
2387	}
2388
2389	if (isSysProc(p) && system_procs_aging_band) {
2390	assert(p->p_memstat_effectivepriority == system_procs_aging_band);
2391	assert(p->p_memstat_idledeadline);
2392	present_in_sysprocs_aging_bucket = TRUE;
2393
2394	} else if (isApp(p) && applications_aging_band) {
2395	assert(p->p_memstat_effectivepriority == applications_aging_band);
2396	assert(p->p_memstat_idledeadline);
2397	present_in_apps_aging_bucket = TRUE;
2398	}
2399	}
2400
2401	MEMORYSTATUS_DEBUG(`1`, "memorystatus_invalidate_idle_demotion(): invalidating demotion to idle band for pid %d (clear_state %d, demotions %d).\n",
2402	p->p_pid, clear_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
2403
2404
2405	if (clear_state) {
2406	p->p_memstat_idledeadline = `0`;
2407	p->p_memstat_dirty &= ~P_DIRTY_AGING_IN_PROGRESS;
2408	}
2409
2410	if (isSysProc(p) &&present_in_sysprocs_aging_bucket == TRUE) {
2411	memorystatus_scheduled_idle_demotions_sysprocs--;
2412	assert(memorystatus_scheduled_idle_demotions_sysprocs >= `0`);
2413
2414	} else if (isApp(p) && present_in_apps_aging_bucket == TRUE) {
2415	memorystatus_scheduled_idle_demotions_apps--;
2416	assert(memorystatus_scheduled_idle_demotions_apps >= `0`);
2417	}
2418
2419	assert((memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps) >= `0`);
2420	}
2421
2422	static void
2423	memorystatus_reschedule_idle_demotion_locked(void) {
2424	if (`0` == (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps)) {
2425	if (memstat_idle_demotion_deadline) {
2426	/ Transitioned 1->0, so cancel next call /
2427	thread_call_cancel(memorystatus_idle_demotion_call);
2428	memstat_idle_demotion_deadline = `0`;
2429	}
2430	} else {
2431	memstat_bucket_t *demotion_bucket;
2432	proc_t p = NULL, p1 = NULL, p2 = NULL;
2433
2434	if (system_procs_aging_band) {
2435
2436	demotion_bucket = &memstat_bucket[system_procs_aging_band];
2437	p1 = TAILQ_FIRST(&demotion_bucket->list);
2438
2439	p = p1;
2440	}
2441
2442	if (applications_aging_band) {
2443
2444	demotion_bucket = &memstat_bucket[applications_aging_band];
2445	p2 = TAILQ_FIRST(&demotion_bucket->list);
2446
2447	if (p1 && p2) {
2448	p = (p1->p_memstat_idledeadline > p2->p_memstat_idledeadline) ? p2 : p1;
2449	} else {
2450	p = (p1 == NULL) ? p2 : p1;
2451	}
2452
2453	}
2454
2455	assert(p);
2456
2457	if (p != NULL) {
2458	assert(p && p->p_memstat_idledeadline);
2459	if (memstat_idle_demotion_deadline != p->p_memstat_idledeadline){
2460	thread_call_enter_delayed(memorystatus_idle_demotion_call, p->p_memstat_idledeadline);
2461	memstat_idle_demotion_deadline = p->p_memstat_idledeadline;
2462	}
2463	}
2464	}
2465	}
2466
2467	/*
2468	* List manipulation
2469	*/
2470
2471	int
2472	memorystatus_add(proc_t p, boolean_t locked)
2473	{
2474	memstat_bucket_t *bucket;
2475
2476	MEMORYSTATUS_DEBUG(`1`, "memorystatus_list_add(): adding pid %d with priority %d.\n", p->p_pid, p->p_memstat_effectivepriority);
2477
2478	if (!locked) {
2479	proc_list_lock();
2480	}
2481
2482	DTRACE_MEMORYSTATUS2(memorystatus_add, proc_t, p, int32_t, p->p_memstat_effectivepriority);
2483
2484	/ Processes marked internal do not have priority tracked /
2485	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2486	goto exit;
2487	}
2488
2489	bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2490
2491	if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
2492	assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs - `1`);
2493
2494	} else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
2495	assert(bucket->count == memorystatus_scheduled_idle_demotions_apps - `1`);
2496
2497	} else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
2498	/*
2499	* Entering the idle band.
2500	* Record idle start time.
2501	*/
2502	p->p_memstat_idle_start = mach_absolute_time();
2503	}
2504
2505	TAILQ_INSERT_TAIL(&bucket->list, p, p_memstat_list);
2506	bucket->count++;
2507
2508	memorystatus_list_count++;
2509
2510	memorystatus_check_levels_locked();
2511
2512	exit:
2513	if (!locked) {
2514	proc_list_unlock();
2515	}
2516
2517	return `0`;
2518	}
2519
2520	/*
2521	* Description:
2522	* Moves a process from one jetsam bucket to another.
2523	* which changes the LRU position of the process.
2524	*
2525	* Monitors transition between buckets and if necessary
2526	* will update cached memory limits accordingly.
2527	*
2528	* skip_demotion_check:
2529	* - if the 'jetsam aging policy' is NOT 'legacy':
2530	* When this flag is TRUE, it means we are going
2531	* to age the ripe processes out of the aging bands and into the
2532	* IDLE band and apply their inactive memory limits.
2533	*
2534	* - if the 'jetsam aging policy' is 'legacy':
2535	* When this flag is TRUE, it might mean the above aging mechanism
2536	* OR
2537	* It might be that we have a process that has used up its 'idle deferral'
2538	* stay that is given to it once per lifetime. And in this case, the process
2539	* won't be going through any aging codepaths. But we still need to apply
2540	* the right inactive limits and so we explicitly set this to TRUE if the
2541	* new priority for the process is the IDLE band.
2542	*/
2543	void
2544	memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check)
2545	{
2546	memstat_bucket_t old_bucket, new_bucket;
2547
2548	assert(priority < MEMSTAT_BUCKET_COUNT);
2549
2550	/ Ensure that exit isn't underway, leaving the proc retained but removed from its bucket /
2551	if ((p->p_listflag & P_LIST_EXITED) != `0`) {
2552	return;
2553	}
2554
2555	MEMORYSTATUS_DEBUG(`1`, "memorystatus_update_priority_locked(): setting %s(%d) to priority %d, inserting at %s\n",
2556	(*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, head_insert ? "head" : "tail");
2557
2558	DTRACE_MEMORYSTATUS3(memorystatus_update_priority, proc_t, p, int32_t, p->p_memstat_effectivepriority, int, priority);
2559
2560	#if DEVELOPMENT \|\| DEBUG
2561	if (priority == JETSAM_PRIORITY_IDLE && / if the process is on its way into the IDLE band /
2562	skip_demotion_check == FALSE && / and it isn't via the path that will set the INACTIVE memlimits /
2563	(p->p_memstat_dirty & P_DIRTY_TRACK) && / and it has 'DIRTY' tracking enabled /
2564	((p->p_memstat_memlimit != p->p_memstat_memlimit_inactive) \|\| / and we notice that the current limit isn't the right value (inactive) /
2565	((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) ? ( ! (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)) : (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)))) / OR type (fatal vs non-fatal) /
2566	panic("memorystatus_update_priority_locked: on %s with 0x%x, prio: %d and %d\n", p->p_name, p->p_memstat_state, priority, p->p_memstat_memlimit); / then we must catch this /
2567	#endif /* DEVELOPMENT \|\| DEBUG */
2568
2569	old_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2570
2571	if (skip_demotion_check == FALSE) {
2572
2573	if (isSysProc(p)) {
2574	/*
2575	* For system processes, the memorystatus_dirty_* routines take care of adding/removing
2576	* the processes from the aging bands and balancing the demotion counts.
2577	* We can, however, override that if the process has an 'elevated inactive jetsam band' attribute.
2578	*/
2579
2580	if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
2581	/*
2582	* 2 types of processes can use the non-standard elevated inactive band:
2583	* - Frozen processes that always land in memorystatus_freeze_jetsam_band
2584	* OR
2585	* - processes that specifically opt-in to the elevated inactive support e.g. docked processes.
2586	*/
2587	#if CONFIG_FREEZE
2588	if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
2589	if (priority <= memorystatus_freeze_jetsam_band) {
2590	priority = memorystatus_freeze_jetsam_band;
2591	}
2592	} else
2593	#endif /* CONFIG_FREEZE */
2594	{
2595	if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE) {
2596	priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
2597	}
2598	}
2599	assert(! (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
2600	}
2601	} else if (isApp(p)) {
2602
2603	/*
2604	* Check to see if the application is being lowered in jetsam priority. If so, and:
2605	* - it has an 'elevated inactive jetsam band' attribute, then put it in the appropriate band.
2606	* - it is a normal application, then let it age in the aging band if that policy is in effect.
2607	*/
2608
2609	if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
2610	#if CONFIG_FREEZE
2611	if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
2612	if (priority <= memorystatus_freeze_jetsam_band) {
2613	priority = memorystatus_freeze_jetsam_band;
2614	}
2615	} else
2616	#endif /* CONFIG_FREEZE */
2617	{
2618	if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE) {
2619	priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
2620	}
2621	}
2622	} else {
2623
2624	if (applications_aging_band) {
2625	if (p->p_memstat_effectivepriority == applications_aging_band) {
2626	assert(old_bucket->count == (memorystatus_scheduled_idle_demotions_apps + `1`));
2627	}
2628
2629	if ((jetsam_aging_policy != kJetsamAgingPolicyLegacy) && (priority <= applications_aging_band)) {
2630	assert(! (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
2631	priority = applications_aging_band;
2632	memorystatus_schedule_idle_demotion_locked(p, TRUE);
2633	}
2634	}
2635	}
2636	}
2637	}
2638
2639	if ((system_procs_aging_band && (priority == system_procs_aging_band)) \|\| (applications_aging_band && (priority == applications_aging_band))) {
2640	assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
2641	}
2642
2643	TAILQ_REMOVE(&old_bucket->list, p, p_memstat_list);
2644	old_bucket->count--;
2645
2646	new_bucket = &memstat_bucket[priority];
2647	if (head_insert)
2648	TAILQ_INSERT_HEAD(&new_bucket->list, p, p_memstat_list);
2649	else
2650	TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
2651	new_bucket->count++;
2652
2653	if (memorystatus_highwater_enabled) {
2654	boolean_t is_fatal;
2655	boolean_t use_active;
2656
2657	/*
2658	* If cached limit data is updated, then the limits
2659	* will be enforced by writing to the ledgers.
2660	*/
2661	boolean_t ledger_update_needed = TRUE;
2662
2663	/*
2664	* Here, we must update the cached memory limit if the task
2665	* is transitioning between:
2666	* active <--> inactive
2667	* FG <--> BG
2668	* but:
2669	* dirty <--> clean is ignored
2670	*
2671	* We bypass non-idle processes that have opted into dirty tracking because
2672	* a move between buckets does not imply a transition between the
2673	* dirty <--> clean state.
2674	*/
2675
2676	if (p->p_memstat_dirty & P_DIRTY_TRACK) {
2677
2678	if (skip_demotion_check == TRUE && priority == JETSAM_PRIORITY_IDLE) {
2679	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2680	use_active = FALSE;
2681	} else {
2682	ledger_update_needed = FALSE;
2683	}
2684
2685	} else if ((priority >= JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority < JETSAM_PRIORITY_FOREGROUND)) {
2686	/*
2687	* inactive --> active
2688	* BG --> FG
2689	* assign active state
2690	*/
2691	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2692	use_active = TRUE;
2693
2694	} else if ((priority < JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) {
2695	/*
2696	* active --> inactive
2697	* FG --> BG
2698	* assign inactive state
2699	*/
2700	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2701	use_active = FALSE;
2702	} else {
2703	/*
2704	* The transition between jetsam priority buckets apparently did
2705	* not affect active/inactive state.
2706	* This is not unusual... especially during startup when
2707	* processes are getting established in their respective bands.
2708	*/
2709	ledger_update_needed = FALSE;
2710	}
2711
2712	/*
2713	* Enforce the new limits by writing to the ledger
2714	*/
2715	if (ledger_update_needed) {
2716	task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > `0`) ? p->p_memstat_memlimit : -`1`, NULL, use_active, is_fatal);
2717
2718	MEMORYSTATUS_DEBUG(`3`, "memorystatus_update_priority_locked: new limit on pid %d (%dMB %s) priority old --> new (%d --> %d) dirty?=0x%x %s\n",
2719	p->p_pid, (p->p_memstat_memlimit > `0` ? p->p_memstat_memlimit : -`1`),
2720	(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, priority, p->p_memstat_dirty,
2721	(p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
2722	}
2723	}
2724
2725	/*
2726	* Record idle start or idle delta.
2727	*/
2728	if (p->p_memstat_effectivepriority == priority) {
2729	/*
2730	* This process is not transitioning between
2731	* jetsam priority buckets. Do nothing.
2732	*/
2733	} else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
2734	uint64_t now;
2735	/*
2736	* Transitioning out of the idle priority bucket.
2737	* Record idle delta.
2738	*/
2739	assert(p->p_memstat_idle_start != `0`);
2740	now = mach_absolute_time();
2741	if (now > p->p_memstat_idle_start) {
2742	p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
2743	}
2744
2745	/*
2746	* About to become active and so memory footprint could change.
2747	* So mark it eligible for freeze-considerations next time around.
2748	*/
2749	if (p->p_memstat_state & P_MEMSTAT_FREEZE_IGNORE) {
2750	p->p_memstat_state &= ~P_MEMSTAT_FREEZE_IGNORE;
2751	}
2752
2753	} else if (priority == JETSAM_PRIORITY_IDLE) {
2754	/*
2755	* Transitioning into the idle priority bucket.
2756	* Record idle start.
2757	*/
2758	p->p_memstat_idle_start = mach_absolute_time();
2759	}
2760
2761	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CHANGE_PRIORITY), p->p_pid, priority, p->p_memstat_effectivepriority, `0`, `0`);
2762
2763	p->p_memstat_effectivepriority = priority;
2764
2765	#if CONFIG_SECLUDED_MEMORY
2766	if (secluded_for_apps &&
2767	task_could_use_secluded_mem(p->task)) {
2768	task_set_can_use_secluded_mem(
2769	p->task,
2770	(priority >= JETSAM_PRIORITY_FOREGROUND));
2771	}
2772	#endif /* CONFIG_SECLUDED_MEMORY */
2773
2774	memorystatus_check_levels_locked();
2775	}
2776
2777	/*
2778	*
2779	* Description: Update the jetsam priority and memory limit attributes for a given process.
2780	*
2781	* Parameters:
2782	* p init this process's jetsam information.
2783	* priority The jetsam priority band
2784	* user_data user specific data, unused by the kernel
2785	* effective guards against race if process's update already occurred
2786	* update_memlimit When true we know this is the init step via the posix_spawn path.
2787	*
2788	* memlimit_active Value in megabytes; The monitored footprint level while the
2789	* process is active. Exceeding it may result in termination
2790	* based on it's associated fatal flag.
2791	*
2792	* memlimit_active_is_fatal When a process is active and exceeds its memory footprint,
2793	* this describes whether or not it should be immediately fatal.
2794	*
2795	* memlimit_inactive Value in megabytes; The monitored footprint level while the
2796	* process is inactive. Exceeding it may result in termination
2797	* based on it's associated fatal flag.
2798	*
2799	* memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint,
2800	* this describes whether or not it should be immediatly fatal.
2801	*
2802	* Returns: 0 Success
2803	* non-0 Failure
2804	*/
2805
2806	int
2807	memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit,
2808	int32_t memlimit_active, boolean_t memlimit_active_is_fatal,
2809	int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal)
2810	{
2811	int ret;
2812	boolean_t head_insert = false;
2813
2814	MEMORYSTATUS_DEBUG(`1`, "memorystatus_update: changing (%s) pid %d: priority %d, user_data 0x%llx\n", (*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, user_data);
2815
2816	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) \| DBG_FUNC_START, p->p_pid, priority, user_data, effective, `0`);
2817
2818	if (priority == -`1`) {
2819	/ Use as shorthand for default priority /
2820	priority = JETSAM_PRIORITY_DEFAULT;
2821	} else if ((priority == system_procs_aging_band) \|\| (priority == applications_aging_band)) {
2822	/ Both the aging bands are reserved for internal use; if requested, adjust to JETSAM_PRIORITY_IDLE. /
2823	priority = JETSAM_PRIORITY_IDLE;
2824	} else if (priority == JETSAM_PRIORITY_IDLE_HEAD) {
2825	/ JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle queue /
2826	priority = JETSAM_PRIORITY_IDLE;
2827	head_insert = TRUE;
2828	} else if ((priority < `0`) \|\| (priority >= MEMSTAT_BUCKET_COUNT)) {
2829	/ Sanity check /
2830	ret = EINVAL;
2831	goto out;
2832	}
2833
2834	proc_list_lock();
2835
2836	assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
2837
2838	if (effective && (p->p_memstat_state & P_MEMSTAT_PRIORITYUPDATED)) {
2839	ret = EALREADY;
2840	proc_list_unlock();
2841	MEMORYSTATUS_DEBUG(`1`, "memorystatus_update: effective change specified for pid %d, but change already occurred.\n", p->p_pid);
2842	goto out;
2843	}
2844
2845	if ((p->p_memstat_state & P_MEMSTAT_TERMINATED) \|\| ((p->p_listflag & P_LIST_EXITED) != `0`)) {
2846	/*
2847	* This could happen when a process calling posix_spawn() is exiting on the jetsam thread.
2848	*/
2849	ret = EBUSY;
2850	proc_list_unlock();
2851	goto out;
2852	}
2853
2854	p->p_memstat_state \|= P_MEMSTAT_PRIORITYUPDATED;
2855	p->p_memstat_userdata = user_data;
2856	p->p_memstat_requestedpriority = priority;
2857
2858	if (update_memlimit) {
2859	boolean_t is_fatal;
2860	boolean_t use_active;
2861
2862	/*
2863	* Posix_spawn'd processes come through this path to instantiate ledger limits.
2864	* Forked processes do not come through this path, so no ledger limits exist.
2865	* (That's why forked processes can consume unlimited memory.)
2866	*/
2867
2868	MEMORYSTATUS_DEBUG(`3`, "memorystatus_update(enter): pid %d, priority %d, dirty=0x%x, Active(%dMB %s), Inactive(%dMB, %s)\n",
2869	p->p_pid, priority, p->p_memstat_dirty,
2870	memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"),
2871	memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF"));
2872
2873	if (memlimit_active <= `0`) {
2874	/*
2875	* This process will have a system_wide task limit when active.
2876	* System_wide task limit is always fatal.
2877	* It's quite common to see non-fatal flag passed in here.
2878	* It's not an error, we just ignore it.
2879	*/
2880
2881	/*
2882	* For backward compatibility with some unexplained launchd behavior,
2883	* we allow a zero sized limit. But we still enforce system_wide limit
2884	* when written to the ledgers.
2885	*/
2886
2887	if (memlimit_active < `0`) {
2888	memlimit_active = -`1`; / enforces system_wide task limit /
2889	}
2890	memlimit_active_is_fatal = TRUE;
2891	}
2892
2893	if (memlimit_inactive <= `0`) {
2894	/*
2895	* This process will have a system_wide task limit when inactive.
2896	* System_wide task limit is always fatal.
2897	*/
2898
2899	memlimit_inactive = -`1`;
2900	memlimit_inactive_is_fatal = TRUE;
2901	}
2902
2903	/*
2904	* Initialize the active limit variants for this process.
2905	*/
2906	SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
2907
2908	/*
2909	* Initialize the inactive limit variants for this process.
2910	*/
2911	SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
2912
2913	/*
2914	* Initialize the cached limits for target process.
2915	* When the target process is dirty tracked, it's typically
2916	* in a clean state. Non dirty tracked processes are
2917	* typically active (Foreground or above).
2918	* But just in case, we don't make assumptions...
2919	*/
2920
2921	if (proc_jetsam_state_is_active_locked(p) == TRUE) {
2922	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2923	use_active = TRUE;
2924	} else {
2925	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2926	use_active = FALSE;
2927	}
2928
2929	/*
2930	* Enforce the cached limit by writing to the ledger.
2931	*/
2932	if (memorystatus_highwater_enabled) {
2933	/ apply now /
2934	task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > `0`) ? p->p_memstat_memlimit : -`1`), NULL, use_active, is_fatal);
2935
2936	MEMORYSTATUS_DEBUG(`3`, "memorystatus_update: init: limit on pid %d (%dMB %s) targeting priority(%d) dirty?=0x%x %s\n",
2937	p->p_pid, (p->p_memstat_memlimit > `0` ? p->p_memstat_memlimit : -`1`),
2938	(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), priority, p->p_memstat_dirty,
2939	(p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
2940	}
2941	}
2942
2943	/*
2944	* We can't add to the aging bands buckets here.
2945	* But, we could be removing it from those buckets.
2946	* Check and take appropriate steps if so.
2947	*/
2948
2949	if (isProcessInAgingBands(p)) {
2950
2951	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2952	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
2953	} else {
2954	if (jetsam_aging_policy == kJetsamAgingPolicyLegacy && priority == JETSAM_PRIORITY_IDLE) {
2955	/*
2956	* Daemons with 'inactive' limits will go through the dirty tracking codepath.
2957	* This path deals with apps that may have 'inactive' limits e.g. WebContent processes.
2958	* If this is the legacy aging policy we explicitly need to apply those limits. If it
2959	* is any other aging policy, then we don't need to worry because all processes
2960	* will go through the aging bands and then the demotion thread will take care to
2961	* move them into the IDLE band and apply the required limits.
2962	*/
2963	memorystatus_update_priority_locked(p, priority, head_insert, TRUE);
2964	}
2965	}
2966
2967	memorystatus_update_priority_locked(p, priority, head_insert, FALSE);
2968
2969	proc_list_unlock();
2970	ret = `0`;
2971
2972	out:
2973	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) \| DBG_FUNC_END, ret, `0`, `0`, `0`, `0`);
2974
2975	return ret;
2976	}
2977
2978	int
2979	memorystatus_remove(proc_t p, boolean_t locked)
2980	{
2981	int ret;
2982	memstat_bucket_t *bucket;
2983	boolean_t reschedule = FALSE;
2984
2985	MEMORYSTATUS_DEBUG(`1`, "memorystatus_list_remove: removing pid %d\n", p->p_pid);
2986
2987	if (!locked) {
2988	proc_list_lock();
2989	}
2990
2991	assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
2992
2993	bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2994
2995	if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
2996
2997	assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs);
2998	reschedule = TRUE;
2999
3000	} else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
3001
3002	assert(bucket->count == memorystatus_scheduled_idle_demotions_apps);
3003	reschedule = TRUE;
3004	}
3005
3006	/*
3007	* Record idle delta
3008	*/
3009
3010	if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
3011	uint64_t now = mach_absolute_time();
3012	if (now > p->p_memstat_idle_start) {
3013	p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
3014	}
3015	}
3016
3017	TAILQ_REMOVE(&bucket->list, p, p_memstat_list);
3018	bucket->count--;
3019
3020	memorystatus_list_count--;
3021
3022	/ If awaiting demotion to the idle band, clean up /
3023	if (reschedule) {
3024	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3025	memorystatus_reschedule_idle_demotion_locked();
3026	}
3027
3028	memorystatus_check_levels_locked();
3029
3030	#if CONFIG_FREEZE
3031	if (p->p_memstat_state & (P_MEMSTAT_FROZEN)) {
3032
3033	if (p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) {
3034	p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
3035	memorystatus_refreeze_eligible_count--;
3036	}
3037
3038	memorystatus_frozen_count--;
3039	memorystatus_frozen_shared_mb -= p->p_memstat_freeze_sharedanon_pages;
3040	p->p_memstat_freeze_sharedanon_pages = `0`;
3041	}
3042
3043	if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
3044	memorystatus_suspended_count--;
3045	}
3046	#endif
3047
3048	if (!locked) {
3049	proc_list_unlock();
3050	}
3051
3052	if (p) {
3053	ret = `0`;
3054	} else {
3055	ret = ESRCH;
3056	}
3057
3058	return ret;
3059	}
3060
3061	/*
3062	* Validate dirty tracking flags with process state.
3063	*
3064	* Return:
3065	* 0 on success
3066	* non-0 on failure
3067	*
3068	* The proc_list_lock is held by the caller.
3069	*/
3070
3071	static int
3072	memorystatus_validate_track_flags(struct proc *target_p, uint32_t pcontrol) {
3073	/ See that the process isn't marked for termination /
3074	if (target_p->p_memstat_dirty & P_DIRTY_TERMINATED) {
3075	return EBUSY;
3076	}
3077
3078	/ Idle exit requires that process be tracked /
3079	if ((pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) &&
3080	!(pcontrol & PROC_DIRTY_TRACK)) {
3081	return EINVAL;
3082	}
3083
3084	/ 'Launch in progress' tracking requires that process have enabled dirty tracking too. /
3085	if ((pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) &&
3086	!(pcontrol & PROC_DIRTY_TRACK)) {
3087	return EINVAL;
3088	}
3089
3090	/ Only one type of DEFER behavior is allowed./
3091	if ((pcontrol & PROC_DIRTY_DEFER) &&
3092	(pcontrol & PROC_DIRTY_DEFER_ALWAYS)) {
3093	return EINVAL;
3094	}
3095
3096	/ Deferral is only relevant if idle exit is specified /
3097	if (((pcontrol & PROC_DIRTY_DEFER) \|\|
3098	(pcontrol & PROC_DIRTY_DEFER_ALWAYS)) &&
3099	!(pcontrol & PROC_DIRTY_ALLOWS_IDLE_EXIT)) {
3100	return EINVAL;
3101	}
3102
3103	return(`0`);
3104	}
3105
3106	static void
3107	memorystatus_update_idle_priority_locked(proc_t p) {
3108	int32_t priority;
3109
3110	MEMORYSTATUS_DEBUG(`1`, "memorystatus_update_idle_priority_locked(): pid %d dirty 0x%X\n", p->p_pid, p->p_memstat_dirty);
3111
3112	assert(isSysProc(p));
3113
3114	if ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED\|P_DIRTY_IS_DIRTY)) == P_DIRTY_IDLE_EXIT_ENABLED) {
3115
3116	priority = (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) ? system_procs_aging_band : JETSAM_PRIORITY_IDLE;
3117	} else {
3118	priority = p->p_memstat_requestedpriority;
3119	}
3120
3121	if (priority != p->p_memstat_effectivepriority) {
3122
3123	if ((jetsam_aging_policy == kJetsamAgingPolicyLegacy) &&
3124	(priority == JETSAM_PRIORITY_IDLE)) {
3125
3126	/*
3127	* This process is on its way into the IDLE band. The system is
3128	* using 'legacy' jetsam aging policy. That means, this process
3129	* has already used up its idle-deferral aging time that is given
3130	* once per its lifetime. So we need to set the INACTIVE limits
3131	* explicitly because it won't be going through the demotion paths
3132	* that take care to apply the limits appropriately.
3133	*/
3134
3135	if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
3136
3137	/*
3138	* This process has the 'elevated inactive jetsam band' attribute.
3139	* So, there will be no trip to IDLE after all.
3140	* Instead, we pin the process in the elevated band,
3141	* where its ACTIVE limits will apply.
3142	*/
3143
3144	priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
3145	}
3146
3147	memorystatus_update_priority_locked(p, priority, false, true);
3148
3149	} else {
3150	memorystatus_update_priority_locked(p, priority, false, false);
3151	}
3152	}
3153	}
3154
3155	/*
3156	* Processes can opt to have their state tracked by the kernel, indicating when they are busy (dirty) or idle
3157	* (clean). They may also indicate that they support termination when idle, with the result that they are promoted
3158	* to their desired, higher, jetsam priority when dirty (and are therefore killed later), and demoted to the low
3159	* priority idle band when clean (and killed earlier, protecting higher priority procesess).
3160	*
3161	* If the deferral flag is set, then newly tracked processes will be protected for an initial period (as determined by
3162	* memorystatus_sysprocs_idle_delay_time); if they go clean during this time, then they will be moved to a deferred-idle band
3163	* with a slightly higher priority, guarding against immediate termination under memory pressure and being unable to
3164	* make forward progress. Finally, when the guard expires, they will be moved to the standard, lowest-priority, idle
3165	* band. The deferral can be cleared early by clearing the appropriate flag.
3166	*
3167	* The deferral timer is active only for the duration that the process is marked as guarded and clean; if the process
3168	* is marked dirty, the timer will be cancelled. Upon being subsequently marked clean, the deferment will either be
3169	* re-enabled or the guard state cleared, depending on whether the guard deadline has passed.
3170	*/
3171
3172	int
3173	memorystatus_dirty_track(proc_t p, uint32_t pcontrol) {
3174	unsigned int old_dirty;
3175	boolean_t reschedule = FALSE;
3176	boolean_t already_deferred = FALSE;
3177	boolean_t defer_now = FALSE;
3178	int ret = `0`;
3179
3180	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_TRACK),
3181	p->p_pid, p->p_memstat_dirty, pcontrol, `0`, `0`);
3182
3183	proc_list_lock();
3184
3185	if ((p->p_listflag & P_LIST_EXITED) != `0`) {
3186	/*
3187	* Process is on its way out.
3188	*/
3189	ret = EBUSY;
3190	goto exit;
3191	}
3192
3193	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
3194	ret = EPERM;
3195	goto exit;
3196	}
3197
3198	if ((ret = memorystatus_validate_track_flags(p, pcontrol)) != `0`) {
3199	/ error /
3200	goto exit;
3201	}
3202
3203	old_dirty = p->p_memstat_dirty;
3204
3205	/ These bits are cumulative, as per <rdar://problem/11159924> /
3206	if (pcontrol & PROC_DIRTY_TRACK) {
3207	p->p_memstat_dirty \|= P_DIRTY_TRACK;
3208	}
3209
3210	if (pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) {
3211	p->p_memstat_dirty \|= P_DIRTY_ALLOW_IDLE_EXIT;
3212	}
3213
3214	if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
3215	p->p_memstat_dirty \|= P_DIRTY_LAUNCH_IN_PROGRESS;
3216	}
3217
3218	if (old_dirty & P_DIRTY_AGING_IN_PROGRESS) {
3219	already_deferred = TRUE;
3220	}
3221
3222
3223	/ This can be set and cleared exactly once. /
3224	if (pcontrol & (PROC_DIRTY_DEFER \| PROC_DIRTY_DEFER_ALWAYS)) {
3225
3226	if ((pcontrol & (PROC_DIRTY_DEFER)) &&
3227	!(old_dirty & P_DIRTY_DEFER)) {
3228	p->p_memstat_dirty \|= P_DIRTY_DEFER;
3229	}
3230
3231	if ((pcontrol & (PROC_DIRTY_DEFER_ALWAYS)) &&
3232	!(old_dirty & P_DIRTY_DEFER_ALWAYS)) {
3233	p->p_memstat_dirty \|= P_DIRTY_DEFER_ALWAYS;
3234	}
3235
3236	defer_now = TRUE;
3237	}
3238
3239	MEMORYSTATUS_DEBUG(`1`, "memorystatus_on_track_dirty(): set idle-exit %s / defer %s / dirty %s for pid %d\n",
3240	((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) ? "Y" : "N",
3241	defer_now ? "Y" : "N",
3242	p->p_memstat_dirty & P_DIRTY ? "Y" : "N",
3243	p->p_pid);
3244
3245	/ Kick off or invalidate the idle exit deferment if there's a state transition. /
3246	if (!(p->p_memstat_dirty & P_DIRTY_IS_DIRTY)) {
3247	if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
3248
3249	if (defer_now && !already_deferred) {
3250
3251	/*
3252	* Request to defer a clean process that's idle-exit enabled
3253	* and not already in the jetsam deferred band. Most likely a
3254	* new launch.
3255	*/
3256	memorystatus_schedule_idle_demotion_locked(p, TRUE);
3257	reschedule = TRUE;
3258
3259	} else if (!defer_now) {
3260
3261	/*
3262	* The process isn't asking for the 'aging' facility.
3263	* Could be that it is:
3264	*/
3265
3266	if (already_deferred) {
3267	/*
3268	* already in the aging bands. Traditionally,
3269	* some processes have tried to use this to
3270	* opt out of the 'aging' facility.
3271	*/
3272
3273	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3274	} else {
3275	/*
3276	* agnostic to the 'aging' facility. In that case,
3277	* we'll go ahead and opt it in because this is likely
3278	* a new launch (clean process, dirty tracking enabled)
3279	*/
3280
3281	memorystatus_schedule_idle_demotion_locked(p, TRUE);
3282	}
3283
3284	reschedule = TRUE;
3285	}
3286	}
3287	} else {
3288
3289	/*
3290	* We are trying to operate on a dirty process. Dirty processes have to
3291	* be removed from the deferred band. The question is do we reset the
3292	* deferred state or not?
3293	*
3294	* This could be a legal request like:
3295	* - this process had opted into the 'aging' band
3296	* - but it's now dirty and requests to opt out.
3297	* In this case, we remove the process from the band and reset its
3298	* state too. It'll opt back in properly when needed.
3299	*
3300	* OR, this request could be a user-space bug. E.g.:
3301	* - this process had opted into the 'aging' band when clean
3302	* - and, then issues another request to again put it into the band except
3303	* this time the process is dirty.
3304	* The process going dirty, as a transition in memorystatus_dirty_set(), will pull the process out of
3305	* the deferred band with its state intact. So our request below is no-op.
3306	* But we do it here anyways for coverage.
3307	*
3308	* memorystatus_update_idle_priority_locked()
3309	* single-mindedly treats a dirty process as "cannot be in the aging band".
3310	*/
3311
3312	if (!defer_now && already_deferred) {
3313	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3314	reschedule = TRUE;
3315	} else {
3316
3317	boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
3318
3319	memorystatus_invalidate_idle_demotion_locked(p, reset_state);
3320	reschedule = TRUE;
3321	}
3322	}
3323
3324	memorystatus_update_idle_priority_locked(p);
3325
3326	if (reschedule) {
3327	memorystatus_reschedule_idle_demotion_locked();
3328	}
3329
3330	ret = `0`;
3331
3332	exit:
3333	proc_list_unlock();
3334
3335	return ret;
3336	}
3337
3338	int
3339	memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol) {
3340	int ret;
3341	boolean_t kill = false;
3342	boolean_t reschedule = FALSE;
3343	boolean_t was_dirty = FALSE;
3344	boolean_t now_dirty = FALSE;
3345
3346	MEMORYSTATUS_DEBUG(`1`, "memorystatus_dirty_set(): %d %d 0x%x 0x%x\n", self, p->p_pid, pcontrol, p->p_memstat_dirty);
3347	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_SET), p->p_pid, self, pcontrol, `0`, `0`);
3348
3349	proc_list_lock();
3350
3351	if ((p->p_listflag & P_LIST_EXITED) != `0`) {
3352	/*
3353	* Process is on its way out.
3354	*/
3355	ret = EBUSY;
3356	goto exit;
3357	}
3358
3359	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
3360	ret = EPERM;
3361	goto exit;
3362	}
3363
3364	if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
3365	was_dirty = TRUE;
3366
3367	if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
3368	/ Dirty tracking not enabled /
3369	ret = EINVAL;
3370	} else if (pcontrol && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
3371	/*
3372	* Process is set to be terminated and we're attempting to mark it dirty.
3373	* Set for termination and marking as clean is OK - see <rdar://problem/10594349>.
3374	*/
3375	ret = EBUSY;
3376	} else {
3377	int flag = (self == TRUE) ? P_DIRTY : P_DIRTY_SHUTDOWN;
3378	if (pcontrol && !(p->p_memstat_dirty & flag)) {
3379	/ Mark the process as having been dirtied at some point /
3380	p->p_memstat_dirty \|= (flag \| P_DIRTY_MARKED);
3381	memorystatus_dirty_count++;
3382	ret = `0`;
3383	} else if ((pcontrol == `0`) && (p->p_memstat_dirty & flag)) {
3384	if ((flag == P_DIRTY_SHUTDOWN) && (!(p->p_memstat_dirty & P_DIRTY))) {
3385	/ Clearing the dirty shutdown flag, and the process is otherwise clean - kill /
3386	p->p_memstat_dirty \|= P_DIRTY_TERMINATED;
3387	kill = true;
3388	} else if ((flag == P_DIRTY) && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
3389	/ Kill previously terminated processes if set clean /
3390	kill = true;
3391	}
3392	p->p_memstat_dirty &= ~flag;
3393	memorystatus_dirty_count--;
3394	ret = `0`;
3395	} else {
3396	/ Already set /
3397	ret = EALREADY;
3398	}
3399	}
3400
3401	if (ret != `0`) {
3402	goto exit;
3403	}
3404
3405	if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
3406	now_dirty = TRUE;
3407
3408	if ((was_dirty == TRUE && now_dirty == FALSE) \|\|
3409	(was_dirty == FALSE && now_dirty == TRUE)) {
3410
3411	/ Manage idle exit deferral, if applied /
3412	if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
3413
3414	/*
3415	* Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band OR it might be heading back
3416	* there once it's clean again. For the legacy case, this only applies if it has some protection window left.
3417	* P_DIRTY_DEFER: one-time protection window given at launch
3418	* P_DIRTY_DEFER_ALWAYS: protection window given for every dirty->clean transition. Like non-legacy mode.
3419	*
3420	* Non-Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band. It will always stop over
3421	* in that band on it's way to IDLE.
3422	*/
3423
3424	if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
3425	/*
3426	* New dirty process i.e. "was_dirty == FALSE && now_dirty == TRUE"
3427	*
3428	* The process will move from its aging band to its higher requested
3429	* jetsam band.
3430	*/
3431	boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
3432
3433	memorystatus_invalidate_idle_demotion_locked(p, reset_state);
3434	reschedule = TRUE;
3435	} else {
3436
3437	/*
3438	* Process is back from "dirty" to "clean".
3439	*/
3440
3441	if (jetsam_aging_policy == kJetsamAgingPolicyLegacy) {
3442	if (((p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) == FALSE) &&
3443	(mach_absolute_time() >= p->p_memstat_idledeadline)) {
3444	/*
3445	* The process' hasn't enrolled in the "always defer after dirty"
3446	* mode and its deadline has expired. It currently
3447	* does not reside in any of the aging buckets.
3448	*
3449	* It's on its way to the JETSAM_PRIORITY_IDLE
3450	* bucket via memorystatus_update_idle_priority_locked()
3451	* below.
3452
3453	* So all we need to do is reset all the state on the
3454	* process that's related to the aging bucket i.e.
3455	* the AGING_IN_PROGRESS flag and the timer deadline.
3456	*/
3457
3458	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3459	reschedule = TRUE;
3460	} else {
3461	/*
3462	* Process enrolled in "always stop in deferral band after dirty" OR
3463	* it still has some protection window left and so
3464	* we just re-arm the timer without modifying any
3465	* state on the process iff it still wants into that band.
3466	*/
3467
3468	if (p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) {
3469	memorystatus_schedule_idle_demotion_locked(p, TRUE);
3470	reschedule = TRUE;
3471	} else if (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) {
3472	memorystatus_schedule_idle_demotion_locked(p, FALSE);
3473	reschedule = TRUE;
3474	}
3475	}
3476	} else {
3477
3478	memorystatus_schedule_idle_demotion_locked(p, TRUE);
3479	reschedule = TRUE;
3480	}
3481	}
3482	}
3483
3484	memorystatus_update_idle_priority_locked(p);
3485
3486	if (memorystatus_highwater_enabled) {
3487	boolean_t ledger_update_needed = TRUE;
3488	boolean_t use_active;
3489	boolean_t is_fatal;
3490	/*
3491	* We are in this path because this process transitioned between
3492	* dirty <--> clean state. Update the cached memory limits.
3493	*/
3494
3495	if (proc_jetsam_state_is_active_locked(p) == TRUE) {
3496	/*
3497	* process is pinned in elevated band
3498	* or
3499	* process is dirty
3500	*/
3501	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
3502	use_active = TRUE;
3503	ledger_update_needed = TRUE;
3504	} else {
3505	/*
3506	* process is clean...but if it has opted into pressured-exit
3507	* we don't apply the INACTIVE limit till the process has aged
3508	* out and is entering the IDLE band.
3509	* See memorystatus_update_priority_locked() for that.
3510	*/
3511
3512	if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
3513	ledger_update_needed = FALSE;
3514	} else {
3515	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
3516	use_active = FALSE;
3517	ledger_update_needed = TRUE;
3518	}
3519	}
3520
3521	/*
3522	* Enforce the new limits by writing to the ledger.
3523	*
3524	* This is a hot path and holding the proc_list_lock while writing to the ledgers,
3525	* (where the task lock is taken) is bad. So, we temporarily drop the proc_list_lock.
3526	* We aren't traversing the jetsam bucket list here, so we should be safe.
3527	* See rdar://21394491.
3528	*/
3529
3530	if (ledger_update_needed && proc_ref_locked(p) == p) {
3531	int ledger_limit;
3532	if (p->p_memstat_memlimit > `0`) {
3533	ledger_limit = p->p_memstat_memlimit;
3534	} else {
3535	ledger_limit = -`1`;
3536	}
3537	proc_list_unlock();
3538	task_set_phys_footprint_limit_internal(p->task, ledger_limit, NULL, use_active, is_fatal);
3539	proc_list_lock();
3540	proc_rele_locked(p);
3541
3542	MEMORYSTATUS_DEBUG(`3`, "memorystatus_dirty_set: new limit on pid %d (%dMB %s) priority(%d) dirty?=0x%x %s\n",
3543	p->p_pid, (p->p_memstat_memlimit > `0` ? p->p_memstat_memlimit : -`1`),
3544	(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
3545	(p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
3546	}
3547
3548	}
3549
3550	/ If the deferral state changed, reschedule the demotion timer /
3551	if (reschedule) {
3552	memorystatus_reschedule_idle_demotion_locked();
3553	}
3554	}
3555
3556	if (kill) {
3557	if (proc_ref_locked(p) == p) {
3558	proc_list_unlock();
3559	psignal(p, SIGKILL);
3560	proc_list_lock();
3561	proc_rele_locked(p);
3562	}
3563	}
3564
3565	exit:
3566	proc_list_unlock();
3567
3568	return ret;
3569	}
3570
3571	int
3572	memorystatus_dirty_clear(proc_t p, uint32_t pcontrol) {
3573
3574	int ret = `0`;
3575
3576	MEMORYSTATUS_DEBUG(`1`, "memorystatus_dirty_clear(): %d 0x%x 0x%x\n", p->p_pid, pcontrol, p->p_memstat_dirty);
3577
3578	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_CLEAR), p->p_pid, pcontrol, `0`, `0`, `0`);
3579
3580	proc_list_lock();
3581
3582	if ((p->p_listflag & P_LIST_EXITED) != `0`) {
3583	/*
3584	* Process is on its way out.
3585	*/
3586	ret = EBUSY;
3587	goto exit;
3588	}
3589
3590	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
3591	ret = EPERM;
3592	goto exit;
3593	}
3594
3595	if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
3596	/ Dirty tracking not enabled /
3597	ret = EINVAL;
3598	goto exit;
3599	}
3600
3601	if (!pcontrol \|\| (pcontrol & (PROC_DIRTY_LAUNCH_IN_PROGRESS \| PROC_DIRTY_DEFER \| PROC_DIRTY_DEFER_ALWAYS)) == `0`) {
3602	ret = EINVAL;
3603	goto exit;
3604	}
3605
3606	if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
3607	p->p_memstat_dirty &= ~P_DIRTY_LAUNCH_IN_PROGRESS;
3608	}
3609
3610	/ This can be set and cleared exactly once. /
3611	if (pcontrol & (PROC_DIRTY_DEFER \| PROC_DIRTY_DEFER_ALWAYS)) {
3612
3613	if (p->p_memstat_dirty & P_DIRTY_DEFER) {
3614	p->p_memstat_dirty &= ~(P_DIRTY_DEFER);
3615	}
3616
3617	if (p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) {
3618	p->p_memstat_dirty &= ~(P_DIRTY_DEFER_ALWAYS);
3619	}
3620
3621	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3622	memorystatus_update_idle_priority_locked(p);
3623	memorystatus_reschedule_idle_demotion_locked();
3624	}
3625
3626	ret = `0`;
3627	exit:
3628	proc_list_unlock();
3629
3630	return ret;
3631	}
3632
3633	int
3634	memorystatus_dirty_get(proc_t p) {
3635	int ret = `0`;
3636
3637	proc_list_lock();
3638
3639	if (p->p_memstat_dirty & P_DIRTY_TRACK) {
3640	ret \|= PROC_DIRTY_TRACKED;
3641	if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
3642	ret \|= PROC_DIRTY_ALLOWS_IDLE_EXIT;
3643	}
3644	if (p->p_memstat_dirty & P_DIRTY) {
3645	ret \|= PROC_DIRTY_IS_DIRTY;
3646	}
3647	if (p->p_memstat_dirty & P_DIRTY_LAUNCH_IN_PROGRESS) {
3648	ret \|= PROC_DIRTY_LAUNCH_IS_IN_PROGRESS;
3649	}
3650	}
3651
3652	proc_list_unlock();
3653
3654	return ret;
3655	}
3656
3657	int
3658	memorystatus_on_terminate(proc_t p) {
3659	int sig;
3660
3661	proc_list_lock();
3662
3663	p->p_memstat_dirty \|= P_DIRTY_TERMINATED;
3664
3665	if ((p->p_memstat_dirty & (P_DIRTY_TRACK\|P_DIRTY_IS_DIRTY)) == P_DIRTY_TRACK) {
3666	/ Clean; mark as terminated and issue SIGKILL /
3667	sig = SIGKILL;
3668	} else {
3669	/ Dirty, terminated, or state tracking is unsupported; issue SIGTERM to allow cleanup /
3670	sig = SIGTERM;
3671	}
3672
3673	proc_list_unlock();
3674
3675	return sig;
3676	}
3677
3678	void
3679	memorystatus_on_suspend(proc_t p)
3680	{
3681	#if CONFIG_FREEZE
3682	uint32_t pages;
3683	memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
3684	#endif
3685	proc_list_lock();
3686	#if CONFIG_FREEZE
3687	memorystatus_suspended_count++;
3688	#endif
3689	p->p_memstat_state \|= P_MEMSTAT_SUSPENDED;
3690	proc_list_unlock();
3691	}
3692
3693	void
3694	memorystatus_on_resume(proc_t p)
3695	{
3696	#if CONFIG_FREEZE
3697	boolean_t frozen;
3698	pid_t pid;
3699	#endif
3700
3701	proc_list_lock();
3702
3703	#if CONFIG_FREEZE
3704	frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN);
3705	if (frozen) {
3706	/*
3707	* Now that we don't _thaw_ a process completely,
3708	* resuming it (and having some on-demand swapins)
3709	* shouldn't preclude it from being counted as frozen.
3710	*
3711	* memorystatus_frozen_count--;
3712	*
3713	* We preserve the P_MEMSTAT_FROZEN state since the process
3714	* could have state on disk AND so will deserve some protection
3715	* in the jetsam bands.
3716	*/
3717	if ((p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) == `0`) {
3718	p->p_memstat_state \|= P_MEMSTAT_REFREEZE_ELIGIBLE;
3719	memorystatus_refreeze_eligible_count++;
3720	}
3721	p->p_memstat_thaw_count++;
3722
3723	memorystatus_thaw_count++;
3724	}
3725
3726	memorystatus_suspended_count--;
3727
3728	pid = p->p_pid;
3729	#endif
3730
3731	/*
3732	* P_MEMSTAT_FROZEN will remain unchanged. This used to be:
3733	* p->p_memstat_state &= ~(P_MEMSTAT_SUSPENDED \| P_MEMSTAT_FROZEN);
3734	*/
3735	p->p_memstat_state &= ~P_MEMSTAT_SUSPENDED;
3736
3737	proc_list_unlock();
3738
3739	#if CONFIG_FREEZE
3740	if (frozen) {
3741	memorystatus_freeze_entry_t data = { pid, FALSE, `0` };
3742	memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
3743	}
3744	#endif
3745	}
3746
3747	void
3748	memorystatus_on_inactivity(proc_t p)
3749	{
3750	#pragma unused(p)
3751	#if CONFIG_FREEZE
3752	/ Wake the freeze thread /
3753	thread_wakeup((event_t)&memorystatus_freeze_wakeup);
3754	#endif
3755	}
3756
3757	/*
3758	* The proc_list_lock is held by the caller.
3759	*/
3760	static uint32_t
3761	memorystatus_build_state(proc_t p) {
3762	uint32_t snapshot_state = `0`;
3763
3764	/ General /
3765	if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
3766	snapshot_state \|= kMemorystatusSuspended;
3767	}
3768	if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
3769	snapshot_state \|= kMemorystatusFrozen;
3770	}
3771	if (p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) {
3772	snapshot_state \|= kMemorystatusWasThawed;
3773	}
3774
3775	/ Tracking /
3776	if (p->p_memstat_dirty & P_DIRTY_TRACK) {
3777	snapshot_state \|= kMemorystatusTracked;
3778	}
3779	if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
3780	snapshot_state \|= kMemorystatusSupportsIdleExit;
3781	}
3782	if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
3783	snapshot_state \|= kMemorystatusDirty;
3784	}
3785
3786	return snapshot_state;
3787	}
3788
3789	static boolean_t
3790	kill_idle_exit_proc(void)
3791	{
3792	proc_t p, victim_p = PROC_NULL;
3793	uint64_t current_time;
3794	boolean_t killed = FALSE;
3795	unsigned int i = `0`;
3796	os_reason_t jetsam_reason = OS_REASON_NULL;
3797
3798	/ Pick next idle exit victim. /
3799	current_time = mach_absolute_time();
3800
3801	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_IDLE_EXIT);
3802	if (jetsam_reason == OS_REASON_NULL) {
3803	printf("kill_idle_exit_proc: failed to allocate jetsam reason\n");
3804	}
3805
3806	proc_list_lock();
3807
3808	p = memorystatus_get_first_proc_locked(&i, FALSE);
3809	while (p) {
3810	/ No need to look beyond the idle band /
3811	if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
3812	break;
3813	}
3814
3815	if ((p->p_memstat_dirty & (P_DIRTY_ALLOW_IDLE_EXIT\|P_DIRTY_IS_DIRTY\|P_DIRTY_TERMINATED)) == (P_DIRTY_ALLOW_IDLE_EXIT)) {
3816	if (current_time >= p->p_memstat_idledeadline) {
3817	p->p_memstat_dirty \|= P_DIRTY_TERMINATED;
3818	victim_p = proc_ref_locked(p);
3819	break;
3820	}
3821	}
3822
3823	p = memorystatus_get_next_proc_locked(&i, p, FALSE);
3824	}
3825
3826	proc_list_unlock();
3827
3828	if (victim_p) {
3829	printf("memorystatus: killing_idle_process pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "unknown"));
3830	killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit, jetsam_reason);
3831	proc_rele(victim_p);
3832	} else {
3833	os_reason_free(jetsam_reason);
3834	}
3835
3836	return killed;
3837	}
3838
3839	static void
3840	memorystatus_thread_wake(void)
3841	{
3842	int thr_id = `0`;
3843	int active_thr = atomic_load(&active_jetsam_threads);
3844
3845	/ Wakeup all the jetsam threads /
3846	for (thr_id = `0`; thr_id < active_thr; thr_id++) {
3847	thread_wakeup((event_t)&jetsam_threads[thr_id].memorystatus_wakeup);
3848	}
3849	}
3850
3851	#if CONFIG_JETSAM
3852
3853	static void
3854	memorystatus_thread_pool_max()
3855	{
3856	/ Increase the jetsam thread pool to max_jetsam_threads /
3857	int max_threads = max_jetsam_threads;
3858	printf("Expanding memorystatus pool to %d!\n", max_threads);
3859	atomic_store(&active_jetsam_threads, max_threads);
3860	}
3861
3862	static void
3863	memorystatus_thread_pool_default()
3864	{
3865	/ Restore the jetsam thread pool to a single thread /
3866	printf("Reverting memorystatus pool back to 1\n");
3867	atomic_store(&active_jetsam_threads, `1`);
3868	}
3869
3870	#endif /* CONFIG_JETSAM */
3871
3872	extern void vm_pressure_response(void);
3873
3874	static int
3875	memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation)
3876	{
3877	struct jetsam_thread_state *jetsam_thread = jetsam_current_thread();
3878
3879	if (interval_ms) {
3880	assert_wait_timeout(&jetsam_thread->memorystatus_wakeup, THREAD_UNINT, interval_ms, NSEC_PER_MSEC);
3881	} else {
3882	assert_wait(&jetsam_thread->memorystatus_wakeup, THREAD_UNINT);
3883	}
3884
3885	return thread_block(continuation);
3886	}
3887
3888	static boolean_t
3889	memorystatus_avail_pages_below_pressure(void)
3890	{
3891	#if CONFIG_EMBEDDED
3892	/*
3893	* Instead of CONFIG_EMBEDDED for these avail_pages routines, we should
3894	* key off of the system having dynamic swap support. With full swap support,
3895	* the system shouldn't really need to worry about various page thresholds.
3896	*/
3897	return (memorystatus_available_pages <= memorystatus_available_pages_pressure);
3898	#else /* CONFIG_EMBEDDED */
3899	return FALSE;
3900	#endif /* CONFIG_EMBEDDED */
3901	}
3902
3903	static boolean_t
3904	memorystatus_avail_pages_below_critical(void)
3905	{
3906	#if CONFIG_EMBEDDED
3907	return (memorystatus_available_pages <= memorystatus_available_pages_critical);
3908	#else /* CONFIG_EMBEDDED */
3909	return FALSE;
3910	#endif /* CONFIG_EMBEDDED */
3911	}
3912
3913	static boolean_t
3914	memorystatus_post_snapshot(int32_t priority, uint32_t cause)
3915	{
3916	#if CONFIG_EMBEDDED
3917	#pragma unused(cause)
3918	/*
3919	* Don't generate logs for steady-state idle-exit kills,
3920	* unless it is overridden for debug or by the device
3921	* tree.
3922	*/
3923
3924	return ((priority != JETSAM_PRIORITY_IDLE) \|\| memorystatus_idle_snapshot);
3925
3926	#else /* CONFIG_EMBEDDED */
3927	/*
3928	* Don't generate logs for steady-state idle-exit kills,
3929	* unless
3930	* - it is overridden for debug or by the device
3931	* tree.
3932	* OR
3933	* - the kill causes are important i.e. not kMemorystatusKilledIdleExit
3934	*/
3935
3936	boolean_t snapshot_eligible_kill_cause = (is_reason_thrashing(cause) \|\| is_reason_zone_map_exhaustion(cause));
3937	return ((priority != JETSAM_PRIORITY_IDLE) \|\| memorystatus_idle_snapshot \|\| snapshot_eligible_kill_cause);
3938	#endif /* CONFIG_EMBEDDED */
3939	}
3940
3941	static boolean_t
3942	memorystatus_action_needed(void)
3943	{
3944	#if CONFIG_EMBEDDED
3945	return (is_reason_thrashing(kill_under_pressure_cause) \|\|
3946	is_reason_zone_map_exhaustion(kill_under_pressure_cause) \|\|
3947	memorystatus_available_pages <= memorystatus_available_pages_pressure);
3948	#else /* CONFIG_EMBEDDED */
3949	return (is_reason_thrashing(kill_under_pressure_cause) \|\|
3950	is_reason_zone_map_exhaustion(kill_under_pressure_cause));
3951	#endif /* CONFIG_EMBEDDED */
3952	}
3953
3954	#if CONFIG_FREEZE
3955	extern void vm_swap_consider_defragmenting(int);
3956
3957	/*
3958	* This routine will _jetsam_ all frozen processes
3959	* and reclaim the swap space immediately.
3960	*
3961	* So freeze has to be DISABLED when we call this routine.
3962	*/
3963
3964	void
3965	memorystatus_disable_freeze(void)
3966	{
3967	memstat_bucket_t *bucket;
3968	int bucket_count = `0`, retries = `0`;
3969	boolean_t retval = FALSE, killed = FALSE;
3970	uint32_t errors = `0`, errors_over_prev_iteration = `0`;
3971	os_reason_t jetsam_reason = `0`;
3972	unsigned int band = `0`;
3973	proc_t p = PROC_NULL, next_p = PROC_NULL;
3974
3975	assert(memorystatus_freeze_enabled == FALSE);
3976
3977	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_DISK_SPACE_SHORTAGE);
3978	if (jetsam_reason == OS_REASON_NULL) {
3979	printf("memorystatus_disable_freeze: failed to allocate jetsam reason\n");
3980	}
3981
3982	/*
3983	* Let's relocate all frozen processes into band 8. Demoted frozen processes
3984	* are sitting in band 0 currently and it's possible to have a frozen process
3985	* in the FG band being actively used. We don't reset its frozen state when
3986	* it is resumed because it has state on disk.
3987	*
3988	* We choose to do this relocation rather than implement a new 'kill frozen'
3989	* process function for these reasons:
3990	* - duplication of code: too many kill functions exist and we need to rework them better.
3991	* - disk-space-shortage kills are rare
3992	* - not having the 'real' jetsam band at time of the this frozen kill won't preclude us
3993	* from answering any imp. questions re. jetsam policy/effectiveness.
3994	*
3995	* This is essentially what memorystatus_update_inactive_jetsam_priority_band() does while
3996	* avoiding the application of memory limits.
3997	*/
3998
3999	again:
4000	proc_list_lock();
4001
4002	band = JETSAM_PRIORITY_IDLE;
4003	p = PROC_NULL;
4004	next_p = PROC_NULL;
4005
4006	next_p = memorystatus_get_first_proc_locked(&band, TRUE);
4007	while (next_p) {
4008
4009	p = next_p;
4010	next_p = memorystatus_get_next_proc_locked(&band, p, TRUE);
4011
4012	if (p->p_memstat_effectivepriority > JETSAM_PRIORITY_FOREGROUND) {
4013	break;
4014	}
4015
4016	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == FALSE) {
4017	continue;
4018	}
4019
4020	if (p->p_memstat_state & P_MEMSTAT_ERROR) {
4021	p->p_memstat_state &= ~P_MEMSTAT_ERROR;
4022	}
4023
4024	if (p->p_memstat_effectivepriority == memorystatus_freeze_jetsam_band) {
4025	continue;
4026	}
4027
4028	/*
4029	* We explicitly add this flag here so the process looks like a normal
4030	* frozen process i.e. P_MEMSTAT_FROZEN and P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND.
4031	* We don't bother with assigning the 'active' memory
4032	* limits at this point because we are going to be killing it soon below.
4033	*/
4034	p->p_memstat_state \|= P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
4035	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
4036
4037	memorystatus_update_priority_locked(p, memorystatus_freeze_jetsam_band, FALSE, TRUE);
4038	}
4039
4040	bucket = &memstat_bucket[memorystatus_freeze_jetsam_band];
4041	bucket_count = bucket->count;
4042	proc_list_unlock();
4043
4044	/*
4045	* Bucket count is already stale at this point. But, we don't expect
4046	* freezing to continue since we have already disabled the freeze functionality.
4047	* However, an existing freeze might be in progress. So we might miss that process
4048	* in the first go-around. We hope to catch it in the next.
4049	*/
4050
4051	errors_over_prev_iteration = `0`;
4052	while (bucket_count) {
4053
4054	bucket_count--;
4055
4056	/*
4057	* memorystatus_kill_elevated_process() drops a reference,
4058	* so take another one so we can continue to use this exit reason
4059	* even after it returns.
4060	*/
4061
4062	os_reason_ref(jetsam_reason);
4063	retval = memorystatus_kill_elevated_process(
4064	kMemorystatusKilledDiskSpaceShortage,
4065	jetsam_reason,
4066	memorystatus_freeze_jetsam_band,
4067	`0`, / the iteration of aggressive jetsam..ignored here /
4068	&errors);
4069
4070	if (errors > `0`) {
4071	printf("memorystatus_disable_freeze: memorystatus_kill_elevated_process returned %d error(s)\n", errors);
4072	errors_over_prev_iteration += errors;
4073	errors = `0`;
4074	}
4075
4076	if (retval == `0`) {
4077	/*
4078	* No frozen processes left to kill.
4079	*/
4080	break;
4081	}
4082
4083	killed = TRUE;
4084	}
4085
4086	proc_list_lock();
4087
4088	if (memorystatus_frozen_count) {
4089	/*
4090	* A frozen process snuck in and so
4091	* go back around to kill it. That
4092	* process may have been resumed and
4093	* put into the FG band too. So we
4094	* have to do the relocation again.
4095	*/
4096	assert(memorystatus_freeze_enabled == FALSE);
4097
4098	retries++;
4099	if (retries < `3`) {
4100	proc_list_unlock();
4101	goto again;
4102	}
4103	#if DEVELOPMENT \|\| DEBUG
4104	panic("memorystatus_disable_freeze: Failed to kill all frozen processes, memorystatus_frozen_count = %d, errors = %d",
4105	memorystatus_frozen_count, errors_over_prev_iteration);
4106	#endif /* DEVELOPMENT \|\| DEBUG */
4107	}
4108	proc_list_unlock();
4109
4110	os_reason_free(jetsam_reason);
4111
4112	if (killed) {
4113
4114	vm_swap_consider_defragmenting(VM_SWAP_FLAGS_FORCE_DEFRAG \| VM_SWAP_FLAGS_FORCE_RECLAIM);
4115
4116	proc_list_lock();
4117	size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
4118	sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
4119	uint64_t timestamp_now = mach_absolute_time();
4120	memorystatus_jetsam_snapshot->notification_time = timestamp_now;
4121	memorystatus_jetsam_snapshot->js_gencount++;
4122	if (memorystatus_jetsam_snapshot_count > `0` && (memorystatus_jetsam_snapshot_last_timestamp == `0` \|\|
4123	timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
4124	proc_list_unlock();
4125	int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
4126	if (!ret) {
4127	proc_list_lock();
4128	memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
4129	proc_list_unlock();
4130	}
4131	} else {
4132	proc_list_unlock();
4133	}
4134	}
4135
4136	return;
4137	}
4138	#endif /* CONFIG_FREEZE */
4139
4140	static boolean_t
4141	memorystatus_act_on_hiwat_processes(uint32_t errors, uint32_t hwm_kill, boolean_t post_snapshot, __unused boolean_t is_critical)
4142	{
4143	boolean_t purged = FALSE;
4144	boolean_t killed = memorystatus_kill_hiwat_proc(errors, &purged);
4145
4146	if (killed) {
4147	hwm_kill = hwm_kill + `1`;
4148	*post_snapshot = TRUE;
4149	return TRUE;
4150	} else {
4151	if (purged == FALSE) {
4152	/ couldn't purge and couldn't kill /
4153	memorystatus_hwm_candidates = FALSE;
4154	}
4155	}
4156
4157	#if CONFIG_JETSAM
4158	/ No highwater processes to kill. Continue or stop for now? /
4159	if (!is_reason_thrashing(kill_under_pressure_cause) &&
4160	!is_reason_zone_map_exhaustion(kill_under_pressure_cause) &&
4161	(memorystatus_available_pages > memorystatus_available_pages_critical)) {
4162	/*
4163	* We are _not_ out of pressure but we are above the critical threshold and there's:
4164	* - no compressor thrashing
4165	* - enough zone memory
4166	* - no more HWM processes left.
4167	* For now, don't kill any other processes.
4168	*/
4169
4170	if (*hwm_kill == `0`) {
4171	memorystatus_thread_wasted_wakeup++;
4172	}
4173
4174	*is_critical = FALSE;
4175
4176	return TRUE;
4177	}
4178	#endif /* CONFIG_JETSAM */
4179
4180	return FALSE;
4181	}
4182
4183	static boolean_t
4184	memorystatus_act_aggressive(uint32_t cause, os_reason_t jetsam_reason, int jld_idle_kills, boolean_t corpse_list_purged, boolean_t *post_snapshot)
4185	{
4186	if (memorystatus_jld_enabled == TRUE) {
4187
4188	boolean_t killed;
4189	uint32_t errors = `0`;
4190
4191	/ Jetsam Loop Detection - locals /
4192	memstat_bucket_t *bucket;
4193	int jld_bucket_count = `0`;
4194	struct timeval jld_now_tstamp = {`0`,`0`};
4195	uint64_t jld_now_msecs = `0`;
4196	int elevated_bucket_count = `0`;
4197
4198	/ Jetsam Loop Detection - statics /
4199	static uint64_t jld_timestamp_msecs = `0`;
4200	static int jld_idle_kill_candidates = `0`; / Number of available processes in band 0,1 at start /
4201	static int jld_eval_aggressive_count = `0`; / Bumps the max priority in aggressive loop /
4202	static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
4203	/*
4204	* Jetsam Loop Detection: attempt to detect
4205	* rapid daemon relaunches in the lower bands.
4206	*/
4207
4208	microuptime(&jld_now_tstamp);
4209
4210	/*
4211	* Ignore usecs in this calculation.
4212	* msecs granularity is close enough.
4213	*/
4214	jld_now_msecs = (jld_now_tstamp.tv_sec * `1000`);
4215
4216	proc_list_lock();
4217	switch (jetsam_aging_policy) {
4218	case kJetsamAgingPolicyLegacy:
4219	bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
4220	jld_bucket_count = bucket->count;
4221	bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1];
4222	jld_bucket_count += bucket->count;
4223	break;
4224	case kJetsamAgingPolicySysProcsReclaimedFirst:
4225	case kJetsamAgingPolicyAppsReclaimedFirst:
4226	bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
4227	jld_bucket_count = bucket->count;
4228	bucket = &memstat_bucket[system_procs_aging_band];
4229	jld_bucket_count += bucket->count;
4230	bucket = &memstat_bucket[applications_aging_band];
4231	jld_bucket_count += bucket->count;
4232	break;
4233	case kJetsamAgingPolicyNone:
4234	default:
4235	bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
4236	jld_bucket_count = bucket->count;
4237	break;
4238	}
4239
4240	bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE];
4241	elevated_bucket_count = bucket->count;
4242
4243	proc_list_unlock();
4244
4245	/*
4246	* memorystatus_jld_eval_period_msecs is a tunable
4247	* memorystatus_jld_eval_aggressive_count is a tunable
4248	* memorystatus_jld_eval_aggressive_priority_band_max is a tunable
4249	*/
4250	if ( (jld_bucket_count == `0`) \|\|
4251	(jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) {
4252
4253	/*
4254	* Refresh evaluation parameters
4255	*/
4256	jld_timestamp_msecs = jld_now_msecs;
4257	jld_idle_kill_candidates = jld_bucket_count;
4258	*jld_idle_kills = `0`;
4259	jld_eval_aggressive_count = `0`;
4260	jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
4261	}
4262
4263	if (*jld_idle_kills > jld_idle_kill_candidates) {
4264	jld_eval_aggressive_count++;
4265
4266	#if DEVELOPMENT \|\| DEBUG
4267	printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n",
4268	jld_eval_aggressive_count,
4269	jld_timestamp_msecs,
4270	jld_now_msecs);
4271	printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n",
4272	jld_eval_aggressive_count,
4273	jld_idle_kill_candidates,
4274	*jld_idle_kills);
4275	#endif /* DEVELOPMENT \|\| DEBUG */
4276
4277	if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) &&
4278	(total_corpses_count() > `0`) && (*corpse_list_purged == FALSE)) {
4279	/*
4280	* If we reach this aggressive cycle, corpses might be causing memory pressure.
4281	* So, in an effort to avoid jetsams in the FG band, we will attempt to purge
4282	* corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT.
4283	*/
4284	task_purge_all_corpses();
4285	*corpse_list_purged = TRUE;
4286	}
4287	else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) {
4288	/*
4289	* Bump up the jetsam priority limit (eg: the bucket index)
4290	* Enforce bucket index sanity.
4291	*/
4292	if ((memorystatus_jld_eval_aggressive_priority_band_max < `0`) \|\|
4293	(memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) {
4294	/*
4295	* Do nothing. Stick with the default level.
4296	*/
4297	} else {
4298	jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max;
4299	}
4300	}
4301
4302	/ Visit elevated processes first /
4303	while (elevated_bucket_count) {
4304
4305	elevated_bucket_count--;
4306
4307	/*
4308	* memorystatus_kill_elevated_process() drops a reference,
4309	* so take another one so we can continue to use this exit reason
4310	* even after it returns.
4311	*/
4312
4313	os_reason_ref(jetsam_reason);
4314	killed = memorystatus_kill_elevated_process(
4315	cause,
4316	jetsam_reason,
4317	JETSAM_PRIORITY_ELEVATED_INACTIVE,
4318	jld_eval_aggressive_count,
4319	&errors);
4320
4321	if (killed) {
4322	*post_snapshot = TRUE;
4323	if (memorystatus_avail_pages_below_pressure()) {
4324	/*
4325	* Still under pressure.
4326	* Find another pinned processes.
4327	*/
4328	continue;
4329	} else {
4330	return TRUE;
4331	}
4332	} else {
4333	/*
4334	* No pinned processes left to kill.
4335	* Abandon elevated band.
4336	*/
4337	break;
4338	}
4339	}
4340
4341	/*
4342	* memorystatus_kill_top_process_aggressive() allocates its own
4343	* jetsam_reason so the kMemorystatusKilledProcThrashing cause
4344	* is consistent throughout the aggressive march.
4345	*/
4346	killed = memorystatus_kill_top_process_aggressive(
4347	kMemorystatusKilledProcThrashing,
4348	jld_eval_aggressive_count,
4349	jld_priority_band_max,
4350	&errors);
4351
4352	if (killed) {
4353	/ Always generate logs after aggressive kill /
4354	*post_snapshot = TRUE;
4355	*jld_idle_kills = `0`;
4356	return TRUE;
4357	}
4358	}
4359
4360	return FALSE;
4361	}
4362
4363	return FALSE;
4364	}
4365
4366
4367	static void
4368	memorystatus_thread(void *param __unused, wait_result_t wr __unused)
4369	{
4370	boolean_t post_snapshot = FALSE;
4371	uint32_t errors = `0`;
4372	uint32_t hwm_kill = `0`;
4373	boolean_t sort_flag = TRUE;
4374	boolean_t corpse_list_purged = FALSE;
4375	int jld_idle_kills = `0`;
4376	struct jetsam_thread_state *jetsam_thread = jetsam_current_thread();
4377
4378	if (jetsam_thread->inited == FALSE) {
4379	/*
4380	* It's the first time the thread has run, so just mark the thread as privileged and block.
4381	* This avoids a spurious pass with unset variables, as set out in <rdar://problem/9609402>.
4382	*/
4383
4384	char name[`32`];
4385	thread_wire(host_priv_self(), current_thread(), TRUE);
4386	snprintf(name, `32`, "VM_memorystatus_%d", jetsam_thread->index + `1`);
4387
4388	if (jetsam_thread->index == `0`) {
4389	if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
4390	thread_vm_bind_group_add();
4391	}
4392	}
4393	thread_set_thread_name(current_thread(), name);
4394	jetsam_thread->inited = TRUE;
4395	memorystatus_thread_block(`0`, memorystatus_thread);
4396	}
4397
4398	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) \| DBG_FUNC_START,
4399	memorystatus_available_pages, memorystatus_jld_enabled, memorystatus_jld_eval_period_msecs, memorystatus_jld_eval_aggressive_count,`0`);
4400
4401	/*
4402	* Jetsam aware version.
4403	*
4404	* The VM pressure notification thread is working it's way through clients in parallel.
4405	*
4406	* So, while the pressure notification thread is targeting processes in order of
4407	* increasing jetsam priority, we can hopefully reduce / stop it's work by killing
4408	* any processes that have exceeded their highwater mark.
4409	*
4410	* If we run out of HWM processes and our available pages drops below the critical threshold, then,
4411	* we target the least recently used process in order of increasing jetsam priority (exception: the FG band).
4412	*/
4413	while (memorystatus_action_needed()) {
4414	boolean_t killed;
4415	int32_t priority;
4416	uint32_t cause;
4417	uint64_t jetsam_reason_code = JETSAM_REASON_INVALID;
4418	os_reason_t jetsam_reason = OS_REASON_NULL;
4419
4420	cause = kill_under_pressure_cause;
4421	switch (cause) {
4422	case kMemorystatusKilledFCThrashing:
4423	jetsam_reason_code = JETSAM_REASON_MEMORY_FCTHRASHING;
4424	break;
4425	case kMemorystatusKilledVMCompressorThrashing:
4426	jetsam_reason_code = JETSAM_REASON_MEMORY_VMCOMPRESSOR_THRASHING;
4427	break;
4428	case kMemorystatusKilledVMCompressorSpaceShortage:
4429	jetsam_reason_code = JETSAM_REASON_MEMORY_VMCOMPRESSOR_SPACE_SHORTAGE;
4430	break;
4431	case kMemorystatusKilledZoneMapExhaustion:
4432	jetsam_reason_code = JETSAM_REASON_ZONE_MAP_EXHAUSTION;
4433	break;
4434	case kMemorystatusKilledVMPageShortage:
4435	/ falls through /
4436	default:
4437	jetsam_reason_code = JETSAM_REASON_MEMORY_VMPAGESHORTAGE;
4438	cause = kMemorystatusKilledVMPageShortage;
4439	break;
4440	}
4441
4442	/ Highwater /
4443	boolean_t is_critical = TRUE;
4444	if (memorystatus_act_on_hiwat_processes(&errors, &hwm_kill, &post_snapshot, &is_critical)) {
4445	if (is_critical == FALSE) {
4446	/*
4447	* For now, don't kill any other processes.
4448	*/
4449	break;
4450	} else {
4451	goto done;
4452	}
4453	}
4454
4455	jetsam_reason = os_reason_create(OS_REASON_JETSAM, jetsam_reason_code);
4456	if (jetsam_reason == OS_REASON_NULL) {
4457	printf("memorystatus_thread: failed to allocate jetsam reason\n");
4458	}
4459
4460	if (memorystatus_act_aggressive(cause, jetsam_reason, &jld_idle_kills, &corpse_list_purged, &post_snapshot)) {
4461	goto done;
4462	}
4463
4464	/*
4465	* memorystatus_kill_top_process() drops a reference,
4466	* so take another one so we can continue to use this exit reason
4467	* even after it returns
4468	*/
4469	os_reason_ref(jetsam_reason);
4470
4471	/ LRU /
4472	killed = memorystatus_kill_top_process(TRUE, sort_flag, cause, jetsam_reason, &priority, &errors);
4473	sort_flag = FALSE;
4474
4475	if (killed) {
4476	if (memorystatus_post_snapshot(priority, cause) == TRUE) {
4477
4478	post_snapshot = TRUE;
4479	}
4480
4481	/ Jetsam Loop Detection /
4482	if (memorystatus_jld_enabled == TRUE) {
4483	if ((priority == JETSAM_PRIORITY_IDLE) \|\| (priority == system_procs_aging_band) \|\| (priority == applications_aging_band)) {
4484	jld_idle_kills++;
4485	} else {
4486	/*
4487	* We've reached into bands beyond idle deferred.
4488	* We make no attempt to monitor them
4489	*/
4490	}
4491	}
4492
4493	if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count() > `0`) && (corpse_list_purged == FALSE)) {
4494	/*
4495	* If we have jetsammed a process in or above JETSAM_PRIORITY_UI_SUPPORT
4496	* then we attempt to relieve pressure by purging corpse memory.
4497	*/
4498	task_purge_all_corpses();
4499	corpse_list_purged = TRUE;
4500	}
4501	goto done;
4502	}
4503
4504	if (memorystatus_avail_pages_below_critical()) {
4505	/*
4506	* Still under pressure and unable to kill a process - purge corpse memory
4507	*/
4508	if (total_corpses_count() > `0`) {
4509	task_purge_all_corpses();
4510	corpse_list_purged = TRUE;
4511	}
4512
4513	if (memorystatus_avail_pages_below_critical()) {
4514	/*
4515	* Still under pressure and unable to kill a process - panic
4516	*/
4517	panic("memorystatus_jetsam_thread: no victim! available pages:%llu\n", (uint64_t)memorystatus_available_pages);
4518	}
4519	}
4520
4521	done:
4522
4523	/*
4524	* We do not want to over-kill when thrashing has been detected.
4525	* To avoid that, we reset the flag here and notify the
4526	* compressor.
4527	*/
4528	if (is_reason_thrashing(kill_under_pressure_cause)) {
4529	kill_under_pressure_cause = `0`;
4530	#if CONFIG_JETSAM
4531	vm_thrashing_jetsam_done();
4532	#endif /* CONFIG_JETSAM */
4533	} else if (is_reason_zone_map_exhaustion(kill_under_pressure_cause)) {
4534	kill_under_pressure_cause = `0`;
4535	}
4536
4537	os_reason_free(jetsam_reason);
4538	}
4539
4540	kill_under_pressure_cause = `0`;
4541
4542	if (errors) {
4543	memorystatus_clear_errors();
4544	}
4545
4546	if (post_snapshot) {
4547	proc_list_lock();
4548	size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
4549	sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
4550	uint64_t timestamp_now = mach_absolute_time();
4551	memorystatus_jetsam_snapshot->notification_time = timestamp_now;
4552	memorystatus_jetsam_snapshot->js_gencount++;
4553	if (memorystatus_jetsam_snapshot_count > `0` && (memorystatus_jetsam_snapshot_last_timestamp == `0` \|\|
4554	timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
4555	proc_list_unlock();
4556	int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
4557	if (!ret) {
4558	proc_list_lock();
4559	memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
4560	proc_list_unlock();
4561	}
4562	} else {
4563	proc_list_unlock();
4564	}
4565	}
4566
4567	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) \| DBG_FUNC_END,
4568	memorystatus_available_pages, `0`, `0`, `0`, `0`);
4569
4570	memorystatus_thread_block(`0`, memorystatus_thread);
4571	}
4572
4573	/*
4574	* Returns TRUE:
4575	* when an idle-exitable proc was killed
4576	* Returns FALSE:
4577	* when there are no more idle-exitable procs found
4578	* when the attempt to kill an idle-exitable proc failed
4579	*/
4580	boolean_t memorystatus_idle_exit_from_VM(void) {
4581
4582	/*
4583	* This routine should no longer be needed since we are
4584	* now using jetsam bands on all platforms and so will deal
4585	* with IDLE processes within the memorystatus thread itself.
4586	*
4587	* But we still use it because we observed that macos systems
4588	* started heavy compression/swapping with a bunch of
4589	* idle-exitable processes alive and doing nothing. We decided
4590	* to rather kill those processes than start swapping earlier.
4591	*/
4592
4593	return(kill_idle_exit_proc());
4594	}
4595
4596	/*
4597	* Callback invoked when allowable physical memory footprint exceeded
4598	* (dirty pages + IOKit mappings)
4599	*
4600	* This is invoked for both advisory, non-fatal per-task high watermarks,
4601	* as well as the fatal task memory limits.
4602	*/
4603	void
4604	memorystatus_on_ledger_footprint_exceeded(boolean_t warning, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal)
4605	{
4606	os_reason_t jetsam_reason = OS_REASON_NULL;
4607
4608	proc_t p = current_proc();
4609
4610	#if VM_PRESSURE_EVENTS
4611	if (warning == TRUE) {
4612	/*
4613	* This is a warning path which implies that the current process is close, but has
4614	* not yet exceeded its per-process memory limit.
4615	*/
4616	if (memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, FALSE / not exceeded /) != TRUE) {
4617	/ Print warning, since it's possible that task has not registered for pressure notifications /
4618	os_log(OS_LOG_DEFAULT, "memorystatus_on_ledger_footprint_exceeded: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid);
4619	}
4620	return;
4621	}
4622	#endif /* VM_PRESSURE_EVENTS */
4623
4624	if (memlimit_is_fatal) {
4625	/*
4626	* If this process has no high watermark or has a fatal task limit, then we have been invoked because the task
4627	* has violated either the system-wide per-task memory limit OR its own task limit.
4628	*/
4629	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_PERPROCESSLIMIT);
4630	if (jetsam_reason == NULL) {
4631	printf("task_exceeded footprint: failed to allocate jetsam reason\n");
4632	} else if (corpse_for_fatal_memkill != `0` && proc_send_synchronous_EXC_RESOURCE(p) == FALSE) {
4633	/ Set OS_REASON_FLAG_GENERATE_CRASH_REPORT to generate corpse /
4634	jetsam_reason->osr_flags \|= OS_REASON_FLAG_GENERATE_CRASH_REPORT;
4635	}
4636
4637	if (memorystatus_kill_process_sync(p->p_pid, kMemorystatusKilledPerProcessLimit, jetsam_reason) != TRUE) {
4638	printf("task_exceeded_footprint: failed to kill the current task (exiting?).\n");
4639	}
4640	} else {
4641	/*
4642	* HWM offender exists. Done without locks or synchronization.
4643	* See comment near its declaration for more details.
4644	*/
4645	memorystatus_hwm_candidates = TRUE;
4646
4647	#if VM_PRESSURE_EVENTS
4648	/*
4649	* The current process is not in the warning path.
4650	* This path implies the current process has exceeded a non-fatal (soft) memory limit.
4651	* Failure to send note is ignored here.
4652	*/
4653	(void)memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, TRUE / exceeded /);
4654
4655	#endif /* VM_PRESSURE_EVENTS */
4656	}
4657	}
4658
4659	void
4660	memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal)
4661	{
4662	proc_t p = current_proc();
4663
4664	/*
4665	* The limit violation is logged here, but only once per process per limit.
4666	* Soft memory limit is a non-fatal high-water-mark
4667	* Hard memory limit is a fatal custom-task-limit or system-wide per-task memory limit.
4668	*/
4669
4670	os_log_with_startup_serial(OS_LOG_DEFAULT, "EXC_RESOURCE -> %s[%d] exceeded mem limit: %s%s %d MB (%s)\n",
4671	(*p->p_name ? p->p_name : "unknown"), p->p_pid, (memlimit_is_active ? "Active" : "Inactive"),
4672	(memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb,
4673	(memlimit_is_fatal ? "fatal" : "non-fatal"));
4674
4675	return;
4676	}
4677
4678
4679	/*
4680	* Description:
4681	* Evaluates process state to determine which limit
4682	* should be applied (active vs. inactive limit).
4683	*
4684	* Processes that have the 'elevated inactive jetsam band' attribute
4685	* are first evaluated based on their current priority band.
4686	* presently elevated ==> active
4687	*
4688	* Processes that opt into dirty tracking are evaluated
4689	* based on clean vs dirty state.
4690	* dirty ==> active
4691	* clean ==> inactive
4692	*
4693	* Process that do not opt into dirty tracking are
4694	* evalulated based on priority level.
4695	* Foreground or above ==> active
4696	* Below Foreground ==> inactive
4697	*
4698	* Return: TRUE if active
4699	* False if inactive
4700	*/
4701
4702	static boolean_t
4703	proc_jetsam_state_is_active_locked(proc_t p) {
4704
4705	if ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) &&
4706	(p->p_memstat_effectivepriority == JETSAM_PRIORITY_ELEVATED_INACTIVE)) {
4707	/*
4708	* process has the 'elevated inactive jetsam band' attribute
4709	* and process is present in the elevated band
4710	* implies active state
4711	*/
4712	return TRUE;
4713	} else if (p->p_memstat_dirty & P_DIRTY_TRACK) {
4714	/*
4715	* process has opted into dirty tracking
4716	* active state is based on dirty vs. clean
4717	*/
4718	if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
4719	/*
4720	* process is dirty
4721	* implies active state
4722	*/
4723	return TRUE;
4724	} else {
4725	/*
4726	* process is clean
4727	* implies inactive state
4728	*/
4729	return FALSE;
4730	}
4731	} else if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
4732	/*
4733	* process is Foreground or higher
4734	* implies active state
4735	*/
4736	return TRUE;
4737	} else {
4738	/*
4739	* process found below Foreground
4740	* implies inactive state
4741	*/
4742	return FALSE;
4743	}
4744	}
4745
4746	static boolean_t
4747	memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) {
4748	boolean_t res;
4749
4750	uint32_t errors = `0`;
4751
4752	if (victim_pid == -`1`) {
4753	/ No pid, so kill first process /
4754	res = memorystatus_kill_top_process(TRUE, TRUE, cause, jetsam_reason, NULL, &errors);
4755	} else {
4756	res = memorystatus_kill_specific_process(victim_pid, cause, jetsam_reason);
4757	}
4758
4759	if (errors) {
4760	memorystatus_clear_errors();
4761	}
4762
4763	if (res == TRUE) {
4764	/ Fire off snapshot notification /
4765	proc_list_lock();
4766	size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
4767	sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_count;
4768	uint64_t timestamp_now = mach_absolute_time();
4769	memorystatus_jetsam_snapshot->notification_time = timestamp_now;
4770	if (memorystatus_jetsam_snapshot_count > `0` && (memorystatus_jetsam_snapshot_last_timestamp == `0` \|\|
4771	timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
4772	proc_list_unlock();
4773	int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
4774	if (!ret) {
4775	proc_list_lock();
4776	memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
4777	proc_list_unlock();
4778	}
4779	} else {
4780	proc_list_unlock();
4781	}
4782	}
4783
4784	return res;
4785	}
4786
4787	/*
4788	* Jetsam a specific process.
4789	*/
4790	static boolean_t
4791	memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) {
4792	boolean_t killed;
4793	proc_t p;
4794	uint64_t killtime = `0`;
4795	clock_sec_t tv_sec;
4796	clock_usec_t tv_usec;
4797	uint32_t tv_msec;
4798
4799	/ TODO - add a victim queue and push this into the main jetsam thread /
4800
4801	p = proc_find(victim_pid);
4802	if (!p) {
4803	os_reason_free(jetsam_reason);
4804	return FALSE;
4805	}
4806
4807	proc_list_lock();
4808
4809	if (memorystatus_jetsam_snapshot_count == `0`) {
4810	memorystatus_init_jetsam_snapshot_locked(NULL,`0`);
4811	}
4812
4813	killtime = mach_absolute_time();
4814	absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
4815	tv_msec = tv_usec / `1000`;
4816
4817	memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
4818
4819	proc_list_unlock();
4820
4821	os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_specific_process pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
4822	(unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "unknown"),
4823	memorystatus_kill_cause_name[cause], p->p_memstat_effectivepriority, (uint64_t)memorystatus_available_pages);
4824
4825	killed = memorystatus_do_kill(p, cause, jetsam_reason);
4826	proc_rele(p);
4827
4828	return killed;
4829	}
4830
4831
4832	/*
4833	* Toggle the P_MEMSTAT_TERMINATED state.
4834	* Takes the proc_list_lock.
4835	*/
4836	void
4837	proc_memstat_terminated(proc_t p, boolean_t set)
4838	{
4839	#if DEVELOPMENT \|\| DEBUG
4840	if (p) {
4841	proc_list_lock();
4842	if (set == TRUE) {
4843	p->p_memstat_state \|= P_MEMSTAT_TERMINATED;
4844	} else {
4845	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
4846	}
4847	proc_list_unlock();
4848	}
4849	#else
4850	#pragma unused(p, set)
4851	/*
4852	* do nothing
4853	*/
4854	#endif /* DEVELOPMENT \|\| DEBUG */
4855	return;
4856	}
4857
4858
4859	#if CONFIG_JETSAM
4860	/*
4861	* This is invoked when cpulimits have been exceeded while in fatal mode.
4862	* The jetsam_flags do not apply as those are for memory related kills.
4863	* We call this routine so that the offending process is killed with
4864	* a non-zero exit status.
4865	*/
4866	void
4867	jetsam_on_ledger_cpulimit_exceeded(void)
4868	{
4869	int retval = `0`;
4870	int jetsam_flags = `0`; / make it obvious /
4871	proc_t p = current_proc();
4872	os_reason_t jetsam_reason = OS_REASON_NULL;
4873
4874	printf("task_exceeded_cpulimit: killing pid %d [%s]\n",
4875	p->p_pid, (*p->p_name ? p->p_name : "(unknown)"));
4876
4877	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_CPULIMIT);
4878	if (jetsam_reason == OS_REASON_NULL) {
4879	printf("task_exceeded_cpulimit: unable to allocate memory for jetsam reason\n");
4880	}
4881
4882	retval = jetsam_do_kill(p, jetsam_flags, jetsam_reason);
4883
4884	if (retval) {
4885	printf("task_exceeded_cpulimit: failed to kill current task (exiting?).\n");
4886	}
4887	}
4888
4889	#endif /* CONFIG_JETSAM */
4890
4891	static void
4892	memorystatus_get_task_memory_region_count(task_t task, uint64_t *count)
4893	{
4894	assert(task);
4895	assert(count);
4896
4897	*count = get_task_memory_region_count(task);
4898	}
4899
4900
4901	#define MEMORYSTATUS_VM_MAP_FORK_ALLOWED 0x100000000
4902	#define MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED 0x200000000
4903
4904	#if DEVELOPMENT \|\| DEBUG
4905
4906	/*
4907	* Sysctl only used to test memorystatus_allowed_vm_map_fork() path.
4908	* set a new pidwatch value
4909	* or
4910	* get the current pidwatch value
4911	*
4912	* The pidwatch_val starts out with a PID to watch for in the map_fork path.
4913	* Its value is:
4914	* - OR'd with MEMORYSTATUS_VM_MAP_FORK_ALLOWED if we allow the map_fork.
4915	* - OR'd with MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED if we disallow the map_fork.
4916	* - set to -1ull if the map_fork() is aborted for other reasons.
4917	*/
4918
4919	uint64_t memorystatus_vm_map_fork_pidwatch_val = `0`;
4920
4921	static int sysctl_memorystatus_vm_map_fork_pidwatch SYSCTL_HANDLER_ARGS {
4922	#pragma unused(oidp, arg1, arg2)
4923
4924	uint64_t new_value = `0`;
4925	uint64_t old_value = `0`;
4926	int error = `0`;
4927
4928	/*
4929	* The pid is held in the low 32 bits.
4930	* The 'allowed' flags are in the upper 32 bits.
4931	*/
4932	old_value = memorystatus_vm_map_fork_pidwatch_val;
4933
4934	error = sysctl_io_number(req, old_value, sizeof(old_value), &new_value, NULL);
4935
4936	if (error \|\| !req->newptr) {
4937	/*
4938	* No new value passed in.
4939	*/
4940	return(error);
4941	}
4942
4943	/*
4944	* A new pid was passed in via req->newptr.
4945	* Ignore any attempt to set the higher order bits.
4946	*/
4947	memorystatus_vm_map_fork_pidwatch_val = new_value & `0xFFFFFFFF`;
4948	printf("memorystatus: pidwatch old_value = 0x%llx, new_value = 0x%llx \n", old_value, new_value);
4949
4950	return(error);
4951	}
4952
4953	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_map_fork_pidwatch, CTLTYPE_QUAD \| CTLFLAG_RW \| CTLFLAG_LOCKED\| CTLFLAG_MASKED,
4954	`0`, `0`, sysctl_memorystatus_vm_map_fork_pidwatch, "Q", "get/set pid watched for in vm_map_fork");
4955
4956
4957	/*
4958	* Record if a watched process fails to qualify for a vm_map_fork().
4959	*/
4960	void
4961	memorystatus_abort_vm_map_fork(task_t task)
4962	{
4963	if (memorystatus_vm_map_fork_pidwatch_val != `0`) {
4964	proc_t p = get_bsdtask_info(task);
4965	if (p != NULL && memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid) {
4966	memorystatus_vm_map_fork_pidwatch_val = -`1ull`;
4967	}
4968	}
4969	}
4970
4971	static void
4972	set_vm_map_fork_pidwatch(task_t task, uint64_t x)
4973	{
4974	if (memorystatus_vm_map_fork_pidwatch_val != `0`) {
4975	proc_t p = get_bsdtask_info(task);
4976	if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) {
4977	memorystatus_vm_map_fork_pidwatch_val \|= x;
4978	}
4979	}
4980	}
4981
4982	#else /* DEVELOPMENT \|\| DEBUG */
4983
4984
4985	static void
4986	set_vm_map_fork_pidwatch(task_t task, uint64_t x)
4987	{
4988	#pragma unused(task)
4989	#pragma unused(x)
4990	}
4991
4992	#endif /* DEVELOPMENT \|\| DEBUG */
4993
4994	/*
4995	* Called during EXC_RESOURCE handling when a process exceeds a soft
4996	* memory limit. This is the corpse fork path and here we decide if
4997	* vm_map_fork will be allowed when creating the corpse.
4998	* The task being considered is suspended.
4999	*
5000	* By default, a vm_map_fork is allowed to proceed.
5001	*
5002	* A few simple policy assumptions:
5003	* Desktop platform is not considered in this path.
5004	* The vm_map_fork is always allowed.
5005	*
5006	* If the device has a zero system-wide task limit,
5007	* then the vm_map_fork is allowed.
5008	*
5009	* And if a process's memory footprint calculates less
5010	* than or equal to half of the system-wide task limit,
5011	* then the vm_map_fork is allowed. This calculation
5012	* is based on the assumption that a process can
5013	* munch memory up to the system-wide task limit.
5014	*/
5015	boolean_t
5016	memorystatus_allowed_vm_map_fork(task_t task)
5017	{
5018	boolean_t is_allowed = TRUE; / default /
5019
5020	#if CONFIG_EMBEDDED
5021
5022	uint64_t footprint_in_bytes;
5023	uint64_t max_allowed_bytes;
5024
5025	if (max_task_footprint_mb == `0`) {
5026	set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_ALLOWED);
5027	return (is_allowed);
5028	}
5029
5030	footprint_in_bytes = get_task_phys_footprint(task);
5031
5032	/*
5033	* Maximum is 1/4 of the system-wide task limit.
5034	*/
5035	max_allowed_bytes = ((uint64_t)max_task_footprint_mb * `1024` * `1024`) >> `2`;
5036
5037	if (footprint_in_bytes > max_allowed_bytes) {
5038	printf("memorystatus disallowed vm_map_fork %lld %lld\n", footprint_in_bytes, max_allowed_bytes);
5039	set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED);
5040	return (!is_allowed);
5041	}
5042	#endif /* CONFIG_EMBEDDED */
5043
5044	set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_ALLOWED);
5045	return (is_allowed);
5046
5047	}
5048
5049	static void
5050	memorystatus_get_task_page_counts(task_t task, uint32_t footprint, uint32_t max_footprint_lifetime, uint32_t *purgeable_pages)
5051	{
5052	assert(task);
5053	assert(footprint);
5054
5055	uint64_t pages;
5056
5057	pages = (get_task_phys_footprint(task) / PAGE_SIZE_64);
5058	assert(((uint32_t)pages) == pages);
5059	*footprint = (uint32_t)pages;
5060
5061	if (max_footprint_lifetime) {
5062	pages = (get_task_resident_max(task) / PAGE_SIZE_64);
5063	assert(((uint32_t)pages) == pages);
5064	*max_footprint_lifetime = (uint32_t)pages;
5065	}
5066	if (purgeable_pages) {
5067	pages = (get_task_purgeable_size(task) / PAGE_SIZE_64);
5068	assert(((uint32_t)pages) == pages);
5069	*purgeable_pages = (uint32_t)pages;
5070	}
5071	}
5072
5073	static void
5074	memorystatus_get_task_phys_footprint_page_counts(task_t task,
5075	uint64_t internal_pages, uint64_t internal_compressed_pages,
5076	uint64_t purgeable_nonvolatile_pages, uint64_t purgeable_nonvolatile_compressed_pages,
5077	uint64_t alternate_accounting_pages, uint64_t alternate_accounting_compressed_pages,
5078	uint64_t iokit_mapped_pages, uint64_t page_table_pages)
5079	{
5080	assert(task);
5081
5082	if (internal_pages) {
5083	*internal_pages = (get_task_internal(task) / PAGE_SIZE_64);
5084	}
5085
5086	if (internal_compressed_pages) {
5087	*internal_compressed_pages = (get_task_internal_compressed(task) / PAGE_SIZE_64);
5088	}
5089
5090	if (purgeable_nonvolatile_pages) {
5091	*purgeable_nonvolatile_pages = (get_task_purgeable_nonvolatile(task) / PAGE_SIZE_64);
5092	}
5093
5094	if (purgeable_nonvolatile_compressed_pages) {
5095	*purgeable_nonvolatile_compressed_pages = (get_task_purgeable_nonvolatile_compressed(task) / PAGE_SIZE_64);
5096	}
5097
5098	if (alternate_accounting_pages) {
5099	*alternate_accounting_pages = (get_task_alternate_accounting(task) / PAGE_SIZE_64);
5100	}
5101
5102	if (alternate_accounting_compressed_pages) {
5103	*alternate_accounting_compressed_pages = (get_task_alternate_accounting_compressed(task) / PAGE_SIZE_64);
5104	}
5105
5106	if (iokit_mapped_pages) {
5107	*iokit_mapped_pages = (get_task_iokit_mapped(task) / PAGE_SIZE_64);
5108	}
5109
5110	if (page_table_pages) {
5111	*page_table_pages = (get_task_page_table(task) / PAGE_SIZE_64);
5112	}
5113	}
5114
5115	/*
5116	* This routine only acts on the global jetsam event snapshot.
5117	* Updating the process's entry can race when the memorystatus_thread
5118	* has chosen to kill a process that is racing to exit on another core.
5119	*/
5120	static void
5121	memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause, uint64_t killtime)
5122	{
5123	memorystatus_jetsam_snapshot_entry_t *entry = NULL;
5124	memorystatus_jetsam_snapshot_t *snapshot = NULL;
5125	memorystatus_jetsam_snapshot_entry_t *snapshot_list = NULL;
5126
5127	unsigned int i;
5128
5129	LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_OWNED);
5130
5131	if (memorystatus_jetsam_snapshot_count == `0`) {
5132	/*
5133	* No active snapshot.
5134	* Nothing to do.
5135	*/
5136	return;
5137	}
5138
5139	/*
5140	* Sanity check as this routine should only be called
5141	* from a jetsam kill path.
5142	*/
5143	assert(kill_cause != `0` && killtime != `0`);
5144
5145	snapshot = memorystatus_jetsam_snapshot;
5146	snapshot_list = memorystatus_jetsam_snapshot->entries;
5147
5148	for (i = `0`; i < memorystatus_jetsam_snapshot_count; i++) {
5149	if (snapshot_list[i].pid == p->p_pid) {
5150
5151	entry = &snapshot_list[i];
5152
5153	if (entry->killed \|\| entry->jse_killtime) {
5154	/*
5155	* We apparently raced on the exit path
5156	* for this process, as it's snapshot entry
5157	* has already recorded a kill.
5158	*/
5159	assert(entry->killed && entry->jse_killtime);
5160	break;
5161	}
5162
5163	/*
5164	* Update the entry we just found in the snapshot.
5165	*/
5166
5167	entry->killed = kill_cause;
5168	entry->jse_killtime = killtime;
5169	entry->jse_gencount = snapshot->js_gencount;
5170	entry->jse_idle_delta = p->p_memstat_idle_delta;
5171	#if CONFIG_FREEZE
5172	entry->jse_thaw_count = p->p_memstat_thaw_count;
5173	#else /* CONFIG_FREEZE */
5174	entry->jse_thaw_count = `0`;
5175	#endif /* CONFIG_FREEZE */
5176
5177	/*
5178	* If a process has moved between bands since snapshot was
5179	* initialized, then likely these fields changed too.
5180	*/
5181	if (entry->priority != p->p_memstat_effectivepriority) {
5182
5183	strlcpy(entry->name, p->p_name, sizeof(entry->name));
5184	entry->priority = p->p_memstat_effectivepriority;
5185	entry->state = memorystatus_build_state(p);
5186	entry->user_data = p->p_memstat_userdata;
5187	entry->fds = p->p_fd->fd_nfiles;
5188	}
5189
5190	/*
5191	* Always update the page counts on a kill.
5192	*/
5193
5194	uint32_t pages = `0`;
5195	uint32_t max_pages_lifetime = `0`;
5196	uint32_t purgeable_pages = `0`;
5197
5198	memorystatus_get_task_page_counts(p->task, &pages, &max_pages_lifetime, &purgeable_pages);
5199	entry->pages = (uint64_t)pages;
5200	entry->max_pages_lifetime = (uint64_t)max_pages_lifetime;
5201	entry->purgeable_pages = (uint64_t)purgeable_pages;
5202
5203	uint64_t internal_pages = `0`;
5204	uint64_t internal_compressed_pages = `0`;
5205	uint64_t purgeable_nonvolatile_pages = `0`;
5206	uint64_t purgeable_nonvolatile_compressed_pages = `0`;
5207	uint64_t alternate_accounting_pages = `0`;
5208	uint64_t alternate_accounting_compressed_pages = `0`;
5209	uint64_t iokit_mapped_pages = `0`;
5210	uint64_t page_table_pages = `0`;
5211
5212	memorystatus_get_task_phys_footprint_page_counts(p->task, &internal_pages, &internal_compressed_pages,
5213	&purgeable_nonvolatile_pages, &purgeable_nonvolatile_compressed_pages,
5214	&alternate_accounting_pages, &alternate_accounting_compressed_pages,
5215	&iokit_mapped_pages, &page_table_pages);
5216
5217	entry->jse_internal_pages = internal_pages;
5218	entry->jse_internal_compressed_pages = internal_compressed_pages;
5219	entry->jse_purgeable_nonvolatile_pages = purgeable_nonvolatile_pages;
5220	entry->jse_purgeable_nonvolatile_compressed_pages = purgeable_nonvolatile_compressed_pages;
5221	entry->jse_alternate_accounting_pages = alternate_accounting_pages;
5222	entry->jse_alternate_accounting_compressed_pages = alternate_accounting_compressed_pages;
5223	entry->jse_iokit_mapped_pages = iokit_mapped_pages;
5224	entry->jse_page_table_pages = page_table_pages;
5225
5226	uint64_t region_count = `0`;
5227	memorystatus_get_task_memory_region_count(p->task, &region_count);
5228	entry->jse_memory_region_count = region_count;
5229
5230	goto exit;
5231	}
5232	}
5233
5234	if (entry == NULL) {
5235	/*
5236	* The entry was not found in the snapshot, so the process must have
5237	* launched after the snapshot was initialized.
5238	* Let's try to append the new entry.
5239	*/
5240	if (memorystatus_jetsam_snapshot_count < memorystatus_jetsam_snapshot_max) {
5241	/*
5242	* A populated snapshot buffer exists
5243	* and there is room to init a new entry.
5244	*/
5245	assert(memorystatus_jetsam_snapshot_count == snapshot->entry_count);
5246
5247	unsigned int next = memorystatus_jetsam_snapshot_count;
5248
5249	if(memorystatus_init_jetsam_snapshot_entry_locked(p, &snapshot_list[next], (snapshot->js_gencount)) == TRUE) {
5250
5251	entry = &snapshot_list[next];
5252	entry->killed = kill_cause;
5253	entry->jse_killtime = killtime;
5254
5255	snapshot->entry_count = ++next;
5256	memorystatus_jetsam_snapshot_count = next;
5257
5258	if (memorystatus_jetsam_snapshot_count >= memorystatus_jetsam_snapshot_max) {
5259	/*
5260	* We just used the last slot in the snapshot buffer.
5261	* We only want to log it once... so we do it here
5262	* when we notice we've hit the max.
5263	*/
5264	printf("memorystatus: WARNING snapshot buffer is full, count %d\n",
5265	memorystatus_jetsam_snapshot_count);
5266	}
5267	}
5268	}
5269	}
5270
5271	exit:
5272	if (entry == NULL) {
5273	/*
5274	* If we reach here, the snapshot buffer could not be updated.
5275	* Most likely, the buffer is full, in which case we would have
5276	* logged a warning in the previous call.
5277	*
5278	* For now, we will stop appending snapshot entries.
5279	* When the buffer is consumed, the snapshot state will reset.
5280	*/
5281
5282	MEMORYSTATUS_DEBUG(`4`, "memorystatus_update_jetsam_snapshot_entry_locked: failed to update pid %d, priority %d, count %d\n",
5283	p->p_pid, p->p_memstat_effectivepriority, memorystatus_jetsam_snapshot_count);
5284	}
5285
5286	return;
5287	}
5288
5289	#if CONFIG_JETSAM
5290	void memorystatus_pages_update(unsigned int pages_avail)
5291	{
5292	memorystatus_available_pages = pages_avail;
5293
5294	#if VM_PRESSURE_EVENTS
5295	/*
5296	* Since memorystatus_available_pages changes, we should
5297	* re-evaluate the pressure levels on the system and
5298	* check if we need to wake the pressure thread.
5299	* We also update memorystatus_level in that routine.
5300	*/
5301	vm_pressure_response();
5302
5303	if (memorystatus_available_pages <= memorystatus_available_pages_pressure) {
5304
5305	if (memorystatus_hwm_candidates \|\| (memorystatus_available_pages <= memorystatus_available_pages_critical)) {
5306	memorystatus_thread_wake();
5307	}
5308	}
5309	#if CONFIG_FREEZE
5310	/*
5311	* We can't grab the freezer_mutex here even though that synchronization would be correct to inspect
5312	* the # of frozen processes and wakeup the freezer thread. Reason being that we come here into this
5313	* code with (possibly) the page-queue locks held and preemption disabled. So trying to grab a mutex here
5314	* will result in the "mutex with preemption disabled" panic.
5315	*/
5316
5317	if (memorystatus_freeze_thread_should_run() == TRUE) {
5318	/*
5319	* The freezer thread is usually woken up by some user-space call i.e. pid_hibernate(any process).
5320	* That trigger isn't invoked often enough and so we are enabling this explicit wakeup here.
5321	*/
5322	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
5323	thread_wakeup((event_t)&memorystatus_freeze_wakeup);
5324	}
5325	}
5326	#endif /* CONFIG_FREEZE */
5327
5328	#else /* VM_PRESSURE_EVENTS */
5329
5330	boolean_t critical, delta;
5331
5332	if (!memorystatus_delta) {
5333	return;
5334	}
5335
5336	critical = (pages_avail < memorystatus_available_pages_critical) ? TRUE : FALSE;
5337	delta = ((pages_avail >= (memorystatus_available_pages + memorystatus_delta))
5338	\|\| (memorystatus_available_pages >= (pages_avail + memorystatus_delta))) ? TRUE : FALSE;
5339
5340	if (critical \|\| delta) {
5341	unsigned int total_pages;
5342
5343	total_pages = (unsigned int) atop_64(max_mem);
5344	#if CONFIG_SECLUDED_MEMORY
5345	total_pages -= vm_page_secluded_count;
5346	#endif /* CONFIG_SECLUDED_MEMORY */
5347	memorystatus_level = memorystatus_available_pages * `100` / total_pages;
5348	memorystatus_thread_wake();
5349	}
5350	#endif /* VM_PRESSURE_EVENTS */
5351	}
5352	#endif /* CONFIG_JETSAM */
5353
5354	static boolean_t
5355	memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount)
5356	{
5357	clock_sec_t tv_sec;
5358	clock_usec_t tv_usec;
5359	uint32_t pages = `0`;
5360	uint32_t max_pages_lifetime = `0`;
5361	uint32_t purgeable_pages = `0`;
5362	uint64_t internal_pages = `0`;
5363	uint64_t internal_compressed_pages = `0`;
5364	uint64_t purgeable_nonvolatile_pages = `0`;
5365	uint64_t purgeable_nonvolatile_compressed_pages = `0`;
5366	uint64_t alternate_accounting_pages = `0`;
5367	uint64_t alternate_accounting_compressed_pages = `0`;
5368	uint64_t iokit_mapped_pages = `0`;
5369	uint64_t page_table_pages =`0`;
5370	uint64_t region_count = `0`;
5371	uint64_t cids[COALITION_NUM_TYPES];
5372
5373	memset(entry, `0`, sizeof(memorystatus_jetsam_snapshot_entry_t));
5374
5375	entry->pid = p->p_pid;
5376	strlcpy(&entry->name[`0`], p->p_name, sizeof(entry->name));
5377	entry->priority = p->p_memstat_effectivepriority;
5378
5379	memorystatus_get_task_page_counts(p->task, &pages, &max_pages_lifetime, &purgeable_pages);
5380	entry->pages = (uint64_t)pages;
5381	entry->max_pages_lifetime = (uint64_t)max_pages_lifetime;
5382	entry->purgeable_pages = (uint64_t)purgeable_pages;
5383
5384	memorystatus_get_task_phys_footprint_page_counts(p->task, &internal_pages, &internal_compressed_pages,
5385	&purgeable_nonvolatile_pages, &purgeable_nonvolatile_compressed_pages,
5386	&alternate_accounting_pages, &alternate_accounting_compressed_pages,
5387	&iokit_mapped_pages, &page_table_pages);
5388
5389	entry->jse_internal_pages = internal_pages;
5390	entry->jse_internal_compressed_pages = internal_compressed_pages;
5391	entry->jse_purgeable_nonvolatile_pages = purgeable_nonvolatile_pages;
5392	entry->jse_purgeable_nonvolatile_compressed_pages = purgeable_nonvolatile_compressed_pages;
5393	entry->jse_alternate_accounting_pages = alternate_accounting_pages;
5394	entry->jse_alternate_accounting_compressed_pages = alternate_accounting_compressed_pages;
5395	entry->jse_iokit_mapped_pages = iokit_mapped_pages;
5396	entry->jse_page_table_pages = page_table_pages;
5397
5398	memorystatus_get_task_memory_region_count(p->task, &region_count);
5399	entry->jse_memory_region_count = region_count;
5400
5401	entry->state = memorystatus_build_state(p);
5402	entry->user_data = p->p_memstat_userdata;
5403	memcpy(&entry->uuid[`0`], &p->p_uuid[`0`], sizeof(p->p_uuid));
5404	entry->fds = p->p_fd->fd_nfiles;
5405
5406	absolutetime_to_microtime(get_task_cpu_time(p->task), &tv_sec, &tv_usec);
5407	entry->cpu_time.tv_sec = (int64_t)tv_sec;
5408	entry->cpu_time.tv_usec = (int64_t)tv_usec;
5409
5410	assert(p->p_stats != NULL);
5411	entry->jse_starttime = p->p_stats->ps_start; / abstime process started /
5412	entry->jse_killtime = `0`; / abstime jetsam chose to kill process /
5413	entry->killed = `0`; / the jetsam kill cause /
5414	entry->jse_gencount = gencount; / indicates a pass through jetsam thread, when process was targeted to be killed /
5415
5416	entry->jse_idle_delta = p->p_memstat_idle_delta; / Most recent timespan spent in idle-band /
5417
5418	#if CONFIG_FREEZE
5419	entry->jse_thaw_count = p->p_memstat_thaw_count;
5420	#else /* CONFIG_FREEZE */
5421	entry->jse_thaw_count = `0`;
5422	#endif /* CONFIG_FREEZE */
5423
5424	proc_coalitionids(p, cids);
5425	entry->jse_coalition_jetsam_id = cids[COALITION_TYPE_JETSAM];
5426
5427	return TRUE;
5428	}
5429
5430	static void
5431	memorystatus_init_snapshot_vmstats(memorystatus_jetsam_snapshot_t *snapshot)
5432	{
5433	kern_return_t kr = KERN_SUCCESS;
5434	mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
5435	vm_statistics64_data_t vm_stat;
5436
5437	if ((kr = host_statistics64(host_self(), HOST_VM_INFO64, (host_info64_t)&vm_stat, &count)) != KERN_SUCCESS) {
5438	printf("memorystatus_init_jetsam_snapshot_stats: host_statistics64 failed with %d\n", kr);
5439	memset(&snapshot->stats, `0`, sizeof(snapshot->stats));
5440	} else {
5441	snapshot->stats.free_pages = vm_stat.free_count;
5442	snapshot->stats.active_pages = vm_stat.active_count;
5443	snapshot->stats.inactive_pages = vm_stat.inactive_count;
5444	snapshot->stats.throttled_pages = vm_stat.throttled_count;
5445	snapshot->stats.purgeable_pages = vm_stat.purgeable_count;
5446	snapshot->stats.wired_pages = vm_stat.wire_count;
5447
5448	snapshot->stats.speculative_pages = vm_stat.speculative_count;
5449	snapshot->stats.filebacked_pages = vm_stat.external_page_count;
5450	snapshot->stats.anonymous_pages = vm_stat.internal_page_count;
5451	snapshot->stats.compressions = vm_stat.compressions;
5452	snapshot->stats.decompressions = vm_stat.decompressions;
5453	snapshot->stats.compressor_pages = vm_stat.compressor_page_count;
5454	snapshot->stats.total_uncompressed_pages_in_compressor = vm_stat.total_uncompressed_pages_in_compressor;
5455	}
5456
5457	get_zone_map_size(&snapshot->stats.zone_map_size, &snapshot->stats.zone_map_capacity);
5458	get_largest_zone_info(snapshot->stats.largest_zone_name, sizeof(snapshot->stats.largest_zone_name),
5459	&snapshot->stats.largest_zone_size);
5460	}
5461
5462	/*
5463	* Collect vm statistics at boot.
5464	* Called only once (see kern_exec.c)
5465	* Data can be consumed at any time.
5466	*/
5467	void
5468	memorystatus_init_at_boot_snapshot() {
5469	memorystatus_init_snapshot_vmstats(&memorystatus_at_boot_snapshot);
5470	memorystatus_at_boot_snapshot.entry_count = `0`;
5471	memorystatus_at_boot_snapshot.notification_time = `0`; / updated when consumed /
5472	memorystatus_at_boot_snapshot.snapshot_time = mach_absolute_time();
5473	}
5474
5475	static void
5476	memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count )
5477	{
5478	proc_t p, next_p;
5479	unsigned int b = `0`, i = `0`;
5480
5481	memorystatus_jetsam_snapshot_t *snapshot = NULL;
5482	memorystatus_jetsam_snapshot_entry_t *snapshot_list = NULL;
5483	unsigned int snapshot_max = `0`;
5484
5485	LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_OWNED);
5486
5487	if (od_snapshot) {
5488	/*
5489	* This is an on_demand snapshot
5490	*/
5491	snapshot = od_snapshot;
5492	snapshot_list = od_snapshot->entries;
5493	snapshot_max = ods_list_count;
5494	} else {
5495	/*
5496	* This is a jetsam event snapshot
5497	*/
5498	snapshot = memorystatus_jetsam_snapshot;
5499	snapshot_list = memorystatus_jetsam_snapshot->entries;
5500	snapshot_max = memorystatus_jetsam_snapshot_max;
5501	}
5502
5503	/*
5504	* Init the snapshot header information
5505	*/
5506	memorystatus_init_snapshot_vmstats(snapshot);
5507	snapshot->snapshot_time = mach_absolute_time();
5508	snapshot->notification_time = `0`;
5509	snapshot->js_gencount = `0`;
5510
5511	next_p = memorystatus_get_first_proc_locked(&b, TRUE);
5512	while (next_p) {
5513	p = next_p;
5514	next_p = memorystatus_get_next_proc_locked(&b, p, TRUE);
5515
5516	if (FALSE == memorystatus_init_jetsam_snapshot_entry_locked(p, &snapshot_list[i], snapshot->js_gencount)) {
5517	continue;
5518	}
5519
5520	MEMORYSTATUS_DEBUG(`0`, "jetsam snapshot pid %d, uuid = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
5521	p->p_pid,
5522	p->p_uuid[`0`], p->p_uuid[`1`], p->p_uuid[`2`], p->p_uuid[`3`], p->p_uuid[`4`], p->p_uuid[`5`], p->p_uuid[`6`], p->p_uuid[`7`],
5523	p->p_uuid[`8`], p->p_uuid[`9`], p->p_uuid[`10`], p->p_uuid[`11`], p->p_uuid[`12`], p->p_uuid[`13`], p->p_uuid[`14`], p->p_uuid[`15`]);
5524
5525	if (++i == snapshot_max) {
5526	break;
5527	}
5528	}
5529
5530	snapshot->entry_count = i;
5531
5532	if (!od_snapshot) {
5533	/ update the system buffer count /
5534	memorystatus_jetsam_snapshot_count = i;
5535	}
5536	}
5537
5538	#if DEVELOPMENT \|\| DEBUG
5539
5540	#if CONFIG_JETSAM
5541	static int
5542	memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) {
5543	int ret;
5544	memorystatus_jetsam_panic_options_t debug;
5545
5546	if (buffer_size != sizeof(memorystatus_jetsam_panic_options_t)) {
5547	return EINVAL;
5548	}
5549
5550	ret = copyin(buffer, &debug, buffer_size);
5551	if (ret) {
5552	return ret;
5553	}
5554
5555	/ Panic bits match kMemorystatusKilled* enum /
5556	memorystatus_jetsam_panic_debug = (memorystatus_jetsam_panic_debug & ~debug.mask) \| (debug.data & debug.mask);
5557
5558	/ Copyout new value /
5559	debug.data = memorystatus_jetsam_panic_debug;
5560	ret = copyout(&debug, buffer, sizeof(memorystatus_jetsam_panic_options_t));
5561
5562	return ret;
5563	}
5564	#endif /* CONFIG_JETSAM */
5565
5566	/*
5567	* Triggers a sort_order on a specified jetsam priority band.
5568	* This is for testing only, used to force a path through the sort
5569	* function.
5570	*/
5571	static int
5572	memorystatus_cmd_test_jetsam_sort(int priority, int sort_order) {
5573
5574	int error = `0`;
5575
5576	unsigned int bucket_index = `0`;
5577
5578	if (priority == -`1`) {
5579	/ Use as shorthand for default priority /
5580	bucket_index = JETSAM_PRIORITY_DEFAULT;
5581	} else {
5582	bucket_index = (unsigned int)priority;
5583	}
5584
5585	error = memorystatus_sort_bucket(bucket_index, sort_order);
5586
5587	return (error);
5588	}
5589
5590	#endif /* DEVELOPMENT \|\| DEBUG */
5591
5592	/*
5593	* Prepare the process to be killed (set state, update snapshot) and kill it.
5594	*/
5595	static uint64_t memorystatus_purge_before_jetsam_success = `0`;
5596
5597	static boolean_t
5598	memorystatus_kill_proc(proc_t p, uint32_t cause, os_reason_t jetsam_reason, boolean_t *killed)
5599	{
5600	pid_t aPid = `0`;
5601	uint32_t aPid_ep = `0`;
5602
5603	uint64_t killtime = `0`;
5604	clock_sec_t tv_sec;
5605	clock_usec_t tv_usec;
5606	uint32_t tv_msec;
5607	boolean_t retval = FALSE;
5608	uint64_t num_pages_purged = `0`;
5609
5610	aPid = p->p_pid;
5611	aPid_ep = p->p_memstat_effectivepriority;
5612
5613	if (cause != kMemorystatusKilledVnodes && cause != kMemorystatusKilledZoneMapExhaustion) {
5614	/*
5615	* Genuine memory pressure and not other (vnode/zone) resource exhaustion.
5616	*/
5617	boolean_t success = FALSE;
5618
5619	networking_memstatus_callout(p, cause);
5620	num_pages_purged = vm_purgeable_purge_task_owned(p->task);
5621
5622	if (num_pages_purged) {
5623	/*
5624	* We actually purged something and so let's
5625	* check if we need to continue with the kill.
5626	*/
5627	if (cause == kMemorystatusKilledHiwat) {
5628	uint64_t footprint_in_bytes = get_task_phys_footprint(p->task);
5629	uint64_t memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * `1024ULL` * `1024ULL`); / convert MB to bytes /
5630	success = (footprint_in_bytes <= memlimit_in_bytes);
5631	} else {
5632	success = (memorystatus_avail_pages_below_pressure() == FALSE);
5633	}
5634
5635	if (success) {
5636
5637	memorystatus_purge_before_jetsam_success++;
5638
5639	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: purged %llu pages from pid %d [%s] and avoided %s\n",
5640	num_pages_purged, aPid, (*p->p_name ? p->p_name : "unknown"), memorystatus_kill_cause_name[cause]);
5641
5642	*killed = FALSE;
5643
5644	return TRUE;
5645	}
5646	}
5647	}
5648
5649	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
5650	MEMORYSTATUS_DEBUG(`1`, "jetsam: %s pid %d [%s] - %lld Mb > 1 (%d Mb)\n",
5651	(memorystatus_jetsam_policy & kPolicyDiagnoseActive) ? "suspending": "killing",
5652	aPid, (*p->p_name ? p->p_name : "unknown"),
5653	(footprint_in_bytes / (`1024ULL` * `1024ULL`)), / converted bytes to MB /
5654	p->p_memstat_memlimit);
5655	#endif /* CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG) */
5656
5657	killtime = mach_absolute_time();
5658	absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
5659	tv_msec = tv_usec / `1000`;
5660
5661	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
5662	if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
5663	if (cause == kMemorystatusKilledHiwat) {
5664	MEMORYSTATUS_DEBUG(`1`, "jetsam: suspending pid %d [%s] for diagnosis - memorystatus_available_pages: %d\n",
5665	aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_available_pages);
5666	} else {
5667	int activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
5668	if (activeProcess) {
5669	MEMORYSTATUS_DEBUG(`1`, "jetsam: suspending pid %d [%s] (active) for diagnosis - memorystatus_available_pages: %d\n",
5670	aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_available_pages);
5671
5672	if (memorystatus_jetsam_policy & kPolicyDiagnoseFirst) {
5673	jetsam_diagnostic_suspended_one_active_proc = `1`;
5674	printf("jetsam: returning after suspending first active proc - %d\n", aPid);
5675	}
5676	}
5677	}
5678
5679	proc_list_lock();
5680	/ This diagnostic code is going away soon. Ignore the kMemorystatusInvalid cause here. /
5681	memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusInvalid, killtime);
5682	proc_list_unlock();
5683
5684	p->p_memstat_state \|= P_MEMSTAT_DIAG_SUSPENDED;
5685
5686	if (p) {
5687	task_suspend(p->task);
5688	*killed = TRUE;
5689	}
5690	} else
5691	#endif /* CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG) */
5692	{
5693	proc_list_lock();
5694	memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
5695	proc_list_unlock();
5696
5697	char kill_reason_string[`128`];
5698
5699	if (cause == kMemorystatusKilledHiwat) {
5700	strlcpy(kill_reason_string, "killing_highwater_process", `128`);
5701	} else {
5702	if (aPid_ep == JETSAM_PRIORITY_IDLE) {
5703	strlcpy(kill_reason_string, "killing_idle_process", `128`);
5704	} else {
5705	strlcpy(kill_reason_string, "killing_top_process", `128`);
5706	}
5707	}
5708
5709	os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: %s pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
5710	(unsigned long)tv_sec, tv_msec, kill_reason_string,
5711	aPid, (*p->p_name ? p->p_name : "unknown"),
5712	memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
5713
5714	/*
5715	* memorystatus_do_kill drops a reference, so take another one so we can
5716	* continue to use this exit reason even after memorystatus_do_kill()
5717	* returns
5718	*/
5719	os_reason_ref(jetsam_reason);
5720
5721	retval = memorystatus_do_kill(p, cause, jetsam_reason);
5722
5723	*killed = retval;
5724	}
5725
5726	return retval;
5727	}
5728
5729	/*
5730	* Jetsam the first process in the queue.
5731	*/
5732	static boolean_t
5733	memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason,
5734	int32_t priority, uint32_t errors)
5735	{
5736	pid_t aPid;
5737	proc_t p = PROC_NULL, next_p = PROC_NULL;
5738	boolean_t new_snapshot = FALSE, force_new_snapshot = FALSE, killed = FALSE, freed_mem = FALSE;
5739	unsigned int i = `0`;
5740	uint32_t aPid_ep;
5741	int32_t local_max_kill_prio = JETSAM_PRIORITY_IDLE;
5742
5743	#ifndef CONFIG_FREEZE
5744	#pragma unused(any)
5745	#endif
5746
5747	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_START,
5748	memorystatus_available_pages, `0`, `0`, `0`, `0`);
5749
5750
5751	#if CONFIG_JETSAM
5752	if (sort_flag == TRUE) {
5753	(void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
5754	}
5755
5756	local_max_kill_prio = max_kill_priority;
5757
5758	force_new_snapshot = FALSE;
5759
5760	#else /* CONFIG_JETSAM */
5761
5762	if (sort_flag == TRUE) {
5763	(void)memorystatus_sort_bucket(JETSAM_PRIORITY_IDLE, JETSAM_SORT_DEFAULT);
5764	}
5765
5766	/*
5767	* On macos, we currently only have 2 reasons to be here:
5768	*
5769	* kMemorystatusKilledZoneMapExhaustion
5770	* AND
5771	* kMemorystatusKilledVMCompressorSpaceShortage
5772	*
5773	* If we are here because of kMemorystatusKilledZoneMapExhaustion, we will consider
5774	* any and all processes as eligible kill candidates since we need to avoid a panic.
5775	*
5776	* Since this function can be called async. it is harder to toggle the max_kill_priority
5777	* value before and after a call. And so we use this local variable to set the upper band
5778	* on the eligible kill bands.
5779	*/
5780	if (cause == kMemorystatusKilledZoneMapExhaustion) {
5781	local_max_kill_prio = JETSAM_PRIORITY_MAX;
5782	} else {
5783	local_max_kill_prio = max_kill_priority;
5784	}
5785
5786	/*
5787	* And, because we are here under extreme circumstances, we force a snapshot even for
5788	* IDLE kills.
5789	*/
5790	force_new_snapshot = TRUE;
5791
5792	#endif /* CONFIG_JETSAM */
5793
5794	proc_list_lock();
5795
5796	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5797	while (next_p && (next_p->p_memstat_effectivepriority <= local_max_kill_prio)) {
5798	#if DEVELOPMENT \|\| DEBUG
5799	int procSuspendedForDiagnosis;
5800	#endif /* DEVELOPMENT \|\| DEBUG */
5801
5802	p = next_p;
5803	next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5804
5805	#if DEVELOPMENT \|\| DEBUG
5806	procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
5807	#endif /* DEVELOPMENT \|\| DEBUG */
5808
5809	aPid = p->p_pid;
5810	aPid_ep = p->p_memstat_effectivepriority;
5811
5812	if (p->p_memstat_state & (P_MEMSTAT_ERROR \| P_MEMSTAT_TERMINATED)) {
5813	continue; / with lock held /
5814	}
5815
5816	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
5817	if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
5818	printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
5819	continue;
5820	}
5821	#endif /* CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG) */
5822
5823	if (cause == kMemorystatusKilledVnodes)
5824	{
5825	/*
5826	* If the system runs out of vnodes, we systematically jetsam
5827	* processes in hopes of stumbling onto a vnode gain that helps
5828	* the system recover. The process that happens to trigger
5829	* this path has no known relationship to the vnode shortage.
5830	* Deadlock avoidance: attempt to safeguard the caller.
5831	*/
5832
5833	if (p == current_proc()) {
5834	/ do not jetsam the current process /
5835	continue;
5836	}
5837	}
5838
5839	#if CONFIG_FREEZE
5840	boolean_t skip;
5841	boolean_t reclaim_proc = !(p->p_memstat_state & P_MEMSTAT_LOCKED);
5842	if (any \|\| reclaim_proc) {
5843	skip = FALSE;
5844	} else {
5845	skip = TRUE;
5846	}
5847
5848	if (skip) {
5849	continue;
5850	} else
5851	#endif
5852	{
5853	if (proc_ref_locked(p) == p) {
5854	/*
5855	* Mark as terminated so that if exit1() indicates success, but the process (for example)
5856	* is blocked in task_exception_notify(), it'll be skipped if encountered again - see
5857	* <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
5858	* acquisition of the proc lock.
5859	*/
5860	p->p_memstat_state \|= P_MEMSTAT_TERMINATED;
5861
5862	} else {
5863	/*
5864	* We need to restart the search again because
5865	* proc_ref_locked _can_ drop the proc_list lock
5866	* and we could have lost our stored next_p via
5867	* an exit() on another core.
5868	*/
5869	i = `0`;
5870	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5871	continue;
5872	}
5873
5874	/*
5875	* Capture a snapshot if none exists and:
5876	* - we are forcing a new snapshot creation, either because:
5877	* - on a particular platform we need these snapshots every time, OR
5878	* - a boot-arg/embedded device tree property has been set.
5879	* - priority was not requested (this is something other than an ambient kill)
5880	* - the priority was requested and the targeted process is not at idle priority
5881	*/
5882	if ((memorystatus_jetsam_snapshot_count == `0`) &&
5883	(force_new_snapshot \|\| memorystatus_idle_snapshot \|\| ((!priority) \|\| (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) {
5884	memorystatus_init_jetsam_snapshot_locked(NULL,`0`);
5885	new_snapshot = TRUE;
5886	}
5887
5888	proc_list_unlock();
5889
5890	freed_mem = memorystatus_kill_proc(p, cause, jetsam_reason, &killed); / purged and/or killed 'p' /
5891	/ Success? /
5892	if (freed_mem) {
5893	if (killed) {
5894	if (priority) {
5895	*priority = aPid_ep;
5896	}
5897	} else {
5898	/ purged /
5899	proc_list_lock();
5900	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5901	proc_list_unlock();
5902	}
5903	proc_rele(p);
5904	goto exit;
5905	}
5906
5907	/*
5908	* Failure - first unwind the state,
5909	* then fall through to restart the search.
5910	*/
5911	proc_list_lock();
5912	proc_rele_locked(p);
5913	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5914	p->p_memstat_state \|= P_MEMSTAT_ERROR;
5915	*errors += `1`;
5916
5917	i = `0`;
5918	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5919	}
5920	}
5921
5922	proc_list_unlock();
5923
5924	exit:
5925	os_reason_free(jetsam_reason);
5926
5927	/ Clear snapshot if freshly captured and no target was found /
5928	if (new_snapshot && !killed) {
5929	proc_list_lock();
5930	memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = `0`;
5931	proc_list_unlock();
5932	}
5933
5934	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_END,
5935	memorystatus_available_pages, killed ? aPid : `0`, `0`, `0`, `0`);
5936
5937	return killed;
5938	}
5939
5940	/*
5941	* Jetsam aggressively
5942	*/
5943	static boolean_t
5944	memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count,
5945	int32_t priority_max, uint32_t *errors)
5946	{
5947	pid_t aPid;
5948	proc_t p = PROC_NULL, next_p = PROC_NULL;
5949	boolean_t new_snapshot = FALSE, killed = FALSE;
5950	int kill_count = `0`;
5951	unsigned int i = `0`;
5952	int32_t aPid_ep = `0`;
5953	unsigned int memorystatus_level_snapshot = `0`;
5954	uint64_t killtime = `0`;
5955	clock_sec_t tv_sec;
5956	clock_usec_t tv_usec;
5957	uint32_t tv_msec;
5958	os_reason_t jetsam_reason = OS_REASON_NULL;
5959
5960	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_START,
5961	memorystatus_available_pages, priority_max, `0`, `0`, `0`);
5962
5963	memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
5964
5965	jetsam_reason = os_reason_create(OS_REASON_JETSAM, cause);
5966	if (jetsam_reason == OS_REASON_NULL) {
5967	printf("memorystatus_kill_top_process_aggressive: failed to allocate exit reason\n");
5968	}
5969
5970	proc_list_lock();
5971
5972	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5973	while (next_p) {
5974	#if DEVELOPMENT \|\| DEBUG
5975	int activeProcess;
5976	int procSuspendedForDiagnosis;
5977	#endif /* DEVELOPMENT \|\| DEBUG */
5978
5979	if (((next_p->p_listflag & P_LIST_EXITED) != `0`) \|\|
5980	((unsigned int)(next_p->p_memstat_effectivepriority) != i)) {
5981
5982	/*
5983	* We have raced with next_p running on another core.
5984	* It may be exiting or it may have moved to a different
5985	* jetsam priority band. This means we have lost our
5986	* place in line while traversing the jetsam list. We
5987	* attempt to recover by rewinding to the beginning of the band
5988	* we were already traversing. By doing this, we do not guarantee
5989	* that no process escapes this aggressive march, but we can make
5990	* skipping an entire range of processes less likely. (PR-21069019)
5991	*/
5992
5993	MEMORYSTATUS_DEBUG(`1`, "memorystatus: aggressive%d: rewinding band %d, %s(%d) moved or exiting.\n",
5994	aggr_count, i, (*next_p->p_name ? next_p->p_name : "unknown"), next_p->p_pid);
5995
5996	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5997	continue;
5998	}
5999
6000	p = next_p;
6001	next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
6002
6003	if (p->p_memstat_effectivepriority > priority_max) {
6004	/*
6005	* Bail out of this killing spree if we have
6006	* reached beyond the priority_max jetsam band.
6007	* That is, we kill up to and through the
6008	* priority_max jetsam band.
6009	*/
6010	proc_list_unlock();
6011	goto exit;
6012	}
6013
6014	#if DEVELOPMENT \|\| DEBUG
6015	activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
6016	procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
6017	#endif /* DEVELOPMENT \|\| DEBUG */
6018
6019	aPid = p->p_pid;
6020	aPid_ep = p->p_memstat_effectivepriority;
6021
6022	if (p->p_memstat_state & (P_MEMSTAT_ERROR \| P_MEMSTAT_TERMINATED)) {
6023	continue;
6024	}
6025
6026	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
6027	if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
6028	printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
6029	continue;
6030	}
6031	#endif /* CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG) */
6032
6033	/*
6034	* Capture a snapshot if none exists.
6035	*/
6036	if (memorystatus_jetsam_snapshot_count == `0`) {
6037	memorystatus_init_jetsam_snapshot_locked(NULL,`0`);
6038	new_snapshot = TRUE;
6039	}
6040
6041	/*
6042	* Mark as terminated so that if exit1() indicates success, but the process (for example)
6043	* is blocked in task_exception_notify(), it'll be skipped if encountered again - see
6044	* <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
6045	* acquisition of the proc lock.
6046	*/
6047	p->p_memstat_state \|= P_MEMSTAT_TERMINATED;
6048
6049	killtime = mach_absolute_time();
6050	absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
6051	tv_msec = tv_usec / `1000`;
6052
6053	/ Shift queue, update stats /
6054	memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
6055
6056	/*
6057	* In order to kill the target process, we will drop the proc_list_lock.
6058	* To guaranteee that p and next_p don't disappear out from under the lock,
6059	* we must take a ref on both.
6060	* If we cannot get a reference, then it's likely we've raced with
6061	* that process exiting on another core.
6062	*/
6063	if (proc_ref_locked(p) == p) {
6064	if (next_p) {
6065	while (next_p && (proc_ref_locked(next_p) != next_p)) {
6066	proc_t temp_p;
6067
6068	/*
6069	* We must have raced with next_p exiting on another core.
6070	* Recover by getting the next eligible process in the band.
6071	*/
6072
6073	MEMORYSTATUS_DEBUG(`1`, "memorystatus: aggressive%d: skipping %d [%s] (exiting?)\n",
6074	aggr_count, next_p->p_pid, (*next_p->p_name ? next_p->p_name : "(unknown)"));
6075
6076	temp_p = next_p;
6077	next_p = memorystatus_get_next_proc_locked(&i, temp_p, TRUE);
6078	}
6079	}
6080	proc_list_unlock();
6081
6082	printf("%lu.%03d memorystatus: %s%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
6083	(unsigned long)tv_sec, tv_msec,
6084	((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process_aggressive" : "killing_top_process_aggressive"),
6085	aggr_count, aPid, (*p->p_name ? p->p_name : "unknown"),
6086	memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
6087
6088	memorystatus_level_snapshot = memorystatus_level;
6089
6090	/*
6091	* memorystatus_do_kill() drops a reference, so take another one so we can
6092	* continue to use this exit reason even after memorystatus_do_kill()
6093	* returns.
6094	*/
6095	os_reason_ref(jetsam_reason);
6096	killed = memorystatus_do_kill(p, cause, jetsam_reason);
6097
6098	/ Success? /
6099	if (killed) {
6100	proc_rele(p);
6101	kill_count++;
6102	p = NULL;
6103	killed = FALSE;
6104
6105	/*
6106	* Continue the killing spree.
6107	*/
6108	proc_list_lock();
6109	if (next_p) {
6110	proc_rele_locked(next_p);
6111	}
6112
6113	if (aPid_ep == JETSAM_PRIORITY_FOREGROUND && memorystatus_aggressive_jetsam_lenient == TRUE) {
6114	if (memorystatus_level > memorystatus_level_snapshot && ((memorystatus_level - memorystatus_level_snapshot) >= AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD)) {
6115	#if DEVELOPMENT \|\| DEBUG
6116	printf("Disabling Lenient mode after one-time deployment.\n");
6117	#endif /* DEVELOPMENT \|\| DEBUG */
6118	memorystatus_aggressive_jetsam_lenient = FALSE;
6119	break;
6120	}
6121	}
6122
6123	continue;
6124	}
6125
6126	/*
6127	* Failure - first unwind the state,
6128	* then fall through to restart the search.
6129	*/
6130	proc_list_lock();
6131	proc_rele_locked(p);
6132	if (next_p) {
6133	proc_rele_locked(next_p);
6134	}
6135	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
6136	p->p_memstat_state \|= P_MEMSTAT_ERROR;
6137	*errors += `1`;
6138	p = NULL;
6139	}
6140
6141	/*
6142	* Failure - restart the search at the beginning of
6143	* the band we were already traversing.
6144	*
6145	* We might have raced with "p" exiting on another core, resulting in no
6146	* ref on "p". Or, we may have failed to kill "p".
6147	*
6148	* Either way, we fall thru to here, leaving the proc in the
6149	* P_MEMSTAT_TERMINATED or P_MEMSTAT_ERROR state.
6150	*
6151	* And, we hold the the proc_list_lock at this point.
6152	*/
6153
6154	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
6155	}
6156
6157	proc_list_unlock();
6158
6159	exit:
6160	os_reason_free(jetsam_reason);
6161
6162	/ Clear snapshot if freshly captured and no target was found /
6163	if (new_snapshot && (kill_count == `0`)) {
6164	proc_list_lock();
6165	memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = `0`;
6166	proc_list_unlock();
6167	}
6168
6169	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_END,
6170	memorystatus_available_pages, killed ? aPid : `0`, kill_count, `0`, `0`);
6171
6172	if (kill_count > `0`) {
6173	return(TRUE);
6174	}
6175	else {
6176	return(FALSE);
6177	}
6178	}
6179
6180	static boolean_t
6181	memorystatus_kill_hiwat_proc(uint32_t errors, boolean_t purged)
6182	{
6183	pid_t aPid = `0`;
6184	proc_t p = PROC_NULL, next_p = PROC_NULL;
6185	boolean_t new_snapshot = FALSE, killed = FALSE, freed_mem = FALSE;
6186	unsigned int i = `0`;
6187	uint32_t aPid_ep;
6188	os_reason_t jetsam_reason = OS_REASON_NULL;
6189	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) \| DBG_FUNC_START,
6190	memorystatus_available_pages, `0`, `0`, `0`, `0`);
6191
6192	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_HIGHWATER);
6193	if (jetsam_reason == OS_REASON_NULL) {
6194	printf("memorystatus_kill_hiwat_proc: failed to allocate exit reason\n");
6195	}
6196
6197	proc_list_lock();
6198
6199	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
6200	while (next_p) {
6201	uint64_t footprint_in_bytes = `0`;
6202	uint64_t memlimit_in_bytes = `0`;
6203	boolean_t skip = `0`;
6204
6205	p = next_p;
6206	next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
6207
6208	aPid = p->p_pid;
6209	aPid_ep = p->p_memstat_effectivepriority;
6210
6211	if (p->p_memstat_state & (P_MEMSTAT_ERROR \| P_MEMSTAT_TERMINATED)) {
6212	continue;
6213	}
6214
6215	/ skip if no limit set /
6216	if (p->p_memstat_memlimit <= `0`) {
6217	continue;
6218	}
6219
6220	footprint_in_bytes = get_task_phys_footprint(p->task);
6221	memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * `1024ULL` * `1024ULL`); / convert MB to bytes /
6222	skip = (footprint_in_bytes <= memlimit_in_bytes);
6223
6224	#if CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG)
6225	if (!skip && (memorystatus_jetsam_policy & kPolicyDiagnoseActive)) {
6226	if (p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED) {
6227	continue;
6228	}
6229	}
6230	#endif /* CONFIG_JETSAM && (DEVELOPMENT \|\| DEBUG) */
6231
6232	#if CONFIG_FREEZE
6233	if (!skip) {
6234	if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
6235	skip = TRUE;
6236	} else {
6237	skip = FALSE;
6238	}
6239	}
6240	#endif
6241
6242	if (skip) {
6243	continue;
6244	} else {
6245
6246	if (memorystatus_jetsam_snapshot_count == `0`) {
6247	memorystatus_init_jetsam_snapshot_locked(NULL,`0`);
6248	new_snapshot = TRUE;
6249	}
6250
6251	if (proc_ref_locked(p) == p) {
6252	/*
6253	* Mark as terminated so that if exit1() indicates success, but the process (for example)
6254	* is blocked in task_exception_notify(), it'll be skipped if encountered again - see
6255	* <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
6256	* acquisition of the proc lock.
6257	*/
6258	p->p_memstat_state \|= P_MEMSTAT_TERMINATED;
6259
6260	proc_list_unlock();
6261	} else {
6262	/*
6263	* We need to restart the search again because
6264	* proc_ref_locked _can_ drop the proc_list lock
6265	* and we could have lost our stored next_p via
6266	* an exit() on another core.
6267	*/
6268	i = `0`;
6269	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
6270	continue;
6271	}
6272
6273	freed_mem = memorystatus_kill_proc(p, kMemorystatusKilledHiwat, jetsam_reason, &killed); / purged and/or killed 'p' /
6274
6275	/ Success? /
6276	if (freed_mem) {
6277	if (killed == FALSE) {
6278	/ purged 'p'..don't reset HWM candidate count /
6279	*purged = TRUE;
6280
6281	proc_list_lock();
6282	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
6283	proc_list_unlock();
6284	}
6285	proc_rele(p);
6286	goto exit;
6287	}
6288	/*
6289	* Failure - first unwind the state,
6290	* then fall through to restart the search.
6291	*/
6292	proc_list_lock();
6293	proc_rele_locked(p);
6294	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
6295	p->p_memstat_state \|= P_MEMSTAT_ERROR;
6296	*errors += `1`;
6297
6298	i = `0`;
6299	next_p = memorystatus_get_first_proc_locked(&i, TRUE);
6300	}
6301	}
6302
6303	proc_list_unlock();
6304
6305	exit:
6306	os_reason_free(jetsam_reason);
6307
6308	/ Clear snapshot if freshly captured and no target was found /
6309	if (new_snapshot && !killed) {
6310	proc_list_lock();
6311	memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = `0`;
6312	proc_list_unlock();
6313	}
6314
6315	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) \| DBG_FUNC_END,
6316	memorystatus_available_pages, killed ? aPid : `0`, `0`, `0`, `0`);
6317
6318	return killed;
6319	}
6320
6321	/*
6322	* Jetsam a process pinned in the elevated band.
6323	*
6324	* Return: true -- at least one pinned process was jetsammed
6325	* false -- no pinned process was jetsammed
6326	*/
6327	static boolean_t
6328	memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, unsigned int band, int aggr_count, uint32_t *errors)
6329	{
6330	pid_t aPid = `0`;
6331	proc_t p = PROC_NULL, next_p = PROC_NULL;
6332	boolean_t new_snapshot = FALSE, killed = FALSE;
6333	int kill_count = `0`;
6334	uint32_t aPid_ep;
6335	uint64_t killtime = `0`;
6336	clock_sec_t tv_sec;
6337	clock_usec_t tv_usec;
6338	uint32_t tv_msec;
6339
6340
6341	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_START,
6342	memorystatus_available_pages, `0`, `0`, `0`, `0`);
6343
6344	#if CONFIG_FREEZE
6345	boolean_t consider_frozen_only = FALSE;
6346
6347	if (band == (unsigned int) memorystatus_freeze_jetsam_band) {
6348	consider_frozen_only = TRUE;
6349	}
6350	#endif /* CONFIG_FREEZE */
6351
6352	proc_list_lock();
6353
6354	next_p = memorystatus_get_first_proc_locked(&band, FALSE);
6355	while (next_p) {
6356
6357	p = next_p;
6358	next_p = memorystatus_get_next_proc_locked(&band, p, FALSE);
6359
6360	aPid = p->p_pid;
6361	aPid_ep = p->p_memstat_effectivepriority;
6362
6363	/*
6364	* Only pick a process pinned in this elevated band
6365	*/
6366	if (!(p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) {
6367	continue;
6368	}
6369
6370	if (p->p_memstat_state & (P_MEMSTAT_ERROR \| P_MEMSTAT_TERMINATED)) {
6371	continue;
6372	}
6373
6374	#if CONFIG_FREEZE
6375	if (consider_frozen_only && ! (p->p_memstat_state & P_MEMSTAT_FROZEN)) {
6376	continue;
6377	}
6378
6379	if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
6380	continue;
6381	}
6382	#endif /* CONFIG_FREEZE */
6383
6384	#if DEVELOPMENT \|\| DEBUG
6385	MEMORYSTATUS_DEBUG(`1`, "jetsam: elevated%d process pid %d [%s] - memorystatus_available_pages: %d\n",
6386	aggr_count,
6387	aPid, (*p->p_name ? p->p_name : "unknown"),
6388	memorystatus_available_pages);
6389	#endif /* DEVELOPMENT \|\| DEBUG */
6390
6391	if (memorystatus_jetsam_snapshot_count == `0`) {
6392	memorystatus_init_jetsam_snapshot_locked(NULL,`0`);
6393	new_snapshot = TRUE;
6394	}
6395
6396	p->p_memstat_state \|= P_MEMSTAT_TERMINATED;
6397
6398	killtime = mach_absolute_time();
6399	absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
6400	tv_msec = tv_usec / `1000`;
6401
6402	memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
6403
6404	if (proc_ref_locked(p) == p) {
6405
6406	proc_list_unlock();
6407
6408	os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_top_process_elevated%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
6409	(unsigned long)tv_sec, tv_msec,
6410	aggr_count,
6411	aPid, (*p->p_name ? p->p_name : "unknown"),
6412	memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
6413
6414	/*
6415	* memorystatus_do_kill drops a reference, so take another one so we can
6416	* continue to use this exit reason even after memorystatus_do_kill()
6417	* returns
6418	*/
6419	os_reason_ref(jetsam_reason);
6420	killed = memorystatus_do_kill(p, cause, jetsam_reason);
6421
6422	/ Success? /
6423	if (killed) {
6424	proc_rele(p);
6425	kill_count++;
6426	goto exit;
6427	}
6428
6429	/*
6430	* Failure - first unwind the state,
6431	* then fall through to restart the search.
6432	*/
6433	proc_list_lock();
6434	proc_rele_locked(p);
6435	p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
6436	p->p_memstat_state \|= P_MEMSTAT_ERROR;
6437	*errors += `1`;
6438	}
6439
6440	/*
6441	* Failure - restart the search.
6442	*
6443	* We might have raced with "p" exiting on another core, resulting in no
6444	* ref on "p". Or, we may have failed to kill "p".
6445	*
6446	* Either way, we fall thru to here, leaving the proc in the
6447	* P_MEMSTAT_TERMINATED state or P_MEMSTAT_ERROR state.
6448	*
6449	* And, we hold the the proc_list_lock at this point.
6450	*/
6451
6452	next_p = memorystatus_get_first_proc_locked(&band, FALSE);
6453	}
6454
6455	proc_list_unlock();
6456
6457	exit:
6458	os_reason_free(jetsam_reason);
6459
6460	/ Clear snapshot if freshly captured and no target was found /
6461	if (new_snapshot && (kill_count == `0`)) {
6462	proc_list_lock();
6463	memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = `0`;
6464	proc_list_unlock();
6465	}
6466
6467	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) \| DBG_FUNC_END,
6468	memorystatus_available_pages, killed ? aPid : `0`, kill_count, `0`, `0`);
6469
6470	return (killed);
6471	}
6472
6473	static boolean_t
6474	memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) {
6475	/*
6476	* TODO: allow a general async path
6477	*
6478	* NOTE: If a new async kill cause is added, make sure to update memorystatus_thread() to
6479	* add the appropriate exit reason code mapping.
6480	*/
6481	if ((victim_pid != -`1`) \|\|
6482	(cause != kMemorystatusKilledVMPageShortage &&
6483	cause != kMemorystatusKilledVMCompressorThrashing &&
6484	cause != kMemorystatusKilledVMCompressorSpaceShortage &&
6485	cause != kMemorystatusKilledFCThrashing &&
6486	cause != kMemorystatusKilledZoneMapExhaustion)) {
6487	return FALSE;
6488	}
6489
6490	kill_under_pressure_cause = cause;
6491	memorystatus_thread_wake();
6492	return TRUE;
6493	}
6494
6495	boolean_t
6496	memorystatus_kill_on_VM_compressor_space_shortage(boolean_t async) {
6497	if (async) {
6498	return memorystatus_kill_process_async(-`1`, kMemorystatusKilledVMCompressorSpaceShortage);
6499	} else {
6500	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMCOMPRESSOR_SPACE_SHORTAGE);
6501	if (jetsam_reason == OS_REASON_NULL) {
6502	printf("memorystatus_kill_on_VM_compressor_space_shortage -- sync: failed to allocate jetsam reason\n");
6503	}
6504
6505	return memorystatus_kill_process_sync(-`1`, kMemorystatusKilledVMCompressorSpaceShortage, jetsam_reason);
6506	}
6507	}
6508
6509	#if CONFIG_JETSAM
6510	boolean_t
6511	memorystatus_kill_on_VM_compressor_thrashing(boolean_t async) {
6512	if (async) {
6513	return memorystatus_kill_process_async(-`1`, kMemorystatusKilledVMCompressorThrashing);
6514	} else {
6515	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMCOMPRESSOR_THRASHING);
6516	if (jetsam_reason == OS_REASON_NULL) {
6517	printf("memorystatus_kill_on_VM_compressor_thrashing -- sync: failed to allocate jetsam reason\n");
6518	}
6519
6520	return memorystatus_kill_process_sync(-`1`, kMemorystatusKilledVMCompressorThrashing, jetsam_reason);
6521	}
6522	}
6523
6524	boolean_t
6525	memorystatus_kill_on_VM_page_shortage(boolean_t async) {
6526	if (async) {
6527	return memorystatus_kill_process_async(-`1`, kMemorystatusKilledVMPageShortage);
6528	} else {
6529	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE);
6530	if (jetsam_reason == OS_REASON_NULL) {
6531	printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n");
6532	}
6533
6534	return memorystatus_kill_process_sync(-`1`, kMemorystatusKilledVMPageShortage, jetsam_reason);
6535	}
6536	}
6537
6538	boolean_t
6539	memorystatus_kill_on_FC_thrashing(boolean_t async) {
6540
6541
6542	if (async) {
6543	return memorystatus_kill_process_async(-`1`, kMemorystatusKilledFCThrashing);
6544	} else {
6545	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_FCTHRASHING);
6546	if (jetsam_reason == OS_REASON_NULL) {
6547	printf("memorystatus_kill_on_FC_thrashing -- sync: failed to allocate jetsam reason\n");
6548	}
6549
6550	return memorystatus_kill_process_sync(-`1`, kMemorystatusKilledFCThrashing, jetsam_reason);
6551	}
6552	}
6553
6554	boolean_t
6555	memorystatus_kill_on_vnode_limit(void) {
6556	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_VNODE);
6557	if (jetsam_reason == OS_REASON_NULL) {
6558	printf("memorystatus_kill_on_vnode_limit: failed to allocate jetsam reason\n");
6559	}
6560
6561	return memorystatus_kill_process_sync(-`1`, kMemorystatusKilledVnodes, jetsam_reason);
6562	}
6563
6564	#endif /* CONFIG_JETSAM */
6565
6566	boolean_t
6567	memorystatus_kill_on_zone_map_exhaustion(pid_t pid) {
6568	boolean_t res = FALSE;
6569	if (pid == -`1`) {
6570	res = memorystatus_kill_process_async(-`1`, kMemorystatusKilledZoneMapExhaustion);
6571	} else {
6572	os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_ZONE_MAP_EXHAUSTION);
6573	if (jetsam_reason == OS_REASON_NULL) {
6574	printf("memorystatus_kill_on_zone_map_exhaustion: failed to allocate jetsam reason\n");
6575	}
6576
6577	res = memorystatus_kill_process_sync(pid, kMemorystatusKilledZoneMapExhaustion, jetsam_reason);
6578	}
6579	return res;
6580	}
6581
6582	#if CONFIG_FREEZE
6583
6584	__private_extern__ void
6585	memorystatus_freeze_init(void)
6586	{
6587	kern_return_t result;
6588	thread_t thread;
6589
6590	freezer_lck_grp_attr = lck_grp_attr_alloc_init();
6591	freezer_lck_grp = lck_grp_alloc_init("freezer", freezer_lck_grp_attr);
6592
6593	lck_mtx_init(&freezer_mutex, freezer_lck_grp, NULL);
6594
6595	/*
6596	* This is just the default value if the underlying
6597	* storage device doesn't have any specific budget.
6598	* We check with the storage layer in memorystatus_freeze_update_throttle()
6599	* before we start our freezing the first time.
6600	*/
6601	memorystatus_freeze_budget_pages_remaining = (memorystatus_freeze_daily_mb_max * `1024` * `1024`) / PAGE_SIZE;
6602
6603	result = kernel_thread_start(memorystatus_freeze_thread, NULL, &thread);
6604	if (result == KERN_SUCCESS) {
6605
6606	proc_set_thread_policy(thread, TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
6607	proc_set_thread_policy(thread, TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE);
6608	thread_set_thread_name(thread, "VM_freezer");
6609
6610	thread_deallocate(thread);
6611	} else {
6612	panic("Could not create memorystatus_freeze_thread");
6613	}
6614	}
6615
6616	static boolean_t
6617	memorystatus_is_process_eligible_for_freeze(proc_t p)
6618	{
6619	/*
6620	* Called with proc_list_lock held.
6621	*/
6622
6623	LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_OWNED);
6624
6625	boolean_t should_freeze = FALSE;
6626	uint32_t state = `0`, entry_count = `0`, pages = `0`, i = `0`;
6627	int probability_of_use = `0`;
6628
6629	if (isApp(p) == FALSE) {
6630	goto out;
6631	}
6632
6633	state = p->p_memstat_state;
6634
6635	if ((state & (P_MEMSTAT_TERMINATED \| P_MEMSTAT_LOCKED \| P_MEMSTAT_FREEZE_DISABLED \| P_MEMSTAT_FREEZE_IGNORE)) \|\|
6636	!(state & P_MEMSTAT_SUSPENDED)) {
6637	goto out;
6638	}
6639
6640	/ Only freeze processes meeting our minimum resident page criteria /
6641	memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
6642	if (pages < memorystatus_freeze_pages_min) {
6643	goto out;
6644	}
6645
6646	entry_count = (memorystatus_global_probabilities_size / sizeof(memorystatus_internal_probabilities_t));
6647
6648	if (entry_count) {
6649
6650	for (i=`0`; i < entry_count; i++ ) {
6651	if (strncmp(memorystatus_global_probabilities_table[i].proc_name,
6652	p->p_name,
6653	MAXCOMLEN + `1`) == `0`) {
6654
6655	probability_of_use = memorystatus_global_probabilities_table[i].use_probability;
6656	break;
6657	}
6658	}
6659
6660	if (probability_of_use == `0`) {
6661	goto out;
6662	}
6663	}
6664
6665	should_freeze = TRUE;
6666	out:
6667	return should_freeze;
6668	}
6669
6670	/*
6671	* Synchronously freeze the passed proc. Called with a reference to the proc held.
6672	*
6673	* Doesn't deal with re-freezing because this is called on a specific process and
6674	* not by the freezer thread. If that changes, we'll have to teach it about
6675	* refreezing a frozen process.
6676	*
6677	* Returns EINVAL or the value returned by task_freeze().
6678	*/
6679	int
6680	memorystatus_freeze_process_sync(proc_t p)
6681	{
6682	int ret = EINVAL;
6683	pid_t aPid = `0`;
6684	boolean_t memorystatus_freeze_swap_low = FALSE;
6685	int freezer_error_code = `0`;
6686
6687	lck_mtx_lock(&freezer_mutex);
6688
6689	if (p == NULL) {
6690	printf("memorystatus_freeze_process_sync: Invalid process\n");
6691	goto exit;
6692	}
6693
6694	if (memorystatus_freeze_enabled == FALSE) {
6695	printf("memorystatus_freeze_process_sync: Freezing is DISABLED\n");
6696	goto exit;
6697	}
6698
6699	if (!memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
6700	printf("memorystatus_freeze_process_sync: Low compressor and/or low swap space...skipping freeze\n");
6701	goto exit;
6702	}
6703
6704	memorystatus_freeze_update_throttle(&memorystatus_freeze_budget_pages_remaining);
6705	if (!memorystatus_freeze_budget_pages_remaining) {
6706	printf("memorystatus_freeze_process_sync: exit with NO available budget\n");
6707	goto exit;
6708	}
6709
6710	proc_list_lock();
6711
6712	if (p != NULL) {
6713	uint32_t purgeable, wired, clean, dirty, shared;
6714	uint32_t max_pages, i;
6715
6716	aPid = p->p_pid;
6717
6718	/ Ensure the process is eligible for freezing /
6719	if (memorystatus_is_process_eligible_for_freeze(p) == FALSE) {
6720	proc_list_unlock();
6721	goto exit;
6722	}
6723
6724	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
6725
6726	max_pages = MIN(memorystatus_freeze_pages_max, memorystatus_freeze_budget_pages_remaining);
6727
6728	} else {
6729	/*
6730	* We only have the compressor without any swap.
6731	*/
6732	max_pages = UINT32_MAX - `1`;
6733	}
6734
6735	/ Mark as locked temporarily to avoid kill /
6736	p->p_memstat_state \|= P_MEMSTAT_LOCKED;
6737	proc_list_unlock();
6738
6739	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) \| DBG_FUNC_START,
6740	memorystatus_available_pages, `0`, `0`, `0`, `0`);
6741
6742	ret = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, FALSE / eval only /);
6743
6744	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) \| DBG_FUNC_END,
6745	memorystatus_available_pages, aPid, `0`, `0`, `0`);
6746
6747	DTRACE_MEMORYSTATUS6(memorystatus_freeze, proc_t, p, unsigned int, memorystatus_available_pages, boolean_t, purgeable, unsigned int, wired, uint32_t, clean, uint32_t, dirty);
6748
6749	MEMORYSTATUS_DEBUG(`1`, "memorystatus_freeze_process_sync: task_freeze %s for pid %d [%s] - "
6750	"memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, max_pages %d, shared %d\n",
6751	(ret == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (*p->p_name ? p->p_name : "(unknown)"),
6752	memorystatus_available_pages, purgeable, wired, clean, dirty, max_pages, shared);
6753
6754	proc_list_lock();
6755
6756	if (ret == KERN_SUCCESS) {
6757
6758	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: freezing (specific) pid %d [%s]...done",
6759	aPid, (*p->p_name ? p->p_name : "unknown"));
6760
6761	memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
6762
6763	p->p_memstat_freeze_sharedanon_pages += shared;
6764
6765	memorystatus_frozen_shared_mb += shared;
6766
6767	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == `0`) {
6768	p->p_memstat_state \|= P_MEMSTAT_FROZEN;
6769	memorystatus_frozen_count++;
6770	}
6771
6772	p->p_memstat_frozen_count++;
6773
6774	/*
6775	* Still keeping the P_MEMSTAT_LOCKED bit till we are actually done elevating this frozen process
6776	* to its higher jetsam band.
6777	*/
6778	proc_list_unlock();
6779
6780	memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
6781
6782	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
6783
6784	ret = memorystatus_update_inactive_jetsam_priority_band(p->p_pid, MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE,
6785	memorystatus_freeze_jetsam_band, TRUE);
6786
6787	if (ret) {
6788	printf("Elevating the frozen process failed with %d\n", ret);
6789	/ not fatal /
6790	ret = `0`;
6791	}
6792
6793	proc_list_lock();
6794
6795	/ Update stats /
6796	for (i = `0`; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
6797	throttle_intervals[i].pageouts += dirty;
6798	}
6799	} else {
6800	proc_list_lock();
6801	}
6802
6803	memorystatus_freeze_pageouts += dirty;
6804
6805	if (memorystatus_frozen_count == (memorystatus_frozen_processes_max - `1`)) {
6806	/*
6807	* Add some eviction logic here? At some point should we
6808	* jetsam a process to get back its swap space so that we
6809	* can freeze a more eligible process at this moment in time?
6810	*/
6811	}
6812	} else {
6813	char reason[`128`];
6814	if (freezer_error_code == FREEZER_ERROR_EXCESS_SHARED_MEMORY) {
6815	strlcpy(reason, "too much shared memory", `128`);
6816	}
6817
6818	if (freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO) {
6819	strlcpy(reason, "low private-shared pages ratio", `128`);
6820	}
6821
6822	if (freezer_error_code == FREEZER_ERROR_NO_COMPRESSOR_SPACE) {
6823	strlcpy(reason, "no compressor space", `128`);
6824	}
6825
6826	if (freezer_error_code == FREEZER_ERROR_NO_SWAP_SPACE) {
6827	strlcpy(reason, "no swap space", `128`);
6828	}
6829
6830	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: freezing (specific) pid %d [%s]...skipped (%s)",
6831	aPid, (*p->p_name ? p->p_name : "unknown"), reason);
6832	p->p_memstat_state \|= P_MEMSTAT_FREEZE_IGNORE;
6833	}
6834
6835	p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
6836	proc_list_unlock();
6837	}
6838
6839	exit:
6840	lck_mtx_unlock(&freezer_mutex);
6841
6842	return ret;
6843	}
6844
6845	static int
6846	memorystatus_freeze_top_process(void)
6847	{
6848	pid_t aPid = `0`;
6849	int ret = -`1`;
6850	proc_t p = PROC_NULL, next_p = PROC_NULL;
6851	unsigned int i = `0`;
6852	unsigned int band = JETSAM_PRIORITY_IDLE;
6853	boolean_t refreeze_processes = FALSE;
6854
6855	proc_list_lock();
6856
6857	if (memorystatus_frozen_count >= memorystatus_frozen_processes_max) {
6858	/*
6859	* Freezer is already full but we are here and so let's
6860	* try to refreeze any processes we might have thawed
6861	* in the past and push out their compressed state out.
6862	*/
6863	refreeze_processes = TRUE;
6864	band = (unsigned int) memorystatus_freeze_jetsam_band;
6865	}
6866
6867	freeze_process:
6868
6869	next_p = memorystatus_get_first_proc_locked(&band, FALSE);
6870	while (next_p) {
6871	kern_return_t kr;
6872	uint32_t purgeable, wired, clean, dirty, shared;
6873	uint32_t max_pages = `0`;
6874	int freezer_error_code = `0`;
6875
6876	p = next_p;
6877	next_p = memorystatus_get_next_proc_locked(&band, p, FALSE);
6878
6879	aPid = p->p_pid;
6880
6881	if (p->p_memstat_effectivepriority != (int32_t) band) {
6882	/*
6883	* We shouldn't be freezing processes outside the
6884	* prescribed band.
6885	*/
6886	break;
6887	}
6888
6889	/ Ensure the process is eligible for (re-)freezing /
6890	if (refreeze_processes) {
6891	/*
6892	* Has to have been frozen once before.
6893	*/
6894	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == FALSE) {
6895	continue;
6896	}
6897
6898	/*
6899	* Has to have been resumed once before.
6900	*/
6901	if ((p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) == FALSE) {
6902	continue;
6903	}
6904
6905	/*
6906	* Not currently being looked at for something.
6907	*/
6908	if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
6909	continue;
6910	}
6911
6912	/*
6913	* We are going to try and refreeze and so re-evaluate
6914	* the process. We don't want to double count the shared
6915	* memory. So deduct the old snapshot here.
6916	*/
6917	memorystatus_frozen_shared_mb -= p->p_memstat_freeze_sharedanon_pages;
6918	p->p_memstat_freeze_sharedanon_pages = `0`;
6919
6920	p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
6921	memorystatus_refreeze_eligible_count--;
6922
6923	} else {
6924	if (memorystatus_is_process_eligible_for_freeze(p) == FALSE) {
6925	continue; // with lock held
6926	}
6927	}
6928
6929	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
6930	/*
6931	* Freezer backed by the compressor and swap file(s)
6932	* will hold compressed data.
6933	*/
6934
6935	max_pages = MIN(memorystatus_freeze_pages_max, memorystatus_freeze_budget_pages_remaining);
6936
6937	} else {
6938	/*
6939	* We only have the compressor pool.
6940	*/
6941	max_pages = UINT32_MAX - `1`;
6942	}
6943
6944	/ Mark as locked temporarily to avoid kill /
6945	p->p_memstat_state \|= P_MEMSTAT_LOCKED;
6946
6947	p = proc_ref_locked(p);
6948	if (!p) {
6949	break;
6950	}
6951
6952	proc_list_unlock();
6953
6954	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) \| DBG_FUNC_START,
6955	memorystatus_available_pages, `0`, `0`, `0`, `0`);
6956
6957	kr = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, FALSE / eval only /);
6958
6959	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) \| DBG_FUNC_END,
6960	memorystatus_available_pages, aPid, `0`, `0`, `0`);
6961
6962	MEMORYSTATUS_DEBUG(`1`, "memorystatus_freeze_top_process: task_freeze %s for pid %d [%s] - "
6963	"memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, max_pages %d, shared %d\n",
6964	(kr == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (*p->p_name ? p->p_name : "(unknown)"),
6965	memorystatus_available_pages, purgeable, wired, clean, dirty, max_pages, shared);
6966
6967	proc_list_lock();
6968
6969	/ Success? /
6970	if (KERN_SUCCESS == kr) {
6971
6972	if (refreeze_processes) {
6973	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: Refreezing (general) pid %d [%s]...done",
6974	aPid, (*p->p_name ? p->p_name : "unknown"));
6975	} else {
6976	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: freezing (general) pid %d [%s]...done",
6977	aPid, (*p->p_name ? p->p_name : "unknown"));
6978	}
6979
6980	memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
6981
6982	p->p_memstat_freeze_sharedanon_pages += shared;
6983
6984	memorystatus_frozen_shared_mb += shared;
6985
6986	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == `0`) {
6987	p->p_memstat_state \|= P_MEMSTAT_FROZEN;
6988	memorystatus_frozen_count++;
6989	}
6990
6991	p->p_memstat_frozen_count++;
6992
6993	/*
6994	* Still keeping the P_MEMSTAT_LOCKED bit till we are actually done elevating this frozen process
6995	* to its higher jetsam band.
6996	*/
6997	proc_list_unlock();
6998
6999	memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
7000
7001	if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
7002
7003	ret = memorystatus_update_inactive_jetsam_priority_band(p->p_pid, MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE, memorystatus_freeze_jetsam_band, TRUE);
7004
7005	if (ret) {
7006	printf("Elevating the frozen process failed with %d\n", ret);
7007	/ not fatal /
7008	ret = `0`;
7009	}
7010
7011	proc_list_lock();
7012
7013	/ Update stats /
7014	for (i = `0`; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
7015	throttle_intervals[i].pageouts += dirty;
7016	}
7017	} else {
7018	proc_list_lock();
7019	}
7020
7021	memorystatus_freeze_pageouts += dirty;
7022
7023	if (memorystatus_frozen_count == (memorystatus_frozen_processes_max - `1`)) {
7024	/*
7025	* Add some eviction logic here? At some point should we
7026	* jetsam a process to get back its swap space so that we
7027	* can freeze a more eligible process at this moment in time?
7028	*/
7029	}
7030
7031	/ Return KERN_SUCCESS /
7032	ret = kr;
7033
7034	p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
7035	proc_rele_locked(p);
7036
7037	/*
7038	* We froze a process successfully. We can stop now
7039	* and see if that helped.
7040	*/
7041
7042	break;
7043	} else {
7044
7045	p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
7046
7047	if (refreeze_processes == TRUE) {
7048	if ((freezer_error_code == FREEZER_ERROR_EXCESS_SHARED_MEMORY) \|\|
7049	(freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO)) {
7050	/*
7051	* Keeping this prior-frozen process in this high band when
7052	* we failed to re-freeze it due to bad shared memory usage
7053	* could cause excessive pressure on the lower bands.
7054	* We need to demote it for now. It'll get re-evaluated next
7055	* time because we don't set the P_MEMSTAT_FREEZE_IGNORE
7056	* bit.
7057	*/
7058
7059	p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
7060	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
7061	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, TRUE, TRUE);
7062	}
7063	} else {
7064	p->p_memstat_state \|= P_MEMSTAT_FREEZE_IGNORE;
7065	}
7066
7067	proc_rele_locked(p);
7068
7069	char reason[`128`];
7070	if (freezer_error_code == FREEZER_ERROR_EXCESS_SHARED_MEMORY) {
7071	strlcpy(reason, "too much shared memory", `128`);
7072	}
7073
7074	if (freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO) {
7075	strlcpy(reason, "low private-shared pages ratio", `128`);
7076	}
7077
7078	if (freezer_error_code == FREEZER_ERROR_NO_COMPRESSOR_SPACE) {
7079	strlcpy(reason, "no compressor space", `128`);
7080	}
7081
7082	if (freezer_error_code == FREEZER_ERROR_NO_SWAP_SPACE) {
7083	strlcpy(reason, "no swap space", `128`);
7084	}
7085
7086	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: freezing (general) pid %d [%s]...skipped (%s)",
7087	aPid, (*p->p_name ? p->p_name : "unknown"), reason);
7088
7089	if (vm_compressor_low_on_space() \|\| vm_swap_low_on_space()) {
7090	break;
7091	}
7092	}
7093	}
7094
7095	if ((ret == -`1`) &&
7096	(memorystatus_refreeze_eligible_count >= MIN_THAW_REFREEZE_THRESHOLD) &&
7097	(refreeze_processes == FALSE)) {
7098	/*
7099	* We failed to freeze a process from the IDLE
7100	* band AND we have some thawed processes
7101	* AND haven't tried refreezing as yet.
7102	* Let's try and re-freeze processes in the
7103	* frozen band that have been resumed in the past
7104	* and so have brought in state from disk.
7105	*/
7106
7107	band = (unsigned int) memorystatus_freeze_jetsam_band;
7108
7109	refreeze_processes = TRUE;
7110
7111	goto freeze_process;
7112	}
7113
7114	proc_list_unlock();
7115
7116	return ret;
7117	}
7118
7119	static inline boolean_t
7120	memorystatus_can_freeze_processes(void)
7121	{
7122	boolean_t ret;
7123
7124	proc_list_lock();
7125
7126	if (memorystatus_suspended_count) {
7127
7128	memorystatus_freeze_suspended_threshold = MIN(memorystatus_freeze_suspended_threshold, FREEZE_SUSPENDED_THRESHOLD_DEFAULT);
7129
7130	if ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold) {
7131	ret = TRUE;
7132	} else {
7133	ret = FALSE;
7134	}
7135	} else {
7136	ret = FALSE;
7137	}
7138
7139	proc_list_unlock();
7140
7141	return ret;
7142	}
7143
7144	static boolean_t
7145	memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low)
7146	{
7147	boolean_t can_freeze = TRUE;
7148
7149	/ Only freeze if we're sufficiently low on memory; this holds off freeze right*
7150	after boot, and is generally is a no-op once we've reached steady state. /*
7151	if (memorystatus_available_pages > memorystatus_freeze_threshold) {
7152	return FALSE;
7153	}
7154
7155	/ Check minimum suspended process threshold. /
7156	if (!memorystatus_can_freeze_processes()) {
7157	return FALSE;
7158	}
7159	assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
7160
7161	if ( !VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
7162	/*
7163	* In-core compressor used for freezing WITHOUT on-disk swap support.
7164	*/
7165	if (vm_compressor_low_on_space()) {
7166	if (*memorystatus_freeze_swap_low) {
7167	*memorystatus_freeze_swap_low = TRUE;
7168	}
7169
7170	can_freeze = FALSE;
7171
7172	} else {
7173	if (*memorystatus_freeze_swap_low) {
7174	*memorystatus_freeze_swap_low = FALSE;
7175	}
7176
7177	can_freeze = TRUE;
7178	}
7179	} else {
7180	/*
7181	* Freezing WITH on-disk swap support.
7182	*
7183	* In-core compressor fronts the swap.
7184	*/
7185	if (vm_swap_low_on_space()) {
7186	if (*memorystatus_freeze_swap_low) {
7187	*memorystatus_freeze_swap_low = TRUE;
7188	}
7189
7190	can_freeze = FALSE;
7191	}
7192
7193	}
7194
7195	return can_freeze;
7196	}
7197
7198	/*
7199	* This function evaluates if the currently frozen processes deserve
7200	* to stay in the higher jetsam band. If the # of thaws of a process
7201	* is below our threshold, then we will demote that process into the IDLE
7202	* band and put it at the head. We don't immediately kill the process here
7203	* because it already has state on disk and so it might be worth giving
7204	* it another shot at getting thawed/resumed and used.
7205	*/
7206	static void
7207	memorystatus_demote_frozen_processes(void)
7208	{
7209	unsigned int band = (unsigned int) memorystatus_freeze_jetsam_band;
7210	unsigned int demoted_proc_count = `0`;
7211	proc_t p = PROC_NULL, next_p = PROC_NULL;
7212
7213	proc_list_lock();
7214
7215	if (memorystatus_freeze_enabled == FALSE) {
7216	/*
7217	* Freeze has been disabled likely to
7218	* reclaim swap space. So don't change
7219	* any state on the frozen processes.
7220	*/
7221	proc_list_unlock();
7222	return;
7223	}
7224
7225	next_p = memorystatus_get_first_proc_locked(&band, FALSE);
7226	while (next_p) {
7227
7228	p = next_p;
7229	next_p = memorystatus_get_next_proc_locked(&band, p, FALSE);
7230
7231	if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == FALSE) {
7232	continue;
7233	}
7234
7235	if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
7236	continue;
7237	}
7238
7239	if (p->p_memstat_thaw_count < memorystatus_thaw_count_demotion_threshold) {
7240	p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
7241	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
7242
7243	memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, TRUE, TRUE);
7244	#if DEVELOPMENT \|\| DEBUG
7245	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus_demote_frozen_process pid %d [%s]",
7246	p->p_pid, (*p->p_name ? p->p_name : "unknown"));
7247	#endif /* DEVELOPMENT \|\| DEBUG */
7248
7249	/*
7250	* The freezer thread will consider this a normal app to be frozen
7251	* because it is in the IDLE band. So we don't need the
7252	* P_MEMSTAT_REFREEZE_ELIGIBLE state here. Also, if it gets resumed
7253	* we'll correctly count it as eligible for re-freeze again.
7254	*
7255	* We don't drop the frozen count because this process still has
7256	* state on disk. So there's a chance it gets resumed and then it
7257	* should land in the higher jetsam band. For that it needs to
7258	* remain marked frozen.
7259	*/
7260	if (p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) {
7261	p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
7262	memorystatus_refreeze_eligible_count--;
7263	}
7264
7265	demoted_proc_count++;
7266	}
7267
7268	if (demoted_proc_count == memorystatus_max_frozen_demotions_daily) {
7269	break;
7270	}
7271	}
7272
7273	memorystatus_thaw_count = `0`;
7274	proc_list_unlock();
7275	}
7276
7277
7278	/*
7279	* This function will do 4 things:
7280	*
7281	* 1) check to see if we are currently in a degraded freezer mode, and if so:
7282	* - check to see if our window has expired and we should exit this mode, OR,
7283	* - return a budget based on the degraded throttle window's max. pageouts vs current pageouts.
7284	*
7285	* 2) check to see if we are in a NEW normal window and update the normal throttle window's params.
7286	*
7287	* 3) check what the current normal window allows for a budget.
7288	*
7289	* 4) calculate the current rate of pageouts for DEGRADED_WINDOW_MINS duration. If that rate is below
7290	* what we would normally expect, then we are running low on our daily budget and need to enter
7291	* degraded perf. mode.
7292	*/
7293
7294	static void
7295	memorystatus_freeze_update_throttle(uint64_t *budget_pages_allowed)
7296	{
7297	clock_sec_t sec;
7298	clock_nsec_t nsec;
7299	mach_timespec_t ts;
7300
7301	unsigned int freeze_daily_pageouts_max = `0`;
7302
7303	#if DEVELOPMENT \|\| DEBUG
7304	if (!memorystatus_freeze_throttle_enabled) {
7305	/*
7306	* No throttling...we can use the full budget everytime.
7307	*/
7308	*budget_pages_allowed = UINT64_MAX;
7309	return;
7310	}
7311	#endif
7312
7313	clock_get_system_nanotime(&sec, &nsec);
7314	ts.tv_sec = sec;
7315	ts.tv_nsec = nsec;
7316
7317	struct throttle_interval_t *interval = NULL;
7318
7319	if (memorystatus_freeze_degradation == TRUE) {
7320
7321	interval = degraded_throttle_window;
7322
7323	if (CMP_MACH_TIMESPEC(&ts, &interval->ts) >= `0`) {
7324	memorystatus_freeze_degradation = FALSE;
7325	interval->pageouts = `0`;
7326	interval->max_pageouts = `0`;
7327
7328	} else {
7329	*budget_pages_allowed = interval->max_pageouts - interval->pageouts;
7330	}
7331	}
7332
7333	interval = normal_throttle_window;
7334
7335	if (CMP_MACH_TIMESPEC(&ts, &interval->ts) >= `0`) {
7336	/*
7337	* New throttle window.
7338	* Rollover any unused budget.
7339	* Also ask the storage layer what the new budget needs to be.
7340	*/
7341	uint64_t freeze_daily_budget = `0`;
7342	unsigned int daily_budget_pageouts = `0`;
7343
7344	if (vm_swap_max_budget(&freeze_daily_budget)) {
7345	memorystatus_freeze_daily_mb_max = (freeze_daily_budget / (`1024` * `1024`));
7346	os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: memorystatus_freeze_daily_mb_max set to %dMB\n", memorystatus_freeze_daily_mb_max);
7347	}
7348
7349	freeze_daily_pageouts_max = memorystatus_freeze_daily_mb_max * (`1024` * `1024` / PAGE_SIZE);
7350
7351	daily_budget_pageouts = (interval->burst_multiple * (((uint64_t)interval->mins * freeze_daily_pageouts_max) / NORMAL_WINDOW_MINS));
7352	interval->max_pageouts = (interval->max_pageouts - interval->pageouts) + daily_budget_pageouts;
7353
7354	interval->ts.tv_sec = interval->mins * `60`;
7355	interval->ts.tv_nsec = `0`;
7356	ADD_MACH_TIMESPEC(&interval->ts, &ts);
7357	/ Since we update the throttle stats pre-freeze, adjust for overshoot here /
7358	if (interval->pageouts > interval->max_pageouts) {
7359	interval->pageouts -= interval->max_pageouts;
7360	} else {
7361	interval->pageouts = `0`;
7362	}
7363	*budget_pages_allowed = interval->max_pageouts;
7364
7365	memorystatus_demote_frozen_processes();
7366
7367	} else {
7368	/*
7369	* Current throttle window.
7370	* Deny freezing if we have no budget left.
7371	* Try graceful degradation if we are within 25% of:
7372	* - the daily budget, and
7373	* - the current budget left is below our normal budget expectations.
7374	*/
7375
7376	#if DEVELOPMENT \|\| DEBUG
7377	/*
7378	* This can only happen in the INTERNAL configs because we allow modifying the daily budget for testing.
7379	*/
7380
7381	if (freeze_daily_pageouts_max > interval->max_pageouts) {
7382	/*
7383	* We just bumped the daily budget. Re-evaluate our normal window params.
7384	*/
7385	interval->max_pageouts = (interval->burst_multiple * (((uint64_t)interval->mins * freeze_daily_pageouts_max) / NORMAL_WINDOW_MINS));
7386	memorystatus_freeze_degradation = FALSE; //we'll re-evaluate this below...
7387	}
7388	#endif /* DEVELOPMENT \|\| DEBUG */
7389
7390	if (memorystatus_freeze_degradation == FALSE) {
7391
7392	if (interval->pageouts >= interval->max_pageouts) {
7393
7394	*budget_pages_allowed = `0`;
7395
7396	} else {
7397
7398	int budget_left = interval->max_pageouts - interval->pageouts;
7399	int budget_threshold = (freeze_daily_pageouts_max * FREEZE_DEGRADATION_BUDGET_THRESHOLD) / `100`;
7400
7401	mach_timespec_t time_left = {`0`,`0`};
7402
7403	time_left.tv_sec = interval->ts.tv_sec;
7404	time_left.tv_nsec = `0`;
7405
7406	SUB_MACH_TIMESPEC(&time_left, &ts);
7407
7408	if (budget_left <= budget_threshold) {
7409
7410	/*
7411	* For the current normal window, calculate how much we would pageout in a DEGRADED_WINDOW_MINS duration.
7412	* And also calculate what we would pageout for the same DEGRADED_WINDOW_MINS duration if we had the full
7413	* daily pageout budget.
7414	*/
7415
7416	unsigned int current_budget_rate_allowed = ((budget_left / time_left.tv_sec) / `60`) * DEGRADED_WINDOW_MINS;
7417	unsigned int normal_budget_rate_allowed = (freeze_daily_pageouts_max / NORMAL_WINDOW_MINS) * DEGRADED_WINDOW_MINS;
7418
7419	/*
7420	* The current rate of pageouts is below what we would expect for
7421	* the normal rate i.e. we have below normal budget left and so...
7422	*/
7423
7424	if (current_budget_rate_allowed < normal_budget_rate_allowed) {
7425
7426	memorystatus_freeze_degradation = TRUE;
7427	degraded_throttle_window->max_pageouts = current_budget_rate_allowed;
7428	degraded_throttle_window->pageouts = `0`;
7429
7430	/*
7431	* Switch over to the degraded throttle window so the budget
7432	* doled out is based on that window.
7433	*/
7434	interval = degraded_throttle_window;
7435	}
7436	}
7437
7438	*budget_pages_allowed = interval->max_pageouts - interval->pageouts;
7439	}
7440	}
7441	}
7442
7443	MEMORYSTATUS_DEBUG(`1`, "memorystatus_freeze_update_throttle_interval: throttle updated - %d frozen (%d max) within %dm; %dm remaining; throttle %s\n",
7444	interval->pageouts, interval->max_pageouts, interval->mins, (interval->ts.tv_sec - ts->tv_sec) / `60`,
7445	interval->throttle ? "on" : "off");
7446	}
7447
7448	static void
7449	memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused)
7450	{
7451	static boolean_t memorystatus_freeze_swap_low = FALSE;
7452
7453	lck_mtx_lock(&freezer_mutex);
7454
7455	if (memorystatus_freeze_enabled) {
7456
7457	if ((memorystatus_frozen_count < memorystatus_frozen_processes_max) \|\|
7458	(memorystatus_refreeze_eligible_count >= MIN_THAW_REFREEZE_THRESHOLD)) {
7459
7460	if (memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
7461
7462	/ Only freeze if we've not exceeded our pageout budgets./
7463	memorystatus_freeze_update_throttle(&memorystatus_freeze_budget_pages_remaining);
7464
7465	if (memorystatus_freeze_budget_pages_remaining) {
7466	memorystatus_freeze_top_process();
7467	}
7468	}
7469	}
7470	}
7471
7472	/*
7473	* We use memorystatus_apps_idle_delay_time because if/when we adopt aging for applications,
7474	* it'll tie neatly into running the freezer once we age an application.
7475	*
7476	* Till then, it serves as a good interval that can be tuned via a sysctl too.
7477	*/
7478	memorystatus_freezer_thread_next_run_ts = mach_absolute_time() + memorystatus_apps_idle_delay_time;
7479
7480	assert_wait((event_t) &memorystatus_freeze_wakeup, THREAD_UNINT);
7481	lck_mtx_unlock(&freezer_mutex);
7482
7483	thread_block((thread_continue_t) memorystatus_freeze_thread);
7484	}
7485
7486	static boolean_t
7487	memorystatus_freeze_thread_should_run(void)
7488	{
7489	/*
7490	* No freezer_mutex held here...see why near call-site
7491	* within memorystatus_pages_update().
7492	*/
7493
7494	boolean_t should_run = FALSE;
7495
7496	if (memorystatus_freeze_enabled == FALSE) {
7497	goto out;
7498	}
7499
7500	if (memorystatus_available_pages > memorystatus_freeze_threshold) {
7501	goto out;
7502	}
7503
7504	if ((memorystatus_frozen_count >= memorystatus_frozen_processes_max) &&
7505	(memorystatus_refreeze_eligible_count < MIN_THAW_REFREEZE_THRESHOLD)) {
7506	goto out;
7507	}
7508
7509	if (memorystatus_frozen_shared_mb_max && (memorystatus_frozen_shared_mb >= memorystatus_frozen_shared_mb_max)) {
7510	goto out;
7511	}
7512
7513	uint64_t curr_time = mach_absolute_time();
7514
7515	if (curr_time < memorystatus_freezer_thread_next_run_ts) {
7516	goto out;
7517	}
7518
7519	should_run = TRUE;
7520
7521	out:
7522	return should_run;
7523	}
7524
7525	static int
7526	sysctl_memorystatus_do_fastwake_warmup_all SYSCTL_HANDLER_ARGS
7527	{
7528	#pragma unused(oidp, req, arg1, arg2)
7529
7530	/ Need to be root or have entitlement /
7531	if (!kauth_cred_issuser(kauth_cred_get()) && !IOTaskHasEntitlement(current_task(), MEMORYSTATUS_ENTITLEMENT)) {
7532	return EPERM;
7533	}
7534
7535	if (memorystatus_freeze_enabled == FALSE) {
7536	return ENOTSUP;
7537	}
7538
7539	do_fastwake_warmup_all();
7540
7541	return `0`;
7542	}
7543
7544	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_do_fastwake_warmup_all, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
7545	`0`, `0`, &sysctl_memorystatus_do_fastwake_warmup_all, "I", "");
7546
7547	#endif /* CONFIG_FREEZE */
7548
7549	#if VM_PRESSURE_EVENTS
7550
7551	#if CONFIG_MEMORYSTATUS
7552
7553	static int
7554	memorystatus_send_note(int event_code, void *data, size_t data_length) {
7555	int ret;
7556	struct kev_msg ev_msg;
7557
7558	ev_msg.vendor_code = KEV_VENDOR_APPLE;
7559	ev_msg.kev_class = KEV_SYSTEM_CLASS;
7560	ev_msg.kev_subclass = KEV_MEMORYSTATUS_SUBCLASS;
7561
7562	ev_msg.event_code = event_code;
7563
7564	ev_msg.dv[`0`].data_length = data_length;
7565	ev_msg.dv[`0`].data_ptr = data;
7566	ev_msg.dv[`1`].data_length = `0`;
7567
7568	ret = kev_post_msg(&ev_msg);
7569	if (ret) {
7570	printf("%s: kev_post_msg() failed, err %d\n", __func__, ret);
7571	}
7572
7573	return ret;
7574	}
7575
7576	boolean_t
7577	memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t limit_exceeded) {
7578
7579	boolean_t ret = FALSE;
7580	boolean_t found_knote = FALSE;
7581	struct knote *kn = NULL;
7582	int send_knote_count = `0`;
7583
7584	/*
7585	* See comment in sysctl_memorystatus_vm_pressure_send.
7586	*/
7587
7588	memorystatus_klist_lock();
7589
7590	SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
7591	proc_t knote_proc = knote_get_kq(kn)->kq_p;
7592	pid_t knote_pid = knote_proc->p_pid;
7593
7594	if (knote_pid == pid) {
7595	/*
7596	* By setting the "fflags" here, we are forcing
7597	* a process to deal with the case where it's
7598	* bumping up into its memory limits. If we don't
7599	* do this here, we will end up depending on the
7600	* system pressure snapshot evaluation in
7601	* filt_memorystatus().
7602	*/
7603
7604	#if CONFIG_EMBEDDED
7605	if (!limit_exceeded) {
7606	/*
7607	* Intentionally set either the unambiguous limit warning,
7608	* the system-wide critical or the system-wide warning
7609	* notification bit.
7610	*/
7611
7612	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
7613	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
7614	found_knote = TRUE;
7615	send_knote_count++;
7616	} else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
7617	kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
7618	found_knote = TRUE;
7619	send_knote_count++;
7620	} else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
7621	kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
7622	found_knote = TRUE;
7623	send_knote_count++;
7624	}
7625	} else {
7626	/*
7627	* Send this notification when a process has exceeded a soft limit.
7628	*/
7629	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
7630	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
7631	found_knote = TRUE;
7632	send_knote_count++;
7633	}
7634	}
7635	#else /* CONFIG_EMBEDDED */
7636	if (!limit_exceeded) {
7637
7638	/*
7639	* Processes on desktop are not expecting to handle a system-wide
7640	* critical or system-wide warning notification from this path.
7641	* Intentionally set only the unambiguous limit warning here.
7642	*
7643	* If the limit is soft, however, limit this to one notification per
7644	* active/inactive limit (per each registered listener).
7645	*/
7646
7647	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
7648	found_knote=TRUE;
7649	if (!is_fatal) {
7650	/*
7651	* Restrict proc_limit_warn notifications when
7652	* non-fatal (soft) limit is at play.
7653	*/
7654	if (is_active) {
7655	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
7656	/*
7657	* Mark this knote for delivery.
7658	*/
7659	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
7660	/*
7661	* And suppress it from future notifications.
7662	*/
7663	kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
7664	send_knote_count++;
7665	}
7666	} else {
7667	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
7668	/*
7669	* Mark this knote for delivery.
7670	*/
7671	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
7672	/*
7673	* And suppress it from future notifications.
7674	*/
7675	kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
7676	send_knote_count++;
7677	}
7678	}
7679	} else {
7680	/*
7681	* No restriction on proc_limit_warn notifications when
7682	* fatal (hard) limit is at play.
7683	*/
7684	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
7685	send_knote_count++;
7686	}
7687	}
7688	} else {
7689	/*
7690	* Send this notification when a process has exceeded a soft limit,
7691	*/
7692
7693	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
7694	found_knote = TRUE;
7695	if (!is_fatal) {
7696	/*
7697	* Restrict critical notifications for soft limits.
7698	*/
7699
7700	if (is_active) {
7701	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
7702	/*
7703	* Suppress future proc_limit_critical notifications
7704	* for the active soft limit.
7705	*/
7706	kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
7707	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
7708	send_knote_count++;
7709
7710	}
7711	} else {
7712	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
7713	/*
7714	* Suppress future proc_limit_critical_notifications
7715	* for the inactive soft limit.
7716	*/
7717	kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
7718	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
7719	send_knote_count++;
7720	}
7721	}
7722	} else {
7723	/*
7724	* We should never be trying to send a critical notification for
7725	* a hard limit... the process would be killed before it could be
7726	* received.
7727	*/
7728	panic("Caught sending pid %d a critical warning for a fatal limit.\n", pid);
7729	}
7730	}
7731	}
7732	#endif /* CONFIG_EMBEDDED */
7733	}
7734	}
7735
7736	if (found_knote) {
7737	if (send_knote_count > `0`) {
7738	KNOTE(&memorystatus_klist, `0`);
7739	}
7740	ret = TRUE;
7741	}
7742
7743	memorystatus_klist_unlock();
7744
7745	return ret;
7746	}
7747
7748	/*
7749	* Can only be set by the current task on itself.
7750	*/
7751	int
7752	memorystatus_low_mem_privileged_listener(uint32_t op_flags)
7753	{
7754	boolean_t set_privilege = FALSE;
7755	/*
7756	* Need an entitlement check here?
7757	*/
7758	if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE) {
7759	set_privilege = TRUE;
7760	} else if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE) {
7761	set_privilege = FALSE;
7762	} else {
7763	return EINVAL;
7764	}
7765
7766	return (task_low_mem_privileged_listener(current_task(), set_privilege, NULL));
7767	}
7768
7769	int
7770	memorystatus_send_pressure_note(pid_t pid) {
7771	MEMORYSTATUS_DEBUG(`1`, "memorystatus_send_pressure_note(): pid %d\n", pid);
7772	return memorystatus_send_note(kMemorystatusPressureNote, &pid, sizeof(pid));
7773	}
7774
7775	void
7776	memorystatus_send_low_swap_note(void) {
7777
7778	struct knote *kn = NULL;
7779
7780	memorystatus_klist_lock();
7781	SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
7782	/ We call is_knote_registered_modify_task_pressure_bits to check if the sfflags for the*
7783	* current note contain NOTE_MEMORYSTATUS_LOW_SWAP. Once we find one note in the memorystatus_klist
7784	* that has the NOTE_MEMORYSTATUS_LOW_SWAP flags in its sfflags set, we call KNOTE with
7785	* kMemoryStatusLowSwap as the hint to process and update all knotes on the memorystatus_klist accordingly. */
7786	if (is_knote_registered_modify_task_pressure_bits(kn, NOTE_MEMORYSTATUS_LOW_SWAP, NULL, `0`, `0`) == TRUE) {
7787	KNOTE(&memorystatus_klist, kMemorystatusLowSwap);
7788	break;
7789	}
7790	}
7791
7792	memorystatus_klist_unlock();
7793	}
7794
7795	boolean_t
7796	memorystatus_bg_pressure_eligible(proc_t p) {
7797	boolean_t eligible = FALSE;
7798
7799	proc_list_lock();
7800
7801	MEMORYSTATUS_DEBUG(`1`, "memorystatus_bg_pressure_eligible: pid %d, state 0x%x\n", p->p_pid, p->p_memstat_state);
7802
7803	/ Foreground processes have already been dealt with at this point, so just test for eligibility /
7804	if (!(p->p_memstat_state & (P_MEMSTAT_TERMINATED \| P_MEMSTAT_LOCKED \| P_MEMSTAT_SUSPENDED \| P_MEMSTAT_FROZEN))) {
7805	eligible = TRUE;
7806	}
7807
7808	if (p->p_memstat_effectivepriority < JETSAM_PRIORITY_BACKGROUND_OPPORTUNISTIC) {
7809	/*
7810	* IDLE and IDLE_DEFERRED bands contain processes
7811	* that have dropped memory to be under their inactive
7812	* memory limits. And so they can't really give back
7813	* anything.
7814	*/
7815	eligible = FALSE;
7816	}
7817
7818	proc_list_unlock();
7819
7820	return eligible;
7821	}
7822
7823	boolean_t
7824	memorystatus_is_foreground_locked(proc_t p) {
7825	return ((p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND) \|\|
7826	(p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND_SUPPORT));
7827	}
7828
7829	/*
7830	* This is meant for stackshot and kperf -- it does not take the proc_list_lock
7831	* to access the p_memstat_dirty field.
7832	*/
7833	void memorystatus_proc_flags_unsafe(void * v, boolean_t is_dirty, boolean_t is_dirty_tracked, boolean_t *allow_idle_exit)
7834	{
7835	if (!v) {
7836	*is_dirty = FALSE;
7837	*is_dirty_tracked = FALSE;
7838	*allow_idle_exit = FALSE;
7839	} else {
7840	proc_t p = (proc_t)v;
7841	*is_dirty = (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) != `0`;
7842	*is_dirty_tracked = (p->p_memstat_dirty & P_DIRTY_TRACK) != `0`;
7843	*allow_idle_exit = (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) != `0`;
7844	}
7845	}
7846
7847	#endif /* CONFIG_MEMORYSTATUS */
7848
7849	/*
7850	* Trigger levels to test the mechanism.
7851	* Can be used via a sysctl.
7852	*/
7853	#define TEST_LOW_MEMORY_TRIGGER_ONE 1
7854	#define TEST_LOW_MEMORY_TRIGGER_ALL 2
7855	#define TEST_PURGEABLE_TRIGGER_ONE 3
7856	#define TEST_PURGEABLE_TRIGGER_ALL 4
7857	#define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE 5
7858	#define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL 6
7859
7860	boolean_t memorystatus_manual_testing_on = FALSE;
7861	vm_pressure_level_t memorystatus_manual_testing_level = kVMPressureNormal;
7862
7863	extern struct knote *
7864	vm_pressure_select_optimal_candidate_to_notify(struct klist , int*, boolean_t);
7865
7866
7867	#define VM_PRESSURE_NOTIFY_WAIT_PERIOD 10000 /* milliseconds */
7868
7869	#if DEBUG
7870	#define VM_PRESSURE_DEBUG(cond, format, ...) \
7871	do { \
7872	if (cond) { printf(format, ##__VA_ARGS__); } \
7873	} while(0)
7874	#else
7875	#define VM_PRESSURE_DEBUG(cond, format, ...)
7876	#endif
7877
7878	#define INTER_NOTIFICATION_DELAY (250000) /* .25 second */
7879
7880	void memorystatus_on_pageout_scan_end(void) {
7881	/ No-op /
7882	}
7883
7884	/*
7885	* kn_max - knote
7886	*
7887	* knote_pressure_level - to check if the knote is registered for this notification level.
7888	*
7889	* task - task whose bits we'll be modifying
7890	*
7891	* pressure_level_to_clear - if the task has been notified of this past level, clear that notification bit so that if/when we revert to that level, the task will be notified again.
7892	*
7893	* pressure_level_to_set - the task is about to be notified of this new level. Update the task's bit notification information appropriately.
7894	*
7895	*/
7896
7897	boolean_t
7898	is_knote_registered_modify_task_pressure_bits(struct knote kn_max, int* knote_pressure_level, task_t task, vm_pressure_level_t pressure_level_to_clear, vm_pressure_level_t pressure_level_to_set)
7899	{
7900	if (kn_max->kn_sfflags & knote_pressure_level) {
7901
7902	if (pressure_level_to_clear && task_has_been_notified(task, pressure_level_to_clear) == TRUE) {
7903
7904	task_clear_has_been_notified(task, pressure_level_to_clear);
7905	}
7906
7907	task_mark_has_been_notified(task, pressure_level_to_set);
7908	return TRUE;
7909	}
7910
7911	return FALSE;
7912	}
7913
7914	void
7915	memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear)
7916	{
7917	struct knote *kn = NULL;
7918
7919	memorystatus_klist_lock();
7920	SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
7921
7922	proc_t p = PROC_NULL;
7923	struct task* t = TASK_NULL;
7924
7925	p = knote_get_kq(kn)->kq_p;
7926	proc_list_lock();
7927	if (p != proc_ref_locked(p)) {
7928	p = PROC_NULL;
7929	proc_list_unlock();
7930	continue;
7931	}
7932	proc_list_unlock();
7933
7934	t = (struct task *)(p->task);
7935
7936	task_clear_has_been_notified(t, pressure_level_to_clear);
7937
7938	proc_rele(p);
7939	}
7940
7941	memorystatus_klist_unlock();
7942	}
7943
7944	extern kern_return_t vm_pressure_notify_dispatch_vm_clients(boolean_t target_foreground_process);
7945
7946	struct knote *
7947	vm_pressure_select_optimal_candidate_to_notify(struct klist candidate_list, int* level, boolean_t target_foreground_process);
7948
7949	/*
7950	* Used by the vm_pressure_thread which is
7951	* signalled from within vm_pageout_scan().
7952	*/
7953	static void vm_dispatch_memory_pressure(void);
7954	void consider_vm_pressure_events(void);
7955
7956	void consider_vm_pressure_events(void)
7957	{
7958	vm_dispatch_memory_pressure();
7959	}
7960	static void vm_dispatch_memory_pressure(void)
7961	{
7962	memorystatus_update_vm_pressure(FALSE);
7963	}
7964
7965	extern vm_pressure_level_t
7966	convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
7967
7968	struct knote *
7969	vm_pressure_select_optimal_candidate_to_notify(struct klist candidate_list, int* level, boolean_t target_foreground_process)
7970	{
7971	struct knote kn = NULL, kn_max = NULL;
7972	uint64_t resident_max = `0`; / MB /
7973	struct timeval curr_tstamp = {`0`, `0`};
7974	int elapsed_msecs = `0`;
7975	int selected_task_importance = `0`;
7976	static int pressure_snapshot = -`1`;
7977	boolean_t pressure_increase = FALSE;
7978
7979	if (pressure_snapshot == -`1`) {
7980	/*
7981	* Initial snapshot.
7982	*/
7983	pressure_snapshot = level;
7984	pressure_increase = TRUE;
7985	} else {
7986
7987	if (level && (level >= pressure_snapshot)) {
7988	pressure_increase = TRUE;
7989	} else {
7990	pressure_increase = FALSE;
7991	}
7992
7993	pressure_snapshot = level;
7994	}
7995
7996	if (pressure_increase == TRUE) {
7997	/*
7998	* We'll start by considering the largest
7999	* unimportant task in our list.
8000	*/
8001	selected_task_importance = INT_MAX;
8002	} else {
8003	/*
8004	* We'll start by considering the largest
8005	* important task in our list.
8006	*/
8007	selected_task_importance = `0`;
8008	}
8009
8010	microuptime(&curr_tstamp);
8011
8012	SLIST_FOREACH(kn, candidate_list, kn_selnext) {
8013
8014	uint64_t resident_size = `0`; / MB /
8015	proc_t p = PROC_NULL;
8016	struct task* t = TASK_NULL;
8017	int curr_task_importance = `0`;
8018	boolean_t consider_knote = FALSE;
8019	boolean_t privileged_listener = FALSE;
8020
8021	p = knote_get_kq(kn)->kq_p;
8022	proc_list_lock();
8023	if (p != proc_ref_locked(p)) {
8024	p = PROC_NULL;
8025	proc_list_unlock();
8026	continue;
8027	}
8028	proc_list_unlock();
8029
8030	#if CONFIG_MEMORYSTATUS
8031	if (target_foreground_process == TRUE && !memorystatus_is_foreground_locked(p)) {
8032	/*
8033	* Skip process not marked foreground.
8034	*/
8035	proc_rele(p);
8036	continue;
8037	}
8038	#endif /* CONFIG_MEMORYSTATUS */
8039
8040	t = (struct task *)(p->task);
8041
8042	timevalsub(&curr_tstamp, &p->vm_pressure_last_notify_tstamp);
8043	elapsed_msecs = curr_tstamp.tv_sec * `1000` + curr_tstamp.tv_usec / `1000`;
8044
8045	vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(level);
8046
8047	if ((kn->kn_sfflags & dispatch_level) == `0`) {
8048	proc_rele(p);
8049	continue;
8050	}
8051
8052	#if CONFIG_MEMORYSTATUS
8053	if (target_foreground_process == FALSE && !memorystatus_bg_pressure_eligible(p)) {
8054	VM_PRESSURE_DEBUG(`1`, "[vm_pressure] skipping process %d\n", p->p_pid);
8055	proc_rele(p);
8056	continue;
8057	}
8058	#endif /* CONFIG_MEMORYSTATUS */
8059
8060	#if CONFIG_EMBEDDED
8061	curr_task_importance = p->p_memstat_effectivepriority;
8062	#else /* CONFIG_EMBEDDED */
8063	curr_task_importance = task_importance_estimate(t);
8064	#endif /* CONFIG_EMBEDDED */
8065
8066	/*
8067	* Privileged listeners are only considered in the multi-level pressure scheme
8068	* AND only if the pressure is increasing.
8069	*/
8070	if (level > `0`) {
8071
8072	if (task_has_been_notified(t, level) == FALSE) {
8073
8074	/*
8075	* Is this a privileged listener?
8076	*/
8077	if (task_low_mem_privileged_listener(t, FALSE, &privileged_listener) == `0`) {
8078
8079	if (privileged_listener) {
8080	kn_max = kn;
8081	proc_rele(p);
8082	goto done_scanning;
8083	}
8084	}
8085	} else {
8086	proc_rele(p);
8087	continue;
8088	}
8089	} else if (level == `0`) {
8090
8091	/*
8092	* Task wasn't notified when the pressure was increasing and so
8093	* no need to notify it that the pressure is decreasing.
8094	*/
8095	if ((task_has_been_notified(t, kVMPressureWarning) == FALSE) && (task_has_been_notified(t, kVMPressureCritical) == FALSE)) {
8096	proc_rele(p);
8097	continue;
8098	}
8099	}
8100
8101	/*
8102	* We don't want a small process to block large processes from
8103	* being notified again. <rdar://problem/7955532>
8104	*/
8105	resident_size = (get_task_phys_footprint(t))/(`1024``1024ULL`); /* MB /
8106
8107	if (resident_size >= vm_pressure_task_footprint_min) {
8108
8109	if (level > `0`) {
8110	/*
8111	* Warning or Critical Pressure.
8112	*/
8113	if (pressure_increase) {
8114	if ((curr_task_importance < selected_task_importance) \|\|
8115	((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
8116
8117	/*
8118	* We have found a candidate process which is:
8119	* a) at a lower importance than the current selected process
8120	* OR
8121	* b) has importance equal to that of the current selected process but is larger
8122	*/
8123
8124	consider_knote = TRUE;
8125	}
8126	} else {
8127	if ((curr_task_importance > selected_task_importance) \|\|
8128	((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
8129
8130	/*
8131	* We have found a candidate process which is:
8132	* a) at a higher importance than the current selected process
8133	* OR
8134	* b) has importance equal to that of the current selected process but is larger
8135	*/
8136
8137	consider_knote = TRUE;
8138	}
8139	}
8140	} else if (level == `0`) {
8141	/*
8142	* Pressure back to normal.
8143	*/
8144	if ((curr_task_importance > selected_task_importance) \|\|
8145	((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
8146
8147	consider_knote = TRUE;
8148	}
8149	}
8150
8151	if (consider_knote) {
8152	resident_max = resident_size;
8153	kn_max = kn;
8154	selected_task_importance = curr_task_importance;
8155	consider_knote = FALSE; / reset for the next candidate /
8156	}
8157	} else {
8158	/ There was no candidate with enough resident memory to scavenge /
8159	VM_PRESSURE_DEBUG(`0`, "[vm_pressure] threshold failed for pid %d with %llu resident...\n", p->p_pid, resident_size);
8160	}
8161	proc_rele(p);
8162	}
8163
8164	done_scanning:
8165	if (kn_max) {
8166	VM_DEBUG_CONSTANT_EVENT(vm_pressure_event, VM_PRESSURE_EVENT, DBG_FUNC_NONE, knote_get_kq(kn_max)->kq_p->p_pid, resident_max, `0`, `0`);
8167	VM_PRESSURE_DEBUG(`1`, "[vm_pressure] sending event to pid %d with %llu resident\n", knote_get_kq(kn_max)->kq_p->p_pid, resident_max);
8168	}
8169
8170	return kn_max;
8171	}
8172
8173	#define VM_PRESSURE_DECREASED_SMOOTHING_PERIOD 5000 /* milliseconds */
8174	#define WARNING_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
8175	#define CRITICAL_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
8176
8177	uint64_t next_warning_notification_sent_at_ts = `0`;
8178	uint64_t next_critical_notification_sent_at_ts = `0`;
8179
8180	kern_return_t
8181	memorystatus_update_vm_pressure(boolean_t target_foreground_process)
8182	{
8183	struct knote *kn_max = NULL;
8184	struct knote kn_cur = NULL, kn_temp = NULL; / for safe list traversal /
8185	pid_t target_pid = -`1`;
8186	struct klist dispatch_klist = { NULL };
8187	proc_t target_proc = PROC_NULL;
8188	struct task *task = NULL;
8189	boolean_t found_candidate = FALSE;
8190
8191	static vm_pressure_level_t level_snapshot = kVMPressureNormal;
8192	static vm_pressure_level_t prev_level_snapshot = kVMPressureNormal;
8193	boolean_t smoothing_window_started = FALSE;
8194	struct timeval smoothing_window_start_tstamp = {`0`, `0`};
8195	struct timeval curr_tstamp = {`0`, `0`};
8196	int elapsed_msecs = `0`;
8197	uint64_t curr_ts = mach_absolute_time();
8198
8199	#if !CONFIG_JETSAM
8200	#define MAX_IDLE_KILLS 100 /* limit the number of idle kills allowed */
8201
8202	int idle_kill_counter = `0`;
8203
8204	/*
8205	* On desktop we take this opportunity to free up memory pressure
8206	* by immediately killing idle exitable processes. We use a delay
8207	* to avoid overkill. And we impose a max counter as a fail safe
8208	* in case daemons re-launch too fast.
8209	*/
8210	while ((memorystatus_vm_pressure_level != kVMPressureNormal) && (idle_kill_counter < MAX_IDLE_KILLS)) {
8211	if (memorystatus_idle_exit_from_VM() == FALSE) {
8212	/ No idle exitable processes left to kill /
8213	break;
8214	}
8215	idle_kill_counter++;
8216
8217	if (memorystatus_manual_testing_on == TRUE) {
8218	/*
8219	* Skip the delay when testing
8220	* the pressure notification scheme.
8221	*/
8222	} else {
8223	delay(`1000000`); / 1 second /
8224	}
8225	}
8226	#endif /* !CONFIG_JETSAM */
8227
8228	if (level_snapshot != kVMPressureNormal) {
8229
8230	/*
8231	* Check to see if we are still in the 'resting' period
8232	* after having notified all clients interested in
8233	* a particular pressure level.
8234	*/
8235
8236	level_snapshot = memorystatus_vm_pressure_level;
8237
8238	if (level_snapshot == kVMPressureWarning \|\| level_snapshot == kVMPressureUrgent) {
8239
8240	if (next_warning_notification_sent_at_ts) {
8241	if (curr_ts < next_warning_notification_sent_at_ts) {
8242	delay(INTER_NOTIFICATION_DELAY * `4` / 1 sec /);
8243	return KERN_SUCCESS;
8244	}
8245
8246	next_warning_notification_sent_at_ts = `0`;
8247	memorystatus_klist_reset_all_for_level(kVMPressureWarning);
8248	}
8249	} else if (level_snapshot == kVMPressureCritical) {
8250
8251	if (next_critical_notification_sent_at_ts) {
8252	if (curr_ts < next_critical_notification_sent_at_ts) {
8253	delay(INTER_NOTIFICATION_DELAY * `4` / 1 sec /);
8254	return KERN_SUCCESS;
8255	}
8256	next_critical_notification_sent_at_ts = `0`;
8257	memorystatus_klist_reset_all_for_level(kVMPressureCritical);
8258	}
8259	}
8260	}
8261
8262	while (`1`) {
8263
8264	/*
8265	* There is a race window here. But it's not clear
8266	* how much we benefit from having extra synchronization.
8267	*/
8268	level_snapshot = memorystatus_vm_pressure_level;
8269
8270	if (prev_level_snapshot > level_snapshot) {
8271	/*
8272	* Pressure decreased? Let's take a little breather
8273	* and see if this condition stays.
8274	*/
8275	if (smoothing_window_started == FALSE) {
8276
8277	smoothing_window_started = TRUE;
8278	microuptime(&smoothing_window_start_tstamp);
8279	}
8280
8281	microuptime(&curr_tstamp);
8282	timevalsub(&curr_tstamp, &smoothing_window_start_tstamp);
8283	elapsed_msecs = curr_tstamp.tv_sec * `1000` + curr_tstamp.tv_usec / `1000`;
8284
8285	if (elapsed_msecs < VM_PRESSURE_DECREASED_SMOOTHING_PERIOD) {
8286
8287	delay(INTER_NOTIFICATION_DELAY);
8288	continue;
8289	}
8290	}
8291
8292	prev_level_snapshot = level_snapshot;
8293	smoothing_window_started = FALSE;
8294
8295	memorystatus_klist_lock();
8296	kn_max = vm_pressure_select_optimal_candidate_to_notify(&memorystatus_klist, level_snapshot, target_foreground_process);
8297
8298	if (kn_max == NULL) {
8299	memorystatus_klist_unlock();
8300
8301	/*
8302	* No more level-based clients to notify.
8303	*
8304	* Start the 'resting' window within which clients will not be re-notified.
8305	*/
8306
8307	if (level_snapshot != kVMPressureNormal) {
8308	if (level_snapshot == kVMPressureWarning \|\| level_snapshot == kVMPressureUrgent) {
8309	nanoseconds_to_absolutetime(WARNING_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
8310
8311	/ Next warning notification (if nothing changes) won't be sent before.../
8312	next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts;
8313	}
8314
8315	if (level_snapshot == kVMPressureCritical) {
8316	nanoseconds_to_absolutetime(CRITICAL_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
8317
8318	/ Next critical notification (if nothing changes) won't be sent before.../
8319	next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts;
8320	}
8321	}
8322	return KERN_FAILURE;
8323	}
8324
8325	target_proc = knote_get_kq(kn_max)->kq_p;
8326
8327	proc_list_lock();
8328	if (target_proc != proc_ref_locked(target_proc)) {
8329	target_proc = PROC_NULL;
8330	proc_list_unlock();
8331	memorystatus_klist_unlock();
8332	continue;
8333	}
8334	proc_list_unlock();
8335
8336	target_pid = target_proc->p_pid;
8337
8338	task = (struct task *)(target_proc->task);
8339
8340	if (level_snapshot != kVMPressureNormal) {
8341
8342	if (level_snapshot == kVMPressureWarning \|\| level_snapshot == kVMPressureUrgent) {
8343
8344	if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_WARN, task, `0`, kVMPressureWarning) == TRUE) {
8345	found_candidate = TRUE;
8346	}
8347	} else {
8348	if (level_snapshot == kVMPressureCritical) {
8349
8350	if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_CRITICAL, task, `0`, kVMPressureCritical) == TRUE) {
8351	found_candidate = TRUE;
8352	}
8353	}
8354	}
8355	} else {
8356	if (kn_max->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
8357
8358	task_clear_has_been_notified(task, kVMPressureWarning);
8359	task_clear_has_been_notified(task, kVMPressureCritical);
8360
8361	found_candidate = TRUE;
8362	}
8363	}
8364
8365	if (found_candidate == FALSE) {
8366	proc_rele(target_proc);
8367	memorystatus_klist_unlock();
8368	continue;
8369	}
8370
8371	SLIST_FOREACH_SAFE(kn_cur, &memorystatus_klist, kn_selnext, kn_temp) {
8372
8373	int knote_pressure_level = convert_internal_pressure_level_to_dispatch_level(level_snapshot);
8374
8375	if (is_knote_registered_modify_task_pressure_bits(kn_cur, knote_pressure_level, task, `0`, level_snapshot) == TRUE) {
8376	proc_t knote_proc = knote_get_kq(kn_cur)->kq_p;
8377	pid_t knote_pid = knote_proc->p_pid;
8378	if (knote_pid == target_pid) {
8379	KNOTE_DETACH(&memorystatus_klist, kn_cur);
8380	KNOTE_ATTACH(&dispatch_klist, kn_cur);
8381	}
8382	}
8383	}
8384
8385	KNOTE(&dispatch_klist, (level_snapshot != kVMPressureNormal) ? kMemorystatusPressure : kMemorystatusNoPressure);
8386
8387	SLIST_FOREACH_SAFE(kn_cur, &dispatch_klist, kn_selnext, kn_temp) {
8388	KNOTE_DETACH(&dispatch_klist, kn_cur);
8389	KNOTE_ATTACH(&memorystatus_klist, kn_cur);
8390	}
8391
8392	memorystatus_klist_unlock();
8393
8394	microuptime(&target_proc->vm_pressure_last_notify_tstamp);
8395	proc_rele(target_proc);
8396
8397	if (memorystatus_manual_testing_on == TRUE && target_foreground_process == TRUE) {
8398	break;
8399	}
8400
8401	if (memorystatus_manual_testing_on == TRUE) {
8402	/*
8403	* Testing out the pressure notification scheme.
8404	* No need for delays etc.
8405	*/
8406	} else {
8407
8408	uint32_t sleep_interval = INTER_NOTIFICATION_DELAY;
8409	#if CONFIG_JETSAM
8410	unsigned int page_delta = `0`;
8411	unsigned int skip_delay_page_threshold = `0`;
8412
8413	assert(memorystatus_available_pages_pressure >= memorystatus_available_pages_critical_base);
8414
8415	page_delta = (memorystatus_available_pages_pressure - memorystatus_available_pages_critical_base) / `2`;
8416	skip_delay_page_threshold = memorystatus_available_pages_pressure - page_delta;
8417
8418	if (memorystatus_available_pages <= skip_delay_page_threshold) {
8419	/*
8420	* We are nearing the critcal mark fast and can't afford to wait between
8421	* notifications.
8422	*/
8423	sleep_interval = `0`;
8424	}
8425	#endif /* CONFIG_JETSAM */
8426
8427	if (sleep_interval) {
8428	delay(sleep_interval);
8429	}
8430	}
8431	}
8432
8433	return KERN_SUCCESS;
8434	}
8435
8436	vm_pressure_level_t
8437	convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t internal_pressure_level)
8438	{
8439	vm_pressure_level_t dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
8440
8441	switch (internal_pressure_level) {
8442
8443	case kVMPressureNormal:
8444	{
8445	dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
8446	break;
8447	}
8448
8449	case kVMPressureWarning:
8450	case kVMPressureUrgent:
8451	{
8452	dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_WARN;
8453	break;
8454	}
8455
8456	case kVMPressureCritical:
8457	{
8458	dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
8459	break;
8460	}
8461
8462	default:
8463	break;
8464	}
8465
8466	return dispatch_level;
8467	}
8468
8469	static int
8470	sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS
8471	{
8472	#pragma unused(arg1, arg2, oidp)
8473	#if CONFIG_EMBEDDED
8474	int error = `0`;
8475
8476	error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, `0`);
8477	if (error)
8478	return (error);
8479
8480	#endif /* CONFIG_EMBEDDED */
8481	vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level);
8482
8483	return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level));
8484	}
8485
8486	#if DEBUG \|\| DEVELOPMENT
8487
8488	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT\|CTLFLAG_RD\|CTLFLAG_LOCKED,
8489	`0`, `0`, &sysctl_memorystatus_vm_pressure_level, "I", "");
8490
8491	#else /* DEBUG \|\| DEVELOPMENT */
8492
8493	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT\|CTLFLAG_RD\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
8494	`0`, `0`, &sysctl_memorystatus_vm_pressure_level, "I", "");
8495
8496	#endif /* DEBUG \|\| DEVELOPMENT */
8497
8498
8499	static int
8500	sysctl_memorypressure_manual_trigger SYSCTL_HANDLER_ARGS
8501	{
8502	#pragma unused(arg1, arg2)
8503
8504	int level = `0`;
8505	int error = `0`;
8506	int pressure_level = `0`;
8507	int trigger_request = `0`;
8508	int force_purge;
8509
8510	error = sysctl_handle_int(oidp, &level, `0`, req);
8511	if (error \|\| !req->newptr) {
8512	return (error);
8513	}
8514
8515	memorystatus_manual_testing_on = TRUE;
8516
8517	trigger_request = (level >> `16`) & `0xFFFF`;
8518	pressure_level = (level & `0xFFFF`);
8519
8520	if (trigger_request < TEST_LOW_MEMORY_TRIGGER_ONE \|\|
8521	trigger_request > TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL) {
8522	return EINVAL;
8523	}
8524	switch (pressure_level) {
8525	case NOTE_MEMORYSTATUS_PRESSURE_NORMAL:
8526	case NOTE_MEMORYSTATUS_PRESSURE_WARN:
8527	case NOTE_MEMORYSTATUS_PRESSURE_CRITICAL:
8528	break;
8529	default:
8530	return EINVAL;
8531	}
8532
8533	/*
8534	* The pressure level is being set from user-space.
8535	* And user-space uses the constants in sys/event.h
8536	* So we translate those events to our internal levels here.
8537	*/
8538	if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
8539
8540	memorystatus_manual_testing_level = kVMPressureNormal;
8541	force_purge = `0`;
8542
8543	} else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_WARN) {
8544
8545	memorystatus_manual_testing_level = kVMPressureWarning;
8546	force_purge = vm_pageout_state.memorystatus_purge_on_warning;
8547
8548	} else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
8549
8550	memorystatus_manual_testing_level = kVMPressureCritical;
8551	force_purge = vm_pageout_state.memorystatus_purge_on_critical;
8552	}
8553
8554	memorystatus_vm_pressure_level = memorystatus_manual_testing_level;
8555
8556	/ purge according to the new pressure level /
8557	switch (trigger_request) {
8558	case TEST_PURGEABLE_TRIGGER_ONE:
8559	case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE:
8560	if (force_purge == `0`) {
8561	/ no purging requested /
8562	break;
8563	}
8564	vm_purgeable_object_purge_one_unlocked(force_purge);
8565	break;
8566	case TEST_PURGEABLE_TRIGGER_ALL:
8567	case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL:
8568	if (force_purge == `0`) {
8569	/ no purging requested /
8570	break;
8571	}
8572	while (vm_purgeable_object_purge_one_unlocked(force_purge));
8573	break;
8574	}
8575
8576	if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ONE) \|\|
8577	(trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE)) {
8578
8579	memorystatus_update_vm_pressure(TRUE);
8580	}
8581
8582	if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ALL) \|\|
8583	(trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL)) {
8584
8585	while (memorystatus_update_vm_pressure(FALSE) == KERN_SUCCESS) {
8586	continue;
8587	}
8588	}
8589
8590	if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
8591	memorystatus_manual_testing_on = FALSE;
8592	}
8593
8594	return `0`;
8595	}
8596
8597	SYSCTL_PROC(_kern, OID_AUTO, memorypressure_manual_trigger, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
8598	`0`, `0`, &sysctl_memorypressure_manual_trigger, "I", "");
8599
8600
8601	SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_warning, CTLFLAG_RW\|CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_warning, `0`, "");
8602	SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_urgent, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_urgent, `0`, "");
8603	SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_critical, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_critical, `0`, "");
8604
8605	#if DEBUG \|\| DEVELOPMENT
8606	SYSCTL_UINT(_kern, OID_AUTO, memorystatus_vm_pressure_events_enabled, CTLFLAG_RW\|CTLFLAG_LOCKED, &vm_pressure_events_enabled, `0`, "");
8607	#endif
8608
8609	#endif /* VM_PRESSURE_EVENTS */
8610
8611	/ Return both allocated and actual size, since there's a race between allocation and list compilation /
8612	static int
8613	memorystatus_get_priority_list(memorystatus_priority_entry_t *list_ptr, size_t buffer_size, size_t *list_size, boolean_t size_only)
8614	{
8615	uint32_t list_count, i = `0`;
8616	memorystatus_priority_entry_t *list_entry;
8617	proc_t p;
8618
8619	list_count = memorystatus_list_count;
8620	list_size = sizeof(memorystatus_priority_entry_t) list_count;
8621
8622	/ Just a size check? /
8623	if (size_only) {
8624	return `0`;
8625	}
8626
8627	/ Otherwise, validate the size of the buffer /
8628	if (buffer_size < list_size) {
8629	return EINVAL;
8630	}
8631
8632	list_ptr = (memorystatus_priority_entry_t)kalloc(*list_size);
8633	if (!*list_ptr) {
8634	return ENOMEM;
8635	}
8636
8637	memset(list_ptr, `0`, list_size);
8638
8639	buffer_size = list_size;
8640	*list_size = `0`;
8641
8642	list_entry = *list_ptr;
8643
8644	proc_list_lock();
8645
8646	p = memorystatus_get_first_proc_locked(&i, TRUE);
8647	while (p && (list_size < buffer_size)) {
8648	list_entry->pid = p->p_pid;
8649	list_entry->priority = p->p_memstat_effectivepriority;
8650	list_entry->user_data = p->p_memstat_userdata;
8651
8652	if (p->p_memstat_memlimit <= `0`) {
8653	task_get_phys_footprint_limit(p->task, &list_entry->limit);
8654	} else {
8655	list_entry->limit = p->p_memstat_memlimit;
8656	}
8657
8658	list_entry->state = memorystatus_build_state(p);
8659	list_entry++;
8660
8661	list_size += sizeof*(memorystatus_priority_entry_t);
8662
8663	p = memorystatus_get_next_proc_locked(&i, p, TRUE);
8664	}
8665
8666	proc_list_unlock();
8667
8668	MEMORYSTATUS_DEBUG(`1`, "memorystatus_get_priority_list: returning %lu for size\n", (unsigned long)*list_size);
8669
8670	return `0`;
8671	}
8672
8673	static int
8674	memorystatus_get_priority_pid(pid_t pid, user_addr_t buffer, size_t buffer_size) {
8675	int error = `0`;
8676	memorystatus_priority_entry_t mp_entry;
8677
8678	/ Validate inputs /
8679	if ((pid == `0`) \|\| (buffer == USER_ADDR_NULL) \|\| (buffer_size != sizeof(memorystatus_priority_entry_t))) {
8680	return EINVAL;
8681	}
8682
8683	proc_t p = proc_find(pid);
8684	if (!p) {
8685	return ESRCH;
8686	}
8687
8688	memset (&mp_entry, `0`, sizeof(memorystatus_priority_entry_t));
8689
8690	mp_entry.pid = p->p_pid;
8691	mp_entry.priority = p->p_memstat_effectivepriority;
8692	mp_entry.user_data = p->p_memstat_userdata;
8693	if (p->p_memstat_memlimit <= `0`) {
8694	task_get_phys_footprint_limit(p->task, &mp_entry.limit);
8695	} else {
8696	mp_entry.limit = p->p_memstat_memlimit;
8697	}
8698	mp_entry.state = memorystatus_build_state(p);
8699
8700	proc_rele(p);
8701
8702	error = copyout(&mp_entry, buffer, buffer_size);
8703
8704	return (error);
8705	}
8706
8707	static int
8708	memorystatus_cmd_get_priority_list(pid_t pid, user_addr_t buffer, size_t buffer_size, int32_t *retval) {
8709	int error = `0`;
8710	boolean_t size_only;
8711	size_t list_size;
8712
8713	/*
8714	* When a non-zero pid is provided, the 'list' has only one entry.
8715	*/
8716
8717	size_only = ((buffer == USER_ADDR_NULL) ? TRUE: FALSE);
8718
8719	if (pid != `0`) {
8720	list_size = sizeof(memorystatus_priority_entry_t) * `1`;
8721	if (!size_only) {
8722	error = memorystatus_get_priority_pid(pid, buffer, buffer_size);
8723	}
8724	} else {
8725	memorystatus_priority_entry_t *list = NULL;
8726	error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only);
8727
8728	if (error == `0`) {
8729	if (!size_only) {
8730	error = copyout(list, buffer, list_size);
8731	}
8732	}
8733
8734	if (list) {
8735	kfree(list, buffer_size);
8736	}
8737	}
8738
8739	if (error == `0`) {
8740	*retval = list_size;
8741	}
8742
8743	return (error);
8744	}
8745
8746	static void
8747	memorystatus_clear_errors(void)
8748	{
8749	proc_t p;
8750	unsigned int i = `0`;
8751
8752	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) \| DBG_FUNC_START, `0`, `0`, `0`, `0`, `0`);
8753
8754	proc_list_lock();
8755
8756	p = memorystatus_get_first_proc_locked(&i, TRUE);
8757	while (p) {
8758	if (p->p_memstat_state & P_MEMSTAT_ERROR) {
8759	p->p_memstat_state &= ~P_MEMSTAT_ERROR;
8760	}
8761	p = memorystatus_get_next_proc_locked(&i, p, TRUE);
8762	}
8763
8764	proc_list_unlock();
8765
8766	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) \| DBG_FUNC_END, `0`, `0`, `0`, `0`, `0`);
8767	}
8768
8769	#if CONFIG_JETSAM
8770	static void
8771	memorystatus_update_levels_locked(boolean_t critical_only) {
8772
8773	memorystatus_available_pages_critical = memorystatus_available_pages_critical_base;
8774
8775	/*
8776	* If there's an entry in the first bucket, we have idle processes.
8777	*/
8778
8779	memstat_bucket_t *first_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
8780	if (first_bucket->count) {
8781	memorystatus_available_pages_critical += memorystatus_available_pages_critical_idle_offset;
8782
8783	if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
8784	/*
8785	* The critical threshold must never exceed the pressure threshold
8786	*/
8787	memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
8788	}
8789	}
8790
8791	#if DEBUG \|\| DEVELOPMENT
8792	if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
8793	memorystatus_available_pages_critical += memorystatus_jetsam_policy_offset_pages_diagnostic;
8794
8795	if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
8796	/*
8797	* The critical threshold must never exceed the pressure threshold
8798	*/
8799	memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
8800	}
8801	}
8802	#endif /* DEBUG \|\| DEVELOPMENT */
8803
8804	if (memorystatus_jetsam_policy & kPolicyMoreFree) {
8805	memorystatus_available_pages_critical += memorystatus_policy_more_free_offset_pages;
8806	}
8807
8808	if (critical_only) {
8809	return;
8810	}
8811
8812	#if VM_PRESSURE_EVENTS
8813	memorystatus_available_pages_pressure = (pressure_threshold_percentage / delta_percentage) * memorystatus_delta;
8814	#if DEBUG \|\| DEVELOPMENT
8815	if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
8816	memorystatus_available_pages_pressure += memorystatus_jetsam_policy_offset_pages_diagnostic;
8817	}
8818	#endif
8819	#endif
8820	}
8821
8822	void
8823	memorystatus_fast_jetsam_override(boolean_t enable_override)
8824	{
8825	/ If fast jetsam is not enabled, simply return /
8826	if (!fast_jetsam_enabled)
8827	return;
8828
8829	if (enable_override) {
8830	if ((memorystatus_jetsam_policy & kPolicyMoreFree) == kPolicyMoreFree)
8831	return;
8832	proc_list_lock();
8833	memorystatus_jetsam_policy \|= kPolicyMoreFree;
8834	memorystatus_thread_pool_max();
8835	memorystatus_update_levels_locked(TRUE);
8836	proc_list_unlock();
8837	} else {
8838	if ((memorystatus_jetsam_policy & kPolicyMoreFree) == `0`)
8839	return;
8840	proc_list_lock();
8841	memorystatus_jetsam_policy &= ~kPolicyMoreFree;
8842	memorystatus_thread_pool_default();
8843	memorystatus_update_levels_locked(TRUE);
8844	proc_list_unlock();
8845	}
8846	}
8847
8848
8849	static int
8850	sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS
8851	{
8852	#pragma unused(arg1, arg2, oidp)
8853	int error = `0`, more_free = `0`;
8854
8855	/*
8856	* TODO: Enable this privilege check?
8857	*
8858	* error = priv_check_cred(kauth_cred_get(), PRIV_VM_JETSAM, 0);
8859	* if (error)
8860	* return (error);
8861	*/
8862
8863	error = sysctl_handle_int(oidp, &more_free, `0`, req);
8864	if (error \|\| !req->newptr)
8865	return (error);
8866
8867	if (more_free) {
8868	memorystatus_fast_jetsam_override(true);
8869	} else {
8870	memorystatus_fast_jetsam_override(false);
8871	}
8872
8873	return `0`;
8874	}
8875	SYSCTL_PROC(_kern, OID_AUTO, memorystatus_policy_more_free, CTLTYPE_INT\|CTLFLAG_WR\|CTLFLAG_LOCKED\|CTLFLAG_MASKED,
8876	`0`, `0`, &sysctl_kern_memorystatus_policy_more_free, "I", "");
8877
8878	#endif /* CONFIG_JETSAM */
8879
8880	/*
8881	* Get the at_boot snapshot
8882	*/
8883	static int
8884	memorystatus_get_at_boot_snapshot(memorystatus_jetsam_snapshot_t *snapshot, size_t snapshot_size, boolean_t size_only) {
8885	size_t input_size = *snapshot_size;
8886
8887	/*
8888	* The at_boot snapshot has no entry list.
8889	*/
8890	snapshot_size = sizeof*(memorystatus_jetsam_snapshot_t);
8891
8892	if (size_only) {
8893	return `0`;
8894	}
8895
8896	/*
8897	* Validate the size of the snapshot buffer
8898	*/
8899	if (input_size < *snapshot_size) {
8900	return EINVAL;
8901	}
8902
8903	/*
8904	* Update the notification_time only
8905	*/
8906	memorystatus_at_boot_snapshot.notification_time = mach_absolute_time();
8907	*snapshot = &memorystatus_at_boot_snapshot;
8908
8909	MEMORYSTATUS_DEBUG(`7`, "memorystatus_get_at_boot_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%d)\n",
8910	(long)input_size, (long)*snapshot_size, `0`);
8911	return `0`;
8912	}
8913
8914	/*
8915	* Get the previous fully populated snapshot
8916	*/
8917	static int
8918	memorystatus_get_jetsam_snapshot_copy(memorystatus_jetsam_snapshot_t *snapshot, size_t snapshot_size, boolean_t size_only) {
8919	size_t input_size = *snapshot_size;
8920
8921	if (memorystatus_jetsam_snapshot_copy_count > `0`) {
8922	snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) (memorystatus_jetsam_snapshot_copy_count));
8923	} else {
8924	*snapshot_size = `0`;
8925	}
8926
8927	if (size_only) {
8928	return `0`;
8929	}
8930
8931	if (input_size < *snapshot_size) {
8932	return EINVAL;
8933	}
8934
8935	*snapshot = memorystatus_jetsam_snapshot_copy;
8936
8937	MEMORYSTATUS_DEBUG(`7`, "memorystatus_get_jetsam_snapshot_copy: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
8938	(long)input_size, (long)snapshot_size, (long*)memorystatus_jetsam_snapshot_copy_count);
8939
8940	return `0`;
8941	}
8942
8943	static int
8944	memorystatus_get_on_demand_snapshot(memorystatus_jetsam_snapshot_t *snapshot, size_t snapshot_size, boolean_t size_only) {
8945	size_t input_size = *snapshot_size;
8946	uint32_t ods_list_count = memorystatus_list_count;
8947	memorystatus_jetsam_snapshot_t ods = NULL; /* The on_demand snapshot buffer /
8948
8949	snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) (ods_list_count));
8950
8951	if (size_only) {
8952	return `0`;
8953	}
8954
8955	/*
8956	* Validate the size of the snapshot buffer.
8957	* This is inherently racey. May want to revisit
8958	* this error condition and trim the output when
8959	* it doesn't fit.
8960	*/
8961	if (input_size < *snapshot_size) {
8962	return EINVAL;
8963	}
8964
8965	/*
8966	* Allocate and initialize a snapshot buffer.
8967	*/
8968	ods = (memorystatus_jetsam_snapshot_t )kalloc(snapshot_size);
8969	if (!ods) {
8970	return (ENOMEM);
8971	}
8972
8973	memset(ods, `0`, *snapshot_size);
8974
8975	proc_list_lock();
8976	memorystatus_init_jetsam_snapshot_locked(ods, ods_list_count);
8977	proc_list_unlock();
8978
8979	/*
8980	* Return the kernel allocated, on_demand buffer.
8981	* The caller of this routine will copy the data out
8982	* to user space and then free the kernel allocated
8983	* buffer.
8984	*/
8985	*snapshot = ods;
8986
8987	MEMORYSTATUS_DEBUG(`7`, "memorystatus_get_on_demand_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
8988	(long)input_size, (long)snapshot_size, (long*)ods_list_count);
8989
8990	return `0`;
8991	}
8992
8993	static int
8994	memorystatus_get_jetsam_snapshot(memorystatus_jetsam_snapshot_t *snapshot, size_t snapshot_size, boolean_t size_only) {
8995	size_t input_size = *snapshot_size;
8996
8997	if (memorystatus_jetsam_snapshot_count > `0`) {
8998	snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) (memorystatus_jetsam_snapshot_count));
8999	} else {
9000	*snapshot_size = `0`;
9001	}
9002
9003	if (size_only) {
9004	return `0`;
9005	}
9006
9007	if (input_size < *snapshot_size) {
9008	return EINVAL;
9009	}
9010
9011	*snapshot = memorystatus_jetsam_snapshot;
9012
9013	MEMORYSTATUS_DEBUG(`7`, "memorystatus_get_jetsam_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
9014	(long)input_size, (long)snapshot_size, (long*)memorystatus_jetsam_snapshot_count);
9015
9016	return `0`;
9017	}
9018
9019
9020	static int
9021	memorystatus_cmd_get_jetsam_snapshot(int32_t flags, user_addr_t buffer, size_t buffer_size, int32_t *retval) {
9022	int error = EINVAL;
9023	boolean_t size_only;
9024	boolean_t is_default_snapshot = FALSE;
9025	boolean_t is_on_demand_snapshot = FALSE;
9026	boolean_t is_at_boot_snapshot = FALSE;
9027	memorystatus_jetsam_snapshot_t *snapshot;
9028
9029	size_only = ((buffer == USER_ADDR_NULL) ? TRUE : FALSE);
9030
9031	if (flags == `0`) {
9032	/ Default /
9033	is_default_snapshot = TRUE;
9034	error = memorystatus_get_jetsam_snapshot(&snapshot, &buffer_size, size_only);
9035	} else {
9036	if (flags & ~(MEMORYSTATUS_SNAPSHOT_ON_DEMAND \| MEMORYSTATUS_SNAPSHOT_AT_BOOT \| MEMORYSTATUS_SNAPSHOT_COPY)) {
9037	/*
9038	* Unsupported bit set in flag.
9039	*/
9040	return EINVAL;
9041	}
9042
9043	if (flags & (flags - `0x1`)) {
9044	/*
9045	* Can't have multiple flags set at the same time.
9046	*/
9047	return EINVAL;
9048	}
9049
9050	if (flags & MEMORYSTATUS_SNAPSHOT_ON_DEMAND) {
9051	is_on_demand_snapshot = TRUE;
9052	/*
9053	* When not requesting the size only, the following call will allocate
9054	* an on_demand snapshot buffer, which is freed below.
9055	*/
9056	error = memorystatus_get_on_demand_snapshot(&snapshot, &buffer_size, size_only);
9057
9058	} else if (flags & MEMORYSTATUS_SNAPSHOT_AT_BOOT) {
9059	is_at_boot_snapshot = TRUE;
9060	error = memorystatus_get_at_boot_snapshot(&snapshot, &buffer_size, size_only);
9061	} else if (flags & MEMORYSTATUS_SNAPSHOT_COPY) {
9062	error = memorystatus_get_jetsam_snapshot_copy(&snapshot, &buffer_size, size_only);
9063	} else {
9064	/*
9065	* Invalid flag setting.
9066	*/
9067	return EINVAL;
9068	}
9069	}
9070
9071	if (error) {
9072	goto out;
9073	}
9074
9075	/*
9076	* Copy the data out to user space and clear the snapshot buffer.
9077	* If working with the jetsam snapshot,
9078	* clearing the buffer means, reset the count.
9079	* If working with an on_demand snapshot
9080	* clearing the buffer means, free it.
9081	* If working with the at_boot snapshot
9082	* there is nothing to clear or update.
9083	* If working with a copy of the snapshot
9084	* there is nothing to clear or update.
9085	*/
9086	if (!size_only) {
9087	if ((error = copyout(snapshot, buffer, buffer_size)) == `0`) {
9088	if (is_default_snapshot) {
9089	/*
9090	* The jetsam snapshot is never freed, its count is simply reset.
9091	* However, we make a copy for any parties that might be interested
9092	* in the previous fully populated snapshot.
9093	*/
9094	proc_list_lock();
9095	memcpy(memorystatus_jetsam_snapshot_copy, memorystatus_jetsam_snapshot, memorystatus_jetsam_snapshot_size);
9096	memorystatus_jetsam_snapshot_copy_count = memorystatus_jetsam_snapshot_count;
9097	snapshot->entry_count = memorystatus_jetsam_snapshot_count = `0`;
9098	memorystatus_jetsam_snapshot_last_timestamp = `0`;
9099	proc_list_unlock();
9100	}
9101	}
9102
9103	if (is_on_demand_snapshot) {
9104	/*
9105	* The on_demand snapshot is always freed,
9106	* even if the copyout failed.
9107	*/
9108	if(snapshot) {
9109	kfree(snapshot, buffer_size);
9110	}
9111	}
9112	}
9113
9114	if (error == `0`) {
9115	*retval = buffer_size;
9116	}
9117	out:
9118	return error;
9119	}
9120
9121	/*
9122	* Routine: memorystatus_cmd_grp_set_priorities
9123	* Purpose: Update priorities for a group of processes.
9124	*
9125	* [priority]
9126	* Move each process out of its effective priority
9127	* band and into a new priority band.
9128	* Maintains relative order from lowest to highest priority.
9129	* In single band, maintains relative order from head to tail.
9130	*
9131	* eg: before [effectivepriority \| pid]
9132	* [18 \| p101 ]
9133	* [17 \| p55, p67, p19 ]
9134	* [12 \| p103 p10 ]
9135	* [ 7 \| p25 ]
9136	* [ 0 \| p71, p82, ]
9137	*
9138	* after [ new band \| pid]
9139	* [ xxx \| p71, p82, p25, p103, p10, p55, p67, p19, p101]
9140	*
9141	* Returns: 0 on success, else non-zero.
9142	*
9143	* Caveat: We know there is a race window regarding recycled pids.
9144	* A process could be killed before the kernel can act on it here.
9145	* If a pid cannot be found in any of the jetsam priority bands,
9146	* then we simply ignore it. No harm.
9147	* But, if the pid has been recycled then it could be an issue.
9148	* In that scenario, we might move an unsuspecting process to the new
9149	* priority band. It's not clear how the kernel can safeguard
9150	* against this, but it would be an extremely rare case anyway.
9151	* The caller of this api might avoid such race conditions by
9152	* ensuring that the processes passed in the pid list are suspended.
9153	*/
9154
9155
9156	static int
9157	memorystatus_cmd_grp_set_priorities(user_addr_t buffer, size_t buffer_size)
9158	{
9159
9160	/*
9161	* We only handle setting priority
9162	* per process
9163	*/
9164
9165	int error = `0`;
9166	memorystatus_properties_entry_v1_t *entries = NULL;
9167	uint32_t entry_count = `0`;
9168
9169	/ This will be the ordered proc list /
9170	typedef struct memorystatus_internal_properties {
9171	proc_t proc;
9172	int32_t priority;
9173	} memorystatus_internal_properties_t;
9174
9175	memorystatus_internal_properties_t *table = NULL;
9176	size_t table_size = `0`;
9177	uint32_t table_count = `0`;
9178
9179	uint32_t i = `0`;
9180	uint32_t bucket_index = `0`;
9181	boolean_t head_insert;
9182	int32_t new_priority;
9183
9184	proc_t p;
9185
9186	/ Verify inputs /
9187	if ((buffer == USER_ADDR_NULL) \|\| (buffer_size == `0`)) {
9188	error = EINVAL;
9189	goto out;
9190	}
9191
9192	entry_count = (buffer_size / sizeof(memorystatus_properties_entry_v1_t));
9193	if ((entries = (memorystatus_properties_entry_v1_t *)kalloc(buffer_size)) == NULL) {
9194	error = ENOMEM;
9195	goto out;
9196	}
9197
9198	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) \| DBG_FUNC_START, MEMORYSTATUS_FLAGS_GRP_SET_PRIORITY, entry_count, `0`, `0`, `0`);
9199
9200	if ((error = copyin(buffer, entries, buffer_size)) != `0`) {
9201	goto out;
9202	}
9203
9204	/ Verify sanity of input priorities /
9205	if (entries[`0`].version == MEMORYSTATUS_MPE_VERSION_1) {
9206	if ((buffer_size % MEMORYSTATUS_MPE_VERSION_1_SIZE) != `0`) {
9207	error = EINVAL;
9208	goto out;
9209	}
9210	} else {
9211	error = EINVAL;
9212	goto out;
9213	}
9214
9215	for (i=`0`; i < entry_count; i++) {
9216	if (entries[i].priority == -`1`) {
9217	/ Use as shorthand for default priority /
9218	entries[i].priority = JETSAM_PRIORITY_DEFAULT;
9219	} else if ((entries[i].priority == system_procs_aging_band) \|\| (entries[i].priority == applications_aging_band)) {
9220	/ Both the aging bands are reserved for internal use;*
9221	* if requested, adjust to JETSAM_PRIORITY_IDLE. */
9222	entries[i].priority = JETSAM_PRIORITY_IDLE;
9223	} else if (entries[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
9224	/ JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle*
9225	* queue */
9226	/ Deal with this later /
9227	} else if ((entries[i].priority < `0`) \|\| (entries[i].priority >= MEMSTAT_BUCKET_COUNT)) {
9228	/ Sanity check /
9229	error = EINVAL;
9230	goto out;
9231	}
9232	}
9233
9234	table_size = sizeof(memorystatus_internal_properties_t) * entry_count;
9235	if ( (table = (memorystatus_internal_properties_t *)kalloc(table_size)) == NULL) {
9236	error = ENOMEM;
9237	goto out;
9238	}
9239	memset(table, `0`, table_size);
9240
9241
9242	/*
9243	* For each jetsam bucket entry, spin through the input property list.
9244	* When a matching pid is found, populate an adjacent table with the
9245	* appropriate proc pointer and new property values.
9246	* This traversal automatically preserves order from lowest
9247	* to highest priority.
9248	*/
9249
9250	bucket_index=`0`;
9251
9252	proc_list_lock();
9253
9254	/ Create the ordered table /
9255	p = memorystatus_get_first_proc_locked(&bucket_index, TRUE);
9256	while (p && (table_count < entry_count)) {
9257	for (i=`0`; i < entry_count; i++ ) {
9258	if (p->p_pid == entries[i].pid) {
9259	/ Build the table data /
9260	table[table_count].proc = p;
9261	table[table_count].priority = entries[i].priority;
9262	table_count++;
9263	break;
9264	}
9265	}
9266	p = memorystatus_get_next_proc_locked(&bucket_index, p, TRUE);
9267	}
9268
9269	/ We now have ordered list of procs ready to move /
9270	for (i=`0`; i < table_count; i++) {
9271	p = table[i].proc;
9272	assert(p != NULL);
9273
9274	/ Allow head inserts -- but relative order is now /
9275	if (table[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
9276	new_priority = JETSAM_PRIORITY_IDLE;
9277	head_insert = true;
9278	} else {
9279	new_priority = table[i].priority;
9280	head_insert = false;
9281	}
9282
9283	/ Not allowed /
9284	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
9285	continue;
9286	}
9287
9288	/*
9289	* Take appropriate steps if moving proc out of
9290	* either of the aging bands.
9291	*/
9292	if ((p->p_memstat_effectivepriority == system_procs_aging_band) \|\| (p->p_memstat_effectivepriority == applications_aging_band)) {
9293	memorystatus_invalidate_idle_demotion_locked(p, TRUE);
9294	}
9295
9296	memorystatus_update_priority_locked(p, new_priority, head_insert, false);
9297	}
9298
9299	proc_list_unlock();
9300
9301	/*
9302	* if (table_count != entry_count)
9303	* then some pids were not found in a jetsam band.
9304	* harmless but interesting...
9305	*/
9306	out:
9307	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) \| DBG_FUNC_END, MEMORYSTATUS_FLAGS_GRP_SET_PRIORITY, entry_count, table_count, `0`, `0`);
9308
9309	if (entries)
9310	kfree(entries, buffer_size);
9311	if (table)
9312	kfree(table, table_size);
9313
9314	return (error);
9315	}
9316
9317	static int
9318	memorystatus_cmd_grp_set_probabilities(user_addr_t buffer, size_t buffer_size)
9319	{
9320	int error = `0`;
9321	memorystatus_properties_entry_v1_t *entries = NULL;
9322	uint32_t entry_count = `0`, i = `0`;
9323	memorystatus_internal_probabilities_t tmp_table_new = NULL, tmp_table_old = NULL;
9324	size_t tmp_table_new_size = `0`, tmp_table_old_size = `0`;
9325
9326	/ Verify inputs /
9327	if ((buffer == USER_ADDR_NULL) \|\| (buffer_size == `0`)) {
9328	error = EINVAL;
9329	goto out;
9330	}
9331
9332	entry_count = (buffer_size / sizeof(memorystatus_properties_entry_v1_t));
9333
9334	if ((entries = (memorystatus_properties_entry_v1_t *) kalloc(buffer_size)) == NULL) {
9335	error = ENOMEM;
9336	goto out;
9337	}
9338
9339	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) \| DBG_FUNC_START, MEMORYSTATUS_FLAGS_GRP_SET_PROBABILITY, entry_count, `0`, `0`, `0`);
9340
9341	if ((error = copyin(buffer, entries, buffer_size)) != `0`) {
9342	goto out;
9343	}
9344
9345	if (entries[`0`].version == MEMORYSTATUS_MPE_VERSION_1) {
9346	if ((buffer_size % MEMORYSTATUS_MPE_VERSION_1_SIZE) != `0`) {
9347	error = EINVAL;
9348	goto out;
9349	}
9350	} else {
9351	error = EINVAL;
9352	goto out;
9353	}
9354
9355	/ Verify sanity of input priorities /
9356	for (i=`0`; i < entry_count; i++) {
9357	/*
9358	* 0 - low probability of use.
9359	* 1 - high probability of use.
9360	*
9361	* Keeping this field an int (& not a bool) to allow
9362	* us to experiment with different values/approaches
9363	* later on.
9364	*/
9365	if (entries[i].use_probability > `1`) {
9366	error = EINVAL;
9367	goto out;
9368	}
9369	}
9370
9371	tmp_table_new_size = sizeof(memorystatus_internal_probabilities_t) * entry_count;
9372
9373	if ( (tmp_table_new = (memorystatus_internal_probabilities_t *) kalloc(tmp_table_new_size)) == NULL) {
9374	error = ENOMEM;
9375	goto out;
9376	}
9377	memset(tmp_table_new, `0`, tmp_table_new_size);
9378
9379	proc_list_lock();
9380
9381	if (memorystatus_global_probabilities_table) {
9382	tmp_table_old = memorystatus_global_probabilities_table;
9383	tmp_table_old_size = memorystatus_global_probabilities_size;
9384	}
9385
9386	memorystatus_global_probabilities_table = tmp_table_new;
9387	memorystatus_global_probabilities_size = tmp_table_new_size;
9388	tmp_table_new = NULL;
9389
9390	for (i=`0`; i < entry_count; i++ ) {
9391	/ Build the table data /
9392	strlcpy(memorystatus_global_probabilities_table[i].proc_name, entries[i].proc_name, MAXCOMLEN + `1`);
9393	memorystatus_global_probabilities_table[i].use_probability = entries[i].use_probability;
9394	}
9395
9396	proc_list_unlock();
9397
9398	out:
9399	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) \| DBG_FUNC_END, MEMORYSTATUS_FLAGS_GRP_SET_PROBABILITY, entry_count, tmp_table_new_size, `0`, `0`);
9400
9401	if (entries) {
9402	kfree(entries, buffer_size);
9403	entries = NULL;
9404	}
9405
9406	if (tmp_table_old) {
9407	kfree(tmp_table_old, tmp_table_old_size);
9408	tmp_table_old = NULL;
9409	}
9410
9411	return (error);
9412
9413	}
9414
9415	static int
9416	memorystatus_cmd_grp_set_properties(int32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval)
9417	{
9418	int error = `0`;
9419
9420	if ((flags & MEMORYSTATUS_FLAGS_GRP_SET_PRIORITY) == MEMORYSTATUS_FLAGS_GRP_SET_PRIORITY) {
9421
9422	error = memorystatus_cmd_grp_set_priorities(buffer, buffer_size);
9423
9424	} else if ((flags & MEMORYSTATUS_FLAGS_GRP_SET_PROBABILITY) == MEMORYSTATUS_FLAGS_GRP_SET_PROBABILITY) {
9425
9426	error = memorystatus_cmd_grp_set_probabilities(buffer, buffer_size);
9427
9428	} else {
9429	error = EINVAL;
9430	}
9431
9432	return error;
9433	}
9434
9435	/*
9436	* This routine is used to update a process's jetsam priority position and stored user_data.
9437	* It is not used for the setting of memory limits, which is why the last 6 args to the
9438	* memorystatus_update() call are 0 or FALSE.
9439	*/
9440
9441	static int
9442	memorystatus_cmd_set_priority_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
9443	int error = `0`;
9444	memorystatus_priority_properties_t mpp_entry;
9445
9446	/ Validate inputs /
9447	if ((pid == `0`) \|\| (buffer == USER_ADDR_NULL) \|\| (buffer_size != sizeof(memorystatus_priority_properties_t))) {
9448	return EINVAL;
9449	}
9450
9451	error = copyin(buffer, &mpp_entry, buffer_size);
9452
9453	if (error == `0`) {
9454	proc_t p;
9455
9456	p = proc_find(pid);
9457	if (!p) {
9458	return ESRCH;
9459	}
9460
9461	if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
9462	proc_rele(p);
9463	return EPERM;
9464	}
9465
9466	error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, `0`, `0`, FALSE, FALSE);
9467	proc_rele(p);
9468	}
9469
9470	return(error);
9471	}
9472
9473	static int
9474	memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
9475	int error = `0`;
9476	memorystatus_memlimit_properties_t mmp_entry;
9477
9478	/ Validate inputs /
9479	if ((pid == `0`) \|\| (buffer == USER_ADDR_NULL) \|\| (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
9480	return EINVAL;
9481	}
9482
9483	error = copyin(buffer, &mmp_entry, buffer_size);
9484
9485	if (error == `0`) {
9486	error = memorystatus_set_memlimit_properties(pid, &mmp_entry);
9487	}
9488
9489	return(error);
9490	}
9491
9492	/*
9493	* When getting the memlimit settings, we can't simply call task_get_phys_footprint_limit().
9494	* That gets the proc's cached memlimit and there is no guarantee that the active/inactive
9495	* limits will be the same in the no-limit case. Instead we convert limits <= 0 using
9496	* task_convert_phys_footprint_limit(). It computes the same limit value that would be written
9497	* to the task's ledgers via task_set_phys_footprint_limit().
9498	*/
9499	static int
9500	memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
9501	int error = `0`;
9502	memorystatus_memlimit_properties_t mmp_entry;
9503
9504	/ Validate inputs /
9505	if ((pid == `0`) \|\| (buffer == USER_ADDR_NULL) \|\| (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
9506	return EINVAL;
9507	}
9508
9509	memset (&mmp_entry, `0`, sizeof(memorystatus_memlimit_properties_t));
9510
9511	proc_t p = proc_find(pid);
9512	if (!p) {
9513	return ESRCH;
9514	}
9515
9516	/*
9517	* Get the active limit and attributes.
9518	* No locks taken since we hold a reference to the proc.
9519	*/
9520
9521	if (p->p_memstat_memlimit_active > `0` ) {
9522	mmp_entry.memlimit_active = p->p_memstat_memlimit_active;
9523	} else {
9524	task_convert_phys_footprint_limit(-`1`, &mmp_entry.memlimit_active);
9525	}
9526
9527	if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) {
9528	mmp_entry.memlimit_active_attr \|= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
9529	}
9530
9531	/*
9532	* Get the inactive limit and attributes
9533	*/
9534	if (p->p_memstat_memlimit_inactive <= `0`) {
9535	task_convert_phys_footprint_limit(-`1`, &mmp_entry.memlimit_inactive);
9536	} else {
9537	mmp_entry.memlimit_inactive = p->p_memstat_memlimit_inactive;
9538	}
9539	if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) {
9540	mmp_entry.memlimit_inactive_attr \|= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
9541	}
9542	proc_rele(p);
9543
9544	error = copyout(&mmp_entry, buffer, buffer_size);
9545
9546	return(error);
9547	}
9548
9549
9550	/*
9551	* SPI for kbd - pr24956468
9552	* This is a very simple snapshot that calculates how much a
9553	* process's phys_footprint exceeds a specific memory limit.
9554	* Only the inactive memory limit is supported for now.
9555	* The delta is returned as bytes in excess or zero.
9556	*/
9557	static int
9558	memorystatus_cmd_get_memlimit_excess_np(pid_t pid, uint32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
9559	int error = `0`;
9560	uint64_t footprint_in_bytes = `0`;
9561	uint64_t delta_in_bytes = `0`;
9562	int32_t memlimit_mb = `0`;
9563	uint64_t memlimit_bytes = `0`;
9564
9565	/ Validate inputs /
9566	if ((pid == `0`) \|\| (buffer == USER_ADDR_NULL) \|\| (buffer_size != sizeof(uint64_t)) \|\| (flags != `0`)) {
9567	return EINVAL;
9568	}
9569
9570	proc_t p = proc_find(pid);
9571	if (!p) {
9572	return ESRCH;
9573	}
9574
9575	/*
9576	* Get the inactive limit.
9577	* No locks taken since we hold a reference to the proc.
9578	*/
9579
9580	if (p->p_memstat_memlimit_inactive <= `0`) {
9581	task_convert_phys_footprint_limit(-`1`, &memlimit_mb);
9582	} else {
9583	memlimit_mb = p->p_memstat_memlimit_inactive;
9584	}
9585
9586	footprint_in_bytes = get_task_phys_footprint(p->task);
9587
9588	proc_rele(p);
9589
9590	memlimit_bytes = memlimit_mb * `1024` * `1024`; / MB to bytes /
9591
9592	/*
9593	* Computed delta always returns >= 0 bytes
9594	*/
9595	if (footprint_in_bytes > memlimit_bytes) {
9596	delta_in_bytes = footprint_in_bytes - memlimit_bytes;
9597	}
9598
9599	error = copyout(&delta_in_bytes, buffer, sizeof(delta_in_bytes));
9600
9601	return(error);
9602	}
9603
9604
9605	static int
9606	memorystatus_cmd_get_pressure_status(int32_t *retval) {
9607	int error;
9608
9609	/ Need privilege for check /
9610	error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, `0`);
9611	if (error) {
9612	return (error);
9613	}
9614
9615	/ Inherently racy, so it's not worth taking a lock here /
9616	*retval = (kVMPressureNormal != memorystatus_vm_pressure_level) ? `1` : `0`;
9617
9618	return error;
9619	}
9620
9621	int
9622	memorystatus_get_pressure_status_kdp() {
9623	return (kVMPressureNormal != memorystatus_vm_pressure_level) ? `1` : `0`;
9624	}
9625
9626	/*
9627	* Every process, including a P_MEMSTAT_INTERNAL process (currently only pid 1), is allowed to set a HWM.
9628	*
9629	* This call is inflexible -- it does not distinguish between active/inactive, fatal/non-fatal
9630	* So, with 2-level HWM preserving previous behavior will map as follows.
9631	* - treat the limit passed in as both an active and inactive limit.
9632	* - treat the is_fatal_limit flag as though it applies to both active and inactive limits.
9633	*
9634	* When invoked via MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK
9635	* - the is_fatal_limit is FALSE, meaning the active and inactive limits are non-fatal/soft
9636	* - so mapping is (active/non-fatal, inactive/non-fatal)
9637	*
9638	* When invoked via MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT
9639	* - the is_fatal_limit is TRUE, meaning the process's active and inactive limits are fatal/hard
9640	* - so mapping is (active/fatal, inactive/fatal)
9641	*/
9642
9643	#if CONFIG_JETSAM
9644	static int
9645	memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit) {
9646	int error = `0`;
9647	memorystatus_memlimit_properties_t entry;
9648
9649	entry.memlimit_active = high_water_mark;
9650	entry.memlimit_active_attr = `0`;
9651	entry.memlimit_inactive = high_water_mark;
9652	entry.memlimit_inactive_attr = `0`;
9653
9654	if (is_fatal_limit == TRUE) {
9655	entry.memlimit_active_attr \|= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
9656	entry.memlimit_inactive_attr \|= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
9657	}
9658
9659	error = memorystatus_set_memlimit_properties(pid, &entry);
9660	return (error);
9661	}
9662	#endif /* CONFIG_JETSAM */
9663
9664	static int
9665	memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry) {
9666
9667	int32_t memlimit_active;
9668	boolean_t memlimit_active_is_fatal;
9669	int32_t memlimit_inactive;
9670	boolean_t memlimit_inactive_is_fatal;
9671	uint32_t valid_attrs = `0`;
9672	int error = `0`;
9673
9674	proc_t p = proc_find(pid);
9675	if (!p) {
9676	return ESRCH;
9677	}
9678
9679	/*
9680	* Check for valid attribute flags.
9681	*/
9682	valid_attrs \|= (MEMORYSTATUS_MEMLIMIT_ATTR_FATAL);
9683	if ((entry->memlimit_active_attr & (~valid_attrs)) != `0`) {
9684	proc_rele(p);
9685	return EINVAL;
9686	}
9687	if ((entry->memlimit_inactive_attr & (~valid_attrs)) != `0`) {
9688	proc_rele(p);
9689	return EINVAL;
9690	}
9691
9692	/*
9693	* Setup the active memlimit properties
9694	*/
9695	memlimit_active = entry->memlimit_active;
9696	if (entry->memlimit_active_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
9697	memlimit_active_is_fatal = TRUE;
9698	} else {
9699	memlimit_active_is_fatal = FALSE;
9700	}
9701
9702	/*
9703	* Setup the inactive memlimit properties
9704	*/
9705	memlimit_inactive = entry->memlimit_inactive;
9706	if (entry->memlimit_inactive_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
9707	memlimit_inactive_is_fatal = TRUE;
9708	} else {
9709	memlimit_inactive_is_fatal = FALSE;
9710	}
9711
9712	/*
9713	* Setting a limit of <= 0 implies that the process has no
9714	* high-water-mark and has no per-task-limit. That means
9715	* the system_wide task limit is in place, which by the way,
9716	* is always fatal.
9717	*/
9718
9719	if (memlimit_active <= `0`) {
9720	/*
9721	* Enforce the fatal system_wide task limit while process is active.
9722	*/
9723	memlimit_active = -`1`;
9724	memlimit_active_is_fatal = TRUE;
9725	}
9726
9727	if (memlimit_inactive <= `0`) {
9728	/*
9729	* Enforce the fatal system_wide task limit while process is inactive.
9730	*/
9731	memlimit_inactive = -`1`;
9732	memlimit_inactive_is_fatal = TRUE;
9733	}
9734
9735	proc_list_lock();
9736
9737	/*
9738	* Store the active limit variants in the proc.
9739	*/
9740	SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
9741
9742	/*
9743	* Store the inactive limit variants in the proc.
9744	*/
9745	SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
9746
9747	/*
9748	* Enforce appropriate limit variant by updating the cached values
9749	* and writing the ledger.
9750	* Limit choice is based on process active/inactive state.
9751	*/
9752
9753	if (memorystatus_highwater_enabled) {
9754	boolean_t is_fatal;
9755	boolean_t use_active;
9756
9757	if (proc_jetsam_state_is_active_locked(p) == TRUE) {
9758	CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
9759	use_active = TRUE;
9760	} else {
9761	CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
9762	use_active = FALSE;
9763	}
9764
9765	/ Enforce the limit by writing to the ledgers /
9766	error = (task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > `0`) ? p->p_memstat_memlimit : -`1`), NULL, use_active, is_fatal) == `0`) ? `0` : EINVAL;
9767
9768	MEMORYSTATUS_DEBUG(`3`, "memorystatus_set_memlimit_properties: new limit on pid %d (%dMB %s) current priority (%d) dirty_state?=0x%x %s\n",
9769	p->p_pid, (p->p_memstat_memlimit > `0` ? p->p_memstat_memlimit : -`1`),
9770	(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
9771	(p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
9772	DTRACE_MEMORYSTATUS2(memorystatus_set_memlimit, proc_t, p, int32_t, (p->p_memstat_memlimit > `0` ? p->p_memstat_memlimit : -`1`));
9773	}
9774
9775	proc_list_unlock();
9776	proc_rele(p);
9777
9778	return error;
9779	}
9780
9781	/*
9782	* Returns the jetsam priority (effective or requested) of the process
9783	* associated with this task.
9784	*/
9785	int
9786	proc_get_memstat_priority(proc_t p, boolean_t effective_priority)
9787	{
9788	if (p) {
9789	if (effective_priority) {
9790	return p->p_memstat_effectivepriority;
9791	} else {
9792	return p->p_memstat_requestedpriority;
9793	}
9794	}
9795	return `0`;
9796	}
9797
9798	static int
9799	memorystatus_get_process_is_managed(pid_t pid, int *is_managed)
9800	{
9801	proc_t p = NULL;
9802
9803	/ Validate inputs /
9804	if (pid == `0`) {
9805	return EINVAL;
9806	}
9807
9808	p = proc_find(pid);
9809	if (!p) {
9810	return ESRCH;
9811	}
9812
9813	proc_list_lock();
9814	*is_managed = ((p->p_memstat_state & P_MEMSTAT_MANAGED) ? `1` : `0`);
9815	proc_rele_locked(p);
9816	proc_list_unlock();
9817
9818	return `0`;
9819	}
9820
9821	static int
9822	memorystatus_set_process_is_managed(pid_t pid, boolean_t set_managed)
9823	{
9824	proc_t p = NULL;
9825
9826	/ Validate inputs /
9827	if (pid == `0`) {
9828	return EINVAL;
9829	}
9830
9831	p = proc_find(pid);
9832	if (!p) {
9833	return ESRCH;
9834	}
9835
9836	proc_list_lock();
9837	if (set_managed == TRUE) {
9838	p->p_memstat_state \|= P_MEMSTAT_MANAGED;
9839	} else {
9840	p->p_memstat_state &= ~P_MEMSTAT_MANAGED;
9841	}
9842	proc_rele_locked(p);
9843	proc_list_unlock();
9844
9845	return `0`;
9846	}
9847
9848	static int
9849	memorystatus_get_process_is_freezable(pid_t pid, int *is_freezable)
9850	{
9851	proc_t p = PROC_NULL;
9852
9853	if (pid == `0`) {
9854	return EINVAL;
9855	}
9856
9857	p = proc_find(pid);
9858	if (!p) {
9859	return ESRCH;
9860	}
9861
9862	/*
9863	* Only allow this on the current proc for now.
9864	* We can check for privileges and allow targeting another process in the future.
9865	*/
9866	if (p != current_proc()) {
9867	proc_rele(p);
9868	return EPERM;
9869	}
9870
9871	proc_list_lock();
9872	*is_freezable = ((p->p_memstat_state & P_MEMSTAT_FREEZE_DISABLED) ? `0` : `1`);
9873	proc_rele_locked(p);
9874	proc_list_unlock();
9875
9876	return `0`;
9877	}
9878
9879	static int
9880	memorystatus_set_process_is_freezable(pid_t pid, boolean_t is_freezable)
9881	{
9882	proc_t p = PROC_NULL;
9883
9884	if (pid == `0`) {
9885	return EINVAL;
9886	}
9887
9888	p = proc_find(pid);
9889	if (!p) {
9890	return ESRCH;
9891	}
9892
9893	/*
9894	* Only allow this on the current proc for now.
9895	* We can check for privileges and allow targeting another process in the future.
9896	*/
9897	if (p != current_proc()) {
9898	proc_rele(p);
9899	return EPERM;
9900	}
9901
9902	proc_list_lock();
9903	if (is_freezable == FALSE) {
9904	/ Freeze preference set to FALSE. Set the P_MEMSTAT_FREEZE_DISABLED bit. /
9905	p->p_memstat_state \|= P_MEMSTAT_FREEZE_DISABLED;
9906	printf("memorystatus_set_process_is_freezable: disabling freeze for pid %d [%s]\n",
9907	p->p_pid, (*p->p_name ? p->p_name : "unknown"));
9908	} else {
9909	p->p_memstat_state &= ~P_MEMSTAT_FREEZE_DISABLED;
9910	printf("memorystatus_set_process_is_freezable: enabling freeze for pid %d [%s]\n",
9911	p->p_pid, (*p->p_name ? p->p_name : "unknown"));
9912	}
9913	proc_rele_locked(p);
9914	proc_list_unlock();
9915
9916	return `0`;
9917	}
9918
9919	int
9920	memorystatus_control(struct proc p __unused, struct* memorystatus_control_args args, int* *ret) {
9921	int error = EINVAL;
9922	boolean_t skip_auth_check = FALSE;
9923	os_reason_t jetsam_reason = OS_REASON_NULL;
9924
9925	#if !CONFIG_JETSAM
9926	#pragma unused(ret)
9927	#pragma unused(jetsam_reason)
9928	#endif
9929
9930	/ We don't need entitlements if we're setting/ querying the freeze preference for a process. Skip the check below. /
9931	if (args->command == MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE \|\| args->command == MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE) {
9932	skip_auth_check = TRUE;
9933	}
9934
9935	/ Need to be root or have entitlement. /
9936	if (!kauth_cred_issuser(kauth_cred_get()) && !IOTaskHasEntitlement(current_task(), MEMORYSTATUS_ENTITLEMENT) && !skip_auth_check) {
9937	error = EPERM;
9938	goto out;
9939	}
9940
9941	/*
9942	* Sanity check.
9943	* Do not enforce it for snapshots.
9944	*/
9945	if (args->command != MEMORYSTATUS_CMD_GET_JETSAM_SNAPSHOT) {
9946	if (args->buffersize > MEMORYSTATUS_BUFFERSIZE_MAX) {
9947	error = EINVAL;
9948	goto out;
9949	}
9950	}
9951
9952	switch (args->command) {
9953	case MEMORYSTATUS_CMD_GET_PRIORITY_LIST:
9954	error = memorystatus_cmd_get_priority_list(args->pid, args->buffer, args->buffersize, ret);
9955	break;
9956	case MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES:
9957	error = memorystatus_cmd_set_priority_properties(args->pid, args->buffer, args->buffersize, ret);
9958	break;
9959	case MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES:
9960	error = memorystatus_cmd_set_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
9961	break;
9962	case MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES:
9963	error = memorystatus_cmd_get_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
9964	break;
9965	case MEMORYSTATUS_CMD_GET_MEMLIMIT_EXCESS:
9966	error = memorystatus_cmd_get_memlimit_excess_np(args->pid, args->flags, args->buffer, args->buffersize, ret);
9967	break;
9968	case MEMORYSTATUS_CMD_GRP_SET_PROPERTIES:
9969	error = memorystatus_cmd_grp_set_properties((int32_t)args->flags, args->buffer, args->buffersize, ret);
9970	break;
9971	case MEMORYSTATUS_CMD_GET_JETSAM_SNAPSHOT:
9972	error = memorystatus_cmd_get_jetsam_snapshot((int32_t)args->flags, args->buffer, args->buffersize, ret);
9973	break;
9974	case MEMORYSTATUS_CMD_GET_PRESSURE_STATUS:
9975	error = memorystatus_cmd_get_pressure_status(ret);
9976	break;
9977	#if CONFIG_JETSAM
9978	case MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK:
9979	/*
9980	* This call does not distinguish between active and inactive limits.
9981	* Default behavior in 2-level HWM world is to set both.
9982	* Non-fatal limit is also assumed for both.
9983	*/
9984	error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, FALSE);
9985	break;
9986	case MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT:
9987	/*
9988	* This call does not distinguish between active and inactive limits.
9989	* Default behavior in 2-level HWM world is to set both.
9990	* Fatal limit is also assumed for both.
9991	*/
9992	error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, TRUE);
9993	break;
9994	#endif /* CONFIG_JETSAM */
9995	/ Test commands /
9996	#if DEVELOPMENT \|\| DEBUG
9997	case MEMORYSTATUS_CMD_TEST_JETSAM:
9998	jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_GENERIC);
9999	if (jetsam_reason == OS_REASON_NULL) {
10000	printf("memorystatus_control: failed to allocate jetsam reason\n");
10001	}
10002
10003	error = memorystatus_kill_process_sync(args->pid, kMemorystatusKilled, jetsam_reason) ? `0` : EINVAL;
10004	break;
10005	case MEMORYSTATUS_CMD_TEST_JETSAM_SORT:
10006	error = memorystatus_cmd_test_jetsam_sort(args->pid, (int32_t)args->flags);
10007	break;
10008	#if CONFIG_JETSAM
10009	case MEMORYSTATUS_CMD_SET_JETSAM_PANIC_BITS:
10010	error = memorystatus_cmd_set_panic_bits(args->buffer, args->buffersize);
10011	break;
10012	#endif /* CONFIG_JETSAM */
10013	#else /* DEVELOPMENT \|\| DEBUG */
10014	#pragma unused(jetsam_reason)
10015	#endif /* DEVELOPMENT \|\| DEBUG */
10016	case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_ENABLE:
10017	if (memorystatus_aggressive_jetsam_lenient_allowed == FALSE) {
10018	#if DEVELOPMENT \|\| DEBUG
10019	printf("Enabling Lenient Mode\n");
10020	#endif /* DEVELOPMENT \|\| DEBUG */
10021
10022	memorystatus_aggressive_jetsam_lenient_allowed = TRUE;
10023	memorystatus_aggressive_jetsam_lenient = TRUE;
10024	error = `0`;
10025	}
10026	break;
10027	case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_DISABLE:
10028	#if DEVELOPMENT \|\| DEBUG
10029	printf("Disabling Lenient mode\n");
10030	#endif /* DEVELOPMENT \|\| DEBUG */
10031	memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
10032	memorystatus_aggressive_jetsam_lenient = FALSE;
10033	error = `0`;
10034	break;
10035	case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE:
10036	case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE:
10037	error = memorystatus_low_mem_privileged_listener(args->command);
10038	break;
10039
10040	case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE:
10041	case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE:
10042	error = memorystatus_update_inactive_jetsam_priority_band(args->pid, args->command, JETSAM_PRIORITY_ELEVATED_INACTIVE, args->flags ? TRUE : FALSE);
10043	break;
10044	case MEMORYSTATUS_CMD_SET_PROCESS_IS_MANAGED:
10045	error = memorystatus_set_process_is_managed(args->pid, args->flags);
10046	break;
10047
10048	case MEMORYSTATUS_CMD_GET_PROCESS_IS_MANAGED:
10049	error = memorystatus_get_process_is_managed(args->pid, ret);
10050	break;
10051
10052	case MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE:
10053	error = memorystatus_set_process_is_freezable(args->pid, args->flags ? TRUE : FALSE);
10054	break;
10055
10056	case MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE:
10057	error = memorystatus_get_process_is_freezable(args->pid, ret);
10058	break;
10059
10060	#if CONFIG_FREEZE
10061	#if DEVELOPMENT \|\| DEBUG
10062	case MEMORYSTATUS_CMD_FREEZER_CONTROL:
10063	error = memorystatus_freezer_control(args->flags, args->buffer, args->buffersize, ret);
10064	break;
10065	#endif /* DEVELOPMENT \|\| DEBUG */
10066	#endif /* CONFIG_FREEZE */
10067
10068	default:
10069	break;
10070	}
10071
10072	out:
10073	return error;
10074	}
10075
10076
10077	static int
10078	filt_memorystatusattach(struct knote kn, __unused struct* kevent_internal_s *kev)
10079	{
10080	int error;
10081
10082	kn->kn_flags \|= EV_CLEAR;
10083	error = memorystatus_knote_register(kn);
10084	if (error) {
10085	kn->kn_flags = EV_ERROR;
10086	kn->kn_data = error;
10087	}
10088	return `0`;
10089	}
10090
10091	static void
10092	filt_memorystatusdetach(struct knote *kn)
10093	{
10094	memorystatus_knote_unregister(kn);
10095	}
10096
10097	static int
10098	filt_memorystatus(struct knote kn __unused, long* hint)
10099	{
10100	if (hint) {
10101	switch (hint) {
10102	case kMemorystatusNoPressure:
10103	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
10104	kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
10105	}
10106	break;
10107	case kMemorystatusPressure:
10108	if (memorystatus_vm_pressure_level == kVMPressureWarning \|\| memorystatus_vm_pressure_level == kVMPressureUrgent) {
10109	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
10110	kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
10111	}
10112	} else if (memorystatus_vm_pressure_level == kVMPressureCritical) {
10113
10114	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
10115	kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
10116	}
10117	}
10118	break;
10119	case kMemorystatusLowSwap:
10120	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_LOW_SWAP) {
10121	kn->kn_fflags = NOTE_MEMORYSTATUS_LOW_SWAP;
10122	}
10123	break;
10124
10125	case kMemorystatusProcLimitWarn:
10126	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
10127	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
10128	}
10129	break;
10130
10131	case kMemorystatusProcLimitCritical:
10132	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
10133	kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
10134	}
10135	break;
10136
10137	default:
10138	break;
10139	}
10140	}
10141
10142	#if 0
10143	if (kn->kn_fflags != `0`) {
10144	proc_t knote_proc = knote_get_kq(kn)->kq_p;
10145	pid_t knote_pid = knote_proc->p_pid;
10146
10147	printf("filt_memorystatus: sending kn 0x%lx (event 0x%x) for pid (%d)\n",
10148	(unsigned long)kn, kn->kn_fflags, knote_pid);
10149	}
10150	#endif
10151
10152	return (kn->kn_fflags != `0`);
10153	}
10154
10155	static int
10156	filt_memorystatustouch(struct knote kn, struct* kevent_internal_s *kev)
10157	{
10158	int res;
10159	int prev_kn_sfflags = `0`;
10160
10161	memorystatus_klist_lock();
10162
10163	/*
10164	* copy in new kevent settings
10165	* (saving the "desired" data and fflags).
10166	*/
10167
10168	prev_kn_sfflags = kn->kn_sfflags;
10169	kn->kn_sfflags = (kev->fflags & EVFILT_MEMORYSTATUS_ALL_MASK);
10170
10171	#if !CONFIG_EMBEDDED
10172	/*
10173	* Only on desktop do we restrict notifications to
10174	* one per active/inactive state (soft limits only).
10175	*/
10176	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
10177	/*
10178	* Is there previous state to preserve?
10179	*/
10180	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
10181	/*
10182	* This knote was previously interested in proc_limit_warn,
10183	* so yes, preserve previous state.
10184	*/
10185	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
10186	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
10187	}
10188	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
10189	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
10190	}
10191	} else {
10192	/*
10193	* This knote was not previously interested in proc_limit_warn,
10194	* but it is now. Set both states.
10195	*/
10196	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
10197	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
10198	}
10199	}
10200
10201	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
10202	/*
10203	* Is there previous state to preserve?
10204	*/
10205	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
10206	/*
10207	* This knote was previously interested in proc_limit_critical,
10208	* so yes, preserve previous state.
10209	*/
10210	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
10211	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
10212	}
10213	if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
10214	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
10215	}
10216	} else {
10217	/*
10218	* This knote was not previously interested in proc_limit_critical,
10219	* but it is now. Set both states.
10220	*/
10221	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
10222	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
10223	}
10224	}
10225	#endif /* !CONFIG_EMBEDDED */
10226
10227	/*
10228	* reset the output flags based on a
10229	* combination of the old events and
10230	* the new desired event list.
10231	*/
10232	//kn->kn_fflags &= kn->kn_sfflags;
10233
10234	res = (kn->kn_fflags != `0`);
10235
10236	memorystatus_klist_unlock();
10237
10238	return res;
10239	}
10240
10241	static int
10242	filt_memorystatusprocess(struct knote kn, struct* filt_process_s data, struct* kevent_internal_s *kev)
10243	{
10244	#pragma unused(data)
10245	int res;
10246
10247	memorystatus_klist_lock();
10248	res = (kn->kn_fflags != `0`);
10249	if (res) {
10250	*kev = kn->kn_kevent;
10251	kn->kn_flags \|= EV_CLEAR; / automatic /
10252	kn->kn_fflags = `0`;
10253	kn->kn_data = `0`;
10254	}
10255	memorystatus_klist_unlock();
10256
10257	return res;
10258	}
10259
10260	static void
10261	memorystatus_klist_lock(void) {
10262	lck_mtx_lock(&memorystatus_klist_mutex);
10263	}
10264
10265	static void
10266	memorystatus_klist_unlock(void) {
10267	lck_mtx_unlock(&memorystatus_klist_mutex);
10268	}
10269
10270	void
10271	memorystatus_kevent_init(lck_grp_t grp, lck_attr_t attr) {
10272	lck_mtx_init(&memorystatus_klist_mutex, grp, attr);
10273	klist_init(&memorystatus_klist);
10274	}
10275
10276	int
10277	memorystatus_knote_register(struct knote *kn) {
10278	int error = `0`;
10279
10280	memorystatus_klist_lock();
10281
10282	/*
10283	* Support only userspace visible flags.
10284	*/
10285	if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == (unsigned int) kn->kn_sfflags) {
10286
10287	#if !CONFIG_EMBEDDED
10288	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
10289	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
10290	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
10291	}
10292
10293	if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
10294	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
10295	kn->kn_sfflags \|= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
10296	}
10297	#endif /* !CONFIG_EMBEDDED */
10298
10299	KNOTE_ATTACH(&memorystatus_klist, kn);
10300
10301	} else {
10302	error = ENOTSUP;
10303	}
10304
10305	memorystatus_klist_unlock();
10306
10307	return error;
10308	}
10309
10310	void
10311	memorystatus_knote_unregister(struct knote *kn __unused) {
10312	memorystatus_klist_lock();
10313	KNOTE_DETACH(&memorystatus_klist, kn);
10314	memorystatus_klist_unlock();
10315	}
10316
10317
10318	#if 0
10319	#if CONFIG_JETSAM && VM_PRESSURE_EVENTS
10320	static boolean_t
10321	memorystatus_issue_pressure_kevent(boolean_t pressured) {
10322	memorystatus_klist_lock();
10323	KNOTE(&memorystatus_klist, pressured ? kMemorystatusPressure : kMemorystatusNoPressure);
10324	memorystatus_klist_unlock();
10325	return TRUE;
10326	}
10327	#endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */
10328	#endif /* 0 */
10329
10330	/ Coalition support /
10331
10332	/ sorting info for a particular priority bucket /
10333	typedef struct memstat_sort_info {
10334	coalition_t msi_coal;
10335	uint64_t msi_page_count;
10336	pid_t msi_pid;
10337	int msi_ntasks;
10338	} memstat_sort_info_t;
10339
10340	/*
10341	* qsort from smallest page count to largest page count
10342	*
10343	* return < 0 for a < b
10344	* 0 for a == b
10345	* > 0 for a > b
10346	*/
10347	static int memstat_asc_cmp(const void a, const* void *b)
10348	{
10349	const memstat_sort_info_t msA = (const* memstat_sort_info_t *)a;
10350	const memstat_sort_info_t msB = (const* memstat_sort_info_t *)b;
10351
10352	return (int)((uint64_t)msA->msi_page_count - (uint64_t)msB->msi_page_count);
10353	}
10354
10355	/*
10356	* Return the number of pids rearranged during this sort.
10357	*/
10358	static int
10359	memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order)
10360	{
10361	#define MAX_SORT_PIDS 80
10362	#define MAX_COAL_LEADERS 10
10363
10364	unsigned int b = bucket_index;
10365	int nleaders = `0`;
10366	int ntasks = `0`;
10367	proc_t p = NULL;
10368	coalition_t coal = COALITION_NULL;
10369	int pids_moved = `0`;
10370	int total_pids_moved = `0`;
10371	int i;
10372
10373	/*
10374	* The system is typically under memory pressure when in this
10375	* path, hence, we want to avoid dynamic memory allocation.
10376	*/
10377	memstat_sort_info_t leaders[MAX_COAL_LEADERS];
10378	pid_t pid_list[MAX_SORT_PIDS];
10379
10380	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
10381	return(`0`);
10382	}
10383
10384	/*
10385	* Clear the array that holds coalition leader information
10386	*/
10387	for (i=`0`; i < MAX_COAL_LEADERS; i++) {
10388	leaders[i].msi_coal = COALITION_NULL;
10389	leaders[i].msi_page_count = `0`; / will hold total coalition page count /
10390	leaders[i].msi_pid = `0`; / will hold coalition leader pid /
10391	leaders[i].msi_ntasks = `0`; / will hold the number of tasks in a coalition /
10392	}
10393
10394	p = memorystatus_get_first_proc_locked(&b, FALSE);
10395	while (p) {
10396	if (coalition_is_leader(p->task, COALITION_TYPE_JETSAM, &coal)) {
10397	if (nleaders < MAX_COAL_LEADERS) {
10398	int coal_ntasks = `0`;
10399	uint64_t coal_page_count = coalition_get_page_count(coal, &coal_ntasks);
10400	leaders[nleaders].msi_coal = coal;
10401	leaders[nleaders].msi_page_count = coal_page_count;
10402	leaders[nleaders].msi_pid = p->p_pid; / the coalition leader /
10403	leaders[nleaders].msi_ntasks = coal_ntasks;
10404	nleaders++;
10405	} else {
10406	/*
10407	* We've hit MAX_COAL_LEADERS meaning we can handle no more coalitions.
10408	* Abandoned coalitions will linger at the tail of the priority band
10409	* when this sort session ends.
10410	* TODO: should this be an assert?
10411	*/
10412	printf("%s: WARNING: more than %d leaders in priority band [%d]\n",
10413	__FUNCTION__, MAX_COAL_LEADERS, bucket_index);
10414	break;
10415	}
10416	}
10417	p=memorystatus_get_next_proc_locked(&b, p, FALSE);
10418	}
10419
10420	if (nleaders == `0`) {
10421	/ Nothing to sort /
10422	return(`0`);
10423	}
10424
10425	/*
10426	* Sort the coalition leader array, from smallest coalition page count
10427	* to largest coalition page count. When inserted in the priority bucket,
10428	* smallest coalition is handled first, resulting in the last to be jetsammed.
10429	*/
10430	if (nleaders > `1`) {
10431	qsort(leaders, nleaders, sizeof(memstat_sort_info_t), memstat_asc_cmp);
10432	}
10433
10434	#if 0
10435	for (i = `0`; i < nleaders; i++) {
10436	printf("%s: coal_leader[%d of %d] pid[%d] pages[%llu] ntasks[%d]\n",
10437	__FUNCTION__, i, nleaders, leaders[i].msi_pid, leaders[i].msi_page_count,
10438	leaders[i].msi_ntasks);
10439	}
10440	#endif
10441
10442	/*
10443	* During coalition sorting, processes in a priority band are rearranged
10444	* by being re-inserted at the head of the queue. So, when handling a
10445	* list, the first process that gets moved to the head of the queue,
10446	* ultimately gets pushed toward the queue tail, and hence, jetsams last.
10447	*
10448	* So, for example, the coalition leader is expected to jetsam last,
10449	* after its coalition members. Therefore, the coalition leader is
10450	* inserted at the head of the queue first.
10451	*
10452	* After processing a coalition, the jetsam order is as follows:
10453	* undefs(jetsam first), extensions, xpc services, leader(jetsam last)
10454	*/
10455
10456	/*
10457	* Coalition members are rearranged in the priority bucket here,
10458	* based on their coalition role.
10459	*/
10460	total_pids_moved = `0`;
10461	for (i=`0`; i < nleaders; i++) {
10462
10463	/ a bit of bookkeeping /
10464	pids_moved = `0`;
10465
10466	/ Coalition leaders are jetsammed last, so move into place first /
10467	pid_list[`0`] = leaders[i].msi_pid;
10468	pids_moved += memorystatus_move_list_locked(bucket_index, pid_list, `1`);
10469
10470	/ xpc services should jetsam after extensions /
10471	ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_XPC,
10472	coal_sort_order, pid_list, MAX_SORT_PIDS);
10473
10474	if (ntasks > `0`) {
10475	pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
10476	(ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
10477	}
10478
10479	/ extensions should jetsam after unmarked processes /
10480	ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_EXT,
10481	coal_sort_order, pid_list, MAX_SORT_PIDS);
10482
10483	if (ntasks > `0`) {
10484	pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
10485	(ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
10486	}
10487
10488	/ undefined coalition members should be the first to jetsam /
10489	ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_UNDEF,
10490	coal_sort_order, pid_list, MAX_SORT_PIDS);
10491
10492	if (ntasks > `0`) {
10493	pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
10494	(ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
10495	}
10496
10497	#if 0
10498	if (pids_moved == leaders[i].msi_ntasks) {
10499	/*
10500	* All the pids in the coalition were found in this band.
10501	*/
10502	printf("%s: pids_moved[%d] equal total coalition ntasks[%d] \n", __FUNCTION__,
10503	pids_moved, leaders[i].msi_ntasks);
10504	} else if (pids_moved > leaders[i].msi_ntasks) {
10505	/*
10506	* Apparently new coalition members showed up during the sort?
10507	*/
10508	printf("%s: pids_moved[%d] were greater than expected coalition ntasks[%d] \n", __FUNCTION__,
10509	pids_moved, leaders[i].msi_ntasks);
10510	} else {
10511	/*
10512	* Apparently not all the pids in the coalition were found in this band?
10513	*/
10514	printf("%s: pids_moved[%d] were less than expected coalition ntasks[%d] \n", __FUNCTION__,
10515	pids_moved, leaders[i].msi_ntasks);
10516	}
10517	#endif
10518
10519	total_pids_moved += pids_moved;
10520
10521	} / end for /
10522
10523	return(total_pids_moved);
10524	}
10525
10526
10527	/*
10528	* Traverse a list of pids, searching for each within the priority band provided.
10529	* If pid is found, move it to the front of the priority band.
10530	* Never searches outside the priority band provided.
10531	*
10532	* Input:
10533	* bucket_index - jetsam priority band.
10534	* pid_list - pointer to a list of pids.
10535	* list_sz - number of pids in the list.
10536	*
10537	* Pid list ordering is important in that,
10538	* pid_list[n] is expected to jetsam ahead of pid_list[n+1].
10539	* The sort_order is set by the coalition default.
10540	*
10541	* Return:
10542	* the number of pids found and hence moved within the priority band.
10543	*/
10544	static int
10545	memorystatus_move_list_locked(unsigned int bucket_index, pid_t pid_list, int* list_sz)
10546	{
10547	memstat_bucket_t *current_bucket;
10548	int i;
10549	int found_pids = `0`;
10550
10551	if ((pid_list == NULL) \|\| (list_sz <= `0`)) {
10552	return(`0`);
10553	}
10554
10555	if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
10556	return(`0`);
10557	}
10558
10559	current_bucket = &memstat_bucket[bucket_index];
10560	for (i=`0`; i < list_sz; i++) {
10561	unsigned int b = bucket_index;
10562	proc_t p = NULL;
10563	proc_t aProc = NULL;
10564	pid_t aPid;
10565	int list_index;
10566
10567	list_index = ((list_sz - `1`) - i);
10568	aPid = pid_list[list_index];
10569
10570	/ never search beyond bucket_index provided /
10571	p = memorystatus_get_first_proc_locked(&b, FALSE);
10572	while (p) {
10573	if (p->p_pid == aPid) {
10574	aProc = p;
10575	break;
10576	}
10577	p = memorystatus_get_next_proc_locked(&b, p, FALSE);
10578	}
10579
10580	if (aProc == NULL) {
10581	/ pid not found in this band, just skip it /
10582	continue;
10583	} else {
10584	TAILQ_REMOVE(&current_bucket->list, aProc, p_memstat_list);
10585	TAILQ_INSERT_HEAD(&current_bucket->list, aProc, p_memstat_list);
10586	found_pids++;
10587	}
10588	}
10589	return(found_pids);
10590	}
10591
10592	int
10593	memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index)
10594	{
10595	int32_t i = JETSAM_PRIORITY_IDLE;
10596	int count = `0`;
10597
10598	if (max_bucket_index >= MEMSTAT_BUCKET_COUNT) {
10599	return(-`1`);
10600	}
10601
10602	while(i <= max_bucket_index) {
10603	count += memstat_bucket[i++].count;
10604	}
10605
10606	return count;
10607	}
10608
10609	int
10610	memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap)
10611	{
10612	#if !CONFIG_JETSAM
10613	if (!p \|\| (!isApp(p)) \|\| (p->p_memstat_state & (P_MEMSTAT_INTERNAL \| P_MEMSTAT_MANAGED))) {
10614	/*
10615	* Ineligible processes OR system processes e.g. launchd.
10616	*
10617	* We also skip processes that have the P_MEMSTAT_MANAGED bit set, i.e.
10618	* they're managed by assertiond. These are iOS apps that have been ported
10619	* to macOS. assertiond might be in the process of modifying the app's
10620	* priority / memory limit - so it might have the proc_list lock, and then try
10621	* to take the task lock. Meanwhile we've entered this function with the task lock
10622	* held, and we need the proc_list lock below. So we'll deadlock with assertiond.
10623	*
10624	* It should be fine to read the P_MEMSTAT_MANAGED bit without the proc_list
10625	* lock here, since assertiond only sets this bit on process launch.
10626	*/
10627	return -`1`;
10628	}
10629
10630	/*
10631	* For macOS only:
10632	* We would like to use memorystatus_update() here to move the processes
10633	* within the bands. Unfortunately memorystatus_update() calls
10634	* memorystatus_update_priority_locked() which uses any band transitions
10635	* as an indication to modify ledgers. For that it needs the task lock
10636	* and since we came into this function with the task lock held, we'll deadlock.
10637	*
10638	* Unfortunately we can't completely disable ledger updates because we still
10639	* need the ledger updates for a subset of processes i.e. daemons.
10640	* When all processes on all platforms support memory limits, we can simply call
10641	* memorystatus_update().
10642
10643	* It also has some logic to deal with 'aging' which, currently, is only applicable
10644	* on CONFIG_JETSAM configs. So, till every platform has CONFIG_JETSAM we'll need
10645	* to do this explicit band transition.
10646	*/
10647
10648	memstat_bucket_t current_bucket, new_bucket;
10649	int32_t priority = `0`;
10650
10651	proc_list_lock();
10652
10653	if (((p->p_listflag & P_LIST_EXITED) != `0`) \|\|
10654	(p->p_memstat_state & (P_MEMSTAT_ERROR \| P_MEMSTAT_TERMINATED))) {
10655	/*
10656	* If the process is on its way out OR
10657	* jetsam has alread tried and failed to kill this process,
10658	* let's skip the whole jetsam band transition.
10659	*/
10660	proc_list_unlock();
10661	return(`0`);
10662	}
10663
10664	if (is_appnap) {
10665	current_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
10666	new_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
10667	priority = JETSAM_PRIORITY_IDLE;
10668	} else {
10669	if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
10670	/*
10671	* It is possible that someone pulled this process
10672	* out of the IDLE band without updating its app-nap
10673	* parameters.
10674	*/
10675	proc_list_unlock();
10676	return (`0`);
10677	}
10678
10679	current_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
10680	new_bucket = &memstat_bucket[p->p_memstat_requestedpriority];
10681	priority = p->p_memstat_requestedpriority;
10682	}
10683
10684	TAILQ_REMOVE(&current_bucket->list, p, p_memstat_list);
10685	current_bucket->count--;
10686
10687	TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
10688	new_bucket->count++;
10689
10690	/*
10691	* Record idle start or idle delta.
10692	*/
10693	if (p->p_memstat_effectivepriority == priority) {
10694	/*
10695	* This process is not transitioning between
10696	* jetsam priority buckets. Do nothing.
10697	*/
10698	} else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
10699	uint64_t now;
10700	/*
10701	* Transitioning out of the idle priority bucket.
10702	* Record idle delta.
10703	*/
10704	assert(p->p_memstat_idle_start != `0`);
10705	now = mach_absolute_time();
10706	if (now > p->p_memstat_idle_start) {
10707	p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
10708	}
10709	} else if (priority == JETSAM_PRIORITY_IDLE) {
10710	/*
10711	* Transitioning into the idle priority bucket.
10712	* Record idle start.
10713	*/
10714	p->p_memstat_idle_start = mach_absolute_time();
10715	}
10716
10717	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CHANGE_PRIORITY), p->p_pid, priority, p->p_memstat_effectivepriority, `0`, `0`);
10718
10719	p->p_memstat_effectivepriority = priority;
10720
10721	proc_list_unlock();
10722
10723	return (`0`);
10724
10725	#else /* !CONFIG_JETSAM */
10726	#pragma unused(p)
10727	#pragma unused(is_appnap)
10728	return -`1`;
10729	#endif /* !CONFIG_JETSAM */
10730	}
10731

Browse the source code of codebrowser/bsd/kern/kern_memorystatus.c