1/*
2 * Copyright (c) 2018 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/errno.h>
30
31#include <mach/mach_types.h>
32#include <mach/mach_traps.h>
33#include <mach/host_priv.h>
34#include <mach/kern_return.h>
35#include <mach/memory_object_control.h>
36#include <mach/memory_object_types.h>
37#include <mach/port.h>
38#include <mach/policy.h>
39#include <mach/upl.h>
40#include <mach/thread_act.h>
41#include <mach/mach_vm.h>
42
43#include <kern/host.h>
44#include <kern/kalloc.h>
45#include <kern/queue.h>
46#include <kern/thread.h>
47#include <kern/ipc_kobject.h>
48
49#include <ipc/ipc_port.h>
50#include <ipc/ipc_space.h>
51
52#include <vm/memory_object.h>
53#include <vm/vm_kern.h>
54#include <vm/vm_fault.h>
55#include <vm/vm_map.h>
56#include <vm/vm_pageout.h>
57#include <vm/vm_pageout.h>
58#include <vm/vm_protos.h>
59#include <vm/vm_shared_region.h>
60
61
62/*
63 * SHARED REGION MEMORY PAGER
64 *
65 * This external memory manager (EMM) handles mappings of a dyld shared cache
66 * in shared regions, applying any necessary modifications (sliding,
67 * pointer signing, ...).
68 *
69 * It mostly handles page-in requests (from memory_object_data_request()) by
70 * getting the original data from its backing VM object, itself backed by
71 * the dyld shared cache file, modifying it if needed and providing it to VM.
72 *
73 * The modified pages will never be dirtied, so the memory manager doesn't
74 * need to handle page-out requests (from memory_object_data_return()). The
75 * pages need to be mapped copy-on-write, so that the originals stay clean.
76 *
77 * We don't expect to have to handle a large number of shared cache files,
78 * so the data structures are very simple (simple linked list) for now.
79 */
80
81/* forward declarations */
82void shared_region_pager_reference(memory_object_t mem_obj);
83void shared_region_pager_deallocate(memory_object_t mem_obj);
84kern_return_t shared_region_pager_init(memory_object_t mem_obj,
85 memory_object_control_t control,
86 memory_object_cluster_size_t pg_size);
87kern_return_t shared_region_pager_terminate(memory_object_t mem_obj);
88kern_return_t shared_region_pager_data_request(memory_object_t mem_obj,
89 memory_object_offset_t offset,
90 memory_object_cluster_size_t length,
91 vm_prot_t protection_required,
92 memory_object_fault_info_t fault_info);
93kern_return_t shared_region_pager_data_return(memory_object_t mem_obj,
94 memory_object_offset_t offset,
95 memory_object_cluster_size_t data_cnt,
96 memory_object_offset_t *resid_offset,
97 int *io_error,
98 boolean_t dirty,
99 boolean_t kernel_copy,
100 int upl_flags);
101kern_return_t shared_region_pager_data_initialize(memory_object_t mem_obj,
102 memory_object_offset_t offset,
103 memory_object_cluster_size_t data_cnt);
104kern_return_t shared_region_pager_data_unlock(memory_object_t mem_obj,
105 memory_object_offset_t offset,
106 memory_object_size_t size,
107 vm_prot_t desired_access);
108kern_return_t shared_region_pager_synchronize(memory_object_t mem_obj,
109 memory_object_offset_t offset,
110 memory_object_size_t length,
111 vm_sync_t sync_flags);
112kern_return_t shared_region_pager_map(memory_object_t mem_obj,
113 vm_prot_t prot);
114kern_return_t shared_region_pager_last_unmap(memory_object_t mem_obj);
115
116/*
117 * Vector of VM operations for this EMM.
118 * These routines are invoked by VM via the memory_object_*() interfaces.
119 */
120const struct memory_object_pager_ops shared_region_pager_ops = {
121 shared_region_pager_reference,
122 shared_region_pager_deallocate,
123 shared_region_pager_init,
124 shared_region_pager_terminate,
125 shared_region_pager_data_request,
126 shared_region_pager_data_return,
127 shared_region_pager_data_initialize,
128 shared_region_pager_data_unlock,
129 shared_region_pager_synchronize,
130 shared_region_pager_map,
131 shared_region_pager_last_unmap,
132 NULL, /* data_reclaim */
133 "shared_region"
134};
135
136/*
137 * The "shared_region_pager" describes a memory object backed by
138 * the "shared_region" EMM.
139 */
140typedef struct shared_region_pager {
141 /* mandatory generic header */
142 struct memory_object sc_pgr_hdr;
143
144 /* pager-specific data */
145 queue_chain_t pager_queue; /* next & prev pagers */
146 unsigned int ref_count; /* reference count */
147 boolean_t is_ready; /* is this pager ready ? */
148 boolean_t is_mapped; /* is this mem_obj mapped ? */
149 vm_object_t backing_object; /* VM obj for shared cache */
150 vm_object_offset_t backing_offset;
151 struct vm_shared_region_slide_info *scp_slide_info;
152} *shared_region_pager_t;
153#define SHARED_REGION_PAGER_NULL ((shared_region_pager_t) NULL)
154
155/*
156 * List of memory objects managed by this EMM.
157 * The list is protected by the "shared_region_pager_lock" lock.
158 */
159int shared_region_pager_count = 0; /* number of pagers */
160int shared_region_pager_count_mapped = 0; /* number of unmapped pagers */
161queue_head_t shared_region_pager_queue;
162decl_lck_mtx_data(,shared_region_pager_lock)
163
164/*
165 * Maximum number of unmapped pagers we're willing to keep around.
166 */
167int shared_region_pager_cache_limit = 0;
168
169/*
170 * Statistics & counters.
171 */
172int shared_region_pager_count_max = 0;
173int shared_region_pager_count_unmapped_max = 0;
174int shared_region_pager_num_trim_max = 0;
175int shared_region_pager_num_trim_total = 0;
176
177
178lck_grp_t shared_region_pager_lck_grp;
179lck_grp_attr_t shared_region_pager_lck_grp_attr;
180lck_attr_t shared_region_pager_lck_attr;
181
182uint64_t shared_region_pager_copied = 0;
183uint64_t shared_region_pager_slid = 0;
184uint64_t shared_region_pager_slid_error = 0;
185uint64_t shared_region_pager_reclaimed = 0;
186
187/* internal prototypes */
188shared_region_pager_t shared_region_pager_create(
189 vm_object_t backing_object,
190 vm_object_offset_t backing_offset,
191 struct vm_shared_region_slide_info *slide_info);
192shared_region_pager_t shared_region_pager_lookup(memory_object_t mem_obj);
193void shared_region_pager_dequeue(shared_region_pager_t pager);
194void shared_region_pager_deallocate_internal(shared_region_pager_t pager,
195 boolean_t locked);
196void shared_region_pager_terminate_internal(shared_region_pager_t pager);
197void shared_region_pager_trim(void);
198
199
200#if DEBUG
201int shared_region_pagerdebug = 0;
202#define PAGER_ALL 0xffffffff
203#define PAGER_INIT 0x00000001
204#define PAGER_PAGEIN 0x00000002
205
206#define PAGER_DEBUG(LEVEL, A) \
207 MACRO_BEGIN \
208 if ((shared_region_pagerdebug & (LEVEL)) == (LEVEL)) { \
209 printf A; \
210 } \
211 MACRO_END
212#else
213#define PAGER_DEBUG(LEVEL, A)
214#endif
215
216
217void
218shared_region_pager_bootstrap(void)
219{
220 lck_grp_attr_setdefault(&shared_region_pager_lck_grp_attr);
221 lck_grp_init(&shared_region_pager_lck_grp, "shared_region", &shared_region_pager_lck_grp_attr);
222 lck_attr_setdefault(&shared_region_pager_lck_attr);
223 lck_mtx_init(&shared_region_pager_lock, &shared_region_pager_lck_grp, &shared_region_pager_lck_attr);
224 queue_init(&shared_region_pager_queue);
225}
226
227/*
228 * shared_region_pager_init()
229 *
230 * Initialize the memory object and makes it ready to be used and mapped.
231 */
232kern_return_t
233shared_region_pager_init(
234 memory_object_t mem_obj,
235 memory_object_control_t control,
236#if !DEBUG
237 __unused
238#endif
239 memory_object_cluster_size_t pg_size)
240{
241 shared_region_pager_t pager;
242 kern_return_t kr;
243 memory_object_attr_info_data_t attributes;
244
245 PAGER_DEBUG(PAGER_ALL,
246 ("shared_region_pager_init: %p, %p, %x\n",
247 mem_obj, control, pg_size));
248
249 if (control == MEMORY_OBJECT_CONTROL_NULL)
250 return KERN_INVALID_ARGUMENT;
251
252 pager = shared_region_pager_lookup(mem_obj);
253
254 memory_object_control_reference(control);
255
256 pager->sc_pgr_hdr.mo_control = control;
257
258 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
259 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
260 attributes.cluster_size = (1 << (PAGE_SHIFT));
261 attributes.may_cache_object = FALSE;
262 attributes.temporary = TRUE;
263
264 kr = memory_object_change_attributes(
265 control,
266 MEMORY_OBJECT_ATTRIBUTE_INFO,
267 (memory_object_info_t) &attributes,
268 MEMORY_OBJECT_ATTR_INFO_COUNT);
269 if (kr != KERN_SUCCESS)
270 panic("shared_region_pager_init: "
271 "memory_object_change_attributes() failed");
272
273#if CONFIG_SECLUDED_MEMORY
274 if (secluded_for_filecache) {
275#if 00
276 /*
277 * XXX FBDP do we want this in the secluded pool?
278 * Ideally, we'd want the shared region used by Camera to
279 * NOT be in the secluded pool, but all other shared regions
280 * in the secluded pool...
281 */
282 memory_object_mark_eligible_for_secluded(control, TRUE);
283#endif /* 00 */
284 }
285#endif /* CONFIG_SECLUDED_MEMORY */
286
287 return KERN_SUCCESS;
288}
289
290/*
291 * shared_region_data_return()
292 *
293 * Handles page-out requests from VM. This should never happen since
294 * the pages provided by this EMM are not supposed to be dirty or dirtied
295 * and VM should simply discard the contents and reclaim the pages if it
296 * needs to.
297 */
298kern_return_t
299shared_region_pager_data_return(
300 __unused memory_object_t mem_obj,
301 __unused memory_object_offset_t offset,
302 __unused memory_object_cluster_size_t data_cnt,
303 __unused memory_object_offset_t *resid_offset,
304 __unused int *io_error,
305 __unused boolean_t dirty,
306 __unused boolean_t kernel_copy,
307 __unused int upl_flags)
308{
309 panic("shared_region_pager_data_return: should never get called");
310 return KERN_FAILURE;
311}
312
313kern_return_t
314shared_region_pager_data_initialize(
315 __unused memory_object_t mem_obj,
316 __unused memory_object_offset_t offset,
317 __unused memory_object_cluster_size_t data_cnt)
318{
319 panic("shared_region_pager_data_initialize: should never get called");
320 return KERN_FAILURE;
321}
322
323kern_return_t
324shared_region_pager_data_unlock(
325 __unused memory_object_t mem_obj,
326 __unused memory_object_offset_t offset,
327 __unused memory_object_size_t size,
328 __unused vm_prot_t desired_access)
329{
330 return KERN_FAILURE;
331}
332
333/*
334 * shared_region_pager_data_request()
335 *
336 * Handles page-in requests from VM.
337 */
338int shared_region_pager_data_request_debug = 0;
339kern_return_t
340shared_region_pager_data_request(
341 memory_object_t mem_obj,
342 memory_object_offset_t offset,
343 memory_object_cluster_size_t length,
344#if !DEBUG
345 __unused
346#endif
347 vm_prot_t protection_required,
348 memory_object_fault_info_t mo_fault_info)
349{
350 shared_region_pager_t pager;
351 memory_object_control_t mo_control;
352 upl_t upl;
353 int upl_flags;
354 upl_size_t upl_size;
355 upl_page_info_t *upl_pl;
356 unsigned int pl_count;
357 vm_object_t src_top_object, src_page_object, dst_object;
358 kern_return_t kr, retval;
359 vm_offset_t src_vaddr, dst_vaddr;
360 vm_offset_t cur_offset;
361 vm_offset_t offset_in_page;
362 kern_return_t error_code;
363 vm_prot_t prot;
364 vm_page_t src_page, top_page;
365 int interruptible;
366 struct vm_object_fault_info fault_info;
367 mach_vm_offset_t slide_start_address;
368
369 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
370
371 retval = KERN_SUCCESS;
372 src_top_object = VM_OBJECT_NULL;
373 src_page_object = VM_OBJECT_NULL;
374 upl = NULL;
375 upl_pl = NULL;
376 fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info);
377 fault_info.stealth = TRUE;
378 fault_info.io_sync = FALSE;
379 fault_info.mark_zf_absent = FALSE;
380 fault_info.batch_pmap_op = FALSE;
381 interruptible = fault_info.interruptible;
382
383 pager = shared_region_pager_lookup(mem_obj);
384 assert(pager->is_ready);
385 assert(pager->ref_count > 1); /* pager is alive and mapped */
386
387 PAGER_DEBUG(PAGER_PAGEIN, ("shared_region_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
388
389 /*
390 * Gather in a UPL all the VM pages requested by VM.
391 */
392 mo_control = pager->sc_pgr_hdr.mo_control;
393
394 upl_size = length;
395 upl_flags =
396 UPL_RET_ONLY_ABSENT |
397 UPL_SET_LITE |
398 UPL_NO_SYNC |
399 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */
400 UPL_SET_INTERNAL;
401 pl_count = 0;
402 kr = memory_object_upl_request(mo_control,
403 offset, upl_size,
404 &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
405 if (kr != KERN_SUCCESS) {
406 retval = kr;
407 goto done;
408 }
409 dst_object = mo_control->moc_object;
410 assert(dst_object != VM_OBJECT_NULL);
411
412 /*
413 * We'll map the original data in the kernel address space from the
414 * backing VM object (itself backed by the shared cache file via
415 * the vnode pager).
416 */
417 src_top_object = pager->backing_object;
418 assert(src_top_object != VM_OBJECT_NULL);
419 vm_object_reference(src_top_object); /* keep the source object alive */
420
421 slide_start_address = pager->scp_slide_info->slid_address;
422
423 fault_info.lo_offset += pager->backing_offset;
424 fault_info.hi_offset += pager->backing_offset;
425
426 /*
427 * Fill in the contents of the pages requested by VM.
428 */
429 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
430 pl_count = length / PAGE_SIZE;
431 for (cur_offset = 0;
432 retval == KERN_SUCCESS && cur_offset < length;
433 cur_offset += PAGE_SIZE) {
434 ppnum_t dst_pnum;
435
436 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
437 /* this page is not in the UPL: skip it */
438 continue;
439 }
440
441 /*
442 * Map the source (dyld shared cache) page in the kernel's
443 * virtual address space.
444 * We already hold a reference on the src_top_object.
445 */
446 retry_src_fault:
447 vm_object_lock(src_top_object);
448 vm_object_paging_begin(src_top_object);
449 error_code = 0;
450 prot = VM_PROT_READ;
451 src_page = VM_PAGE_NULL;
452 kr = vm_fault_page(src_top_object,
453 pager->backing_offset + offset + cur_offset,
454 VM_PROT_READ,
455 FALSE,
456 FALSE, /* src_page not looked up */
457 &prot,
458 &src_page,
459 &top_page,
460 NULL,
461 &error_code,
462 FALSE,
463 FALSE,
464 &fault_info);
465 switch (kr) {
466 case VM_FAULT_SUCCESS:
467 break;
468 case VM_FAULT_RETRY:
469 goto retry_src_fault;
470 case VM_FAULT_MEMORY_SHORTAGE:
471 if (vm_page_wait(interruptible)) {
472 goto retry_src_fault;
473 }
474 /* fall thru */
475 case VM_FAULT_INTERRUPTED:
476 retval = MACH_SEND_INTERRUPTED;
477 goto done;
478 case VM_FAULT_SUCCESS_NO_VM_PAGE:
479 /* success but no VM page: fail */
480 vm_object_paging_end(src_top_object);
481 vm_object_unlock(src_top_object);
482 /*FALLTHROUGH*/
483 case VM_FAULT_MEMORY_ERROR:
484 /* the page is not there ! */
485 if (error_code) {
486 retval = error_code;
487 } else {
488 retval = KERN_MEMORY_ERROR;
489 }
490 goto done;
491 default:
492 panic("shared_region_pager_data_request: "
493 "vm_fault_page() unexpected error 0x%x\n",
494 kr);
495 }
496 assert(src_page != VM_PAGE_NULL);
497 assert(src_page->vmp_busy);
498
499 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
500 vm_page_lockspin_queues();
501 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
502 vm_page_speculate(src_page, FALSE);
503 }
504 vm_page_unlock_queues();
505 }
506
507 /*
508 * Establish pointers to the source
509 * and destination physical pages.
510 */
511 dst_pnum = (ppnum_t)
512 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
513 assert(dst_pnum != 0);
514#if __x86_64__
515 src_vaddr = (vm_map_offset_t)
516 PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
517 << PAGE_SHIFT);
518 dst_vaddr = (vm_map_offset_t)
519 PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
520
521#elif __arm__ || __arm64__
522 src_vaddr = (vm_map_offset_t)
523 phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
524 << PAGE_SHIFT);
525 dst_vaddr = (vm_map_offset_t)
526 phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
527#else
528#error "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
529 src_vaddr = 0;
530 dst_vaddr = 0;
531#endif
532 src_page_object = VM_PAGE_OBJECT(src_page);
533
534 /*
535 * Validate the original page...
536 */
537 if (src_page_object->code_signed) {
538 vm_page_validate_cs_mapped(
539 src_page,
540 (const void *) src_vaddr);
541 }
542 /*
543 * ... and transfer the results to the destination page.
544 */
545 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
546 src_page->vmp_cs_validated);
547 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
548 src_page->vmp_cs_tainted);
549 UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
550 src_page->vmp_cs_nx);
551
552 /*
553 * The page provider might access a mapped file, so let's
554 * release the object lock for the source page to avoid a
555 * potential deadlock.
556 * The source page is kept busy and we have a
557 * "paging_in_progress" reference on its object, so it's safe
558 * to unlock the object here.
559 */
560 assert(src_page->vmp_busy);
561 assert(src_page_object->paging_in_progress > 0);
562 vm_object_unlock(src_page_object);
563
564 /*
565 * Process the original contents of the source page
566 * into the destination page.
567 */
568 for (offset_in_page = 0;
569 offset_in_page < PAGE_SIZE;
570 offset_in_page += PAGE_SIZE_FOR_SR_SLIDE) {
571 vm_object_offset_t chunk_offset;
572 vm_object_offset_t offset_in_backing_object;
573 vm_object_offset_t offset_in_sliding_range;
574
575 chunk_offset = offset + cur_offset + offset_in_page;
576
577 bcopy((const char *)(src_vaddr +
578 offset_in_page),
579 (char *)(dst_vaddr + offset_in_page),
580 PAGE_SIZE_FOR_SR_SLIDE);
581
582 offset_in_backing_object = (chunk_offset +
583 pager->backing_offset);
584 if ((offset_in_backing_object < pager->scp_slide_info->start) ||
585 (offset_in_backing_object >= pager->scp_slide_info->end)) {
586 /* chunk is outside of sliding range: done */
587 shared_region_pager_copied++;
588 continue;
589 }
590
591 offset_in_sliding_range =
592 (offset_in_backing_object -
593 pager->scp_slide_info->start);
594 kr = vm_shared_region_slide_page(
595 pager->scp_slide_info,
596 dst_vaddr + offset_in_page,
597 (mach_vm_offset_t) (offset_in_sliding_range +
598 slide_start_address),
599 (uint32_t) (offset_in_sliding_range /
600 PAGE_SIZE_FOR_SR_SLIDE));
601 if (shared_region_pager_data_request_debug) {
602 printf("shared_region_data_request"
603 "(%p,0x%llx+0x%llx+0x%04llx): 0x%llx "
604 "in sliding range [0x%llx:0x%llx]: "
605 "SLIDE offset 0x%llx="
606 "(0x%llx+0x%llx+0x%llx+0x%04llx)"
607 "[0x%016llx 0x%016llx] "
608 "code_signed=%d "
609 "cs_validated=%d "
610 "cs_tainted=%d "
611 "cs_nx=%d "
612 "kr=0x%x\n",
613 pager,
614 offset,
615 (uint64_t) cur_offset,
616 (uint64_t) offset_in_page,
617 chunk_offset,
618 pager->scp_slide_info->start,
619 pager->scp_slide_info->end,
620 (pager->backing_offset +
621 offset +
622 cur_offset +
623 offset_in_page),
624 pager->backing_offset,
625 offset,
626 (uint64_t) cur_offset,
627 (uint64_t) offset_in_page,
628 *(uint64_t *)(dst_vaddr+offset_in_page),
629 *(uint64_t *)(dst_vaddr+offset_in_page+8),
630 src_page_object->code_signed,
631 src_page->vmp_cs_validated,
632 src_page->vmp_cs_tainted,
633 src_page->vmp_cs_nx,
634 kr);
635 }
636 if (kr != KERN_SUCCESS) {
637 shared_region_pager_slid_error++;
638 break;
639 }
640 shared_region_pager_slid++;
641 }
642
643 assert(VM_PAGE_OBJECT(src_page) == src_page_object);
644 assert(src_page->vmp_busy);
645 assert(src_page_object->paging_in_progress > 0);
646 vm_object_lock(src_page_object);
647
648 /*
649 * Cleanup the result of vm_fault_page() of the source page.
650 */
651 PAGE_WAKEUP_DONE(src_page);
652 src_page = VM_PAGE_NULL;
653 vm_object_paging_end(src_page_object);
654 vm_object_unlock(src_page_object);
655
656 if (top_page != VM_PAGE_NULL) {
657 assert(VM_PAGE_OBJECT(top_page) == src_top_object);
658 vm_object_lock(src_top_object);
659 VM_PAGE_FREE(top_page);
660 vm_object_paging_end(src_top_object);
661 vm_object_unlock(src_top_object);
662 }
663 }
664
665done:
666 if (upl != NULL) {
667 /* clean up the UPL */
668
669 /*
670 * The pages are currently dirty because we've just been
671 * writing on them, but as far as we're concerned, they're
672 * clean since they contain their "original" contents as
673 * provided by us, the pager.
674 * Tell the UPL to mark them "clean".
675 */
676 upl_clear_dirty(upl, TRUE);
677
678 /* abort or commit the UPL */
679 if (retval != KERN_SUCCESS) {
680 upl_abort(upl, 0);
681 } else {
682 boolean_t empty;
683 upl_commit_range(upl, 0, upl->size,
684 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
685 upl_pl, pl_count, &empty);
686 }
687
688 /* and deallocate the UPL */
689 upl_deallocate(upl);
690 upl = NULL;
691 }
692 if (src_top_object != VM_OBJECT_NULL) {
693 vm_object_deallocate(src_top_object);
694 }
695 return retval;
696}
697
698/*
699 * shared_region_pager_reference()
700 *
701 * Get a reference on this memory object.
702 * For external usage only. Assumes that the initial reference count is not 0,
703 * i.e one should not "revive" a dead pager this way.
704 */
705void
706shared_region_pager_reference(
707 memory_object_t mem_obj)
708{
709 shared_region_pager_t pager;
710
711 pager = shared_region_pager_lookup(mem_obj);
712
713 lck_mtx_lock(&shared_region_pager_lock);
714 assert(pager->ref_count > 0);
715 pager->ref_count++;
716 lck_mtx_unlock(&shared_region_pager_lock);
717}
718
719
720/*
721 * shared_region_pager_dequeue:
722 *
723 * Removes a pager from the list of pagers.
724 *
725 * The caller must hold "shared_region_pager_lock".
726 */
727void
728shared_region_pager_dequeue(
729 shared_region_pager_t pager)
730{
731 assert(!pager->is_mapped);
732
733 queue_remove(&shared_region_pager_queue,
734 pager,
735 shared_region_pager_t,
736 pager_queue);
737 pager->pager_queue.next = NULL;
738 pager->pager_queue.prev = NULL;
739
740 shared_region_pager_count--;
741}
742
743/*
744 * shared_region_pager_terminate_internal:
745 *
746 * Trigger the asynchronous termination of the memory object associated
747 * with this pager.
748 * When the memory object is terminated, there will be one more call
749 * to memory_object_deallocate() (i.e. shared_region_pager_deallocate())
750 * to finish the clean up.
751 *
752 * "shared_region_pager_lock" should not be held by the caller.
753 * We don't need the lock because the pager has already been removed from
754 * the pagers' list and is now ours exclusively.
755 */
756void
757shared_region_pager_terminate_internal(
758 shared_region_pager_t pager)
759{
760 assert(pager->is_ready);
761 assert(!pager->is_mapped);
762
763 if (pager->backing_object != VM_OBJECT_NULL) {
764 vm_object_deallocate(pager->backing_object);
765 pager->backing_object = VM_OBJECT_NULL;
766 }
767 /* trigger the destruction of the memory object */
768 memory_object_destroy(pager->sc_pgr_hdr.mo_control, 0);
769}
770
771/*
772 * shared_region_pager_deallocate_internal()
773 *
774 * Release a reference on this pager and free it when the last
775 * reference goes away.
776 * Can be called with shared_region_pager_lock held or not but always returns
777 * with it unlocked.
778 */
779void
780shared_region_pager_deallocate_internal(
781 shared_region_pager_t pager,
782 boolean_t locked)
783{
784 boolean_t needs_trimming;
785 int count_unmapped;
786
787 if (! locked) {
788 lck_mtx_lock(&shared_region_pager_lock);
789 }
790
791 count_unmapped = (shared_region_pager_count -
792 shared_region_pager_count_mapped);
793 if (count_unmapped > shared_region_pager_cache_limit) {
794 /* we have too many unmapped pagers: trim some */
795 needs_trimming = TRUE;
796 } else {
797 needs_trimming = FALSE;
798 }
799
800 /* drop a reference on this pager */
801 pager->ref_count--;
802
803 if (pager->ref_count == 1) {
804 /*
805 * Only the "named" reference is left, which means that
806 * no one is really holding on to this pager anymore.
807 * Terminate it.
808 */
809 shared_region_pager_dequeue(pager);
810 /* the pager is all ours: no need for the lock now */
811 lck_mtx_unlock(&shared_region_pager_lock);
812 shared_region_pager_terminate_internal(pager);
813 } else if (pager->ref_count == 0) {
814 /*
815 * Dropped the existence reference; the memory object has
816 * been terminated. Do some final cleanup and release the
817 * pager structure.
818 */
819 lck_mtx_unlock(&shared_region_pager_lock);
820 if (pager->sc_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
821 memory_object_control_deallocate(pager->sc_pgr_hdr.mo_control);
822 pager->sc_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
823 }
824 kfree(pager, sizeof (*pager));
825 pager = SHARED_REGION_PAGER_NULL;
826 } else {
827 /* there are still plenty of references: keep going... */
828 lck_mtx_unlock(&shared_region_pager_lock);
829 }
830
831 if (needs_trimming) {
832 shared_region_pager_trim();
833 }
834 /* caution: lock is not held on return... */
835}
836
837/*
838 * shared_region_pager_deallocate()
839 *
840 * Release a reference on this pager and free it when the last
841 * reference goes away.
842 */
843void
844shared_region_pager_deallocate(
845 memory_object_t mem_obj)
846{
847 shared_region_pager_t pager;
848
849 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_deallocate: %p\n", mem_obj));
850 pager = shared_region_pager_lookup(mem_obj);
851 shared_region_pager_deallocate_internal(pager, FALSE);
852}
853
854/*
855 *
856 */
857kern_return_t
858shared_region_pager_terminate(
859#if !DEBUG
860 __unused
861#endif
862 memory_object_t mem_obj)
863{
864 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_terminate: %p\n", mem_obj));
865
866 return KERN_SUCCESS;
867}
868
869/*
870 *
871 */
872kern_return_t
873shared_region_pager_synchronize(
874 __unused memory_object_t mem_obj,
875 __unused memory_object_offset_t offset,
876 __unused memory_object_size_t length,
877 __unused vm_sync_t sync_flags)
878{
879 panic("shared_region_pager_synchronize: memory_object_synchronize no longer supported\n");
880 return KERN_FAILURE;
881}
882
883/*
884 * shared_region_pager_map()
885 *
886 * This allows VM to let us, the EMM, know that this memory object
887 * is currently mapped one or more times. This is called by VM each time
888 * the memory object gets mapped and we take one extra reference on the
889 * memory object to account for all its mappings.
890 */
891kern_return_t
892shared_region_pager_map(
893 memory_object_t mem_obj,
894 __unused vm_prot_t prot)
895{
896 shared_region_pager_t pager;
897
898 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_map: %p\n", mem_obj));
899
900 pager = shared_region_pager_lookup(mem_obj);
901
902 lck_mtx_lock(&shared_region_pager_lock);
903 assert(pager->is_ready);
904 assert(pager->ref_count > 0); /* pager is alive */
905 if (pager->is_mapped == FALSE) {
906 /*
907 * First mapping of this pager: take an extra reference
908 * that will remain until all the mappings of this pager
909 * are removed.
910 */
911 pager->is_mapped = TRUE;
912 pager->ref_count++;
913 shared_region_pager_count_mapped++;
914 }
915 lck_mtx_unlock(&shared_region_pager_lock);
916
917 return KERN_SUCCESS;
918}
919
920/*
921 * shared_region_pager_last_unmap()
922 *
923 * This is called by VM when this memory object is no longer mapped anywhere.
924 */
925kern_return_t
926shared_region_pager_last_unmap(
927 memory_object_t mem_obj)
928{
929 shared_region_pager_t pager;
930 int count_unmapped;
931
932 PAGER_DEBUG(PAGER_ALL,
933 ("shared_region_pager_last_unmap: %p\n", mem_obj));
934
935 pager = shared_region_pager_lookup(mem_obj);
936
937 lck_mtx_lock(&shared_region_pager_lock);
938 if (pager->is_mapped) {
939 /*
940 * All the mappings are gone, so let go of the one extra
941 * reference that represents all the mappings of this pager.
942 */
943 shared_region_pager_count_mapped--;
944 count_unmapped = (shared_region_pager_count -
945 shared_region_pager_count_mapped);
946 if (count_unmapped > shared_region_pager_count_unmapped_max) {
947 shared_region_pager_count_unmapped_max = count_unmapped;
948 }
949 pager->is_mapped = FALSE;
950 shared_region_pager_deallocate_internal(pager, TRUE);
951 /* caution: deallocate_internal() released the lock ! */
952 } else {
953 lck_mtx_unlock(&shared_region_pager_lock);
954 }
955
956 return KERN_SUCCESS;
957}
958
959
960/*
961 *
962 */
963shared_region_pager_t
964shared_region_pager_lookup(
965 memory_object_t mem_obj)
966{
967 shared_region_pager_t pager;
968
969 assert(mem_obj->mo_pager_ops == &shared_region_pager_ops);
970 pager = (shared_region_pager_t)(uintptr_t) mem_obj;
971 assert(pager->ref_count > 0);
972 return pager;
973}
974
975shared_region_pager_t
976shared_region_pager_create(
977 vm_object_t backing_object,
978 vm_object_offset_t backing_offset,
979 struct vm_shared_region_slide_info *slide_info)
980{
981 shared_region_pager_t pager;
982 memory_object_control_t control;
983 kern_return_t kr;
984
985 pager = (shared_region_pager_t) kalloc(sizeof (*pager));
986 if (pager == SHARED_REGION_PAGER_NULL) {
987 return SHARED_REGION_PAGER_NULL;
988 }
989
990 /*
991 * The vm_map call takes both named entry ports and raw memory
992 * objects in the same parameter. We need to make sure that
993 * vm_map does not see this object as a named entry port. So,
994 * we reserve the first word in the object for a fake ip_kotype
995 * setting - that will tell vm_map to use it as a memory object.
996 */
997 pager->sc_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
998 pager->sc_pgr_hdr.mo_pager_ops = &shared_region_pager_ops;
999 pager->sc_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
1000
1001 pager->is_ready = FALSE;/* not ready until it has a "name" */
1002 pager->ref_count = 1; /* existence reference (for the cache) */
1003 pager->ref_count++; /* for the caller */
1004 pager->is_mapped = FALSE;
1005 pager->backing_object = backing_object;
1006 pager->backing_offset = backing_offset;
1007 pager->scp_slide_info = slide_info;
1008
1009 vm_object_reference(backing_object);
1010
1011 lck_mtx_lock(&shared_region_pager_lock);
1012 /* enter new pager at the head of our list of pagers */
1013 queue_enter_first(&shared_region_pager_queue,
1014 pager,
1015 shared_region_pager_t,
1016 pager_queue);
1017 shared_region_pager_count++;
1018 if (shared_region_pager_count > shared_region_pager_count_max) {
1019 shared_region_pager_count_max = shared_region_pager_count;
1020 }
1021 lck_mtx_unlock(&shared_region_pager_lock);
1022
1023 kr = memory_object_create_named((memory_object_t) pager,
1024 0,
1025 &control);
1026 assert(kr == KERN_SUCCESS);
1027
1028 lck_mtx_lock(&shared_region_pager_lock);
1029 /* the new pager is now ready to be used */
1030 pager->is_ready = TRUE;
1031 lck_mtx_unlock(&shared_region_pager_lock);
1032
1033 /* wakeup anyone waiting for this pager to be ready */
1034 thread_wakeup(&pager->is_ready);
1035
1036 return pager;
1037}
1038
1039/*
1040 * shared_region_pager_setup()
1041 *
1042 * Provide the caller with a memory object backed by the provided
1043 * "backing_object" VM object.
1044 */
1045memory_object_t
1046shared_region_pager_setup(
1047 vm_object_t backing_object,
1048 vm_object_offset_t backing_offset,
1049 struct vm_shared_region_slide_info *slide_info)
1050{
1051 shared_region_pager_t pager;
1052
1053 /* create new pager */
1054 pager = shared_region_pager_create(
1055 backing_object,
1056 backing_offset,
1057 slide_info);
1058 if (pager == SHARED_REGION_PAGER_NULL) {
1059 /* could not create a new pager */
1060 return MEMORY_OBJECT_NULL;
1061 }
1062
1063 lck_mtx_lock(&shared_region_pager_lock);
1064 while (!pager->is_ready) {
1065 lck_mtx_sleep(&shared_region_pager_lock,
1066 LCK_SLEEP_DEFAULT,
1067 &pager->is_ready,
1068 THREAD_UNINT);
1069 }
1070 lck_mtx_unlock(&shared_region_pager_lock);
1071
1072 return (memory_object_t) pager;
1073}
1074
1075void
1076shared_region_pager_trim(void)
1077{
1078 shared_region_pager_t pager, prev_pager;
1079 queue_head_t trim_queue;
1080 int num_trim;
1081 int count_unmapped;
1082
1083 lck_mtx_lock(&shared_region_pager_lock);
1084
1085 /*
1086 * We have too many pagers, try and trim some unused ones,
1087 * starting with the oldest pager at the end of the queue.
1088 */
1089 queue_init(&trim_queue);
1090 num_trim = 0;
1091
1092 for (pager = (shared_region_pager_t)
1093 queue_last(&shared_region_pager_queue);
1094 !queue_end(&shared_region_pager_queue,
1095 (queue_entry_t) pager);
1096 pager = prev_pager) {
1097 /* get prev elt before we dequeue */
1098 prev_pager = (shared_region_pager_t)
1099 queue_prev(&pager->pager_queue);
1100
1101 if (pager->ref_count == 2 &&
1102 pager->is_ready &&
1103 !pager->is_mapped) {
1104 /* this pager can be trimmed */
1105 num_trim++;
1106 /* remove this pager from the main list ... */
1107 shared_region_pager_dequeue(pager);
1108 /* ... and add it to our trim queue */
1109 queue_enter_first(&trim_queue,
1110 pager,
1111 shared_region_pager_t,
1112 pager_queue);
1113
1114 count_unmapped = (shared_region_pager_count -
1115 shared_region_pager_count_mapped);
1116 if (count_unmapped <= shared_region_pager_cache_limit) {
1117 /* we have enough pagers to trim */
1118 break;
1119 }
1120 }
1121 }
1122 if (num_trim > shared_region_pager_num_trim_max) {
1123 shared_region_pager_num_trim_max = num_trim;
1124 }
1125 shared_region_pager_num_trim_total += num_trim;
1126
1127 lck_mtx_unlock(&shared_region_pager_lock);
1128
1129 /* terminate the trimmed pagers */
1130 while (!queue_empty(&trim_queue)) {
1131 queue_remove_first(&trim_queue,
1132 pager,
1133 shared_region_pager_t,
1134 pager_queue);
1135 pager->pager_queue.next = NULL;
1136 pager->pager_queue.prev = NULL;
1137 assert(pager->ref_count == 2);
1138 /*
1139 * We can't call deallocate_internal() because the pager
1140 * has already been dequeued, but we still need to remove
1141 * a reference.
1142 */
1143 pager->ref_count--;
1144 shared_region_pager_terminate_internal(pager);
1145 }
1146}
1147