1/*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/memory_object.c
60 * Author: Michael Wayne Young
61 *
62 * External memory management interface control functions.
63 */
64
65/*
66 * Interface dependencies:
67 */
68
69#include <mach/std_types.h> /* For pointer_t */
70#include <mach/mach_types.h>
71
72#include <mach/mig.h>
73#include <mach/kern_return.h>
74#include <mach/memory_object.h>
75#include <mach/memory_object_default.h>
76#include <mach/memory_object_control_server.h>
77#include <mach/host_priv_server.h>
78#include <mach/boolean.h>
79#include <mach/vm_prot.h>
80#include <mach/message.h>
81
82/*
83 * Implementation dependencies:
84 */
85#include <string.h> /* For memcpy() */
86
87#include <kern/xpr.h>
88#include <kern/host.h>
89#include <kern/thread.h> /* For current_thread() */
90#include <kern/ipc_mig.h>
91#include <kern/misc_protos.h>
92
93#include <vm/vm_object.h>
94#include <vm/vm_fault.h>
95#include <vm/memory_object.h>
96#include <vm/vm_page.h>
97#include <vm/vm_pageout.h>
98#include <vm/pmap.h> /* For pmap_clear_modify */
99#include <vm/vm_kern.h> /* For kernel_map, vm_move */
100#include <vm/vm_map.h> /* For vm_map_pageable */
101#include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
102#include <vm/vm_shared_region.h>
103
104#include <vm/vm_external.h>
105
106#include <vm/vm_protos.h>
107
108memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
109decl_lck_mtx_data(, memory_manager_default_lock)
110
111
112/*
113 * Routine: memory_object_should_return_page
114 *
115 * Description:
116 * Determine whether the given page should be returned,
117 * based on the page's state and on the given return policy.
118 *
119 * We should return the page if one of the following is true:
120 *
121 * 1. Page is dirty and should_return is not RETURN_NONE.
122 * 2. Page is precious and should_return is RETURN_ALL.
123 * 3. Should_return is RETURN_ANYTHING.
124 *
125 * As a side effect, m->vmp_dirty will be made consistent
126 * with pmap_is_modified(m), if should_return is not
127 * MEMORY_OBJECT_RETURN_NONE.
128 */
129
130#define memory_object_should_return_page(m, should_return) \
131 (should_return != MEMORY_OBJECT_RETURN_NONE && \
132 (((m)->vmp_dirty || ((m)->vmp_dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \
133 ((m)->vmp_precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
134 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
135
136typedef int memory_object_lock_result_t;
137
138#define MEMORY_OBJECT_LOCK_RESULT_DONE 0
139#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
140#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
141#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
142
143memory_object_lock_result_t memory_object_lock_page(
144 vm_page_t m,
145 memory_object_return_t should_return,
146 boolean_t should_flush,
147 vm_prot_t prot);
148
149/*
150 * Routine: memory_object_lock_page
151 *
152 * Description:
153 * Perform the appropriate lock operations on the
154 * given page. See the description of
155 * "memory_object_lock_request" for the meanings
156 * of the arguments.
157 *
158 * Returns an indication that the operation
159 * completed, blocked, or that the page must
160 * be cleaned.
161 */
162memory_object_lock_result_t
163memory_object_lock_page(
164 vm_page_t m,
165 memory_object_return_t should_return,
166 boolean_t should_flush,
167 vm_prot_t prot)
168{
169 XPR(XPR_MEMORY_OBJECT,
170 "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n",
171 m, should_return, should_flush, prot, 0);
172
173
174 if (m->vmp_busy || m->vmp_cleaning)
175 return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
176
177 if (m->vmp_laundry)
178 vm_pageout_steal_laundry(m, FALSE);
179
180 /*
181 * Don't worry about pages for which the kernel
182 * does not have any data.
183 */
184 if (m->vmp_absent || m->vmp_error || m->vmp_restart) {
185 if (m->vmp_error && should_flush && !VM_PAGE_WIRED(m)) {
186 /*
187 * dump the page, pager wants us to
188 * clean it up and there is no
189 * relevant data to return
190 */
191 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
192 }
193 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
194 }
195 assert(!m->vmp_fictitious);
196
197 if (VM_PAGE_WIRED(m)) {
198 /*
199 * The page is wired... just clean or return the page if needed.
200 * Wired pages don't get flushed or disconnected from the pmap.
201 */
202 if (memory_object_should_return_page(m, should_return))
203 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
204
205 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
206 }
207
208 if (should_flush) {
209 /*
210 * must do the pmap_disconnect before determining the
211 * need to return the page... otherwise it's possible
212 * for the page to go from the clean to the dirty state
213 * after we've made our decision
214 */
215 if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
216 SET_PAGE_DIRTY(m, FALSE);
217 }
218 } else {
219 /*
220 * If we are decreasing permission, do it now;
221 * let the fault handler take care of increases
222 * (pmap_page_protect may not increase protection).
223 */
224 if (prot != VM_PROT_NO_CHANGE)
225 pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_ALL & ~prot);
226 }
227 /*
228 * Handle returning dirty or precious pages
229 */
230 if (memory_object_should_return_page(m, should_return)) {
231 /*
232 * we use to do a pmap_disconnect here in support
233 * of memory_object_lock_request, but that routine
234 * no longer requires this... in any event, in
235 * our world, it would turn into a big noop since
236 * we don't lock the page in any way and as soon
237 * as we drop the object lock, the page can be
238 * faulted back into an address space
239 *
240 * if (!should_flush)
241 * pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
242 */
243 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
244 }
245
246 /*
247 * Handle flushing clean pages
248 */
249 if (should_flush)
250 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
251
252 /*
253 * we use to deactivate clean pages at this point,
254 * but we do not believe that an msync should change
255 * the 'age' of a page in the cache... here is the
256 * original comment and code concerning this...
257 *
258 * XXX Make clean but not flush a paging hint,
259 * and deactivate the pages. This is a hack
260 * because it overloads flush/clean with
261 * implementation-dependent meaning. This only
262 * happens to pages that are already clean.
263 *
264 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
265 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
266 */
267
268 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
269}
270
271
272
273/*
274 * Routine: memory_object_lock_request [user interface]
275 *
276 * Description:
277 * Control use of the data associated with the given
278 * memory object. For each page in the given range,
279 * perform the following operations, in order:
280 * 1) restrict access to the page (disallow
281 * forms specified by "prot");
282 * 2) return data to the manager (if "should_return"
283 * is RETURN_DIRTY and the page is dirty, or
284 * "should_return" is RETURN_ALL and the page
285 * is either dirty or precious); and,
286 * 3) flush the cached copy (if "should_flush"
287 * is asserted).
288 * The set of pages is defined by a starting offset
289 * ("offset") and size ("size"). Only pages with the
290 * same page alignment as the starting offset are
291 * considered.
292 *
293 * A single acknowledgement is sent (to the "reply_to"
294 * port) when these actions are complete. If successful,
295 * the naked send right for reply_to is consumed.
296 */
297
298kern_return_t
299memory_object_lock_request(
300 memory_object_control_t control,
301 memory_object_offset_t offset,
302 memory_object_size_t size,
303 memory_object_offset_t * resid_offset,
304 int * io_errno,
305 memory_object_return_t should_return,
306 int flags,
307 vm_prot_t prot)
308{
309 vm_object_t object;
310
311 /*
312 * Check for bogus arguments.
313 */
314 object = memory_object_control_to_vm_object(control);
315 if (object == VM_OBJECT_NULL)
316 return (KERN_INVALID_ARGUMENT);
317
318 if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
319 return (KERN_INVALID_ARGUMENT);
320
321 size = round_page_64(size);
322
323 /*
324 * Lock the object, and acquire a paging reference to
325 * prevent the memory_object reference from being released.
326 */
327 vm_object_lock(object);
328 vm_object_paging_begin(object);
329
330 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
331 if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) {
332 flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL;
333 flags |= MEMORY_OBJECT_DATA_FLUSH;
334 }
335 }
336 offset -= object->paging_offset;
337
338 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL)
339 vm_object_reap_pages(object, REAP_DATA_FLUSH);
340 else
341 (void)vm_object_update(object, offset, size, resid_offset,
342 io_errno, should_return, flags, prot);
343
344 vm_object_paging_end(object);
345 vm_object_unlock(object);
346
347 return (KERN_SUCCESS);
348}
349
350/*
351 * memory_object_release_name: [interface]
352 *
353 * Enforces name semantic on memory_object reference count decrement
354 * This routine should not be called unless the caller holds a name
355 * reference gained through the memory_object_named_create or the
356 * memory_object_rename call.
357 * If the TERMINATE_IDLE flag is set, the call will return if the
358 * reference count is not 1. i.e. idle with the only remaining reference
359 * being the name.
360 * If the decision is made to proceed the name field flag is set to
361 * false and the reference count is decremented. If the RESPECT_CACHE
362 * flag is set and the reference count has gone to zero, the
363 * memory_object is checked to see if it is cacheable otherwise when
364 * the reference count is zero, it is simply terminated.
365 */
366
367kern_return_t
368memory_object_release_name(
369 memory_object_control_t control,
370 int flags)
371{
372 vm_object_t object;
373
374 object = memory_object_control_to_vm_object(control);
375 if (object == VM_OBJECT_NULL)
376 return (KERN_INVALID_ARGUMENT);
377
378 return vm_object_release_name(object, flags);
379}
380
381
382
383/*
384 * Routine: memory_object_destroy [user interface]
385 * Purpose:
386 * Shut down a memory object, despite the
387 * presence of address map (or other) references
388 * to the vm_object.
389 */
390kern_return_t
391memory_object_destroy(
392 memory_object_control_t control,
393 kern_return_t reason)
394{
395 vm_object_t object;
396
397 object = memory_object_control_to_vm_object(control);
398 if (object == VM_OBJECT_NULL)
399 return (KERN_INVALID_ARGUMENT);
400
401 return (vm_object_destroy(object, reason));
402}
403
404/*
405 * Routine: vm_object_sync
406 *
407 * Kernel internal function to synch out pages in a given
408 * range within an object to its memory manager. Much the
409 * same as memory_object_lock_request but page protection
410 * is not changed.
411 *
412 * If the should_flush and should_return flags are true pages
413 * are flushed, that is dirty & precious pages are written to
414 * the memory manager and then discarded. If should_return
415 * is false, only precious pages are returned to the memory
416 * manager.
417 *
418 * If should flush is false and should_return true, the memory
419 * manager's copy of the pages is updated. If should_return
420 * is also false, only the precious pages are updated. This
421 * last option is of limited utility.
422 *
423 * Returns:
424 * FALSE if no pages were returned to the pager
425 * TRUE otherwise.
426 */
427
428boolean_t
429vm_object_sync(
430 vm_object_t object,
431 vm_object_offset_t offset,
432 vm_object_size_t size,
433 boolean_t should_flush,
434 boolean_t should_return,
435 boolean_t should_iosync)
436{
437 boolean_t rv;
438 int flags;
439
440 XPR(XPR_VM_OBJECT,
441 "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n",
442 object, offset, size, should_flush, should_return);
443
444 /*
445 * Lock the object, and acquire a paging reference to
446 * prevent the memory_object and control ports from
447 * being destroyed.
448 */
449 vm_object_lock(object);
450 vm_object_paging_begin(object);
451
452 if (should_flush) {
453 flags = MEMORY_OBJECT_DATA_FLUSH;
454 /*
455 * This flush is from an msync(), not a truncate(), so the
456 * contents of the file are not affected.
457 * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know
458 * that the data is not changed and that there's no need to
459 * push the old contents to a copy object.
460 */
461 flags |= MEMORY_OBJECT_DATA_NO_CHANGE;
462 } else
463 flags = 0;
464
465 if (should_iosync)
466 flags |= MEMORY_OBJECT_IO_SYNC;
467
468 rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,
469 (should_return) ?
470 MEMORY_OBJECT_RETURN_ALL :
471 MEMORY_OBJECT_RETURN_NONE,
472 flags,
473 VM_PROT_NO_CHANGE);
474
475
476 vm_object_paging_end(object);
477 vm_object_unlock(object);
478 return rv;
479}
480
481
482
483#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
484MACRO_BEGIN \
485 \
486 int upl_flags; \
487 memory_object_t pager; \
488 \
489 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
490 vm_object_paging_begin(object); \
491 vm_object_unlock(object); \
492 \
493 if (iosync) \
494 upl_flags = UPL_MSYNC | UPL_IOSYNC; \
495 else \
496 upl_flags = UPL_MSYNC; \
497 \
498 (void) memory_object_data_return(pager, \
499 po, \
500 (memory_object_cluster_size_t)data_cnt, \
501 ro, \
502 ioerr, \
503 FALSE, \
504 FALSE, \
505 upl_flags); \
506 \
507 vm_object_lock(object); \
508 vm_object_paging_end(object); \
509 } \
510MACRO_END
511
512extern struct vnode *
513vnode_pager_lookup_vnode(memory_object_t);
514
515static int
516vm_object_update_extent(
517 vm_object_t object,
518 vm_object_offset_t offset,
519 vm_object_offset_t offset_end,
520 vm_object_offset_t *offset_resid,
521 int *io_errno,
522 boolean_t should_flush,
523 memory_object_return_t should_return,
524 boolean_t should_iosync,
525 vm_prot_t prot)
526{
527 vm_page_t m;
528 int retval = 0;
529 vm_object_offset_t paging_offset = 0;
530 vm_object_offset_t next_offset = offset;
531 memory_object_lock_result_t page_lock_result;
532 memory_object_cluster_size_t data_cnt = 0;
533 struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
534 struct vm_page_delayed_work *dwp;
535 int dw_count;
536 int dw_limit;
537 int dirty_count;
538
539 dwp = &dw_array[0];
540 dw_count = 0;
541 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
542 dirty_count = 0;
543
544 for (;
545 offset < offset_end && object->resident_page_count;
546 offset += PAGE_SIZE_64) {
547
548 /*
549 * Limit the number of pages to be cleaned at once to a contiguous
550 * run, or at most MAX_UPL_TRANSFER_BYTES
551 */
552 if (data_cnt) {
553 if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) {
554
555 if (dw_count) {
556 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
557 dwp = &dw_array[0];
558 dw_count = 0;
559 }
560 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
561 paging_offset, offset_resid, io_errno, should_iosync);
562 data_cnt = 0;
563 }
564 }
565 while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
566
567 dwp->dw_mask = 0;
568
569 page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
570
571 if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
572 /*
573 * End of a run of dirty/precious pages.
574 */
575 if (dw_count) {
576 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
577 dwp = &dw_array[0];
578 dw_count = 0;
579 }
580 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
581 paging_offset, offset_resid, io_errno, should_iosync);
582 /*
583 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
584 * allow the state of page 'm' to change... we need to re-lookup
585 * the current offset
586 */
587 data_cnt = 0;
588 continue;
589 }
590
591 switch (page_lock_result) {
592
593 case MEMORY_OBJECT_LOCK_RESULT_DONE:
594 break;
595
596 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
597 if (m->vmp_dirty == TRUE)
598 dirty_count++;
599 dwp->dw_mask |= DW_vm_page_free;
600 break;
601
602 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
603 PAGE_SLEEP(object, m, THREAD_UNINT);
604 continue;
605
606 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
607 if (data_cnt == 0)
608 paging_offset = offset;
609
610 data_cnt += PAGE_SIZE;
611 next_offset = offset + PAGE_SIZE_64;
612
613 /*
614 * wired pages shouldn't be flushed and
615 * since they aren't on any queue,
616 * no need to remove them
617 */
618 if (!VM_PAGE_WIRED(m)) {
619
620 if (should_flush) {
621 /*
622 * add additional state for the flush
623 */
624 m->vmp_free_when_done = TRUE;
625 }
626 /*
627 * we use to remove the page from the queues at this
628 * point, but we do not believe that an msync
629 * should cause the 'age' of a page to be changed
630 *
631 * else
632 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
633 */
634 }
635 retval = 1;
636 break;
637 }
638 if (dwp->dw_mask) {
639 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
640
641 if (dw_count >= dw_limit) {
642 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
643 dwp = &dw_array[0];
644 dw_count = 0;
645 }
646 }
647 break;
648 }
649 }
650
651 if (object->pager)
652 task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_INVALIDATED, vnode_pager_lookup_vnode(object->pager));
653 /*
654 * We have completed the scan for applicable pages.
655 * Clean any pages that have been saved.
656 */
657 if (dw_count)
658 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
659
660 if (data_cnt) {
661 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
662 paging_offset, offset_resid, io_errno, should_iosync);
663 }
664 return (retval);
665}
666
667
668
669/*
670 * Routine: vm_object_update
671 * Description:
672 * Work function for m_o_lock_request(), vm_o_sync().
673 *
674 * Called with object locked and paging ref taken.
675 */
676kern_return_t
677vm_object_update(
678 vm_object_t object,
679 vm_object_offset_t offset,
680 vm_object_size_t size,
681 vm_object_offset_t *resid_offset,
682 int *io_errno,
683 memory_object_return_t should_return,
684 int flags,
685 vm_prot_t protection)
686{
687 vm_object_t copy_object = VM_OBJECT_NULL;
688 boolean_t data_returned = FALSE;
689 boolean_t update_cow;
690 boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
691 boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
692 vm_fault_return_t result;
693 int num_of_extents;
694 int n;
695#define MAX_EXTENTS 8
696#define EXTENT_SIZE (1024 * 1024 * 256)
697#define RESIDENT_LIMIT (1024 * 32)
698 struct extent {
699 vm_object_offset_t e_base;
700 vm_object_offset_t e_min;
701 vm_object_offset_t e_max;
702 } extents[MAX_EXTENTS];
703
704 /*
705 * To avoid blocking while scanning for pages, save
706 * dirty pages to be cleaned all at once.
707 *
708 * XXXO A similar strategy could be used to limit the
709 * number of times that a scan must be restarted for
710 * other reasons. Those pages that would require blocking
711 * could be temporarily collected in another list, or
712 * their offsets could be recorded in a small array.
713 */
714
715 /*
716 * XXX NOTE: May want to consider converting this to a page list
717 * XXX vm_map_copy interface. Need to understand object
718 * XXX coalescing implications before doing so.
719 */
720
721 update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH)
722 && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
723 !(flags & MEMORY_OBJECT_DATA_PURGE)))
724 || (flags & MEMORY_OBJECT_COPY_SYNC);
725
726 if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
727 int collisions = 0;
728
729 while ((copy_object = object->copy) != VM_OBJECT_NULL) {
730 /*
731 * need to do a try here since we're swimming upstream
732 * against the normal lock ordering... however, we need
733 * to hold the object stable until we gain control of the
734 * copy object so we have to be careful how we approach this
735 */
736 if (vm_object_lock_try(copy_object)) {
737 /*
738 * we 'won' the lock on the copy object...
739 * no need to hold the object lock any longer...
740 * take a real reference on the copy object because
741 * we're going to call vm_fault_page on it which may
742 * under certain conditions drop the lock and the paging
743 * reference we're about to take... the reference
744 * will keep the copy object from going away if that happens
745 */
746 vm_object_unlock(object);
747 vm_object_reference_locked(copy_object);
748 break;
749 }
750 vm_object_unlock(object);
751
752 collisions++;
753 mutex_pause(collisions);
754
755 vm_object_lock(object);
756 }
757 }
758 if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
759 vm_map_size_t i;
760 vm_map_size_t copy_size;
761 vm_map_offset_t copy_offset;
762 vm_prot_t prot;
763 vm_page_t page;
764 vm_page_t top_page;
765 kern_return_t error = 0;
766 struct vm_object_fault_info fault_info = {};
767
768 if (copy_object != VM_OBJECT_NULL) {
769 /*
770 * translate offset with respect to shadow's offset
771 */
772 copy_offset = (offset >= copy_object->vo_shadow_offset) ?
773 (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) :
774 (vm_map_offset_t) 0;
775
776 if (copy_offset > copy_object->vo_size)
777 copy_offset = copy_object->vo_size;
778
779 /*
780 * clip size with respect to shadow offset
781 */
782 if (offset >= copy_object->vo_shadow_offset) {
783 copy_size = size;
784 } else if (size >= copy_object->vo_shadow_offset - offset) {
785 copy_size = size - (copy_object->vo_shadow_offset - offset);
786 } else {
787 copy_size = 0;
788 }
789
790 if (copy_offset + copy_size > copy_object->vo_size) {
791 if (copy_object->vo_size >= copy_offset) {
792 copy_size = copy_object->vo_size - copy_offset;
793 } else {
794 copy_size = 0;
795 }
796 }
797 copy_size+=copy_offset;
798
799 } else {
800 copy_object = object;
801
802 copy_size = offset + size;
803 copy_offset = offset;
804 }
805 fault_info.interruptible = THREAD_UNINT;
806 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
807 fault_info.lo_offset = copy_offset;
808 fault_info.hi_offset = copy_size;
809 fault_info.stealth = TRUE;
810 assert(fault_info.cs_bypass == FALSE);
811 assert(fault_info.pmap_cs_associated == FALSE);
812
813 vm_object_paging_begin(copy_object);
814
815 for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
816 RETRY_COW_OF_LOCK_REQUEST:
817 fault_info.cluster_size = (vm_size_t) (copy_size - i);
818 assert(fault_info.cluster_size == copy_size - i);
819
820 prot = VM_PROT_WRITE|VM_PROT_READ;
821 page = VM_PAGE_NULL;
822 result = vm_fault_page(copy_object, i,
823 VM_PROT_WRITE|VM_PROT_READ,
824 FALSE,
825 FALSE, /* page not looked up */
826 &prot,
827 &page,
828 &top_page,
829 (int *)0,
830 &error,
831 FALSE,
832 FALSE, &fault_info);
833
834 switch (result) {
835 case VM_FAULT_SUCCESS:
836 if (top_page) {
837 vm_fault_cleanup(
838 VM_PAGE_OBJECT(page), top_page);
839 vm_object_lock(copy_object);
840 vm_object_paging_begin(copy_object);
841 }
842 if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {
843
844 vm_page_lockspin_queues();
845
846 if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {
847 vm_page_deactivate(page);
848 }
849 vm_page_unlock_queues();
850 }
851 PAGE_WAKEUP_DONE(page);
852 break;
853 case VM_FAULT_RETRY:
854 prot = VM_PROT_WRITE|VM_PROT_READ;
855 vm_object_lock(copy_object);
856 vm_object_paging_begin(copy_object);
857 goto RETRY_COW_OF_LOCK_REQUEST;
858 case VM_FAULT_INTERRUPTED:
859 prot = VM_PROT_WRITE|VM_PROT_READ;
860 vm_object_lock(copy_object);
861 vm_object_paging_begin(copy_object);
862 goto RETRY_COW_OF_LOCK_REQUEST;
863 case VM_FAULT_MEMORY_SHORTAGE:
864 VM_PAGE_WAIT();
865 prot = VM_PROT_WRITE|VM_PROT_READ;
866 vm_object_lock(copy_object);
867 vm_object_paging_begin(copy_object);
868 goto RETRY_COW_OF_LOCK_REQUEST;
869 case VM_FAULT_SUCCESS_NO_VM_PAGE:
870 /* success but no VM page: fail */
871 vm_object_paging_end(copy_object);
872 vm_object_unlock(copy_object);
873 /*FALLTHROUGH*/
874 case VM_FAULT_MEMORY_ERROR:
875 if (object != copy_object)
876 vm_object_deallocate(copy_object);
877 vm_object_lock(object);
878 goto BYPASS_COW_COPYIN;
879 default:
880 panic("vm_object_update: unexpected error 0x%x"
881 " from vm_fault_page()\n", result);
882 }
883
884 }
885 vm_object_paging_end(copy_object);
886 }
887 if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
888 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
889 vm_object_unlock(copy_object);
890 vm_object_deallocate(copy_object);
891 vm_object_lock(object);
892 }
893 return KERN_SUCCESS;
894 }
895 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
896 if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
897 vm_object_lock_assert_exclusive(copy_object);
898 copy_object->shadow_severed = TRUE;
899 copy_object->shadowed = FALSE;
900 copy_object->shadow = NULL;
901 /*
902 * delete the ref the COW was holding on the target object
903 */
904 vm_object_deallocate(object);
905 }
906 vm_object_unlock(copy_object);
907 vm_object_deallocate(copy_object);
908 vm_object_lock(object);
909 }
910BYPASS_COW_COPYIN:
911
912 /*
913 * when we have a really large range to check relative
914 * to the number of actual resident pages, we'd like
915 * to use the resident page list to drive our checks
916 * however, the object lock will get dropped while processing
917 * the page which means the resident queue can change which
918 * means we can't walk the queue as we process the pages
919 * we also want to do the processing in offset order to allow
920 * 'runs' of pages to be collected if we're being told to
921 * flush to disk... the resident page queue is NOT ordered.
922 *
923 * a temporary solution (until we figure out how to deal with
924 * large address spaces more generically) is to pre-flight
925 * the resident page queue (if it's small enough) and develop
926 * a collection of extents (that encompass actual resident pages)
927 * to visit. This will at least allow us to deal with some of the
928 * more pathological cases in a more efficient manner. The current
929 * worst case (a single resident page at the end of an extremely large
930 * range) can take minutes to complete for ranges in the terrabyte
931 * category... since this routine is called when truncating a file,
932 * and we currently support files up to 16 Tbytes in size, this
933 * is not a theoretical problem
934 */
935
936 if ((object->resident_page_count < RESIDENT_LIMIT) &&
937 (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) {
938 vm_page_t next;
939 vm_object_offset_t start;
940 vm_object_offset_t end;
941 vm_object_size_t e_mask;
942 vm_page_t m;
943
944 start = offset;
945 end = offset + size;
946 num_of_extents = 0;
947 e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
948
949 m = (vm_page_t) vm_page_queue_first(&object->memq);
950
951 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) m)) {
952 next = (vm_page_t) vm_page_queue_next(&m->vmp_listq);
953
954 if ((m->vmp_offset >= start) && (m->vmp_offset < end)) {
955 /*
956 * this is a page we're interested in
957 * try to fit it into a current extent
958 */
959 for (n = 0; n < num_of_extents; n++) {
960 if ((m->vmp_offset & e_mask) == extents[n].e_base) {
961 /*
962 * use (PAGE_SIZE - 1) to determine the
963 * max offset so that we don't wrap if
964 * we're at the last page of the space
965 */
966 if (m->vmp_offset < extents[n].e_min)
967 extents[n].e_min = m->vmp_offset;
968 else if ((m->vmp_offset + (PAGE_SIZE - 1)) > extents[n].e_max)
969 extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1);
970 break;
971 }
972 }
973 if (n == num_of_extents) {
974 /*
975 * didn't find a current extent that can encompass
976 * this page
977 */
978 if (n < MAX_EXTENTS) {
979 /*
980 * if we still have room,
981 * create a new extent
982 */
983 extents[n].e_base = m->vmp_offset & e_mask;
984 extents[n].e_min = m->vmp_offset;
985 extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1);
986
987 num_of_extents++;
988 } else {
989 /*
990 * no room to create a new extent...
991 * fall back to a single extent based
992 * on the min and max page offsets
993 * we find in the range we're interested in...
994 * first, look through the extent list and
995 * develop the overall min and max for the
996 * pages we've looked at up to this point
997 */
998 for (n = 1; n < num_of_extents; n++) {
999 if (extents[n].e_min < extents[0].e_min)
1000 extents[0].e_min = extents[n].e_min;
1001 if (extents[n].e_max > extents[0].e_max)
1002 extents[0].e_max = extents[n].e_max;
1003 }
1004 /*
1005 * now setup to run through the remaining pages
1006 * to determine the overall min and max
1007 * offset for the specified range
1008 */
1009 extents[0].e_base = 0;
1010 e_mask = 0;
1011 num_of_extents = 1;
1012
1013 /*
1014 * by continuing, we'll reprocess the
1015 * page that forced us to abandon trying
1016 * to develop multiple extents
1017 */
1018 continue;
1019 }
1020 }
1021 }
1022 m = next;
1023 }
1024 } else {
1025 extents[0].e_min = offset;
1026 extents[0].e_max = offset + (size - 1);
1027
1028 num_of_extents = 1;
1029 }
1030 for (n = 0; n < num_of_extents; n++) {
1031 if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
1032 should_flush, should_return, should_iosync, protection))
1033 data_returned = TRUE;
1034 }
1035 return (data_returned);
1036}
1037
1038
1039static kern_return_t
1040vm_object_set_attributes_common(
1041 vm_object_t object,
1042 boolean_t may_cache,
1043 memory_object_copy_strategy_t copy_strategy)
1044{
1045 boolean_t object_became_ready;
1046
1047 XPR(XPR_MEMORY_OBJECT,
1048 "m_o_set_attr_com, object 0x%X flg %x strat %d\n",
1049 object, (may_cache&1), copy_strategy, 0, 0);
1050
1051 if (object == VM_OBJECT_NULL)
1052 return(KERN_INVALID_ARGUMENT);
1053
1054 /*
1055 * Verify the attributes of importance
1056 */
1057
1058 switch(copy_strategy) {
1059 case MEMORY_OBJECT_COPY_NONE:
1060 case MEMORY_OBJECT_COPY_DELAY:
1061 break;
1062 default:
1063 return(KERN_INVALID_ARGUMENT);
1064 }
1065
1066 if (may_cache)
1067 may_cache = TRUE;
1068
1069 vm_object_lock(object);
1070
1071 /*
1072 * Copy the attributes
1073 */
1074 assert(!object->internal);
1075 object_became_ready = !object->pager_ready;
1076 object->copy_strategy = copy_strategy;
1077 object->can_persist = may_cache;
1078
1079 /*
1080 * Wake up anyone waiting for the ready attribute
1081 * to become asserted.
1082 */
1083
1084 if (object_became_ready) {
1085 object->pager_ready = TRUE;
1086 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
1087 }
1088
1089 vm_object_unlock(object);
1090
1091 return(KERN_SUCCESS);
1092}
1093
1094
1095kern_return_t
1096memory_object_synchronize_completed(
1097 __unused memory_object_control_t control,
1098 __unused memory_object_offset_t offset,
1099 __unused memory_object_size_t length)
1100{
1101 panic("memory_object_synchronize_completed no longer supported\n");
1102 return(KERN_FAILURE);
1103}
1104
1105
1106/*
1107 * Set the memory object attribute as provided.
1108 *
1109 * XXX This routine cannot be completed until the vm_msync, clean
1110 * in place, and cluster work is completed. See ifdef notyet
1111 * below and note that vm_object_set_attributes_common()
1112 * may have to be expanded.
1113 */
1114kern_return_t
1115memory_object_change_attributes(
1116 memory_object_control_t control,
1117 memory_object_flavor_t flavor,
1118 memory_object_info_t attributes,
1119 mach_msg_type_number_t count)
1120{
1121 vm_object_t object;
1122 kern_return_t result = KERN_SUCCESS;
1123 boolean_t may_cache;
1124 boolean_t invalidate;
1125 memory_object_copy_strategy_t copy_strategy;
1126
1127 object = memory_object_control_to_vm_object(control);
1128 if (object == VM_OBJECT_NULL)
1129 return (KERN_INVALID_ARGUMENT);
1130
1131 vm_object_lock(object);
1132
1133 may_cache = object->can_persist;
1134 copy_strategy = object->copy_strategy;
1135#if notyet
1136 invalidate = object->invalidate;
1137#endif
1138 vm_object_unlock(object);
1139
1140 switch (flavor) {
1141 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1142 {
1143 old_memory_object_behave_info_t behave;
1144
1145 if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1146 result = KERN_INVALID_ARGUMENT;
1147 break;
1148 }
1149
1150 behave = (old_memory_object_behave_info_t) attributes;
1151
1152 invalidate = behave->invalidate;
1153 copy_strategy = behave->copy_strategy;
1154
1155 break;
1156 }
1157
1158 case MEMORY_OBJECT_BEHAVIOR_INFO:
1159 {
1160 memory_object_behave_info_t behave;
1161
1162 if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1163 result = KERN_INVALID_ARGUMENT;
1164 break;
1165 }
1166
1167 behave = (memory_object_behave_info_t) attributes;
1168
1169 invalidate = behave->invalidate;
1170 copy_strategy = behave->copy_strategy;
1171 break;
1172 }
1173
1174 case MEMORY_OBJECT_PERFORMANCE_INFO:
1175 {
1176 memory_object_perf_info_t perf;
1177
1178 if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
1179 result = KERN_INVALID_ARGUMENT;
1180 break;
1181 }
1182
1183 perf = (memory_object_perf_info_t) attributes;
1184
1185 may_cache = perf->may_cache;
1186
1187 break;
1188 }
1189
1190 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1191 {
1192 old_memory_object_attr_info_t attr;
1193
1194 if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1195 result = KERN_INVALID_ARGUMENT;
1196 break;
1197 }
1198
1199 attr = (old_memory_object_attr_info_t) attributes;
1200
1201 may_cache = attr->may_cache;
1202 copy_strategy = attr->copy_strategy;
1203
1204 break;
1205 }
1206
1207 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1208 {
1209 memory_object_attr_info_t attr;
1210
1211 if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
1212 result = KERN_INVALID_ARGUMENT;
1213 break;
1214 }
1215
1216 attr = (memory_object_attr_info_t) attributes;
1217
1218 copy_strategy = attr->copy_strategy;
1219 may_cache = attr->may_cache_object;
1220
1221 break;
1222 }
1223
1224 default:
1225 result = KERN_INVALID_ARGUMENT;
1226 break;
1227 }
1228
1229 if (result != KERN_SUCCESS)
1230 return(result);
1231
1232 if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
1233 copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1234 }
1235
1236 /*
1237 * XXX may_cache may become a tri-valued variable to handle
1238 * XXX uncache if not in use.
1239 */
1240 return (vm_object_set_attributes_common(object,
1241 may_cache,
1242 copy_strategy));
1243}
1244
1245kern_return_t
1246memory_object_get_attributes(
1247 memory_object_control_t control,
1248 memory_object_flavor_t flavor,
1249 memory_object_info_t attributes, /* pointer to OUT array */
1250 mach_msg_type_number_t *count) /* IN/OUT */
1251{
1252 kern_return_t ret = KERN_SUCCESS;
1253 vm_object_t object;
1254
1255 object = memory_object_control_to_vm_object(control);
1256 if (object == VM_OBJECT_NULL)
1257 return (KERN_INVALID_ARGUMENT);
1258
1259 vm_object_lock(object);
1260
1261 switch (flavor) {
1262 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1263 {
1264 old_memory_object_behave_info_t behave;
1265
1266 if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1267 ret = KERN_INVALID_ARGUMENT;
1268 break;
1269 }
1270
1271 behave = (old_memory_object_behave_info_t) attributes;
1272 behave->copy_strategy = object->copy_strategy;
1273 behave->temporary = FALSE;
1274#if notyet /* remove when vm_msync complies and clean in place fini */
1275 behave->invalidate = object->invalidate;
1276#else
1277 behave->invalidate = FALSE;
1278#endif
1279
1280 *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1281 break;
1282 }
1283
1284 case MEMORY_OBJECT_BEHAVIOR_INFO:
1285 {
1286 memory_object_behave_info_t behave;
1287
1288 if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1289 ret = KERN_INVALID_ARGUMENT;
1290 break;
1291 }
1292
1293 behave = (memory_object_behave_info_t) attributes;
1294 behave->copy_strategy = object->copy_strategy;
1295 behave->temporary = FALSE;
1296#if notyet /* remove when vm_msync complies and clean in place fini */
1297 behave->invalidate = object->invalidate;
1298#else
1299 behave->invalidate = FALSE;
1300#endif
1301 behave->advisory_pageout = FALSE;
1302 behave->silent_overwrite = FALSE;
1303 *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1304 break;
1305 }
1306
1307 case MEMORY_OBJECT_PERFORMANCE_INFO:
1308 {
1309 memory_object_perf_info_t perf;
1310
1311 if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
1312 ret = KERN_INVALID_ARGUMENT;
1313 break;
1314 }
1315
1316 perf = (memory_object_perf_info_t) attributes;
1317 perf->cluster_size = PAGE_SIZE;
1318 perf->may_cache = object->can_persist;
1319
1320 *count = MEMORY_OBJECT_PERF_INFO_COUNT;
1321 break;
1322 }
1323
1324 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1325 {
1326 old_memory_object_attr_info_t attr;
1327
1328 if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1329 ret = KERN_INVALID_ARGUMENT;
1330 break;
1331 }
1332
1333 attr = (old_memory_object_attr_info_t) attributes;
1334 attr->may_cache = object->can_persist;
1335 attr->copy_strategy = object->copy_strategy;
1336
1337 *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
1338 break;
1339 }
1340
1341 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1342 {
1343 memory_object_attr_info_t attr;
1344
1345 if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
1346 ret = KERN_INVALID_ARGUMENT;
1347 break;
1348 }
1349
1350 attr = (memory_object_attr_info_t) attributes;
1351 attr->copy_strategy = object->copy_strategy;
1352 attr->cluster_size = PAGE_SIZE;
1353 attr->may_cache_object = object->can_persist;
1354 attr->temporary = FALSE;
1355
1356 *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
1357 break;
1358 }
1359
1360 default:
1361 ret = KERN_INVALID_ARGUMENT;
1362 break;
1363 }
1364
1365 vm_object_unlock(object);
1366
1367 return(ret);
1368}
1369
1370
1371kern_return_t
1372memory_object_iopl_request(
1373 ipc_port_t port,
1374 memory_object_offset_t offset,
1375 upl_size_t *upl_size,
1376 upl_t *upl_ptr,
1377 upl_page_info_array_t user_page_list,
1378 unsigned int *page_list_count,
1379 upl_control_flags_t *flags,
1380 vm_tag_t tag)
1381{
1382 vm_object_t object;
1383 kern_return_t ret;
1384 upl_control_flags_t caller_flags;
1385
1386 caller_flags = *flags;
1387
1388 if (caller_flags & ~UPL_VALID_FLAGS) {
1389 /*
1390 * For forward compatibility's sake,
1391 * reject any unknown flag.
1392 */
1393 return KERN_INVALID_VALUE;
1394 }
1395
1396 if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
1397 vm_named_entry_t named_entry;
1398
1399 named_entry = (vm_named_entry_t)port->ip_kobject;
1400 /* a few checks to make sure user is obeying rules */
1401 if(*upl_size == 0) {
1402 if(offset >= named_entry->size)
1403 return(KERN_INVALID_RIGHT);
1404 *upl_size = (upl_size_t)(named_entry->size - offset);
1405 if (*upl_size != named_entry->size - offset)
1406 return KERN_INVALID_ARGUMENT;
1407 }
1408 if(caller_flags & UPL_COPYOUT_FROM) {
1409 if((named_entry->protection & VM_PROT_READ)
1410 != VM_PROT_READ) {
1411 return(KERN_INVALID_RIGHT);
1412 }
1413 } else {
1414 if((named_entry->protection &
1415 (VM_PROT_READ | VM_PROT_WRITE))
1416 != (VM_PROT_READ | VM_PROT_WRITE)) {
1417 return(KERN_INVALID_RIGHT);
1418 }
1419 }
1420 if(named_entry->size < (offset + *upl_size))
1421 return(KERN_INVALID_ARGUMENT);
1422
1423 /* the callers parameter offset is defined to be the */
1424 /* offset from beginning of named entry offset in object */
1425 offset = offset + named_entry->offset;
1426
1427 if (named_entry->is_sub_map ||
1428 named_entry->is_copy)
1429 return KERN_INVALID_ARGUMENT;
1430
1431 named_entry_lock(named_entry);
1432
1433 object = named_entry->backing.object;
1434 vm_object_reference(object);
1435 named_entry_unlock(named_entry);
1436 } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
1437 memory_object_control_t control;
1438 control = (memory_object_control_t) port;
1439 if (control == NULL)
1440 return (KERN_INVALID_ARGUMENT);
1441 object = memory_object_control_to_vm_object(control);
1442 if (object == VM_OBJECT_NULL)
1443 return (KERN_INVALID_ARGUMENT);
1444 vm_object_reference(object);
1445 } else {
1446 return KERN_INVALID_ARGUMENT;
1447 }
1448 if (object == VM_OBJECT_NULL)
1449 return (KERN_INVALID_ARGUMENT);
1450
1451 if (!object->private) {
1452 if (object->phys_contiguous) {
1453 *flags = UPL_PHYS_CONTIG;
1454 } else {
1455 *flags = 0;
1456 }
1457 } else {
1458 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
1459 }
1460
1461 ret = vm_object_iopl_request(object,
1462 offset,
1463 *upl_size,
1464 upl_ptr,
1465 user_page_list,
1466 page_list_count,
1467 caller_flags,
1468 tag);
1469 vm_object_deallocate(object);
1470 return ret;
1471}
1472
1473/*
1474 * Routine: memory_object_upl_request [interface]
1475 * Purpose:
1476 * Cause the population of a portion of a vm_object.
1477 * Depending on the nature of the request, the pages
1478 * returned may be contain valid data or be uninitialized.
1479 *
1480 */
1481
1482kern_return_t
1483memory_object_upl_request(
1484 memory_object_control_t control,
1485 memory_object_offset_t offset,
1486 upl_size_t size,
1487 upl_t *upl_ptr,
1488 upl_page_info_array_t user_page_list,
1489 unsigned int *page_list_count,
1490 int cntrl_flags,
1491 int tag)
1492{
1493 vm_object_t object;
1494
1495 object = memory_object_control_to_vm_object(control);
1496 if (object == VM_OBJECT_NULL)
1497 return (KERN_TERMINATED);
1498
1499 return vm_object_upl_request(object,
1500 offset,
1501 size,
1502 upl_ptr,
1503 user_page_list,
1504 page_list_count,
1505 (upl_control_flags_t)(unsigned int) cntrl_flags,
1506 tag);
1507}
1508
1509/*
1510 * Routine: memory_object_super_upl_request [interface]
1511 * Purpose:
1512 * Cause the population of a portion of a vm_object
1513 * in much the same way as memory_object_upl_request.
1514 * Depending on the nature of the request, the pages
1515 * returned may be contain valid data or be uninitialized.
1516 * However, the region may be expanded up to the super
1517 * cluster size provided.
1518 */
1519
1520kern_return_t
1521memory_object_super_upl_request(
1522 memory_object_control_t control,
1523 memory_object_offset_t offset,
1524 upl_size_t size,
1525 upl_size_t super_cluster,
1526 upl_t *upl,
1527 upl_page_info_t *user_page_list,
1528 unsigned int *page_list_count,
1529 int cntrl_flags,
1530 int tag)
1531{
1532 vm_object_t object;
1533
1534 object = memory_object_control_to_vm_object(control);
1535 if (object == VM_OBJECT_NULL)
1536 return (KERN_INVALID_ARGUMENT);
1537
1538 return vm_object_super_upl_request(object,
1539 offset,
1540 size,
1541 super_cluster,
1542 upl,
1543 user_page_list,
1544 page_list_count,
1545 (upl_control_flags_t)(unsigned int) cntrl_flags,
1546 tag);
1547}
1548
1549kern_return_t
1550memory_object_cluster_size(
1551 memory_object_control_t control,
1552 memory_object_offset_t *start,
1553 vm_size_t *length,
1554 uint32_t *io_streaming,
1555 memory_object_fault_info_t mo_fault_info)
1556{
1557 vm_object_t object;
1558 vm_object_fault_info_t fault_info;
1559
1560 object = memory_object_control_to_vm_object(control);
1561
1562 if (object == VM_OBJECT_NULL || object->paging_offset > *start)
1563 return KERN_INVALID_ARGUMENT;
1564
1565 *start -= object->paging_offset;
1566
1567 fault_info = (vm_object_fault_info_t)(uintptr_t) mo_fault_info;
1568 vm_object_cluster_size(object,
1569 (vm_object_offset_t *)start,
1570 length,
1571 fault_info,
1572 io_streaming);
1573
1574 *start += object->paging_offset;
1575
1576 return KERN_SUCCESS;
1577}
1578
1579
1580/*
1581 * Routine: host_default_memory_manager [interface]
1582 * Purpose:
1583 * set/get the default memory manager port and default cluster
1584 * size.
1585 *
1586 * If successful, consumes the supplied naked send right.
1587 */
1588kern_return_t
1589host_default_memory_manager(
1590 host_priv_t host_priv,
1591 memory_object_default_t *default_manager,
1592 __unused memory_object_cluster_size_t cluster_size)
1593{
1594 memory_object_default_t current_manager;
1595 memory_object_default_t new_manager;
1596 memory_object_default_t returned_manager;
1597 kern_return_t result = KERN_SUCCESS;
1598
1599 if (host_priv == HOST_PRIV_NULL)
1600 return(KERN_INVALID_HOST);
1601
1602 assert(host_priv == &realhost);
1603
1604 new_manager = *default_manager;
1605 lck_mtx_lock(&memory_manager_default_lock);
1606 current_manager = memory_manager_default;
1607 returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
1608
1609 if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1610 /*
1611 * Retrieve the current value.
1612 */
1613 returned_manager = current_manager;
1614 memory_object_default_reference(returned_manager);
1615 } else {
1616 /*
1617 * Only allow the kernel to change the value.
1618 */
1619 extern task_t kernel_task;
1620 if (current_task() != kernel_task) {
1621 result = KERN_NO_ACCESS;
1622 goto out;
1623 }
1624
1625 /*
1626 * If this is the first non-null manager, start
1627 * up the internal pager support.
1628 */
1629 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1630 result = vm_pageout_internal_start();
1631 if (result != KERN_SUCCESS)
1632 goto out;
1633 }
1634
1635 /*
1636 * Retrieve the current value,
1637 * and replace it with the supplied value.
1638 * We return the old reference to the caller
1639 * but we have to take a reference on the new
1640 * one.
1641 */
1642 returned_manager = current_manager;
1643 memory_manager_default = new_manager;
1644 memory_object_default_reference(new_manager);
1645
1646 /*
1647 * In case anyone's been waiting for a memory
1648 * manager to be established, wake them up.
1649 */
1650
1651 thread_wakeup((event_t) &memory_manager_default);
1652
1653 /*
1654 * Now that we have a default pager for anonymous memory,
1655 * reactivate all the throttled pages (i.e. dirty pages with
1656 * no pager).
1657 */
1658 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL)
1659 {
1660 vm_page_reactivate_all_throttled();
1661 }
1662 }
1663 out:
1664 lck_mtx_unlock(&memory_manager_default_lock);
1665
1666 *default_manager = returned_manager;
1667 return(result);
1668}
1669
1670/*
1671 * Routine: memory_manager_default_reference
1672 * Purpose:
1673 * Returns a naked send right for the default
1674 * memory manager. The returned right is always
1675 * valid (not IP_NULL or IP_DEAD).
1676 */
1677
1678__private_extern__ memory_object_default_t
1679memory_manager_default_reference(void)
1680{
1681 memory_object_default_t current_manager;
1682
1683 lck_mtx_lock(&memory_manager_default_lock);
1684 current_manager = memory_manager_default;
1685 while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1686 wait_result_t res;
1687
1688 res = lck_mtx_sleep(&memory_manager_default_lock,
1689 LCK_SLEEP_DEFAULT,
1690 (event_t) &memory_manager_default,
1691 THREAD_UNINT);
1692 assert(res == THREAD_AWAKENED);
1693 current_manager = memory_manager_default;
1694 }
1695 memory_object_default_reference(current_manager);
1696 lck_mtx_unlock(&memory_manager_default_lock);
1697
1698 return current_manager;
1699}
1700
1701/*
1702 * Routine: memory_manager_default_check
1703 *
1704 * Purpose:
1705 * Check whether a default memory manager has been set
1706 * up yet, or not. Returns KERN_SUCCESS if dmm exists,
1707 * and KERN_FAILURE if dmm does not exist.
1708 *
1709 * If there is no default memory manager, log an error,
1710 * but only the first time.
1711 *
1712 */
1713__private_extern__ kern_return_t
1714memory_manager_default_check(void)
1715{
1716 memory_object_default_t current;
1717
1718 lck_mtx_lock(&memory_manager_default_lock);
1719 current = memory_manager_default;
1720 if (current == MEMORY_OBJECT_DEFAULT_NULL) {
1721 static boolean_t logged; /* initialized to 0 */
1722 boolean_t complain = !logged;
1723 logged = TRUE;
1724 lck_mtx_unlock(&memory_manager_default_lock);
1725 if (complain)
1726 printf("Warning: No default memory manager\n");
1727 return(KERN_FAILURE);
1728 } else {
1729 lck_mtx_unlock(&memory_manager_default_lock);
1730 return(KERN_SUCCESS);
1731 }
1732}
1733
1734__private_extern__ void
1735memory_manager_default_init(void)
1736{
1737 memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
1738 lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr);
1739}
1740
1741
1742
1743/* Allow manipulation of individual page state. This is actually part of */
1744/* the UPL regimen but takes place on the object rather than on a UPL */
1745
1746kern_return_t
1747memory_object_page_op(
1748 memory_object_control_t control,
1749 memory_object_offset_t offset,
1750 int ops,
1751 ppnum_t *phys_entry,
1752 int *flags)
1753{
1754 vm_object_t object;
1755
1756 object = memory_object_control_to_vm_object(control);
1757 if (object == VM_OBJECT_NULL)
1758 return (KERN_INVALID_ARGUMENT);
1759
1760 return vm_object_page_op(object, offset, ops, phys_entry, flags);
1761}
1762
1763/*
1764 * memory_object_range_op offers performance enhancement over
1765 * memory_object_page_op for page_op functions which do not require page
1766 * level state to be returned from the call. Page_op was created to provide
1767 * a low-cost alternative to page manipulation via UPLs when only a single
1768 * page was involved. The range_op call establishes the ability in the _op
1769 * family of functions to work on multiple pages where the lack of page level
1770 * state handling allows the caller to avoid the overhead of the upl structures.
1771 */
1772
1773kern_return_t
1774memory_object_range_op(
1775 memory_object_control_t control,
1776 memory_object_offset_t offset_beg,
1777 memory_object_offset_t offset_end,
1778 int ops,
1779 int *range)
1780{
1781 vm_object_t object;
1782
1783 object = memory_object_control_to_vm_object(control);
1784 if (object == VM_OBJECT_NULL)
1785 return (KERN_INVALID_ARGUMENT);
1786
1787 return vm_object_range_op(object,
1788 offset_beg,
1789 offset_end,
1790 ops,
1791 (uint32_t *) range);
1792}
1793
1794
1795void
1796memory_object_mark_used(
1797 memory_object_control_t control)
1798{
1799 vm_object_t object;
1800
1801 if (control == NULL)
1802 return;
1803
1804 object = memory_object_control_to_vm_object(control);
1805
1806 if (object != VM_OBJECT_NULL)
1807 vm_object_cache_remove(object);
1808}
1809
1810
1811void
1812memory_object_mark_unused(
1813 memory_object_control_t control,
1814 __unused boolean_t rage)
1815{
1816 vm_object_t object;
1817
1818 if (control == NULL)
1819 return;
1820
1821 object = memory_object_control_to_vm_object(control);
1822
1823 if (object != VM_OBJECT_NULL)
1824 vm_object_cache_add(object);
1825}
1826
1827void
1828memory_object_mark_io_tracking(
1829 memory_object_control_t control)
1830{
1831 vm_object_t object;
1832
1833 if (control == NULL)
1834 return;
1835 object = memory_object_control_to_vm_object(control);
1836
1837 if (object != VM_OBJECT_NULL) {
1838 vm_object_lock(object);
1839 object->io_tracking = TRUE;
1840 vm_object_unlock(object);
1841 }
1842}
1843
1844#if CONFIG_SECLUDED_MEMORY
1845void
1846memory_object_mark_eligible_for_secluded(
1847 memory_object_control_t control,
1848 boolean_t eligible_for_secluded)
1849{
1850 vm_object_t object;
1851
1852 if (control == NULL)
1853 return;
1854 object = memory_object_control_to_vm_object(control);
1855
1856 if (object == VM_OBJECT_NULL) {
1857 return;
1858 }
1859
1860 vm_object_lock(object);
1861 if (eligible_for_secluded &&
1862 secluded_for_filecache && /* global boot-arg */
1863 !object->eligible_for_secluded) {
1864 object->eligible_for_secluded = TRUE;
1865 vm_page_secluded.eligible_for_secluded += object->resident_page_count;
1866 } else if (!eligible_for_secluded &&
1867 object->eligible_for_secluded) {
1868 object->eligible_for_secluded = FALSE;
1869 vm_page_secluded.eligible_for_secluded -= object->resident_page_count;
1870 if (object->resident_page_count) {
1871 /* XXX FBDP TODO: flush pages from secluded queue? */
1872 // printf("FBDP TODO: flush %d pages from %p from secluded queue\n", object->resident_page_count, object);
1873 }
1874 }
1875 vm_object_unlock(object);
1876}
1877#endif /* CONFIG_SECLUDED_MEMORY */
1878
1879kern_return_t
1880memory_object_pages_resident(
1881 memory_object_control_t control,
1882 boolean_t * has_pages_resident)
1883{
1884 vm_object_t object;
1885
1886 *has_pages_resident = FALSE;
1887
1888 object = memory_object_control_to_vm_object(control);
1889 if (object == VM_OBJECT_NULL)
1890 return (KERN_INVALID_ARGUMENT);
1891
1892 if (object->resident_page_count)
1893 *has_pages_resident = TRUE;
1894
1895 return (KERN_SUCCESS);
1896}
1897
1898kern_return_t
1899memory_object_signed(
1900 memory_object_control_t control,
1901 boolean_t is_signed)
1902{
1903 vm_object_t object;
1904
1905 object = memory_object_control_to_vm_object(control);
1906 if (object == VM_OBJECT_NULL)
1907 return KERN_INVALID_ARGUMENT;
1908
1909 vm_object_lock(object);
1910 object->code_signed = is_signed;
1911 vm_object_unlock(object);
1912
1913 return KERN_SUCCESS;
1914}
1915
1916boolean_t
1917memory_object_is_signed(
1918 memory_object_control_t control)
1919{
1920 boolean_t is_signed;
1921 vm_object_t object;
1922
1923 object = memory_object_control_to_vm_object(control);
1924 if (object == VM_OBJECT_NULL)
1925 return FALSE;
1926
1927 vm_object_lock_shared(object);
1928 is_signed = object->code_signed;
1929 vm_object_unlock(object);
1930
1931 return is_signed;
1932}
1933
1934boolean_t
1935memory_object_is_shared_cache(
1936 memory_object_control_t control)
1937{
1938 vm_object_t object = VM_OBJECT_NULL;
1939
1940 object = memory_object_control_to_vm_object(control);
1941 if (object == VM_OBJECT_NULL)
1942 return FALSE;
1943
1944 return object->object_is_shared_cache;
1945}
1946
1947static zone_t mem_obj_control_zone;
1948
1949__private_extern__ void
1950memory_object_control_bootstrap(void)
1951{
1952 int i;
1953
1954 i = (vm_size_t) sizeof (struct memory_object_control);
1955 mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control");
1956 zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE);
1957 zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE);
1958 return;
1959}
1960
1961__private_extern__ memory_object_control_t
1962memory_object_control_allocate(
1963 vm_object_t object)
1964{
1965 memory_object_control_t control;
1966
1967 control = (memory_object_control_t)zalloc(mem_obj_control_zone);
1968 if (control != MEMORY_OBJECT_CONTROL_NULL) {
1969 control->moc_object = object;
1970 control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
1971 }
1972 return (control);
1973}
1974
1975__private_extern__ void
1976memory_object_control_collapse(
1977 memory_object_control_t control,
1978 vm_object_t object)
1979{
1980 assert((control->moc_object != VM_OBJECT_NULL) &&
1981 (control->moc_object != object));
1982 control->moc_object = object;
1983}
1984
1985__private_extern__ vm_object_t
1986memory_object_control_to_vm_object(
1987 memory_object_control_t control)
1988{
1989 if (control == MEMORY_OBJECT_CONTROL_NULL ||
1990 control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
1991 return VM_OBJECT_NULL;
1992
1993 return (control->moc_object);
1994}
1995
1996__private_extern__ vm_object_t
1997memory_object_to_vm_object(
1998 memory_object_t mem_obj)
1999{
2000 memory_object_control_t mo_control;
2001
2002 if (mem_obj == MEMORY_OBJECT_NULL) {
2003 return VM_OBJECT_NULL;
2004 }
2005 mo_control = mem_obj->mo_control;
2006 if (mo_control == NULL) {
2007 return VM_OBJECT_NULL;
2008 }
2009 return memory_object_control_to_vm_object(mo_control);
2010}
2011
2012memory_object_control_t
2013convert_port_to_mo_control(
2014 __unused mach_port_t port)
2015{
2016 return MEMORY_OBJECT_CONTROL_NULL;
2017}
2018
2019
2020mach_port_t
2021convert_mo_control_to_port(
2022 __unused memory_object_control_t control)
2023{
2024 return MACH_PORT_NULL;
2025}
2026
2027void
2028memory_object_control_reference(
2029 __unused memory_object_control_t control)
2030{
2031 return;
2032}
2033
2034/*
2035 * We only every issue one of these references, so kill it
2036 * when that gets released (should switch the real reference
2037 * counting in true port-less EMMI).
2038 */
2039void
2040memory_object_control_deallocate(
2041 memory_object_control_t control)
2042{
2043 zfree(mem_obj_control_zone, control);
2044}
2045
2046void
2047memory_object_control_disable(
2048 memory_object_control_t control)
2049{
2050 assert(control->moc_object != VM_OBJECT_NULL);
2051 control->moc_object = VM_OBJECT_NULL;
2052}
2053
2054void
2055memory_object_default_reference(
2056 memory_object_default_t dmm)
2057{
2058 ipc_port_make_send(dmm);
2059}
2060
2061void
2062memory_object_default_deallocate(
2063 memory_object_default_t dmm)
2064{
2065 ipc_port_release_send(dmm);
2066}
2067
2068memory_object_t
2069convert_port_to_memory_object(
2070 __unused mach_port_t port)
2071{
2072 return (MEMORY_OBJECT_NULL);
2073}
2074
2075
2076mach_port_t
2077convert_memory_object_to_port(
2078 __unused memory_object_t object)
2079{
2080 return (MACH_PORT_NULL);
2081}
2082
2083
2084/* Routine memory_object_reference */
2085void memory_object_reference(
2086 memory_object_t memory_object)
2087{
2088 (memory_object->mo_pager_ops->memory_object_reference)(
2089 memory_object);
2090}
2091
2092/* Routine memory_object_deallocate */
2093void memory_object_deallocate(
2094 memory_object_t memory_object)
2095{
2096 (memory_object->mo_pager_ops->memory_object_deallocate)(
2097 memory_object);
2098}
2099
2100
2101/* Routine memory_object_init */
2102kern_return_t memory_object_init
2103(
2104 memory_object_t memory_object,
2105 memory_object_control_t memory_control,
2106 memory_object_cluster_size_t memory_object_page_size
2107)
2108{
2109 return (memory_object->mo_pager_ops->memory_object_init)(
2110 memory_object,
2111 memory_control,
2112 memory_object_page_size);
2113}
2114
2115/* Routine memory_object_terminate */
2116kern_return_t memory_object_terminate
2117(
2118 memory_object_t memory_object
2119)
2120{
2121 return (memory_object->mo_pager_ops->memory_object_terminate)(
2122 memory_object);
2123}
2124
2125/* Routine memory_object_data_request */
2126kern_return_t memory_object_data_request
2127(
2128 memory_object_t memory_object,
2129 memory_object_offset_t offset,
2130 memory_object_cluster_size_t length,
2131 vm_prot_t desired_access,
2132 memory_object_fault_info_t fault_info
2133)
2134{
2135 return (memory_object->mo_pager_ops->memory_object_data_request)(
2136 memory_object,
2137 offset,
2138 length,
2139 desired_access,
2140 fault_info);
2141}
2142
2143/* Routine memory_object_data_return */
2144kern_return_t memory_object_data_return
2145(
2146 memory_object_t memory_object,
2147 memory_object_offset_t offset,
2148 memory_object_cluster_size_t size,
2149 memory_object_offset_t *resid_offset,
2150 int *io_error,
2151 boolean_t dirty,
2152 boolean_t kernel_copy,
2153 int upl_flags
2154)
2155{
2156 return (memory_object->mo_pager_ops->memory_object_data_return)(
2157 memory_object,
2158 offset,
2159 size,
2160 resid_offset,
2161 io_error,
2162 dirty,
2163 kernel_copy,
2164 upl_flags);
2165}
2166
2167/* Routine memory_object_data_initialize */
2168kern_return_t memory_object_data_initialize
2169(
2170 memory_object_t memory_object,
2171 memory_object_offset_t offset,
2172 memory_object_cluster_size_t size
2173)
2174{
2175 return (memory_object->mo_pager_ops->memory_object_data_initialize)(
2176 memory_object,
2177 offset,
2178 size);
2179}
2180
2181/* Routine memory_object_data_unlock */
2182kern_return_t memory_object_data_unlock
2183(
2184 memory_object_t memory_object,
2185 memory_object_offset_t offset,
2186 memory_object_size_t size,
2187 vm_prot_t desired_access
2188)
2189{
2190 return (memory_object->mo_pager_ops->memory_object_data_unlock)(
2191 memory_object,
2192 offset,
2193 size,
2194 desired_access);
2195}
2196
2197/* Routine memory_object_synchronize */
2198kern_return_t memory_object_synchronize
2199(
2200 memory_object_t memory_object,
2201 memory_object_offset_t offset,
2202 memory_object_size_t size,
2203 vm_sync_t sync_flags
2204)
2205{
2206 panic("memory_object_syncrhonize no longer supported\n");
2207
2208 return (memory_object->mo_pager_ops->memory_object_synchronize)(
2209 memory_object,
2210 offset,
2211 size,
2212 sync_flags);
2213}
2214
2215
2216/*
2217 * memory_object_map() is called by VM (in vm_map_enter() and its variants)
2218 * each time a "named" VM object gets mapped directly or indirectly
2219 * (copy-on-write mapping). A "named" VM object has an extra reference held
2220 * by the pager to keep it alive until the pager decides that the
2221 * memory object (and its VM object) can be reclaimed.
2222 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
2223 * the mappings of that memory object have been removed.
2224 *
2225 * For a given VM object, calls to memory_object_map() and memory_object_unmap()
2226 * are serialized (through object->mapping_in_progress), to ensure that the
2227 * pager gets a consistent view of the mapping status of the memory object.
2228 *
2229 * This allows the pager to keep track of how many times a memory object
2230 * has been mapped and with which protections, to decide when it can be
2231 * reclaimed.
2232 */
2233
2234/* Routine memory_object_map */
2235kern_return_t memory_object_map
2236(
2237 memory_object_t memory_object,
2238 vm_prot_t prot
2239)
2240{
2241 return (memory_object->mo_pager_ops->memory_object_map)(
2242 memory_object,
2243 prot);
2244}
2245
2246/* Routine memory_object_last_unmap */
2247kern_return_t memory_object_last_unmap
2248(
2249 memory_object_t memory_object
2250)
2251{
2252 return (memory_object->mo_pager_ops->memory_object_last_unmap)(
2253 memory_object);
2254}
2255
2256/* Routine memory_object_data_reclaim */
2257kern_return_t memory_object_data_reclaim
2258(
2259 memory_object_t memory_object,
2260 boolean_t reclaim_backing_store
2261)
2262{
2263 if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL)
2264 return KERN_NOT_SUPPORTED;
2265 return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
2266 memory_object,
2267 reclaim_backing_store);
2268}
2269
2270upl_t
2271convert_port_to_upl(
2272 ipc_port_t port)
2273{
2274 upl_t upl;
2275
2276 ip_lock(port);
2277 if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) {
2278 ip_unlock(port);
2279 return (upl_t)NULL;
2280 }
2281 upl = (upl_t) port->ip_kobject;
2282 ip_unlock(port);
2283 upl_lock(upl);
2284 upl->ref_count+=1;
2285 upl_unlock(upl);
2286 return upl;
2287}
2288
2289mach_port_t
2290convert_upl_to_port(
2291 __unused upl_t upl)
2292{
2293 return MACH_PORT_NULL;
2294}
2295
2296__private_extern__ void
2297upl_no_senders(
2298 __unused ipc_port_t port,
2299 __unused mach_port_mscount_t mscount)
2300{
2301 return;
2302}
2303