1/*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34/*
35 * File: vnode_pager.c
36 *
37 * "Swap" pager that pages to/from vnodes. Also
38 * handles demand paging from files.
39 *
40 */
41
42#include <mach/boolean.h>
43#include <sys/param.h>
44#include <sys/systm.h>
45#include <sys/user.h>
46#include <sys/proc.h>
47#include <sys/kauth.h>
48#include <sys/buf.h>
49#include <sys/uio.h>
50#include <sys/vnode_internal.h>
51#include <sys/namei.h>
52#include <sys/mount_internal.h> /* needs internal due to fhandle_t */
53#include <sys/ubc_internal.h>
54#include <sys/lock.h>
55#include <sys/disk.h> /* For DKIOC calls */
56
57#include <mach/mach_types.h>
58#include <mach/memory_object_types.h>
59#include <mach/memory_object_control.h>
60#include <mach/vm_map.h>
61#include <mach/mach_vm.h>
62#include <mach/upl.h>
63#include <mach/sdt.h>
64
65#include <vm/vm_map.h>
66#include <vm/vm_kern.h>
67#include <kern/zalloc.h>
68#include <kern/kalloc.h>
69#include <libkern/libkern.h>
70
71#include <vm/vnode_pager.h>
72#include <vm/vm_pageout.h>
73
74#include <kern/assert.h>
75#include <sys/kdebug.h>
76#include <nfs/rpcv2.h>
77#include <nfs/nfsproto.h>
78#include <nfs/nfs.h>
79
80#include <vm/vm_protos.h>
81
82#include <vfs/vfs_disk_conditioner.h>
83
84void
85vnode_pager_throttle()
86{
87 struct uthread *ut;
88
89 ut = get_bsdthread_info(current_thread());
90
91 if (ut->uu_lowpri_window)
92 throttle_lowpri_io(1);
93}
94
95boolean_t
96vnode_pager_isSSD(vnode_t vp)
97{
98 return disk_conditioner_mount_is_ssd(vp->v_mount);
99}
100
101#if CONFIG_IOSCHED
102void
103vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
104{
105 u_int32_t blocksize = 0;
106 dk_extent_t extent;
107 dk_set_tier_t set_tier;
108 int error = 0;
109
110 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
111 if (error)
112 return;
113
114 memset(&extent, 0, sizeof(dk_extent_t));
115 memset(&set_tier, 0, sizeof(dk_set_tier_t));
116
117 extent.offset = blkno * (u_int64_t) blocksize;
118 extent.length = len;
119
120 set_tier.extents = &extent;
121 set_tier.extentsCount = 1;
122 set_tier.tier = priority;
123
124 error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
125 return;
126}
127#endif
128
129void
130vnode_pager_was_dirtied(
131 struct vnode *vp,
132 vm_object_offset_t s_offset,
133 vm_object_offset_t e_offset)
134{
135 cluster_update_state(vp, s_offset, e_offset, TRUE);
136}
137
138uint32_t
139vnode_pager_isinuse(struct vnode *vp)
140{
141 if (vp->v_usecount > vp->v_kusecount)
142 return (1);
143 return (0);
144}
145
146uint32_t
147vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)
148{
149 return(cluster_throttle_io_limit(vp, limit));
150}
151
152vm_object_offset_t
153vnode_pager_get_filesize(struct vnode *vp)
154{
155 return (vm_object_offset_t) ubc_getsize(vp);
156}
157
158extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
159
160kern_return_t
161vnode_pager_get_name(
162 struct vnode *vp,
163 char *pathname,
164 vm_size_t pathname_len,
165 char *filename,
166 vm_size_t filename_len,
167 boolean_t *truncated_path_p)
168{
169 *truncated_path_p = FALSE;
170 if (pathname != NULL) {
171 /* get the path name */
172 safe_getpath(vp, NULL,
173 pathname, (int) pathname_len,
174 truncated_path_p);
175 }
176 if ((pathname == NULL || *truncated_path_p) &&
177 filename != NULL) {
178 /* get the file name */
179 const char *name;
180
181 name = vnode_getname_printable(vp);
182 strlcpy(filename, name, (size_t) filename_len);
183 vnode_putname_printable(name);
184 }
185 return KERN_SUCCESS;
186}
187
188kern_return_t
189vnode_pager_get_mtime(
190 struct vnode *vp,
191 struct timespec *current_mtime,
192 struct timespec *cs_mtime)
193{
194 vnode_mtime(vp, current_mtime, vfs_context_current());
195 if (cs_mtime != NULL) {
196 ubc_get_cs_mtime(vp, cs_mtime);
197 }
198 return KERN_SUCCESS;
199}
200
201kern_return_t
202vnode_pager_get_cs_blobs(
203 struct vnode *vp,
204 void **blobs)
205{
206 *blobs = ubc_get_cs_blobs(vp);
207 return KERN_SUCCESS;
208}
209
210/*
211 * vnode_trim:
212 * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
213 * Trims the region at offset bytes into the file, for length bytes.
214 *
215 * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this
216 * function is called; no iocounts or usecounts are taken on the vnode.
217 * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them
218 * are successfully discarded.
219 */
220u_int32_t vnode_trim (
221 struct vnode *vp,
222 off_t offset,
223 size_t length)
224{
225 daddr64_t io_blockno; /* Block number corresponding to the start of the extent */
226 size_t io_bytecount; /* Number of bytes in current extent for the specified range */
227 size_t trimmed = 0;
228 off_t current_offset = offset;
229 size_t remaining_length = length;
230 int error = 0;
231 u_int32_t blocksize = 0;
232 struct vnode *devvp;
233 dk_extent_t extent;
234 dk_unmap_t unmap;
235
236
237 /* Get the underlying device vnode */
238 devvp = vp->v_mount->mnt_devvp;
239
240 /* Figure out the underlying device block size */
241 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
242 if (error) {
243 goto trim_exit;
244 }
245
246 /*
247 * We may not get the entire range from offset -> offset+length in a single
248 * extent from the blockmap call. Keep looping/going until we are sure we've hit
249 * the whole range or if we encounter an error.
250 */
251 while (trimmed < length) {
252 /*
253 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
254 * specified offset. It returns blocks in contiguous chunks, so if the logical range is
255 * broken into multiple extents, it must be called multiple times, increasing the offset
256 * in each call to ensure that the entire range is covered.
257 */
258 error = VNOP_BLOCKMAP (vp, current_offset, remaining_length,
259 &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL);
260
261 if (error) {
262 goto trim_exit;
263 }
264 /*
265 * We have a contiguous run. Prepare & issue the ioctl for the device.
266 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
267 */
268 memset (&extent, 0, sizeof(dk_extent_t));
269 memset (&unmap, 0, sizeof(dk_unmap_t));
270 extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
271 extent.length = io_bytecount;
272 unmap.extents = &extent;
273 unmap.extentsCount = 1;
274 error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel());
275
276 if (error) {
277 goto trim_exit;
278 }
279 remaining_length = remaining_length - io_bytecount;
280 trimmed = trimmed + io_bytecount;
281 current_offset = current_offset + io_bytecount;
282 }
283trim_exit:
284
285 return error;
286
287}
288
289pager_return_t
290vnode_pageout(struct vnode *vp,
291 upl_t upl,
292 upl_offset_t upl_offset,
293 vm_object_offset_t f_offset,
294 upl_size_t size,
295 int flags,
296 int *errorp)
297{
298 int result = PAGER_SUCCESS;
299 int error = 0;
300 int error_ret = 0;
301 daddr64_t blkno;
302 int isize;
303 int pg_index;
304 int base_index;
305 upl_offset_t offset;
306 upl_page_info_t *pl;
307 vfs_context_t ctx = vfs_context_current(); /* pager context */
308
309 isize = (int)size;
310
311 if (isize <= 0) {
312 result = PAGER_ERROR;
313 error_ret = EINVAL;
314 goto out;
315 }
316
317 if (UBCINFOEXISTS(vp) == 0) {
318 result = PAGER_ERROR;
319 error_ret = EINVAL;
320
321 if (upl && !(flags & UPL_NOCOMMIT))
322 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
323 goto out;
324 }
325 if ( !(flags & UPL_VNODE_PAGER)) {
326 /*
327 * This is a pageout from the default pager,
328 * just go ahead and call vnop_pageout since
329 * it has already sorted out the dirty ranges
330 */
331 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
332 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
333 size, 1, 0, 0, 0);
334
335 if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
336 (size_t)size, flags, ctx)) )
337 result = PAGER_ERROR;
338
339 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
340 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
341 size, 1, 0, 0, 0);
342
343 goto out;
344 }
345 if (upl == NULL) {
346 int request_flags;
347
348 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
349 /*
350 * filesystem has requested the new form of VNOP_PAGEOUT for file
351 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT...
352 * it is the fileystem's responsibility to grab the range we're denoting
353 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
354 * take any locks it needs, before effectively locking the pages into a UPL...
355 */
356 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
357 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
358 size, (int)f_offset, 0, 0, 0);
359
360 if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
361 size, flags, ctx)) ) {
362 result = PAGER_ERROR;
363 }
364 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
365 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
366 size, 0, 0, 0, 0);
367
368 goto out;
369 }
370 if (flags & UPL_MSYNC)
371 request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
372 else
373 request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
374
375 if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) {
376 result = PAGER_ERROR;
377 error_ret = EINVAL;
378 goto out;
379 }
380 upl_offset = 0;
381 } else
382 pl = ubc_upl_pageinfo(upl);
383
384 /*
385 * Ignore any non-present pages at the end of the
386 * UPL so that we aren't looking at a upl that
387 * may already have been freed by the preceeding
388 * aborts/completions.
389 */
390 base_index = upl_offset / PAGE_SIZE;
391
392 for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
393 if (upl_page_present(pl, --pg_index))
394 break;
395 if (pg_index == base_index) {
396 /*
397 * no pages were returned, so release
398 * our hold on the upl and leave
399 */
400 if ( !(flags & UPL_NOCOMMIT))
401 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
402
403 goto out;
404 }
405 }
406 isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
407
408 /*
409 * we come here for pageouts to 'real' files and
410 * for msyncs... the upl may not contain any
411 * dirty pages.. it's our responsibility to sort
412 * through it and find the 'runs' of dirty pages
413 * to call VNOP_PAGEOUT on...
414 */
415
416 if (ubc_getsize(vp) == 0) {
417 /*
418 * if the file has been effectively deleted, then
419 * we need to go through the UPL and invalidate any
420 * buffer headers we might have that reference any
421 * of it's pages
422 */
423 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
424#if NFSCLIENT
425 if (vp->v_tag == VT_NFS)
426 /* check with nfs if page is OK to drop */
427 error = nfs_buf_page_inval(vp, (off_t)f_offset);
428 else
429#endif
430 {
431 blkno = ubc_offtoblk(vp, (off_t)f_offset);
432 error = buf_invalblkno(vp, blkno, 0);
433 }
434 if (error) {
435 if ( !(flags & UPL_NOCOMMIT))
436 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
437 if (error_ret == 0)
438 error_ret = error;
439 result = PAGER_ERROR;
440
441 } else if ( !(flags & UPL_NOCOMMIT)) {
442 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
443 }
444 f_offset += PAGE_SIZE;
445 }
446 goto out;
447 }
448
449 offset = upl_offset;
450 pg_index = base_index;
451
452 while (isize) {
453 int xsize;
454 int num_of_pages;
455
456 if ( !upl_page_present(pl, pg_index)) {
457 /*
458 * we asked for RET_ONLY_DIRTY, so it's possible
459 * to get back empty slots in the UPL
460 * just skip over them
461 */
462 f_offset += PAGE_SIZE;
463 offset += PAGE_SIZE;
464 isize -= PAGE_SIZE;
465 pg_index++;
466
467 continue;
468 }
469 if ( !upl_dirty_page(pl, pg_index)) {
470 /*
471 * if the page is not dirty and reached here it is
472 * marked precious or it is due to invalidation in
473 * memory_object_lock request as part of truncation
474 * We also get here from vm_object_terminate()
475 * So all you need to do in these
476 * cases is to invalidate incore buffer if it is there
477 * Note we must not sleep here if the buffer is busy - that is
478 * a lock inversion which causes deadlock.
479 */
480#if NFSCLIENT
481 if (vp->v_tag == VT_NFS)
482 /* check with nfs if page is OK to drop */
483 error = nfs_buf_page_inval(vp, (off_t)f_offset);
484 else
485#endif
486 {
487 blkno = ubc_offtoblk(vp, (off_t)f_offset);
488 error = buf_invalblkno(vp, blkno, 0);
489 }
490 if (error) {
491 if ( !(flags & UPL_NOCOMMIT))
492 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
493 if (error_ret == 0)
494 error_ret = error;
495 result = PAGER_ERROR;
496
497 } else if ( !(flags & UPL_NOCOMMIT)) {
498 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
499 }
500 f_offset += PAGE_SIZE;
501 offset += PAGE_SIZE;
502 isize -= PAGE_SIZE;
503 pg_index++;
504
505 continue;
506 }
507 num_of_pages = 1;
508 xsize = isize - PAGE_SIZE;
509
510 while (xsize) {
511 if ( !upl_dirty_page(pl, pg_index + num_of_pages))
512 break;
513 num_of_pages++;
514 xsize -= PAGE_SIZE;
515 }
516 xsize = num_of_pages * PAGE_SIZE;
517
518 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
519 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
520 xsize, (int)f_offset, 0, 0, 0);
521
522 if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
523 xsize, flags, ctx)) ) {
524 if (error_ret == 0)
525 error_ret = error;
526 result = PAGER_ERROR;
527 }
528 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
529 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
530 xsize, 0, 0, 0, 0);
531
532 f_offset += xsize;
533 offset += xsize;
534 isize -= xsize;
535 pg_index += num_of_pages;
536 }
537out:
538 if (errorp)
539 *errorp = error_ret;
540
541 return (result);
542}
543
544
545pager_return_t
546vnode_pagein(
547 struct vnode *vp,
548 upl_t upl,
549 upl_offset_t upl_offset,
550 vm_object_offset_t f_offset,
551 upl_size_t size,
552 int flags,
553 int *errorp)
554{
555 upl_page_info_t *pl;
556 int result = PAGER_SUCCESS;
557 int error = 0;
558 int pages_in_upl;
559 int start_pg;
560 int last_pg;
561 int first_pg;
562 int xsize;
563 int must_commit = 1;
564 int ignore_valid_page_check = 0;
565
566 if (flags & UPL_NOCOMMIT)
567 must_commit = 0;
568
569 if (flags & UPL_IGNORE_VALID_PAGE_CHECK)
570 ignore_valid_page_check = 1;
571
572 if (UBCINFOEXISTS(vp) == 0) {
573 result = PAGER_ERROR;
574 error = PAGER_ERROR;
575
576 if (upl && must_commit)
577 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
578
579 goto out;
580 }
581 if (upl == (upl_t)NULL) {
582 flags &= ~UPL_NOCOMMIT;
583
584 if (size > MAX_UPL_SIZE_BYTES) {
585 result = PAGER_ERROR;
586 error = PAGER_ERROR;
587 goto out;
588 }
589 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) {
590 /*
591 * filesystem has requested the new form of VNOP_PAGEIN for file
592 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN...
593 * it is the fileystem's responsibility to grab the range we're denoting
594 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
595 * take any locks it needs, before effectively locking the pages into a UPL...
596 * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
597 * is used to identify the "must have" page in the extent... the filesystem is free
598 * to clip the extent to better fit the underlying FS blocksize if it desires as
599 * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
600 * identifies that page
601 */
602 if ( (error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
603 size, flags, vfs_context_current())) ) {
604 result = PAGER_ERROR;
605 error = PAGER_ERROR;
606 }
607 goto out;
608 }
609 ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE);
610
611 if (upl == (upl_t)NULL) {
612 result = PAGER_ABSENT;
613 error = PAGER_ABSENT;
614 goto out;
615 }
616 ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
617
618 upl_offset = 0;
619 first_pg = 0;
620
621 /*
622 * if we get here, we've created the upl and
623 * are responsible for commiting/aborting it
624 * regardless of what the caller has passed in
625 */
626 must_commit = 1;
627 } else {
628 pl = ubc_upl_pageinfo(upl);
629 first_pg = upl_offset / PAGE_SIZE;
630 }
631 pages_in_upl = size / PAGE_SIZE;
632 DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
633
634 /*
635 * before we start marching forward, we must make sure we end on
636 * a present page, otherwise we will be working with a freed
637 * upl
638 */
639 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
640 if (upl_page_present(pl, last_pg))
641 break;
642 if (last_pg == first_pg) {
643 /*
644 * empty UPL, no pages are present
645 */
646 if (must_commit)
647 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
648 goto out;
649 }
650 }
651 pages_in_upl = last_pg + 1;
652 last_pg = first_pg;
653
654 while (last_pg < pages_in_upl) {
655 /*
656 * skip over missing pages...
657 */
658 for ( ; last_pg < pages_in_upl; last_pg++) {
659 if (upl_page_present(pl, last_pg))
660 break;
661 }
662
663 if (ignore_valid_page_check == 1) {
664 start_pg = last_pg;
665 } else {
666 /*
667 * skip over 'valid' pages... we don't want to issue I/O for these
668 */
669 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
670 if (!upl_valid_page(pl, last_pg))
671 break;
672 }
673 }
674
675 if (last_pg > start_pg) {
676 /*
677 * we've found a range of valid pages
678 * if we've got COMMIT responsibility
679 * commit this range of pages back to the
680 * cache unchanged
681 */
682 xsize = (last_pg - start_pg) * PAGE_SIZE;
683
684 if (must_commit)
685 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
686 }
687 if (last_pg == pages_in_upl)
688 /*
689 * we're done... all pages that were present
690 * have either had I/O issued on them or
691 * were aborted unchanged...
692 */
693 break;
694
695 if (!upl_page_present(pl, last_pg)) {
696 /*
697 * we found a range of valid pages
698 * terminated by a missing page...
699 * bump index to the next page and continue on
700 */
701 last_pg++;
702 continue;
703 }
704 /*
705 * scan from the found invalid page looking for a valid
706 * or non-present page before the end of the upl is reached, if we
707 * find one, then it will be the last page of the request to
708 * 'cluster_io'
709 */
710 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
711 if (( !ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg))
712 break;
713 }
714 if (last_pg > start_pg) {
715 int xoff;
716 xsize = (last_pg - start_pg) * PAGE_SIZE;
717 xoff = start_pg * PAGE_SIZE;
718
719 if ( (error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
720 (off_t)f_offset + xoff,
721 xsize, flags, vfs_context_current())) ) {
722 /*
723 * Usually this UPL will be aborted/committed by the lower cluster layer.
724 *
725 * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
726 * a deadlock with another thread already inflating the file.
727 *
728 * b) In the case of content protection, EPERM is a valid error and we should respect it.
729 *
730 * In those cases, we must take care of our UPL at this layer itself.
731 */
732 if (must_commit) {
733 if(error == EAGAIN) {
734 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
735 }
736 if(error == EPERM) {
737 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
738 }
739 }
740 result = PAGER_ERROR;
741 error = PAGER_ERROR;
742
743 }
744 }
745 }
746out:
747 if (errorp)
748 *errorp = result;
749
750 return (error);
751}
752
753void *
754upl_get_internal_page_list(upl_t upl)
755{
756 return(UPL_GET_INTERNAL_PAGE_LIST(upl));
757
758}
759