1/*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29/*
30 * Copyright (c) 1982, 1986, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
67 */
68
69#include <sys/param.h>
70#include <sys/systm.h>
71#include <sys/proc_internal.h>
72#include <sys/malloc.h>
73#include <sys/queue.h>
74#include <vm/pmap.h>
75#include <sys/uio_internal.h>
76#include <kern/kalloc.h>
77
78#include <kdebug.h>
79
80#include <sys/kdebug.h>
81#define DBG_UIO_COPYOUT 16
82#define DBG_UIO_COPYIN 17
83
84#if DEBUG
85#include <kern/simple_lock.h>
86
87static uint32_t uio_t_count = 0;
88#endif /* DEBUG */
89
90#define IS_VALID_UIO_SEGFLG(segflg) \
91 ( (segflg) == UIO_USERSPACE || \
92 (segflg) == UIO_SYSSPACE || \
93 (segflg) == UIO_USERSPACE32 || \
94 (segflg) == UIO_USERSPACE64 || \
95 (segflg) == UIO_SYSSPACE32 || \
96 (segflg) == UIO_USERISPACE || \
97 (segflg) == UIO_PHYS_USERSPACE || \
98 (segflg) == UIO_PHYS_SYSSPACE || \
99 (segflg) == UIO_USERISPACE32 || \
100 (segflg) == UIO_PHYS_USERSPACE32 || \
101 (segflg) == UIO_USERISPACE64 || \
102 (segflg) == UIO_PHYS_USERSPACE64 )
103
104/*
105 * Returns: 0 Success
106 * uiomove64:EFAULT
107 *
108 * Notes: The first argument should be a caddr_t, but const poisoning
109 * for typedef'ed types doesn't work in gcc.
110 */
111int
112uiomove(const char * cp, int n, uio_t uio)
113{
114 return uiomove64((const addr64_t)(uintptr_t)cp, n, uio);
115}
116
117/*
118 * Returns: 0 Success
119 * EFAULT
120 * copyout:EFAULT
121 * copyin:EFAULT
122 * copywithin:EFAULT
123 * copypv:EFAULT
124 */
125int
126uiomove64(const addr64_t c_cp, int n, struct uio *uio)
127{
128 addr64_t cp = c_cp;
129 uint64_t acnt;
130 int error = 0;
131
132#if DIAGNOSTIC
133 if (uio->uio_rw != UIO_READ && uio->uio_rw != UIO_WRITE)
134 panic("uiomove: mode");
135#endif
136
137#if LP64_DEBUG
138 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
139 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
140 }
141#endif /* LP64_DEBUG */
142
143 while (n > 0 && uio_resid(uio)) {
144 uio_update(uio, 0);
145 acnt = uio_curriovlen(uio);
146 if (acnt == 0) {
147 continue;
148 }
149 if (n > 0 && acnt > (uint64_t)n)
150 acnt = n;
151
152 switch ((int) uio->uio_segflg) {
153
154 case UIO_USERSPACE64:
155 case UIO_USERISPACE64:
156 case UIO_USERSPACE32:
157 case UIO_USERISPACE32:
158 case UIO_USERSPACE:
159 case UIO_USERISPACE:
160 // LP64 - 3rd argument in debug code is 64 bit, expected to be 32 bit
161 if (uio->uio_rw == UIO_READ)
162 {
163 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
164 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0,0);
165
166 error = copyout( CAST_DOWN(caddr_t, cp), uio->uio_iovs.uiovp->iov_base, acnt );
167
168 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
169 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0,0);
170 }
171 else
172 {
173 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
174 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0,0);
175
176 error = copyin(uio->uio_iovs.uiovp->iov_base, CAST_DOWN(caddr_t, cp), acnt);
177
178 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
179 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0,0);
180 }
181 if (error)
182 return (error);
183 break;
184
185 case UIO_SYSSPACE32:
186 case UIO_SYSSPACE:
187 if (uio->uio_rw == UIO_READ)
188 error = copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base),
189 acnt);
190 else
191 error = copywithin(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base), CAST_DOWN(caddr_t, cp),
192 acnt);
193 break;
194
195 case UIO_PHYS_USERSPACE64:
196 case UIO_PHYS_USERSPACE32:
197 case UIO_PHYS_USERSPACE:
198 if (uio->uio_rw == UIO_READ)
199 {
200 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
201 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1,0);
202
203 error = copypv((addr64_t)cp, uio->uio_iovs.uiovp->iov_base, acnt, cppvPsrc | cppvNoRefSrc);
204 if (error) /* Copy physical to virtual */
205 error = EFAULT;
206
207 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
208 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1,0);
209 }
210 else
211 {
212 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
213 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1,0);
214
215 error = copypv(uio->uio_iovs.uiovp->iov_base, (addr64_t)cp, acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
216 if (error) /* Copy virtual to physical */
217 error = EFAULT;
218
219 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
220 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1,0);
221 }
222 if (error)
223 return (error);
224 break;
225
226 case UIO_PHYS_SYSSPACE:
227 if (uio->uio_rw == UIO_READ)
228 {
229 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
230 (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2,0);
231
232 error = copypv((addr64_t)cp, uio->uio_iovs.kiovp->iov_base, acnt, cppvKmap | cppvPsrc | cppvNoRefSrc);
233 if (error) /* Copy physical to virtual */
234 error = EFAULT;
235
236 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
237 (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2,0);
238 }
239 else
240 {
241 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
242 (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2,0);
243
244 error = copypv(uio->uio_iovs.kiovp->iov_base, (addr64_t)cp, acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
245 if (error) /* Copy virtual to physical */
246 error = EFAULT;
247
248 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
249 (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2,0);
250 }
251 if (error)
252 return (error);
253 break;
254
255 default:
256 break;
257 }
258 uio_update(uio, acnt);
259 cp += acnt;
260 n -= acnt;
261 }
262 return (error);
263}
264
265/*
266 * Give next character to user as result of read.
267 */
268int
269ureadc(int c, struct uio *uio)
270{
271 if (uio_resid(uio) <= 0)
272 panic("ureadc: non-positive resid");
273 uio_update(uio, 0);
274 if (uio->uio_iovcnt == 0)
275 panic("ureadc: non-positive iovcnt");
276 if (uio_curriovlen(uio) <= 0)
277 panic("ureadc: non-positive iovlen");
278
279 switch ((int) uio->uio_segflg) {
280
281 case UIO_USERSPACE32:
282 case UIO_USERSPACE:
283 case UIO_USERISPACE32:
284 case UIO_USERISPACE:
285 case UIO_USERSPACE64:
286 case UIO_USERISPACE64:
287 if (subyte((user_addr_t)uio->uio_iovs.uiovp->iov_base, c) < 0)
288 return (EFAULT);
289 break;
290
291 case UIO_SYSSPACE32:
292 case UIO_SYSSPACE:
293 *(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base)) = c;
294 break;
295
296 default:
297 break;
298 }
299 uio_update(uio, 1);
300 return (0);
301}
302
303/*
304 * General routine to allocate a hash table.
305 */
306void *
307hashinit(int elements, int type, u_long *hashmask)
308{
309 long hashsize;
310 LIST_HEAD(generic, generic) *hashtbl;
311 int i;
312
313 if (elements <= 0)
314 panic("hashinit: bad cnt");
315 for (hashsize = 1; hashsize <= elements; hashsize <<= 1)
316 continue;
317 hashsize >>= 1;
318 MALLOC(hashtbl, struct generic *,
319 hashsize * sizeof(*hashtbl), type, M_WAITOK|M_ZERO);
320 if (hashtbl != NULL) {
321 for (i = 0; i < hashsize; i++)
322 LIST_INIT(&hashtbl[i]);
323 *hashmask = hashsize - 1;
324 }
325 return (hashtbl);
326}
327
328/*
329 * uio_resid - return the residual IO value for the given uio_t
330 */
331user_ssize_t uio_resid( uio_t a_uio )
332{
333#if DEBUG
334 if (a_uio == NULL) {
335 printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
336 }
337/* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */
338/* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */
339/* } */
340#endif /* DEBUG */
341
342 /* return 0 if there are no active iovecs */
343 if (a_uio == NULL) {
344 return( 0 );
345 }
346
347 return( a_uio->uio_resid_64 );
348}
349
350/*
351 * uio_setresid - set the residual IO value for the given uio_t
352 */
353void uio_setresid( uio_t a_uio, user_ssize_t a_value )
354{
355#if DEBUG
356 if (a_uio == NULL) {
357 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
358 }
359/* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */
360/* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */
361/* } */
362#endif /* DEBUG */
363
364 if (a_uio == NULL) {
365 return;
366 }
367
368 a_uio->uio_resid_64 = a_value;
369 return;
370}
371
372/*
373 * uio_curriovbase - return the base address of the current iovec associated
374 * with the given uio_t. May return 0.
375 */
376user_addr_t uio_curriovbase( uio_t a_uio )
377{
378#if LP64_DEBUG
379 if (a_uio == NULL) {
380 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
381 }
382#endif /* LP64_DEBUG */
383
384 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
385 return(0);
386 }
387
388 if (UIO_IS_USER_SPACE(a_uio)) {
389 return(a_uio->uio_iovs.uiovp->iov_base);
390 }
391 return((user_addr_t)a_uio->uio_iovs.kiovp->iov_base);
392
393}
394
395/*
396 * uio_curriovlen - return the length value of the current iovec associated
397 * with the given uio_t.
398 */
399user_size_t uio_curriovlen( uio_t a_uio )
400{
401#if LP64_DEBUG
402 if (a_uio == NULL) {
403 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
404 }
405#endif /* LP64_DEBUG */
406
407 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
408 return(0);
409 }
410
411 if (UIO_IS_USER_SPACE(a_uio)) {
412 return(a_uio->uio_iovs.uiovp->iov_len);
413 }
414 return((user_size_t)a_uio->uio_iovs.kiovp->iov_len);
415}
416
417/*
418 * uio_setcurriovlen - set the length value of the current iovec associated
419 * with the given uio_t.
420 */
421__private_extern__ void uio_setcurriovlen( uio_t a_uio, user_size_t a_value )
422{
423#if LP64_DEBUG
424 if (a_uio == NULL) {
425 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
426 }
427#endif /* LP64_DEBUG */
428
429 if (a_uio == NULL) {
430 return;
431 }
432
433 if (UIO_IS_USER_SPACE(a_uio)) {
434 a_uio->uio_iovs.uiovp->iov_len = a_value;
435 }
436 else {
437#if LP64_DEBUG
438 if (a_value > 0xFFFFFFFFull) {
439 panic("%s :%d - invalid a_value\n", __FILE__, __LINE__);
440 }
441#endif /* LP64_DEBUG */
442 a_uio->uio_iovs.kiovp->iov_len = (size_t)a_value;
443 }
444 return;
445}
446
447/*
448 * uio_iovcnt - return count of active iovecs for the given uio_t
449 */
450int uio_iovcnt( uio_t a_uio )
451{
452#if LP64_DEBUG
453 if (a_uio == NULL) {
454 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
455 }
456#endif /* LP64_DEBUG */
457
458 if (a_uio == NULL) {
459 return(0);
460 }
461
462 return( a_uio->uio_iovcnt );
463}
464
465/*
466 * uio_offset - return the current offset value for the given uio_t
467 */
468off_t uio_offset( uio_t a_uio )
469{
470#if LP64_DEBUG
471 if (a_uio == NULL) {
472 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
473 }
474#endif /* LP64_DEBUG */
475
476 if (a_uio == NULL) {
477 return(0);
478 }
479 return( a_uio->uio_offset );
480}
481
482/*
483 * uio_setoffset - set the current offset value for the given uio_t
484 */
485void uio_setoffset( uio_t a_uio, off_t a_offset )
486{
487#if LP64_DEBUG
488 if (a_uio == NULL) {
489 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
490 }
491#endif /* LP64_DEBUG */
492
493 if (a_uio == NULL) {
494 return;
495 }
496 a_uio->uio_offset = a_offset;
497 return;
498}
499
500/*
501 * uio_rw - return the read / write flag for the given uio_t
502 */
503int uio_rw( uio_t a_uio )
504{
505#if LP64_DEBUG
506 if (a_uio == NULL) {
507 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
508 }
509#endif /* LP64_DEBUG */
510
511 if (a_uio == NULL) {
512 return(-1);
513 }
514 return( a_uio->uio_rw );
515}
516
517/*
518 * uio_setrw - set the read / write flag for the given uio_t
519 */
520void uio_setrw( uio_t a_uio, int a_value )
521{
522 if (a_uio == NULL) {
523#if LP64_DEBUG
524 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
525#endif /* LP64_DEBUG */
526 return;
527 }
528
529#if LP64_DEBUG
530 if (!(a_value == UIO_READ || a_value == UIO_WRITE)) {
531 panic("%s :%d - invalid a_value\n", __FILE__, __LINE__);
532 }
533#endif /* LP64_DEBUG */
534
535 if (a_value == UIO_READ || a_value == UIO_WRITE) {
536 a_uio->uio_rw = a_value;
537 }
538 return;
539}
540
541/*
542 * uio_isuserspace - return non zero value if the address space
543 * flag is for a user address space (could be 32 or 64 bit).
544 */
545int uio_isuserspace( uio_t a_uio )
546{
547 if (a_uio == NULL) {
548#if LP64_DEBUG
549 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
550#endif /* LP64_DEBUG */
551 return(0);
552 }
553
554 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
555 return( 1 );
556 }
557 return( 0 );
558}
559
560
561/*
562 * uio_create - create an uio_t.
563 * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t
564 * is not fully initialized until all iovecs are added using uio_addiov calls.
565 * a_iovcount is the maximum number of iovecs you may add.
566 */
567uio_t uio_create( int a_iovcount, /* number of iovecs */
568 off_t a_offset, /* current offset */
569 int a_spacetype, /* type of address space */
570 int a_iodirection ) /* read or write flag */
571{
572 void * my_buf_p;
573 size_t my_size;
574 uio_t my_uio;
575
576 my_size = UIO_SIZEOF(a_iovcount);
577 my_buf_p = kalloc(my_size);
578 my_uio = uio_createwithbuffer( a_iovcount,
579 a_offset,
580 a_spacetype,
581 a_iodirection,
582 my_buf_p,
583 my_size );
584 if (my_uio != 0) {
585 /* leave a note that we allocated this uio_t */
586 my_uio->uio_flags |= UIO_FLAGS_WE_ALLOCED;
587#if DEBUG
588 (void)hw_atomic_add(&uio_t_count, 1);
589#endif
590 }
591
592 return( my_uio );
593}
594
595
596/*
597 * uio_createwithbuffer - create an uio_t.
598 * Create a uio_t using the given buffer. The uio_t
599 * is not fully initialized until all iovecs are added using uio_addiov calls.
600 * a_iovcount is the maximum number of iovecs you may add.
601 * This call may fail if the given buffer is not large enough.
602 */
603__private_extern__ uio_t
604 uio_createwithbuffer( int a_iovcount, /* number of iovecs */
605 off_t a_offset, /* current offset */
606 int a_spacetype, /* type of address space */
607 int a_iodirection, /* read or write flag */
608 void *a_buf_p, /* pointer to a uio_t buffer */
609 size_t a_buffer_size ) /* size of uio_t buffer */
610{
611 uio_t my_uio = (uio_t) a_buf_p;
612 size_t my_size;
613
614 my_size = UIO_SIZEOF(a_iovcount);
615 if (a_buffer_size < my_size) {
616#if DEBUG
617 panic("%s :%d - a_buffer_size is too small\n", __FILE__, __LINE__);
618#endif /* DEBUG */
619 return( NULL );
620 }
621 my_size = a_buffer_size;
622
623#if DEBUG
624 if (my_uio == 0) {
625 panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__);
626 }
627 if (!IS_VALID_UIO_SEGFLG(a_spacetype)) {
628 panic("%s :%d - invalid address space type\n", __FILE__, __LINE__);
629 }
630 if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) {
631 panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__);
632 }
633 if (a_iovcount > UIO_MAXIOV) {
634 panic("%s :%d - invalid a_iovcount\n", __FILE__, __LINE__);
635 }
636#endif /* DEBUG */
637
638 bzero(my_uio, my_size);
639 my_uio->uio_size = my_size;
640
641 /*
642 * we use uio_segflg to indicate if the uio_t is the new format or
643 * old (pre LP64 support) legacy format
644 * This switch statement should canonicalize incoming space type
645 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
646 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
647 */
648 switch (a_spacetype) {
649 case UIO_USERSPACE:
650 my_uio->uio_segflg = UIO_USERSPACE32;
651 break;
652 case UIO_SYSSPACE32:
653 my_uio->uio_segflg = UIO_SYSSPACE;
654 break;
655 case UIO_PHYS_USERSPACE:
656 my_uio->uio_segflg = UIO_PHYS_USERSPACE32;
657 break;
658 default:
659 my_uio->uio_segflg = a_spacetype;
660 break;
661 }
662
663 if (a_iovcount > 0) {
664 my_uio->uio_iovs.uiovp = (struct user_iovec *)
665 (((uint8_t *)my_uio) + sizeof(struct uio));
666 }
667 else {
668 my_uio->uio_iovs.uiovp = NULL;
669 }
670
671 my_uio->uio_max_iovs = a_iovcount;
672 my_uio->uio_offset = a_offset;
673 my_uio->uio_rw = a_iodirection;
674 my_uio->uio_flags = UIO_FLAGS_INITED;
675
676 return( my_uio );
677}
678
679/*
680 * uio_spacetype - return the address space type for the given uio_t
681 */
682__private_extern__ int uio_spacetype( uio_t a_uio )
683{
684 if (a_uio == NULL) {
685#if LP64_DEBUG
686 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
687#endif /* LP64_DEBUG */
688 return(-1);
689 }
690
691 return( a_uio->uio_segflg );
692}
693
694/*
695 * uio_iovsaddr - get the address of the iovec array for the given uio_t.
696 * This returns the location of the iovecs within the uio.
697 * NOTE - for compatibility mode we just return the current value in uio_iovs
698 * which will increase as the IO is completed and is NOT embedded within the
699 * uio, it is a seperate array of one or more iovecs.
700 */
701__private_extern__ struct user_iovec * uio_iovsaddr( uio_t a_uio )
702{
703 struct user_iovec * my_addr;
704
705 if (a_uio == NULL) {
706 return(NULL);
707 }
708
709 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
710 /* we need this for compatibility mode. */
711 my_addr = (struct user_iovec *) a_uio->uio_iovs.uiovp;
712 }
713 else {
714#if DEBUG
715 panic("uio_iovsaddr called for UIO_SYSSPACE request");
716#endif
717 my_addr = 0;
718 }
719 return(my_addr);
720}
721
722/*
723 * uio_reset - reset an uio_t.
724 * Reset the given uio_t to initial values. The uio_t is not fully initialized
725 * until all iovecs are added using uio_addiov calls.
726 * The a_iovcount value passed in the uio_create is the maximum number of
727 * iovecs you may add.
728 */
729void uio_reset( uio_t a_uio,
730 off_t a_offset, /* current offset */
731 int a_spacetype, /* type of address space */
732 int a_iodirection ) /* read or write flag */
733{
734 vm_size_t my_size;
735 int my_max_iovs;
736 u_int32_t my_old_flags;
737
738#if LP64_DEBUG
739 if (a_uio == NULL) {
740 panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__);
741 }
742 if (!IS_VALID_UIO_SEGFLG(a_spacetype)) {
743 panic("%s :%d - invalid address space type\n", __FILE__, __LINE__);
744 }
745 if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) {
746 panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__);
747 }
748#endif /* LP64_DEBUG */
749
750 if (a_uio == NULL) {
751 return;
752 }
753
754 my_size = a_uio->uio_size;
755 my_old_flags = a_uio->uio_flags;
756 my_max_iovs = a_uio->uio_max_iovs;
757 bzero(a_uio, my_size);
758 a_uio->uio_size = my_size;
759
760 /*
761 * we use uio_segflg to indicate if the uio_t is the new format or
762 * old (pre LP64 support) legacy format
763 * This switch statement should canonicalize incoming space type
764 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
765 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
766 */
767 switch (a_spacetype) {
768 case UIO_USERSPACE:
769 a_uio->uio_segflg = UIO_USERSPACE32;
770 break;
771 case UIO_SYSSPACE32:
772 a_uio->uio_segflg = UIO_SYSSPACE;
773 break;
774 case UIO_PHYS_USERSPACE:
775 a_uio->uio_segflg = UIO_PHYS_USERSPACE32;
776 break;
777 default:
778 a_uio->uio_segflg = a_spacetype;
779 break;
780 }
781
782 if (my_max_iovs > 0) {
783 a_uio->uio_iovs.uiovp = (struct user_iovec *)
784 (((uint8_t *)a_uio) + sizeof(struct uio));
785 }
786 else {
787 a_uio->uio_iovs.uiovp = NULL;
788 }
789
790 a_uio->uio_max_iovs = my_max_iovs;
791 a_uio->uio_offset = a_offset;
792 a_uio->uio_rw = a_iodirection;
793 a_uio->uio_flags = my_old_flags;
794
795 return;
796}
797
798/*
799 * uio_free - free a uio_t allocated via uio_init. this also frees all
800 * associated iovecs.
801 */
802void uio_free( uio_t a_uio )
803{
804#if DEBUG
805 if (a_uio == NULL) {
806 panic("%s :%d - passing NULL uio_t\n", __FILE__, __LINE__);
807 }
808#endif /* LP64_DEBUG */
809
810 if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) {
811#if DEBUG
812 if (hw_atomic_sub(&uio_t_count, 1) == UINT_MAX)
813 panic("%s :%d - uio_t_count underflow\n", __FILE__, __LINE__);
814#endif
815 kfree(a_uio, a_uio->uio_size);
816 }
817
818
819}
820
821/*
822 * uio_addiov - add an iovec to the given uio_t. You may call this up to
823 * the a_iovcount number that was passed to uio_create. This call will
824 * increment the residual IO count as iovecs are added to the uio_t.
825 * returns 0 if add was successful else non zero.
826 */
827int uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length )
828{
829 int i;
830
831 if (a_uio == NULL) {
832#if DEBUG
833 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
834#endif /* LP64_DEBUG */
835 return(-1);
836 }
837
838 if (UIO_IS_USER_SPACE(a_uio)) {
839 for ( i = 0; i < a_uio->uio_max_iovs; i++ ) {
840 if (a_uio->uio_iovs.uiovp[i].iov_len == 0 && a_uio->uio_iovs.uiovp[i].iov_base == 0) {
841 a_uio->uio_iovs.uiovp[i].iov_len = a_length;
842 a_uio->uio_iovs.uiovp[i].iov_base = a_baseaddr;
843 a_uio->uio_iovcnt++;
844 a_uio->uio_resid_64 += a_length;
845 return( 0 );
846 }
847 }
848 }
849 else {
850 for ( i = 0; i < a_uio->uio_max_iovs; i++ ) {
851 if (a_uio->uio_iovs.kiovp[i].iov_len == 0 && a_uio->uio_iovs.kiovp[i].iov_base == 0) {
852 a_uio->uio_iovs.kiovp[i].iov_len = (u_int64_t)a_length;
853 a_uio->uio_iovs.kiovp[i].iov_base = (u_int64_t)a_baseaddr;
854 a_uio->uio_iovcnt++;
855 a_uio->uio_resid_64 += a_length;
856 return( 0 );
857 }
858 }
859 }
860
861 return( -1 );
862}
863
864/*
865 * uio_getiov - get iovec data associated with the given uio_t. Use
866 * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)).
867 * a_baseaddr_p and a_length_p may be NULL.
868 * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t.
869 * returns 0 when data is returned.
870 */
871int uio_getiov( uio_t a_uio,
872 int a_index,
873 user_addr_t * a_baseaddr_p,
874 user_size_t * a_length_p )
875{
876 if (a_uio == NULL) {
877#if DEBUG
878 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
879#endif /* DEBUG */
880 return(-1);
881 }
882 if ( a_index < 0 || a_index >= a_uio->uio_iovcnt) {
883 return(-1);
884 }
885
886 if (UIO_IS_USER_SPACE(a_uio)) {
887 if (a_baseaddr_p != NULL) {
888 *a_baseaddr_p = a_uio->uio_iovs.uiovp[a_index].iov_base;
889 }
890 if (a_length_p != NULL) {
891 *a_length_p = a_uio->uio_iovs.uiovp[a_index].iov_len;
892 }
893 }
894 else {
895 if (a_baseaddr_p != NULL) {
896 *a_baseaddr_p = a_uio->uio_iovs.kiovp[a_index].iov_base;
897 }
898 if (a_length_p != NULL) {
899 *a_length_p = a_uio->uio_iovs.kiovp[a_index].iov_len;
900 }
901 }
902
903 return( 0 );
904}
905
906/*
907 * uio_calculateresid - runs through all iovecs associated with this
908 * uio_t and calculates (and sets) the residual IO count.
909 */
910__private_extern__ int uio_calculateresid( uio_t a_uio )
911{
912 int i;
913 u_int64_t resid = 0;
914
915 if (a_uio == NULL) {
916#if LP64_DEBUG
917 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
918#endif /* LP64_DEBUG */
919 return EINVAL;
920 }
921
922 a_uio->uio_iovcnt = a_uio->uio_max_iovs;
923 if (UIO_IS_USER_SPACE(a_uio)) {
924 a_uio->uio_resid_64 = 0;
925 for ( i = 0; i < a_uio->uio_max_iovs; i++ ) {
926 if (a_uio->uio_iovs.uiovp[i].iov_len != 0 && a_uio->uio_iovs.uiovp[i].iov_base != 0) {
927 if (a_uio->uio_iovs.uiovp[i].iov_len > LONG_MAX)
928 return EINVAL;
929 resid += a_uio->uio_iovs.uiovp[i].iov_len;
930 if (resid > LONG_MAX)
931 return EINVAL;
932 }
933 }
934 a_uio->uio_resid_64 = resid;
935
936 /* position to first non zero length iovec (4235922) */
937 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) {
938 a_uio->uio_iovcnt--;
939 if (a_uio->uio_iovcnt > 0) {
940 a_uio->uio_iovs.uiovp++;
941 }
942 }
943 }
944 else {
945 a_uio->uio_resid_64 = 0;
946 for ( i = 0; i < a_uio->uio_max_iovs; i++ ) {
947 if (a_uio->uio_iovs.kiovp[i].iov_len != 0 && a_uio->uio_iovs.kiovp[i].iov_base != 0) {
948 if (a_uio->uio_iovs.kiovp[i].iov_len > LONG_MAX)
949 return EINVAL;
950 resid += a_uio->uio_iovs.kiovp[i].iov_len;
951 if (resid > LONG_MAX)
952 return EINVAL;
953 }
954 }
955 a_uio->uio_resid_64 = resid;
956
957 /* position to first non zero length iovec (4235922) */
958 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) {
959 a_uio->uio_iovcnt--;
960 if (a_uio->uio_iovcnt > 0) {
961 a_uio->uio_iovs.kiovp++;
962 }
963 }
964 }
965
966 return 0;
967}
968
969/*
970 * uio_update - update the given uio_t for a_count of completed IO.
971 * This call decrements the current iovec length and residual IO value
972 * and increments the current iovec base address and offset value.
973 * If the current iovec length is 0 then advance to the next
974 * iovec (if any).
975 * If the a_count passed in is 0, than only do the advancement
976 * over any 0 length iovec's.
977 */
978void uio_update( uio_t a_uio, user_size_t a_count )
979{
980#if LP64_DEBUG
981 if (a_uio == NULL) {
982 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
983 }
984 if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) {
985 panic("%s :%d - invalid count value \n", __FILE__, __LINE__);
986 }
987#endif /* LP64_DEBUG */
988
989 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
990 return;
991 }
992
993 if (UIO_IS_USER_SPACE(a_uio)) {
994 /*
995 * if a_count == 0, then we are asking to skip over
996 * any empty iovs
997 */
998 if (a_count) {
999 if (a_count > a_uio->uio_iovs.uiovp->iov_len) {
1000 a_uio->uio_iovs.uiovp->iov_base += a_uio->uio_iovs.uiovp->iov_len;
1001 a_uio->uio_iovs.uiovp->iov_len = 0;
1002 }
1003 else {
1004 a_uio->uio_iovs.uiovp->iov_base += a_count;
1005 a_uio->uio_iovs.uiovp->iov_len -= a_count;
1006 }
1007 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1008 a_uio->uio_offset += a_uio->uio_resid_64;
1009 a_uio->uio_resid_64 = 0;
1010 }
1011 else {
1012 a_uio->uio_offset += a_count;
1013 a_uio->uio_resid_64 -= a_count;
1014 }
1015 }
1016 /*
1017 * advance to next iovec if current one is totally consumed
1018 */
1019 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) {
1020 a_uio->uio_iovcnt--;
1021 if (a_uio->uio_iovcnt > 0) {
1022 a_uio->uio_iovs.uiovp++;
1023 }
1024 }
1025 }
1026 else {
1027 /*
1028 * if a_count == 0, then we are asking to skip over
1029 * any empty iovs
1030 */
1031 if (a_count) {
1032 if (a_count > a_uio->uio_iovs.kiovp->iov_len) {
1033 a_uio->uio_iovs.kiovp->iov_base += a_uio->uio_iovs.kiovp->iov_len;
1034 a_uio->uio_iovs.kiovp->iov_len = 0;
1035 }
1036 else {
1037 a_uio->uio_iovs.kiovp->iov_base += a_count;
1038 a_uio->uio_iovs.kiovp->iov_len -= a_count;
1039 }
1040 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1041 a_uio->uio_offset += a_uio->uio_resid_64;
1042 a_uio->uio_resid_64 = 0;
1043 }
1044 else {
1045 a_uio->uio_offset += a_count;
1046 a_uio->uio_resid_64 -= a_count;
1047 }
1048 }
1049 /*
1050 * advance to next iovec if current one is totally consumed
1051 */
1052 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) {
1053 a_uio->uio_iovcnt--;
1054 if (a_uio->uio_iovcnt > 0) {
1055 a_uio->uio_iovs.kiovp++;
1056 }
1057 }
1058 }
1059 return;
1060}
1061
1062/*
1063 * uio_pushback - undo uncommitted I/O by subtracting from the
1064 * current base address and offset, and incrementing the residiual
1065 * IO. If the UIO was previously exhausted, this call will panic.
1066 * New code should not use this functionality.
1067 */
1068__private_extern__ void uio_pushback( uio_t a_uio, user_size_t a_count )
1069{
1070#if LP64_DEBUG
1071 if (a_uio == NULL) {
1072 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
1073 }
1074 if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) {
1075 panic("%s :%d - invalid count value \n", __FILE__, __LINE__);
1076 }
1077#endif /* LP64_DEBUG */
1078
1079 if (a_uio == NULL || a_count == 0) {
1080 return;
1081 }
1082
1083 if (a_uio->uio_iovcnt < 1) {
1084 panic("Invalid uio for pushback");
1085 }
1086
1087 if (UIO_IS_USER_SPACE(a_uio)) {
1088 a_uio->uio_iovs.uiovp->iov_base -= a_count;
1089 a_uio->uio_iovs.uiovp->iov_len += a_count;
1090 }
1091 else {
1092 a_uio->uio_iovs.kiovp->iov_base -= a_count;
1093 a_uio->uio_iovs.kiovp->iov_len += a_count;
1094 }
1095
1096 a_uio->uio_offset -= a_count;
1097 a_uio->uio_resid_64 += a_count;
1098
1099 return;
1100}
1101
1102
1103/*
1104 * uio_duplicate - allocate a new uio and make a copy of the given uio_t.
1105 * may return NULL.
1106 */
1107uio_t uio_duplicate( uio_t a_uio )
1108{
1109 uio_t my_uio;
1110 int i;
1111
1112 if (a_uio == NULL) {
1113 return(NULL);
1114 }
1115
1116 my_uio = (uio_t) kalloc(a_uio->uio_size);
1117 if (my_uio == 0) {
1118 panic("%s :%d - allocation failed\n", __FILE__, __LINE__);
1119 }
1120
1121 bcopy((void *)a_uio, (void *)my_uio, a_uio->uio_size);
1122 /* need to set our iovec pointer to point to first active iovec */
1123 if (my_uio->uio_max_iovs > 0) {
1124 my_uio->uio_iovs.uiovp = (struct user_iovec *)
1125 (((uint8_t *)my_uio) + sizeof(struct uio));
1126
1127 /* advance to first nonzero iovec */
1128 if (my_uio->uio_iovcnt > 0) {
1129 for ( i = 0; i < my_uio->uio_max_iovs; i++ ) {
1130 if (UIO_IS_USER_SPACE(a_uio)) {
1131 if (my_uio->uio_iovs.uiovp->iov_len != 0) {
1132 break;
1133 }
1134 my_uio->uio_iovs.uiovp++;
1135 }
1136 else {
1137 if (my_uio->uio_iovs.kiovp->iov_len != 0) {
1138 break;
1139 }
1140 my_uio->uio_iovs.kiovp++;
1141 }
1142 }
1143 }
1144 }
1145
1146 my_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED;
1147#if DEBUG
1148 (void)hw_atomic_add(&uio_t_count, 1);
1149#endif
1150
1151
1152 return(my_uio);
1153}
1154
1155int copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst)
1156{
1157 size_t size_of_iovec = ( spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec));
1158 int error;
1159 int i;
1160
1161 // copyin to the front of "dst", without regard for putting records in the right places
1162 error = copyin(uaddr, dst, count * size_of_iovec);
1163 if (error)
1164 return (error);
1165
1166 // now, unpack the entries in reverse order, so we don't overwrite anything
1167 for (i = count - 1; i >= 0; i--) {
1168 if (spacetype == UIO_USERSPACE64) {
1169 struct user64_iovec iovec = ((struct user64_iovec *)dst)[i];
1170 dst[i].iov_base = iovec.iov_base;
1171 dst[i].iov_len = iovec.iov_len;
1172 } else {
1173 struct user32_iovec iovec = ((struct user32_iovec *)dst)[i];
1174 dst[i].iov_base = iovec.iov_base;
1175 dst[i].iov_len = iovec.iov_len;
1176 }
1177 }
1178
1179 return (0);
1180}
1181