1/*
2 * Copyright (c) 2009-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <mach_assert.h>
29
30#include <sys/errno.h>
31#include <i386/param.h>
32#include <i386/misc_protos.h>
33#include <i386/cpu_data.h>
34#include <i386/machine_routines.h>
35#include <i386/cpuid.h>
36#include <i386/vmx.h>
37#include <vm/pmap.h>
38#include <vm/vm_map.h>
39#include <vm/vm_kern.h>
40#include <vm/vm_fault.h>
41#include <san/kasan.h>
42
43#include <sys/kdebug.h>
44
45#include <kern/copyout_shim.h>
46
47#undef copyin
48#undef copyout
49
50static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int);
51static int copyio_phys(addr64_t, addr64_t, vm_size_t, int);
52
53/*
54 * Copy sizes bigger than this value will cause a kernel panic.
55 *
56 * Yes, this is an arbitrary fixed limit, but it's almost certainly
57 * a programming error to be copying more than this amount between
58 * user and wired kernel memory in a single invocation on this
59 * platform.
60 */
61const int copysize_limit_panic = (64 * MB);
62
63/*
64 * The copy engine has the following characteristics
65 * - copyio() handles copies to/from user or kernel space
66 * - copypv() deals with physical or virtual addresses
67 *
68 * Readers familiar with the 32-bit kernel will expect Joe's thesis at this
69 * point describing the full glory of the copy window implementation. In K64,
70 * however, there is no need for windowing. Thanks to the vast shared address
71 * space, the kernel has direct access to userspace and to physical memory.
72 *
73 * User virtual addresses are accessible provided the user's cr3 is loaded.
74 * Physical addresses are accessible via the direct map and the PHYSMAP_PTOV()
75 * translation.
76 *
77 * Copyin/out variants all boil done to just these 2 routines in locore.s which
78 * provide fault-recoverable copying:
79 */
80extern int _bcopy(const void *, void *, vm_size_t);
81extern int _bcopystr(const void *, void *, vm_size_t, vm_size_t *);
82extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len);
83
84/* On by default, optionally disabled by boot-arg */
85extern boolean_t copyio_zalloc_check;
86
87/*
88 * Types of copies:
89 */
90#define COPYIN 0 /* from user virtual to kernel virtual */
91#define COPYOUT 1 /* from kernel virtual to user virtual */
92#define COPYINSTR 2 /* string variant of copyout */
93#define COPYINPHYS 3 /* from user virtual to kernel physical */
94#define COPYOUTPHYS 4 /* from kernel physical to user virtual */
95#define COPYINWORD 5 /* from user virtual to kernel virtual */
96
97#if ENABLE_SMAPLOG
98typedef struct {
99 uint64_t timestamp;
100 thread_t thread;
101 uintptr_t cr4;
102 uint8_t cpuid;
103 uint8_t smap_state;
104 uint8_t copyio_active;
105} smaplog_entry_t;
106
107#define SMAPLOG_BUFFER_SIZE (50)
108static smaplog_entry_t smaplog_cbuf[SMAPLOG_BUFFER_SIZE];
109static uint32_t smaplog_head = 0;
110
111static void
112smaplog_add_entry(boolean_t enabling)
113{
114 uint32_t index = 0;
115 thread_t thread = current_thread();
116
117 do {
118 index = smaplog_head;
119 } while (!OSCompareAndSwap(index, (index + 1) % SMAPLOG_BUFFER_SIZE, &smaplog_head));
120
121 assert(index < SMAPLOG_BUFFER_SIZE);
122 assert(smaplog_head < SMAPLOG_BUFFER_SIZE);
123 assert(thread);
124
125 smaplog_cbuf[index].timestamp = mach_absolute_time();
126 smaplog_cbuf[index].thread = thread;
127 smaplog_cbuf[index].cpuid = cpu_number();
128 smaplog_cbuf[index].cr4 = get_cr4();
129 smaplog_cbuf[index].smap_state = enabling;
130 smaplog_cbuf[index].copyio_active = (thread->machine.specFlags & CopyIOActive) ? 1 : 0;
131}
132#endif /* ENABLE_SMAPLOG */
133
134extern boolean_t pmap_smap_enabled;
135static inline void user_access_enable(void) {
136 if (pmap_smap_enabled) {
137 stac();
138#if ENABLE_SMAPLOG
139 smaplog_add_entry(TRUE);
140#endif
141 }
142}
143static inline void user_access_disable(void) {
144 if (pmap_smap_enabled) {
145 clac();
146#if ENABLE_SMAPLOG
147 smaplog_add_entry(FALSE);
148#endif
149 }
150}
151
152#if COPYIO_TRACE_ENABLED
153#define COPYIO_TRACE(x, a, b, c, d, e) KERNEL_DEBUG_CONSTANT(x, a, b, c, d, e)
154#else
155#define COPYIO_TRACE(x, a, b, c, d, e) do { } while(0)
156#endif
157
158static int
159copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
160 vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
161{
162 thread_t thread = current_thread();
163 pmap_t pmap;
164 vm_size_t bytes_copied;
165 int error = 0;
166 boolean_t istate = FALSE;
167 boolean_t recursive_CopyIOActive;
168#if COPYIO_TRACE_ENABLED
169 int debug_type = 0xeff70010;
170 debug_type += (copy_type << 2);
171#endif
172 vm_size_t kernel_buf_size = 0;
173
174 if (__improbable(nbytes > copysize_limit_panic))
175 panic("%s(%p, %p, %lu) - transfer too large", __func__,
176 (void *)user_addr, (void *)kernel_addr, nbytes);
177
178 COPYIO_TRACE(debug_type | DBG_FUNC_START,
179 user_addr, kernel_addr, nbytes, use_kernel_map, 0);
180
181 if (__improbable(nbytes == 0))
182 goto out;
183
184 pmap = thread->map->pmap;
185 boolean_t nopagezero = thread->map->pmap->pagezero_accessible;
186
187 if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS)) {
188 if (__improbable((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS))
189 panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
190 if (__probable(copyio_zalloc_check)) {
191 kernel_buf_size = zone_element_size(kernel_addr, NULL);
192 if (__improbable(kernel_buf_size && kernel_buf_size < nbytes))
193 panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
194 }
195 }
196
197 /* Sanity and security check for addresses to/from a user */
198
199 if (__improbable(((pmap != kernel_pmap) && (use_kernel_map == 0)) &&
200 ((nbytes && (user_addr+nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map))))) {
201 error = EFAULT;
202 goto out;
203 }
204
205#if KASAN
206 if (copy_type == COPYIN || copy_type == COPYINSTR || copy_type == COPYINWORD) {
207 __asan_storeN((uptr)kernel_addr, nbytes);
208 } else if (copy_type == COPYOUT) {
209 __asan_loadN((uptr)kernel_addr, nbytes);
210 }
211#endif
212
213 /*
214 * If the no_shared_cr3 boot-arg is set (true), the kernel runs on
215 * its own pmap and cr3 rather than the user's -- so that wild accesses
216 * from kernel or kexts can be trapped. So, during copyin and copyout,
217 * we need to switch back to the user's map/cr3. The thread is flagged
218 * "CopyIOActive" at this time so that if the thread is pre-empted,
219 * we will later restore the correct cr3.
220 */
221 recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive;
222
223 boolean_t pdswitch = no_shared_cr3 || nopagezero;
224
225 if (__improbable(pdswitch)) {
226 istate = ml_set_interrupts_enabled(FALSE);
227 if (nopagezero && pmap_pcid_ncpus) {
228 pmap_pcid_activate(pmap, cpu_number(), TRUE, TRUE);
229 } else if (get_cr3_base() != pmap->pm_cr3) {
230 set_cr3_raw(pmap->pm_cr3);
231 }
232 thread->machine.specFlags |= CopyIOActive;
233 } else {
234 thread->machine.specFlags |= CopyIOActive;
235 }
236
237 user_access_enable();
238
239#if DEVELOPMENT || DEBUG
240 /*
241 * Ensure that we're running on the target thread's cr3.
242 */
243 if ((pmap != kernel_pmap) && !use_kernel_map &&
244 (get_cr3_base() != pmap->pm_cr3)) {
245 panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p",
246 copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map,
247 (void *) get_cr3_raw(), (void *) pmap->pm_cr3);
248 }
249#endif
250
251 if (__improbable(pdswitch)) {
252 (void) ml_set_interrupts_enabled(istate);
253 }
254
255 COPYIO_TRACE(0xeff70044 | DBG_FUNC_NONE, user_addr,
256 kernel_addr, nbytes, 0, 0);
257
258 switch (copy_type) {
259
260 case COPYIN:
261 error = _bcopy((const void *) user_addr,
262 kernel_addr,
263 nbytes);
264 break;
265
266 case COPYOUT:
267 error = _bcopy(kernel_addr,
268 (void *) user_addr,
269 nbytes);
270 break;
271
272 case COPYINPHYS:
273 error = _bcopy((const void *) user_addr,
274 PHYSMAP_PTOV(kernel_addr),
275 nbytes);
276 break;
277
278 case COPYOUTPHYS:
279 error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr),
280 (void *) user_addr,
281 nbytes);
282 break;
283
284 case COPYINWORD:
285 error = _copyin_word((const void *) user_addr,
286 (void *) kernel_addr,
287 nbytes);
288 break;
289
290 case COPYINSTR:
291 error = _bcopystr((const void *) user_addr,
292 kernel_addr,
293 (int) nbytes,
294 &bytes_copied);
295
296 /*
297 * lencopied should be updated on success
298 * or ENAMETOOLONG... but not EFAULT
299 */
300 if (error != EFAULT)
301 *lencopied = bytes_copied;
302
303 if (error) {
304#if KDEBUG
305 nbytes = *lencopied;
306#endif
307 break;
308 }
309 if (*(kernel_addr + bytes_copied - 1) == 0) {
310 /*
311 * we found a NULL terminator... we're done
312 */
313#if KDEBUG
314 nbytes = *lencopied;
315#endif
316 break;
317 } else {
318 /*
319 * no more room in the buffer and we haven't
320 * yet come across a NULL terminator
321 */
322#if KDEBUG
323 nbytes = *lencopied;
324#endif
325 error = ENAMETOOLONG;
326 break;
327 }
328 }
329
330 user_access_disable();
331
332 if (__improbable(pdswitch)) {
333 istate = ml_set_interrupts_enabled(FALSE);
334 if (!recursive_CopyIOActive && (get_cr3_raw() != kernel_pmap->pm_cr3)) {
335 if (nopagezero && pmap_pcid_ncpus) {
336 pmap_pcid_activate(pmap, cpu_number(), TRUE, FALSE);
337 } else {
338 set_cr3_raw(kernel_pmap->pm_cr3);
339 }
340 }
341
342 if (!recursive_CopyIOActive) {
343 thread->machine.specFlags &= ~CopyIOActive;
344 }
345 (void) ml_set_interrupts_enabled(istate);
346 } else if (!recursive_CopyIOActive) {
347 thread->machine.specFlags &= ~CopyIOActive;
348 }
349
350out:
351 COPYIO_TRACE(debug_type | DBG_FUNC_END, user_addr, kernel_addr, nbytes, error, 0);
352
353 return (error);
354}
355
356
357static int
358copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
359{
360 char *paddr;
361 user_addr_t vaddr;
362 int ctype;
363
364 if (which & cppvPsnk) {
365 paddr = (char *)sink;
366 vaddr = (user_addr_t)source;
367 ctype = COPYINPHYS;
368 } else {
369 paddr = (char *)source;
370 vaddr = (user_addr_t)sink;
371 ctype = COPYOUTPHYS;
372 CALL_COPYOUT_SHIM_PHYS((void *)PHYSMAP_PTOV(source),sink,csize)
373 }
374 return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap);
375}
376
377int
378copyinmsg(const user_addr_t user_addr, char *kernel_addr, mach_msg_size_t nbytes)
379{
380 return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
381}
382
383int
384copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
385{
386 return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
387}
388
389/*
390 * copyin_word
391 * Read an aligned value from userspace as a single memory transaction.
392 * This function supports userspace synchronization features
393 */
394int
395copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes)
396{
397 /* Verify sizes */
398 if ((nbytes != 4) && (nbytes != 8))
399 return EINVAL;
400
401 /* Test alignment */
402 if (user_addr & (nbytes - 1))
403 return EINVAL;
404 return copyio(COPYINWORD, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
405}
406
407int
408copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
409{
410 *lencopied = 0;
411
412 return copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0);
413}
414
415int
416copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes)
417{
418 CALL_COPYOUT_SHIM_MSG(kernel_addr,user_addr,(vm_size_t)nbytes)
419 return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
420}
421
422int
423copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
424{
425 CALL_COPYOUT_SHIM_NRML(kernel_addr,user_addr,nbytes)
426 return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
427}
428
429
430kern_return_t
431copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
432{
433 unsigned int lop, csize;
434 int bothphys = 0;
435
436 KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64,
437 (unsigned)snk64, size, which, 0);
438
439 if ((which & (cppvPsrc | cppvPsnk)) == 0 ) /* Make sure that only one is virtual */
440 panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */
441
442 if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk))
443 bothphys = 1; /* both are physical */
444
445 while (size) {
446
447 if (bothphys) {
448 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1))); /* Assume sink smallest */
449
450 if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))))
451 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))); /* No, source is smaller */
452 } else {
453 /*
454 * only need to compute the resid for the physical page
455 * address... we don't care about where we start/finish in
456 * the virtual since we just call the normal copyin/copyout
457 */
458 if (which & cppvPsrc)
459 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));
460 else
461 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));
462 }
463 csize = size; /* Assume we can copy it all */
464 if (lop < size)
465 csize = lop; /* Nope, we can't do it all */
466#if 0
467 /*
468 * flush_dcache64 is currently a nop on the i386...
469 * it's used when copying to non-system memory such
470 * as video capture cards... on PPC there was a need
471 * to flush due to how we mapped this memory... not
472 * sure if it's needed on i386.
473 */
474 if (which & cppvFsrc)
475 flush_dcache64(src64, csize, 1); /* If requested, flush source before move */
476 if (which & cppvFsnk)
477 flush_dcache64(snk64, csize, 1); /* If requested, flush sink before move */
478#endif
479 if (bothphys)
480 bcopy_phys(src64, snk64, csize); /* Do a physical copy, virtually */
481 else {
482 if (copyio_phys(src64, snk64, csize, which))
483 return (KERN_FAILURE);
484 }
485#if 0
486 if (which & cppvFsrc)
487 flush_dcache64(src64, csize, 1); /* If requested, flush source after move */
488 if (which & cppvFsnk)
489 flush_dcache64(snk64, csize, 1); /* If requested, flush sink after move */
490#endif
491 size -= csize; /* Calculate what is left */
492 snk64 += csize; /* Bump sink to next physical address */
493 src64 += csize; /* Bump source to next physical address */
494 }
495 KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64,
496 (unsigned)snk64, size, which, 0);
497
498 return KERN_SUCCESS;
499}
500