1/*
2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#define _IOMEMORYDESCRIPTOR_INTERNAL_
30
31#include <IOKit/assert.h>
32#include <IOKit/system.h>
33
34#include <IOKit/IOLib.h>
35#include <IOKit/IOMapper.h>
36#include <IOKit/IOBufferMemoryDescriptor.h>
37#include <libkern/OSDebug.h>
38#include <mach/mach_vm.h>
39
40#include "IOKitKernelInternal.h"
41
42#ifdef IOALLOCDEBUG
43#include <libkern/c++/OSCPPDebug.h>
44#endif
45#include <IOKit/IOStatisticsPrivate.h>
46
47#if IOKITSTATS
48#define IOStatisticsAlloc(type, size) \
49do { \
50 IOStatistics::countAlloc(type, size); \
51} while (0)
52#else
53#define IOStatisticsAlloc(type, size)
54#endif /* IOKITSTATS */
55
56
57__BEGIN_DECLS
58void ipc_port_release_send(ipc_port_t port);
59#include <vm/pmap.h>
60
61__END_DECLS
62
63/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
64
65enum
66{
67 kInternalFlagPhysical = 0x00000001,
68 kInternalFlagPageSized = 0x00000002,
69 kInternalFlagPageAllocated = 0x00000004,
70 kInternalFlagInit = 0x00000008
71};
72
73/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
74
75#define super IOGeneralMemoryDescriptor
76OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
77 IOGeneralMemoryDescriptor);
78
79/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80
81static uintptr_t IOBMDPageProc(iopa_t * a)
82{
83 kern_return_t kr;
84 vm_address_t vmaddr = 0;
85 int options = 0; // KMA_LOMEM;
86
87 kr = kernel_memory_allocate(kernel_map, &vmaddr,
88 page_size, 0, options, VM_KERN_MEMORY_IOKIT);
89
90 if (KERN_SUCCESS != kr) vmaddr = 0;
91 else bzero((void *) vmaddr, page_size);
92
93 return ((uintptr_t) vmaddr);
94}
95
96/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
97
98#ifndef __LP64__
99bool IOBufferMemoryDescriptor::initWithOptions(
100 IOOptionBits options,
101 vm_size_t capacity,
102 vm_offset_t alignment,
103 task_t inTask)
104{
105 mach_vm_address_t physicalMask = 0;
106 return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
107}
108#endif /* !__LP64__ */
109
110bool IOBufferMemoryDescriptor::initWithPhysicalMask(
111 task_t inTask,
112 IOOptionBits options,
113 mach_vm_size_t capacity,
114 mach_vm_address_t alignment,
115 mach_vm_address_t physicalMask)
116{
117 task_t mapTask = NULL;
118 vm_map_t vmmap = NULL;
119 mach_vm_address_t highestMask = 0;
120 IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
121 IODMAMapSpecification mapSpec;
122 bool mapped = false;
123 bool needZero;
124
125 if (!capacity) return false;
126
127 _options = options;
128 _capacity = capacity;
129 _internalFlags = 0;
130 _internalReserved = 0;
131 _buffer = 0;
132
133 _ranges.v64 = IONew(IOAddressRange, 1);
134 if (!_ranges.v64)
135 return (false);
136 _ranges.v64->address = 0;
137 _ranges.v64->length = 0;
138 // make sure super::free doesn't dealloc _ranges before super::init
139 _flags = kIOMemoryAsReference;
140
141 // Grab IOMD bits from the Buffer MD options
142 iomdOptions |= (options & kIOBufferDescriptorMemoryFlags);
143
144 if (!(kIOMemoryMapperNone & options))
145 {
146 IOMapper::checkForSystemMapper();
147 mapped = (0 != IOMapper::gSystem);
148 }
149 needZero = (mapped || (0 != (kIOMemorySharingTypeMask & options)));
150
151 if (physicalMask && (alignment <= 1))
152 {
153 alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
154 highestMask = (physicalMask | alignment);
155 alignment++;
156 if (alignment < page_size)
157 alignment = page_size;
158 }
159
160 if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size))
161 alignment = page_size;
162
163 if (alignment >= page_size)
164 capacity = round_page(capacity);
165
166 if (alignment > page_size)
167 options |= kIOMemoryPhysicallyContiguous;
168
169 _alignment = alignment;
170
171 if ((capacity + alignment) < _capacity) return (false);
172
173 if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
174 return false;
175
176 bzero(&mapSpec, sizeof(mapSpec));
177 mapSpec.alignment = _alignment;
178 mapSpec.numAddressBits = 64;
179 if (highestMask && mapped)
180 {
181 if (highestMask <= 0xFFFFFFFF)
182 mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask));
183 else
184 mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32)));
185 highestMask = 0;
186 }
187
188 // set memory entry cache mode, pageable, purgeable
189 iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
190 if (options & kIOMemoryPageable)
191 {
192 iomdOptions |= kIOMemoryBufferPageable;
193 if (options & kIOMemoryPurgeable) iomdOptions |= kIOMemoryBufferPurgeable;
194 }
195 else
196 {
197 vmmap = kernel_map;
198
199 // Buffer shouldn't auto prepare they should be prepared explicitly
200 // But it never was enforced so what are you going to do?
201 iomdOptions |= kIOMemoryAutoPrepare;
202
203 /* Allocate a wired-down buffer inside kernel space. */
204
205 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));
206
207 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous)))
208 {
209 contig |= (!mapped);
210 contig |= (0 != (kIOMemoryMapperNone & options));
211#if 0
212 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
213 contig |= true;
214#endif
215 }
216
217 if (contig || highestMask || (alignment > page_size))
218 {
219 _internalFlags |= kInternalFlagPhysical;
220 if (highestMask)
221 {
222 _internalFlags |= kInternalFlagPageSized;
223 capacity = round_page(capacity);
224 }
225 _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(
226 capacity, highestMask, alignment, contig);
227 }
228 else if (needZero
229 && ((capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)))
230 {
231 _internalFlags |= kInternalFlagPageAllocated;
232 needZero = false;
233 _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment);
234 if (_buffer)
235 {
236 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
237#if IOALLOCDEBUG
238 OSAddAtomic(capacity, &debug_iomalloc_size);
239#endif
240 }
241 }
242 else if (alignment > 1)
243 {
244 _buffer = IOMallocAligned(capacity, alignment);
245 }
246 else
247 {
248 _buffer = IOMalloc(capacity);
249 }
250 if (!_buffer)
251 {
252 return false;
253 }
254 if (needZero) bzero(_buffer, capacity);
255 }
256
257 if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {
258 vm_size_t size = round_page(capacity);
259
260 // initWithOptions will create memory entry
261 iomdOptions |= kIOMemoryPersistent;
262
263 if( options & kIOMemoryPageable) {
264#if IOALLOCDEBUG
265 OSAddAtomicLong(size, &debug_iomallocpageable_size);
266#endif
267 mapTask = inTask;
268 if (NULL == inTask)
269 inTask = kernel_task;
270 }
271 else if (options & kIOMapCacheMask)
272 {
273 // Prefetch each page to put entries into the pmap
274 volatile UInt8 * startAddr = (UInt8 *)_buffer;
275 volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity;
276
277 while (startAddr < endAddr)
278 {
279 UInt8 dummyVar = *startAddr;
280 (void) dummyVar;
281 startAddr += page_size;
282 }
283 }
284 }
285
286 _ranges.v64->address = (mach_vm_address_t) _buffer;;
287 _ranges.v64->length = _capacity;
288
289 if (!super::initWithOptions(_ranges.v64, 1, 0,
290 inTask, iomdOptions, /* System mapper */ 0))
291 return false;
292
293 _internalFlags |= kInternalFlagInit;
294#if IOTRACKING
295 if (!(options & kIOMemoryPageable)) trackingAccumSize(capacity);
296#endif /* IOTRACKING */
297
298 // give any system mapper the allocation params
299 if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec,
300 &mapSpec, sizeof(mapSpec)))
301 return false;
302
303 if (mapTask)
304 {
305 if (!reserved) {
306 reserved = IONew( ExpansionData, 1 );
307 if( !reserved)
308 return( false );
309 }
310 reserved->map = createMappingInTask(mapTask, 0,
311 kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0);
312 if (!reserved->map)
313 {
314 _buffer = 0;
315 return( false );
316 }
317 release(); // map took a retain on this
318 reserved->map->retain();
319 removeMapping(reserved->map);
320 mach_vm_address_t buffer = reserved->map->getAddress();
321 _buffer = (void *) buffer;
322 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions))
323 _ranges.v64->address = buffer;
324 }
325
326 setLength(_capacity);
327
328 return true;
329}
330
331IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
332 task_t inTask,
333 IOOptionBits options,
334 vm_size_t capacity,
335 vm_offset_t alignment)
336{
337 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
338
339 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
340 me->release();
341 me = 0;
342 }
343 return me;
344}
345
346IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
347 task_t inTask,
348 IOOptionBits options,
349 mach_vm_size_t capacity,
350 mach_vm_address_t physicalMask)
351{
352 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
353
354 if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
355 {
356 me->release();
357 me = 0;
358 }
359 return me;
360}
361
362#ifndef __LP64__
363bool IOBufferMemoryDescriptor::initWithOptions(
364 IOOptionBits options,
365 vm_size_t capacity,
366 vm_offset_t alignment)
367{
368 return (initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0));
369}
370#endif /* !__LP64__ */
371
372IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
373 IOOptionBits options,
374 vm_size_t capacity,
375 vm_offset_t alignment)
376{
377 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
378
379 if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
380 me->release();
381 me = 0;
382 }
383 return me;
384}
385
386
387/*
388 * withCapacity:
389 *
390 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
391 * hold capacity bytes. The descriptor's length is initially set to the capacity.
392 */
393IOBufferMemoryDescriptor *
394IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity,
395 IODirection inDirection,
396 bool inContiguous)
397{
398 return( IOBufferMemoryDescriptor::withOptions(
399 inDirection | kIOMemoryUnshared
400 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
401 inCapacity, inContiguous ? inCapacity : 1 ));
402}
403
404#ifndef __LP64__
405/*
406 * initWithBytes:
407 *
408 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
409 * The descriptor's length and capacity are set to the input buffer's size.
410 */
411bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
412 vm_size_t inLength,
413 IODirection inDirection,
414 bool inContiguous)
415{
416 if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
417 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
418 inLength, inLength, (mach_vm_address_t)0))
419 return false;
420
421 // start out with no data
422 setLength(0);
423
424 if (!appendBytes(inBytes, inLength))
425 return false;
426
427 return true;
428}
429#endif /* !__LP64__ */
430
431/*
432 * withBytes:
433 *
434 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
435 * The descriptor's length and capacity are set to the input buffer's size.
436 */
437IOBufferMemoryDescriptor *
438IOBufferMemoryDescriptor::withBytes(const void * inBytes,
439 vm_size_t inLength,
440 IODirection inDirection,
441 bool inContiguous)
442{
443 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
444
445 if (me && !me->initWithPhysicalMask(
446 kernel_task, inDirection | kIOMemoryUnshared
447 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
448 inLength, inLength, 0 ))
449 {
450 me->release();
451 me = 0;
452 }
453
454 if (me)
455 {
456 // start out with no data
457 me->setLength(0);
458
459 if (!me->appendBytes(inBytes, inLength))
460 {
461 me->release();
462 me = 0;
463 }
464 }
465 return me;
466}
467
468/*
469 * free:
470 *
471 * Free resources
472 */
473void IOBufferMemoryDescriptor::free()
474{
475 // Cache all of the relevant information on the stack for use
476 // after we call super::free()!
477 IOOptionBits flags = _flags;
478 IOOptionBits internalFlags = _internalFlags;
479 IOOptionBits options = _options;
480 vm_size_t size = _capacity;
481 void * buffer = _buffer;
482 IOMemoryMap * map = 0;
483 IOAddressRange * range = _ranges.v64;
484 vm_offset_t alignment = _alignment;
485
486 if (alignment >= page_size)
487 size = round_page(size);
488
489 if (reserved)
490 {
491 map = reserved->map;
492 IODelete( reserved, ExpansionData, 1 );
493 if (map)
494 map->release();
495 }
496
497 if ((options & kIOMemoryPageable)
498 || (kInternalFlagPageSized & internalFlags)) size = round_page(size);
499
500#if IOTRACKING
501 if (!(options & kIOMemoryPageable)
502 && buffer
503 && (kInternalFlagInit & _internalFlags)) trackingAccumSize(-size);
504#endif /* IOTRACKING */
505
506 /* super::free may unwire - deallocate buffer afterwards */
507 super::free();
508
509 if (options & kIOMemoryPageable)
510 {
511#if IOALLOCDEBUG
512 OSAddAtomicLong(-size, &debug_iomallocpageable_size);
513#endif
514 }
515 else if (buffer)
516 {
517 if (kInternalFlagPhysical & internalFlags)
518 {
519 IOKernelFreePhysical((mach_vm_address_t) buffer, size);
520 }
521 else if (kInternalFlagPageAllocated & internalFlags)
522 {
523 uintptr_t page;
524 page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
525 if (page)
526 {
527 kmem_free(kernel_map, page, page_size);
528 }
529#if IOALLOCDEBUG
530 OSAddAtomic(-size, &debug_iomalloc_size);
531#endif
532 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
533 }
534 else if (alignment > 1)
535 {
536 IOFreeAligned(buffer, size);
537 }
538 else
539 {
540 IOFree(buffer, size);
541 }
542 }
543 if (range && (kIOMemoryAsReference & flags))
544 IODelete(range, IOAddressRange, 1);
545}
546
547/*
548 * getCapacity:
549 *
550 * Get the buffer capacity
551 */
552vm_size_t IOBufferMemoryDescriptor::getCapacity() const
553{
554 return _capacity;
555}
556
557/*
558 * setLength:
559 *
560 * Change the buffer length of the memory descriptor. When a new buffer
561 * is created, the initial length of the buffer is set to be the same as
562 * the capacity. The length can be adjusted via setLength for a shorter
563 * transfer (there is no need to create more buffer descriptors when you
564 * can reuse an existing one, even for different transfer sizes). Note
565 * that the specified length must not exceed the capacity of the buffer.
566 */
567void IOBufferMemoryDescriptor::setLength(vm_size_t length)
568{
569 assert(length <= _capacity);
570 if (length > _capacity) return;
571
572 _length = length;
573 _ranges.v64->length = length;
574}
575
576/*
577 * setDirection:
578 *
579 * Change the direction of the transfer. This method allows one to redirect
580 * the descriptor's transfer direction. This eliminates the need to destroy
581 * and create new buffers when different transfer directions are needed.
582 */
583void IOBufferMemoryDescriptor::setDirection(IODirection direction)
584{
585 _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
586#ifndef __LP64__
587 _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
588#endif /* !__LP64__ */
589}
590
591/*
592 * appendBytes:
593 *
594 * Add some data to the end of the buffer. This method automatically
595 * maintains the memory descriptor buffer length. Note that appendBytes
596 * will not copy past the end of the memory descriptor's current capacity.
597 */
598bool
599IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
600{
601 vm_size_t actualBytesToCopy = min(withLength, _capacity - _length);
602 IOByteCount offset;
603
604 assert(_length <= _capacity);
605
606 offset = _length;
607 _length += actualBytesToCopy;
608 _ranges.v64->length += actualBytesToCopy;
609
610 if (_task == kernel_task)
611 bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
612 actualBytesToCopy);
613 else
614 writeBytes(offset, bytes, actualBytesToCopy);
615
616 return true;
617}
618
619/*
620 * getBytesNoCopy:
621 *
622 * Return the virtual address of the beginning of the buffer
623 */
624void * IOBufferMemoryDescriptor::getBytesNoCopy()
625{
626 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
627 return _buffer;
628 else
629 return (void *)_ranges.v64->address;
630}
631
632
633/*
634 * getBytesNoCopy:
635 *
636 * Return the virtual address of an offset from the beginning of the buffer
637 */
638void *
639IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
640{
641 IOVirtualAddress address;
642
643 if ((start + withLength) < start) return 0;
644
645 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
646 address = (IOVirtualAddress) _buffer;
647 else
648 address = _ranges.v64->address;
649
650 if (start < _length && (start + withLength) <= _length)
651 return (void *)(address + start);
652 return 0;
653}
654
655#ifndef __LP64__
656void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
657 IOByteCount * lengthOfSegment)
658{
659 void * bytes = getBytesNoCopy(offset, 0);
660
661 if (bytes && lengthOfSegment)
662 *lengthOfSegment = _length - offset;
663
664 return bytes;
665}
666#endif /* !__LP64__ */
667
668#ifdef __LP64__
669OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
670OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
671#else /* !__LP64__ */
672OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
673OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
674#endif /* !__LP64__ */
675OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
676OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
677OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
678OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
679OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
680OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
681OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
682OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
683OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
684OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
685OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
686OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
687OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
688OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);
689