1/*
2 * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <IOKit/assert.h>
30
31#include <libkern/OSTypes.h>
32#include <libkern/OSByteOrder.h>
33#include <libkern/OSDebug.h>
34
35#include <IOKit/IOReturn.h>
36#include <IOKit/IOLib.h>
37#include <IOKit/IODMACommand.h>
38#include <IOKit/IOMapper.h>
39#include <IOKit/IOMemoryDescriptor.h>
40#include <IOKit/IOBufferMemoryDescriptor.h>
41
42#include "IOKitKernelInternal.h"
43
44#define MAPTYPE(type) ((UInt) (type) & kTypeMask)
45#define IS_NONCOHERENT(type) (MAPTYPE(type) == kNonCoherent)
46
47enum
48{
49 kWalkSyncIn = 0x01, // bounce -> md
50 kWalkSyncOut = 0x02, // bounce <- md
51 kWalkSyncAlways = 0x04,
52 kWalkPreflight = 0x08,
53 kWalkDoubleBuffer = 0x10,
54 kWalkPrepare = 0x20,
55 kWalkComplete = 0x40,
56 kWalkClient = 0x80
57};
58
59
60#define fInternalState reserved
61#define fState reserved->fState
62#define fMDSummary reserved->fMDSummary
63
64
65#if 1
66// no direction => OutIn
67#define SHOULD_COPY_DIR(op, direction) \
68 ((kIODirectionNone == (direction)) \
69 || (kWalkSyncAlways & (op)) \
70 || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut) \
71 & (direction)))
72
73#else
74#define SHOULD_COPY_DIR(state, direction) (true)
75#endif
76
77#if 0
78#define DEBG(fmt, args...) { IOLog(fmt, ## args); kprintf(fmt, ## args); }
79#else
80#define DEBG(fmt, args...) {}
81#endif
82
83/**************************** class IODMACommand ***************************/
84
85#undef super
86#define super IOCommand
87OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
88
89OSMetaClassDefineReservedUsed(IODMACommand, 0);
90OSMetaClassDefineReservedUsed(IODMACommand, 1);
91OSMetaClassDefineReservedUsed(IODMACommand, 2);
92OSMetaClassDefineReservedUsed(IODMACommand, 3);
93OSMetaClassDefineReservedUsed(IODMACommand, 4);
94OSMetaClassDefineReservedUsed(IODMACommand, 5);
95OSMetaClassDefineReservedUsed(IODMACommand, 6);
96OSMetaClassDefineReservedUnused(IODMACommand, 7);
97OSMetaClassDefineReservedUnused(IODMACommand, 8);
98OSMetaClassDefineReservedUnused(IODMACommand, 9);
99OSMetaClassDefineReservedUnused(IODMACommand, 10);
100OSMetaClassDefineReservedUnused(IODMACommand, 11);
101OSMetaClassDefineReservedUnused(IODMACommand, 12);
102OSMetaClassDefineReservedUnused(IODMACommand, 13);
103OSMetaClassDefineReservedUnused(IODMACommand, 14);
104OSMetaClassDefineReservedUnused(IODMACommand, 15);
105
106IODMACommand *
107IODMACommand::withRefCon(void * refCon)
108{
109 IODMACommand * me = new IODMACommand;
110
111 if (me && !me->initWithRefCon(refCon))
112 {
113 me->release();
114 return 0;
115 }
116
117 return me;
118}
119
120IODMACommand *
121IODMACommand::withSpecification(SegmentFunction outSegFunc,
122 const SegmentOptions * segmentOptions,
123 uint32_t mappingOptions,
124 IOMapper * mapper,
125 void * refCon)
126{
127 IODMACommand * me = new IODMACommand;
128
129 if (me && !me->initWithSpecification(outSegFunc, segmentOptions, mappingOptions,
130 mapper, refCon))
131 {
132 me->release();
133 return 0;
134 }
135
136 return me;
137}
138
139IODMACommand *
140IODMACommand::withSpecification(SegmentFunction outSegFunc,
141 UInt8 numAddressBits,
142 UInt64 maxSegmentSize,
143 MappingOptions mappingOptions,
144 UInt64 maxTransferSize,
145 UInt32 alignment,
146 IOMapper *mapper,
147 void *refCon)
148{
149 IODMACommand * me = new IODMACommand;
150
151 if (me && !me->initWithSpecification(outSegFunc,
152 numAddressBits, maxSegmentSize,
153 mappingOptions, maxTransferSize,
154 alignment, mapper, refCon))
155 {
156 me->release();
157 return 0;
158 }
159
160 return me;
161}
162
163IODMACommand *
164IODMACommand::cloneCommand(void *refCon)
165{
166 SegmentOptions segmentOptions =
167 {
168 .fStructSize = sizeof(segmentOptions),
169 .fNumAddressBits = (uint8_t)fNumAddressBits,
170 .fMaxSegmentSize = fMaxSegmentSize,
171 .fMaxTransferSize = fMaxTransferSize,
172 .fAlignment = fAlignMask + 1,
173 .fAlignmentLength = fAlignMaskInternalSegments + 1,
174 .fAlignmentInternalSegments = fAlignMaskLength + 1
175 };
176
177 return (IODMACommand::withSpecification(fOutSeg, &segmentOptions,
178 fMappingOptions, fMapper, refCon));
179}
180
181#define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)
182
183bool
184IODMACommand::initWithRefCon(void * refCon)
185{
186 if (!super::init()) return (false);
187
188 if (!reserved)
189 {
190 reserved = IONew(IODMACommandInternal, 1);
191 if (!reserved) return false;
192 }
193 bzero(reserved, sizeof(IODMACommandInternal));
194 fRefCon = refCon;
195
196 return (true);
197}
198
199bool
200IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
201 const SegmentOptions * segmentOptions,
202 uint32_t mappingOptions,
203 IOMapper * mapper,
204 void * refCon)
205{
206 if (!initWithRefCon(refCon)) return false;
207
208 if (kIOReturnSuccess != setSpecification(outSegFunc, segmentOptions,
209 mappingOptions, mapper)) return false;
210
211 return (true);
212}
213
214bool
215IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
216 UInt8 numAddressBits,
217 UInt64 maxSegmentSize,
218 MappingOptions mappingOptions,
219 UInt64 maxTransferSize,
220 UInt32 alignment,
221 IOMapper *mapper,
222 void *refCon)
223{
224 SegmentOptions segmentOptions =
225 {
226 .fStructSize = sizeof(segmentOptions),
227 .fNumAddressBits = numAddressBits,
228 .fMaxSegmentSize = maxSegmentSize,
229 .fMaxTransferSize = maxTransferSize,
230 .fAlignment = alignment,
231 .fAlignmentLength = 1,
232 .fAlignmentInternalSegments = alignment
233 };
234
235 return (initWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper, refCon));
236}
237
238IOReturn
239IODMACommand::setSpecification(SegmentFunction outSegFunc,
240 const SegmentOptions * segmentOptions,
241 uint32_t mappingOptions,
242 IOMapper * mapper)
243{
244 IOService * device = 0;
245 UInt8 numAddressBits;
246 UInt64 maxSegmentSize;
247 UInt64 maxTransferSize;
248 UInt32 alignment;
249
250 bool is32Bit;
251
252 if (!outSegFunc || !segmentOptions) return (kIOReturnBadArgument);
253
254 is32Bit = ((OutputHost32 == outSegFunc)
255 || (OutputBig32 == outSegFunc)
256 || (OutputLittle32 == outSegFunc));
257
258 numAddressBits = segmentOptions->fNumAddressBits;
259 maxSegmentSize = segmentOptions->fMaxSegmentSize;
260 maxTransferSize = segmentOptions->fMaxTransferSize;
261 alignment = segmentOptions->fAlignment;
262 if (is32Bit)
263 {
264 if (!numAddressBits)
265 numAddressBits = 32;
266 else if (numAddressBits > 32)
267 return (kIOReturnBadArgument); // Wrong output function for bits
268 }
269
270 if (numAddressBits && (numAddressBits < PAGE_SHIFT)) return (kIOReturnBadArgument);
271
272 if (!maxSegmentSize) maxSegmentSize--; // Set Max segment to -1
273 if (!maxTransferSize) maxTransferSize--; // Set Max transfer to -1
274
275 if (mapper && !OSDynamicCast(IOMapper, mapper))
276 {
277 device = mapper;
278 mapper = 0;
279 }
280 if (!mapper && (kUnmapped != MAPTYPE(mappingOptions)))
281 {
282 IOMapper::checkForSystemMapper();
283 mapper = IOMapper::gSystem;
284 }
285
286 fNumSegments = 0;
287 fOutSeg = outSegFunc;
288 fNumAddressBits = numAddressBits;
289 fMaxSegmentSize = maxSegmentSize;
290 fMappingOptions = mappingOptions;
291 fMaxTransferSize = maxTransferSize;
292 if (!alignment) alignment = 1;
293 fAlignMask = alignment - 1;
294
295 alignment = segmentOptions->fAlignmentLength;
296 if (!alignment) alignment = 1;
297 fAlignMaskLength = alignment - 1;
298
299 alignment = segmentOptions->fAlignmentInternalSegments;
300 if (!alignment) alignment = (fAlignMask + 1);
301 fAlignMaskInternalSegments = alignment - 1;
302
303 switch (MAPTYPE(mappingOptions))
304 {
305 case kMapped: break;
306 case kUnmapped: break;
307 case kNonCoherent: break;
308
309 case kBypassed:
310 if (!mapper) break;
311 return (kIOReturnBadArgument);
312
313 default:
314 return (kIOReturnBadArgument);
315 };
316
317 if (mapper != fMapper)
318 {
319 if (mapper) mapper->retain();
320 if (fMapper) fMapper->release();
321 fMapper = mapper;
322 }
323
324 fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
325 fInternalState->fDevice = device;
326
327 return (kIOReturnSuccess);
328}
329
330void
331IODMACommand::free()
332{
333 if (reserved) IODelete(reserved, IODMACommandInternal, 1);
334
335 if (fMapper) fMapper->release();
336
337 super::free();
338}
339
340IOReturn
341IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
342{
343 IOReturn err = kIOReturnSuccess;
344
345 if (mem == fMemory)
346 {
347 if (!autoPrepare)
348 {
349 while (fActive)
350 complete();
351 }
352 return kIOReturnSuccess;
353 }
354
355 if (fMemory) {
356 // As we are almost certainly being called from a work loop thread
357 // if fActive is true it is probably not a good time to potentially
358 // block. Just test for it and return an error
359 if (fActive)
360 return kIOReturnBusy;
361 clearMemoryDescriptor();
362 }
363
364 if (mem) {
365 bzero(&fMDSummary, sizeof(fMDSummary));
366 err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)),
367 &fMDSummary, sizeof(fMDSummary));
368 if (err)
369 return err;
370
371 ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
372
373 if ((kMapped == MAPTYPE(fMappingOptions))
374 && fMapper)
375 fInternalState->fCheckAddressing = false;
376 else
377 fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
378
379 fInternalState->fNewMD = true;
380 mem->retain();
381 fMemory = mem;
382 fInternalState->fSetActiveNoMapper = (!fMapper);
383 if (fInternalState->fSetActiveNoMapper) mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
384 if (autoPrepare) {
385 err = prepare();
386 if (err) {
387 clearMemoryDescriptor();
388 }
389 }
390 }
391
392 return err;
393}
394
395IOReturn
396IODMACommand::clearMemoryDescriptor(bool autoComplete)
397{
398 if (fActive && !autoComplete) return (kIOReturnNotReady);
399
400 if (fMemory)
401 {
402 while (fActive) complete();
403 if (fInternalState->fSetActiveNoMapper) fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0);
404 fMemory->release();
405 fMemory = 0;
406 }
407
408 return (kIOReturnSuccess);
409}
410
411const IOMemoryDescriptor *
412IODMACommand::getMemoryDescriptor() const
413{
414 return fMemory;
415}
416
417IOMemoryDescriptor *
418IODMACommand::getIOMemoryDescriptor() const
419{
420 IOMemoryDescriptor * mem;
421
422 mem = reserved->fCopyMD;
423 if (!mem) mem = __IODEQUALIFY(IOMemoryDescriptor *, fMemory);
424
425 return (mem);
426}
427
428IOReturn
429IODMACommand::segmentOp(
430 void *reference,
431 IODMACommand *target,
432 Segment64 segment,
433 void *segments,
434 UInt32 segmentIndex)
435{
436 IOOptionBits op = (uintptr_t) reference;
437 addr64_t maxPhys, address;
438 uint64_t length;
439 uint32_t numPages;
440 uint32_t mask;
441
442 IODMACommandInternal * state = target->reserved;
443
444 if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper))
445 maxPhys = (1ULL << target->fNumAddressBits);
446 else
447 maxPhys = 0;
448 maxPhys--;
449
450 address = segment.fIOVMAddr;
451 length = segment.fLength;
452
453 assert(length);
454
455 if (!state->fMisaligned)
456 {
457 mask = (segmentIndex ? target->fAlignMaskInternalSegments : state->fSourceAlignMask);
458 state->fMisaligned |= (0 != (mask & address));
459 if (state->fMisaligned) DEBG("misaligned address %qx:%qx, %x\n", address, length, mask);
460 }
461 if (!state->fMisaligned)
462 {
463 mask = target->fAlignMaskLength;
464 state->fMisaligned |= (0 != (mask & length));
465 if (state->fMisaligned) DEBG("misaligned length %qx:%qx, %x\n", address, length, mask);
466 }
467
468 if (state->fMisaligned && (kWalkPreflight & op))
469 return (kIOReturnNotAligned);
470
471 if (!state->fDoubleBuffer)
472 {
473 if ((address + length - 1) <= maxPhys)
474 {
475 length = 0;
476 }
477 else if (address <= maxPhys)
478 {
479 DEBG("tail %qx, %qx", address, length);
480 length = (address + length - maxPhys - 1);
481 address = maxPhys + 1;
482 DEBG("-> %qx, %qx\n", address, length);
483 }
484 }
485
486 if (!length)
487 return (kIOReturnSuccess);
488
489 numPages = atop_64(round_page_64((address & PAGE_MASK) + length));
490
491 if (kWalkPreflight & op)
492 {
493 state->fCopyPageCount += numPages;
494 }
495 else
496 {
497 vm_page_t lastPage;
498 lastPage = NULL;
499 if (kWalkPrepare & op)
500 {
501 lastPage = state->fCopyNext;
502 for (IOItemCount idx = 0; idx < numPages; idx++)
503 {
504 vm_page_set_offset(lastPage, atop_64(address) + idx);
505 lastPage = vm_page_get_next(lastPage);
506 }
507 }
508
509 if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
510 {
511 lastPage = state->fCopyNext;
512 for (IOItemCount idx = 0; idx < numPages; idx++)
513 {
514 if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
515 {
516 addr64_t cpuAddr = address;
517 addr64_t remapAddr;
518 uint64_t chunk;
519
520 if ((kMapped == MAPTYPE(target->fMappingOptions))
521 && target->fMapper)
522 {
523 cpuAddr = target->fMapper->mapToPhysicalAddress(address);
524 }
525
526 remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
527 if (!state->fDoubleBuffer)
528 {
529 remapAddr += (address & PAGE_MASK);
530 }
531 chunk = PAGE_SIZE - (address & PAGE_MASK);
532 if (chunk > length)
533 chunk = length;
534
535 DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
536 (kWalkSyncIn & op) ? "->" : "<-",
537 address, chunk, op);
538
539 if (kWalkSyncIn & op)
540 { // cppvNoModSnk
541 copypv(remapAddr, cpuAddr, chunk,
542 cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
543 }
544 else
545 {
546 copypv(cpuAddr, remapAddr, chunk,
547 cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
548 }
549 address += chunk;
550 length -= chunk;
551 }
552 lastPage = vm_page_get_next(lastPage);
553 }
554 }
555 state->fCopyNext = lastPage;
556 }
557
558 return kIOReturnSuccess;
559}
560
561IOBufferMemoryDescriptor *
562IODMACommand::createCopyBuffer(IODirection direction, UInt64 length)
563{
564 mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask
565 return (IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
566 direction, length, mask));
567}
568
569IOReturn
570IODMACommand::walkAll(UInt8 op)
571{
572 IODMACommandInternal * state = fInternalState;
573
574 IOReturn ret = kIOReturnSuccess;
575 UInt32 numSegments;
576 UInt64 offset;
577
578 if (kWalkPreflight & op)
579 {
580 state->fMisaligned = false;
581 state->fDoubleBuffer = false;
582 state->fPrepared = false;
583 state->fCopyNext = NULL;
584 state->fCopyPageAlloc = 0;
585 state->fCopyPageCount = 0;
586 state->fNextRemapPage = NULL;
587 state->fCopyMD = 0;
588
589 if (!(kWalkDoubleBuffer & op))
590 {
591 offset = 0;
592 numSegments = 0-1;
593 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
594 }
595
596 op &= ~kWalkPreflight;
597
598 state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer);
599 state->fForceDoubleBuffer = false;
600 if (state->fDoubleBuffer)
601 state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
602
603 if (state->fCopyPageCount)
604 {
605 vm_page_t mapBase = NULL;
606
607 DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
608
609 if (!fMapper && !state->fDoubleBuffer)
610 {
611 kern_return_t kr;
612
613 if (fMapper) panic("fMapper copying");
614
615 kr = vm_page_alloc_list(state->fCopyPageCount,
616 KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
617 if (KERN_SUCCESS != kr)
618 {
619 DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
620 mapBase = NULL;
621 }
622 }
623
624 if (mapBase)
625 {
626 state->fCopyPageAlloc = mapBase;
627 state->fCopyNext = state->fCopyPageAlloc;
628 offset = 0;
629 numSegments = 0-1;
630 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
631 state->fPrepared = true;
632 op &= ~(kWalkSyncIn | kWalkSyncOut);
633 }
634 else
635 {
636 DEBG("alloc IOBMD\n");
637 state->fCopyMD = createCopyBuffer(fMDSummary.fDirection, state->fPreparedLength);
638
639 if (state->fCopyMD)
640 {
641 ret = kIOReturnSuccess;
642 state->fPrepared = true;
643 }
644 else
645 {
646 DEBG("IODMACommand !alloc IOBMD");
647 return (kIOReturnNoResources);
648 }
649 }
650 }
651 }
652
653 if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
654 {
655 if (state->fCopyPageCount)
656 {
657 DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
658
659 if (state->fCopyPageAlloc)
660 {
661 state->fCopyNext = state->fCopyPageAlloc;
662 offset = 0;
663 numSegments = 0-1;
664 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
665 }
666 else if (state->fCopyMD)
667 {
668 DEBG("sync IOBMD\n");
669
670 if (SHOULD_COPY_DIR(op, fMDSummary.fDirection))
671 {
672 IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
673
674 IOByteCount bytes;
675
676 if (kWalkSyncIn & op)
677 bytes = poMD->writeBytes(state->fPreparedOffset,
678 state->fCopyMD->getBytesNoCopy(),
679 state->fPreparedLength);
680 else
681 bytes = poMD->readBytes(state->fPreparedOffset,
682 state->fCopyMD->getBytesNoCopy(),
683 state->fPreparedLength);
684 DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
685 ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
686 }
687 else
688 ret = kIOReturnSuccess;
689 }
690 }
691 }
692
693 if (kWalkComplete & op)
694 {
695 if (state->fCopyPageAlloc)
696 {
697 vm_page_free_list(state->fCopyPageAlloc, FALSE);
698 state->fCopyPageAlloc = 0;
699 state->fCopyPageCount = 0;
700 }
701 if (state->fCopyMD)
702 {
703 state->fCopyMD->release();
704 state->fCopyMD = 0;
705 }
706
707 state->fPrepared = false;
708 }
709 return (ret);
710}
711
712UInt8
713IODMACommand::getNumAddressBits(void)
714{
715 return (fNumAddressBits);
716}
717
718UInt32
719IODMACommand::getAlignment(void)
720{
721 return (fAlignMask + 1);
722}
723
724uint32_t
725IODMACommand::getAlignmentLength(void)
726{
727 return (fAlignMaskLength + 1);
728}
729
730uint32_t
731IODMACommand::getAlignmentInternalSegments(void)
732{
733 return (fAlignMaskInternalSegments + 1);
734}
735
736IOReturn
737IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc,
738 const SegmentOptions * segmentOptions,
739 uint32_t mappingOptions,
740 IOMapper * mapper,
741 UInt64 offset,
742 UInt64 length,
743 bool flushCache,
744 bool synchronize)
745{
746 IOReturn ret;
747
748 if (fActive) return kIOReturnNotPermitted;
749
750 ret = setSpecification(outSegFunc, segmentOptions, mappingOptions, mapper);
751 if (kIOReturnSuccess != ret) return (ret);
752
753 ret = prepare(offset, length, flushCache, synchronize);
754
755 return (ret);
756}
757
758IOReturn
759IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc,
760 UInt8 numAddressBits,
761 UInt64 maxSegmentSize,
762 MappingOptions mappingOptions,
763 UInt64 maxTransferSize,
764 UInt32 alignment,
765 IOMapper *mapper,
766 UInt64 offset,
767 UInt64 length,
768 bool flushCache,
769 bool synchronize)
770{
771 SegmentOptions segmentOptions =
772 {
773 .fStructSize = sizeof(segmentOptions),
774 .fNumAddressBits = numAddressBits,
775 .fMaxSegmentSize = maxSegmentSize,
776 .fMaxTransferSize = maxTransferSize,
777 .fAlignment = alignment,
778 .fAlignmentLength = 1,
779 .fAlignmentInternalSegments = alignment
780 };
781
782 return (prepareWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper,
783 offset, length, flushCache, synchronize));
784}
785
786
787IOReturn
788IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
789{
790 IODMACommandInternal * state = fInternalState;
791 IOReturn ret = kIOReturnSuccess;
792 uint32_t mappingOptions = fMappingOptions;
793
794 // check specification has been set
795 if (!fOutSeg) return (kIOReturnNotReady);
796
797 if (!length) length = fMDSummary.fLength;
798
799 if (length > fMaxTransferSize) return kIOReturnNoSpace;
800
801 if (fActive++)
802 {
803 if ((state->fPreparedOffset != offset)
804 || (state->fPreparedLength != length))
805 ret = kIOReturnNotReady;
806 }
807 else
808 {
809 if (fAlignMaskLength & length) return (kIOReturnNotAligned);
810
811 state->fPreparedOffset = offset;
812 state->fPreparedLength = length;
813
814 state->fMapContig = false;
815 state->fMisaligned = false;
816 state->fDoubleBuffer = false;
817 state->fPrepared = false;
818 state->fCopyNext = NULL;
819 state->fCopyPageAlloc = 0;
820 state->fCopyPageCount = 0;
821 state->fNextRemapPage = NULL;
822 state->fCopyMD = 0;
823 state->fLocalMapperAlloc = 0;
824 state->fLocalMapperAllocValid = false;
825 state->fLocalMapperAllocLength = 0;
826
827 state->fSourceAlignMask = fAlignMask;
828 if (fMapper)
829 state->fSourceAlignMask &= page_mask;
830
831 state->fCursor = state->fIterateOnly
832 || (!state->fCheckAddressing
833 && (!state->fSourceAlignMask
834 || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
835
836 if (!state->fCursor)
837 {
838 IOOptionBits op = kWalkPrepare | kWalkPreflight;
839 if (synchronize)
840 op |= kWalkSyncOut;
841 ret = walkAll(op);
842 }
843
844 if (IS_NONCOHERENT(mappingOptions) && flushCache)
845 {
846 if (state->fCopyMD)
847 {
848 state->fCopyMD->performOperation(kIOMemoryIncoherentIOStore, 0, length);
849 }
850 else
851 {
852 IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory);
853 md->performOperation(kIOMemoryIncoherentIOStore, offset, length);
854 }
855 }
856
857 if (fMapper)
858 {
859 IOMDDMAMapArgs mapArgs;
860 bzero(&mapArgs, sizeof(mapArgs));
861 mapArgs.fMapper = fMapper;
862 mapArgs.fCommand = this;
863 mapArgs.fMapSpec.device = state->fDevice;
864 mapArgs.fMapSpec.alignment = fAlignMask + 1;
865 mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64;
866 mapArgs.fLength = state->fPreparedLength;
867 const IOMemoryDescriptor * md = state->fCopyMD;
868 if (md) { mapArgs.fOffset = 0; } else
869 {
870 md = fMemory;
871 mapArgs.fOffset = state->fPreparedOffset;
872 }
873 ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs));
874//IOLog("dma %p 0x%x 0x%qx-0x%qx 0x%qx-0x%qx\n", this, ret, state->fPreparedOffset, state->fPreparedLength, mapArgs.fAlloc, mapArgs.fAllocLength);
875
876 if (kIOReturnSuccess == ret)
877 {
878 state->fLocalMapperAlloc = mapArgs.fAlloc;
879 state->fLocalMapperAllocValid = true;
880 state->fLocalMapperAllocLength = mapArgs.fAllocLength;
881 state->fMapContig = mapArgs.fMapContig;
882 }
883 if (NULL != IOMapper::gSystem) ret = kIOReturnSuccess;
884 }
885 if (kIOReturnSuccess == ret) state->fPrepared = true;
886 }
887 return ret;
888}
889
890IOReturn
891IODMACommand::complete(bool invalidateCache, bool synchronize)
892{
893 IODMACommandInternal * state = fInternalState;
894 IOReturn ret = kIOReturnSuccess;
895 IOMemoryDescriptor * copyMD;
896
897 if (fActive < 1)
898 return kIOReturnNotReady;
899
900 if (!--fActive)
901 {
902 copyMD = state->fCopyMD;
903 if (copyMD) copyMD->retain();
904
905 if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
906 {
907 if (copyMD)
908 {
909 copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength);
910 }
911 else
912 {
913 IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory);
914 md->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
915 }
916 }
917
918 if (!state->fCursor)
919 {
920 IOOptionBits op = kWalkComplete;
921 if (synchronize)
922 op |= kWalkSyncIn;
923 ret = walkAll(op);
924 }
925
926 if (state->fLocalMapperAllocValid)
927 {
928 IOMDDMAMapArgs mapArgs;
929 bzero(&mapArgs, sizeof(mapArgs));
930 mapArgs.fMapper = fMapper;
931 mapArgs.fCommand = this;
932 mapArgs.fAlloc = state->fLocalMapperAlloc;
933 mapArgs.fAllocLength = state->fLocalMapperAllocLength;
934 const IOMemoryDescriptor * md = copyMD;
935 if (md) { mapArgs.fOffset = 0; }
936 else
937 {
938 md = fMemory;
939 mapArgs.fOffset = state->fPreparedOffset;
940 }
941
942 ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs));
943
944 state->fLocalMapperAlloc = 0;
945 state->fLocalMapperAllocValid = false;
946 state->fLocalMapperAllocLength = 0;
947 }
948 if (copyMD) copyMD->release();
949 state->fPrepared = false;
950 }
951
952 return ret;
953}
954
955IOReturn
956IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length)
957{
958 IODMACommandInternal * state = fInternalState;
959 if (fActive < 1)
960 return (kIOReturnNotReady);
961
962 if (offset)
963 *offset = state->fPreparedOffset;
964 if (length)
965 *length = state->fPreparedLength;
966
967 return (kIOReturnSuccess);
968}
969
970IOReturn
971IODMACommand::synchronize(IOOptionBits options)
972{
973 IODMACommandInternal * state = fInternalState;
974 IOReturn ret = kIOReturnSuccess;
975 IOOptionBits op;
976
977 if (kIODirectionOutIn == (kIODirectionOutIn & options))
978 return kIOReturnBadArgument;
979
980 if (fActive < 1)
981 return kIOReturnNotReady;
982
983 op = 0;
984 if (kForceDoubleBuffer & options)
985 {
986 if (state->fDoubleBuffer) return kIOReturnSuccess;
987 ret = complete(false /* invalidateCache */, true /* synchronize */);
988 state->fCursor = false;
989 state->fForceDoubleBuffer = true;
990 ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */);
991
992 return (ret);
993 }
994 else if (state->fCursor)
995 return kIOReturnSuccess;
996
997 if (kIODirectionIn & options)
998 op |= kWalkSyncIn | kWalkSyncAlways;
999 else if (kIODirectionOut & options)
1000 op |= kWalkSyncOut | kWalkSyncAlways;
1001
1002 ret = walkAll(op);
1003
1004 return ret;
1005}
1006
1007struct IODMACommandTransferContext
1008{
1009 void * buffer;
1010 UInt64 bufferOffset;
1011 UInt64 remaining;
1012 UInt32 op;
1013};
1014enum
1015{
1016 kIODMACommandTransferOpReadBytes = 1,
1017 kIODMACommandTransferOpWriteBytes = 2
1018};
1019
1020IOReturn
1021IODMACommand::transferSegment(void *reference,
1022 IODMACommand *target,
1023 Segment64 segment,
1024 void *segments,
1025 UInt32 segmentIndex)
1026{
1027 IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference;
1028 UInt64 length = min(segment.fLength, context->remaining);
1029 addr64_t ioAddr = segment.fIOVMAddr;
1030 addr64_t cpuAddr = ioAddr;
1031
1032 context->remaining -= length;
1033
1034 while (length)
1035 {
1036 UInt64 copyLen = length;
1037 if ((kMapped == MAPTYPE(target->fMappingOptions))
1038 && target->fMapper)
1039 {
1040 cpuAddr = target->fMapper->mapToPhysicalAddress(ioAddr);
1041 copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
1042 ioAddr += copyLen;
1043 }
1044
1045 switch (context->op)
1046 {
1047 case kIODMACommandTransferOpReadBytes:
1048 copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen,
1049 cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
1050 break;
1051 case kIODMACommandTransferOpWriteBytes:
1052 copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen,
1053 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
1054 break;
1055 }
1056 length -= copyLen;
1057 context->bufferOffset += copyLen;
1058 }
1059
1060 return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun);
1061}
1062
1063UInt64
1064IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
1065{
1066 IODMACommandInternal * state = fInternalState;
1067 IODMACommandTransferContext context;
1068 Segment64 segments[1];
1069 UInt32 numSegments = 0-1;
1070
1071 if (fActive < 1)
1072 return (0);
1073
1074 if (offset >= state->fPreparedLength)
1075 return (0);
1076 length = min(length, state->fPreparedLength - offset);
1077
1078 context.buffer = buffer;
1079 context.bufferOffset = 0;
1080 context.remaining = length;
1081 context.op = transferOp;
1082 (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments);
1083
1084 return (length - context.remaining);
1085}
1086
1087UInt64
1088IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
1089{
1090 return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length));
1091}
1092
1093UInt64
1094IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
1095{
1096 return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length));
1097}
1098
1099IOReturn
1100IODMACommand::genIOVMSegments(UInt64 *offsetP,
1101 void *segmentsP,
1102 UInt32 *numSegmentsP)
1103{
1104 return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg,
1105 offsetP, segmentsP, numSegmentsP));
1106}
1107
1108IOReturn
1109IODMACommand::genIOVMSegments(uint32_t op,
1110 InternalSegmentFunction outSegFunc,
1111 void *reference,
1112 UInt64 *offsetP,
1113 void *segmentsP,
1114 UInt32 *numSegmentsP)
1115{
1116 IODMACommandInternal * internalState = fInternalState;
1117 IOOptionBits mdOp = kIOMDWalkSegments;
1118 IOReturn ret = kIOReturnSuccess;
1119
1120 if (!(kWalkComplete & op) && !fActive)
1121 return kIOReturnNotReady;
1122
1123 if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
1124 return kIOReturnBadArgument;
1125
1126 IOMDDMAWalkSegmentArgs *state =
1127 (IOMDDMAWalkSegmentArgs *)(void *) fState;
1128
1129 UInt64 offset = *offsetP + internalState->fPreparedOffset;
1130 UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
1131
1132 if (offset >= memLength)
1133 return kIOReturnOverrun;
1134
1135 if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
1136 state->fOffset = 0;
1137 internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1138 internalState->fNextRemapPage = NULL;
1139 internalState->fNewMD = false;
1140 mdOp = kIOMDFirstSegment;
1141 if (fMapper)
1142 {
1143 if (internalState->fLocalMapperAllocValid)
1144 {
1145 state->fMapped = kIOMDDMAWalkMappedLocal;
1146 state->fMappedBase = internalState->fLocalMapperAlloc;
1147 }
1148 else state->fMapped = true;
1149 }
1150 };
1151
1152 UInt32 segIndex = 0;
1153 UInt32 numSegments = *numSegmentsP;
1154 Segment64 curSeg = { 0, 0 };
1155 bool curSegValid = false;
1156 addr64_t maxPhys;
1157
1158 if (fNumAddressBits && (fNumAddressBits < 64))
1159 maxPhys = (1ULL << fNumAddressBits);
1160 else
1161 maxPhys = 0;
1162 maxPhys--;
1163
1164 while (internalState->fIOVMAddrValid || (state->fOffset < memLength))
1165 {
1166 // state = next seg
1167 if (!internalState->fIOVMAddrValid) {
1168
1169 IOReturn rtn;
1170
1171 state->fOffset = offset;
1172 state->fLength = memLength - offset;
1173
1174 if (internalState->fMapContig && internalState->fLocalMapperAllocValid)
1175 {
1176 state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset;
1177 rtn = kIOReturnSuccess;
1178#if 0
1179 {
1180 uint64_t checkOffset;
1181 IOPhysicalLength segLen;
1182 for (checkOffset = 0; checkOffset < state->fLength; )
1183 {
1184 addr64_t phys = const_cast<IOMemoryDescriptor *>(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone);
1185 if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys)
1186 {
1187 panic("%llx != %llx:%llx, %llx phys: %llx %llx\n", offset,
1188 state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength,
1189 phys, checkOffset);
1190 }
1191 checkOffset += page_size - (phys & page_mask);
1192 }
1193 }
1194#endif
1195 }
1196 else
1197 {
1198 const IOMemoryDescriptor * memory =
1199 internalState->fCopyMD ? internalState->fCopyMD : fMemory;
1200 rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
1201 mdOp = kIOMDWalkSegments;
1202 }
1203
1204 if (rtn == kIOReturnSuccess)
1205 {
1206 internalState->fIOVMAddrValid = true;
1207 assert(state->fLength);
1208 if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) {
1209 UInt64 length = state->fLength;
1210 offset += length;
1211 curSeg.fLength += length;
1212 internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1213 }
1214 }
1215 else if (rtn == kIOReturnOverrun)
1216 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
1217 else
1218 return rtn;
1219 }
1220
1221 // seg = state, offset = end of seg
1222 if (!curSegValid)
1223 {
1224 UInt64 length = state->fLength;
1225 offset += length;
1226 curSeg.fIOVMAddr = state->fIOVMAddr;
1227 curSeg.fLength = length;
1228 curSegValid = true;
1229 internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1230 }
1231
1232 if (!internalState->fIOVMAddrValid)
1233 {
1234 // maxPhys
1235 if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
1236 {
1237 if (internalState->fCursor)
1238 {
1239 curSegValid = curSeg.fIOVMAddr = 0;
1240 ret = kIOReturnMessageTooLarge;
1241 break;
1242 }
1243 else if (curSeg.fIOVMAddr <= maxPhys)
1244 {
1245 UInt64 remain, newLength;
1246
1247 newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
1248 DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
1249 remain = curSeg.fLength - newLength;
1250 state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
1251 internalState->fIOVMAddrValid = true;
1252 curSeg.fLength = newLength;
1253 state->fLength = remain;
1254 offset -= remain;
1255 }
1256 else
1257 {
1258 UInt64 addr = curSeg.fIOVMAddr;
1259 ppnum_t addrPage = atop_64(addr);
1260 vm_page_t remap = NULL;
1261 UInt64 remain, newLength;
1262
1263 DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
1264
1265 remap = internalState->fNextRemapPage;
1266 if (remap && (addrPage == vm_page_get_offset(remap)))
1267 {
1268 }
1269 else for (remap = internalState->fCopyPageAlloc;
1270 remap && (addrPage != vm_page_get_offset(remap));
1271 remap = vm_page_get_next(remap))
1272 {
1273 }
1274
1275 if (!remap) panic("no remap page found");
1276
1277 curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
1278 + (addr & PAGE_MASK);
1279 curSegValid = true;
1280 internalState->fNextRemapPage = vm_page_get_next(remap);
1281
1282 newLength = PAGE_SIZE - (addr & PAGE_MASK);
1283 if (newLength < curSeg.fLength)
1284 {
1285 remain = curSeg.fLength - newLength;
1286 state->fIOVMAddr = addr + newLength;
1287 internalState->fIOVMAddrValid = true;
1288 curSeg.fLength = newLength;
1289 state->fLength = remain;
1290 offset -= remain;
1291 }
1292 DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
1293 }
1294 }
1295
1296 // reduce size of output segment
1297 uint64_t reduce, leftover = 0;
1298
1299 // fMaxSegmentSize
1300 if (curSeg.fLength > fMaxSegmentSize)
1301 {
1302 leftover += curSeg.fLength - fMaxSegmentSize;
1303 curSeg.fLength = fMaxSegmentSize;
1304 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1305 internalState->fIOVMAddrValid = true;
1306 }
1307
1308 // alignment current length
1309
1310 reduce = (curSeg.fLength & fAlignMaskLength);
1311 if (reduce && (curSeg.fLength > reduce))
1312 {
1313 leftover += reduce;
1314 curSeg.fLength -= reduce;
1315 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1316 internalState->fIOVMAddrValid = true;
1317 }
1318
1319 // alignment next address
1320
1321 reduce = (state->fIOVMAddr & fAlignMaskInternalSegments);
1322 if (reduce && (curSeg.fLength > reduce))
1323 {
1324 leftover += reduce;
1325 curSeg.fLength -= reduce;
1326 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1327 internalState->fIOVMAddrValid = true;
1328 }
1329
1330 if (leftover)
1331 {
1332 DEBG("reduce seg by 0x%llx @ 0x%llx [0x%llx, 0x%llx]\n",
1333 leftover, offset,
1334 curSeg.fIOVMAddr, curSeg.fLength);
1335 state->fLength = leftover;
1336 offset -= leftover;
1337 }
1338
1339 //
1340
1341 if (internalState->fCursor)
1342 {
1343 bool misaligned;
1344 uint32_t mask;
1345
1346 mask = (segIndex ? fAlignMaskInternalSegments : internalState->fSourceAlignMask);
1347 misaligned = (0 != (mask & curSeg.fIOVMAddr));
1348 if (!misaligned)
1349 {
1350 mask = fAlignMaskLength;
1351 misaligned |= (0 != (mask & curSeg.fLength));
1352 }
1353 if (misaligned)
1354 {
1355 if (misaligned) DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength);
1356 curSegValid = curSeg.fIOVMAddr = 0;
1357 ret = kIOReturnNotAligned;
1358 break;
1359 }
1360 }
1361
1362 if (offset >= memLength)
1363 {
1364 curSeg.fLength -= (offset - memLength);
1365 offset = memLength;
1366 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
1367 break;
1368 }
1369 }
1370
1371 if (internalState->fIOVMAddrValid) {
1372 if ((segIndex + 1 == numSegments))
1373 break;
1374
1375 ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1376 curSegValid = curSeg.fIOVMAddr = 0;
1377 if (kIOReturnSuccess != ret)
1378 break;
1379 }
1380 }
1381
1382 if (curSegValid) {
1383 ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1384 }
1385
1386 if (kIOReturnSuccess == ret)
1387 {
1388 state->fOffset = offset;
1389 *offsetP = offset - internalState->fPreparedOffset;
1390 *numSegmentsP = segIndex;
1391 }
1392 return ret;
1393}
1394
1395IOReturn
1396IODMACommand::clientOutputSegment(
1397 void *reference, IODMACommand *target,
1398 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1399{
1400 SegmentFunction segmentFunction = (SegmentFunction) reference;
1401 IOReturn ret = kIOReturnSuccess;
1402
1403 if (target->fNumAddressBits && (target->fNumAddressBits < 64)
1404 && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
1405 && (target->reserved->fLocalMapperAllocValid || !target->fMapper))
1406 {
1407 DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1408 ret = kIOReturnMessageTooLarge;
1409 }
1410
1411 if (!(*segmentFunction)(target, segment, vSegList, outSegIndex))
1412 {
1413 DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1414 ret = kIOReturnMessageTooLarge;
1415 }
1416
1417 return (ret);
1418}
1419
1420IOReturn
1421IODMACommand::genIOVMSegments(SegmentFunction segmentFunction,
1422 UInt64 *offsetP,
1423 void *segmentsP,
1424 UInt32 *numSegmentsP)
1425{
1426 return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction,
1427 offsetP, segmentsP, numSegmentsP));
1428}
1429
1430bool
1431IODMACommand::OutputHost32(IODMACommand *,
1432 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1433{
1434 Segment32 *base = (Segment32 *) vSegList;
1435 base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
1436 base[outSegIndex].fLength = (UInt32) segment.fLength;
1437 return true;
1438}
1439
1440bool
1441IODMACommand::OutputBig32(IODMACommand *,
1442 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1443{
1444 const UInt offAddr = outSegIndex * sizeof(Segment32);
1445 const UInt offLen = offAddr + sizeof(UInt32);
1446 OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1447 OSWriteBigInt32(vSegList, offLen, (UInt32) segment.fLength);
1448 return true;
1449}
1450
1451bool
1452IODMACommand::OutputLittle32(IODMACommand *,
1453 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1454{
1455 const UInt offAddr = outSegIndex * sizeof(Segment32);
1456 const UInt offLen = offAddr + sizeof(UInt32);
1457 OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1458 OSWriteLittleInt32(vSegList, offLen, (UInt32) segment.fLength);
1459 return true;
1460}
1461
1462bool
1463IODMACommand::OutputHost64(IODMACommand *,
1464 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1465{
1466 Segment64 *base = (Segment64 *) vSegList;
1467 base[outSegIndex] = segment;
1468 return true;
1469}
1470
1471bool
1472IODMACommand::OutputBig64(IODMACommand *,
1473 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1474{
1475 const UInt offAddr = outSegIndex * sizeof(Segment64);
1476 const UInt offLen = offAddr + sizeof(UInt64);
1477 OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1478 OSWriteBigInt64(vSegList, offLen, (UInt64) segment.fLength);
1479 return true;
1480}
1481
1482bool
1483IODMACommand::OutputLittle64(IODMACommand *,
1484 Segment64 segment, void *vSegList, UInt32 outSegIndex)
1485{
1486 const UInt offAddr = outSegIndex * sizeof(Segment64);
1487 const UInt offLen = offAddr + sizeof(UInt64);
1488 OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1489 OSWriteLittleInt64(vSegList, offLen, (UInt64) segment.fLength);
1490 return true;
1491}
1492
1493
1494