1/*
2 * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/uio.h>
30#include <sys/conf.h>
31
32#include <IOKit/IOLib.h>
33#include <IOKit/IOBSD.h>
34#include <IOKit/IOService.h>
35#include <IOKit/IOPlatformExpert.h>
36#include <IOKit/IOPolledInterface.h>
37#include <IOKit/IOHibernatePrivate.h>
38#include <IOKit/IOBufferMemoryDescriptor.h>
39#include <IOKit/AppleKeyStoreInterface.h>
40#include "IOKitKernelInternal.h"
41
42
43/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
44
45OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
46
47OSMetaClassDefineReservedUsed(IOPolledInterface, 0);
48OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
49OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
50OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
51OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
52OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
53OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
54OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
55OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
56OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
57OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
58OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
59OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
60OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
61OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
62OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
63
64/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
65
66#ifndef kIOMediaPreferredBlockSizeKey
67#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
68#endif
69
70enum { kDefaultIOSize = 128*1024 };
71
72/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
73
74class IOPolledFilePollers : public OSObject
75{
76 OSDeclareDefaultStructors(IOPolledFilePollers)
77
78public:
79 IOService * media;
80 OSArray * pollers;
81 IOBufferMemoryDescriptor * ioBuffer;
82 bool abortable;
83 bool io;
84 IOReturn ioStatus;
85 uint32_t openCount;
86
87 static IOPolledFilePollers * copyPollers(IOService * media);
88};
89
90OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
91
92/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
93
94IOPolledFilePollers *
95IOPolledFilePollers::copyPollers(IOService * media)
96{
97 IOPolledFilePollers * vars;
98 IOReturn err;
99 IOService * service;
100 OSObject * obj;
101 IORegistryEntry * next;
102 IORegistryEntry * child;
103
104 if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
105 {
106 return (OSDynamicCast(IOPolledFilePollers, obj));
107 }
108
109 do
110 {
111 vars = OSTypeAlloc(IOPolledFilePollers);
112 vars->init();
113
114 vars->pollers = OSArray::withCapacity(4);
115 if (!vars->pollers)
116 {
117 err = kIOReturnNoMemory;
118 break;
119 }
120
121 next = vars->media = media;
122 do
123 {
124 IOPolledInterface * poller;
125 OSObject * obj;
126
127 obj = next->getProperty(kIOPolledInterfaceSupportKey);
128 if (kOSBooleanFalse == obj)
129 {
130 vars->pollers->flushCollection();
131 break;
132 }
133 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
134 vars->pollers->setObject(poller);
135
136 if ((service = OSDynamicCast(IOService, next))
137 && service->getDeviceMemory()
138 && !vars->pollers->getCount()) break;
139
140 child = next;
141 }
142 while ((next = child->getParentEntry(gIOServicePlane))
143 && child->isParent(next, gIOServicePlane, true));
144
145 if (!vars->pollers->getCount())
146 {
147 err = kIOReturnUnsupported;
148 break;
149 }
150 }
151 while (false);
152
153 media->setProperty(kIOPolledInterfaceStackKey, vars);
154
155 return (vars);
156}
157
158/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
159
160static IOReturn
161IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
162
163/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164
165static IOReturn
166IOPolledFilePollersProbe(IOPolledFilePollers * vars)
167{
168 IOReturn err = kIOReturnError;
169 int32_t idx;
170 IOPolledInterface * poller;
171
172 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
173 {
174 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
175 err = poller->probe(vars->media);
176 if (err)
177 {
178 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
179 break;
180 }
181 }
182
183 return (err);
184}
185
186/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
187
188IOReturn
189IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
190{
191
192 IOPolledFilePollers * vars = filevars->pollers;
193 IOBufferMemoryDescriptor * ioBuffer;
194 IOPolledInterface * poller;
195 IOService * next;
196 IOReturn err = kIOReturnError;
197 int32_t idx;
198
199 vars->abortable = abortable;
200 ioBuffer = 0;
201
202 if (kIOPolledAfterSleepState == state)
203 {
204 vars->ioStatus = 0;
205 vars->io = false;
206 }
207 (void) IOPolledFilePollersIODone(vars, false);
208
209 if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
210 {
211 ioBuffer = vars->ioBuffer;
212 if (!ioBuffer)
213 {
214 vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
215 2 * kDefaultIOSize, page_size);
216 if (!ioBuffer) return (kIOReturnNoMemory);
217 }
218 }
219
220 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
221 {
222 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
223 err = poller->open(state, ioBuffer);
224 if (kIOReturnSuccess != err)
225 {
226 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
227 break;
228 }
229 }
230 if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state))
231 {
232 next = vars->media;
233 while (next)
234 {
235 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
236 next = next->getProvider();
237 }
238 }
239
240 return (err);
241}
242
243/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
244
245IOReturn
246IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
247{
248 IOPolledFilePollers * vars = filevars->pollers;
249 IOPolledInterface * poller;
250 IORegistryEntry * next;
251 IOReturn err;
252 int32_t idx;
253
254 (void) IOPolledFilePollersIODone(vars, false);
255
256 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state))
257 {
258 vars->openCount--;
259 }
260
261 for (idx = 0, err = kIOReturnSuccess;
262 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
263 idx++)
264 {
265 err = poller->close(state);
266 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state))
267 {
268 err = poller->close(kIOPolledBeforeSleepState);
269 }
270 if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
271 }
272
273 if (kIOPolledPostflightState == state)
274 {
275 next = vars->media;
276 while (next)
277 {
278 next->removeProperty(kIOPolledInterfaceActiveKey);
279 next = next->getParentEntry(gIOServicePlane);
280 }
281 }
282
283 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do
284 {
285 if (vars->openCount) break;
286 if (vars->ioBuffer)
287 {
288 vars->ioBuffer->release();
289 vars->ioBuffer = 0;
290 }
291 }
292 while (false);
293
294 return (err);
295}
296
297/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
298
299IOReturn IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
300{
301 return (kIOReturnUnsupported);
302}
303
304/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
305
306IOReturn
307IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
308 const uint8_t * key, size_t keySize)
309{
310 IOReturn ret = kIOReturnUnsupported;
311 IOReturn err;
312 int32_t idx;
313 IOPolledFilePollers * vars = filevars->pollers;
314 IOPolledInterface * poller;
315
316 for (idx = 0;
317 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
318 idx++)
319 {
320 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
321 err = poller->setEncryptionKey(key, keySize);
322 if (kIOReturnSuccess == err) ret = err;
323 }
324
325 return (ret);
326}
327
328/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
329
330IOMemoryDescriptor *
331IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
332{
333 return (vars->pollers->ioBuffer);
334}
335
336/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
337
338static void
339IOPolledIOComplete(void * target,
340 void * parameter,
341 IOReturn status,
342 UInt64 actualByteCount)
343{
344 IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
345
346 vars->ioStatus = status;
347}
348
349/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
350
351static IOReturn
352IOStartPolledIO(IOPolledFilePollers * vars,
353 uint32_t operation, uint32_t bufferOffset,
354 uint64_t deviceOffset, uint64_t length)
355{
356 IOReturn err;
357 IOPolledInterface * poller;
358 IOPolledCompletion completion;
359
360 err = vars->ioStatus;
361 if (kIOReturnSuccess != err) return (err);
362
363 completion.target = 0;
364 completion.action = &IOPolledIOComplete;
365 completion.parameter = vars;
366
367 vars->ioStatus = -1;
368
369 poller = (IOPolledInterface *) vars->pollers->getObject(0);
370 err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
371 if (err) {
372 if (kernel_debugger_entry_count) {
373 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
374 } else {
375 HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
376 vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
377 }
378 }
379 return (err);
380}
381
382/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
383
384static IOReturn
385IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
386{
387 IOReturn err = kIOReturnSuccess;
388 int32_t idx = 0;
389 IOPolledInterface * poller;
390 AbsoluteTime deadline;
391
392 if (!vars->io) return (kIOReturnSuccess);
393
394 abortable &= vars->abortable;
395
396 clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
397
398 while (-1 == vars->ioStatus)
399 {
400 for (idx = 0;
401 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
402 idx++)
403 {
404 IOReturn newErr;
405 newErr = poller->checkForWork();
406 if ((newErr == kIOReturnAborted) && !abortable)
407 newErr = kIOReturnSuccess;
408 if (kIOReturnSuccess == err)
409 err = newErr;
410 }
411 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
412 {
413 HIBLOG("IOPolledInterface::forced timeout\n");
414 vars->ioStatus = kIOReturnTimeout;
415 }
416 }
417 vars->io = false;
418
419#if HIBERNATION
420 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
421 {
422 err = kIOReturnAborted;
423 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
424 }
425#endif
426
427 if (err)
428 {
429 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
430 }
431 else
432 {
433 err = vars->ioStatus;
434 if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
435 }
436
437 return (err);
438}
439
440/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
441/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
442
443struct _OpenFileContext
444{
445 OSData * extents;
446 uint64_t size;
447};
448
449static void
450file_extent_callback(void * ref, uint64_t start, uint64_t length)
451{
452 _OpenFileContext * ctx = (_OpenFileContext *) ref;
453 IOPolledFileExtent extent;
454
455 extent.start = start;
456 extent.length = length;
457 ctx->extents->appendBytes(&extent, sizeof(extent));
458 ctx->size += length;
459}
460
461/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
462
463static IOService *
464IOCopyMediaForDev(dev_t device)
465{
466 OSDictionary * matching;
467 OSNumber * num;
468 OSIterator * iter;
469 IOService * result = 0;
470
471 matching = IOService::serviceMatching("IOMedia");
472 if (!matching)
473 return (0);
474 do
475 {
476 num = OSNumber::withNumber(major(device), 32);
477 if (!num)
478 break;
479 matching->setObject(kIOBSDMajorKey, num);
480 num->release();
481 num = OSNumber::withNumber(minor(device), 32);
482 if (!num)
483 break;
484 matching->setObject(kIOBSDMinorKey, num);
485 num->release();
486 if (!num)
487 break;
488 iter = IOService::getMatchingServices(matching);
489 if (iter)
490 {
491 result = (IOService *) iter->getNextObject();
492 result->retain();
493 iter->release();
494 }
495 }
496 while (false);
497 matching->release();
498
499 return (result);
500}
501
502#define APFSMEDIA_GETHIBERKEY "getHiberKey"
503
504static IOReturn
505IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID,
506 uint8_t * volumeCryptKey, size_t * keySize)
507{
508 IOReturn err;
509 IOService * part;
510 OSString * keyUUID = 0;
511 OSString * keyStoreUUID = 0;
512 uuid_t volumeKeyUUID;
513 aks_volume_key_t vek;
514 size_t callerKeySize;
515
516 static IOService * sKeyStore;
517
518 part = IOCopyMediaForDev(block_dev);
519 if (!part) return (kIOReturnNotFound);
520
521 callerKeySize = *keySize;
522 // Try APFS first
523 {
524 uuid_t volUuid = {0};
525 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
526 if (kIOReturnBadArgument == err)
527 {
528 // apfs fails on buffer size >32
529 *keySize = 32;
530 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
531 }
532 if (err != kIOReturnSuccess) *keySize = 0;
533 else
534 {
535 // No need to create uuid string if it's not requested
536 if (pKeyUUID)
537 {
538 uuid_string_t volUuidStr;
539 uuid_unparse(volUuid, volUuidStr);
540 *pKeyUUID = OSString::withCString(volUuidStr);
541 }
542
543 part->release();
544 return kIOReturnSuccess;
545 }
546 }
547
548 // Then old CS path
549 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
550 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
551 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
552 {
553// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
554
555 if (!sKeyStore)
556 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
557 if (sKeyStore)
558 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
559 else
560 err = kIOReturnNoResources;
561 if (kIOReturnSuccess == err)
562 err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
563 if (kIOReturnSuccess != err)
564 IOLog("volume key err 0x%x\n", err);
565 else
566 {
567 if (vek.key.keybytecount <= callerKeySize) *keySize = vek.key.keybytecount;
568 bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
569 }
570 bzero(&vek, sizeof(vek));
571
572 if (pKeyUUID)
573 {
574 // Create a copy because the caller would release it
575 *pKeyUUID = OSString::withString(keyUUID);
576 }
577 }
578
579 part->release();
580 return (err);
581}
582
583/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
584
585IOReturn
586IOPolledFileOpen(const char * filename,
587 uint32_t flags,
588 uint64_t setFileSize, uint64_t fsFreeSize,
589 void * write_file_addr, size_t write_file_len,
590 IOPolledFileIOVars ** fileVars,
591 OSData ** imagePath,
592 uint8_t * volumeCryptKey, size_t * keySize)
593{
594 IOReturn err = kIOReturnSuccess;
595 IOPolledFileIOVars * vars;
596 _OpenFileContext ctx;
597 OSData * extentsData = NULL;
598 OSNumber * num;
599 IOService * part = 0;
600 dev_t block_dev;
601 dev_t image_dev;
602 AbsoluteTime startTime, endTime;
603 uint64_t nsec;
604
605 vars = IONew(IOPolledFileIOVars, 1);
606 if (!vars) return (kIOReturnNoMemory);
607 bzero(vars, sizeof(*vars));
608 vars->allocated = true;
609
610 do
611 {
612 extentsData = OSData::withCapacity(32);
613 ctx.extents = extentsData;
614 ctx.size = 0;
615 clock_get_uptime(&startTime);
616
617 vars->fileRef = kern_open_file_for_direct_io(filename,
618 flags,
619 &file_extent_callback, &ctx,
620 setFileSize,
621 fsFreeSize,
622 // write file:
623 0, write_file_addr, write_file_len,
624 // results
625 &block_dev,
626 &image_dev,
627 &vars->block0,
628 &vars->maxiobytes,
629 &vars->flags);
630#if 0
631 uint32_t msDelay = (131071 & random());
632 HIBLOG("sleep %d\n", msDelay);
633 IOSleep(msDelay);
634#endif
635 clock_get_uptime(&endTime);
636 SUB_ABSOLUTETIME(&endTime, &startTime);
637 absolutetime_to_nanoseconds(endTime, &nsec);
638
639 if (!vars->fileRef) err = kIOReturnNoSpace;
640
641 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
642 if (kIOReturnSuccess != err) break;
643
644 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
645 (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
646 vars->maxiobytes, kIOPolledFileSSD & vars->flags);
647 assert(!vars->block0);
648 if (extentsData->getLength() < sizeof(IOPolledFileExtent))
649 {
650 err = kIOReturnNoSpace;
651 break;
652 }
653
654 vars->fileSize = ctx.size;
655 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
656
657 part = IOCopyMediaForDev(image_dev);
658 if (!part)
659 {
660 err = kIOReturnNotFound;
661 break;
662 }
663
664 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
665
666 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
667 vars->blockSize = num->unsigned32BitValue();
668 if (vars->blockSize < 4096) vars->blockSize = 4096;
669
670 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
671 major(image_dev), minor(image_dev), (long)vars->blockSize,
672 vars->pollers->pollers->getCount());
673
674 OSString * keyUUID = NULL;
675 if (volumeCryptKey)
676 {
677 err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
678 }
679
680 *fileVars = vars;
681 vars->fileExtents = extentsData;
682
683 // make imagePath
684 OSData * data;
685 if (imagePath)
686 {
687#if defined(__i386__) || defined(__x86_64__)
688 char str2[24 + sizeof(uuid_string_t) + 2];
689
690 if (keyUUID)
691 snprintf(str2, sizeof(str2), "%qx:%s",
692 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
693 else
694 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
695
696 err = IOService::getPlatform()->callPlatformFunction(
697 gIOCreateEFIDevicePathSymbol, false,
698 (void *) part, (void *) str2,
699 (void *) (uintptr_t) true, (void *) &data);
700#else
701 data = 0;
702 err = kIOReturnSuccess;
703#endif
704 if (kIOReturnSuccess != err)
705 {
706 HIBLOG("error 0x%x getting path\n", err);
707 break;
708 }
709 *imagePath = data;
710 }
711
712 // Release key UUID if we have one
713 if (keyUUID)
714 {
715 keyUUID->release();
716 keyUUID = NULL; // Just in case
717 }
718 }
719 while (false);
720
721 if (kIOReturnSuccess != err)
722 {
723 HIBLOG("error 0x%x opening polled file\n", err);
724 IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
725 if (extentsData) extentsData->release();
726 }
727
728 if (part) part->release();
729
730 return (err);
731}
732
733/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
734
735IOReturn
736IOPolledFileClose(IOPolledFileIOVars ** pVars,
737 off_t write_offset, void * addr, size_t write_length,
738 off_t discard_offset, off_t discard_end)
739{
740 IOPolledFileIOVars * vars;
741
742 vars = *pVars;
743 if (!vars) return(kIOReturnSuccess);
744
745 if (vars->fileRef)
746 {
747 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
748 discard_offset, discard_end);
749 vars->fileRef = NULL;
750 }
751 if (vars->fileExtents)
752 {
753 vars->fileExtents->release();
754 vars->fileExtents = 0;
755 }
756 if (vars->pollers)
757 {
758 vars->pollers->release();
759 vars->pollers = 0;
760 }
761
762 if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
763 else bzero(vars, sizeof(IOPolledFileIOVars));
764 *pVars = NULL;
765
766 return (kIOReturnSuccess);
767}
768
769/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
770
771IOReturn
772IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
773 uint32_t openState)
774{
775 IOReturn err;
776
777 err = kIOReturnSuccess;
778 do
779 {
780 if (!vars->pollers->openCount)
781 {
782 err = IOPolledFilePollersProbe(vars->pollers);
783 if (kIOReturnSuccess != err) break;
784 }
785 err = IOPolledFilePollersOpen(vars, openState, false);
786 if (kIOReturnSuccess != err) break;
787 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState))
788 {
789 vars->pollers->openCount++;
790 }
791 vars->pollers->io = false;
792 vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
793 vars->bufferHalf = 0;
794 vars->bufferOffset = 0;
795 vars->bufferSize = (vars->pollers->ioBuffer->getLength() >> 1);
796
797 if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
798 }
799 while (false);
800
801 if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
802
803 return (err);
804}
805
806
807/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
808
809IOReturn
810IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
811{
812 IOPolledFileExtent * extentMap;
813
814 extentMap = vars->extentMap;
815
816 vars->position = position;
817
818 if (position > vars->fileSize) {
819 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
820 return kIOReturnNoSpace;
821 }
822
823 while (position >= extentMap->length)
824 {
825 position -= extentMap->length;
826 extentMap++;
827 }
828
829 vars->currentExtent = extentMap;
830 vars->extentRemaining = extentMap->length - position;
831 vars->extentPosition = vars->position - position;
832
833 if (vars->bufferSize <= vars->extentRemaining)
834 vars->bufferLimit = vars->bufferSize;
835 else
836 vars->bufferLimit = vars->extentRemaining;
837
838 return (kIOReturnSuccess);
839}
840
841/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
842
843IOReturn
844IOPolledFileWrite(IOPolledFileIOVars * vars,
845 const uint8_t * bytes, IOByteCount size,
846 IOPolledFileCryptVars * cryptvars)
847{
848 IOReturn err = kIOReturnSuccess;
849 IOByteCount copy, original_size = size;
850 bool flush = false;
851
852 do
853 {
854 if (!bytes && !size)
855 {
856 // seek to end of block & flush
857 size = vars->position & (vars->blockSize - 1);
858 if (size)
859 size = vars->blockSize - size;
860 flush = true;
861 // use some garbage for the fill
862 bytes = vars->buffer + vars->bufferOffset;
863 }
864
865 copy = vars->bufferLimit - vars->bufferOffset;
866 if (copy > size)
867 copy = size;
868 else
869 flush = true;
870
871 if (bytes)
872 {
873#if KASAN
874 /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
875 * avoid triggering global redzone sanitizer violations when accessing
876 * interstices between 'C' structures
877 */
878 __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
879#else
880 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
881#endif
882 bytes += copy;
883 }
884 else
885 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
886
887 size -= copy;
888 vars->bufferOffset += copy;
889 vars->position += copy;
890
891 if (flush && vars->bufferOffset)
892 {
893 uint64_t offset = (vars->position - vars->bufferOffset
894 - vars->extentPosition + vars->currentExtent->start);
895 uint32_t length = (vars->bufferOffset);
896
897#if CRYPTO
898 if (cryptvars && vars->encryptStart
899 && (vars->position > vars->encryptStart)
900 && ((vars->position - length) < vars->encryptEnd))
901 {
902 AbsoluteTime startTime, endTime;
903
904 uint64_t encryptLen, encryptStart;
905 encryptLen = vars->position - vars->encryptStart;
906 if (encryptLen > length)
907 encryptLen = length;
908 encryptStart = length - encryptLen;
909 if (vars->position > vars->encryptEnd)
910 encryptLen -= (vars->position - vars->encryptEnd);
911
912 clock_get_uptime(&startTime);
913
914 // encrypt the buffer
915 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
916 &cryptvars->aes_iv[0],
917 encryptLen / AES_BLOCK_SIZE,
918 vars->buffer + vars->bufferHalf + encryptStart,
919 &cryptvars->ctx.encrypt);
920
921 clock_get_uptime(&endTime);
922 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
923 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
924 vars->cryptBytes += encryptLen;
925
926 // save initial vector for following encrypts
927 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
928 &cryptvars->aes_iv[0],
929 AES_BLOCK_SIZE);
930 }
931#endif /* CRYPTO */
932
933 err = IOPolledFilePollersIODone(vars->pollers, true);
934 if (kIOReturnSuccess != err)
935 break;
936
937if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
938//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
939
940 err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
941 if (kIOReturnSuccess != err) {
942 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
943 vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
944 break;
945 }
946 vars->pollers->io = true;
947
948 vars->extentRemaining -= vars->bufferOffset;
949 if (!vars->extentRemaining)
950 {
951 vars->currentExtent++;
952 vars->extentRemaining = vars->currentExtent->length;
953 vars->extentPosition = vars->position;
954 }
955
956 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
957 vars->bufferOffset = 0;
958 if (vars->bufferSize <= vars->extentRemaining)
959 vars->bufferLimit = vars->bufferSize;
960 else
961 vars->bufferLimit = vars->extentRemaining;
962
963 if (!vars->extentRemaining)
964 {
965 err = kIOReturnOverrun;
966 break;
967 }
968
969 flush = false;
970 }
971 }
972 while (size);
973
974 return (err);
975}
976
977/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
978
979IOReturn
980IOPolledFileFlush(IOPolledFileIOVars * vars)
981{
982 // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
983 IOReturn err = kIOReturnSuccess;
984
985 err = IOPolledFilePollersIODone(vars->pollers, true);
986 if (kIOReturnSuccess != err)
987 return err;
988
989 err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
990 if (kIOReturnSuccess != err) {
991 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
992 vars, vars->pollers, err);
993 return err;
994 }
995 vars->pollers->io = true;
996
997 return err;
998}
999
1000/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1001
1002IOReturn
1003IOPolledFileRead(IOPolledFileIOVars * vars,
1004 uint8_t * bytes, IOByteCount size,
1005 IOPolledFileCryptVars * cryptvars)
1006{
1007 IOReturn err = kIOReturnSuccess;
1008 IOByteCount copy;
1009
1010// bytesWritten += size;
1011
1012 do
1013 {
1014 copy = vars->bufferLimit - vars->bufferOffset;
1015 if (copy > size)
1016 copy = size;
1017
1018 if (bytes)
1019 {
1020#if KASAN
1021 __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1022#else
1023 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1024#endif
1025 bytes += copy;
1026 }
1027 size -= copy;
1028 vars->bufferOffset += copy;
1029// vars->position += copy;
1030
1031 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1032 {
1033 if (!vars->pollers->io) cryptvars = 0;
1034 err = IOPolledFilePollersIODone(vars->pollers, true);
1035 if (kIOReturnSuccess != err)
1036 break;
1037
1038if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1039
1040 vars->position += vars->lastRead;
1041 vars->extentRemaining -= vars->lastRead;
1042 vars->bufferLimit = vars->lastRead;
1043
1044 if (!vars->extentRemaining)
1045 {
1046 vars->currentExtent++;
1047 vars->extentRemaining = vars->currentExtent->length;
1048 vars->extentPosition = vars->position;
1049 if (!vars->extentRemaining)
1050 {
1051 err = kIOReturnOverrun;
1052 break;
1053 }
1054 }
1055
1056 uint64_t length;
1057 uint64_t lastReadLength = vars->lastRead;
1058 uint64_t offset = (vars->position
1059 - vars->extentPosition + vars->currentExtent->start);
1060 if (vars->extentRemaining <= vars->bufferSize)
1061 length = vars->extentRemaining;
1062 else
1063 length = vars->bufferSize;
1064 if ((length + vars->position) > vars->readEnd)
1065 length = vars->readEnd - vars->position;
1066
1067 vars->lastRead = length;
1068 if (length)
1069 {
1070//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1071 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1072 if (kIOReturnSuccess != err)
1073 break;
1074 vars->pollers->io = true;
1075 }
1076
1077 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1078 vars->bufferOffset = 0;
1079
1080#if CRYPTO
1081 if (cryptvars)
1082 {
1083 uint8_t thisVector[AES_BLOCK_SIZE];
1084 AbsoluteTime startTime, endTime;
1085
1086 // save initial vector for following decrypts
1087 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1088 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1089 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1090
1091 // decrypt the buffer
1092 clock_get_uptime(&startTime);
1093
1094 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1095 &thisVector[0],
1096 lastReadLength / AES_BLOCK_SIZE,
1097 vars->buffer + vars->bufferHalf,
1098 &cryptvars->ctx.decrypt);
1099
1100 clock_get_uptime(&endTime);
1101 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1102 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1103 vars->cryptBytes += lastReadLength;
1104 }
1105#endif /* CRYPTO */
1106 }
1107 }
1108 while (size);
1109
1110 return (err);
1111}
1112
1113/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1114
1115