1/*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30
31Sleep:
32
33- PMRootDomain calls IOHibernateSystemSleep() before system sleep
34(devices awake, normal execution context)
35- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48- Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71Boot/Wake:
72
73- BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99Polled Mode I/O:
100
101IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102registry, specifying an object of calls IOPolledInterface.
103
104Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105partition) that the image is going to live, looking for polled interface properties. If it finds
106one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107interfaces found are kept in an ordered list.
108
109There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110few different contexts things happen in:
111
112- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113up and running) and after wake - this is safe to allocate memory and do anything. The device
114ignores sleep requests from that point since its a waste of time if it goes to sleep and
115immediately wakes back up for the image write.
116
117- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120used to flush and set the disk to sleep.
121
122- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123immediately after sleep. These can't block or allocate memory. This is happening after the platform
124expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125one thread running.
126
127For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131that is called for the hardware to check for events, and complete the I/O via the callback.
132The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133to restrict I/O ops.
134*/
135
136#include <sys/systm.h>
137
138#include <IOKit/IOWorkLoop.h>
139#include <IOKit/IOCommandGate.h>
140#include <IOKit/IOTimerEventSource.h>
141#include <IOKit/IOPlatformExpert.h>
142#include <IOKit/IOKitDebug.h>
143#include <IOKit/IOTimeStamp.h>
144#include <IOKit/pwr_mgt/RootDomain.h>
145#include <IOKit/pwr_mgt/IOPMPrivate.h>
146#include <IOKit/IOMessage.h>
147#include <IOKit/IODeviceTreeSupport.h>
148#include <IOKit/IOBSD.h>
149#include <IOKit/IOKitKeysPrivate.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
154#include <IOKit/AppleKeyStoreInterface.h>
155#include <libkern/crypto/aes.h>
156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
161#include <sys/sysctl.h>
162#include <sys/kdebug.h>
163#include <stdint.h>
164
165#include <IOKit/IOHibernatePrivate.h>
166#include <IOKit/IOPolledInterface.h>
167#include <IOKit/IONVRAM.h>
168#include "IOHibernateInternal.h"
169#include <vm/WKdm_new.h>
170#include <vm/vm_protos.h>
171#include "IOKitKernelInternal.h"
172#include <pexpert/device_tree.h>
173
174#include <machine/pal_routines.h>
175#include <machine/pal_hibernate.h>
176#include <i386/tsc.h>
177#include <i386/cpuid.h>
178#include <san/kasan.h>
179
180extern "C" addr64_t kvtophys(vm_offset_t va);
181extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
182
183/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
184
185#define DISABLE_TRIM 0
186#define TRIM_DELAY 25000
187
188extern unsigned int save_kdebug_enable;
189extern uint32_t gIOHibernateState;
190uint32_t gIOHibernateMode;
191static char gIOHibernateBootSignature[256+1];
192static char gIOHibernateFilename[MAXPATHLEN+1];
193
194static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
195
196static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
198static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
199boolean_t gIOHibernateStandbyDisabled;
200
201static IODTNVRAM * gIOOptionsEntry;
202static IORegistryEntry * gIOChosenEntry;
203
204static const OSSymbol * gIOHibernateBootImageKey;
205static const OSSymbol * gIOHibernateBootSignatureKey;
206static const OSSymbol * gIOBridgeBootSessionUUIDKey;
207
208#if defined(__i386__) || defined(__x86_64__)
209
210static const OSSymbol * gIOHibernateRTCVariablesKey;
211static const OSSymbol * gIOHibernateBoot0082Key;
212static const OSSymbol * gIOHibernateBootNextKey;
213static OSData * gIOHibernateBoot0082Data;
214static OSData * gIOHibernateBootNextData;
215static OSObject * gIOHibernateBootNextSave;
216
217#endif /* defined(__i386__) || defined(__x86_64__) */
218
219static IOLock * gFSLock;
220 uint32_t gFSState;
221static thread_call_t gIOHibernateTrimCalloutEntry;
222static IOPolledFileIOVars gFileVars;
223static IOHibernateVars gIOHibernateVars;
224static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
225static hibernate_graphics_t _hibernateGraphics;
226static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
227static hibernate_statistics_t _hibernateStats;
228static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
229
230enum
231{
232 kFSIdle = 0,
233 kFSOpening = 2,
234 kFSOpened = 3,
235 kFSTimedOut = 4,
236 kFSTrimDelay = 5
237};
238
239static IOReturn IOHibernateDone(IOHibernateVars * vars);
240static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
241static void IOSetBootImageNVRAM(OSData * data);
242static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
243
244/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
245
246enum { kDefaultIOSize = 128 * 1024 };
247enum { kVideoMapSize = 80 * 1024 * 1024 };
248
249/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
250
251// copy from phys addr to MD
252
253static IOReturn
254IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
255 IOByteCount offset, addr64_t bytes, IOByteCount length)
256{
257 addr64_t srcAddr = bytes;
258 IOByteCount remaining;
259
260 remaining = length = min(length, md->getLength() - offset);
261 while (remaining) { // (process another target segment?)
262 addr64_t dstAddr64;
263 IOByteCount dstLen;
264
265 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
266 if (!dstAddr64)
267 break;
268
269 // Clip segment length to remaining
270 if (dstLen > remaining)
271 dstLen = remaining;
272
273#if 1
274 bcopy_phys(srcAddr, dstAddr64, dstLen);
275#else
276 copypv(srcAddr, dstAddr64, dstLen,
277 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
278#endif
279 srcAddr += dstLen;
280 offset += dstLen;
281 remaining -= dstLen;
282 }
283
284 assert(!remaining);
285
286 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
287}
288
289// copy from MD to phys addr
290
291static IOReturn
292IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
293 IOByteCount offset, addr64_t bytes, IOByteCount length)
294{
295 addr64_t dstAddr = bytes;
296 IOByteCount remaining;
297
298 remaining = length = min(length, md->getLength() - offset);
299 while (remaining) { // (process another target segment?)
300 addr64_t srcAddr64;
301 IOByteCount dstLen;
302
303 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
304 if (!srcAddr64)
305 break;
306
307 // Clip segment length to remaining
308 if (dstLen > remaining)
309 dstLen = remaining;
310
311#if 1
312 bcopy_phys(srcAddr64, dstAddr, dstLen);
313#else
314 copypv(srcAddr, dstAddr64, dstLen,
315 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
316#endif
317 dstAddr += dstLen;
318 offset += dstLen;
319 remaining -= dstLen;
320 }
321
322 assert(!remaining);
323
324 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
325}
326
327/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
328
329void
330hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
331 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
332{
333 count += ppnum;
334 switch (kind)
335 {
336 case kIOHibernatePageStateUnwiredSave:
337 // unwired save
338 for (; ppnum < count; ppnum++)
339 {
340 hibernate_page_bitset(page_list, FALSE, ppnum);
341 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
342 }
343 break;
344 case kIOHibernatePageStateWiredSave:
345 // wired save
346 for (; ppnum < count; ppnum++)
347 {
348 hibernate_page_bitset(page_list, FALSE, ppnum);
349 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
350 }
351 break;
352 case kIOHibernatePageStateFree:
353 // free page
354 for (; ppnum < count; ppnum++)
355 {
356 hibernate_page_bitset(page_list, TRUE, ppnum);
357 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
358 }
359 break;
360 default:
361 panic("hibernate_set_page_state");
362 }
363}
364
365static vm_offset_t
366hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
367{
368 uint32_t page = *pPage;
369 uint32_t count;
370 hibernate_bitmap_t * bitmap;
371
372 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
373 {
374 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
375 if (!count)
376 break;
377 page += count;
378 if (page <= bitmap->last_page)
379 break;
380 }
381
382 *pPage = page;
383 if (bitmap)
384 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
385 else
386 count = 0;
387
388 return (count);
389}
390
391/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
392
393IOReturn
394IOHibernateSystemSleep(void)
395{
396 IOReturn err;
397 OSData * nvramData;
398 OSObject * obj;
399 OSString * str;
400 OSNumber * num;
401 bool dsSSD, vmflush, swapPinned;
402 IOHibernateVars * vars;
403 uint64_t setFileSize = 0;
404
405 gIOHibernateState = kIOHibernateStateInactive;
406
407 gIOHibernateDebugFlags = 0;
408 if (kIOLogHibernate & gIOKitDebug)
409 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
410
411 if (IOService::getPMRootDomain()->getHibernateSettings(
412 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
413 {
414 if (kIOHibernateModeSleep & gIOHibernateMode)
415 // default to discard clean for safe sleep
416 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
417 | kIOHibernateModeDiscardCleanActive);
418 }
419
420 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
421 {
422 if ((str = OSDynamicCast(OSString, obj)))
423 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
424 sizeof(gIOHibernateFilename));
425 obj->release();
426 }
427
428 if (!gIOHibernateMode || !gIOHibernateFilename[0])
429 return (kIOReturnUnsupported);
430
431 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
432
433 vars = IONew(IOHibernateVars, 1);
434 if (!vars) return (kIOReturnNoMemory);
435 bzero(vars, sizeof(*vars));
436
437 IOLockLock(gFSLock);
438 if (!gIOHibernateTrimCalloutEntry)
439 {
440 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
441 }
442 IOHibernateSystemPostWakeTrim(NULL, NULL);
443 thread_call_cancel(gIOHibernateTrimCalloutEntry);
444 if (kFSIdle != gFSState)
445 {
446 HIBLOG("hibernate file busy\n");
447 IOLockUnlock(gFSLock);
448 IODelete(vars, IOHibernateVars, 1);
449 return (kIOReturnBusy);
450 }
451 gFSState = kFSOpening;
452 IOLockUnlock(gFSLock);
453
454 swapPinned = false;
455 do
456 {
457 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
458 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
459
460 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
461 ptoa_64(gIOHibernateHandoffPageCount), page_size);
462
463 if (!vars->srcBuffer || !vars->handoffBuffer)
464 {
465 err = kIOReturnNoMemory;
466 break;
467 }
468
469 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
470 {
471 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
472 obj->release();
473 }
474 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
475 {
476 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
477 obj->release();
478 }
479
480 boolean_t encryptedswap = true;
481 uint32_t pageCount;
482 AbsoluteTime startTime, endTime;
483 uint64_t nsec;
484
485 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
486 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
487 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
488
489 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
490 err = hibernate_alloc_page_lists(&vars->page_list,
491 &vars->page_list_wired,
492 &vars->page_list_pal);
493 if (KERN_SUCCESS != err) break;
494
495 err = hibernate_pin_swap(TRUE);
496 if (KERN_SUCCESS != err) break;
497 swapPinned = true;
498
499 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
500 {
501 hibernate_page_list_setall(vars->page_list,
502 vars->page_list_wired,
503 vars->page_list_pal,
504 true /* preflight */,
505 vmflush /* discard */,
506 &pageCount);
507 PE_Video consoleInfo;
508 bzero(&consoleInfo, sizeof(consoleInfo));
509 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
510
511 // estimate: 6% increase in pages compressed
512 // screen preview 2 images compressed 0%
513 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
514 + vars->page_list->list_size
515 + (consoleInfo.v_width * consoleInfo.v_height * 8);
516 enum { setFileRound = 1024*1024ULL };
517 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
518
519 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
520 pageCount, (100ULL * gIOHibernateCompression) >> 8,
521 setFileSize, vars->fileMinSize);
522
523 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
524 && (setFileSize < vars->fileMinSize))
525 {
526 setFileSize = vars->fileMinSize;
527 }
528 }
529
530 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
531 err = IOPolledFileOpen(gIOHibernateFilename,
532 (kIOPolledFileCreate | kIOPolledFileHibernate),
533 setFileSize, 0,
534 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
535 &vars->fileVars, &nvramData,
536 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
537
538 if (KERN_SUCCESS != err)
539 {
540 IOLockLock(gFSLock);
541 if (kFSOpening != gFSState) err = kIOReturnTimeout;
542 IOLockUnlock(gFSLock);
543 }
544
545 if (KERN_SUCCESS != err)
546 {
547 HIBLOG("IOPolledFileOpen(%x)\n", err);
548 break;
549 }
550
551 // write extents for debug data usage in EFI
552 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
553
554 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
555 if (KERN_SUCCESS != err) break;
556
557 clock_get_uptime(&startTime);
558 err = hibernate_setup(gIOHibernateCurrentHeader,
559 vmflush,
560 vars->page_list, vars->page_list_wired, vars->page_list_pal);
561 clock_get_uptime(&endTime);
562 SUB_ABSOLUTETIME(&endTime, &startTime);
563 absolutetime_to_nanoseconds(endTime, &nsec);
564
565 boolean_t haveSwapPin, hibFileSSD;
566 haveSwapPin = vm_swap_files_pinned();
567
568 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
569
570 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
571 err, nsec / 1000000ULL,
572 haveSwapPin, hibFileSSD);
573 if (KERN_SUCCESS != err) break;
574
575 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
576
577 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
578 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
579
580 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
581 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
582
583
584#if defined(__i386__) || defined(__x86_64__)
585 if (vars->volumeCryptKeySize &&
586 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
587 {
588 uintptr_t smcVars[2];
589 smcVars[0] = vars->volumeCryptKeySize;
590 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
591
592 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
593 bzero(smcVars, sizeof(smcVars));
594 }
595#endif
596
597
598 if (encryptedswap || vars->volumeCryptKeySize)
599 gIOHibernateMode ^= kIOHibernateModeEncrypt;
600
601 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
602 {
603 vars->videoAllocSize = kVideoMapSize;
604 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
605 vars->videoMapping = 0;
606 }
607
608 // generate crypt keys
609 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
610 vars->wiredCryptKey[i] = random();
611 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
612 vars->cryptKey[i] = random();
613
614 // set nvram
615
616 IOSetBootImageNVRAM(nvramData);
617 nvramData->release();
618
619#if defined(__i386__) || defined(__x86_64__)
620 {
621 struct AppleRTCHibernateVars
622 {
623 uint8_t signature[4];
624 uint32_t revision;
625 uint8_t booterSignature[20];
626 uint8_t wiredCryptKey[16];
627 };
628 AppleRTCHibernateVars rtcVars;
629 OSData * data;
630
631 rtcVars.signature[0] = 'A';
632 rtcVars.signature[1] = 'A';
633 rtcVars.signature[2] = 'P';
634 rtcVars.signature[3] = 'L';
635 rtcVars.revision = 1;
636 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
637
638 if (gIOChosenEntry
639 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
640 && (sizeof(rtcVars.booterSignature) <= data->getLength()))
641 {
642 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
643 }
644 else if (gIOHibernateBootSignature[0])
645 {
646 char c;
647 uint8_t value = 0;
648 uint32_t in, out, digits;
649 for (in = out = digits = 0;
650 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
651 in++)
652 {
653 if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10;
654 else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10;
655 else if ((c >= '0') && (c <= '9')) c -= '0';
656 else
657 {
658 if (c == '=') out = digits = value = 0;
659 continue;
660 }
661 value = (value << 4) | c;
662 if (digits & 1)
663 {
664 rtcVars.booterSignature[out++] = value;
665 if (out >= sizeof(rtcVars.booterSignature)) break;
666 }
667 digits++;
668 }
669 }
670#if DEBUG || DEVELOPMENT
671 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> rtc:",
672 &rtcVars, sizeof(rtcVars), &kprintf);
673#endif /* DEBUG || DEVELOPMENT */
674
675 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
676 if (data)
677 {
678 if (gIOHibernateRTCVariablesKey)
679 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
680 data->release();
681 }
682 if (gIOChosenEntry && gIOOptionsEntry)
683 {
684 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
685 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
686 // set BootNext
687 if (!gIOHibernateBoot0082Data)
688 {
689 OSData * fileData = 0;
690 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
691 if (data && data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
692 if (data)
693 {
694 // AppleNVRAM_EFI_LOAD_OPTION
695 struct {
696 uint32_t Attributes;
697 uint16_t FilePathLength;
698 uint16_t Desc;
699 } loadOptionHeader;
700 loadOptionHeader.Attributes = 1;
701 loadOptionHeader.FilePathLength = data->getLength();
702 loadOptionHeader.Desc = 0;
703 if (fileData)
704 {
705 loadOptionHeader.FilePathLength -= 4;
706 loadOptionHeader.FilePathLength += fileData->getLength();
707 }
708 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
709 if (gIOHibernateBoot0082Data)
710 {
711 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
712 if (fileData)
713 {
714 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
715 gIOHibernateBoot0082Data->appendBytes(fileData);
716 }
717 else gIOHibernateBoot0082Data->appendBytes(data);
718 }
719 }
720 }
721 if (!gIOHibernateBootNextData)
722 {
723 uint16_t bits = 0x0082;
724 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
725 }
726
727#if DEBUG || DEVELOPMENT
728 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> bootnext:",
729 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
730#endif /* DEBUG || DEVELOPMENT */
731 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
732 {
733 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
734 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
735 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
736 }
737 // BootNext
738 }
739 }
740#endif /* !i386 && !x86_64 */
741 }
742 while (false);
743
744 if (swapPinned) hibernate_pin_swap(FALSE);
745
746 IOLockLock(gFSLock);
747 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
748 {
749 HIBLOG("hibernate file close due timeout\n");
750 err = kIOReturnTimeout;
751 }
752 if (kIOReturnSuccess == err)
753 {
754 gFSState = kFSOpened;
755 gIOHibernateVars = *vars;
756 gFileVars = *vars->fileVars;
757 gFileVars.allocated = false;
758 gIOHibernateVars.fileVars = &gFileVars;
759 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
760 gIOHibernateState = kIOHibernateStateHibernating;
761
762#if DEBUG || DEVELOPMENT
763 if (kIOLogHibernate & gIOKitDebug)
764 {
765 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
766 if (data)
767 {
768 uintptr_t * smcVars = (typeof(smcVars)) data->getBytesNoCopy();
769 IOKitKernelLogBuffer("H> smc:",
770 (const void *)smcVars[1], smcVars[0], &kprintf);
771 }
772 }
773#endif /* DEBUG || DEVELOPMENT */
774 }
775 else
776 {
777 IOPolledFileIOVars * fileVars = vars->fileVars;
778 IOHibernateDone(vars);
779 IOPolledFileClose(&fileVars,
780#if DISABLE_TRIM
781 0, NULL, 0, 0, 0);
782#else
783 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
784#endif
785 gFSState = kFSIdle;
786 }
787 IOLockUnlock(gFSLock);
788
789 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
790 IODelete(vars, IOHibernateVars, 1);
791
792 return (err);
793}
794
795/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
796
797static void
798IOSetBootImageNVRAM(OSData * data)
799{
800 IORegistryEntry * regEntry;
801
802 if (!gIOOptionsEntry)
803 {
804 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
805 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
806 if (regEntry && !gIOOptionsEntry)
807 regEntry->release();
808 }
809 if (gIOOptionsEntry && gIOHibernateBootImageKey)
810 {
811 if (data)
812 {
813 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
814#if DEBUG || DEVELOPMENT
815 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> boot-image:",
816 data->getBytesNoCopy(), data->getLength(), &kprintf);
817#endif /* DEBUG || DEVELOPMENT */
818 }
819 else
820 {
821 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
822 gIOOptionsEntry->sync();
823 }
824 }
825}
826
827/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
828/*
829 * Writes header to disk with signature, block size and file extents data.
830 * If there are more than 2 extents, then they are written on second block.
831 */
832static IOReturn
833IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
834{
835 IOHibernateImageHeader hdr;
836 IOItemCount count;
837 IOReturn err = kIOReturnSuccess;
838 int rc;
839 IOPolledFileExtent * fileExtents;
840
841 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
842
843 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
844 count = vars->fileExtents->getLength();
845 if (count > sizeof(hdr.fileExtentMap))
846 {
847 hdr.fileExtentMapSize = count;
848 count = sizeof(hdr.fileExtentMap);
849 }
850 else
851 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
852
853 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
854
855 // copy file block extent list if larger than header
856 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
857 {
858 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
859 rc = kern_write_file(vars->fileRef, vars->blockSize,
860 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
861 count, IO_SKIP_ENCRYPTION);
862 if (rc != 0) {
863 HIBLOG("kern_write_file returned %d\n", rc);
864 err = kIOReturnIOError;
865 goto exit;
866 }
867 }
868 hdr.signature = signature;
869 hdr.deviceBlockSize = vars->blockSize;
870
871 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
872 if (rc != 0) {
873 HIBLOG("kern_write_file returned %d\n", rc);
874 err = kIOReturnIOError;
875 goto exit;
876 }
877
878exit:
879 return err;
880}
881
882/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
883
884DECLARE_IOHIBERNATEPROGRESSALPHA
885
886static void
887ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
888{
889 uint32_t rowBytes, pixelShift;
890 uint32_t x, y;
891 int32_t blob;
892 uint32_t alpha, in, color, result;
893 uint8_t * out;
894 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
895
896 rowBytes = display->rowBytes;
897 pixelShift = display->depth >> 4;
898 if (pixelShift < 1) return;
899
900 screen += ((display->width
901 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
902 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
903
904 for (y = 0; y < kIOHibernateProgressHeight; y++)
905 {
906 out = screen + y * rowBytes;
907 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
908 {
909 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
910 for (x = 0; x < kIOHibernateProgressWidth; x++)
911 {
912 alpha = gIOHibernateProgressAlpha[y][x];
913 result = color;
914 if (alpha)
915 {
916 if (0xff != alpha)
917 {
918 if (1 == pixelShift)
919 {
920 in = *((uint16_t *)out) & 0x1f; // 16
921 in = (in << 3) | (in >> 2);
922 }
923 else
924 in = *((uint32_t *)out) & 0xff; // 32
925 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
926 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
927 }
928 if (1 == pixelShift)
929 {
930 result >>= 3;
931 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
932 }
933 else
934 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
935 }
936 out += (1 << pixelShift);
937 }
938 out += (kIOHibernateProgressSpacing << pixelShift);
939 }
940 }
941}
942
943
944static void
945ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
946{
947 uint32_t rowBytes, pixelShift;
948 uint32_t x, y;
949 int32_t blob, lastBlob;
950 uint32_t alpha, in, color, result;
951 uint8_t * out;
952 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
953
954 pixelShift = display->depth >> 4;
955 if (pixelShift < 1)
956 return;
957
958 rowBytes = display->rowBytes;
959
960 screen += ((display->width
961 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
962 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
963
964 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
965
966 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
967
968 for (y = 0; y < kIOHibernateProgressHeight; y++)
969 {
970 out = screen + y * rowBytes;
971 for (blob = firstBlob; blob <= lastBlob; blob++)
972 {
973 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
974 for (x = 0; x < kIOHibernateProgressWidth; x++)
975 {
976 alpha = gIOHibernateProgressAlpha[y][x];
977 result = color;
978 if (alpha)
979 {
980 if (0xff != alpha)
981 {
982 in = display->progressSaveUnder[blob][saveindex[blob]++];
983 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
984 }
985 if (1 == pixelShift)
986 {
987 result >>= 3;
988 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
989 }
990 else
991 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
992 }
993 out += (1 << pixelShift);
994 }
995 out += (kIOHibernateProgressSpacing << pixelShift);
996 }
997 }
998}
999
1000/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1001
1002IOReturn
1003IOHibernateIOKitSleep(void)
1004{
1005 IOReturn ret = kIOReturnSuccess;
1006 IOLockLock(gFSLock);
1007 if (kFSOpening == gFSState)
1008 {
1009 gFSState = kFSTimedOut;
1010 HIBLOG("hibernate file open timed out\n");
1011 ret = kIOReturnTimeout;
1012 }
1013 IOLockUnlock(gFSLock);
1014 return (ret);
1015}
1016
1017/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1018
1019IOReturn
1020IOHibernateSystemHasSlept(void)
1021{
1022 IOReturn ret = kIOReturnSuccess;
1023 IOHibernateVars * vars = &gIOHibernateVars;
1024 OSObject * obj = 0;
1025 OSData * data;
1026
1027 IOLockLock(gFSLock);
1028 if ((kFSOpened != gFSState) && gIOHibernateMode)
1029 {
1030 ret = kIOReturnTimeout;
1031 }
1032 IOLockUnlock(gFSLock);
1033 if (kIOReturnSuccess != ret) return (ret);
1034
1035 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1036 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1037 if (obj && !vars->previewBuffer)
1038 obj->release();
1039
1040 vars->consoleMapping = NULL;
1041 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1042 {
1043 vars->previewBuffer->release();
1044 vars->previewBuffer = 0;
1045 }
1046
1047 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1048 && vars->previewBuffer
1049 && (data = OSDynamicCast(OSData,
1050 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1051 {
1052 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1053 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1054
1055 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1056
1057 if (kIOHibernatePreviewUpdates & flags)
1058 {
1059 PE_Video consoleInfo;
1060 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1061
1062 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1063
1064 graphicsInfo->width = consoleInfo.v_width;
1065 graphicsInfo->height = consoleInfo.v_height;
1066 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1067 graphicsInfo->depth = consoleInfo.v_depth;
1068 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1069
1070 HIBPRINT("video %p %d %d %d\n",
1071 vars->consoleMapping, graphicsInfo->depth,
1072 graphicsInfo->width, graphicsInfo->height);
1073 if (vars->consoleMapping)
1074 ProgressInit(graphicsInfo, vars->consoleMapping,
1075 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1076 }
1077 }
1078
1079 if (gIOOptionsEntry)
1080 gIOOptionsEntry->sync();
1081
1082 return (ret);
1083}
1084
1085/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1086
1087static DeviceTreeNode *
1088MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1089{
1090 DeviceTreeNodeProperty * prop;
1091 DeviceTreeNode * child;
1092 IORegistryEntry * childRegEntry;
1093 const char * nameProp;
1094 unsigned int propLen, idx;
1095
1096 prop = (DeviceTreeNodeProperty *) (entry + 1);
1097 for (idx = 0; idx < entry->nProperties; idx++)
1098 {
1099 if (regEntry && (0 != strcmp("name", prop->name)))
1100 {
1101 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1102// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1103 }
1104 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1105 }
1106
1107 child = (DeviceTreeNode *) prop;
1108 for (idx = 0; idx < entry->nChildren; idx++)
1109 {
1110 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1111 panic("no name");
1112 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1113// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1114 child = MergeDeviceTree(child, childRegEntry);
1115 }
1116 return (child);
1117}
1118
1119IOReturn
1120IOHibernateSystemWake(void)
1121{
1122 if (kFSOpened == gFSState)
1123 {
1124 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1125 IOHibernateDone(&gIOHibernateVars);
1126 }
1127 else
1128 {
1129 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1130 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1131 }
1132 return (kIOReturnSuccess);
1133}
1134
1135static IOReturn
1136IOHibernateDone(IOHibernateVars * vars)
1137{
1138 IOReturn err;
1139 OSData * data;
1140
1141 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1142
1143 if (vars->videoMapping)
1144 {
1145 if (vars->videoMapSize)
1146 // remove mappings
1147 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1148 if (vars->videoAllocSize)
1149 // dealloc range
1150 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1151 }
1152
1153 if (vars->previewBuffer)
1154 {
1155 vars->previewBuffer->release();
1156 vars->previewBuffer = 0;
1157 }
1158
1159 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1160 {
1161 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1162 gIOHibernateCurrentHeader->options, 32);
1163 }
1164 else
1165 {
1166 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1167 }
1168
1169 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1170 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1171 {
1172 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1173 &gIOHibernateGraphicsInfo->gfxStatus,
1174 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1175 }
1176 else
1177 {
1178 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1179 }
1180
1181 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1182
1183#if defined(__i386__) || defined(__x86_64__)
1184 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1185 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1186
1187 /*
1188 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1189 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1190 */
1191 if (gIOOptionsEntry) {
1192
1193 if (gIOHibernateRTCVariablesKey) {
1194 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1195 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1196 }
1197 }
1198
1199 if (gIOHibernateBootNextKey)
1200 {
1201 if (gIOHibernateBootNextSave)
1202 {
1203 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1204 gIOHibernateBootNextSave->release();
1205 gIOHibernateBootNextSave = NULL;
1206 }
1207 else
1208 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1209 }
1210 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1211 }
1212#endif
1213
1214 if (vars->srcBuffer) vars->srcBuffer->release();
1215 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1216 if (vars->handoffBuffer)
1217 {
1218 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1219 {
1220 IOHibernateHandoff * handoff;
1221 bool done = false;
1222 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1223 !done;
1224 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1225 {
1226 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1227 uint8_t * data = &handoff->data[0];
1228 switch (handoff->type)
1229 {
1230 case kIOHibernateHandoffTypeEnd:
1231 done = true;
1232 break;
1233
1234 case kIOHibernateHandoffTypeDeviceTree:
1235 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1236 break;
1237
1238 case kIOHibernateHandoffTypeKeyStore:
1239#if defined(__i386__) || defined(__x86_64__)
1240 {
1241 IOBufferMemoryDescriptor *
1242 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1243 if (md)
1244 {
1245 IOSetKeyStoreData(md);
1246 }
1247 }
1248#endif
1249 break;
1250
1251 default:
1252 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1253 break;
1254 }
1255 }
1256#if defined(__i386__) || defined(__x86_64__)
1257 if (vars->volumeCryptKeySize)
1258 {
1259 IOBufferMemoryDescriptor *
1260 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1261 vars->volumeCryptKeySize, kIODirectionOutIn);
1262 if (!bmd) panic("IOBufferMemoryDescriptor");
1263 IOSetAPFSKeyStoreData(bmd);
1264 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1265 }
1266#endif
1267
1268 }
1269 vars->handoffBuffer->release();
1270 }
1271
1272 if (gIOChosenEntry
1273 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1274 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength()))
1275 {
1276 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1277 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1278 }
1279
1280 if (vars->hwEncrypt)
1281 {
1282 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1283 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1284 }
1285
1286 bzero(vars, sizeof(*vars));
1287
1288// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1289
1290 return (kIOReturnSuccess);
1291}
1292
1293static void
1294IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1295{
1296 // invalidate & close the image file
1297 if (p1) IOLockLock(gFSLock);
1298 if (kFSTrimDelay == gFSState)
1299 {
1300 IOPolledFileIOVars * vars = &gFileVars;
1301 IOPolledFileClose(&vars,
1302#if DISABLE_TRIM
1303 0, NULL, 0, 0, 0);
1304#else
1305 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1306 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1307#endif
1308 gFSState = kFSIdle;
1309 }
1310 if (p1) IOLockUnlock(gFSLock);
1311}
1312
1313IOReturn
1314IOHibernateSystemPostWake(bool now)
1315{
1316 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1317 IOSetBootImageNVRAM(0);
1318
1319 IOLockLock(gFSLock);
1320 if (kFSTrimDelay == gFSState)
1321 {
1322 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1323 IOHibernateSystemPostWakeTrim(NULL, NULL);
1324 }
1325 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1326 else
1327 {
1328 gFSState = kFSTrimDelay;
1329 if (now)
1330 {
1331 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1332 IOHibernateSystemPostWakeTrim(NULL, NULL);
1333 }
1334 else
1335 {
1336 AbsoluteTime deadline;
1337 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1338 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1339 }
1340 }
1341 IOLockUnlock(gFSLock);
1342
1343 return (kIOReturnSuccess);
1344}
1345
1346uint32_t IOHibernateWasScreenLocked(void)
1347{
1348 uint32_t ret = 0;
1349 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1350 {
1351 OSData *
1352 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1353 if (data)
1354 {
1355 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1356 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1357 }
1358 }
1359 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1360
1361 return (ret);
1362}
1363
1364/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1365
1366SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1367 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1368 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1369SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1370 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1371 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1372SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1373 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1374 &gIOHibernateMode, 0, "");
1375SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1376 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1377 &_hibernateStats, hibernate_statistics_t, "");
1378SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
1379 CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1380 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
1381
1382SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1383 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1384 &_hibernateStats.graphicsReadyTime, 0, "");
1385SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1386 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1387 &_hibernateStats.wakeNotificationTime, 0, "");
1388SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1389 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1390 &_hibernateStats.lockScreenReadyTime, 0, "");
1391SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1392 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1393 &_hibernateStats.hidReadyTime, 0, "");
1394
1395void
1396IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1397{
1398 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1399 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1400 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1401
1402#if defined(__i386__) || defined(__x86_64__)
1403 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1404 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1405 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1406 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1407#endif /* defined(__i386__) || defined(__x86_64__) */
1408
1409 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1410 if (data)
1411 {
1412 rootDomain->setProperty(kIOHibernateStateKey, data);
1413 data->release();
1414 }
1415
1416 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1417 gIOHibernateMode = kIOHibernateModeOn;
1418 else
1419 gIOHibernateFilename[0] = 0;
1420
1421 sysctl_register_oid(&sysctl__kern_hibernatefile);
1422 sysctl_register_oid(&sysctl__kern_bootsignature);
1423 sysctl_register_oid(&sysctl__kern_hibernatemode);
1424 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1425 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1426 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1427 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1428 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1429
1430 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1431
1432 if (gIOChosenEntry
1433 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1434 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength()))
1435 {
1436 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1437 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1438 }
1439
1440 gFSLock = IOLockAlloc();
1441}
1442
1443/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1444
1445static IOReturn
1446IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1447 const uint8_t * bytes, IOByteCount size,
1448 IOPolledFileCryptVars * cryptvars)
1449{
1450 IOReturn err;
1451
1452 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1453 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1454
1455 return (err);
1456}
1457
1458/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1459
1460extern "C" uint32_t
1461hibernate_write_image(void)
1462{
1463 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1464 IOHibernateVars * vars = &gIOHibernateVars;
1465 IOPolledFileExtent * fileExtents;
1466
1467 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1468
1469 uint32_t pageCount, pagesDone;
1470 IOReturn err;
1471 vm_offset_t ppnum, page;
1472 IOItemCount count;
1473 uint8_t * src;
1474 uint8_t * data;
1475 uint8_t * compressed;
1476 uint8_t * scratch;
1477 IOByteCount pageCompressedSize;
1478 uint64_t compressedSize, uncompressedSize;
1479 uint64_t image1Size = 0;
1480 uint32_t bitmap_size;
1481 bool iterDone, pollerOpen, needEncrypt;
1482 uint32_t restore1Sum, sum, sum1, sum2;
1483 int wkresult;
1484 uint32_t tag;
1485 uint32_t pageType;
1486 uint32_t pageAndCount[2];
1487 addr64_t phys64;
1488 IOByteCount segLen;
1489 uintptr_t hibernateBase;
1490 uintptr_t hibernateEnd;
1491
1492 AbsoluteTime startTime, endTime;
1493 AbsoluteTime allTime, compTime;
1494 uint64_t compBytes;
1495 uint64_t nsec;
1496 uint32_t lastProgressStamp = 0;
1497 uint32_t progressStamp;
1498 uint32_t blob, lastBlob = (uint32_t) -1L;
1499
1500 uint32_t wiredPagesEncrypted;
1501 uint32_t dirtyPagesEncrypted;
1502 uint32_t wiredPagesClear;
1503 uint32_t svPageCount;
1504 uint32_t zvPageCount;
1505
1506 IOPolledFileCryptVars _cryptvars;
1507 IOPolledFileCryptVars * cryptvars = 0;
1508
1509 wiredPagesEncrypted = 0;
1510 dirtyPagesEncrypted = 0;
1511 wiredPagesClear = 0;
1512 svPageCount = 0;
1513 zvPageCount = 0;
1514
1515 if (!vars->fileVars
1516 || !vars->fileVars->pollers
1517 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1518
1519 if (kIOHibernateModeSleep & gIOHibernateMode)
1520 kdebug_enable = save_kdebug_enable;
1521
1522 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1523 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1524
1525 restore1Sum = sum1 = sum2 = 0;
1526
1527#if CRYPTO
1528 // encryption data. "iv" is the "initial vector".
1529 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1530 {
1531 static const unsigned char first_iv[AES_BLOCK_SIZE]
1532 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1533 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1534
1535 cryptvars = &gIOHibernateCryptWakeContext;
1536 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1537 aes_encrypt_key(vars->cryptKey,
1538 kIOHibernateAESKeySize,
1539 &cryptvars->ctx.encrypt);
1540 aes_decrypt_key(vars->cryptKey,
1541 kIOHibernateAESKeySize,
1542 &cryptvars->ctx.decrypt);
1543
1544 cryptvars = &_cryptvars;
1545 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1546 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1547 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1548 aes_encrypt_key(vars->wiredCryptKey,
1549 kIOHibernateAESKeySize,
1550 &cryptvars->ctx.encrypt);
1551
1552 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1553 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1554 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1555 }
1556#endif /* CRYPTO */
1557
1558 hibernate_page_list_setall(vars->page_list,
1559 vars->page_list_wired,
1560 vars->page_list_pal,
1561 false /* !preflight */,
1562 /* discard_all */
1563 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1564 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1565 &pageCount);
1566
1567 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1568
1569 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1570
1571#if 0
1572 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1573 for (page = 0; page < count; page++)
1574 {
1575 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1576 fileExtents[page].start, fileExtents[page].length,
1577 fileExtents[page].start + fileExtents[page].length);
1578 }
1579#endif
1580
1581 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1582 AbsoluteTime_to_scalar(&compTime) = 0;
1583 compBytes = 0;
1584
1585 clock_get_uptime(&allTime);
1586 IOService::getPMRootDomain()->pmStatsRecordEvent(
1587 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1588 do
1589 {
1590 compressedSize = 0;
1591 uncompressedSize = 0;
1592 svPageCount = 0;
1593 zvPageCount = 0;
1594
1595 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1596
1597 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1598 ml_get_interrupts_enabled());
1599 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1600 // abortable if not low battery
1601 !IOService::getPMRootDomain()->mustHibernate());
1602 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1603 pollerOpen = (kIOReturnSuccess == err);
1604 if (!pollerOpen)
1605 break;
1606
1607 if (vars->volumeCryptKeySize)
1608 {
1609 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1610 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1611 vars->hwEncrypt = (kIOReturnSuccess == err);
1612 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1613 if (vars->hwEncrypt) header->options |= kIOHibernateOptionHWEncrypt;
1614 }
1615
1616 // copy file block extent list if larger than header
1617
1618 count = vars->fileVars->fileExtents->getLength();
1619 if (count > sizeof(header->fileExtentMap))
1620 {
1621 count -= sizeof(header->fileExtentMap);
1622 err = IOHibernatePolledFileWrite(vars->fileVars,
1623 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1624 if (kIOReturnSuccess != err)
1625 break;
1626 }
1627
1628 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1629 hibernateEnd = (segHIBB + segSizeHIB);
1630
1631 // copy out restore1 code
1632
1633 for (count = 0;
1634 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1635 count += segLen)
1636 {
1637 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1638 {
1639 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1640 }
1641 }
1642
1643 page = atop_32(kvtophys(hibernateBase));
1644 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1645 header->restore1CodePhysPage = page;
1646 header->restore1CodeVirt = hibernateBase;
1647 header->restore1PageCount = count;
1648 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1649 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1650
1651 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0]))
1652 {
1653 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1654 }
1655
1656 // sum __HIB seg, with zeros for the stack
1657 src = (uint8_t *) trunc_page(hibernateBase);
1658 for (page = 0; page < count; page++)
1659 {
1660 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1661 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1662 else
1663 restore1Sum += 0x00000000;
1664 src += page_size;
1665 }
1666 sum1 = restore1Sum;
1667
1668 // write the __HIB seg, with zeros for the stack
1669
1670 src = (uint8_t *) trunc_page(hibernateBase);
1671 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1672 if (count)
1673 {
1674 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1675 if (kIOReturnSuccess != err)
1676 break;
1677 }
1678 err = IOHibernatePolledFileWrite(vars->fileVars,
1679 (uint8_t *) 0,
1680 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1681 cryptvars);
1682 if (kIOReturnSuccess != err)
1683 break;
1684 src = &gIOHibernateRestoreStackEnd[0];
1685 count = round_page(hibernateEnd) - ((uintptr_t) src);
1686 if (count)
1687 {
1688 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1689 if (kIOReturnSuccess != err)
1690 break;
1691 }
1692
1693 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode))
1694 {
1695 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1696 vars->fileVars->encryptEnd = UINT64_MAX;
1697 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1698 }
1699
1700 // write the preview buffer
1701
1702 if (vars->previewBuffer)
1703 {
1704 ppnum = 0;
1705 count = 0;
1706 do
1707 {
1708 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1709 pageAndCount[0] = atop_64(phys64);
1710 pageAndCount[1] = atop_32(segLen);
1711 err = IOHibernatePolledFileWrite(vars->fileVars,
1712 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1713 cryptvars);
1714 if (kIOReturnSuccess != err)
1715 break;
1716 count += segLen;
1717 ppnum += sizeof(pageAndCount);
1718 }
1719 while (phys64);
1720 if (kIOReturnSuccess != err)
1721 break;
1722
1723 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1724
1725 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1726
1727 count = vars->previewBuffer->getLength();
1728
1729 header->previewPageListSize = ppnum;
1730 header->previewSize = count + ppnum;
1731
1732 for (page = 0; page < count; page += page_size)
1733 {
1734 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1735 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1736 }
1737 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1738 if (kIOReturnSuccess != err)
1739 break;
1740 }
1741
1742 // mark areas for no save
1743 IOMemoryDescriptor * ioBuffer;
1744 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1745 for (count = 0;
1746 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1747 count += segLen)
1748 {
1749 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1750 atop_64(phys64), atop_32(segLen),
1751 kIOHibernatePageStateFree);
1752 pageCount -= atop_32(segLen);
1753 }
1754
1755 for (count = 0;
1756 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1757 count += segLen)
1758 {
1759 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1760 atop_64(phys64), atop_32(segLen),
1761 kIOHibernatePageStateFree);
1762 pageCount -= atop_32(segLen);
1763 }
1764
1765 // copy out bitmap of pages available for trashing during restore
1766
1767 bitmap_size = vars->page_list_wired->list_size;
1768 src = (uint8_t *) vars->page_list_wired;
1769 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1770 if (kIOReturnSuccess != err)
1771 break;
1772
1773 // mark more areas for no save, but these are not available
1774 // for trashing during restore
1775
1776 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1777
1778
1779 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1780 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1781 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1782 page, count,
1783 kIOHibernatePageStateFree);
1784 pageCount -= count;
1785
1786 if (vars->previewBuffer) for (count = 0;
1787 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1788 count += segLen)
1789 {
1790 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1791 atop_64(phys64), atop_32(segLen),
1792 kIOHibernatePageStateFree);
1793 pageCount -= atop_32(segLen);
1794 }
1795
1796 for (count = 0;
1797 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1798 count += segLen)
1799 {
1800 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1801 atop_64(phys64), atop_32(segLen),
1802 kIOHibernatePageStateFree);
1803 pageCount -= atop_32(segLen);
1804 }
1805
1806#if KASAN
1807 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1808
1809 /* no need to save unused shadow pages */
1810 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1811 atop_64(shadow_pnext),
1812 shadow_pages_free,
1813 kIOHibernatePageStateFree);
1814#endif
1815
1816 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1817 compressed = src + page_size;
1818 scratch = compressed + page_size;
1819
1820 pagesDone = 0;
1821 lastBlob = 0;
1822
1823 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1824 bitmap_size, header->previewSize,
1825 pageCount, vars->fileVars->position);
1826
1827 enum
1828 // pageType
1829 {
1830 kWired = 0x02,
1831 kEncrypt = 0x01,
1832 kWiredEncrypt = kWired | kEncrypt,
1833 kWiredClear = kWired,
1834 kUnwiredEncrypt = kEncrypt
1835 };
1836
1837 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1838
1839 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1840 {
1841 if (kUnwiredEncrypt == pageType)
1842 {
1843 // start unwired image
1844 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode))
1845 {
1846 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1847 vars->fileVars->encryptEnd = UINT64_MAX;
1848 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1849 }
1850 bcopy(&cryptvars->aes_iv[0],
1851 &gIOHibernateCryptWakeContext.aes_iv[0],
1852 sizeof(cryptvars->aes_iv));
1853 cryptvars = &gIOHibernateCryptWakeContext;
1854 }
1855 for (iterDone = false, ppnum = 0; !iterDone; )
1856 {
1857 if (cpuAES && (pageType == kWiredClear))
1858 {
1859 count = 0;
1860 }
1861 else
1862 {
1863 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
1864 &ppnum);
1865 }
1866// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1867 iterDone = !count;
1868
1869 if (!cpuAES)
1870 {
1871 if (count && (kWired & pageType) && needEncrypt)
1872 {
1873 uint32_t checkIndex;
1874 for (checkIndex = 0;
1875 (checkIndex < count)
1876 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
1877 checkIndex++)
1878 {}
1879 if (!checkIndex)
1880 {
1881 ppnum++;
1882 continue;
1883 }
1884 count = checkIndex;
1885 }
1886 }
1887
1888 switch (pageType)
1889 {
1890 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1891 case kWiredClear: wiredPagesClear += count; break;
1892 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1893 }
1894
1895 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1896 else
1897 {
1898 pageAndCount[0] = ppnum;
1899 pageAndCount[1] = count;
1900 err = IOHibernatePolledFileWrite(vars->fileVars,
1901 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1902 cryptvars);
1903 if (kIOReturnSuccess != err)
1904 break;
1905 }
1906
1907 for (page = ppnum; page < (ppnum + count); page++)
1908 {
1909 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1910 if (err)
1911 {
1912 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1913 break;
1914 }
1915
1916 sum = hibernate_sum_page(src, page);
1917 if (kWired & pageType)
1918 sum1 += sum;
1919 else
1920 sum2 += sum;
1921
1922 clock_get_uptime(&startTime);
1923 wkresult = WKdm_compress_new((const WK_word*) src,
1924 (WK_word*) compressed,
1925 (WK_word*) scratch,
1926 page_size - 4);
1927
1928 clock_get_uptime(&endTime);
1929 ADD_ABSOLUTETIME(&compTime, &endTime);
1930 SUB_ABSOLUTETIME(&compTime, &startTime);
1931
1932 compBytes += page_size;
1933 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1934
1935 if (pageCompressedSize == 0)
1936 {
1937 pageCompressedSize = 4;
1938 data = src;
1939
1940 if (*(uint32_t *)src)
1941 svPageCount++;
1942 else
1943 zvPageCount++;
1944 }
1945 else
1946 {
1947 if (pageCompressedSize != page_size)
1948 data = compressed;
1949 else
1950 data = src;
1951 }
1952
1953 tag = pageCompressedSize | kIOHibernateTagSignature;
1954 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1955 if (kIOReturnSuccess != err)
1956 break;
1957
1958 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1959 if (kIOReturnSuccess != err)
1960 break;
1961
1962 compressedSize += pageCompressedSize;
1963 uncompressedSize += page_size;
1964 pagesDone++;
1965
1966 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1967 {
1968 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1969 if (blob != lastBlob)
1970 {
1971 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1972 lastBlob = blob;
1973 }
1974 }
1975 if (0 == (8191 & pagesDone))
1976 {
1977 clock_get_uptime(&endTime);
1978 SUB_ABSOLUTETIME(&endTime, &allTime);
1979 absolutetime_to_nanoseconds(endTime, &nsec);
1980 progressStamp = nsec / 750000000ULL;
1981 if (progressStamp != lastProgressStamp)
1982 {
1983 lastProgressStamp = progressStamp;
1984 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1985 }
1986 }
1987 }
1988 if (kIOReturnSuccess != err)
1989 break;
1990 ppnum = page;
1991 }
1992
1993 if (kIOReturnSuccess != err)
1994 break;
1995
1996 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1997 {
1998 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1999 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2000 }
2001
2002 if (kWiredEncrypt != pageType)
2003 {
2004 // end of image1/2 - fill to next block
2005 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2006 if (kIOReturnSuccess != err)
2007 break;
2008 }
2009 if (kWiredClear == pageType)
2010 {
2011 // enlarge wired image for test
2012// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2013
2014 // end wired image
2015 header->encryptStart = vars->fileVars->encryptStart;
2016 header->encryptEnd = vars->fileVars->encryptEnd;
2017 image1Size = vars->fileVars->position;
2018 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2019 image1Size, header->encryptStart, header->encryptEnd);
2020 }
2021 }
2022 if (kIOReturnSuccess != err)
2023 {
2024 if (kIOReturnOverrun == err)
2025 {
2026 // update actual compression ratio on not enough space (for retry)
2027 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2028 }
2029
2030 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2031 header->imageSize = vars->fileVars->position;
2032 break;
2033 }
2034
2035 // Header:
2036
2037 header->imageSize = vars->fileVars->position;
2038 header->image1Size = image1Size;
2039 header->bitmapSize = bitmap_size;
2040 header->pageCount = pageCount;
2041
2042 header->restore1Sum = restore1Sum;
2043 header->image1Sum = sum1;
2044 header->image2Sum = sum2;
2045 header->sleepTime = gIOLastSleepTime.tv_sec;
2046
2047 header->compression = (compressedSize << 8) / uncompressedSize;
2048 gIOHibernateCompression = header->compression;
2049
2050 count = vars->fileVars->fileExtents->getLength();
2051 if (count > sizeof(header->fileExtentMap))
2052 {
2053 header->fileExtentMapSize = count;
2054 count = sizeof(header->fileExtentMap);
2055 }
2056 else
2057 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2058 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2059
2060 header->deviceBase = vars->fileVars->block0;
2061 header->deviceBlockSize = vars->fileVars->blockSize;
2062
2063 IOPolledFileSeek(vars->fileVars, 0);
2064 err = IOHibernatePolledFileWrite(vars->fileVars,
2065 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2066 cryptvars);
2067 if (kIOReturnSuccess != err)
2068 break;
2069 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2070 }
2071 while (false);
2072
2073 clock_get_uptime(&endTime);
2074
2075 IOService::getPMRootDomain()->pmStatsRecordEvent(
2076 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2077
2078 SUB_ABSOLUTETIME(&endTime, &allTime);
2079 absolutetime_to_nanoseconds(endTime, &nsec);
2080 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2081
2082 absolutetime_to_nanoseconds(compTime, &nsec);
2083 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2084 compBytes,
2085 nsec / 1000000ULL,
2086 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2087
2088 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2089 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2090 vars->fileVars->cryptBytes,
2091 nsec / 1000000ULL,
2092 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2093
2094 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2095 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2096 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2097 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2098 sum1, sum2);
2099
2100 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2101 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2102
2103 if (pollerOpen)
2104 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2105
2106 if (vars->consoleMapping)
2107 ProgressUpdate(gIOHibernateGraphicsInfo,
2108 vars->consoleMapping, 0, kIOHibernateProgressCount);
2109
2110 HIBLOG("hibernate_write_image done(%x)\n", err);
2111
2112 // should we come back via regular wake, set the state in memory.
2113 gIOHibernateState = kIOHibernateStateInactive;
2114
2115 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2116 wiredPagesClear, dirtyPagesEncrypted);
2117
2118 if (kIOReturnSuccess == err)
2119 {
2120 if (kIOHibernateModeSleep & gIOHibernateMode)
2121 {
2122 return (kIOHibernatePostWriteSleep);
2123 }
2124 else if(kIOHibernateModeRestart & gIOHibernateMode)
2125 {
2126 return (kIOHibernatePostWriteRestart);
2127 }
2128 else
2129 {
2130 /* by default, power down */
2131 return (kIOHibernatePostWriteHalt);
2132 }
2133 }
2134 else if (kIOReturnAborted == err)
2135 {
2136 return (kIOHibernatePostWriteWake);
2137 }
2138 else
2139 {
2140 /* on error, sleep */
2141 return (kIOHibernatePostWriteSleep);
2142 }
2143}
2144
2145/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2146
2147extern "C" void
2148hibernate_machine_init(void)
2149{
2150 IOReturn err;
2151 uint32_t sum;
2152 uint32_t pagesDone;
2153 uint32_t pagesRead = 0;
2154 AbsoluteTime startTime, compTime;
2155 AbsoluteTime allTime, endTime;
2156 AbsoluteTime startIOTime, endIOTime;
2157 uint64_t nsec, nsecIO;
2158 uint64_t compBytes;
2159 uint32_t lastProgressStamp = 0;
2160 uint32_t progressStamp;
2161 IOPolledFileCryptVars * cryptvars = 0;
2162
2163 IOHibernateVars * vars = &gIOHibernateVars;
2164 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2165
2166 if (!vars->fileVars || !vars->fileVars->pollers)
2167 return;
2168
2169 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2170 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2171
2172 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2173 {
2174 HIBLOG("regular wake\n");
2175 return;
2176 }
2177
2178 HIBPRINT("diag %x %x %x %x\n",
2179 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2180 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2181
2182#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2183#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2184 tStat(booterStart, booterStart);
2185 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2186 tStat(booterDuration0, booterTime0);
2187 tStat(booterDuration1, booterTime1);
2188 tStat(booterDuration2, booterTime2);
2189 tStat(booterDuration, booterTime);
2190 tStat(booterConnectDisplayDuration, connectDisplayTime);
2191 tStat(booterSplashDuration, splashTime);
2192 tStat(trampolineDuration, trampolineTime);
2193
2194 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2195 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2196 gIOHibernateStats->image1Pages = pagesDone;
2197
2198 /* HIBERNATE_stats */
2199 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2200 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2201 gIOHibernateStats->trampolineDuration);
2202
2203 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2204 gIOHibernateStats->booterStart,
2205 gIOHibernateStats->smcStart,
2206 gIOHibernateStats->booterDuration0,
2207 gIOHibernateStats->booterDuration1,
2208 gIOHibernateStats->booterDuration2,
2209 gIOHibernateStats->booterDuration,
2210 gIOHibernateStats->booterConnectDisplayDuration,
2211 gIOHibernateStats->booterSplashDuration,
2212 gIOHibernateStats->trampolineDuration);
2213
2214 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2215 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2216 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2217
2218 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2219 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2220 {
2221 hibernate_page_list_discard(vars->page_list);
2222 }
2223
2224 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2225
2226 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2227 panic("handoff overflow");
2228
2229 IOHibernateHandoff * handoff;
2230 bool done = false;
2231 bool foundCryptData = false;
2232 bool foundVolumeEncryptData = false;
2233
2234 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2235 !done;
2236 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2237 {
2238// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2239 uint8_t * data = &handoff->data[0];
2240 switch (handoff->type)
2241 {
2242 case kIOHibernateHandoffTypeEnd:
2243 done = true;
2244 break;
2245
2246 case kIOHibernateHandoffTypeGraphicsInfo:
2247 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2248 {
2249 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2250 }
2251 break;
2252
2253 case kIOHibernateHandoffTypeCryptVars:
2254 if (cryptvars)
2255 {
2256 hibernate_cryptwakevars_t *
2257 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2258 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2259 }
2260 foundCryptData = true;
2261 bzero(data, handoff->bytecount);
2262 break;
2263
2264 case kIOHibernateHandoffTypeVolumeCryptKey:
2265 if (handoff->bytecount == vars->volumeCryptKeySize)
2266 {
2267 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2268 foundVolumeEncryptData = true;
2269 }
2270 else panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2271 break;
2272
2273 case kIOHibernateHandoffTypeMemoryMap:
2274
2275 clock_get_uptime(&allTime);
2276
2277 hibernate_newruntime_map(data, handoff->bytecount,
2278 gIOHibernateCurrentHeader->systemTableOffset);
2279
2280 clock_get_uptime(&endTime);
2281
2282 SUB_ABSOLUTETIME(&endTime, &allTime);
2283 absolutetime_to_nanoseconds(endTime, &nsec);
2284
2285 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2286
2287 break;
2288
2289 case kIOHibernateHandoffTypeDeviceTree:
2290 {
2291// DTEntry chosen = NULL;
2292// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2293 }
2294 break;
2295
2296 default:
2297 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2298 break;
2299 }
2300 }
2301
2302 if (vars->hwEncrypt && !foundVolumeEncryptData)
2303 panic("no volumeCryptKey");
2304 else if (cryptvars && !foundCryptData)
2305 panic("hibernate handoff");
2306
2307 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2308 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2309 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2310
2311 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2312 {
2313 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2314 * gIOHibernateGraphicsInfo->rowBytes);
2315 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2316 else
2317 {
2318 IOMapPages(kernel_map,
2319 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2320 vars->videoMapSize, kIOMapInhibitCache );
2321 }
2322 }
2323
2324 if (vars->videoMapSize)
2325 ProgressUpdate(gIOHibernateGraphicsInfo,
2326 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2327
2328 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2329 uint8_t * compressed = src + page_size;
2330 uint8_t * scratch = compressed + page_size;
2331 uint32_t decoOffset;
2332
2333 clock_get_uptime(&allTime);
2334 AbsoluteTime_to_scalar(&compTime) = 0;
2335 compBytes = 0;
2336
2337 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2338 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2339 clock_get_uptime(&startIOTime);
2340 endTime = startIOTime;
2341 SUB_ABSOLUTETIME(&endTime, &allTime);
2342 absolutetime_to_nanoseconds(endTime, &nsec);
2343 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2344
2345 if (vars->hwEncrypt)
2346 {
2347 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2348 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2349 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2350 if (kIOReturnSuccess != err) panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2351 cryptvars = 0;
2352 }
2353
2354 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2355
2356 // kick off the read ahead
2357 vars->fileVars->bufferHalf = 0;
2358 vars->fileVars->bufferLimit = 0;
2359 vars->fileVars->lastRead = 0;
2360 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2361 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2362 vars->fileVars->cryptBytes = 0;
2363 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2364
2365 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2366 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2367 // --
2368
2369 HIBLOG("hibernate_machine_init reading\n");
2370
2371 uint32_t * header = (uint32_t *) src;
2372 sum = 0;
2373
2374 while (kIOReturnSuccess == err)
2375 {
2376 unsigned int count;
2377 unsigned int page;
2378 uint32_t tag;
2379 vm_offset_t ppnum, compressedSize;
2380
2381 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2382 if (kIOReturnSuccess != err)
2383 break;
2384
2385 ppnum = header[0];
2386 count = header[1];
2387
2388// HIBPRINT("(%x, %x)\n", ppnum, count);
2389
2390 if (!count)
2391 break;
2392
2393 for (page = 0; page < count; page++)
2394 {
2395 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2396 if (kIOReturnSuccess != err)
2397 break;
2398
2399 compressedSize = kIOHibernateTagLength & tag;
2400 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2401 {
2402 err = kIOReturnIPCError;
2403 break;
2404 }
2405
2406 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2407 if (kIOReturnSuccess != err) break;
2408
2409 if (compressedSize < page_size)
2410 {
2411 decoOffset = page_size;
2412 clock_get_uptime(&startTime);
2413
2414 if (compressedSize == 4) {
2415 int i;
2416 uint32_t *s, *d;
2417
2418 s = (uint32_t *)src;
2419 d = (uint32_t *)(uintptr_t)compressed;
2420
2421 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2422 *d++ = *s;
2423 }
2424 else
2425 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2426 clock_get_uptime(&endTime);
2427 ADD_ABSOLUTETIME(&compTime, &endTime);
2428 SUB_ABSOLUTETIME(&compTime, &startTime);
2429 compBytes += page_size;
2430 }
2431 else decoOffset = 0;
2432
2433 sum += hibernate_sum_page((src + decoOffset), ppnum);
2434 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2435 if (err)
2436 {
2437 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2438 break;
2439 }
2440
2441 ppnum++;
2442 pagesDone++;
2443 pagesRead++;
2444
2445 if (0 == (8191 & pagesDone))
2446 {
2447 clock_get_uptime(&endTime);
2448 SUB_ABSOLUTETIME(&endTime, &allTime);
2449 absolutetime_to_nanoseconds(endTime, &nsec);
2450 progressStamp = nsec / 750000000ULL;
2451 if (progressStamp != lastProgressStamp)
2452 {
2453 lastProgressStamp = progressStamp;
2454 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2455 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2456 }
2457 }
2458 }
2459 }
2460 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2461 err = kIOReturnLockedRead;
2462
2463 if (kIOReturnSuccess != err)
2464 panic("Hibernate restore error %x", err);
2465
2466 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2467 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2468
2469 clock_get_uptime(&endIOTime);
2470
2471 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2472
2473 clock_get_uptime(&endTime);
2474
2475 IOService::getPMRootDomain()->pmStatsRecordEvent(
2476 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2477 IOService::getPMRootDomain()->pmStatsRecordEvent(
2478 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2479
2480 SUB_ABSOLUTETIME(&endTime, &allTime);
2481 absolutetime_to_nanoseconds(endTime, &nsec);
2482
2483 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2484 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2485
2486 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2487 gIOHibernateStats->imagePages = pagesDone;
2488
2489 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2490 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2491 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2492
2493 absolutetime_to_nanoseconds(compTime, &nsec);
2494 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2495 compBytes,
2496 nsec / 1000000ULL,
2497 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2498
2499 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2500 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2501 vars->fileVars->cryptBytes,
2502 nsec / 1000000ULL,
2503 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2504
2505 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2506}
2507
2508/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2509
2510void IOHibernateSetWakeCapabilities(uint32_t capability)
2511{
2512 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2513 {
2514 gIOHibernateStats->wakeCapability = capability;
2515
2516 if (kIOPMSystemCapabilityGraphics & capability)
2517 {
2518 vm_compressor_do_warmup();
2519 }
2520 }
2521}
2522
2523/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2524
2525void IOHibernateSystemRestart(void)
2526{
2527 static uint8_t noteStore[32] __attribute__((aligned(32)));
2528 IORegistryEntry * regEntry;
2529 const OSSymbol * sym;
2530 OSData * noteProp;
2531 OSData * data;
2532 uintptr_t * smcVars;
2533 uint8_t * smcBytes;
2534 size_t len;
2535 addr64_t element;
2536
2537 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2538 if (!data) return;
2539
2540 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2541 smcBytes = (typeof(smcBytes)) smcVars[1];
2542 len = smcVars[0];
2543 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2544 noteProp = OSData::withCapacity(3 * sizeof(element));
2545 if (!noteProp) return;
2546 element = len;
2547 noteProp->appendBytes(&element, sizeof(element));
2548 element = crc32(0, smcBytes, len);
2549 noteProp->appendBytes(&element, sizeof(element));
2550
2551 bcopy(smcBytes, noteStore, len);
2552 element = (addr64_t) &noteStore[0];
2553 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2554 noteProp->appendBytes(&element, sizeof(element));
2555
2556 if (!gIOOptionsEntry)
2557 {
2558 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2559 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2560 if (regEntry && !gIOOptionsEntry)
2561 regEntry->release();
2562 }
2563
2564 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2565 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2566 if (noteProp) noteProp->release();
2567 if (sym) sym->release();
2568}
2569