1/*
2 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <stdint.h>
30#include <mach/mach_types.h>
31#include <mach/vm_param.h>
32#include <IOKit/IOHibernatePrivate.h>
33#include <IOKit/IOLib.h>
34#include <pexpert/boot.h>
35#include <libkern/libkern.h>
36
37#include <vm/WKdm_new.h>
38#include "IOHibernateInternal.h"
39
40#include <machine/pal_hibernate.h>
41
42/*
43This code is linked into the kernel but part of the "__HIB" section, which means
44its used by code running in the special context of restoring the kernel text and data
45from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
46it calls or references needs to be careful to only touch memory also in the "__HIB" section.
47*/
48
49uint32_t gIOHibernateState;
50
51uint32_t gIOHibernateDebugFlags;
52
53static IOHibernateImageHeader _hibernateHeader;
54IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
55
56ppnum_t gIOHibernateHandoffPages[64];
57uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
58 / sizeof(gIOHibernateHandoffPages[0]);
59
60#if CONFIG_DEBUG
61void hibprintf(const char *fmt, ...);
62#else
63#define hibprintf(x...)
64#endif
65
66
67#if CONFIG_SLEEP
68#if defined(__i386__) || defined(__x86_64__)
69extern void acpi_wake_prot_entry(void);
70#endif
71#endif
72
73/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
74
75#if defined(__i386__) || defined(__x86_64__)
76#include <i386/proc_reg.h>
77#else
78
79static inline uint64_t rdtsc64(void)
80{
81 return (0);
82}
83
84#endif /* defined(__i386__) || defined(__x86_64__) */
85
86#if defined(__i386__) || defined(__x86_64__)
87
88#define DBGLOG 1
89
90#include <architecture/i386/pio.h>
91
92/* standard port addresses */
93enum {
94 COM1_PORT_ADDR = 0x3f8,
95 COM2_PORT_ADDR = 0x2f8
96};
97
98/* UART register offsets */
99enum {
100 UART_RBR = 0, /* receive buffer Register (R) */
101 UART_THR = 0, /* transmit holding register (W) */
102 UART_DLL = 0, /* DLAB = 1, divisor latch (LSB) */
103 UART_IER = 1, /* interrupt enable register */
104 UART_DLM = 1, /* DLAB = 1, divisor latch (MSB) */
105 UART_IIR = 2, /* interrupt ident register (R) */
106 UART_FCR = 2, /* fifo control register (W) */
107 UART_LCR = 3, /* line control register */
108 UART_MCR = 4, /* modem control register */
109 UART_LSR = 5, /* line status register */
110 UART_MSR = 6, /* modem status register */
111 UART_SCR = 7 /* scratch register */
112};
113
114enum {
115 UART_LCR_8BITS = 0x03,
116 UART_LCR_DLAB = 0x80
117};
118
119enum {
120 UART_MCR_DTR = 0x01,
121 UART_MCR_RTS = 0x02,
122 UART_MCR_OUT1 = 0x04,
123 UART_MCR_OUT2 = 0x08,
124 UART_MCR_LOOP = 0x10
125};
126
127enum {
128 UART_LSR_DR = 0x01,
129 UART_LSR_OE = 0x02,
130 UART_LSR_PE = 0x04,
131 UART_LSR_FE = 0x08,
132 UART_LSR_THRE = 0x20
133};
134
135static void uart_putc(char c)
136{
137 while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
138 {}
139 outb(COM1_PORT_ADDR + UART_THR, c);
140}
141
142static int debug_probe( void )
143{
144 /* Verify that the Scratch Register is accessible */
145 outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
146 if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
147 outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
148 if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
149 uart_putc('\n');
150 return true;
151}
152
153static void uart_puthex(uint64_t num)
154{
155 int bit;
156 char c;
157 bool leading = true;
158
159 for (bit = 60; bit >= 0; bit -= 4)
160 {
161 c = 0xf & (num >> bit);
162 if (c)
163 leading = false;
164 else if (leading && bit)
165 continue;
166 if (c <= 9)
167 c += '0';
168 else
169 c+= 'a' - 10;
170 uart_putc(c);
171 }
172}
173
174static void debug_code(uint32_t code, uint64_t value)
175{
176 int bit;
177 char c;
178
179 if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
180 return;
181
182 for (bit = 24; bit >= 0; bit -= 8)
183 {
184 c = 0xFF & (code >> bit);
185 if (c)
186 uart_putc(c);
187 }
188 uart_putc('=');
189 uart_puthex(value);
190 uart_putc('\n');
191 uart_putc('\r');
192}
193
194#endif /* defined(__i386__) || defined(__x86_64__) */
195
196#if !defined(DBGLOG)
197#define debug_probe() (false)
198#define debug_code(c, v) {}
199#endif
200
201enum
202{
203 kIOHibernateRestoreCodeImageStart = 'imgS',
204 kIOHibernateRestoreCodeImageEnd = 'imgE',
205 kIOHibernateRestoreCodePageIndexStart = 'pgiS',
206 kIOHibernateRestoreCodePageIndexEnd = 'pgiE',
207 kIOHibernateRestoreCodeMapStart = 'mapS',
208 kIOHibernateRestoreCodeMapEnd = 'mapE',
209 kIOHibernateRestoreCodeWakeMapSize = 'wkms',
210 kIOHibernateRestoreCodeConflictPage = 'cfpg',
211 kIOHibernateRestoreCodeConflictSource = 'cfsr',
212 kIOHibernateRestoreCodeNoMemory = 'nomm',
213 kIOHibernateRestoreCodeTag = 'tag ',
214 kIOHibernateRestoreCodeSignature = 'sign',
215 kIOHibernateRestoreCodeMapVirt = 'mapV',
216 kIOHibernateRestoreCodeHandoffPages = 'hand',
217 kIOHibernateRestoreCodeHandoffCount = 'hndc',
218};
219
220/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
221
222
223static void fatal(void)
224{
225#if defined(__i386__) || defined(__x86_64__)
226 outb(0xcf9, 6);
227#else
228 while (true) {}
229#endif
230}
231
232/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
233
234uint32_t
235hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
236{
237 return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
238}
239
240/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241
242static hibernate_bitmap_t *
243hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
244{
245 uint32_t bank;
246 hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
247
248 for (bank = 0; bank < list->bank_count; bank++)
249 {
250 if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
251 break;
252 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
253 }
254 if (bank == list->bank_count)
255 bitmap = NULL;
256
257 return (bitmap);
258}
259
260hibernate_bitmap_t *
261hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
262{
263 uint32_t bank, page = *pPage;
264 hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
265
266 for (bank = 0; bank < list->bank_count; bank++)
267 {
268 if (page <= bitmap->first_page)
269 {
270 *pPage = bitmap->first_page;
271 break;
272 }
273 if (page <= bitmap->last_page)
274 break;
275 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
276 }
277 if (bank == list->bank_count)
278 bitmap = NULL;
279
280 return (bitmap);
281}
282
283void
284hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
285{
286 hibernate_bitmap_t * bitmap;
287
288 bitmap = hibernate_page_bitmap(list, page);
289 if (bitmap)
290 {
291 page -= bitmap->first_page;
292 if (set)
293 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
294 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
295 else
296 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
297 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
298 }
299}
300
301boolean_t
302hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
303{
304 boolean_t result = TRUE;
305 hibernate_bitmap_t * bitmap;
306
307 bitmap = hibernate_page_bitmap(list, page);
308 if (bitmap)
309 {
310 page -= bitmap->first_page;
311 result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
312 }
313 return (result);
314}
315
316// count bits clear or set (set == TRUE) starting at page.
317uint32_t
318hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
319{
320 uint32_t index, bit, bits;
321 uint32_t count;
322
323 count = 0;
324
325 index = (page - bitmap->first_page) >> 5;
326 bit = (page - bitmap->first_page) & 31;
327
328 bits = bitmap->bitmap[index];
329 if (set)
330 bits = ~bits;
331 bits = (bits << bit);
332 if (bits)
333 count += __builtin_clz(bits);
334 else
335 {
336 count += 32 - bit;
337 while (++index < bitmap->bitmapwords)
338 {
339 bits = bitmap->bitmap[index];
340 if (set)
341 bits = ~bits;
342 if (bits)
343 {
344 count += __builtin_clz(bits);
345 break;
346 }
347 count += 32;
348 }
349 }
350
351 if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
352
353 return (count);
354}
355
356static ppnum_t
357hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
358{
359 uint32_t nextFree = *pNextFree;
360 uint32_t nextFreeInBank;
361 hibernate_bitmap_t * bitmap;
362
363 nextFreeInBank = nextFree + 1;
364 while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
365 {
366 nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
367 if (nextFreeInBank <= bitmap->last_page)
368 {
369 *pNextFree = nextFreeInBank;
370 break;
371 }
372 }
373
374 if (!bitmap)
375 {
376 debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
377 fatal();
378 nextFree = 0;
379 }
380
381 return (nextFree);
382}
383
384static uint32_t
385store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
386 uint32_t * buffer, uint32_t ppnum)
387{
388 uint64_t dst = ptoa_64(ppnum);
389 uint8_t scratch[WKdm_SCRATCH_BUF_SIZE_INTERNAL] __attribute__ ((aligned (16)));
390
391 if (compressedSize != PAGE_SIZE)
392 {
393 dst = pal_hib_map(DEST_COPY_AREA, dst);
394 if (compressedSize != 4) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], compressedSize);
395 else
396 {
397 size_t i;
398 uint32_t s, *d;
399
400 s = *src;
401 d = (uint32_t *)(uintptr_t)dst;
402 if (!s) __nosan_bzero((void *) dst, PAGE_SIZE);
403 else for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s;
404 }
405 }
406 else
407 {
408 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
409 }
410
411 return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
412}
413
414long
415hibernate_kernel_entrypoint(uint32_t p1,
416 uint32_t p2, uint32_t p3, uint32_t p4)
417{
418 uint64_t headerPhys;
419 uint64_t mapPhys;
420 uint64_t srcPhys;
421 uint64_t imageReadPhys;
422 uint64_t pageIndexPhys;
423 uint32_t * pageIndexSource;
424 hibernate_page_list_t * map;
425 uint32_t stage;
426 uint32_t count;
427 uint32_t ppnum;
428 uint32_t page;
429 uint32_t conflictCount;
430 uint32_t compressedSize;
431 uint32_t uncompressedPages;
432 uint32_t copyPageListHeadPage;
433 uint32_t pageListPage;
434 uint32_t * copyPageList;
435 uint32_t * src;
436 uint32_t copyPageIndex;
437 uint32_t sum;
438 uint32_t pageSum;
439 uint32_t nextFree;
440 uint32_t lastImagePage;
441 uint32_t lastMapPage;
442 uint32_t lastPageIndexPage;
443 uint32_t handoffPages;
444 uint32_t handoffPageCount;
445
446 uint64_t timeStart;
447 timeStart = rdtsc64();
448
449 static_assert(sizeof(IOHibernateImageHeader) == 512);
450
451 headerPhys = ptoa_64(p1);
452
453 if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
454 gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
455
456 debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
457
458 __nosan_memcpy(gIOHibernateCurrentHeader,
459 (void *) pal_hib_map(IMAGE_AREA, headerPhys),
460 sizeof(IOHibernateImageHeader));
461
462 debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
463
464 mapPhys = headerPhys
465 + (offsetof(IOHibernateImageHeader, fileExtentMap)
466 + gIOHibernateCurrentHeader->fileExtentMapSize
467 + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
468 + gIOHibernateCurrentHeader->previewSize);
469
470 map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
471
472 lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
473 lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
474
475 handoffPages = gIOHibernateCurrentHeader->handoffPages;
476 handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
477
478 debug_code(kIOHibernateRestoreCodeImageEnd, ptoa_64(lastImagePage));
479 debug_code(kIOHibernateRestoreCodeMapStart, mapPhys);
480 debug_code(kIOHibernateRestoreCodeMapEnd, ptoa_64(lastMapPage));
481
482 debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
483 debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
484 debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
485
486 // knock all the image pages to be used out of free map
487 for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
488 {
489 hibernate_page_bitset(map, FALSE, ppnum);
490 }
491 // knock all the handoff pages to be used out of free map
492 for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
493 {
494 hibernate_page_bitset(map, FALSE, ppnum);
495 }
496
497 nextFree = 0;
498 hibernate_page_list_grab(map, &nextFree);
499
500 sum = gIOHibernateCurrentHeader->actualRestore1Sum;
501 gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
502 gIOHibernateCurrentHeader->diag[1] = sum;
503 gIOHibernateCurrentHeader->trampolineTime = 0;
504
505 uncompressedPages = 0;
506 conflictCount = 0;
507 copyPageListHeadPage = 0;
508 copyPageList = 0;
509 copyPageIndex = PAGE_SIZE >> 2;
510
511 compressedSize = PAGE_SIZE;
512 stage = 2;
513 count = 0;
514 srcPhys = 0;
515
516 if (gIOHibernateCurrentHeader->previewSize)
517 {
518 pageIndexPhys = headerPhys
519 + (offsetof(IOHibernateImageHeader, fileExtentMap)
520 + gIOHibernateCurrentHeader->fileExtentMapSize
521 + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
522 imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
523 lastPageIndexPage = atop_64(imageReadPhys);
524 pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
525 }
526 else
527 {
528 pageIndexPhys = 0;
529 lastPageIndexPage = 0;
530 imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
531 }
532
533 debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
534 debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage));
535
536 while (1)
537 {
538 switch (stage)
539 {
540 case 2:
541 // copy handoff data
542 count = srcPhys ? 0 : handoffPageCount;
543 if (!count)
544 break;
545 if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
546 srcPhys = ptoa_64(handoffPages);
547 break;
548
549 case 1:
550 // copy pageIndexSource pages == preview image data
551 if (!srcPhys)
552 {
553 if (!pageIndexPhys) break;
554 srcPhys = imageReadPhys;
555 }
556 ppnum = pageIndexSource[0];
557 count = pageIndexSource[1];
558 pageIndexSource += 2;
559 pageIndexPhys += 2 * sizeof(pageIndexSource[0]);
560 imageReadPhys = srcPhys;
561 break;
562
563 case 0:
564 // copy pages
565 if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
566 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
567 ppnum = src[0];
568 count = src[1];
569 srcPhys += 2 * sizeof(*src);
570 imageReadPhys = srcPhys;
571 break;
572 }
573
574
575 if (!count)
576 {
577 if (!stage)
578 break;
579 stage--;
580 srcPhys = 0;
581 continue;
582 }
583
584 for (page = 0; page < count; page++, ppnum++)
585 {
586 uint32_t tag;
587 int conflicts;
588
589 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
590
591 if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
592 else if (!stage)
593 {
594 tag = *src++;
595// debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
596 srcPhys += sizeof(*src);
597 compressedSize = kIOHibernateTagLength & tag;
598 }
599
600 conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
601
602 conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
603
604 if (stage >= 2)
605 conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
606
607 if (stage >= 1)
608 conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
609
610 if (!conflicts)
611 {
612 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
613 src, compressedSize, 0, ppnum);
614 if (stage != 2)
615 sum += pageSum;
616 uncompressedPages++;
617 }
618 else
619 {
620 uint32_t bufferPage = 0;
621 uint32_t * dst;
622
623// debug_code(kIOHibernateRestoreCodeConflictPage, ppnum);
624// debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
625 conflictCount++;
626 if (compressedSize)
627 {
628 // alloc new buffer page
629 bufferPage = hibernate_page_list_grab(map, &nextFree);
630 dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
631 __nosan_memcpy(dst, src, compressedSize);
632 }
633 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
634 {
635 // alloc new copy list page
636 pageListPage = hibernate_page_list_grab(map, &nextFree);
637 // link to current
638 if (copyPageList) {
639 copyPageList[1] = pageListPage;
640 } else {
641 copyPageListHeadPage = pageListPage;
642 }
643 copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
644 ptoa_64(pageListPage));
645 copyPageList[1] = 0;
646 copyPageIndex = 2;
647 }
648 copyPageList[copyPageIndex++] = ppnum;
649 copyPageList[copyPageIndex++] = bufferPage;
650 copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
651 copyPageList[0] = copyPageIndex;
652 }
653 srcPhys += ((compressedSize + 3) & ~3);
654 src += ((compressedSize + 3) >> 2);
655 }
656 }
657
658 /* src points to the last page restored, so we need to skip over that */
659 hibernateRestorePALState(src);
660
661 // -- copy back conflicts
662
663 pageListPage = copyPageListHeadPage;
664 while (pageListPage)
665 {
666 copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
667 for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
668 {
669 ppnum = copyPageList[copyPageIndex + 0];
670 srcPhys = ptoa_64(copyPageList[copyPageIndex + 1]);
671 src = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
672 compressedSize = copyPageList[copyPageIndex + 2];
673 stage = compressedSize >> 24;
674 compressedSize &= 0x1FFF;
675 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
676 src, compressedSize, 0, ppnum);
677 if (stage != 2)
678 sum += pageSum;
679 uncompressedPages++;
680 }
681 pageListPage = copyPageList[1];
682 }
683
684 pal_hib_patchup();
685
686 // -- image has been destroyed...
687
688 gIOHibernateCurrentHeader->actualImage1Sum = sum;
689 gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
690 gIOHibernateCurrentHeader->conflictCount = conflictCount;
691 gIOHibernateCurrentHeader->nextFree = nextFree;
692
693 gIOHibernateState = kIOHibernateStateWakingFromHibernate;
694
695 gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8);
696
697// debug_code('done', 0);
698
699#if CONFIG_SLEEP
700#if defined(__i386__) || defined(__x86_64__)
701 typedef void (*ResetProc)(void);
702 ResetProc proc;
703 proc = HIB_ENTRYPOINT;
704 // flush caches
705 __asm__("wbinvd");
706 proc();
707#else
708// implement me
709#endif
710#endif
711
712 return -1;
713}
714
715#if CONFIG_DEBUG
716/* standalone printf implementation */
717/*-
718 * Copyright (c) 1986, 1988, 1991, 1993
719 * The Regents of the University of California. All rights reserved.
720 * (c) UNIX System Laboratories, Inc.
721 * All or some portions of this file are derived from material licensed
722 * to the University of California by American Telephone and Telegraph
723 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
724 * the permission of UNIX System Laboratories, Inc.
725 *
726 * Redistribution and use in source and binary forms, with or without
727 * modification, are permitted provided that the following conditions
728 * are met:
729 * 1. Redistributions of source code must retain the above copyright
730 * notice, this list of conditions and the following disclaimer.
731 * 2. Redistributions in binary form must reproduce the above copyright
732 * notice, this list of conditions and the following disclaimer in the
733 * documentation and/or other materials provided with the distribution.
734 * 4. Neither the name of the University nor the names of its contributors
735 * may be used to endorse or promote products derived from this software
736 * without specific prior written permission.
737 *
738 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
739 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
740 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
741 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
742 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
743 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
744 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
745 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
746 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
747 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
748 * SUCH DAMAGE.
749 *
750 * @(#)subr_prf.c 8.3 (Berkeley) 1/21/94
751 */
752
753typedef long ptrdiff_t;
754char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
755#define hibhex2ascii(hex) (hibhex2ascii_data[hex])
756#define toupper(c) ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
757static size_t
758hibstrlen(const char *s)
759{
760 size_t l = 0;
761 while (*s++)
762 l++;
763 return l;
764}
765
766/* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
767#define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
768
769/*
770 * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
771 * order; return an optional length and a pointer to the last character
772 * written in the buffer (i.e., the first character of the string).
773 * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
774 */
775static char *
776ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
777{
778 char *p, c;
779
780 /* Truncate so we don't call umoddi3, which isn't in __HIB */
781#if !defined(__LP64__)
782 uint32_t num2 = (uint32_t) num;
783#else
784 uintmax_t num2 = num;
785#endif
786
787 p = nbuf;
788 *p = '\0';
789 do {
790 c = hibhex2ascii(num2 % base);
791 *++p = upper ? toupper(c) : c;
792 } while (num2 /= base);
793 if (lenp)
794 *lenp = (int)(p - nbuf);
795 return (p);
796}
797
798/*
799 * Scaled down version of printf(3).
800 *
801 * Two additional formats:
802 *
803 * The format %b is supported to decode error registers.
804 * Its usage is:
805 *
806 * printf("reg=%b\n", regval, "*");
807 *
808 * where is the output base expressed as a control character, e.g.
809 * \10 gives octal; \20 gives hex. Each arg is a sequence of characters,
810 * the first of which gives the bit number to be inspected (origin 1), and
811 * the next characters (up to a control character, i.e. a character <= 32),
812 * give the name of the register. Thus:
813 *
814 * kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
815 *
816 * would produce output:
817 *
818 * reg=3
819 *
820 * XXX: %D -- Hexdump, takes pointer and separator string:
821 * ("%6D", ptr, ":") -> XX:XX:XX:XX:XX:XX
822 * ("%*D", len, ptr, " " -> XX XX XX XX ...
823 */
824static int
825hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
826{
827#define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
828 char nbuf[MAXNBUF];
829 char *d;
830 const char *p, *percent, *q;
831 u_char *up;
832 int ch, n;
833 uintmax_t num;
834 int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
835 int cflag, hflag, jflag, tflag, zflag;
836 int dwidth, upper;
837 char padc;
838 int stop = 0, retval = 0;
839
840 num = 0;
841 if (!func)
842 d = (char *) arg;
843 else
844 d = NULL;
845
846 if (fmt == NULL)
847 fmt = "(fmt null)\n";
848
849 if (radix < 2 || radix > 36)
850 radix = 10;
851
852 for (;;) {
853 padc = ' ';
854 width = 0;
855 while ((ch = (u_char)*fmt++) != '%' || stop) {
856 if (ch == '\0')
857 return (retval);
858 PCHAR(ch);
859 }
860 percent = fmt - 1;
861 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
862 sign = 0; dot = 0; dwidth = 0; upper = 0;
863 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
864reswitch: switch (ch = (u_char)*fmt++) {
865 case '.':
866 dot = 1;
867 goto reswitch;
868 case '#':
869 sharpflag = 1;
870 goto reswitch;
871 case '+':
872 sign = 1;
873 goto reswitch;
874 case '-':
875 ladjust = 1;
876 goto reswitch;
877 case '%':
878 PCHAR(ch);
879 break;
880 case '*':
881 if (!dot) {
882 width = va_arg(ap, int);
883 if (width < 0) {
884 ladjust = !ladjust;
885 width = -width;
886 }
887 } else {
888 dwidth = va_arg(ap, int);
889 }
890 goto reswitch;
891 case '0':
892 if (!dot) {
893 padc = '0';
894 goto reswitch;
895 }
896 case '1': case '2': case '3': case '4':
897 case '5': case '6': case '7': case '8': case '9':
898 for (n = 0;; ++fmt) {
899 n = n * 10 + ch - '0';
900 ch = *fmt;
901 if (ch < '0' || ch > '9')
902 break;
903 }
904 if (dot)
905 dwidth = n;
906 else
907 width = n;
908 goto reswitch;
909 case 'b':
910 num = (u_int)va_arg(ap, int);
911 p = va_arg(ap, char *);
912 for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
913 PCHAR(*q--);
914
915 if (num == 0)
916 break;
917
918 for (tmp = 0; *p;) {
919 n = *p++;
920 if (num & (1 << (n - 1))) {
921 PCHAR(tmp ? ',' : '<');
922 for (; (n = *p) > ' '; ++p)
923 PCHAR(n);
924 tmp = 1;
925 } else
926 for (; *p > ' '; ++p)
927 continue;
928 }
929 if (tmp)
930 PCHAR('>');
931 break;
932 case 'c':
933 PCHAR(va_arg(ap, int));
934 break;
935 case 'D':
936 up = va_arg(ap, u_char *);
937 p = va_arg(ap, char *);
938 if (!width)
939 width = 16;
940 while(width--) {
941 PCHAR(hibhex2ascii(*up >> 4));
942 PCHAR(hibhex2ascii(*up & 0x0f));
943 up++;
944 if (width)
945 for (q=p;*q;q++)
946 PCHAR(*q);
947 }
948 break;
949 case 'd':
950 case 'i':
951 base = 10;
952 sign = 1;
953 goto handle_sign;
954 case 'h':
955 if (hflag) {
956 hflag = 0;
957 cflag = 1;
958 } else
959 hflag = 1;
960 goto reswitch;
961 case 'j':
962 jflag = 1;
963 goto reswitch;
964 case 'l':
965 if (lflag) {
966 lflag = 0;
967 qflag = 1;
968 } else
969 lflag = 1;
970 goto reswitch;
971 case 'n':
972 if (jflag)
973 *(va_arg(ap, intmax_t *)) = retval;
974 else if (qflag)
975 *(va_arg(ap, quad_t *)) = retval;
976 else if (lflag)
977 *(va_arg(ap, long *)) = retval;
978 else if (zflag)
979 *(va_arg(ap, size_t *)) = retval;
980 else if (hflag)
981 *(va_arg(ap, short *)) = retval;
982 else if (cflag)
983 *(va_arg(ap, char *)) = retval;
984 else
985 *(va_arg(ap, int *)) = retval;
986 break;
987 case 'o':
988 base = 8;
989 goto handle_nosign;
990 case 'p':
991 base = 16;
992 sharpflag = (width == 0);
993 sign = 0;
994 num = (uintptr_t)va_arg(ap, void *);
995 goto number;
996 case 'q':
997 qflag = 1;
998 goto reswitch;
999 case 'r':
1000 base = radix;
1001 if (sign)
1002 goto handle_sign;
1003 goto handle_nosign;
1004 case 's':
1005 p = va_arg(ap, char *);
1006 if (p == NULL)
1007 p = "(null)";
1008 if (!dot)
1009 n = (typeof(n))hibstrlen (p);
1010 else
1011 for (n = 0; n < dwidth && p[n]; n++)
1012 continue;
1013
1014 width -= n;
1015
1016 if (!ladjust && width > 0)
1017 while (width--)
1018 PCHAR(padc);
1019 while (n--)
1020 PCHAR(*p++);
1021 if (ladjust && width > 0)
1022 while (width--)
1023 PCHAR(padc);
1024 break;
1025 case 't':
1026 tflag = 1;
1027 goto reswitch;
1028 case 'u':
1029 base = 10;
1030 goto handle_nosign;
1031 case 'X':
1032 upper = 1;
1033 case 'x':
1034 base = 16;
1035 goto handle_nosign;
1036 case 'y':
1037 base = 16;
1038 sign = 1;
1039 goto handle_sign;
1040 case 'z':
1041 zflag = 1;
1042 goto reswitch;
1043handle_nosign:
1044 sign = 0;
1045 if (jflag)
1046 num = va_arg(ap, uintmax_t);
1047 else if (qflag)
1048 num = va_arg(ap, u_quad_t);
1049 else if (tflag)
1050 num = va_arg(ap, ptrdiff_t);
1051 else if (lflag)
1052 num = va_arg(ap, u_long);
1053 else if (zflag)
1054 num = va_arg(ap, size_t);
1055 else if (hflag)
1056 num = (u_short)va_arg(ap, int);
1057 else if (cflag)
1058 num = (u_char)va_arg(ap, int);
1059 else
1060 num = va_arg(ap, u_int);
1061 goto number;
1062handle_sign:
1063 if (jflag)
1064 num = va_arg(ap, intmax_t);
1065 else if (qflag)
1066 num = va_arg(ap, quad_t);
1067 else if (tflag)
1068 num = va_arg(ap, ptrdiff_t);
1069 else if (lflag)
1070 num = va_arg(ap, long);
1071 else if (zflag)
1072 num = va_arg(ap, ssize_t);
1073 else if (hflag)
1074 num = (short)va_arg(ap, int);
1075 else if (cflag)
1076 num = (char)va_arg(ap, int);
1077 else
1078 num = va_arg(ap, int);
1079number:
1080 if (sign && (intmax_t)num < 0) {
1081 neg = 1;
1082 num = -(intmax_t)num;
1083 }
1084 p = ksprintn(nbuf, num, base, &tmp, upper);
1085 if (sharpflag && num != 0) {
1086 if (base == 8)
1087 tmp++;
1088 else if (base == 16)
1089 tmp += 2;
1090 }
1091 if (neg)
1092 tmp++;
1093
1094 if (!ladjust && padc != '0' && width
1095 && (width -= tmp) > 0)
1096 while (width--)
1097 PCHAR(padc);
1098 if (neg)
1099 PCHAR('-');
1100 if (sharpflag && num != 0) {
1101 if (base == 8) {
1102 PCHAR('0');
1103 } else if (base == 16) {
1104 PCHAR('0');
1105 PCHAR('x');
1106 }
1107 }
1108 if (!ladjust && width && (width -= tmp) > 0)
1109 while (width--)
1110 PCHAR(padc);
1111
1112 while (*p)
1113 PCHAR(*p--);
1114
1115 if (ladjust && width && (width -= tmp) > 0)
1116 while (width--)
1117 PCHAR(padc);
1118
1119 break;
1120 default:
1121 while (percent < fmt)
1122 PCHAR(*percent++);
1123 /*
1124 * Since we ignore an formatting argument it is no
1125 * longer safe to obey the remaining formatting
1126 * arguments as the arguments will no longer match
1127 * the format specs.
1128 */
1129 stop = 1;
1130 break;
1131 }
1132 }
1133#undef PCHAR
1134}
1135
1136
1137static void
1138putchar(int c, void *arg)
1139{
1140 (void)arg;
1141 uart_putc(c);
1142}
1143
1144void
1145hibprintf(const char *fmt, ...)
1146{
1147 /* http://www.pagetable.com/?p=298 */
1148 va_list ap;
1149
1150 va_start(ap, fmt);
1151 hibkvprintf(fmt, putchar, NULL, 10, ap);
1152 va_end(ap);
1153}
1154#endif /* CONFIG_DEBUG */
1155
1156