1/*
2 * Copyright (c) 2004-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <kern/machine.h>
30#include <kern/misc_protos.h>
31#include <kern/thread.h>
32#include <kern/processor.h>
33#include <kern/kalloc.h>
34#include <mach/machine.h>
35#include <mach/processor_info.h>
36#include <mach/mach_types.h>
37#include <i386/pmap.h>
38#include <kern/cpu_data.h>
39#include <IOKit/IOPlatformExpert.h>
40
41#include <pexpert/i386/efi.h>
42
43#include <IOKit/IOHibernatePrivate.h>
44#include <vm/vm_page.h>
45#include <i386/i386_lowmem.h>
46#include <san/kasan.h>
47
48extern ppnum_t max_ppnum;
49
50#define MAX_BANKS 32
51
52/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
53
54hibernate_page_list_t *
55hibernate_page_list_allocate(boolean_t log)
56{
57 ppnum_t base, num;
58 vm_size_t size;
59 uint32_t bank, num_banks;
60 uint32_t pages, page_count;
61 hibernate_page_list_t * list;
62 hibernate_bitmap_t * bitmap;
63
64 EfiMemoryRange * mptr;
65 uint32_t mcount, msize, i;
66 hibernate_bitmap_t dram_ranges[MAX_BANKS];
67 boot_args * args = (boot_args *) PE_state.bootArgs;
68 uint32_t non_os_pagecount;
69 ppnum_t pnmax = max_ppnum;
70
71 mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap);
72 if (args->MemoryMapDescriptorSize == 0)
73 panic("Invalid memory map descriptor size");
74 msize = args->MemoryMapDescriptorSize;
75 mcount = args->MemoryMapSize / msize;
76
77#if KASAN
78 /* adjust max page number to include stolen memory */
79 if (atop(shadow_ptop) > pnmax) {
80 pnmax = (ppnum_t)atop(shadow_ptop);
81 }
82#endif
83
84 num_banks = 0;
85 non_os_pagecount = 0;
86 for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize))
87 {
88 base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
89 num = (ppnum_t) mptr->NumberOfPages;
90
91#if KASAN
92 if (i == shadow_stolen_idx) {
93 /*
94 * Add all stolen pages to the bitmap. Later we will prune the unused
95 * pages.
96 */
97 num += shadow_pages_total;
98 }
99#endif
100
101 if (base > pnmax)
102 continue;
103 if ((base + num - 1) > pnmax)
104 num = pnmax - base + 1;
105 if (!num)
106 continue;
107
108 switch (mptr->Type)
109 {
110 // any kind of dram
111 case kEfiACPIMemoryNVS:
112 case kEfiPalCode:
113 non_os_pagecount += num;
114
115 // OS used dram
116 case kEfiLoaderCode:
117 case kEfiLoaderData:
118 case kEfiBootServicesCode:
119 case kEfiBootServicesData:
120 case kEfiConventionalMemory:
121
122 for (bank = 0; bank < num_banks; bank++)
123 {
124 if (dram_ranges[bank].first_page <= base)
125 continue;
126 if ((base + num) == dram_ranges[bank].first_page)
127 {
128 dram_ranges[bank].first_page = base;
129 num = 0;
130 }
131 break;
132 }
133 if (!num) break;
134
135 if (bank && (base == (1 + dram_ranges[bank - 1].last_page)))
136 bank--;
137 else
138 {
139 num_banks++;
140 if (num_banks >= MAX_BANKS) break;
141 bcopy(&dram_ranges[bank],
142 &dram_ranges[bank + 1],
143 (num_banks - bank - 1) * sizeof(hibernate_bitmap_t));
144 dram_ranges[bank].first_page = base;
145 }
146 dram_ranges[bank].last_page = base + num - 1;
147 break;
148
149 // runtime services will be restarted, so no save
150 case kEfiRuntimeServicesCode:
151 case kEfiRuntimeServicesData:
152 // contents are volatile once the platform expert starts
153 case kEfiACPIReclaimMemory:
154 // non dram
155 case kEfiReservedMemoryType:
156 case kEfiUnusableMemory:
157 case kEfiMemoryMappedIO:
158 case kEfiMemoryMappedIOPortSpace:
159 default:
160 break;
161 }
162 }
163
164 if (num_banks >= MAX_BANKS)
165 return (NULL);
166
167 // size the hibernation bitmap
168
169 size = sizeof(hibernate_page_list_t);
170 page_count = 0;
171 for (bank = 0; bank < num_banks; bank++) {
172 pages = dram_ranges[bank].last_page + 1 - dram_ranges[bank].first_page;
173 page_count += pages;
174 size += sizeof(hibernate_bitmap_t) + ((pages + 31) >> 5) * sizeof(uint32_t);
175 }
176
177 list = (hibernate_page_list_t *)kalloc(size);
178 if (!list)
179 return (list);
180
181 list->list_size = (uint32_t)size;
182 list->page_count = page_count;
183 list->bank_count = num_banks;
184
185 // convert to hibernation bitmap.
186
187 bitmap = &list->bank_bitmap[0];
188 for (bank = 0; bank < num_banks; bank++)
189 {
190 bitmap->first_page = dram_ranges[bank].first_page;
191 bitmap->last_page = dram_ranges[bank].last_page;
192 bitmap->bitmapwords = (bitmap->last_page + 1
193 - bitmap->first_page + 31) >> 5;
194 if (log) kprintf("hib bank[%d]: 0x%x000 end 0x%xfff\n",
195 bank, bitmap->first_page, bitmap->last_page);
196 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
197 }
198 if (log) printf("efi pagecount %d\n", non_os_pagecount);
199
200 return (list);
201}
202
203// mark pages not to be saved, but available for scratch usage during restore
204
205void
206hibernate_page_list_setall_machine( __unused hibernate_page_list_t * page_list,
207 __unused hibernate_page_list_t * page_list_wired,
208 __unused boolean_t preflight,
209 __unused uint32_t * pagesOut)
210{
211}
212
213// mark pages not to be saved and not for scratch usage during restore
214void
215hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
216 hibernate_page_list_t * page_list_wired,
217 uint32_t * pagesOut)
218{
219 boot_args * args = (boot_args *) PE_state.bootArgs;
220
221 if (args->efiRuntimeServicesPageStart)
222 {
223 hibernate_set_page_state(page_list, page_list_wired,
224 args->efiRuntimeServicesPageStart, args->efiRuntimeServicesPageCount,
225 kIOHibernatePageStateFree);
226 *pagesOut -= args->efiRuntimeServicesPageCount;
227 }
228}
229
230kern_return_t
231hibernate_processor_setup(IOHibernateImageHeader * header)
232{
233 boot_args * args = (boot_args *) PE_state.bootArgs;
234
235 cpu_datap(0)->cpu_hibernate = 1;
236 header->processorFlags = 0;
237
238 header->runtimePages = args->efiRuntimeServicesPageStart;
239 header->runtimePageCount = args->efiRuntimeServicesPageCount;
240 header->runtimeVirtualPages = args->efiRuntimeServicesVirtualPageStart;
241 header->performanceDataStart = args->performanceDataStart;
242 header->performanceDataSize = args->performanceDataSize;
243
244 return (KERN_SUCCESS);
245}
246
247static boolean_t hibernate_vm_locks_safe;
248
249void
250hibernate_vm_lock(void)
251{
252 if (current_cpu_datap()->cpu_hibernate) {
253 hibernate_vm_lock_queues();
254 hibernate_vm_locks_safe = TRUE;
255 }
256}
257
258void
259hibernate_vm_unlock(void)
260{
261 assert(FALSE == ml_get_interrupts_enabled());
262 if (current_cpu_datap()->cpu_hibernate) hibernate_vm_unlock_queues();
263 ml_set_is_quiescing(TRUE);
264}
265
266// ACPI calls hibernate_vm_lock(), interrupt disable, hibernate_vm_unlock() on sleep,
267// hibernate_vm_lock_end() and interrupt enable on wake.
268// VM locks are safely single threaded between hibernate_vm_lock() and hibernate_vm_lock_end().
269
270void
271hibernate_vm_lock_end(void)
272{
273 assert(FALSE == ml_get_interrupts_enabled());
274 hibernate_vm_locks_safe = FALSE;
275 ml_set_is_quiescing(FALSE);
276}
277
278boolean_t
279hibernate_vm_locks_are_safe(void)
280{
281 assert(FALSE == ml_get_interrupts_enabled());
282 return (hibernate_vm_locks_safe);
283}
284