1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * File: kern/gzalloc.c
30 * Author: Derek Kumar
31 *
32 * "Guard mode" zone allocator, used to trap use-after-free errors,
33 * overruns, underruns, mismatched allocations/frees, uninitialized
34 * zone element use, timing dependent races etc.
35 *
36 * The allocator is configured by these boot-args:
37 * gzalloc_size=<size>: target all zones with elements of <size> bytes
38 * gzalloc_min=<size>: target zones with elements >= size
39 * gzalloc_max=<size>: target zones with elements <= size
40 * gzalloc_min/max can be specified in conjunction to target a range of
41 * sizes
42 * gzalloc_fc_size=<size>: number of zone elements (effectively page
43 * multiple sized) to retain in the free VA cache. This cache is evicted
44 * (backing pages and VA released) in a least-recently-freed fashion.
45 * Larger free VA caches allow for a longer window of opportunity to trap
46 * delayed use-after-free operations, but use more memory.
47 * -gzalloc_wp: Write protect, rather than unmap, freed allocations
48 * lingering in the free VA cache. Useful to disambiguate between
49 * read-after-frees/read overruns and writes. Also permits direct inspection
50 * of the freed element in the cache via the kernel debugger. As each
51 * element has a "header" (trailer in underflow detection mode), the zone
52 * of origin of the element can be easily determined in this mode.
53 * -gzalloc_uf_mode: Underflow detection mode, where the guard page
54 * adjoining each element is placed *before* the element page rather than
55 * after. The element is also located at the top of the page, rather than
56 * abutting the bottom as with the standard overflow detection mode.
57 * -gzalloc_noconsistency: disable consistency checks that flag mismatched
58 * frees, corruptions of the header/trailer signatures etc.
59 * -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
60 * enables the guard allocator for zones sized 1K (if present) by
61 * default, this option can disable that behaviour.
62 * gzname=<name> target a zone by name. Can be coupled with size-based
63 * targeting. Naming conventions match those of the zlog boot-arg, i.e.
64 * "a period in the logname will match a space in the zone name"
65 * -gzalloc_no_dfree_check Eliminate double free checks
66 * gzalloc_zscale=<value> specify size multiplier for the dedicated gzalloc submap
67 */
68
69#include <zone_debug.h>
70
71#include <mach/mach_types.h>
72#include <mach/vm_param.h>
73#include <mach/kern_return.h>
74#include <mach/machine/vm_types.h>
75#include <mach_debug/zone_info.h>
76#include <mach/vm_map.h>
77
78#include <kern/kern_types.h>
79#include <kern/assert.h>
80#include <kern/sched.h>
81#include <kern/locks.h>
82#include <kern/misc_protos.h>
83#include <kern/zalloc.h>
84#include <kern/kalloc.h>
85
86#include <vm/pmap.h>
87#include <vm/vm_map.h>
88#include <vm/vm_kern.h>
89#include <vm/vm_page.h>
90
91#include <pexpert/pexpert.h>
92
93#include <machine/machparam.h>
94
95#include <libkern/OSDebug.h>
96#include <libkern/OSAtomic.h>
97#include <sys/kdebug.h>
98
99extern boolean_t vm_kernel_ready, kmem_ready;
100boolean_t gzalloc_mode = FALSE;
101uint32_t pdzalloc_count, pdzfree_count;
102
103#define GZALLOC_MIN_DEFAULT (1024)
104#define GZDEADZONE ((zone_t) 0xDEAD201E)
105#define GZALLOC_SIGNATURE (0xABADCAFE)
106#define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)
107#define GZFC_DEFAULT_SIZE (1536)
108
109char gzalloc_fill_pattern = 0x67; /* 'g' */
110
111uint32_t gzalloc_min = ~0U;
112uint32_t gzalloc_max = 0;
113uint32_t gzalloc_size = 0;
114uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted;
115boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE, gzalloc_dfree_check = TRUE;
116vm_prot_t gzalloc_prot = VM_PROT_NONE;
117uint32_t gzalloc_guard = KMA_GUARD_LAST;
118uint32_t gzfc_size = GZFC_DEFAULT_SIZE;
119uint32_t gzalloc_zonemap_scale = 6;
120
121vm_map_t gzalloc_map;
122vm_offset_t gzalloc_map_min, gzalloc_map_max;
123vm_offset_t gzalloc_reserve;
124vm_size_t gzalloc_reserve_size;
125
126typedef struct gzalloc_header {
127 zone_t gzone;
128 uint32_t gzsize;
129 uint32_t gzsig;
130} gzhdr_t;
131
132#define GZHEADER_SIZE (sizeof(gzhdr_t))
133
134extern zone_t vm_page_zone;
135
136static zone_t gztrackzone = NULL;
137static char gznamedzone[MAX_ZONE_NAME] = "";
138
139void gzalloc_reconfigure(__unused zone_t z) {
140 /* Nothing for now */
141}
142
143boolean_t gzalloc_enabled(void) {
144 return gzalloc_mode;
145}
146
147static inline boolean_t gzalloc_tracked(zone_t z) {
148 return (gzalloc_mode &&
149 (((z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max)) || (z == gztrackzone)) &&
150 (z->gzalloc_exempt == 0));
151}
152
153void gzalloc_zone_init(zone_t z) {
154 if (gzalloc_mode) {
155 bzero(&z->gz, sizeof(z->gz));
156
157 if (track_this_zone(z->zone_name, gznamedzone)) {
158 gztrackzone = z;
159 }
160
161 if (gzfc_size &&
162 gzalloc_tracked(z)) {
163 vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size);
164
165 /* If the VM/kmem system aren't yet configured, carve
166 * out the free element cache structure directly from the
167 * gzalloc_reserve supplied by the pmap layer.
168 */
169 if (!kmem_ready) {
170 if (gzalloc_reserve_size < gzfcsz)
171 panic("gzalloc reserve exhausted");
172
173 z->gz.gzfc = (vm_offset_t *)gzalloc_reserve;
174 gzalloc_reserve += gzfcsz;
175 gzalloc_reserve_size -= gzfcsz;
176 } else {
177 kern_return_t kr;
178
179 if ((kr = kernel_memory_allocate(kernel_map, (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0, KMA_KOBJECT, VM_KERN_MEMORY_OSFMK)) != KERN_SUCCESS) {
180 panic("zinit/gzalloc: kernel_memory_allocate failed (%d) for 0x%lx bytes", kr, (unsigned long) gzfcsz);
181 }
182 }
183 bzero((void *)z->gz.gzfc, gzfcsz);
184 }
185 }
186}
187
188/* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */
189void gzalloc_empty_free_cache(zone_t zone) {
190 if (__improbable(gzalloc_tracked(zone))) {
191 kern_return_t kr;
192 int freed_elements = 0;
193 vm_offset_t free_addr = 0;
194 vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
195 vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size);
196 vm_offset_t gzfc_copy;
197
198 kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK);
199 if (kr != KERN_SUCCESS) {
200 panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr);
201 }
202
203 /* Reset gzalloc_data. */
204 lock_zone(zone);
205 memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz);
206 bzero((void *)zone->gz.gzfc, gzfcsz);
207 zone->gz.gzfc_index = 0;
208 unlock_zone(zone);
209
210 /* Free up all the cached elements. */
211 for (uint32_t index = 0; index < gzfc_size; index++) {
212 free_addr = ((vm_offset_t *)gzfc_copy)[index];
213 if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) {
214 kr = vm_map_remove(
215 gzalloc_map,
216 free_addr,
217 free_addr + rounded_size + (1 * PAGE_SIZE),
218 VM_MAP_REMOVE_KUNWIRE);
219 if (kr != KERN_SUCCESS) {
220 panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
221 }
222 OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
223 OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
224
225 freed_elements++;
226 }
227 }
228 /*
229 * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool.
230 * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a
231 * subsequent zinit() as well.
232 */
233
234 /* Decrement zone counters. */
235 lock_zone(zone);
236 zone->count -= freed_elements;
237 zone->cur_size -= (freed_elements * rounded_size);
238 unlock_zone(zone);
239
240 kmem_free(kernel_map, gzfc_copy, gzfcsz);
241 }
242}
243
244void gzalloc_configure(void) {
245 char temp_buf[16];
246
247 if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof (temp_buf))) {
248 gzalloc_mode = TRUE;
249 gzalloc_min = GZALLOC_MIN_DEFAULT;
250 gzalloc_max = ~0U;
251 }
252
253 if (PE_parse_boot_argn("gzalloc_min", &gzalloc_min, sizeof(gzalloc_min))) {
254 gzalloc_mode = TRUE;
255 gzalloc_max = ~0U;
256 }
257
258 if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) {
259 gzalloc_mode = TRUE;
260 if (gzalloc_min == ~0U)
261 gzalloc_min = 0;
262 }
263
264 if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) {
265 gzalloc_min = gzalloc_max = gzalloc_size;
266 gzalloc_mode = TRUE;
267 }
268
269 (void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size));
270
271 if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof (temp_buf))) {
272 gzalloc_prot = VM_PROT_READ;
273 }
274
275 if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof (temp_buf))) {
276 gzalloc_uf_mode = TRUE;
277 gzalloc_guard = KMA_GUARD_FIRST;
278 }
279
280 if (PE_parse_boot_argn("-gzalloc_no_dfree_check", temp_buf, sizeof(temp_buf))) {
281 gzalloc_dfree_check = FALSE;
282 }
283
284 (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale));
285
286 if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) {
287 gzalloc_consistency_checks = FALSE;
288 }
289
290 if (PE_parse_boot_argn("gzname", gznamedzone, sizeof(gznamedzone))) {
291 gzalloc_mode = TRUE;
292 }
293#if DEBUG
294 if (gzalloc_mode == FALSE) {
295 gzalloc_min = 1024;
296 gzalloc_max = 1024;
297 strlcpy(gznamedzone, "pmap", sizeof(gznamedzone));
298 gzalloc_prot = VM_PROT_READ;
299 gzalloc_mode = TRUE;
300 }
301#endif
302 if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof (temp_buf)))
303 gzalloc_mode = FALSE;
304
305 if (gzalloc_mode) {
306 gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT;
307 gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size);
308 }
309}
310
311void gzalloc_init(vm_size_t max_zonemap_size) {
312 kern_return_t retval;
313
314 if (gzalloc_mode) {
315 vm_map_kernel_flags_t vmk_flags;
316
317 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
318 vmk_flags.vmkf_permanent = TRUE;
319 retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale),
320 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE,
321 &gzalloc_map);
322
323 if (retval != KERN_SUCCESS) {
324 panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed", max_zonemap_size, gzalloc_zonemap_scale);
325 }
326 gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale);
327 }
328}
329
330vm_offset_t
331gzalloc_alloc(zone_t zone, boolean_t canblock) {
332 vm_offset_t addr = 0;
333
334 if (__improbable(gzalloc_tracked(zone))) {
335
336 if (get_preemption_level() != 0) {
337 if (canblock == TRUE) {
338 pdzalloc_count++;
339 }
340 else
341 return 0;
342 }
343
344 vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
345 vm_offset_t residue = rounded_size - zone->elem_size;
346 vm_offset_t gzaddr = 0;
347 gzhdr_t *gzh, *gzhcopy = NULL;
348
349 if (!kmem_ready || (vm_page_zone == ZONE_NULL)) {
350 /* Early allocations are supplied directly from the
351 * reserve.
352 */
353 if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE))
354 panic("gzalloc reserve exhausted");
355 gzaddr = gzalloc_reserve;
356 /* No guard page for these early allocations, just
357 * waste an additional page.
358 */
359 gzalloc_reserve += rounded_size + PAGE_SIZE;
360 gzalloc_reserve_size -= rounded_size + PAGE_SIZE;
361 OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc);
362 }
363 else {
364 kern_return_t kr = kernel_memory_allocate(gzalloc_map,
365 &gzaddr, rounded_size + (1*PAGE_SIZE),
366 0, KMA_KOBJECT | KMA_ATOMIC | gzalloc_guard,
367 VM_KERN_MEMORY_OSFMK);
368 if (kr != KERN_SUCCESS)
369 panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d", (uint64_t)rounded_size, kr);
370
371 }
372
373 if (gzalloc_uf_mode) {
374 gzaddr += PAGE_SIZE;
375 /* The "header" becomes a "footer" in underflow
376 * mode.
377 */
378 gzh = (gzhdr_t *) (gzaddr + zone->elem_size);
379 addr = gzaddr;
380 gzhcopy = (gzhdr_t *) (gzaddr + rounded_size - sizeof(gzhdr_t));
381 } else {
382 gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE);
383 addr = (gzaddr + residue);
384 }
385
386 /* Fill with a pattern on allocation to trap uninitialized
387 * data use. Since the element size may be "rounded up"
388 * by higher layers such as the kalloc layer, this may
389 * also identify overruns between the originally requested
390 * size and the rounded size via visual inspection.
391 * TBD: plumb through the originally requested size,
392 * prior to rounding by kalloc/IOMalloc etc.
393 * We also add a signature and the zone of origin in a header
394 * prefixed to the allocation.
395 */
396 memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size);
397
398 gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE;
399 gzh->gzsize = (uint32_t) zone->elem_size;
400 gzh->gzsig = GZALLOC_SIGNATURE;
401
402 /* In underflow detection mode, stash away a copy of the
403 * metadata at the edge of the allocated range, for
404 * retrieval by gzalloc_element_size()
405 */
406 if (gzhcopy) {
407 *gzhcopy = *gzh;
408 }
409
410 lock_zone(zone);
411 assert(zone->zone_valid);
412 zone->count++;
413 zone->sum_count++;
414 zone->cur_size += rounded_size;
415 unlock_zone(zone);
416
417 OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated);
418 OSAddAtomic64((SInt32) (rounded_size - zone->elem_size), &gzalloc_wasted);
419 }
420 return addr;
421}
422
423boolean_t gzalloc_free(zone_t zone, void *addr) {
424 boolean_t gzfreed = FALSE;
425 kern_return_t kr;
426
427 if (__improbable(gzalloc_tracked(zone))) {
428 gzhdr_t *gzh;
429 vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
430 vm_offset_t residue = rounded_size - zone->elem_size;
431 vm_offset_t saddr;
432 vm_offset_t free_addr = 0;
433
434 if (gzalloc_uf_mode) {
435 gzh = (gzhdr_t *)((vm_offset_t)addr + zone->elem_size);
436 saddr = (vm_offset_t) addr - PAGE_SIZE;
437 } else {
438 gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE);
439 saddr = ((vm_offset_t)addr) - residue;
440 }
441
442 if ((saddr & PAGE_MASK) != 0) {
443 panic("gzalloc_free: invalid address supplied: %p (adjusted: 0x%lx) for zone with element sized 0x%lx\n", addr, saddr, zone->elem_size);
444 }
445
446 if (gzfc_size) {
447 if (gzalloc_dfree_check) {
448 uint32_t gd;
449
450 lock_zone(zone);
451 assert(zone->zone_valid);
452 for (gd = 0; gd < gzfc_size; gd++) {
453 if (zone->gz.gzfc[gd] == saddr) {
454 panic("gzalloc: double free detected, freed address: 0x%lx, current free cache index: %d, freed index: %d", saddr, zone->gz.gzfc_index, gd);
455 }
456 }
457 unlock_zone(zone);
458 }
459 }
460
461 if (gzalloc_consistency_checks) {
462 if (gzh->gzsig != GZALLOC_SIGNATURE) {
463 panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", addr, GZALLOC_SIGNATURE, gzh->gzsig);
464 }
465
466 if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE))
467 panic("%s: Mismatched zone or under/overflow, current zone: %p, recorded zone: %p, address: %p", __FUNCTION__, zone, gzh->gzone, (void *)addr);
468 /* Partially redundant given the zone check, but may flag header corruption */
469 if (gzh->gzsize != zone->elem_size) {
470 panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr);
471 }
472
473 char *gzc, *checkstart, *checkend;
474 if (gzalloc_uf_mode) {
475 checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh));
476 checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE);
477 } else {
478 checkstart = (char *) trunc_page_64(addr);
479 checkend = (char *)gzh;
480 }
481
482 for (gzc = checkstart; gzc < checkend; gzc++) {
483 if (*gzc != gzalloc_fill_pattern) {
484 panic("GZALLOC: detected over/underflow, byte at %p, element %p, contents 0x%x from 0x%lx byte sized zone (%s) doesn't match fill pattern (%c)", gzc, addr, *gzc, zone->elem_size, zone->zone_name, gzalloc_fill_pattern);
485 }
486 }
487 }
488
489 if (!kmem_ready || gzh->gzone == GZDEADZONE) {
490 /* For now, just leak frees of early allocations
491 * performed before kmem is fully configured.
492 * They don't seem to get freed currently;
493 * consider ml_static_mfree in the future.
494 */
495 OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free);
496 return TRUE;
497 }
498
499 if (get_preemption_level() != 0) {
500 pdzfree_count++;
501 }
502
503 if (gzfc_size) {
504 /* Either write protect or unmap the newly freed
505 * allocation
506 */
507 kr = vm_map_protect(
508 gzalloc_map,
509 saddr,
510 saddr + rounded_size + (1 * PAGE_SIZE),
511 gzalloc_prot,
512 FALSE);
513 if (kr != KERN_SUCCESS)
514 panic("%s: vm_map_protect: %p, 0x%x", __FUNCTION__, (void *)saddr, kr);
515 } else {
516 free_addr = saddr;
517 }
518
519 lock_zone(zone);
520 assert(zone->zone_valid);
521
522 /* Insert newly freed element into the protected free element
523 * cache, and rotate out the LRU element.
524 */
525 if (gzfc_size) {
526 if (zone->gz.gzfc_index >= gzfc_size) {
527 zone->gz.gzfc_index = 0;
528 }
529 free_addr = zone->gz.gzfc[zone->gz.gzfc_index];
530 zone->gz.gzfc[zone->gz.gzfc_index++] = saddr;
531 }
532
533 if (free_addr) {
534 zone->count--;
535 zone->cur_size -= rounded_size;
536 }
537
538 unlock_zone(zone);
539
540 if (free_addr) {
541 // TODO: consider using physical reads to check for
542 // corruption while on the protected freelist
543 // (i.e. physical corruption)
544 kr = vm_map_remove(
545 gzalloc_map,
546 free_addr,
547 free_addr + rounded_size + (1 * PAGE_SIZE),
548 VM_MAP_REMOVE_KUNWIRE);
549 if (kr != KERN_SUCCESS)
550 panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
551 // TODO: sysctl-ize for quick reference
552 OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
553 OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
554 }
555
556 gzfreed = TRUE;
557 }
558 return gzfreed;
559}
560
561boolean_t gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) {
562 uintptr_t a = (uintptr_t)gzaddr;
563 if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) {
564 gzhdr_t *gzh;
565 boolean_t vmef;
566 vm_map_entry_t gzvme = NULL;
567 vm_map_lock_read(gzalloc_map);
568 vmef = vm_map_lookup_entry(gzalloc_map, (vm_map_offset_t)a, &gzvme);
569 vm_map_unlock(gzalloc_map);
570 if (vmef == FALSE) {
571 panic("GZALLOC: unable to locate map entry for %p\n", (void *)a);
572 }
573 assertf(gzvme->vme_atomic != 0, "GZALLOC: VM map entry inconsistency, vme: %p, start: %llu end: %llu", gzvme, gzvme->vme_start, gzvme->vme_end);
574
575 /* Locate the gzalloc metadata adjoining the element */
576 if (gzalloc_uf_mode == TRUE) {
577
578 /* In underflow detection mode, locate the map entry describing
579 * the element, and then locate the copy of the gzalloc
580 * header at the trailing edge of the range.
581 */
582 gzh = (gzhdr_t *)(gzvme->vme_end - GZHEADER_SIZE);
583 } else {
584 /* In overflow detection mode, scan forward from
585 * the base of the map entry to locate the
586 * gzalloc header.
587 */
588 uint32_t *p = (uint32_t*) gzvme->vme_start;
589 while (p < (uint32_t *) gzvme->vme_end) {
590 if (*p == GZALLOC_SIGNATURE)
591 break;
592 else {
593 p++;
594 }
595 }
596 if (p >= (uint32_t *) gzvme->vme_end) {
597 panic("GZALLOC signature missing addr %p, zone %p", gzaddr, z);
598 }
599 p++;
600 uintptr_t q = (uintptr_t) p;
601 gzh = (gzhdr_t *) (q - sizeof(gzhdr_t));
602 }
603
604 if (gzh->gzsig != GZALLOC_SIGNATURE) {
605 panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", (void *)a, GZALLOC_SIGNATURE, gzh->gzsig);
606 }
607
608 *gzsz = gzh->gzone->elem_size;
609 if (__improbable((gzalloc_tracked(gzh->gzone)) == FALSE)) {
610 panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone);
611 }
612
613 if (z) {
614 *z = gzh->gzone;
615 }
616 return TRUE;
617 } else {
618 return FALSE;
619 }
620}
621