malloc.c source code [glibc/malloc/malloc.c]

1	/ Malloc implementation for multiple threads without lock contention.*
2	Copyright (C) 1996-2023 Free Software Foundation, Inc.
3	Copyright The GNU Toolchain Authors.
4	This file is part of the GNU C Library.
5
6	The GNU C Library is free software; you can redistribute it and/or
7	modify it under the terms of the GNU Lesser General Public License as
8	published by the Free Software Foundation; either version 2.1 of the
9	License, or (at your option) any later version.
10
11	The GNU C Library is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	Lesser General Public License for more details.
15
16	You should have received a copy of the GNU Lesser General Public
17	License along with the GNU C Library; see the file COPYING.LIB. If
18	not, see <https://www.gnu.org/licenses/>. /*
19
20	/*
21	This is a version (aka ptmalloc2) of malloc/free/realloc written by
22	Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
23
24	There have been substantial changes made after the integration into
25	glibc in all parts of the code. Do not look for much commonality
26	with the ptmalloc2 version.
27
28	* Version ptmalloc2-20011215
29	based on:
30	VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
31
32	* Quickstart
33
34	In order to compile this implementation, a Makefile is provided with
35	the ptmalloc2 distribution, which has pre-defined targets for some
36	popular systems (e.g. "make posix" for Posix threads). All that is
37	typically required with regard to compiler flags is the selection of
38	the thread package via defining one out of USE_PTHREADS, USE_THR or
39	USE_SPROC. Check the thread-m.h file for what effects this has.
40	Many/most systems will additionally require USE_TSD_DATA_HACK to be
41	defined, so this is the default for "make posix".
42
43	* Why use this malloc?
44
45	This is not the fastest, most space-conserving, most portable, or
46	most tunable malloc ever written. However it is among the fastest
47	while also being among the most space-conserving, portable and tunable.
48	Consistent balance across these factors results in a good general-purpose
49	allocator for malloc-intensive programs.
50
51	The main properties of the algorithms are:
52	* For large (>= 512 bytes) requests, it is a pure best-fit allocator,
53	with ties normally decided via FIFO (i.e. least recently used).
54	* For small (<= 64 bytes by default) requests, it is a caching
55	allocator, that maintains pools of quickly recycled chunks.
56	* In between, and for combinations of large and small requests, it does
57	the best it can trying to meet both goals at once.
58	* For very large requests (>= 128KB by default), it relies on system
59	memory mapping facilities, if supported.
60
61	For a longer but slightly out of date high-level description, see
62	http://gee.cs.oswego.edu/dl/html/malloc.html
63
64	You may already by default be using a C library containing a malloc
65	that is based on some version of this malloc (for example in
66	linux). You might still want to use the one in this file in order to
67	customize settings or to avoid overheads associated with library
68	versions.
69
70	* Contents, described in more detail in "description of public routines" below.
71
72	Standard (ANSI/SVID/...) functions:
73	malloc(size_t n);
74	calloc(size_t n_elements, size_t element_size);
75	free(void p);*
76	realloc(void p, size_t n);*
77	memalign(size_t alignment, size_t n);
78	valloc(size_t n);
79	mallinfo()
80	mallopt(int parameter_number, int parameter_value)
81
82	Additional functions:
83	independent_calloc(size_t n_elements, size_t size, void chunks[]);*
84	independent_comalloc(size_t n_elements, size_t sizes[], void chunks[]);*
85	pvalloc(size_t n);
86	malloc_trim(size_t pad);
87	malloc_usable_size(void p);*
88	malloc_stats();
89
90	* Vital statistics:
91
92	Supported pointer representation: 4 or 8 bytes
93	Supported size_t representation: 4 or 8 bytes
94	Note that size_t is allowed to be 4 bytes even if pointers are 8.
95	You can adjust this by defining INTERNAL_SIZE_T
96
97	Alignment: 2 sizeof(size_t) (default)*
98	(i.e., 8 byte alignment with 4byte size_t). This suffices for
99	nearly all current machines and C compilers. However, you can
100	define MALLOC_ALIGNMENT to be wider than this if necessary.
101
102	Minimum overhead per allocated chunk: 4 or 8 bytes
103	Each malloced chunk has a hidden word of overhead holding size
104	and status information.
105
106	Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
107	8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
108
109	When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
110	ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
111	needed; 4 (8) for a trailing size field and 8 (16) bytes for
112	free list pointers. Thus, the minimum allocatable size is
113	16/24/32 bytes.
114
115	Even a request for zero bytes (i.e., malloc(0)) returns a
116	pointer to something of the minimum allocatable size.
117
118	The maximum overhead wastage (i.e., number of extra bytes
119	allocated than were requested in malloc) is less than or equal
120	to the minimum size, except for requests >= mmap_threshold that
121	are serviced via mmap(), where the worst case wastage is 2 *
122	sizeof(size_t) bytes plus the remainder from a system page (the
123	minimal mmap unit); typically 4096 or 8192 bytes.
124
125	Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
126	8-byte size_t: 2^64 minus about two pages
127
128	It is assumed that (possibly signed) size_t values suffice to
129	represent chunk sizes. `Possibly signed' is due to the fact
130	that `size_t' may be defined on a system as either a signed or
131	an unsigned type. The ISO C standard says that it must be
132	unsigned, but a few systems are known not to adhere to this.
133	Additionally, even when size_t is unsigned, sbrk (which is by
134	default used to obtain memory from system) accepts signed
135	arguments, and may not be able to handle size_t-wide arguments
136	with negative sign bit. Generally, values that would
137	appear as negative after accounting for overhead and alignment
138	are supported only via mmap(), which does not have this
139	limitation.
140
141	Requests for sizes outside the allowed range will perform an optional
142	failure action and then return null. (Requests may also
143	also fail because a system is out of memory.)
144
145	Thread-safety: thread-safe
146
147	Compliance: I believe it is compliant with the 1997 Single Unix Specification
148	Also SVID/XPG, ANSI C, and probably others as well.
149
150	* Synopsis of compile-time options:
151
152	People have reported using previous versions of this malloc on all
153	versions of Unix, sometimes by tweaking some of the defines
154	below. It has been tested most extensively on Solaris and Linux.
155	People also report using it in stand-alone embedded systems.
156
157	The implementation is in straight, hand-tuned ANSI C. It is not
158	at all modular. (Sorry!) It uses a lot of macros. To be at all
159	usable, this code should be compiled using an optimizing compiler
160	(for example gcc -O3) that can simplify expressions and control
161	paths. (FAQ: some macros import variables as arguments rather than
162	declare locals because people reported that some debuggers
163	otherwise get confused.)
164
165	OPTION DEFAULT VALUE
166
167	Compilation Environment options:
168
169	HAVE_MREMAP 0
170
171	Changing default word sizes:
172
173	INTERNAL_SIZE_T size_t
174
175	Configuration and functionality options:
176
177	USE_PUBLIC_MALLOC_WRAPPERS NOT defined
178	USE_MALLOC_LOCK NOT defined
179	MALLOC_DEBUG NOT defined
180	REALLOC_ZERO_BYTES_FREES 1
181	TRIM_FASTBINS 0
182
183	Options for customizing MORECORE:
184
185	MORECORE sbrk
186	MORECORE_FAILURE -1
187	MORECORE_CONTIGUOUS 1
188	MORECORE_CANNOT_TRIM NOT defined
189	MORECORE_CLEARS 1
190	MMAP_AS_MORECORE_SIZE (1024 1024)*
191
192	Tuning options that are also dynamically changeable via mallopt:
193
194	DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
195	DEFAULT_TRIM_THRESHOLD 128 1024*
196	DEFAULT_TOP_PAD 0
197	DEFAULT_MMAP_THRESHOLD 128 1024*
198	DEFAULT_MMAP_MAX 65536
199
200	There are several other #defined constants and macros that you
201	probably don't want to touch unless you are extending or adapting malloc. /*
202
203	/*
204	void is the pointer type that malloc should say it returns*
205	*/
206
207	#ifndef void
208	#define void void
209	#endif /void/
210
211	#include <stddef.h> /* for size_t */
212	#include <stdlib.h> /* for getenv(), abort() */
213	#include <unistd.h> /* for __libc_enable_secure */
214
215	#include <atomic.h>
216	#include <_itoa.h>
217	#include <bits/wordsize.h>
218	#include <sys/sysinfo.h>
219
220	#include <ldsodefs.h>
221
222	#include <unistd.h>
223	#include <stdio.h> /* needed for malloc_stats */
224	#include <errno.h>
225	#include <assert.h>
226
227	#include <shlib-compat.h>
228
229	/ For uintptr_t. /
230	#include <stdint.h>
231
232	/ For va_arg, va_start, va_end. /
233	#include <stdarg.h>
234
235	/ For MIN, MAX, powerof2. /
236	#include <sys/param.h>
237
238	/ For ALIGN_UP et. al. /
239	#include <libc-pointer-arith.h>
240
241	/ For DIAG_PUSH/POP_NEEDS_COMMENT et al. /
242	#include <libc-diag.h>
243
244	/ For memory tagging. /
245	#include <libc-mtag.h>
246
247	#include <malloc/malloc-internal.h>
248
249	/ For SINGLE_THREAD_P. /
250	#include <sysdep-cancel.h>
251
252	#include <libc-internal.h>
253
254	/ For tcache double-free check. /
255	#include <random-bits.h>
256	#include <sys/random.h>
257	#include <not-cancel.h>
258
259	/*
260	Debugging:
261
262	Because freed chunks may be overwritten with bookkeeping fields, this
263	malloc will often die when freed memory is overwritten by user
264	programs. This can be very effective (albeit in an annoying way)
265	in helping track down dangling pointers.
266
267	If you compile with -DMALLOC_DEBUG, a number of assertion checks are
268	enabled that will catch more memory errors. You probably won't be
269	able to make much sense of the actual assertion errors, but they
270	should help you locate incorrectly overwritten memory. The checking
271	is fairly extensive, and will slow down execution
272	noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
273	will attempt to check every non-mmapped allocated and free chunk in
274	the course of computing the summmaries. (By nature, mmapped regions
275	cannot be checked very much automatically.)
276
277	Setting MALLOC_DEBUG may also be helpful if you are trying to modify
278	this code. The assertions in the check routines spell out in more
279	detail the assumptions and invariants underlying the algorithms.
280
281	Setting MALLOC_DEBUG does NOT provide an automated mechanism for
282	checking that all accesses to malloced memory stay within their
283	bounds. However, there are several add-ons and adaptations of this
284	or other mallocs available that do this.
285	*/
286
287	#ifndef MALLOC_DEBUG
288	#define MALLOC_DEBUG 0
289	#endif
290
291	#if USE_TCACHE
292	/ We want 64 entries. This is an arbitrary limit, which tunables can reduce. /
293	# define TCACHE_MAX_BINS 64
294	# define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1)
295
296	/ Only used to pre-fill the tunables. /
297	# define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
298
299	/ When "x" is from chunksize(). /
300	# define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
301	/ When "x" is a user-provided size. /
302	# define usize2tidx(x) csize2tidx (request2size (x))
303
304	/ With rounding and alignment, the bins are...*
305	idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit)
306	idx 1 bytes 25..40 or 13..20
307	idx 2 bytes 41..56 or 21..28
308	etc. /*
309
310	/ This is another arbitrary limit, which tunables can change. Each*
311	tcache bin will hold at most this number of chunks. /*
312	# define TCACHE_FILL_COUNT 7
313
314	/ Maximum chunks in tcache bins for tunables. This value must fit the range*
315	of tcache->counts[] entries, else they may overflow. /*
316	# define MAX_TCACHE_COUNT UINT16_MAX
317	#endif
318
319	/ Safe-Linking:*
320	Use randomness from ASLR (mmap_base) to protect single-linked lists
321	of Fast-Bins and TCache. That is, mask the "next" pointers of the
322	lists' chunks, and also perform allocation alignment checks on them.
323	This mechanism reduces the risk of pointer hijacking, as was done with
324	Safe-Unlinking in the double-linked lists of Small-Bins.
325	It assumes a minimum page size of 4096 bytes (12 bits). Systems with
326	larger pages provide less entropy, although the pointer mangling
327	still works. /*
328	#define PROTECT_PTR(pos, ptr) \
329	((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
330	#define REVEAL_PTR(ptr) PROTECT_PTR (&ptr, ptr)
331
332	/*
333	The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
334	when p is nonnull. If the macro is nonzero, the realloc call returns NULL;
335	otherwise, the call returns what malloc (0) would. In either case,
336	p is freed. Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
337	implements common historical practice.
338
339	ISO C17 says the realloc call has implementation-defined behavior,
340	and it might not even free p.
341	*/
342
343	#ifndef REALLOC_ZERO_BYTES_FREES
344	#define REALLOC_ZERO_BYTES_FREES 1
345	#endif
346
347	/*
348	TRIM_FASTBINS controls whether free() of a very small chunk can
349	immediately lead to trimming. Setting to true (1) can reduce memory
350	footprint, but will almost always slow down programs that use a lot
351	of small chunks.
352
353	Define this only if you are willing to give up some speed to more
354	aggressively reduce system-level memory footprint when releasing
355	memory in programs that use many small chunks. You can get
356	essentially the same effect by setting MXFAST to 0, but this can
357	lead to even greater slowdowns in programs using many small chunks.
358	TRIM_FASTBINS is an in-between compile-time option, that disables
359	only those chunks bordering topmost memory from being placed in
360	fastbins.
361	*/
362
363	#ifndef TRIM_FASTBINS
364	#define TRIM_FASTBINS 0
365	#endif
366
367	/ Definition for getting more memory from the OS. /
368	#include "morecore.c"
369
370	#define MORECORE (*__glibc_morecore)
371	#define MORECORE_FAILURE 0
372
373	/ Memory tagging. /
374
375	/ Some systems support the concept of tagging (sometimes known as*
376	coloring) memory locations on a fine grained basis. Each memory
377	location is given a color (normally allocated randomly) and
378	pointers are also colored. When the pointer is dereferenced, the
379	pointer's color is checked against the memory's color and if they
380	differ the access is faulted (sometimes lazily).
381
382	We use this in glibc by maintaining a single color for the malloc
383	data structures that are interleaved with the user data and then
384	assigning separate colors for each block allocation handed out. In
385	this way simple buffer overruns will be rapidly detected. When
386	memory is freed, the memory is recolored back to the glibc default
387	so that simple use-after-free errors can also be detected.
388
389	If memory is reallocated the buffer is recolored even if the
390	address remains the same. This has a performance impact, but
391	guarantees that the old pointer cannot mistakenly be reused (code
392	that compares old against new will see a mismatch and will then
393	need to behave as though realloc moved the data to a new location).
394
395	Internal API for memory tagging support.
396
397	The aim is to keep the code for memory tagging support as close to
398	the normal APIs in glibc as possible, so that if tagging is not
399	enabled in the library, or is disabled at runtime then standard
400	operations can continue to be used. Support macros are used to do
401	this:
402
403	void tag_new_zero_region (void ptr, size_t size)
404
405	Allocates a new tag, colors the memory with that tag, zeros the
406	memory and returns a pointer that is correctly colored for that
407	location. The non-tagging version will simply call memset with 0.
408
409	void tag_region (void ptr, size_t size)
410
411	Color the region of memory pointed to by PTR and size SIZE with
412	the color of PTR. Returns the original pointer.
413
414	void tag_new_usable (void ptr)
415
416	Allocate a new random color and use it to color the user region of
417	a chunk; this may include data from the subsequent chunk's header
418	if tagging is sufficiently fine grained. Returns PTR suitably
419	recolored for accessing the memory there.
420
421	void tag_at (void ptr)
422
423	Read the current color of the memory at the address pointed to by
424	PTR (ignoring it's current color) and return PTR recolored to that
425	color. PTR must be valid address in all other respects. When
426	tagging is not enabled, it simply returns the original pointer.
427	*/
428
429	#ifdef USE_MTAG
430	static bool mtag_enabled = false;
431	static int mtag_mmap_flags = `0`;
432	#else
433	# define mtag_enabled false
434	# define mtag_mmap_flags 0
435	#endif
436
437	static __always_inline void *
438	tag_region (void *ptr, size_t size)
439	{
440	if (__glibc_unlikely (mtag_enabled))
441	return __libc_mtag_tag_region (ptr, size);
442	return ptr;
443	}
444
445	static __always_inline void *
446	tag_new_zero_region (void *ptr, size_t size)
447	{
448	if (__glibc_unlikely (mtag_enabled))
449	return __libc_mtag_tag_zero_region (__libc_mtag_new_tag (ptr), size);
450	return memset (ptr, `0`, size);
451	}
452
453	/ Defined later. /
454	static void *
455	tag_new_usable (void *ptr);
456
457	static __always_inline void *
458	tag_at (void *ptr)
459	{
460	if (__glibc_unlikely (mtag_enabled))
461	return __libc_mtag_address_get_tag (ptr);
462	return ptr;
463	}
464
465	#include <string.h>
466
467	/*
468	MORECORE-related declarations. By default, rely on sbrk
469	*/
470
471
472	/*
473	MORECORE is the name of the routine to call to obtain more memory
474	from the system. See below for general guidance on writing
475	alternative MORECORE functions, as well as a version for WIN32 and a
476	sample version for pre-OSX macos.
477	*/
478
479	#ifndef MORECORE
480	#define MORECORE sbrk
481	#endif
482
483	/*
484	MORECORE_FAILURE is the value returned upon failure of MORECORE
485	as well as mmap. Since it cannot be an otherwise valid memory address,
486	and must reflect values of standard sys calls, you probably ought not
487	try to redefine it.
488	*/
489
490	#ifndef MORECORE_FAILURE
491	#define MORECORE_FAILURE (-1)
492	#endif
493
494	/*
495	If MORECORE_CONTIGUOUS is true, take advantage of fact that
496	consecutive calls to MORECORE with positive arguments always return
497	contiguous increasing addresses. This is true of unix sbrk. Even
498	if not defined, when regions happen to be contiguous, malloc will
499	permit allocations spanning regions obtained from different
500	calls. But defining this when applicable enables some stronger
501	consistency checks and space efficiencies.
502	*/
503
504	#ifndef MORECORE_CONTIGUOUS
505	#define MORECORE_CONTIGUOUS 1
506	#endif
507
508	/*
509	Define MORECORE_CANNOT_TRIM if your version of MORECORE
510	cannot release space back to the system when given negative
511	arguments. This is generally necessary only if you are using
512	a hand-crafted MORECORE function that cannot handle negative arguments.
513	*/
514
515	/ #define MORECORE_CANNOT_TRIM /
516
517	/ MORECORE_CLEARS (default 1)*
518	The degree to which the routine mapped to MORECORE zeroes out
519	memory: never (0), only for newly allocated space (1) or always
520	(2). The distinction between (1) and (2) is necessary because on
521	some systems, if the application first decrements and then
522	increments the break value, the contents of the reallocated space
523	are unspecified.
524	*/
525
526	#ifndef MORECORE_CLEARS
527	# define MORECORE_CLEARS 1
528	#endif
529
530
531	/*
532	MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
533	sbrk fails, and mmap is used as a backup. The value must be a
534	multiple of page size. This backup strategy generally applies only
535	when systems have "holes" in address space, so sbrk cannot perform
536	contiguous expansion, but there is still space available on system.
537	On systems for which this is known to be useful (i.e. most linux
538	kernels), this occurs only when programs allocate huge amounts of
539	memory. Between this, and the fact that mmap regions tend to be
540	limited, the size should be large, to avoid too many mmap calls and
541	thus avoid running out of kernel resources. /*
542
543	#ifndef MMAP_AS_MORECORE_SIZE
544	#define MMAP_AS_MORECORE_SIZE (1024 * 1024)
545	#endif
546
547	/*
548	Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
549	large blocks.
550	*/
551
552	#ifndef HAVE_MREMAP
553	#define HAVE_MREMAP 0
554	#endif
555
556	/*
557	This version of malloc supports the standard SVID/XPG mallinfo
558	routine that returns a struct containing usage properties and
559	statistics. It should work on any SVID/XPG compliant system that has
560	a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
561	install such a thing yourself, cut out the preliminary declarations
562	as described above and below and save them in a malloc.h file. But
563	there's no compelling reason to bother to do this.)
564
565	The main declaration needed is the mallinfo struct that is returned
566	(by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
567	bunch of fields that are not even meaningful in this version of
568	malloc. These fields are are instead filled by mallinfo() with
569	other numbers that might be of interest.
570	*/
571
572
573	/ ---------- description of public routines ------------ /
574
575	#if IS_IN (libc)
576	/*
577	malloc(size_t n)
578	Returns a pointer to a newly allocated chunk of at least n bytes, or null
579	if no space is available. Additionally, on failure, errno is
580	set to ENOMEM on ANSI C systems.
581
582	If n is zero, malloc returns a minimum-sized chunk. (The minimum
583	size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
584	systems.) On most systems, size_t is an unsigned type, so calls
585	with negative arguments are interpreted as requests for huge amounts
586	of space, which will often fail. The maximum supported value of n
587	differs across systems, but is in all cases less than the maximum
588	representable value of a size_t.
589	*/
590	void* __libc_malloc(size_t);
591	libc_hidden_proto (__libc_malloc)
592
593	/*
594	free(void p)*
595	Releases the chunk of memory pointed to by p, that had been previously
596	allocated using malloc or a related routine such as realloc.
597	It has no effect if p is null. It can have arbitrary (i.e., bad!)
598	effects if p has already been freed.
599
600	Unless disabled (using mallopt), freeing very large spaces will
601	when possible, automatically trigger operations that give
602	back unused memory to the system, thus reducing program footprint.
603	*/
604	void __libc_free(void*);
605	libc_hidden_proto (__libc_free)
606
607	/*
608	calloc(size_t n_elements, size_t element_size);
609	Returns a pointer to n_elements element_size bytes, with all locations*
610	set to zero.
611	*/
612	void* __libc_calloc(size_t, size_t);
613
614	/*
615	realloc(void p, size_t n)*
616	Returns a pointer to a chunk of size n that contains the same data
617	as does chunk p up to the minimum of (n, p's size) bytes, or null
618	if no space is available.
619
620	The returned pointer may or may not be the same as p. The algorithm
621	prefers extending p when possible, otherwise it employs the
622	equivalent of a malloc-copy-free sequence.
623
624	If p is null, realloc is equivalent to malloc.
625
626	If space is not available, realloc returns null, errno is set (if on
627	ANSI) and p is NOT freed.
628
629	if n is for fewer bytes than already held by p, the newly unused
630	space is lopped off and freed if possible. Unless the #define
631	REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
632	zero (re)allocates a minimum-sized chunk.
633
634	Large chunks that were internally obtained via mmap will always be
635	grown using malloc-copy-free sequences unless the system supports
636	MREMAP (currently only linux).
637
638	The old unix realloc convention of allowing the last-free'd chunk
639	to be used as an argument to realloc is not supported.
640	*/
641	void* __libc_realloc(void*, size_t);
642	libc_hidden_proto (__libc_realloc)
643
644	/*
645	memalign(size_t alignment, size_t n);
646	Returns a pointer to a newly allocated chunk of n bytes, aligned
647	in accord with the alignment argument.
648
649	The alignment argument should be a power of two. If the argument is
650	not a power of two, the nearest greater power is used.
651	8-byte alignment is guaranteed by normal malloc calls, so don't
652	bother calling memalign with an argument of 8 or less.
653
654	Overreliance on memalign is a sure way to fragment space.
655	*/
656	void* __libc_memalign(size_t, size_t);
657	libc_hidden_proto (__libc_memalign)
658
659	/*
660	valloc(size_t n);
661	Equivalent to memalign(pagesize, n), where pagesize is the page
662	size of the system. If the pagesize is unknown, 4096 is used.
663	*/
664	void* __libc_valloc(size_t);
665
666
667
668	/*
669	mallinfo()
670	Returns (by copy) a struct containing various summary statistics:
671
672	arena: current total non-mmapped bytes allocated from system
673	ordblks: the number of free chunks
674	smblks: the number of fastbin blocks (i.e., small chunks that
675	have been freed but not use resused or consolidated)
676	hblks: current number of mmapped regions
677	hblkhd: total bytes held in mmapped regions
678	usmblks: always 0
679	fsmblks: total bytes held in fastbin blocks
680	uordblks: current total allocated space (normal or mmapped)
681	fordblks: total free space
682	keepcost: the maximum number of bytes that could ideally be released
683	back to system via malloc_trim. ("ideally" means that
684	it ignores page restrictions etc.)
685
686	Because these fields are ints, but internal bookkeeping may
687	be kept as longs, the reported values may wrap around zero and
688	thus be inaccurate.
689	*/
690	struct mallinfo2 __libc_mallinfo2(void);
691	libc_hidden_proto (__libc_mallinfo2)
692
693	struct mallinfo __libc_mallinfo(void);
694
695
696	/*
697	pvalloc(size_t n);
698	Equivalent to valloc(minimum-page-that-holds(n)), that is,
699	round up n to nearest pagesize.
700	*/
701	void* __libc_pvalloc(size_t);
702
703	/*
704	malloc_trim(size_t pad);
705
706	If possible, gives memory back to the system (via negative
707	arguments to sbrk) if there is unused memory at the `high' end of
708	the malloc pool. You can call this after freeing large blocks of
709	memory to potentially reduce the system-level memory requirements
710	of a program. However, it cannot guarantee to reduce memory. Under
711	some allocation patterns, some large free blocks of memory will be
712	locked between two used chunks, so they cannot be given back to
713	the system.
714
715	The `pad' argument to malloc_trim represents the amount of free
716	trailing space to leave untrimmed. If this argument is zero,
717	only the minimum amount of memory to maintain internal data
718	structures will be left (one page or less). Non-zero arguments
719	can be supplied to maintain enough trailing space to service
720	future expected allocations without having to re-obtain memory
721	from the system.
722
723	Malloc_trim returns 1 if it actually released any memory, else 0.
724	On systems that do not support "negative sbrks", it will always
725	return 0.
726	*/
727	int __malloc_trim(size_t);
728
729	/*
730	malloc_usable_size(void p);*
731
732	Returns the number of bytes you can actually use in
733	an allocated chunk, which may be more than you requested (although
734	often not) due to alignment and minimum size constraints.
735	You can use this many bytes without worrying about
736	overwriting other allocated objects. This is not a particularly great
737	programming practice. malloc_usable_size can be more useful in
738	debugging and assertions, for example:
739
740	p = malloc(n);
741	assert(malloc_usable_size(p) >= 256);
742
743	*/
744	size_t __malloc_usable_size(void*);
745
746	/*
747	malloc_stats();
748	Prints on stderr the amount of space obtained from the system (both
749	via sbrk and mmap), the maximum amount (which may be more than
750	current if malloc_trim and/or munmap got called), and the current
751	number of bytes allocated via malloc (or realloc, etc) but not yet
752	freed. Note that this is the number of bytes allocated, not the
753	number requested. It will be larger than the number requested
754	because of alignment and bookkeeping overhead. Because it includes
755	alignment wastage as being in use, this figure may be greater than
756	zero even when no user-level chunks are allocated.
757
758	The reported current and maximum system memory can be inaccurate if
759	a program makes other calls to system memory allocation functions
760	(normally sbrk) outside of malloc.
761
762	malloc_stats prints only the most commonly interesting statistics.
763	More information can be obtained by calling mallinfo.
764
765	*/
766	void __malloc_stats(void);
767
768	/*
769	posix_memalign(void memptr, size_t alignment, size_t size);
770
771	POSIX wrapper like memalign(), checking for validity of size.
772	*/
773	int __posix_memalign(void **, size_t, size_t);
774	#endif /* IS_IN (libc) */
775
776	/*
777	mallopt(int parameter_number, int parameter_value)
778	Sets tunable parameters The format is to provide a
779	(parameter-number, parameter-value) pair. mallopt then sets the
780	corresponding parameter to the argument value if it can (i.e., so
781	long as the value is meaningful), and returns 1 if successful else
782	0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
783	normally defined in malloc.h. Only one of these (M_MXFAST) is used
784	in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
785	so setting them has no effect. But this malloc also supports four
786	other options in mallopt. See below for details. Briefly, supported
787	parameters are as follows (listed defaults are for "typical"
788	configurations).
789
790	Symbol param # default allowed param values
791	M_MXFAST 1 64 0-80 (0 disables fastbins)
792	M_TRIM_THRESHOLD -1 1281024 any (-1U disables trimming)*
793	M_TOP_PAD -2 0 any
794	M_MMAP_THRESHOLD -3 1281024 any (or 0 if no MMAP support)*
795	M_MMAP_MAX -4 65536 any (0 disables use of mmap)
796	*/
797	int __libc_mallopt(int, int);
798	#if IS_IN (libc)
799	libc_hidden_proto (__libc_mallopt)
800	#endif
801
802	/ mallopt tuning options /
803
804	/*
805	M_MXFAST is the maximum request size used for "fastbins", special bins
806	that hold returned chunks without consolidating their spaces. This
807	enables future requests for chunks of the same size to be handled
808	very quickly, but can increase fragmentation, and thus increase the
809	overall memory footprint of a program.
810
811	This malloc manages fastbins very conservatively yet still
812	efficiently, so fragmentation is rarely a problem for values less
813	than or equal to the default. The maximum supported value of MXFAST
814	is 80. You wouldn't want it any higher than this anyway. Fastbins
815	are designed especially for use with many small structs, objects or
816	strings -- the default handles structs/objects/arrays with sizes up
817	to 8 4byte fields, or small strings representing words, tokens,
818	etc. Using fastbins for larger objects normally worsens
819	fragmentation without improving speed.
820
821	M_MXFAST is set in REQUEST size units. It is internally used in
822	chunksize units, which adds padding and alignment. You can reduce
823	M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
824	algorithm to be a closer approximation of fifo-best-fit in all cases,
825	not just for larger requests, but will generally cause it to be
826	slower.
827	*/
828
829
830	/ M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h /
831	#ifndef M_MXFAST
832	#define M_MXFAST 1
833	#endif
834
835	#ifndef DEFAULT_MXFAST
836	#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
837	#endif
838
839
840	/*
841	M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
842	to keep before releasing via malloc_trim in free().
843
844	Automatic trimming is mainly useful in long-lived programs.
845	Because trimming via sbrk can be slow on some systems, and can
846	sometimes be wasteful (in cases where programs immediately
847	afterward allocate more large chunks) the value should be high
848	enough so that your overall system performance would improve by
849	releasing this much memory.
850
851	The trim threshold and the mmap control parameters (see below)
852	can be traded off with one another. Trimming and mmapping are
853	two different ways of releasing unused memory back to the
854	system. Between these two, it is often possible to keep
855	system-level demands of a long-lived program down to a bare
856	minimum. For example, in one test suite of sessions measuring
857	the XF86 X server on Linux, using a trim threshold of 128K and a
858	mmap threshold of 192K led to near-minimal long term resource
859	consumption.
860
861	If you are using this malloc in a long-lived program, it should
862	pay to experiment with these values. As a rough guide, you
863	might set to a value close to the average size of a process
864	(program) running on your system. Releasing this much memory
865	would allow such a process to run in memory. Generally, it's
866	worth it to tune for trimming rather tham memory mapping when a
867	program undergoes phases where several large chunks are
868	allocated and released in ways that can reuse each other's
869	storage, perhaps mixed with phases where there are no such
870	chunks at all. And in well-behaved long-lived programs,
871	controlling release of large blocks via trimming versus mapping
872	is usually faster.
873
874	However, in most programs, these parameters serve mainly as
875	protection against the system-level effects of carrying around
876	massive amounts of unneeded memory. Since frequent calls to
877	sbrk, mmap, and munmap otherwise degrade performance, the default
878	parameters are set to relatively high values that serve only as
879	safeguards.
880
881	The trim value It must be greater than page size to have any useful
882	effect. To disable trimming completely, you can set to
883	(unsigned long)(-1)
884
885	Trim settings interact with fastbin (MXFAST) settings: Unless
886	TRIM_FASTBINS is defined, automatic trimming never takes place upon
887	freeing a chunk with size less than or equal to MXFAST. Trimming is
888	instead delayed until subsequent freeing of larger chunks. However,
889	you can still force an attempted trim by calling malloc_trim.
890
891	Also, trimming is not generally possible in cases where
892	the main arena is obtained via mmap.
893
894	Note that the trick some people use of mallocing a huge space and
895	then freeing it at program startup, in an attempt to reserve system
896	memory, doesn't have the intended effect under automatic trimming,
897	since that memory will immediately be returned to the system.
898	*/
899
900	#define M_TRIM_THRESHOLD -1
901
902	#ifndef DEFAULT_TRIM_THRESHOLD
903	#define DEFAULT_TRIM_THRESHOLD (128 * 1024)
904	#endif
905
906	/*
907	M_TOP_PAD is the amount of extra `padding' space to allocate or
908	retain whenever sbrk is called. It is used in two ways internally:
909
910	* When sbrk is called to extend the top of the arena to satisfy
911	a new malloc request, this much padding is added to the sbrk
912	request.
913
914	* When malloc_trim is called automatically from free(),
915	it is used as the `pad' argument.
916
917	In both cases, the actual amount of padding is rounded
918	so that the end of the arena is always a system page boundary.
919
920	The main reason for using padding is to avoid calling sbrk so
921	often. Having even a small pad greatly reduces the likelihood
922	that nearly every malloc request during program start-up (or
923	after trimming) will invoke sbrk, which needlessly wastes
924	time.
925
926	Automatic rounding-up to page-size units is normally sufficient
927	to avoid measurable overhead, so the default is 0. However, in
928	systems where sbrk is relatively slow, it can pay to increase
929	this value, at the expense of carrying around more memory than
930	the program needs.
931	*/
932
933	#define M_TOP_PAD -2
934
935	#ifndef DEFAULT_TOP_PAD
936	#define DEFAULT_TOP_PAD (0)
937	#endif
938
939	/*
940	MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
941	adjusted MMAP_THRESHOLD.
942	*/
943
944	#ifndef DEFAULT_MMAP_THRESHOLD_MIN
945	#define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
946	#endif
947
948	#ifndef DEFAULT_MMAP_THRESHOLD_MAX
949	/ For 32-bit platforms we cannot increase the maximum mmap*
950	threshold much because it is also the minimum value for the
951	maximum heap size and its alignment. Going above 512k (i.e., 1M
952	for new heaps) wastes too much address space. /*
953	# if __WORDSIZE == 32
954	# define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
955	# else
956	# define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
957	# endif
958	#endif
959
960	/*
961	M_MMAP_THRESHOLD is the request size threshold for using mmap()
962	to service a request. Requests of at least this size that cannot
963	be allocated using already-existing space will be serviced via mmap.
964	(If enough normal freed space already exists it is used instead.)
965
966	Using mmap segregates relatively large chunks of memory so that
967	they can be individually obtained and released from the host
968	system. A request serviced through mmap is never reused by any
969	other request (at least not directly; the system may just so
970	happen to remap successive requests to the same locations).
971
972	Segregating space in this way has the benefits that:
973
974	1. Mmapped space can ALWAYS be individually released back
975	to the system, which helps keep the system level memory
976	demands of a long-lived program low.
977	2. Mapped memory can never become `locked' between
978	other chunks, as can happen with normally allocated chunks, which
979	means that even trimming via malloc_trim would not release them.
980	3. On some systems with "holes" in address spaces, mmap can obtain
981	memory that sbrk cannot.
982
983	However, it has the disadvantages that:
984
985	1. The space cannot be reclaimed, consolidated, and then
986	used to service later requests, as happens with normal chunks.
987	2. It can lead to more wastage because of mmap page alignment
988	requirements
989	3. It causes malloc performance to be more dependent on host
990	system memory management support routines which may vary in
991	implementation quality and may impose arbitrary
992	limitations. Generally, servicing a request via normal
993	malloc steps is faster than going through a system's mmap.
994
995	The advantages of mmap nearly always outweigh disadvantages for
996	"large" chunks, but the value of "large" varies across systems. The
997	default is an empirically derived value that works well in most
998	systems.
999
1000
1001	Update in 2006:
1002	The above was written in 2001. Since then the world has changed a lot.
1003	Memory got bigger. Applications got bigger. The virtual address space
1004	layout in 32 bit linux changed.
1005
1006	In the new situation, brk() and mmap space is shared and there are no
1007	artificial limits on brk size imposed by the kernel. What is more,
1008	applications have started using transient allocations larger than the
1009	128Kb as was imagined in 2001.
1010
1011	The price for mmap is also high now; each time glibc mmaps from the
1012	kernel, the kernel is forced to zero out the memory it gives to the
1013	application. Zeroing memory is expensive and eats a lot of cache and
1014	memory bandwidth. This has nothing to do with the efficiency of the
1015	virtual memory system, by doing mmap the kernel just has no choice but
1016	to zero.
1017
1018	In 2001, the kernel had a maximum size for brk() which was about 800
1019	megabytes on 32 bit x86, at that point brk() would hit the first
1020	mmaped shared libaries and couldn't expand anymore. With current 2.6
1021	kernels, the VA space layout is different and brk() and mmap
1022	both can span the entire heap at will.
1023
1024	Rather than using a static threshold for the brk/mmap tradeoff,
1025	we are now using a simple dynamic one. The goal is still to avoid
1026	fragmentation. The old goals we kept are
1027	1) try to get the long lived large allocations to use mmap()
1028	2) really large allocations should always use mmap()
1029	and we're adding now:
1030	3) transient allocations should use brk() to avoid forcing the kernel
1031	having to zero memory over and over again
1032
1033	The implementation works with a sliding threshold, which is by default
1034	limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
1035	out at 128Kb as per the 2001 default.
1036
1037	This allows us to satisfy requirement 1) under the assumption that long
1038	lived allocations are made early in the process' lifespan, before it has
1039	started doing dynamic allocations of the same size (which will
1040	increase the threshold).
1041
1042	The upperbound on the threshold satisfies requirement 2)
1043
1044	The threshold goes up in value when the application frees memory that was
1045	allocated with the mmap allocator. The idea is that once the application
1046	starts freeing memory of a certain size, it's highly probable that this is
1047	a size the application uses for transient allocations. This estimator
1048	is there to satisfy the new third requirement.
1049
1050	*/
1051
1052	#define M_MMAP_THRESHOLD -3
1053
1054	#ifndef DEFAULT_MMAP_THRESHOLD
1055	#define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
1056	#endif
1057
1058	/*
1059	M_MMAP_MAX is the maximum number of requests to simultaneously
1060	service using mmap. This parameter exists because
1061	some systems have a limited number of internal tables for
1062	use by mmap, and using more than a few of them may degrade
1063	performance.
1064
1065	The default is set to a value that serves only as a safeguard.
1066	Setting to 0 disables use of mmap for servicing large requests.
1067	*/
1068
1069	#define M_MMAP_MAX -4
1070
1071	#ifndef DEFAULT_MMAP_MAX
1072	#define DEFAULT_MMAP_MAX (65536)
1073	#endif
1074
1075	#include <malloc.h>
1076
1077	#ifndef RETURN_ADDRESS
1078	#define RETURN_ADDRESS(X_) (NULL)
1079	#endif
1080
1081	/ Forward declarations. /
1082	struct malloc_chunk;
1083	typedef struct malloc_chunk* mchunkptr;
1084
1085	/ Internal routines. /
1086
1087	static void* _int_malloc(mstate, size_t);
1088	static void _int_free(mstate, mchunkptr, int);
1089	static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
1090	INTERNAL_SIZE_T);
1091	static void* _int_memalign(mstate, size_t, size_t);
1092	#if IS_IN (libc)
1093	static void* _mid_memalign(size_t, size_t, void *);
1094	#endif
1095
1096	static void malloc_printerr(const char str) __attribute__* ((noreturn));
1097
1098	static void munmap_chunk(mchunkptr p);
1099	#if HAVE_MREMAP
1100	static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
1101	#endif
1102
1103	static size_t musable (void *mem);
1104
1105	/ ------------------ MMAP support ------------------ /
1106
1107
1108	#include <fcntl.h>
1109	#include <sys/mman.h>
1110
1111	#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1112	# define MAP_ANONYMOUS MAP_ANON
1113	#endif
1114
1115	#define MMAP(addr, size, prot, flags) \
1116	__mmap((addr), (size), (prot), (flags)\|MAP_ANONYMOUS\|MAP_PRIVATE, -1, 0)
1117
1118
1119	/*
1120	----------------------- Chunk representations -----------------------
1121	*/
1122
1123
1124	/*
1125	This struct declaration is misleading (but accurate and necessary).
1126	It declares a "view" into memory allowing access to necessary
1127	fields at known offsets from a given base. See explanation below.
1128	*/
1129
1130	struct malloc_chunk {
1131
1132	INTERNAL_SIZE_T mchunk_prev_size; / Size of previous chunk (if free). /
1133	INTERNAL_SIZE_T mchunk_size; / Size in bytes, including overhead. /
1134
1135	struct malloc_chunk* fd; / double links -- used only if free. /
1136	struct malloc_chunk* bk;
1137
1138	/ Only used for large blocks: pointer to next larger size. /
1139	struct malloc_chunk* fd_nextsize; / double links -- used only if free. /
1140	struct malloc_chunk* bk_nextsize;
1141	};
1142
1143
1144	/*
1145	malloc_chunk details:
1146
1147	(The following includes lightly edited explanations by Colin Plumb.)
1148
1149	Chunks of memory are maintained using a `boundary tag' method as
1150	described in e.g., Knuth or Standish. (See the paper by Paul
1151	Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1152	survey of such techniques.) Sizes of free chunks are stored both
1153	in the front of each chunk and at the end. This makes
1154	consolidating fragmented chunks into bigger chunks very fast. The
1155	size fields also hold bits representing whether chunks are free or
1156	in use.
1157
1158	An allocated chunk looks like this:
1159
1160
1161	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1162	\| Size of previous chunk, if unallocated (P clear) \|
1163	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1164	\| Size of chunk, in bytes \|A\|M\|P\|
1165	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1166	\| User data starts here... .
1167	. .
1168	. (malloc_usable_size() bytes) .
1169	. \|
1170	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1171	\| (size of chunk, but used for application data) \|
1172	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1173	\| Size of next chunk, in bytes \|A\|0\|1\|
1174	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1175
1176	Where "chunk" is the front of the chunk for the purpose of most of
1177	the malloc code, but "mem" is the pointer that is returned to the
1178	user. "Nextchunk" is the beginning of the next contiguous chunk.
1179
1180	Chunks always begin on even word boundaries, so the mem portion
1181	(which is returned to the user) is also on an even word boundary, and
1182	thus at least double-word aligned.
1183
1184	Free chunks are stored in circular doubly-linked lists, and look like this:
1185
1186	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1187	\| Size of previous chunk, if unallocated (P clear) \|
1188	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1189	`head:' \| Size of chunk, in bytes \|A\|0\|P\|
1190	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1191	\| Forward pointer to next chunk in list \|
1192	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1193	\| Back pointer to previous chunk in list \|
1194	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1195	\| Unused space (may be 0 bytes long) .
1196	. .
1197	. \|
1198	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1199	`foot:' \| Size of chunk, in bytes \|
1200	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1201	\| Size of next chunk, in bytes \|A\|0\|0\|
1202	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1203
1204	The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1205	chunk size (which is always a multiple of two words), is an in-use
1206	bit for the previous* chunk. If that bit is clear, then the*
1207	word before the current chunk size contains the previous chunk
1208	size, and can be used to find the front of the previous chunk.
1209	The very first chunk allocated always has this bit set,
1210	preventing access to non-existent (or non-owned) memory. If
1211	prev_inuse is set for any given chunk, then you CANNOT determine
1212	the size of the previous chunk, and might even get a memory
1213	addressing fault when trying to do so.
1214
1215	The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
1216	main arena, described by the main_arena variable. When additional
1217	threads are spawned, each thread receives its own arena (up to a
1218	configurable limit, after which arenas are reused for multiple
1219	threads), and the chunks in these arenas have the A bit set. To
1220	find the arena for a chunk on such a non-main arena, heap_for_ptr
1221	performs a bit mask operation and indirection through the ar_ptr
1222	member of the per-heap header heap_info (see arena.c).
1223
1224	Note that the `foot' of the current chunk is actually represented
1225	as the prev_size of the NEXT chunk. This makes it easier to
1226	deal with alignments etc but can be very confusing when trying
1227	to extend or adapt this code.
1228
1229	The three exceptions to all this are:
1230
1231	1. The special chunk `top' doesn't bother using the
1232	trailing size field since there is no next contiguous chunk
1233	that would have to index off it. After initialization, `top'
1234	is forced to always exist. If it would become less than
1235	MINSIZE bytes long, it is replenished.
1236
1237	2. Chunks allocated via mmap, which have the second-lowest-order
1238	bit M (IS_MMAPPED) set in their size fields. Because they are
1239	allocated one-by-one, each must contain its own trailing size
1240	field. If the M bit is set, the other bits are ignored
1241	(because mmapped chunks are neither in an arena, nor adjacent
1242	to a freed chunk). The M bit is also used for chunks which
1243	originally came from a dumped heap via malloc_set_state in
1244	hooks.c.
1245
1246	3. Chunks in fastbins are treated as allocated chunks from the
1247	point of view of the chunk allocator. They are consolidated
1248	with their neighbors only in bulk, in malloc_consolidate.
1249	*/
1250
1251	/*
1252	---------- Size and alignment checks and conversions ----------
1253	*/
1254
1255	/ Conversion from malloc headers to user pointers, and back. When*
1256	using memory tagging the user data and the malloc data structure
1257	headers have distinct tags. Converting fully from one to the other
1258	involves extracting the tag at the other address and creating a
1259	suitable pointer using it. That can be quite expensive. There are
1260	cases when the pointers are not dereferenced (for example only used
1261	for alignment check) so the tags are not relevant, and there are
1262	cases when user data is not tagged distinctly from malloc headers
1263	(user data is untagged because tagging is done late in malloc and
1264	early in free). User memory tagging across internal interfaces:
1265
1266	sysmalloc: Returns untagged memory.
1267	_int_malloc: Returns untagged memory.
1268	_int_free: Takes untagged memory.
1269	_int_memalign: Returns untagged memory.
1270	_int_memalign: Returns untagged memory.
1271	_mid_memalign: Returns tagged memory.
1272	_int_realloc: Takes and returns tagged memory.
1273	*/
1274
1275	/ The chunk header is two SIZE_SZ elements, but this is used widely, so*
1276	we define it here for clarity later. /*
1277	#define CHUNK_HDR_SZ (2 * SIZE_SZ)
1278
1279	/ Convert a chunk address to a user mem pointer without correcting*
1280	the tag. /*
1281	#define chunk2mem(p) ((void)((char)(p) + CHUNK_HDR_SZ))
1282
1283	/ Convert a chunk address to a user mem pointer and extract the right tag. /
1284	#define chunk2mem_tag(p) ((void)tag_at ((char)(p) + CHUNK_HDR_SZ))
1285
1286	/ Convert a user mem pointer to a chunk address and extract the right tag. /
1287	#define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
1288
1289	/ The smallest possible chunk /
1290	#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1291
1292	/ The smallest size we can malloc is an aligned minimal chunk /
1293
1294	#define MINSIZE \
1295	(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1296
1297	/ Check if m has acceptable alignment /
1298
1299	#define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1300
1301	#define misaligned_chunk(p) \
1302	((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
1303	& MALLOC_ALIGN_MASK)
1304
1305	/ pad request bytes into a usable size -- internal version /
1306	/ Note: This must be a macro that evaluates to a compile time constant*
1307	if passed a literal constant. /*
1308	#define request2size(req) \
1309	(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1310	MINSIZE : \
1311	((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1312
1313	/ Check if REQ overflows when padded and aligned and if the resulting*
1314	value is less than PTRDIFF_T. Returns the requested size or
1315	MINSIZE in case the value is less than MINSIZE, or 0 if any of the
1316	previous checks fail. /*
1317	static inline size_t
1318	checked_request2size (size_t req) __nonnull (`1`)
1319	{
1320	if (__glibc_unlikely (req > PTRDIFF_MAX))
1321	return `0`;
1322
1323	/ When using tagged memory, we cannot share the end of the user*
1324	block with the header for the next chunk, so ensure that we
1325	allocate blocks that are rounded up to the granule size. Take
1326	care not to overflow from close to MAX_SIZE_T to a small
1327	number. Ideally, this would be part of request2size(), but that
1328	must be a macro that produces a compile time constant if passed
1329	a constant literal. /*
1330	if (__glibc_unlikely (mtag_enabled))
1331	{
1332	/ Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551. /
1333	asm ("");
1334
1335	req = (req + (__MTAG_GRANULE_SIZE - `1`)) &
1336	~(size_t)(__MTAG_GRANULE_SIZE - `1`);
1337	}
1338
1339	return request2size (req);
1340	}
1341
1342	/*
1343	--------------- Physical chunk operations ---------------
1344	*/
1345
1346
1347	/ size field is or'ed with PREV_INUSE when previous adjacent chunk in use /
1348	#define PREV_INUSE 0x1
1349
1350	/ extract inuse bit of previous chunk /
1351	#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
1352
1353
1354	/ size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() /
1355	#define IS_MMAPPED 0x2
1356
1357	/ check for mmap()'ed chunk /
1358	#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
1359
1360
1361	/ size field is or'ed with NON_MAIN_ARENA if the chunk was obtained*
1362	from a non-main arena. This is only set immediately before handing
1363	the chunk to the user, if necessary. /*
1364	#define NON_MAIN_ARENA 0x4
1365
1366	/ Check for chunk from main arena. /
1367	#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
1368
1369	/ Mark a chunk as not being on the main arena. /
1370	#define set_non_main_arena(p) ((p)->mchunk_size \|= NON_MAIN_ARENA)
1371
1372
1373	/*
1374	Bits to mask off when extracting size
1375
1376	Note: IS_MMAPPED is intentionally not masked off from size field in
1377	macros for which mmapped chunks should never be seen. This should
1378	cause helpful core dumps to occur if it is tried by accident by
1379	people extending or adapting this malloc.
1380	*/
1381	#define SIZE_BITS (PREV_INUSE \| IS_MMAPPED \| NON_MAIN_ARENA)
1382
1383	/ Get size, ignoring use bits /
1384	#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
1385
1386	/ Like chunksize, but do not mask SIZE_BITS. /
1387	#define chunksize_nomask(p) ((p)->mchunk_size)
1388
1389	/ Ptr to next physical malloc_chunk. /
1390	#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
1391
1392	/ Size of the chunk below P. Only valid if !prev_inuse (P). /
1393	#define prev_size(p) ((p)->mchunk_prev_size)
1394
1395	/ Set the size of the chunk below P. Only valid if !prev_inuse (P). /
1396	#define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
1397
1398	/ Ptr to previous physical malloc_chunk. Only valid if !prev_inuse (P). /
1399	#define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
1400
1401	/ Treat space at ptr + offset as a chunk /
1402	#define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s)))
1403
1404	/ extract p's inuse bit /
1405	#define inuse(p) \
1406	((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
1407
1408	/ set/clear chunk as being inuse without otherwise disturbing /
1409	#define set_inuse(p) \
1410	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size \|= PREV_INUSE
1411
1412	#define clear_inuse(p) \
1413	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
1414
1415
1416	/ check/set/clear inuse bits in known places /
1417	#define inuse_bit_at_offset(p, s) \
1418	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
1419
1420	#define set_inuse_bit_at_offset(p, s) \
1421	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size \|= PREV_INUSE)
1422
1423	#define clear_inuse_bit_at_offset(p, s) \
1424	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
1425
1426
1427	/ Set size at head, without disturbing its use bit /
1428	#define set_head_size(p, s) ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) \| (s)))
1429
1430	/ Set size/use field /
1431	#define set_head(p, s) ((p)->mchunk_size = (s))
1432
1433	/ Set size at footer (only when chunk is not in use) /
1434	#define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
1435
1436	#pragma GCC poison mchunk_size
1437	#pragma GCC poison mchunk_prev_size
1438
1439	/ This is the size of the real usable data in the chunk. Not valid for*
1440	dumped heap chunks. /*
1441	#define memsize(p) \
1442	(__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
1443	chunksize (p) - CHUNK_HDR_SZ : \
1444	chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
1445
1446	/ If memory tagging is enabled the layout changes to accommodate the granule*
1447	size, this is wasteful for small allocations so not done by default.
1448	Both the chunk header and user data has to be granule aligned. /*
1449	_Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
1450	"memory tagging is not supported with large granule.");
1451
1452	static __always_inline void *
1453	tag_new_usable (void *ptr)
1454	{
1455	if (__glibc_unlikely (mtag_enabled) && ptr)
1456	{
1457	mchunkptr cp = mem2chunk(ptr);
1458	ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp));
1459	}
1460	return ptr;
1461	}
1462
1463	/*
1464	-------------------- Internal data structures --------------------
1465
1466	All internal state is held in an instance of malloc_state defined
1467	below. There are no other static variables, except in two optional
1468	cases:
1469	* If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1470	* If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1471	for mmap.
1472
1473	Beware of lots of tricks that minimize the total bookkeeping space
1474	requirements. The result is a little over 1K bytes (for 4byte
1475	pointers and size_t.)
1476	*/
1477
1478	/*
1479	Bins
1480
1481	An array of bin headers for free chunks. Each bin is doubly
1482	linked. The bins are approximately proportionally (log) spaced.
1483	There are a lot of these bins (128). This may look excessive, but
1484	works very well in practice. Most bins hold sizes that are
1485	unusual as malloc request sizes, but are more usual for fragments
1486	and consolidated sets of chunks, which is what these bins hold, so
1487	they can be found quickly. All procedures maintain the invariant
1488	that no consolidated chunk physically borders another one, so each
1489	chunk in a list is known to be preceeded and followed by either
1490	inuse chunks or the ends of memory.
1491
1492	Chunks in bins are kept in size order, with ties going to the
1493	approximately least recently used chunk. Ordering isn't needed
1494	for the small bins, which all contain the same-sized chunks, but
1495	facilitates best-fit allocation for larger chunks. These lists
1496	are just sequential. Keeping them in order almost never requires
1497	enough traversal to warrant using fancier ordered data
1498	structures.
1499
1500	Chunks of the same size are linked with the most
1501	recently freed at the front, and allocations are taken from the
1502	back. This results in LRU (FIFO) allocation order, which tends
1503	to give each chunk an equal opportunity to be consolidated with
1504	adjacent freed chunks, resulting in larger free chunks and less
1505	fragmentation.
1506
1507	To simplify use in double-linked lists, each bin header acts
1508	as a malloc_chunk. This avoids special-casing for headers.
1509	But to conserve space and improve locality, we allocate
1510	only the fd/bk pointers of bins, and then use repositioning tricks
1511	to treat these as the fields of a malloc_chunk.*
1512	*/
1513
1514	typedef struct malloc_chunk *mbinptr;
1515
1516	/ addressing -- note that bin_at(0) does not exist /
1517	#define bin_at(m, i) \
1518	(mbinptr) (((char ) &((m)->bins[((i) - 1) 2])) \
1519	- offsetof (struct malloc_chunk, fd))
1520
1521	/ analog of ++bin /
1522	#define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
1523
1524	/ Reminders about list directionality within bins /
1525	#define first(b) ((b)->fd)
1526	#define last(b) ((b)->bk)
1527
1528	/*
1529	Indexing
1530
1531	Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1532	8 bytes apart. Larger bins are approximately logarithmically spaced:
1533
1534	64 bins of size 8
1535	32 bins of size 64
1536	16 bins of size 512
1537	8 bins of size 4096
1538	4 bins of size 32768
1539	2 bins of size 262144
1540	1 bin of size what's left
1541
1542	There is actually a little bit of slop in the numbers in bin_index
1543	for the sake of speed. This makes no difference elsewhere.
1544
1545	The bins top out around 1MB because we expect to service large
1546	requests via mmap.
1547
1548	Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1549	a valid chunk size the small bins are bumped up one.
1550	*/
1551
1552	#define NBINS 128
1553	#define NSMALLBINS 64
1554	#define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1555	#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
1556	#define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1557
1558	#define in_smallbin_range(sz) \
1559	((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
1560
1561	#define smallbin_index(sz) \
1562	((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
1563	+ SMALLBIN_CORRECTION)
1564
1565	#define largebin_index_32(sz) \
1566	(((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\
1567	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1568	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1569	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1570	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1571	126)
1572
1573	#define largebin_index_32_big(sz) \
1574	(((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\
1575	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1576	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1577	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1578	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1579	126)
1580
1581	// XXX It remains to be seen whether it is good to keep the widths of
1582	// XXX the buckets the same or whether it should be scaled by a factor
1583	// XXX of two as well.
1584	#define largebin_index_64(sz) \
1585	(((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\
1586	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1587	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1588	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1589	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1590	126)
1591
1592	#define largebin_index(sz) \
1593	(SIZE_SZ == 8 ? largebin_index_64 (sz) \
1594	: MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1595	: largebin_index_32 (sz))
1596
1597	#define bin_index(sz) \
1598	((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
1599
1600	/ Take a chunk off a bin list. /
1601	static void
1602	unlink_chunk (mstate av, mchunkptr p)
1603	{
1604	if (chunksize (p) != prev_size (next_chunk (p)))
1605	malloc_printerr ("corrupted size vs. prev_size");
1606
1607	mchunkptr fd = p->fd;
1608	mchunkptr bk = p->bk;
1609
1610	if (__builtin_expect (fd->bk != p \|\| bk->fd != p, `0`))
1611	malloc_printerr ("corrupted double-linked list");
1612
1613	fd->bk = bk;
1614	bk->fd = fd;
1615	if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
1616	{
1617	if (p->fd_nextsize->bk_nextsize != p
1618	\|\| p->bk_nextsize->fd_nextsize != p)
1619	malloc_printerr ("corrupted double-linked list (not small)");
1620
1621	if (fd->fd_nextsize == NULL)
1622	{
1623	if (p->fd_nextsize == p)
1624	fd->fd_nextsize = fd->bk_nextsize = fd;
1625	else
1626	{
1627	fd->fd_nextsize = p->fd_nextsize;
1628	fd->bk_nextsize = p->bk_nextsize;
1629	p->fd_nextsize->bk_nextsize = fd;
1630	p->bk_nextsize->fd_nextsize = fd;
1631	}
1632	}
1633	else
1634	{
1635	p->fd_nextsize->bk_nextsize = p->bk_nextsize;
1636	p->bk_nextsize->fd_nextsize = p->fd_nextsize;
1637	}
1638	}
1639	}
1640
1641	/*
1642	Unsorted chunks
1643
1644	All remainders from chunk splits, as well as all returned chunks,
1645	are first placed in the "unsorted" bin. They are then placed
1646	in regular bins after malloc gives them ONE chance to be used before
1647	binning. So, basically, the unsorted_chunks list acts as a queue,
1648	with chunks being placed on it in free (and malloc_consolidate),
1649	and taken off (to be either used or placed in bins) in malloc.
1650
1651	The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1652	does not have to be taken into account in size comparisons.
1653	*/
1654
1655	/ The otherwise unindexable 1-bin is used to hold unsorted chunks. /
1656	#define unsorted_chunks(M) (bin_at (M, 1))
1657
1658	/*
1659	Top
1660
1661	The top-most available chunk (i.e., the one bordering the end of
1662	available memory) is treated specially. It is never included in
1663	any bin, is used only if no other chunk is available, and is
1664	released back to the system if it is very large (see
1665	M_TRIM_THRESHOLD). Because top initially
1666	points to its own bin with initial zero size, thus forcing
1667	extension on the first malloc request, we avoid having any special
1668	code in malloc to check whether it even exists yet. But we still
1669	need to do so when getting memory from system, so we make
1670	initial_top treat the bin as a legal but unusable chunk during the
1671	interval between initialization and the first call to
1672	sysmalloc. (This is somewhat delicate, since it relies on
1673	the 2 preceding words to be zero during this interval as well.)
1674	*/
1675
1676	/ Conveniently, the unsorted bin can be used as dummy top on first call /
1677	#define initial_top(M) (unsorted_chunks (M))
1678
1679	/*
1680	Binmap
1681
1682	To help compensate for the large number of bins, a one-level index
1683	structure is used for bin-by-bin searching. `binmap' is a
1684	bitvector recording whether bins are definitely empty so they can
1685	be skipped over during during traversals. The bits are NOT always
1686	cleared as soon as bins are empty, but instead only
1687	when they are noticed to be empty during traversal in malloc.
1688	*/
1689
1690	/ Conservatively use 32 bits per map word, even if on 64bit system /
1691	#define BINMAPSHIFT 5
1692	#define BITSPERMAP (1U << BINMAPSHIFT)
1693	#define BINMAPSIZE (NBINS / BITSPERMAP)
1694
1695	#define idx2block(i) ((i) >> BINMAPSHIFT)
1696	#define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
1697
1698	#define mark_bin(m, i) ((m)->binmap[idx2block (i)] \|= idx2bit (i))
1699	#define unmark_bin(m, i) ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
1700	#define get_binmap(m, i) ((m)->binmap[idx2block (i)] & idx2bit (i))
1701
1702	/*
1703	Fastbins
1704
1705	An array of lists holding recently freed small chunks. Fastbins
1706	are not doubly linked. It is faster to single-link them, and
1707	since chunks are never removed from the middles of these lists,
1708	double linking is not necessary. Also, unlike regular bins, they
1709	are not even processed in FIFO order (they use faster LIFO) since
1710	ordering doesn't much matter in the transient contexts in which
1711	fastbins are normally used.
1712
1713	Chunks in fastbins keep their inuse bit set, so they cannot
1714	be consolidated with other free chunks. malloc_consolidate
1715	releases all chunks in fastbins and consolidates them with
1716	other free chunks.
1717	*/
1718
1719	typedef struct malloc_chunk *mfastbinptr;
1720	#define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1721
1722	/ offset 2 to use otherwise unindexable first 2 bins /
1723	#define fastbin_index(sz) \
1724	((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1725
1726
1727	/ The maximum fastbin request size we support /
1728	#define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1729
1730	#define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
1731
1732	/*
1733	FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1734	that triggers automatic consolidation of possibly-surrounding
1735	fastbin chunks. This is a heuristic, so the exact value should not
1736	matter too much. It is defined at half the default trim threshold as a
1737	compromise heuristic to only attempt consolidation if it is likely
1738	to lead to trimming. However, it is not dynamically tunable, since
1739	consolidation reduces fragmentation surrounding large chunks even
1740	if trimming is not used.
1741	*/
1742
1743	#define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1744
1745	/*
1746	NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1747	regions. Otherwise, contiguity is exploited in merging together,
1748	when possible, results from consecutive MORECORE calls.
1749
1750	The initial value comes from MORECORE_CONTIGUOUS, but is
1751	changed dynamically if mmap is ever used as an sbrk substitute.
1752	*/
1753
1754	#define NONCONTIGUOUS_BIT (2U)
1755
1756	#define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1757	#define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1758	#define set_noncontiguous(M) ((M)->flags \|= NONCONTIGUOUS_BIT)
1759	#define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1760
1761	/ Maximum size of memory handled in fastbins. /
1762	static uint8_t global_max_fast;
1763
1764	/*
1765	Set value of max_fast.
1766	Use impossibly small value if 0.
1767	Precondition: there are no existing fastbin chunks in the main arena.
1768	Since do_check_malloc_state () checks this, we call malloc_consolidate ()
1769	before changing max_fast. Note other arenas will leak their fast bin
1770	entries if max_fast is reduced.
1771	*/
1772
1773	#define set_max_fast(s) \
1774	global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \
1775	? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1776
1777	static inline INTERNAL_SIZE_T
1778	get_max_fast (void)
1779	{
1780	/ Tell the GCC optimizers that global_max_fast is never larger*
1781	than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in
1782	_int_malloc after constant propagation of the size parameter.
1783	(The code never executes because malloc preserves the
1784	global_max_fast invariant, but the optimizers may not recognize
1785	this.) /*
1786	if (global_max_fast > MAX_FAST_SIZE)
1787	__builtin_unreachable ();
1788	return global_max_fast;
1789	}
1790
1791	/*
1792	----------- Internal state representation and initialization -----------
1793	*/
1794
1795	/*
1796	have_fastchunks indicates that there are probably some fastbin chunks.
1797	It is set true on entering a chunk into any fastbin, and cleared early in
1798	malloc_consolidate. The value is approximate since it may be set when there
1799	are no fastbin chunks, or it may be clear even if there are fastbin chunks
1800	available. Given it's sole purpose is to reduce number of redundant calls to
1801	malloc_consolidate, it does not affect correctness. As a result we can safely
1802	use relaxed atomic accesses.
1803	*/
1804
1805
1806	struct malloc_state
1807	{
1808	/ Serialize access. /
1809	__libc_lock_define (, mutex);
1810
1811	/ Flags (formerly in max_fast). /
1812	int flags;
1813
1814	/ Set if the fastbin chunks contain recently inserted free blocks. /
1815	/ Note this is a bool but not all targets support atomics on booleans. /
1816	int have_fastchunks;
1817
1818	/ Fastbins /
1819	mfastbinptr fastbinsY[NFASTBINS];
1820
1821	/ Base of the topmost chunk -- not otherwise kept in a bin /
1822	mchunkptr top;
1823
1824	/ The remainder from the most recent split of a small request /
1825	mchunkptr last_remainder;
1826
1827	/ Normal bins packed as described above /
1828	mchunkptr bins[NBINS * `2` - `2`];
1829
1830	/ Bitmap of bins /
1831	unsigned int binmap[BINMAPSIZE];
1832
1833	/ Linked list /
1834	struct malloc_state *next;
1835
1836	/ Linked list for free arenas. Access to this field is serialized*
1837	by free_list_lock in arena.c. /*
1838	struct malloc_state *next_free;
1839
1840	/ Number of threads attached to this arena. 0 if the arena is on*
1841	the free list. Access to this field is serialized by
1842	free_list_lock in arena.c. /*
1843	INTERNAL_SIZE_T attached_threads;
1844
1845	/ Memory allocated from the system in this arena. /
1846	INTERNAL_SIZE_T system_mem;
1847	INTERNAL_SIZE_T max_system_mem;
1848	};
1849
1850	struct malloc_par
1851	{
1852	/ Tunable parameters /
1853	unsigned long trim_threshold;
1854	INTERNAL_SIZE_T top_pad;
1855	INTERNAL_SIZE_T mmap_threshold;
1856	INTERNAL_SIZE_T arena_test;
1857	INTERNAL_SIZE_T arena_max;
1858
1859	#if HAVE_TUNABLES
1860	/ Transparent Large Page support. /
1861	INTERNAL_SIZE_T thp_pagesize;
1862	/ A value different than 0 means to align mmap allocation to hp_pagesize*
1863	add hp_flags on flags. /*
1864	INTERNAL_SIZE_T hp_pagesize;
1865	int hp_flags;
1866	#endif
1867
1868	/ Memory map support /
1869	int n_mmaps;
1870	int n_mmaps_max;
1871	int max_n_mmaps;
1872	/ the mmap_threshold is dynamic, until the user sets*
1873	it manually, at which point we need to disable any
1874	dynamic behavior. /*
1875	int no_dyn_threshold;
1876
1877	/ Statistics /
1878	INTERNAL_SIZE_T mmapped_mem;
1879	INTERNAL_SIZE_T max_mmapped_mem;
1880
1881	/ First address handed out by MORECORE/sbrk. /
1882	char *sbrk_base;
1883
1884	#if USE_TCACHE
1885	/ Maximum number of buckets to use. /
1886	size_t tcache_bins;
1887	size_t tcache_max_bytes;
1888	/ Maximum number of chunks in each bucket. /
1889	size_t tcache_count;
1890	/ Maximum number of chunks to remove from the unsorted list, which*
1891	aren't used to prefill the cache. /*
1892	size_t tcache_unsorted_limit;
1893	#endif
1894	};
1895
1896	/ There are several instances of this struct ("arenas") in this*
1897	malloc. If you are adapting this malloc in a way that does NOT use
1898	a static or mmapped malloc_state, you MUST explicitly zero-fill it
1899	before using. This malloc relies on the property that malloc_state
1900	is initialized to all zeroes (as is true of C statics). /*
1901
1902	static struct malloc_state main_arena =
1903	{
1904	.mutex = _LIBC_LOCK_INITIALIZER,
1905	.next = &main_arena,
1906	.attached_threads = `1`
1907	};
1908
1909	/ There is only one instance of the malloc parameters. /
1910
1911	static struct malloc_par mp_ =
1912	{
1913	.top_pad = DEFAULT_TOP_PAD,
1914	.n_mmaps_max = DEFAULT_MMAP_MAX,
1915	.mmap_threshold = DEFAULT_MMAP_THRESHOLD,
1916	.trim_threshold = DEFAULT_TRIM_THRESHOLD,
1917	#define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
1918	.arena_test = NARENAS_FROM_NCORES (`1`)
1919	#if USE_TCACHE
1920	,
1921	.tcache_count = TCACHE_FILL_COUNT,
1922	.tcache_bins = TCACHE_MAX_BINS,
1923	.tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-`1`),
1924	.tcache_unsorted_limit = `0` / No limit. /
1925	#endif
1926	};
1927
1928	/*
1929	Initialize a malloc_state struct.
1930
1931	This is called from ptmalloc_init () or from _int_new_arena ()
1932	when creating a new arena.
1933	*/
1934
1935	static void
1936	malloc_init_state (mstate av)
1937	{
1938	int i;
1939	mbinptr bin;
1940
1941	/ Establish circular links for normal bins /
1942	for (i = `1`; i < NBINS; ++i)
1943	{
1944	bin = bin_at (av, i);
1945	bin->fd = bin->bk = bin;
1946	}
1947
1948	#if MORECORE_CONTIGUOUS
1949	if (av != &main_arena)
1950	#endif
1951	set_noncontiguous (av);
1952	if (av == &main_arena)
1953	set_max_fast (DEFAULT_MXFAST);
1954	atomic_store_relaxed (&av->have_fastchunks, false);
1955
1956	av->top = initial_top (av);
1957	}
1958
1959	/*
1960	Other internal utilities operating on mstates
1961	*/
1962
1963	static void *sysmalloc (INTERNAL_SIZE_T, mstate);
1964	static int systrim (size_t, mstate);
1965	static void malloc_consolidate (mstate);
1966
1967
1968	/ -------------- Early definitions for debugging hooks ---------------- /
1969
1970	/ This function is called from the arena shutdown hook, to free the*
1971	thread cache (if it exists). /*
1972	static void tcache_thread_shutdown (void);
1973
1974	/ ------------------ Testing support ----------------------------------/
1975
1976	static int perturb_byte;
1977
1978	static void
1979	alloc_perturb (char *p, size_t n)
1980	{
1981	if (__glibc_unlikely (perturb_byte))
1982	memset (p, perturb_byte ^ `0xff`, n);
1983	}
1984
1985	static void
1986	free_perturb (char *p, size_t n)
1987	{
1988	if (__glibc_unlikely (perturb_byte))
1989	memset (p, perturb_byte, n);
1990	}
1991
1992
1993
1994	#include <stap-probe.h>
1995
1996	/ ----------- Routines dealing with transparent huge pages ----------- /
1997
1998	static inline void
1999	madvise_thp (void *p, INTERNAL_SIZE_T size)
2000	{
2001	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2002	/ Do not consider areas smaller than a huge page or if the tunable is*
2003	not active. /*
2004	if (mp_.thp_pagesize == `0` \|\| size < mp_.thp_pagesize)
2005	return;
2006
2007	/ Linux requires the input address to be page-aligned, and unaligned*
2008	inputs happens only for initial data segment. /*
2009	if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
2010	{
2011	void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
2012	size += PTR_DIFF (p, q);
2013	p = q;
2014	}
2015
2016	__madvise (p, size, MADV_HUGEPAGE);
2017	#endif
2018	}
2019
2020	/ ------------------- Support for multiple arenas -------------------- /
2021	#include "arena.c"
2022
2023	/*
2024	Debugging support
2025
2026	These routines make a number of assertions about the states
2027	of data structures that should be true at all times. If any
2028	are not true, it's very likely that a user program has somehow
2029	trashed memory. (It's also possible that there is a coding error
2030	in malloc. In which case, please report it!)
2031	*/
2032
2033	#if !MALLOC_DEBUG
2034
2035	# define check_chunk(A, P)
2036	# define check_free_chunk(A, P)
2037	# define check_inuse_chunk(A, P)
2038	# define check_remalloced_chunk(A, P, N)
2039	# define check_malloced_chunk(A, P, N)
2040	# define check_malloc_state(A)
2041
2042	#else
2043
2044	# define check_chunk(A, P) do_check_chunk (A, P)
2045	# define check_free_chunk(A, P) do_check_free_chunk (A, P)
2046	# define check_inuse_chunk(A, P) do_check_inuse_chunk (A, P)
2047	# define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
2048	# define check_malloced_chunk(A, P, N) do_check_malloced_chunk (A, P, N)
2049	# define check_malloc_state(A) do_check_malloc_state (A)
2050
2051	/*
2052	Properties of all chunks
2053	*/
2054
2055	static void
2056	do_check_chunk (mstate av, mchunkptr p)
2057	{
2058	unsigned long sz = chunksize (p);
2059	/ min and max possible addresses assuming contiguous allocation /
2060	char max_address = (char* *) (av->top) + chunksize (av->top);
2061	char *min_address = max_address - av->system_mem;
2062
2063	if (!chunk_is_mmapped (p))
2064	{
2065	/ Has legal address ... /
2066	if (p != av->top)
2067	{
2068	if (contiguous (av))
2069	{
2070	assert (((char *) p) >= min_address);
2071	assert (((char ) p + sz) <= ((char* *) (av->top)));
2072	}
2073	}
2074	else
2075	{
2076	/ top size is always at least MINSIZE /
2077	assert ((unsigned long) (sz) >= MINSIZE);
2078	/ top predecessor always marked inuse /
2079	assert (prev_inuse (p));
2080	}
2081	}
2082	else
2083	{
2084	/ address is outside main heap /
2085	if (contiguous (av) && av->top != initial_top (av))
2086	{
2087	assert (((char ) p) < min_address \|\| ((char* *) p) >= max_address);
2088	}
2089	/ chunk is page-aligned /
2090	assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - `1`)) == `0`);
2091	/ mem is aligned /
2092	assert (aligned_OK (chunk2mem (p)));
2093	}
2094	}
2095
2096	/*
2097	Properties of free chunks
2098	*/
2099
2100	static void
2101	do_check_free_chunk (mstate av, mchunkptr p)
2102	{
2103	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2104	mchunkptr next = chunk_at_offset (p, sz);
2105
2106	do_check_chunk (av, p);
2107
2108	/ Chunk must claim to be free ... /
2109	assert (!inuse (p));
2110	assert (!chunk_is_mmapped (p));
2111
2112	/ Unless a special marker, must have OK fields /
2113	if ((unsigned long) (sz) >= MINSIZE)
2114	{
2115	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2116	assert (aligned_OK (chunk2mem (p)));
2117	/ ... matching footer field /
2118	assert (prev_size (next_chunk (p)) == sz);
2119	/ ... and is fully consolidated /
2120	assert (prev_inuse (p));
2121	assert (next == av->top \|\| inuse (next));
2122
2123	/ ... and has minimally sane links /
2124	assert (p->fd->bk == p);
2125	assert (p->bk->fd == p);
2126	}
2127	else / markers are always of size SIZE_SZ /
2128	assert (sz == SIZE_SZ);
2129	}
2130
2131	/*
2132	Properties of inuse chunks
2133	*/
2134
2135	static void
2136	do_check_inuse_chunk (mstate av, mchunkptr p)
2137	{
2138	mchunkptr next;
2139
2140	do_check_chunk (av, p);
2141
2142	if (chunk_is_mmapped (p))
2143	return; / mmapped chunks have no next/prev /
2144
2145	/ Check whether it claims to be in use ... /
2146	assert (inuse (p));
2147
2148	next = next_chunk (p);
2149
2150	/ ... and is surrounded by OK chunks.*
2151	Since more things can be checked with free chunks than inuse ones,
2152	if an inuse chunk borders them and debug is on, it's worth doing them.
2153	*/
2154	if (!prev_inuse (p))
2155	{
2156	/ Note that we cannot even look at prev unless it is not inuse /
2157	mchunkptr prv = prev_chunk (p);
2158	assert (next_chunk (prv) == p);
2159	do_check_free_chunk (av, prv);
2160	}
2161
2162	if (next == av->top)
2163	{
2164	assert (prev_inuse (next));
2165	assert (chunksize (next) >= MINSIZE);
2166	}
2167	else if (!inuse (next))
2168	do_check_free_chunk (av, next);
2169	}
2170
2171	/*
2172	Properties of chunks recycled from fastbins
2173	*/
2174
2175	static void
2176	do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2177	{
2178	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2179
2180	if (!chunk_is_mmapped (p))
2181	{
2182	assert (av == arena_for_chunk (p));
2183	if (chunk_main_arena (p))
2184	assert (av == &main_arena);
2185	else
2186	assert (av != &main_arena);
2187	}
2188
2189	do_check_inuse_chunk (av, p);
2190
2191	/ Legal size ... /
2192	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2193	assert ((unsigned long) (sz) >= MINSIZE);
2194	/ ... and alignment /
2195	assert (aligned_OK (chunk2mem (p)));
2196	/ chunk is less than MINSIZE more than request /
2197	assert ((long) (sz) - (long) (s) >= `0`);
2198	assert ((long) (sz) - (long) (s + MINSIZE) < `0`);
2199	}
2200
2201	/*
2202	Properties of nonrecycled chunks at the point they are malloced
2203	*/
2204
2205	static void
2206	do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2207	{
2208	/ same as recycled case ... /
2209	do_check_remalloced_chunk (av, p, s);
2210
2211	/*
2212	... plus, must obey implementation invariant that prev_inuse is
2213	always true of any allocated chunk; i.e., that each allocated
2214	chunk borders either a previously allocated and still in-use
2215	chunk, or the base of its memory arena. This is ensured
2216	by making all allocations from the `lowest' part of any found
2217	chunk. This does not necessarily hold however for chunks
2218	recycled via fastbins.
2219	*/
2220
2221	assert (prev_inuse (p));
2222	}
2223
2224
2225	/*
2226	Properties of malloc_state.
2227
2228	This may be useful for debugging malloc, as well as detecting user
2229	programmer errors that somehow write into malloc_state.
2230
2231	If you are extending or experimenting with this malloc, you can
2232	probably figure out how to hack this routine to print out or
2233	display chunk addresses, sizes, bins, and other instrumentation.
2234	*/
2235
2236	static void
2237	do_check_malloc_state (mstate av)
2238	{
2239	int i;
2240	mchunkptr p;
2241	mchunkptr q;
2242	mbinptr b;
2243	unsigned int idx;
2244	INTERNAL_SIZE_T size;
2245	unsigned long total = `0`;
2246	int max_fast_bin;
2247
2248	/ internal size_t must be no wider than pointer type /
2249	assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
2250
2251	/ alignment is a power of 2 /
2252	assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - `1`)) == `0`);
2253
2254	/ Check the arena is initialized. /
2255	assert (av->top != `0`);
2256
2257	/ No memory has been allocated yet, so doing more tests is not possible. /
2258	if (av->top == initial_top (av))
2259	return;
2260
2261	/ pagesize is a power of 2 /
2262	assert (powerof2(GLRO (dl_pagesize)));
2263
2264	/ A contiguous main_arena is consistent with sbrk_base. /
2265	if (av == &main_arena && contiguous (av))
2266	assert ((char *) mp_.sbrk_base + av->system_mem ==
2267	(char *) av->top + chunksize (av->top));
2268
2269	/ properties of fastbins /
2270
2271	/ max_fast is in allowed range /
2272	assert ((get_max_fast () & ~`1`) <= request2size (MAX_FAST_SIZE));
2273
2274	max_fast_bin = fastbin_index (get_max_fast ());
2275
2276	for (i = `0`; i < NFASTBINS; ++i)
2277	{
2278	p = fastbin (av, i);
2279
2280	/ The following test can only be performed for the main arena.*
2281	While mallopt calls malloc_consolidate to get rid of all fast
2282	bins (especially those larger than the new maximum) this does
2283	only happen for the main arena. Trying to do this for any
2284	other arena would mean those arenas have to be locked and
2285	malloc_consolidate be called for them. This is excessive. And
2286	even if this is acceptable to somebody it still cannot solve
2287	the problem completely since if the arena is locked a
2288	concurrent malloc call might create a new arena which then
2289	could use the newly invalid fast bins. /*
2290
2291	/ all bins past max_fast are empty /
2292	if (av == &main_arena && i > max_fast_bin)
2293	assert (p == `0`);
2294
2295	while (p != `0`)
2296	{
2297	if (__glibc_unlikely (misaligned_chunk (p)))
2298	malloc_printerr ("do_check_malloc_state(): "
2299	"unaligned fastbin chunk detected");
2300	/ each chunk claims to be inuse /
2301	do_check_inuse_chunk (av, p);
2302	total += chunksize (p);
2303	/ chunk belongs in this bin /
2304	assert (fastbin_index (chunksize (p)) == i);
2305	p = REVEAL_PTR (p->fd);
2306	}
2307	}
2308
2309	/ check normal bins /
2310	for (i = `1`; i < NBINS; ++i)
2311	{
2312	b = bin_at (av, i);
2313
2314	/ binmap is accurate (except for bin 1 == unsorted_chunks) /
2315	if (i >= `2`)
2316	{
2317	unsigned int binbit = get_binmap (av, i);
2318	int empty = last (b) == b;
2319	if (!binbit)
2320	assert (empty);
2321	else if (!empty)
2322	assert (binbit);
2323	}
2324
2325	for (p = last (b); p != b; p = p->bk)
2326	{
2327	/ each chunk claims to be free /
2328	do_check_free_chunk (av, p);
2329	size = chunksize (p);
2330	total += size;
2331	if (i >= `2`)
2332	{
2333	/ chunk belongs in bin /
2334	idx = bin_index (size);
2335	assert (idx == i);
2336	/ lists are sorted /
2337	assert (p->bk == b \|\|
2338	(unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
2339
2340	if (!in_smallbin_range (size))
2341	{
2342	if (p->fd_nextsize != NULL)
2343	{
2344	if (p->fd_nextsize == p)
2345	assert (p->bk_nextsize == p);
2346	else
2347	{
2348	if (p->fd_nextsize == first (b))
2349	assert (chunksize (p) < chunksize (p->fd_nextsize));
2350	else
2351	assert (chunksize (p) > chunksize (p->fd_nextsize));
2352
2353	if (p == first (b))
2354	assert (chunksize (p) > chunksize (p->bk_nextsize));
2355	else
2356	assert (chunksize (p) < chunksize (p->bk_nextsize));
2357	}
2358	}
2359	else
2360	assert (p->bk_nextsize == NULL);
2361	}
2362	}
2363	else if (!in_smallbin_range (size))
2364	assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
2365	/ chunk is followed by a legal chain of inuse chunks /
2366	for (q = next_chunk (p);
2367	(q != av->top && inuse (q) &&
2368	(unsigned long) (chunksize (q)) >= MINSIZE);
2369	q = next_chunk (q))
2370	do_check_inuse_chunk (av, q);
2371	}
2372	}
2373
2374	/ top chunk is OK /
2375	check_chunk (av, av->top);
2376	}
2377	#endif
2378
2379
2380	/ ----------------- Support for debugging hooks -------------------- /
2381	#if IS_IN (libc)
2382	#include "hooks.c"
2383	#endif
2384
2385
2386	/ ----------- Routines dealing with system allocation -------------- /
2387
2388	/*
2389	sysmalloc handles malloc cases requiring more memory from the system.
2390	On entry, it is assumed that av->top does not have enough
2391	space to service request for nb bytes, thus requiring that av->top
2392	be extended or replaced.
2393	*/
2394
2395	static void *
2396	sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av)
2397	{
2398	long int size;
2399
2400	/*
2401	Round up size to nearest page. For mmapped chunks, the overhead is one
2402	SIZE_SZ unit larger than for normal chunks, because there is no
2403	following chunk whose prev_size field could be used.
2404
2405	See the front_misalign handling below, for glibc there is no need for
2406	further alignments unless we have have high alignment.
2407	*/
2408	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2409	size = ALIGN_UP (nb + SIZE_SZ, pagesize);
2410	else
2411	size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
2412
2413	/ Don't try if size wraps around 0. /
2414	if ((unsigned long) (size) <= (unsigned long) (nb))
2415	return MAP_FAILED;
2416
2417	char mm = (char* *) MMAP (`0`, size,
2418	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2419	extra_flags);
2420	if (mm == MAP_FAILED)
2421	return mm;
2422
2423	#ifdef MAP_HUGETLB
2424	if (!(extra_flags & MAP_HUGETLB))
2425	madvise_thp (mm, size);
2426	#endif
2427
2428	/*
2429	The offset to the start of the mmapped region is stored in the prev_size
2430	field of the chunk. This allows us to adjust returned start address to
2431	meet alignment requirements here and in memalign(), and still be able to
2432	compute proper address argument for later munmap in free() and realloc().
2433	*/
2434
2435	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2436
2437	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2438	{
2439	/ For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and*
2440	MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1. Each mmap'ed area is page
2441	aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. /*
2442	assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == `0`);
2443	front_misalign = `0`;
2444	}
2445	else
2446	front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
2447
2448	mchunkptr p; / the allocated/returned chunk /
2449
2450	if (front_misalign > `0`)
2451	{
2452	ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign;
2453	p = (mchunkptr) (mm + correction);
2454	set_prev_size (p, correction);
2455	set_head (p, (size - correction) \| IS_MMAPPED);
2456	}
2457	else
2458	{
2459	p = (mchunkptr) mm;
2460	set_prev_size (p, `0`);
2461	set_head (p, size \| IS_MMAPPED);
2462	}
2463
2464	/ update statistics /
2465	int new = atomic_fetch_add_relaxed (&mp_.n_mmaps, `1`) + `1`;
2466	atomic_max (&mp_.max_n_mmaps, new);
2467
2468	unsigned long sum;
2469	sum = atomic_fetch_add_relaxed (&mp_.mmapped_mem, size) + size;
2470	atomic_max (&mp_.max_mmapped_mem, sum);
2471
2472	check_chunk (av, p);
2473
2474	return chunk2mem (p);
2475	}
2476
2477	/*
2478	Allocate memory using mmap() based on S and NB requested size, aligning to
2479	PAGESIZE if required. The EXTRA_FLAGS is used on mmap() call. If the call
2480	succeedes S is updated with the allocated size. This is used as a fallback
2481	if MORECORE fails.
2482	*/
2483	static void *
2484	sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb,
2485	INTERNAL_SIZE_T old_size, size_t minsize,
2486	size_t pagesize, int extra_flags, mstate av)
2487	{
2488	long int size = *s;
2489
2490	/ Cannot merge with old top, so add its size back in /
2491	if (contiguous (av))
2492	size = ALIGN_UP (size + old_size, pagesize);
2493
2494	/ If we are relying on mmap as backup, then use larger units /
2495	if ((unsigned long) (size) < minsize)
2496	size = minsize;
2497
2498	/ Don't try if size wraps around 0 /
2499	if ((unsigned long) (size) <= (unsigned long) (nb))
2500	return MORECORE_FAILURE;
2501
2502	char mbrk = (char* *) (MMAP (`0`, size,
2503	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2504	extra_flags));
2505	if (mbrk == MAP_FAILED)
2506	return MAP_FAILED;
2507
2508	#ifdef MAP_HUGETLB
2509	if (!(extra_flags & MAP_HUGETLB))
2510	madvise_thp (mbrk, size);
2511	#endif
2512
2513	/ Record that we no longer have a contiguous sbrk region. After the first*
2514	time mmap is used as backup, we do not ever rely on contiguous space
2515	since this could incorrectly bridge regions. /*
2516	set_noncontiguous (av);
2517
2518	*s = size;
2519	return mbrk;
2520	}
2521
2522	static void *
2523	sysmalloc (INTERNAL_SIZE_T nb, mstate av)
2524	{
2525	mchunkptr old_top; / incoming value of av->top /
2526	INTERNAL_SIZE_T old_size; / its size /
2527	char old_end; /* its end address /
2528
2529	long size; / arg to first MORECORE or mmap call /
2530	char brk; /* return value from MORECORE /
2531
2532	long correction; / arg to 2nd MORECORE call /
2533	char snd_brk; /* 2nd return val /
2534
2535	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2536	INTERNAL_SIZE_T end_misalign; / partial page left at end of new space /
2537	char aligned_brk; /* aligned offset into brk /
2538
2539	mchunkptr p; / the allocated/returned chunk /
2540	mchunkptr remainder; / remainder from allocation /
2541	unsigned long remainder_size; / its size /
2542
2543
2544	size_t pagesize = GLRO (dl_pagesize);
2545	bool tried_mmap = false;
2546
2547
2548	/*
2549	If have mmap, and the request size meets the mmap threshold, and
2550	the system supports mmap, and there are few enough currently
2551	allocated mmapped regions, try to directly map this request
2552	rather than expanding top.
2553	*/
2554
2555	if (av == NULL
2556	\|\| ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
2557	&& (mp_.n_mmaps < mp_.n_mmaps_max)))
2558	{
2559	char *mm;
2560	#if HAVE_TUNABLES
2561	if (mp_.hp_pagesize > `0` && nb >= mp_.hp_pagesize)
2562	{
2563	/ There is no need to isse the THP madvise call if Huge Pages are*
2564	used directly. /*
2565	mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av);
2566	if (mm != MAP_FAILED)
2567	return mm;
2568	}
2569	#endif
2570	mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2571	if (mm != MAP_FAILED)
2572	return mm;
2573	tried_mmap = true;
2574	}
2575
2576	/ There are no usable arenas and mmap also failed. /
2577	if (av == NULL)
2578	return `0`;
2579
2580	/ Record incoming configuration of top /
2581
2582	old_top = av->top;
2583	old_size = chunksize (old_top);
2584	old_end = (char *) (chunk_at_offset (old_top, old_size));
2585
2586	brk = snd_brk = (char *) (MORECORE_FAILURE);
2587
2588	/*
2589	If not the first time through, we require old_size to be
2590	at least MINSIZE and to have prev_inuse set.
2591	*/
2592
2593	assert ((old_top == initial_top (av) && old_size == `0`) \|\|
2594	((unsigned long) (old_size) >= MINSIZE &&
2595	prev_inuse (old_top) &&
2596	((unsigned long) old_end & (pagesize - `1`)) == `0`));
2597
2598	/ Precondition: not enough current space to satisfy nb request /
2599	assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
2600
2601
2602	if (av != &main_arena)
2603	{
2604	heap_info old_heap, heap;
2605	size_t old_heap_size;
2606
2607	/ First try to extend the current heap. /
2608	old_heap = heap_for_ptr (old_top);
2609	old_heap_size = old_heap->size;
2610	if ((long) (MINSIZE + nb - old_size) > `0`
2611	&& grow_heap (old_heap, MINSIZE + nb - old_size) == `0`)
2612	{
2613	av->system_mem += old_heap->size - old_heap_size;
2614	set_head (old_top, (((char ) old_heap + old_heap->size) - (char* *) old_top)
2615	\| PREV_INUSE);
2616	}
2617	else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad)))
2618	{
2619	/ Use a newly allocated heap. /
2620	heap->ar_ptr = av;
2621	heap->prev = old_heap;
2622	av->system_mem += heap->size;
2623	/ Set up the new top. /
2624	top (av) = chunk_at_offset (heap, sizeof (*heap));
2625	set_head (top (av), (heap->size - sizeof (*heap)) \| PREV_INUSE);
2626
2627	/ Setup fencepost and free the old top chunk with a multiple of*
2628	MALLOC_ALIGNMENT in size. /*
2629	/ The fencepost takes at least MINSIZE bytes, because it might*
2630	become the top chunk again later. Note that a footer is set
2631	up, too, although the chunk is marked in use. /*
2632	old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
2633	set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
2634	`0` \| PREV_INUSE);
2635	if (old_size >= MINSIZE)
2636	{
2637	set_head (chunk_at_offset (old_top, old_size),
2638	CHUNK_HDR_SZ \| PREV_INUSE);
2639	set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
2640	set_head (old_top, old_size \| PREV_INUSE \| NON_MAIN_ARENA);
2641	_int_free (av, old_top, `1`);
2642	}
2643	else
2644	{
2645	set_head (old_top, (old_size + CHUNK_HDR_SZ) \| PREV_INUSE);
2646	set_foot (old_top, (old_size + CHUNK_HDR_SZ));
2647	}
2648	}
2649	else if (!tried_mmap)
2650	{
2651	/ We can at least try to use to mmap memory. If new_heap fails*
2652	it is unlikely that trying to allocate huge pages will
2653	succeed. /*
2654	char *mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2655	if (mm != MAP_FAILED)
2656	return mm;
2657	}
2658	}
2659	else / av == main_arena /
2660
2661
2662	{ / Request enough space for nb + pad + overhead /
2663	size = nb + mp_.top_pad + MINSIZE;
2664
2665	/*
2666	If contiguous, we can subtract out existing space that we hope to
2667	combine with new space. We add it back later only if
2668	we don't actually get contiguous space.
2669	*/
2670
2671	if (contiguous (av))
2672	size -= old_size;
2673
2674	/*
2675	Round to a multiple of page size or huge page size.
2676	If MORECORE is not contiguous, this ensures that we only call it
2677	with whole-page arguments. And if MORECORE is contiguous and
2678	this is not first time through, this preserves page-alignment of
2679	previous calls. Otherwise, we correct to page-align below.
2680	*/
2681
2682	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2683	/ Defined in brk.c. /
2684	extern void *__curbrk;
2685	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2686	{
2687	uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
2688	mp_.thp_pagesize);
2689	size = top - (uintptr_t) __curbrk;
2690	}
2691	else
2692	#endif
2693	size = ALIGN_UP (size, GLRO(dl_pagesize));
2694
2695	/*
2696	Don't try to call MORECORE if argument is so big as to appear
2697	negative. Note that since mmap takes size_t arg, it may succeed
2698	below even if we cannot call MORECORE.
2699	*/
2700
2701	if (size > `0`)
2702	{
2703	brk = (char *) (MORECORE (size));
2704	if (brk != (char *) (MORECORE_FAILURE))
2705	madvise_thp (brk, size);
2706	LIBC_PROBE (memory_sbrk_more, `2`, brk, size);
2707	}
2708
2709	if (brk == (char *) (MORECORE_FAILURE))
2710	{
2711	/*
2712	If have mmap, try using it as a backup when MORECORE fails or
2713	cannot be used. This is worth doing on systems that have "holes" in
2714	address space, so sbrk cannot extend to give contiguous space, but
2715	space is available elsewhere. Note that we ignore mmap max count
2716	and threshold limits, since the space will not be used as a
2717	segregated mmap region.
2718	*/
2719
2720	char *mbrk = MAP_FAILED;
2721	#if HAVE_TUNABLES
2722	if (mp_.hp_pagesize > `0`)
2723	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size,
2724	mp_.hp_pagesize, mp_.hp_pagesize,
2725	mp_.hp_flags, av);
2726	#endif
2727	if (mbrk == MAP_FAILED)
2728	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, pagesize,
2729	MMAP_AS_MORECORE_SIZE, `0`, av);
2730	if (mbrk != MAP_FAILED)
2731	{
2732	/ We do not need, and cannot use, another sbrk call to find end /
2733	brk = mbrk;
2734	snd_brk = brk + size;
2735	}
2736	}
2737
2738	if (brk != (char *) (MORECORE_FAILURE))
2739	{
2740	if (mp_.sbrk_base == `0`)
2741	mp_.sbrk_base = brk;
2742	av->system_mem += size;
2743
2744	/*
2745	If MORECORE extends previous space, we can likewise extend top size.
2746	*/
2747
2748	if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
2749	set_head (old_top, (size + old_size) \| PREV_INUSE);
2750
2751	else if (contiguous (av) && old_size && brk < old_end)
2752	/ Oops! Someone else killed our space.. Can't touch anything. /
2753	malloc_printerr ("break adjusted to free malloc space");
2754
2755	/*
2756	Otherwise, make adjustments:
2757
2758	* If the first time through or noncontiguous, we need to call sbrk
2759	just to find out where the end of memory lies.
2760
2761	* We need to ensure that all returned chunks from malloc will meet
2762	MALLOC_ALIGNMENT
2763
2764	* If there was an intervening foreign sbrk, we need to adjust sbrk
2765	request size to account for fact that we will not be able to
2766	combine new space with existing space in old_top.
2767
2768	* Almost all systems internally allocate whole pages at a time, in
2769	which case we might as well use the whole last page of request.
2770	So we allocate enough more memory to hit a page boundary now,
2771	which in turn causes future contiguous calls to page-align.
2772	*/
2773
2774	else
2775	{
2776	front_misalign = `0`;
2777	end_misalign = `0`;
2778	correction = `0`;
2779	aligned_brk = brk;
2780
2781	/ handle contiguous cases /
2782	if (contiguous (av))
2783	{
2784	/ Count foreign sbrk as system_mem. /
2785	if (old_size)
2786	av->system_mem += brk - old_end;
2787
2788	/ Guarantee alignment of first new chunk made from this space /
2789
2790	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2791	if (front_misalign > `0`)
2792	{
2793	/*
2794	Skip over some bytes to arrive at an aligned position.
2795	We don't need to specially mark these wasted front bytes.
2796	They will never be accessed anyway because
2797	prev_inuse of av->top (and any chunk created from its start)
2798	is always true after initialization.
2799	*/
2800
2801	correction = MALLOC_ALIGNMENT - front_misalign;
2802	aligned_brk += correction;
2803	}
2804
2805	/*
2806	If this isn't adjacent to existing space, then we will not
2807	be able to merge with old_top space, so must add to 2nd request.
2808	*/
2809
2810	correction += old_size;
2811
2812	/ Extend the end address to hit a page boundary /
2813	end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
2814	correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
2815
2816	assert (correction >= `0`);
2817	snd_brk = (char *) (MORECORE (correction));
2818
2819	/*
2820	If can't allocate correction, try to at least find out current
2821	brk. It might be enough to proceed without failing.
2822
2823	Note that if second sbrk did NOT fail, we assume that space
2824	is contiguous with first sbrk. This is a safe assumption unless
2825	program is multithreaded but doesn't use locks and a foreign sbrk
2826	occurred between our first and second calls.
2827	*/
2828
2829	if (snd_brk == (char *) (MORECORE_FAILURE))
2830	{
2831	correction = `0`;
2832	snd_brk = (char *) (MORECORE (`0`));
2833	}
2834	else
2835	madvise_thp (snd_brk, correction);
2836	}
2837
2838	/ handle non-contiguous cases /
2839	else
2840	{
2841	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2842	/ MORECORE/mmap must correctly align /
2843	assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == `0`);
2844	else
2845	{
2846	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2847	if (front_misalign > `0`)
2848	{
2849	/*
2850	Skip over some bytes to arrive at an aligned position.
2851	We don't need to specially mark these wasted front bytes.
2852	They will never be accessed anyway because
2853	prev_inuse of av->top (and any chunk created from its start)
2854	is always true after initialization.
2855	*/
2856
2857	aligned_brk += MALLOC_ALIGNMENT - front_misalign;
2858	}
2859	}
2860
2861	/ Find out current end of memory /
2862	if (snd_brk == (char *) (MORECORE_FAILURE))
2863	{
2864	snd_brk = (char *) (MORECORE (`0`));
2865	}
2866	}
2867
2868	/ Adjust top based on results of second sbrk /
2869	if (snd_brk != (char *) (MORECORE_FAILURE))
2870	{
2871	av->top = (mchunkptr) aligned_brk;
2872	set_head (av->top, (snd_brk - aligned_brk + correction) \| PREV_INUSE);
2873	av->system_mem += correction;
2874
2875	/*
2876	If not the first time through, we either have a
2877	gap due to foreign sbrk or a non-contiguous region. Insert a
2878	double fencepost at old_top to prevent consolidation with space
2879	we don't own. These fenceposts are artificial chunks that are
2880	marked as inuse and are in any case too small to use. We need
2881	two to make sizes and alignments work out.
2882	*/
2883
2884	if (old_size != `0`)
2885	{
2886	/*
2887	Shrink old_top to insert fenceposts, keeping size a
2888	multiple of MALLOC_ALIGNMENT. We know there is at least
2889	enough space in old_top to do this.
2890	*/
2891	old_size = (old_size - `2` * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
2892	set_head (old_top, old_size \| PREV_INUSE);
2893
2894	/*
2895	Note that the following assignments completely overwrite
2896	old_top when old_size was previously MINSIZE. This is
2897	intentional. We need the fencepost, even if old_top otherwise gets
2898	lost.
2899	*/
2900	set_head (chunk_at_offset (old_top, old_size),
2901	CHUNK_HDR_SZ \| PREV_INUSE);
2902	set_head (chunk_at_offset (old_top,
2903	old_size + CHUNK_HDR_SZ),
2904	CHUNK_HDR_SZ \| PREV_INUSE);
2905
2906	/ If possible, release the rest. /
2907	if (old_size >= MINSIZE)
2908	{
2909	_int_free (av, old_top, `1`);
2910	}
2911	}
2912	}
2913	}
2914	}
2915	} / if (av != &main_arena) /
2916
2917	if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
2918	av->max_system_mem = av->system_mem;
2919	check_malloc_state (av);
2920
2921	/ finally, do the allocation /
2922	p = av->top;
2923	size = chunksize (p);
2924
2925	/ check that one of the above allocation paths succeeded /
2926	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
2927	{
2928	remainder_size = size - nb;
2929	remainder = chunk_at_offset (p, nb);
2930	av->top = remainder;
2931	set_head (p, nb \| PREV_INUSE \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
2932	set_head (remainder, remainder_size \| PREV_INUSE);
2933	check_malloced_chunk (av, p, nb);
2934	return chunk2mem (p);
2935	}
2936
2937	/ catch all failure paths /
2938	__set_errno (ENOMEM);
2939	return `0`;
2940	}
2941
2942
2943	/*
2944	systrim is an inverse of sorts to sysmalloc. It gives memory back
2945	to the system (via negative arguments to sbrk) if there is unused
2946	memory at the `high' end of the malloc pool. It is called
2947	automatically by free() when top space exceeds the trim
2948	threshold. It is also called by the public malloc_trim routine. It
2949	returns 1 if it actually released any memory, else 0.
2950	*/
2951
2952	static int
2953	systrim (size_t pad, mstate av)
2954	{
2955	long top_size; / Amount of top-most memory /
2956	long extra; / Amount to release /
2957	long released; / Amount actually released /
2958	char current_brk; /* address returned by pre-check sbrk call /
2959	char new_brk; /* address returned by post-check sbrk call /
2960	long top_area;
2961
2962	top_size = chunksize (av->top);
2963
2964	top_area = top_size - MINSIZE - `1`;
2965	if (top_area <= pad)
2966	return `0`;
2967
2968	/ Release in pagesize units and round down to the nearest page. /
2969	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2970	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2971	extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
2972	else
2973	#endif
2974	extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
2975
2976	if (extra == `0`)
2977	return `0`;
2978
2979	/*
2980	Only proceed if end of memory is where we last set it.
2981	This avoids problems if there were foreign sbrk calls.
2982	*/
2983	current_brk = (char *) (MORECORE (`0`));
2984	if (current_brk == (char *) (av->top) + top_size)
2985	{
2986	/*
2987	Attempt to release memory. We ignore MORECORE return value,
2988	and instead call again to find out where new end of memory is.
2989	This avoids problems if first call releases less than we asked,
2990	of if failure somehow altered brk value. (We could still
2991	encounter problems if it altered brk in some very bad way,
2992	but the only thing we can do is adjust anyway, which will cause
2993	some downstream failure.)
2994	*/
2995
2996	MORECORE (-extra);
2997	new_brk = (char *) (MORECORE (`0`));
2998
2999	LIBC_PROBE (memory_sbrk_less, `2`, new_brk, extra);
3000
3001	if (new_brk != (char *) MORECORE_FAILURE)
3002	{
3003	released = (long) (current_brk - new_brk);
3004
3005	if (released != `0`)
3006	{
3007	/ Success. Adjust top. /
3008	av->system_mem -= released;
3009	set_head (av->top, (top_size - released) \| PREV_INUSE);
3010	check_malloc_state (av);
3011	return `1`;
3012	}
3013	}
3014	}
3015	return `0`;
3016	}
3017
3018	static void
3019	munmap_chunk (mchunkptr p)
3020	{
3021	size_t pagesize = GLRO (dl_pagesize);
3022	INTERNAL_SIZE_T size = chunksize (p);
3023
3024	assert (chunk_is_mmapped (p));
3025
3026	uintptr_t mem = (uintptr_t) chunk2mem (p);
3027	uintptr_t block = (uintptr_t) p - prev_size (p);
3028	size_t total_size = prev_size (p) + size;
3029	/ Unfortunately we have to do the compilers job by hand here. Normally*
3030	we would test BLOCK and TOTAL-SIZE separately for compliance with the
3031	page size. But gcc does not recognize the optimization possibility
3032	(in the moment at least) so we combine the two values into one before
3033	the bit test. /*
3034	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3035	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3036	malloc_printerr ("munmap_chunk(): invalid pointer");
3037
3038	atomic_fetch_add_relaxed (&mp_.n_mmaps, -`1`);
3039	atomic_fetch_add_relaxed (&mp_.mmapped_mem, -total_size);
3040
3041	/ If munmap failed the process virtual memory address space is in a*
3042	bad shape. Just leave the block hanging around, the process will
3043	terminate shortly anyway since not much can be done. /*
3044	__munmap ((char *) block, total_size);
3045	}
3046
3047	#if HAVE_MREMAP
3048
3049	static mchunkptr
3050	mremap_chunk (mchunkptr p, size_t new_size)
3051	{
3052	size_t pagesize = GLRO (dl_pagesize);
3053	INTERNAL_SIZE_T offset = prev_size (p);
3054	INTERNAL_SIZE_T size = chunksize (p);
3055	char *cp;
3056
3057	assert (chunk_is_mmapped (p));
3058
3059	uintptr_t block = (uintptr_t) p - offset;
3060	uintptr_t mem = (uintptr_t) chunk2mem(p);
3061	size_t total_size = offset + size;
3062	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3063	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3064	malloc_printerr("mremap_chunk(): invalid pointer");
3065
3066	/ Note the extra SIZE_SZ overhead as in mmap_chunk(). /
3067	new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
3068
3069	/ No need to remap if the number of pages does not change. /
3070	if (total_size == new_size)
3071	return p;
3072
3073	cp = (char ) __mremap ((char* *) block, total_size, new_size,
3074	MREMAP_MAYMOVE);
3075
3076	if (cp == MAP_FAILED)
3077	return `0`;
3078
3079	madvise_thp (cp, new_size);
3080
3081	p = (mchunkptr) (cp + offset);
3082
3083	assert (aligned_OK (chunk2mem (p)));
3084
3085	assert (prev_size (p) == offset);
3086	set_head (p, (new_size - offset) \| IS_MMAPPED);
3087
3088	INTERNAL_SIZE_T new;
3089	new = atomic_fetch_add_relaxed (&mp_.mmapped_mem, new_size - size - offset)
3090	+ new_size - size - offset;
3091	atomic_max (&mp_.max_mmapped_mem, new);
3092	return p;
3093	}
3094	#endif /* HAVE_MREMAP */
3095
3096	/------------------------ Public wrappers. --------------------------------/
3097
3098	#if USE_TCACHE
3099
3100	/ We overlay this structure on the user-data portion of a chunk when*
3101	the chunk is stored in the per-thread cache. /*
3102	typedef struct tcache_entry
3103	{
3104	struct tcache_entry *next;
3105	/ This field exists to detect double frees. /
3106	uintptr_t key;
3107	} tcache_entry;
3108
3109	/ There is one of these for each thread, which contains the*
3110	per-thread cache (hence "tcache_perthread_struct"). Keeping
3111	overall size low is mildly important. Note that COUNTS and ENTRIES
3112	are redundant (we could have just counted the linked list each
3113	time), this is for performance reasons. /*
3114	typedef struct tcache_perthread_struct
3115	{
3116	uint16_t counts[TCACHE_MAX_BINS];
3117	tcache_entry *entries[TCACHE_MAX_BINS];
3118	} tcache_perthread_struct;
3119
3120	static __thread bool tcache_shutting_down = false;
3121	static __thread tcache_perthread_struct *tcache = NULL;
3122
3123	/ Process-wide key to try and catch a double-free in the same thread. /
3124	static uintptr_t tcache_key;
3125
3126	/ The value of tcache_key does not really have to be a cryptographically*
3127	secure random number. It only needs to be arbitrary enough so that it does
3128	not collide with values present in applications. If a collision does happen
3129	consistently enough, it could cause a degradation in performance since the
3130	entire list is checked to check if the block indeed has been freed the
3131	second time. The odds of this happening are exceedingly low though, about 1
3132	in 2^wordsize. There is probably a higher chance of the performance
3133	degradation being due to a double free where the first free happened in a
3134	different thread; that's a case this check does not cover. /*
3135	static void
3136	tcache_key_initialize (void)
3137	{
3138	if (__getrandom_nocancel (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
3139	!= sizeof (tcache_key))
3140	{
3141	tcache_key = random_bits ();
3142	#if __WORDSIZE == 64
3143	tcache_key = (tcache_key << `32`) \| random_bits ();
3144	#endif
3145	}
3146	}
3147
3148	/ Caller must ensure that we know tc_idx is valid and there's room*
3149	for more chunks. /*
3150	static __always_inline void
3151	tcache_put (mchunkptr chunk, size_t tc_idx)
3152	{
3153	tcache_entry e = (tcache_entry ) chunk2mem (chunk);
3154
3155	/ Mark this chunk as "in the tcache" so the test in _int_free will*
3156	detect a double free. /*
3157	e->key = tcache_key;
3158
3159	e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
3160	tcache->entries[tc_idx] = e;
3161	++(tcache->counts[tc_idx]);
3162	}
3163
3164	/ Caller must ensure that we know tc_idx is valid and there's*
3165	available chunks to remove. /*
3166	static __always_inline void *
3167	tcache_get (size_t tc_idx)
3168	{
3169	tcache_entry *e = tcache->entries[tc_idx];
3170	if (__glibc_unlikely (!aligned_OK (e)))
3171	malloc_printerr ("malloc(): unaligned tcache chunk detected");
3172	tcache->entries[tc_idx] = REVEAL_PTR (e->next);
3173	--(tcache->counts[tc_idx]);
3174	e->key = `0`;
3175	return (void *) e;
3176	}
3177
3178	static void
3179	tcache_thread_shutdown (void)
3180	{
3181	int i;
3182	tcache_perthread_struct *tcache_tmp = tcache;
3183
3184	tcache_shutting_down = true;
3185
3186	if (!tcache)
3187	return;
3188
3189	/ Disable the tcache and prevent it from being reinitialized. /
3190	tcache = NULL;
3191
3192	/ Free all of the entries and the tcache itself back to the arena*
3193	heap for coalescing. /*
3194	for (i = `0`; i < TCACHE_MAX_BINS; ++i)
3195	{
3196	while (tcache_tmp->entries[i])
3197	{
3198	tcache_entry *e = tcache_tmp->entries[i];
3199	if (__glibc_unlikely (!aligned_OK (e)))
3200	malloc_printerr ("tcache_thread_shutdown(): "
3201	"unaligned tcache chunk detected");
3202	tcache_tmp->entries[i] = REVEAL_PTR (e->next);
3203	__libc_free (e);
3204	}
3205	}
3206
3207	__libc_free (tcache_tmp);
3208	}
3209
3210	static void
3211	tcache_init(void)
3212	{
3213	mstate ar_ptr;
3214	void *victim = `0`;
3215	const size_t bytes = sizeof (tcache_perthread_struct);
3216
3217	if (tcache_shutting_down)
3218	return;
3219
3220	arena_get (ar_ptr, bytes);
3221	victim = _int_malloc (ar_ptr, bytes);
3222	if (!victim && ar_ptr != NULL)
3223	{
3224	ar_ptr = arena_get_retry (ar_ptr, bytes);
3225	victim = _int_malloc (ar_ptr, bytes);
3226	}
3227
3228
3229	if (ar_ptr != NULL)
3230	__libc_lock_unlock (ar_ptr->mutex);
3231
3232	/ In a low memory situation, we may not be able to allocate memory*
3233	- in which case, we just keep trying later. However, we
3234	typically do this very early, so either there is sufficient
3235	memory, or there isn't enough memory to do non-trivial
3236	allocations anyway. /*
3237	if (victim)
3238	{
3239	tcache = (tcache_perthread_struct *) victim;
3240	memset (tcache, `0`, sizeof (tcache_perthread_struct));
3241	}
3242
3243	}
3244
3245	# define MAYBE_INIT_TCACHE() \
3246	if (__glibc_unlikely (tcache == NULL)) \
3247	tcache_init();
3248
3249	#else /* !USE_TCACHE */
3250	# define MAYBE_INIT_TCACHE()
3251
3252	static void
3253	tcache_thread_shutdown (void)
3254	{
3255	/ Nothing to do if there is no thread cache. /
3256	}
3257
3258	#endif /* !USE_TCACHE */
3259
3260	#if IS_IN (libc)
3261	void *
3262	__libc_malloc (size_t bytes)
3263	{
3264	mstate ar_ptr;
3265	void *victim;
3266
3267	_Static_assert (PTRDIFF_MAX <= SIZE_MAX / `2`,
3268	"PTRDIFF_MAX is not more than half of SIZE_MAX");
3269
3270	if (!__malloc_initialized)
3271	ptmalloc_init ();
3272	#if USE_TCACHE
3273	/ int_free also calls request2size, be careful to not pad twice. /
3274	size_t tbytes = checked_request2size (bytes);
3275	if (tbytes == `0`)
3276	{
3277	__set_errno (ENOMEM);
3278	return NULL;
3279	}
3280	size_t tc_idx = csize2tidx (tbytes);
3281
3282	MAYBE_INIT_TCACHE ();
3283
3284	DIAG_PUSH_NEEDS_COMMENT;
3285	if (tc_idx < mp_.tcache_bins
3286	&& tcache
3287	&& tcache->counts[tc_idx] > `0`)
3288	{
3289	victim = tcache_get (tc_idx);
3290	return tag_new_usable (victim);
3291	}
3292	DIAG_POP_NEEDS_COMMENT;
3293	#endif
3294
3295	if (SINGLE_THREAD_P)
3296	{
3297	victim = tag_new_usable (_int_malloc (&main_arena, bytes));
3298	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3299	&main_arena == arena_for_chunk (mem2chunk (victim)));
3300	return victim;
3301	}
3302
3303	arena_get (ar_ptr, bytes);
3304
3305	victim = _int_malloc (ar_ptr, bytes);
3306	/ Retry with another arena only if we were able to find a usable arena*
3307	before. /*
3308	if (!victim && ar_ptr != NULL)
3309	{
3310	LIBC_PROBE (memory_malloc_retry, `1`, bytes);
3311	ar_ptr = arena_get_retry (ar_ptr, bytes);
3312	victim = _int_malloc (ar_ptr, bytes);
3313	}
3314
3315	if (ar_ptr != NULL)
3316	__libc_lock_unlock (ar_ptr->mutex);
3317
3318	victim = tag_new_usable (victim);
3319
3320	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3321	ar_ptr == arena_for_chunk (mem2chunk (victim)));
3322	return victim;
3323	}
3324	libc_hidden_def (__libc_malloc)
3325
3326	void
3327	__libc_free (void *mem)
3328	{
3329	mstate ar_ptr;
3330	mchunkptr p; / chunk corresponding to mem /
3331
3332	if (mem == `0`) / free(0) has no effect /
3333	return;
3334
3335	/ Quickly check that the freed pointer matches the tag for the memory.*
3336	This gives a useful double-free detection. /*
3337	if (__glibc_unlikely (mtag_enabled))
3338	(volatile* char *)mem;
3339
3340	int err = errno;
3341
3342	p = mem2chunk (mem);
3343
3344	if (chunk_is_mmapped (p)) / release mmapped memory. /
3345	{
3346	/ See if the dynamic brk/mmap threshold needs adjusting.*
3347	Dumped fake mmapped chunks do not affect the threshold. /*
3348	if (!mp_.no_dyn_threshold
3349	&& chunksize_nomask (p) > mp_.mmap_threshold
3350	&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
3351	{
3352	mp_.mmap_threshold = chunksize (p);
3353	mp_.trim_threshold = `2` * mp_.mmap_threshold;
3354	LIBC_PROBE (memory_mallopt_free_dyn_thresholds, `2`,
3355	mp_.mmap_threshold, mp_.trim_threshold);
3356	}
3357	munmap_chunk (p);
3358	}
3359	else
3360	{
3361	MAYBE_INIT_TCACHE ();
3362
3363	/ Mark the chunk as belonging to the library again. /
3364	(void)tag_region (chunk2mem (p), memsize (p));
3365
3366	ar_ptr = arena_for_chunk (p);
3367	_int_free (ar_ptr, p, `0`);
3368	}
3369
3370	__set_errno (err);
3371	}
3372	libc_hidden_def (__libc_free)
3373
3374	void *
3375	__libc_realloc (void *oldmem, size_t bytes)
3376	{
3377	mstate ar_ptr;
3378	INTERNAL_SIZE_T nb; / padded request size /
3379
3380	void newp; /* chunk to return /
3381
3382	if (!__malloc_initialized)
3383	ptmalloc_init ();
3384
3385	#if REALLOC_ZERO_BYTES_FREES
3386	if (bytes == `0` && oldmem != NULL)
3387	{
3388	__libc_free (oldmem); return `0`;
3389	}
3390	#endif
3391
3392	/ realloc of null is supposed to be same as malloc /
3393	if (oldmem == `0`)
3394	return __libc_malloc (bytes);
3395
3396	/ Perform a quick check to ensure that the pointer's tag matches the*
3397	memory's tag. /*
3398	if (__glibc_unlikely (mtag_enabled))
3399	(volatile* char*) oldmem;
3400
3401	/ Return the chunk as is whenever possible, i.e. there's enough usable space*
3402	but not so much that we end up fragmenting the block. We use the trim
3403	threshold as the heuristic to decide the latter. /*
3404	size_t usable = musable (oldmem);
3405	if (bytes <= usable
3406	&& (unsigned long) (usable - bytes) <= mp_.trim_threshold)
3407	return oldmem;
3408
3409	/ chunk corresponding to oldmem /
3410	const mchunkptr oldp = mem2chunk (oldmem);
3411	/ its size /
3412	const INTERNAL_SIZE_T oldsize = chunksize (oldp);
3413
3414	if (chunk_is_mmapped (oldp))
3415	ar_ptr = NULL;
3416	else
3417	{
3418	MAYBE_INIT_TCACHE ();
3419	ar_ptr = arena_for_chunk (oldp);
3420	}
3421
3422	/ Little security check which won't hurt performance: the allocator*
3423	never wrapps around at the end of the address space. Therefore
3424	we can exclude some size values which might appear here by
3425	accident or by "design" from some intruder. /*
3426	if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, `0`)
3427	\|\| __builtin_expect (misaligned_chunk (oldp), `0`)))
3428	malloc_printerr ("realloc(): invalid pointer");
3429
3430	nb = checked_request2size (bytes);
3431	if (nb == `0`)
3432	{
3433	__set_errno (ENOMEM);
3434	return NULL;
3435	}
3436
3437	if (chunk_is_mmapped (oldp))
3438	{
3439	void *newmem;
3440
3441	#if HAVE_MREMAP
3442	newp = mremap_chunk (oldp, nb);
3443	if (newp)
3444	{
3445	void *newmem = chunk2mem_tag (newp);
3446	/ Give the new block a different tag. This helps to ensure*
3447	that stale handles to the previous mapping are not
3448	reused. There's a performance hit for both us and the
3449	caller for doing this, so we might want to
3450	reconsider. /*
3451	return tag_new_usable (newmem);
3452	}
3453	#endif
3454	/ Note the extra SIZE_SZ overhead. /
3455	if (oldsize - SIZE_SZ >= nb)
3456	return oldmem; / do nothing /
3457
3458	/ Must alloc, copy, free. /
3459	newmem = __libc_malloc (bytes);
3460	if (newmem == `0`)
3461	return `0`; / propagate failure /
3462
3463	memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
3464	munmap_chunk (oldp);
3465	return newmem;
3466	}
3467
3468	if (SINGLE_THREAD_P)
3469	{
3470	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3471	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3472	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3473
3474	return newp;
3475	}
3476
3477	__libc_lock_lock (ar_ptr->mutex);
3478
3479	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3480
3481	__libc_lock_unlock (ar_ptr->mutex);
3482	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3483	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3484
3485	if (newp == NULL)
3486	{
3487	/ Try harder to allocate memory in other arenas. /
3488	LIBC_PROBE (memory_realloc_retry, `2`, bytes, oldmem);
3489	newp = __libc_malloc (bytes);
3490	if (newp != NULL)
3491	{
3492	size_t sz = memsize (oldp);
3493	memcpy (newp, oldmem, sz);
3494	(void) tag_region (chunk2mem (oldp), sz);
3495	_int_free (ar_ptr, oldp, `0`);
3496	}
3497	}
3498
3499	return newp;
3500	}
3501	libc_hidden_def (__libc_realloc)
3502
3503	void *
3504	__libc_memalign (size_t alignment, size_t bytes)
3505	{
3506	if (!__malloc_initialized)
3507	ptmalloc_init ();
3508
3509	void *address = RETURN_ADDRESS (`0`);
3510	return _mid_memalign (alignment, bytes, address);
3511	}
3512
3513	static void *
3514	_mid_memalign (size_t alignment, size_t bytes, void *address)
3515	{
3516	mstate ar_ptr;
3517	void *p;
3518
3519	/ If we need less alignment than we give anyway, just relay to malloc. /
3520	if (alignment <= MALLOC_ALIGNMENT)
3521	return __libc_malloc (bytes);
3522
3523	/ Otherwise, ensure that it is at least a minimum chunk size /
3524	if (alignment < MINSIZE)
3525	alignment = MINSIZE;
3526
3527	/ If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a*
3528	power of 2 and will cause overflow in the check below. /*
3529	if (alignment > SIZE_MAX / `2` + `1`)
3530	{
3531	__set_errno (EINVAL);
3532	return `0`;
3533	}
3534
3535
3536	/ Make sure alignment is power of 2. /
3537	if (!powerof2 (alignment))
3538	{
3539	size_t a = MALLOC_ALIGNMENT * `2`;
3540	while (a < alignment)
3541	a <<= `1`;
3542	alignment = a;
3543	}
3544
3545	if (SINGLE_THREAD_P)
3546	{
3547	p = _int_memalign (&main_arena, alignment, bytes);
3548	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3549	&main_arena == arena_for_chunk (mem2chunk (p)));
3550	return tag_new_usable (p);
3551	}
3552
3553	arena_get (ar_ptr, bytes + alignment + MINSIZE);
3554
3555	p = _int_memalign (ar_ptr, alignment, bytes);
3556	if (!p && ar_ptr != NULL)
3557	{
3558	LIBC_PROBE (memory_memalign_retry, `2`, bytes, alignment);
3559	ar_ptr = arena_get_retry (ar_ptr, bytes);
3560	p = _int_memalign (ar_ptr, alignment, bytes);
3561	}
3562
3563	if (ar_ptr != NULL)
3564	__libc_lock_unlock (ar_ptr->mutex);
3565
3566	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3567	ar_ptr == arena_for_chunk (mem2chunk (p)));
3568	return tag_new_usable (p);
3569	}
3570	/ For ISO C11. /
3571	weak_alias (__libc_memalign, aligned_alloc)
3572	libc_hidden_def (__libc_memalign)
3573
3574	void *
3575	__libc_valloc (size_t bytes)
3576	{
3577	if (!__malloc_initialized)
3578	ptmalloc_init ();
3579
3580	void *address = RETURN_ADDRESS (`0`);
3581	size_t pagesize = GLRO (dl_pagesize);
3582	return _mid_memalign (pagesize, bytes, address);
3583	}
3584
3585	void *
3586	__libc_pvalloc (size_t bytes)
3587	{
3588	if (!__malloc_initialized)
3589	ptmalloc_init ();
3590
3591	void *address = RETURN_ADDRESS (`0`);
3592	size_t pagesize = GLRO (dl_pagesize);
3593	size_t rounded_bytes;
3594	/ ALIGN_UP with overflow check. /
3595	if (__glibc_unlikely (__builtin_add_overflow (bytes,
3596	pagesize - `1`,
3597	&rounded_bytes)))
3598	{
3599	__set_errno (ENOMEM);
3600	return `0`;
3601	}
3602	rounded_bytes = rounded_bytes & -(pagesize - `1`);
3603
3604	return _mid_memalign (pagesize, rounded_bytes, address);
3605	}
3606
3607	void *
3608	__libc_calloc (size_t n, size_t elem_size)
3609	{
3610	mstate av;
3611	mchunkptr oldtop;
3612	INTERNAL_SIZE_T sz, oldtopsize;
3613	void *mem;
3614	unsigned long clearsize;
3615	unsigned long nclears;
3616	INTERNAL_SIZE_T *d;
3617	ptrdiff_t bytes;
3618
3619	if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
3620	{
3621	__set_errno (ENOMEM);
3622	return NULL;
3623	}
3624
3625	sz = bytes;
3626
3627	if (!__malloc_initialized)
3628	ptmalloc_init ();
3629
3630	MAYBE_INIT_TCACHE ();
3631
3632	if (SINGLE_THREAD_P)
3633	av = &main_arena;
3634	else
3635	arena_get (av, sz);
3636
3637	if (av)
3638	{
3639	/ Check if we hand out the top chunk, in which case there may be no*
3640	need to clear. /*
3641	#if MORECORE_CLEARS
3642	oldtop = top (av);
3643	oldtopsize = chunksize (top (av));
3644	# if MORECORE_CLEARS < 2
3645	/ Only newly allocated memory is guaranteed to be cleared. /
3646	if (av == &main_arena &&
3647	oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
3648	oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
3649	# endif
3650	if (av != &main_arena)
3651	{
3652	heap_info *heap = heap_for_ptr (oldtop);
3653	if (oldtopsize < (char ) heap + heap->mprotect_size - (char* *) oldtop)
3654	oldtopsize = (char ) heap + heap->mprotect_size - (char* *) oldtop;
3655	}
3656	#endif
3657	}
3658	else
3659	{
3660	/ No usable arenas. /
3661	oldtop = `0`;
3662	oldtopsize = `0`;
3663	}
3664	mem = _int_malloc (av, sz);
3665
3666	assert (!mem \|\| chunk_is_mmapped (mem2chunk (mem)) \|\|
3667	av == arena_for_chunk (mem2chunk (mem)));
3668
3669	if (!SINGLE_THREAD_P)
3670	{
3671	if (mem == `0` && av != NULL)
3672	{
3673	LIBC_PROBE (memory_calloc_retry, `1`, sz);
3674	av = arena_get_retry (av, sz);
3675	mem = _int_malloc (av, sz);
3676	}
3677
3678	if (av != NULL)
3679	__libc_lock_unlock (av->mutex);
3680	}
3681
3682	/ Allocation failed even after a retry. /
3683	if (mem == `0`)
3684	return `0`;
3685
3686	mchunkptr p = mem2chunk (mem);
3687
3688	/ If we are using memory tagging, then we need to set the tags*
3689	regardless of MORECORE_CLEARS, so we zero the whole block while
3690	doing so. /*
3691	if (__glibc_unlikely (mtag_enabled))
3692	return tag_new_zero_region (mem, memsize (p));
3693
3694	INTERNAL_SIZE_T csz = chunksize (p);
3695
3696	/ Two optional cases in which clearing not necessary /
3697	if (chunk_is_mmapped (p))
3698	{
3699	if (__builtin_expect (perturb_byte, `0`))
3700	return memset (mem, `0`, sz);
3701
3702	return mem;
3703	}
3704
3705	#if MORECORE_CLEARS
3706	if (perturb_byte == `0` && (p == oldtop && csz > oldtopsize))
3707	{
3708	/ clear only the bytes from non-freshly-sbrked memory /
3709	csz = oldtopsize;
3710	}
3711	#endif
3712
3713	/ Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that*
3714	contents have an odd number of INTERNAL_SIZE_T-sized words;
3715	minimally 3. /*
3716	d = (INTERNAL_SIZE_T *) mem;
3717	clearsize = csz - SIZE_SZ;
3718	nclears = clearsize / sizeof (INTERNAL_SIZE_T);
3719	assert (nclears >= `3`);
3720
3721	if (nclears > `9`)
3722	return memset (d, `0`, clearsize);
3723
3724	else
3725	{
3726	*(d + `0`) = `0`;
3727	*(d + `1`) = `0`;
3728	*(d + `2`) = `0`;
3729	if (nclears > `4`)
3730	{
3731	*(d + `3`) = `0`;
3732	*(d + `4`) = `0`;
3733	if (nclears > `6`)
3734	{
3735	*(d + `5`) = `0`;
3736	*(d + `6`) = `0`;
3737	if (nclears > `8`)
3738	{
3739	*(d + `7`) = `0`;
3740	*(d + `8`) = `0`;
3741	}
3742	}
3743	}
3744	}
3745
3746	return mem;
3747	}
3748	#endif /* IS_IN (libc) */
3749
3750	/*
3751	------------------------------ malloc ------------------------------
3752	*/
3753
3754	static void *
3755	_int_malloc (mstate av, size_t bytes)
3756	{
3757	INTERNAL_SIZE_T nb; / normalized request size /
3758	unsigned int idx; / associated bin index /
3759	mbinptr bin; / associated bin /
3760
3761	mchunkptr victim; / inspected/selected chunk /
3762	INTERNAL_SIZE_T size; / its size /
3763	int victim_index; / its bin index /
3764
3765	mchunkptr remainder; / remainder from a split /
3766	unsigned long remainder_size; / its size /
3767
3768	unsigned int block; / bit map traverser /
3769	unsigned int bit; / bit map traverser /
3770	unsigned int map; / current word of binmap /
3771
3772	mchunkptr fwd; / misc temp for linking /
3773	mchunkptr bck; / misc temp for linking /
3774
3775	#if USE_TCACHE
3776	size_t tcache_unsorted_count; / count of unsorted chunks processed /
3777	#endif
3778
3779	/*
3780	Convert request size to internal form by adding SIZE_SZ bytes
3781	overhead plus possibly more to obtain necessary alignment and/or
3782	to obtain a size of at least MINSIZE, the smallest allocatable
3783	size. Also, checked_request2size returns false for request sizes
3784	that are so large that they wrap around zero when padded and
3785	aligned.
3786	*/
3787
3788	nb = checked_request2size (bytes);
3789	if (nb == `0`)
3790	{
3791	__set_errno (ENOMEM);
3792	return NULL;
3793	}
3794
3795	/ There are no usable arenas. Fall back to sysmalloc to get a chunk from*
3796	mmap. /*
3797	if (__glibc_unlikely (av == NULL))
3798	{
3799	void *p = sysmalloc (nb, av);
3800	if (p != NULL)
3801	alloc_perturb (p, bytes);
3802	return p;
3803	}
3804
3805	/*
3806	If the size qualifies as a fastbin, first check corresponding bin.
3807	This code is safe to execute even if av is not yet initialized, so we
3808	can try it without checking, which saves some time on this fast path.
3809	*/
3810
3811	#define REMOVE_FB(fb, victim, pp) \
3812	do \
3813	{ \
3814	victim = pp; \
3815	if (victim == NULL) \
3816	break; \
3817	pp = REVEAL_PTR (victim->fd); \
3818	if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp))) \
3819	malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
3820	} \
3821	while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
3822	!= victim); \
3823
3824	if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
3825	{
3826	idx = fastbin_index (nb);
3827	mfastbinptr *fb = &fastbin (av, idx);
3828	mchunkptr pp;
3829	victim = *fb;
3830
3831	if (victim != NULL)
3832	{
3833	if (__glibc_unlikely (misaligned_chunk (victim)))
3834	malloc_printerr ("malloc(): unaligned fastbin chunk detected 2");
3835
3836	if (SINGLE_THREAD_P)
3837	*fb = REVEAL_PTR (victim->fd);
3838	else
3839	REMOVE_FB (fb, pp, victim);
3840	if (__glibc_likely (victim != NULL))
3841	{
3842	size_t victim_idx = fastbin_index (chunksize (victim));
3843	if (__builtin_expect (victim_idx != idx, `0`))
3844	malloc_printerr ("malloc(): memory corruption (fast)");
3845	check_remalloced_chunk (av, victim, nb);
3846	#if USE_TCACHE
3847	/ While we're here, if we see other chunks of the same size,*
3848	stash them in the tcache. /*
3849	size_t tc_idx = csize2tidx (nb);
3850	if (tcache && tc_idx < mp_.tcache_bins)
3851	{
3852	mchunkptr tc_victim;
3853
3854	/ While bin not empty and tcache not full, copy chunks. /
3855	while (tcache->counts[tc_idx] < mp_.tcache_count
3856	&& (tc_victim = *fb) != NULL)
3857	{
3858	if (__glibc_unlikely (misaligned_chunk (tc_victim)))
3859	malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
3860	if (SINGLE_THREAD_P)
3861	*fb = REVEAL_PTR (tc_victim->fd);
3862	else
3863	{
3864	REMOVE_FB (fb, pp, tc_victim);
3865	if (__glibc_unlikely (tc_victim == NULL))
3866	break;
3867	}
3868	tcache_put (tc_victim, tc_idx);
3869	}
3870	}
3871	#endif
3872	void *p = chunk2mem (victim);
3873	alloc_perturb (p, bytes);
3874	return p;
3875	}
3876	}
3877	}
3878
3879	/*
3880	If a small request, check regular bin. Since these "smallbins"
3881	hold one size each, no searching within bins is necessary.
3882	(For a large request, we need to wait until unsorted chunks are
3883	processed to find best fit. But for small ones, fits are exact
3884	anyway, so we can check now, which is faster.)
3885	*/
3886
3887	if (in_smallbin_range (nb))
3888	{
3889	idx = smallbin_index (nb);
3890	bin = bin_at (av, idx);
3891
3892	if ((victim = last (bin)) != bin)
3893	{
3894	bck = victim->bk;
3895	if (__glibc_unlikely (bck->fd != victim))
3896	malloc_printerr ("malloc(): smallbin double linked list corrupted");
3897	set_inuse_bit_at_offset (victim, nb);
3898	bin->bk = bck;
3899	bck->fd = bin;
3900
3901	if (av != &main_arena)
3902	set_non_main_arena (victim);
3903	check_malloced_chunk (av, victim, nb);
3904	#if USE_TCACHE
3905	/ While we're here, if we see other chunks of the same size,*
3906	stash them in the tcache. /*
3907	size_t tc_idx = csize2tidx (nb);
3908	if (tcache && tc_idx < mp_.tcache_bins)
3909	{
3910	mchunkptr tc_victim;
3911
3912	/ While bin not empty and tcache not full, copy chunks over. /
3913	while (tcache->counts[tc_idx] < mp_.tcache_count
3914	&& (tc_victim = last (bin)) != bin)
3915	{
3916	if (tc_victim != `0`)
3917	{
3918	bck = tc_victim->bk;
3919	set_inuse_bit_at_offset (tc_victim, nb);
3920	if (av != &main_arena)
3921	set_non_main_arena (tc_victim);
3922	bin->bk = bck;
3923	bck->fd = bin;
3924
3925	tcache_put (tc_victim, tc_idx);
3926	}
3927	}
3928	}
3929	#endif
3930	void *p = chunk2mem (victim);
3931	alloc_perturb (p, bytes);
3932	return p;
3933	}
3934	}
3935
3936	/*
3937	If this is a large request, consolidate fastbins before continuing.
3938	While it might look excessive to kill all fastbins before
3939	even seeing if there is space available, this avoids
3940	fragmentation problems normally associated with fastbins.
3941	Also, in practice, programs tend to have runs of either small or
3942	large requests, but less often mixtures, so consolidation is not
3943	invoked all that often in most programs. And the programs that
3944	it is called frequently in otherwise tend to fragment.
3945	*/
3946
3947	else
3948	{
3949	idx = largebin_index (nb);
3950	if (atomic_load_relaxed (&av->have_fastchunks))
3951	malloc_consolidate (av);
3952	}
3953
3954	/*
3955	Process recently freed or remaindered chunks, taking one only if
3956	it is exact fit, or, if this a small request, the chunk is remainder from
3957	the most recent non-exact fit. Place other traversed chunks in
3958	bins. Note that this step is the only place in any routine where
3959	chunks are placed in bins.
3960
3961	The outer loop here is needed because we might not realize until
3962	near the end of malloc that we should have consolidated, so must
3963	do so and retry. This happens at most once, and only when we would
3964	otherwise need to expand memory to service a "small" request.
3965	*/
3966
3967	#if USE_TCACHE
3968	INTERNAL_SIZE_T tcache_nb = `0`;
3969	size_t tc_idx = csize2tidx (nb);
3970	if (tcache && tc_idx < mp_.tcache_bins)
3971	tcache_nb = nb;
3972	int return_cached = `0`;
3973
3974	tcache_unsorted_count = `0`;
3975	#endif
3976
3977	for (;; )
3978	{
3979	int iters = `0`;
3980	while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
3981	{
3982	bck = victim->bk;
3983	size = chunksize (victim);
3984	mchunkptr next = chunk_at_offset (victim, size);
3985
3986	if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
3987	\|\| __glibc_unlikely (size > av->system_mem))
3988	malloc_printerr ("malloc(): invalid size (unsorted)");
3989	if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
3990	\|\| __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
3991	malloc_printerr ("malloc(): invalid next size (unsorted)");
3992	if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
3993	malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)");
3994	if (__glibc_unlikely (bck->fd != victim)
3995	\|\| __glibc_unlikely (victim->fd != unsorted_chunks (av)))
3996	malloc_printerr ("malloc(): unsorted double linked list corrupted");
3997	if (__glibc_unlikely (prev_inuse (next)))
3998	malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)");
3999
4000	/*
4001	If a small request, try to use last remainder if it is the
4002	only chunk in unsorted bin. This helps promote locality for
4003	runs of consecutive small requests. This is the only
4004	exception to best-fit, and applies only when there is
4005	no exact fit for a small chunk.
4006	*/
4007
4008	if (in_smallbin_range (nb) &&
4009	bck == unsorted_chunks (av) &&
4010	victim == av->last_remainder &&
4011	(unsigned long) (size) > (unsigned long) (nb + MINSIZE))
4012	{
4013	/ split and reattach remainder /
4014	remainder_size = size - nb;
4015	remainder = chunk_at_offset (victim, nb);
4016	unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
4017	av->last_remainder = remainder;
4018	remainder->bk = remainder->fd = unsorted_chunks (av);
4019	if (!in_smallbin_range (remainder_size))
4020	{
4021	remainder->fd_nextsize = NULL;
4022	remainder->bk_nextsize = NULL;
4023	}
4024
4025	set_head (victim, nb \| PREV_INUSE \|
4026	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4027	set_head (remainder, remainder_size \| PREV_INUSE);
4028	set_foot (remainder, remainder_size);
4029
4030	check_malloced_chunk (av, victim, nb);
4031	void *p = chunk2mem (victim);
4032	alloc_perturb (p, bytes);
4033	return p;
4034	}
4035
4036	/ remove from unsorted list /
4037	if (__glibc_unlikely (bck->fd != victim))
4038	malloc_printerr ("malloc(): corrupted unsorted chunks 3");
4039	unsorted_chunks (av)->bk = bck;
4040	bck->fd = unsorted_chunks (av);
4041
4042	/ Take now instead of binning if exact fit /
4043
4044	if (size == nb)
4045	{
4046	set_inuse_bit_at_offset (victim, size);
4047	if (av != &main_arena)
4048	set_non_main_arena (victim);
4049	#if USE_TCACHE
4050	/ Fill cache first, return to user only if cache fills.*
4051	We may return one of these chunks later. /*
4052	if (tcache_nb
4053	&& tcache->counts[tc_idx] < mp_.tcache_count)
4054	{
4055	tcache_put (victim, tc_idx);
4056	return_cached = `1`;
4057	continue;
4058	}
4059	else
4060	{
4061	#endif
4062	check_malloced_chunk (av, victim, nb);
4063	void *p = chunk2mem (victim);
4064	alloc_perturb (p, bytes);
4065	return p;
4066	#if USE_TCACHE
4067	}
4068	#endif
4069	}
4070
4071	/ place chunk in bin /
4072
4073	if (in_smallbin_range (size))
4074	{
4075	victim_index = smallbin_index (size);
4076	bck = bin_at (av, victim_index);
4077	fwd = bck->fd;
4078	}
4079	else
4080	{
4081	victim_index = largebin_index (size);
4082	bck = bin_at (av, victim_index);
4083	fwd = bck->fd;
4084
4085	/ maintain large bins in sorted order /
4086	if (fwd != bck)
4087	{
4088	/ Or with inuse bit to speed comparisons /
4089	size \|= PREV_INUSE;
4090	/ if smaller than smallest, bypass loop below /
4091	assert (chunk_main_arena (bck->bk));
4092	if ((unsigned long) (size)
4093	< (unsigned long) chunksize_nomask (bck->bk))
4094	{
4095	fwd = bck;
4096	bck = bck->bk;
4097
4098	victim->fd_nextsize = fwd->fd;
4099	victim->bk_nextsize = fwd->fd->bk_nextsize;
4100	fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
4101	}
4102	else
4103	{
4104	assert (chunk_main_arena (fwd));
4105	while ((unsigned long) size < chunksize_nomask (fwd))
4106	{
4107	fwd = fwd->fd_nextsize;
4108	assert (chunk_main_arena (fwd));
4109	}
4110
4111	if ((unsigned long) size
4112	== (unsigned long) chunksize_nomask (fwd))
4113	/ Always insert in the second position. /
4114	fwd = fwd->fd;
4115	else
4116	{
4117	victim->fd_nextsize = fwd;
4118	victim->bk_nextsize = fwd->bk_nextsize;
4119	if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
4120	malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)");
4121	fwd->bk_nextsize = victim;
4122	victim->bk_nextsize->fd_nextsize = victim;
4123	}
4124	bck = fwd->bk;
4125	if (bck->fd != fwd)
4126	malloc_printerr ("malloc(): largebin double linked list corrupted (bk)");
4127	}
4128	}
4129	else
4130	victim->fd_nextsize = victim->bk_nextsize = victim;
4131	}
4132
4133	mark_bin (av, victim_index);
4134	victim->bk = bck;
4135	victim->fd = fwd;
4136	fwd->bk = victim;
4137	bck->fd = victim;
4138
4139	#if USE_TCACHE
4140	/ If we've processed as many chunks as we're allowed while*
4141	filling the cache, return one of the cached ones. /*
4142	++tcache_unsorted_count;
4143	if (return_cached
4144	&& mp_.tcache_unsorted_limit > `0`
4145	&& tcache_unsorted_count > mp_.tcache_unsorted_limit)
4146	{
4147	return tcache_get (tc_idx);
4148	}
4149	#endif
4150
4151	#define MAX_ITERS 10000
4152	if (++iters >= MAX_ITERS)
4153	break;
4154	}
4155
4156	#if USE_TCACHE
4157	/ If all the small chunks we found ended up cached, return one now. /
4158	if (return_cached)
4159	{
4160	return tcache_get (tc_idx);
4161	}
4162	#endif
4163
4164	/*
4165	If a large request, scan through the chunks of current bin in
4166	sorted order to find smallest that fits. Use the skip list for this.
4167	*/
4168
4169	if (!in_smallbin_range (nb))
4170	{
4171	bin = bin_at (av, idx);
4172
4173	/ skip scan if empty or largest chunk is too small /
4174	if ((victim = first (bin)) != bin
4175	&& (unsigned long) chunksize_nomask (victim)
4176	>= (unsigned long) (nb))
4177	{
4178	victim = victim->bk_nextsize;
4179	while (((unsigned long) (size = chunksize (victim)) <
4180	(unsigned long) (nb)))
4181	victim = victim->bk_nextsize;
4182
4183	/ Avoid removing the first entry for a size so that the skip*
4184	list does not have to be rerouted. /*
4185	if (victim != last (bin)
4186	&& chunksize_nomask (victim)
4187	== chunksize_nomask (victim->fd))
4188	victim = victim->fd;
4189
4190	remainder_size = size - nb;
4191	unlink_chunk (av, victim);
4192
4193	/ Exhaust /
4194	if (remainder_size < MINSIZE)
4195	{
4196	set_inuse_bit_at_offset (victim, size);
4197	if (av != &main_arena)
4198	set_non_main_arena (victim);
4199	}
4200	/ Split /
4201	else
4202	{
4203	remainder = chunk_at_offset (victim, nb);
4204	/ We cannot assume the unsorted list is empty and therefore*
4205	have to perform a complete insert here. /*
4206	bck = unsorted_chunks (av);
4207	fwd = bck->fd;
4208	if (__glibc_unlikely (fwd->bk != bck))
4209	malloc_printerr ("malloc(): corrupted unsorted chunks");
4210	remainder->bk = bck;
4211	remainder->fd = fwd;
4212	bck->fd = remainder;
4213	fwd->bk = remainder;
4214	if (!in_smallbin_range (remainder_size))
4215	{
4216	remainder->fd_nextsize = NULL;
4217	remainder->bk_nextsize = NULL;
4218	}
4219	set_head (victim, nb \| PREV_INUSE \|
4220	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4221	set_head (remainder, remainder_size \| PREV_INUSE);
4222	set_foot (remainder, remainder_size);
4223	}
4224	check_malloced_chunk (av, victim, nb);
4225	void *p = chunk2mem (victim);
4226	alloc_perturb (p, bytes);
4227	return p;
4228	}
4229	}
4230
4231	/*
4232	Search for a chunk by scanning bins, starting with next largest
4233	bin. This search is strictly by best-fit; i.e., the smallest
4234	(with ties going to approximately the least recently used) chunk
4235	that fits is selected.
4236
4237	The bitmap avoids needing to check that most blocks are nonempty.
4238	The particular case of skipping all bins during warm-up phases
4239	when no chunks have been returned yet is faster than it might look.
4240	*/
4241
4242	++idx;
4243	bin = bin_at (av, idx);
4244	block = idx2block (idx);
4245	map = av->binmap[block];
4246	bit = idx2bit (idx);
4247
4248	for (;; )
4249	{
4250	/ Skip rest of block if there are no more set bits in this block. /
4251	if (bit > map \|\| bit == `0`)
4252	{
4253	do
4254	{
4255	if (++block >= BINMAPSIZE) / out of bins /
4256	goto use_top;
4257	}
4258	while ((map = av->binmap[block]) == `0`);
4259
4260	bin = bin_at (av, (block << BINMAPSHIFT));
4261	bit = `1`;
4262	}
4263
4264	/ Advance to bin with set bit. There must be one. /
4265	while ((bit & map) == `0`)
4266	{
4267	bin = next_bin (bin);
4268	bit <<= `1`;
4269	assert (bit != `0`);
4270	}
4271
4272	/ Inspect the bin. It is likely to be non-empty /
4273	victim = last (bin);
4274
4275	/ If a false alarm (empty bin), clear the bit. /
4276	if (victim == bin)
4277	{
4278	av->binmap[block] = map &= ~bit; / Write through /
4279	bin = next_bin (bin);
4280	bit <<= `1`;
4281	}
4282
4283	else
4284	{
4285	size = chunksize (victim);
4286
4287	/ We know the first chunk in this bin is big enough to use. /
4288	assert ((unsigned long) (size) >= (unsigned long) (nb));
4289
4290	remainder_size = size - nb;
4291
4292	/ unlink /
4293	unlink_chunk (av, victim);
4294
4295	/ Exhaust /
4296	if (remainder_size < MINSIZE)
4297	{
4298	set_inuse_bit_at_offset (victim, size);
4299	if (av != &main_arena)
4300	set_non_main_arena (victim);
4301	}
4302
4303	/ Split /
4304	else
4305	{
4306	remainder = chunk_at_offset (victim, nb);
4307
4308	/ We cannot assume the unsorted list is empty and therefore*
4309	have to perform a complete insert here. /*
4310	bck = unsorted_chunks (av);
4311	fwd = bck->fd;
4312	if (__glibc_unlikely (fwd->bk != bck))
4313	malloc_printerr ("malloc(): corrupted unsorted chunks 2");
4314	remainder->bk = bck;
4315	remainder->fd = fwd;
4316	bck->fd = remainder;
4317	fwd->bk = remainder;
4318
4319	/ advertise as last remainder /
4320	if (in_smallbin_range (nb))
4321	av->last_remainder = remainder;
4322	if (!in_smallbin_range (remainder_size))
4323	{
4324	remainder->fd_nextsize = NULL;
4325	remainder->bk_nextsize = NULL;
4326	}
4327	set_head (victim, nb \| PREV_INUSE \|
4328	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4329	set_head (remainder, remainder_size \| PREV_INUSE);
4330	set_foot (remainder, remainder_size);
4331	}
4332	check_malloced_chunk (av, victim, nb);
4333	void *p = chunk2mem (victim);
4334	alloc_perturb (p, bytes);
4335	return p;
4336	}
4337	}
4338
4339	use_top:
4340	/*
4341	If large enough, split off the chunk bordering the end of memory
4342	(held in av->top). Note that this is in accord with the best-fit
4343	search rule. In effect, av->top is treated as larger (and thus
4344	less well fitting) than any other available chunk since it can
4345	be extended to be as large as necessary (up to system
4346	limitations).
4347
4348	We require that av->top always exists (i.e., has size >=
4349	MINSIZE) after initialization, so if it would otherwise be
4350	exhausted by current request, it is replenished. (The main
4351	reason for ensuring it exists is that we may need MINSIZE space
4352	to put in fenceposts in sysmalloc.)
4353	*/
4354
4355	victim = av->top;
4356	size = chunksize (victim);
4357
4358	if (__glibc_unlikely (size > av->system_mem))
4359	malloc_printerr ("malloc(): corrupted top size");
4360
4361	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
4362	{
4363	remainder_size = size - nb;
4364	remainder = chunk_at_offset (victim, nb);
4365	av->top = remainder;
4366	set_head (victim, nb \| PREV_INUSE \|
4367	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4368	set_head (remainder, remainder_size \| PREV_INUSE);
4369
4370	check_malloced_chunk (av, victim, nb);
4371	void *p = chunk2mem (victim);
4372	alloc_perturb (p, bytes);
4373	return p;
4374	}
4375
4376	/ When we are using atomic ops to free fast chunks we can get*
4377	here for all block sizes. /*
4378	else if (atomic_load_relaxed (&av->have_fastchunks))
4379	{
4380	malloc_consolidate (av);
4381	/ restore original bin index /
4382	if (in_smallbin_range (nb))
4383	idx = smallbin_index (nb);
4384	else
4385	idx = largebin_index (nb);
4386	}
4387
4388	/*
4389	Otherwise, relay to handle system-dependent cases
4390	*/
4391	else
4392	{
4393	void *p = sysmalloc (nb, av);
4394	if (p != NULL)
4395	alloc_perturb (p, bytes);
4396	return p;
4397	}
4398	}
4399	}
4400
4401	/*
4402	------------------------------ free ------------------------------
4403	*/
4404
4405	static void
4406	_int_free (mstate av, mchunkptr p, int have_lock)
4407	{
4408	INTERNAL_SIZE_T size; / its size /
4409	mfastbinptr fb; /* associated fastbin /
4410	mchunkptr nextchunk; / next contiguous chunk /
4411	INTERNAL_SIZE_T nextsize; / its size /
4412	int nextinuse; / true if nextchunk is used /
4413	INTERNAL_SIZE_T prevsize; / size of previous contiguous chunk /
4414	mchunkptr bck; / misc temp for linking /
4415	mchunkptr fwd; / misc temp for linking /
4416
4417	size = chunksize (p);
4418
4419	/ Little security check which won't hurt performance: the*
4420	allocator never wrapps around at the end of the address space.
4421	Therefore we can exclude some size values which might appear
4422	here by accident or by "design" from some intruder. /*
4423	if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, `0`)
4424	\|\| __builtin_expect (misaligned_chunk (p), `0`))
4425	malloc_printerr ("free(): invalid pointer");
4426	/ We know that each chunk is at least MINSIZE bytes in size or a*
4427	multiple of MALLOC_ALIGNMENT. /*
4428	if (__glibc_unlikely (size < MINSIZE \|\| !aligned_OK (size)))
4429	malloc_printerr ("free(): invalid size");
4430
4431	check_inuse_chunk(av, p);
4432
4433	#if USE_TCACHE
4434	{
4435	size_t tc_idx = csize2tidx (size);
4436	if (tcache != NULL && tc_idx < mp_.tcache_bins)
4437	{
4438	/ Check to see if it's already in the tcache. /
4439	tcache_entry e = (tcache_entry ) chunk2mem (p);
4440
4441	/ This test succeeds on double free. However, we don't 100%*
4442	trust it (it also matches random payload data at a 1 in
4443	2^<size_t> chance), so verify it's not an unlikely
4444	coincidence before aborting. /*
4445	if (__glibc_unlikely (e->key == tcache_key))
4446	{
4447	tcache_entry *tmp;
4448	size_t cnt = `0`;
4449	LIBC_PROBE (memory_tcache_double_free, `2`, e, tc_idx);
4450	for (tmp = tcache->entries[tc_idx];
4451	tmp;
4452	tmp = REVEAL_PTR (tmp->next), ++cnt)
4453	{
4454	if (cnt >= mp_.tcache_count)
4455	malloc_printerr ("free(): too many chunks detected in tcache");
4456	if (__glibc_unlikely (!aligned_OK (tmp)))
4457	malloc_printerr ("free(): unaligned chunk detected in tcache 2");
4458	if (tmp == e)
4459	malloc_printerr ("free(): double free detected in tcache 2");
4460	/ If we get here, it was a coincidence. We've wasted a*
4461	few cycles, but don't abort. /*
4462	}
4463	}
4464
4465	if (tcache->counts[tc_idx] < mp_.tcache_count)
4466	{
4467	tcache_put (p, tc_idx);
4468	return;
4469	}
4470	}
4471	}
4472	#endif
4473
4474	/*
4475	If eligible, place chunk on a fastbin so it can be found
4476	and used quickly in malloc.
4477	*/
4478
4479	if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
4480
4481	#if TRIM_FASTBINS
4482	/*
4483	If TRIM_FASTBINS set, don't place chunks
4484	bordering top into fastbins
4485	*/
4486	&& (chunk_at_offset(p, size) != av->top)
4487	#endif
4488	) {
4489
4490	if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
4491	<= CHUNK_HDR_SZ, `0`)
4492	\|\| __builtin_expect (chunksize (chunk_at_offset (p, size))
4493	>= av->system_mem, `0`))
4494	{
4495	bool fail = true;
4496	/ We might not have a lock at this point and concurrent modifications*
4497	of system_mem might result in a false positive. Redo the test after
4498	getting the lock. /*
4499	if (!have_lock)
4500	{
4501	__libc_lock_lock (av->mutex);
4502	fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
4503	\|\| chunksize (chunk_at_offset (p, size)) >= av->system_mem);
4504	__libc_lock_unlock (av->mutex);
4505	}
4506
4507	if (fail)
4508	malloc_printerr ("free(): invalid next size (fast)");
4509	}
4510
4511	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4512
4513	atomic_store_relaxed (&av->have_fastchunks, true);
4514	unsigned int idx = fastbin_index(size);
4515	fb = &fastbin (av, idx);
4516
4517	/ Atomically link P to its fastbin: P->FD = FB; FB = P; /
4518	mchunkptr old = *fb, old2;
4519
4520	if (SINGLE_THREAD_P)
4521	{
4522	/ Check that the top of the bin is not the record we are going to*
4523	add (i.e., double free). /*
4524	if (__builtin_expect (old == p, `0`))
4525	malloc_printerr ("double free or corruption (fasttop)");
4526	p->fd = PROTECT_PTR (&p->fd, old);
4527	*fb = p;
4528	}
4529	else
4530	do
4531	{
4532	/ Check that the top of the bin is not the record we are going to*
4533	add (i.e., double free). /*
4534	if (__builtin_expect (old == p, `0`))
4535	malloc_printerr ("double free or corruption (fasttop)");
4536	old2 = old;
4537	p->fd = PROTECT_PTR (&p->fd, old);
4538	}
4539	while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
4540	!= old2);
4541
4542	/ Check that size of fastbin chunk at the top is the same as*
4543	size of the chunk that we are adding. We can dereference OLD
4544	only if we have the lock, otherwise it might have already been
4545	allocated again. /*
4546	if (have_lock && old != NULL
4547	&& __builtin_expect (fastbin_index (chunksize (old)) != idx, `0`))
4548	malloc_printerr ("invalid fastbin entry (free)");
4549	}
4550
4551	/*
4552	Consolidate other non-mmapped chunks as they arrive.
4553	*/
4554
4555	else if (!chunk_is_mmapped(p)) {
4556
4557	/ If we're single-threaded, don't lock the arena. /
4558	if (SINGLE_THREAD_P)
4559	have_lock = true;
4560
4561	if (!have_lock)
4562	__libc_lock_lock (av->mutex);
4563
4564	nextchunk = chunk_at_offset(p, size);
4565
4566	/ Lightweight tests: check whether the block is already the*
4567	top block. /*
4568	if (__glibc_unlikely (p == av->top))
4569	malloc_printerr ("double free or corruption (top)");
4570	/ Or whether the next chunk is beyond the boundaries of the arena. /
4571	if (__builtin_expect (contiguous (av)
4572	&& (char *) nextchunk
4573	>= ((char *) av->top + chunksize(av->top)), `0`))
4574	malloc_printerr ("double free or corruption (out)");
4575	/ Or whether the block is actually not marked used. /
4576	if (__glibc_unlikely (!prev_inuse(nextchunk)))
4577	malloc_printerr ("double free or corruption (!prev)");
4578
4579	nextsize = chunksize(nextchunk);
4580	if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, `0`)
4581	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4582	malloc_printerr ("free(): invalid next size (normal)");
4583
4584	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4585
4586	/ consolidate backward /
4587	if (!prev_inuse(p)) {
4588	prevsize = prev_size (p);
4589	size += prevsize;
4590	p = chunk_at_offset(p, -((long) prevsize));
4591	if (__glibc_unlikely (chunksize(p) != prevsize))
4592	malloc_printerr ("corrupted size vs. prev_size while consolidating");
4593	unlink_chunk (av, p);
4594	}
4595
4596	if (nextchunk != av->top) {
4597	/ get and clear inuse bit /
4598	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4599
4600	/ consolidate forward /
4601	if (!nextinuse) {
4602	unlink_chunk (av, nextchunk);
4603	size += nextsize;
4604	} else
4605	clear_inuse_bit_at_offset(nextchunk, `0`);
4606
4607	/*
4608	Place the chunk in unsorted chunk list. Chunks are
4609	not placed into regular bins until after they have
4610	been given one chance to be used in malloc.
4611	*/
4612
4613	bck = unsorted_chunks(av);
4614	fwd = bck->fd;
4615	if (__glibc_unlikely (fwd->bk != bck))
4616	malloc_printerr ("free(): corrupted unsorted chunks");
4617	p->fd = fwd;
4618	p->bk = bck;
4619	if (!in_smallbin_range(size))
4620	{
4621	p->fd_nextsize = NULL;
4622	p->bk_nextsize = NULL;
4623	}
4624	bck->fd = p;
4625	fwd->bk = p;
4626
4627	set_head(p, size \| PREV_INUSE);
4628	set_foot(p, size);
4629
4630	check_free_chunk(av, p);
4631	}
4632
4633	/*
4634	If the chunk borders the current high end of memory,
4635	consolidate into top
4636	*/
4637
4638	else {
4639	size += nextsize;
4640	set_head(p, size \| PREV_INUSE);
4641	av->top = p;
4642	check_chunk(av, p);
4643	}
4644
4645	/*
4646	If freeing a large space, consolidate possibly-surrounding
4647	chunks. Then, if the total unused topmost memory exceeds trim
4648	threshold, ask malloc_trim to reduce top.
4649
4650	Unless max_fast is 0, we don't know if there are fastbins
4651	bordering top, so we cannot tell for sure whether threshold
4652	has been reached unless fastbins are consolidated. But we
4653	don't want to consolidate on each free. As a compromise,
4654	consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
4655	is reached.
4656	*/
4657
4658	if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
4659	if (atomic_load_relaxed (&av->have_fastchunks))
4660	malloc_consolidate(av);
4661
4662	if (av == &main_arena) {
4663	#ifndef MORECORE_CANNOT_TRIM
4664	if ((unsigned long)(chunksize(av->top)) >=
4665	(unsigned long)(mp_.trim_threshold))
4666	systrim(mp_.top_pad, av);
4667	#endif
4668	} else {
4669	/ Always try heap_trim(), even if the top chunk is not*
4670	large, because the corresponding heap might go away. /*
4671	heap_info *heap = heap_for_ptr(top(av));
4672
4673	assert(heap->ar_ptr == av);
4674	heap_trim(heap, mp_.top_pad);
4675	}
4676	}
4677
4678	if (!have_lock)
4679	__libc_lock_unlock (av->mutex);
4680	}
4681	/*
4682	If the chunk was allocated via mmap, release via munmap().
4683	*/
4684
4685	else {
4686	munmap_chunk (p);
4687	}
4688	}
4689
4690	/*
4691	------------------------- malloc_consolidate -------------------------
4692
4693	malloc_consolidate is a specialized version of free() that tears
4694	down chunks held in fastbins. Free itself cannot be used for this
4695	purpose since, among other things, it might place chunks back onto
4696	fastbins. So, instead, we need to use a minor variant of the same
4697	code.
4698	*/
4699
4700	static void malloc_consolidate(mstate av)
4701	{
4702	mfastbinptr* fb; / current fastbin being consolidated /
4703	mfastbinptr* maxfb; / last fastbin (for loop control) /
4704	mchunkptr p; / current chunk being consolidated /
4705	mchunkptr nextp; / next chunk to consolidate /
4706	mchunkptr unsorted_bin; / bin header /
4707	mchunkptr first_unsorted; / chunk to link to /
4708
4709	/ These have same use as in free() /
4710	mchunkptr nextchunk;
4711	INTERNAL_SIZE_T size;
4712	INTERNAL_SIZE_T nextsize;
4713	INTERNAL_SIZE_T prevsize;
4714	int nextinuse;
4715
4716	atomic_store_relaxed (&av->have_fastchunks, false);
4717
4718	unsorted_bin = unsorted_chunks(av);
4719
4720	/*
4721	Remove each chunk from fast bin and consolidate it, placing it
4722	then in unsorted bin. Among other reasons for doing this,
4723	placing in unsorted bin avoids needing to calculate actual bins
4724	until malloc is sure that chunks aren't immediately going to be
4725	reused anyway.
4726	*/
4727
4728	maxfb = &fastbin (av, NFASTBINS - `1`);
4729	fb = &fastbin (av, `0`);
4730	do {
4731	p = atomic_exchange_acquire (fb, NULL);
4732	if (p != `0`) {
4733	do {
4734	{
4735	if (__glibc_unlikely (misaligned_chunk (p)))
4736	malloc_printerr ("malloc_consolidate(): "
4737	"unaligned fastbin chunk detected");
4738
4739	unsigned int idx = fastbin_index (chunksize (p));
4740	if ((&fastbin (av, idx)) != fb)
4741	malloc_printerr ("malloc_consolidate(): invalid chunk size");
4742	}
4743
4744	check_inuse_chunk(av, p);
4745	nextp = REVEAL_PTR (p->fd);
4746
4747	/ Slightly streamlined version of consolidation code in free() /
4748	size = chunksize (p);
4749	nextchunk = chunk_at_offset(p, size);
4750	nextsize = chunksize(nextchunk);
4751
4752	if (!prev_inuse(p)) {
4753	prevsize = prev_size (p);
4754	size += prevsize;
4755	p = chunk_at_offset(p, -((long) prevsize));
4756	if (__glibc_unlikely (chunksize(p) != prevsize))
4757	malloc_printerr ("corrupted size vs. prev_size in fastbins");
4758	unlink_chunk (av, p);
4759	}
4760
4761	if (nextchunk != av->top) {
4762	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4763
4764	if (!nextinuse) {
4765	size += nextsize;
4766	unlink_chunk (av, nextchunk);
4767	} else
4768	clear_inuse_bit_at_offset(nextchunk, `0`);
4769
4770	first_unsorted = unsorted_bin->fd;
4771	unsorted_bin->fd = p;
4772	first_unsorted->bk = p;
4773
4774	if (!in_smallbin_range (size)) {
4775	p->fd_nextsize = NULL;
4776	p->bk_nextsize = NULL;
4777	}
4778
4779	set_head(p, size \| PREV_INUSE);
4780	p->bk = unsorted_bin;
4781	p->fd = first_unsorted;
4782	set_foot(p, size);
4783	}
4784
4785	else {
4786	size += nextsize;
4787	set_head(p, size \| PREV_INUSE);
4788	av->top = p;
4789	}
4790
4791	} while ( (p = nextp) != `0`);
4792
4793	}
4794	} while (fb++ != maxfb);
4795	}
4796
4797	/*
4798	------------------------------ realloc ------------------------------
4799	*/
4800
4801	static void *
4802	_int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
4803	INTERNAL_SIZE_T nb)
4804	{
4805	mchunkptr newp; / chunk to return /
4806	INTERNAL_SIZE_T newsize; / its size /
4807	void* newmem; / corresponding user mem /
4808
4809	mchunkptr next; / next contiguous chunk after oldp /
4810
4811	mchunkptr remainder; / extra space at end of newp /
4812	unsigned long remainder_size; / its size /
4813
4814	/ oldmem size /
4815	if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, `0`)
4816	\|\| __builtin_expect (oldsize >= av->system_mem, `0`)
4817	\|\| __builtin_expect (oldsize != chunksize (oldp), `0`))
4818	malloc_printerr ("realloc(): invalid old size");
4819
4820	check_inuse_chunk (av, oldp);
4821
4822	/ All callers already filter out mmap'ed chunks. /
4823	assert (!chunk_is_mmapped (oldp));
4824
4825	next = chunk_at_offset (oldp, oldsize);
4826	INTERNAL_SIZE_T nextsize = chunksize (next);
4827	if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, `0`)
4828	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4829	malloc_printerr ("realloc(): invalid next size");
4830
4831	if ((unsigned long) (oldsize) >= (unsigned long) (nb))
4832	{
4833	/ already big enough; split below /
4834	newp = oldp;
4835	newsize = oldsize;
4836	}
4837
4838	else
4839	{
4840	/ Try to expand forward into top /
4841	if (next == av->top &&
4842	(unsigned long) (newsize = oldsize + nextsize) >=
4843	(unsigned long) (nb + MINSIZE))
4844	{
4845	set_head_size (oldp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4846	av->top = chunk_at_offset (oldp, nb);
4847	set_head (av->top, (newsize - nb) \| PREV_INUSE);
4848	check_inuse_chunk (av, oldp);
4849	return tag_new_usable (chunk2mem (oldp));
4850	}
4851
4852	/ Try to expand forward into next chunk; split off remainder below /
4853	else if (next != av->top &&
4854	!inuse (next) &&
4855	(unsigned long) (newsize = oldsize + nextsize) >=
4856	(unsigned long) (nb))
4857	{
4858	newp = oldp;
4859	unlink_chunk (av, next);
4860	}
4861
4862	/ allocate, copy, free /
4863	else
4864	{
4865	newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK);
4866	if (newmem == `0`)
4867	return `0`; / propagate failure /
4868
4869	newp = mem2chunk (newmem);
4870	newsize = chunksize (newp);
4871
4872	/*
4873	Avoid copy if newp is next chunk after oldp.
4874	*/
4875	if (newp == next)
4876	{
4877	newsize += oldsize;
4878	newp = oldp;
4879	}
4880	else
4881	{
4882	void *oldmem = chunk2mem (oldp);
4883	size_t sz = memsize (oldp);
4884	(void) tag_region (oldmem, sz);
4885	newmem = tag_new_usable (newmem);
4886	memcpy (newmem, oldmem, sz);
4887	_int_free (av, oldp, `1`);
4888	check_inuse_chunk (av, newp);
4889	return newmem;
4890	}
4891	}
4892	}
4893
4894	/ If possible, free extra space in old or extended chunk /
4895
4896	assert ((unsigned long) (newsize) >= (unsigned long) (nb));
4897
4898	remainder_size = newsize - nb;
4899
4900	if (remainder_size < MINSIZE) / not enough extra to split off /
4901	{
4902	set_head_size (newp, newsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4903	set_inuse_bit_at_offset (newp, newsize);
4904	}
4905	else / split remainder /
4906	{
4907	remainder = chunk_at_offset (newp, nb);
4908	/ Clear any user-space tags before writing the header. /
4909	remainder = tag_region (remainder, remainder_size);
4910	set_head_size (newp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4911	set_head (remainder, remainder_size \| PREV_INUSE \|
4912	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4913	/ Mark remainder as inuse so free() won't complain /
4914	set_inuse_bit_at_offset (remainder, remainder_size);
4915	_int_free (av, remainder, `1`);
4916	}
4917
4918	check_inuse_chunk (av, newp);
4919	return tag_new_usable (chunk2mem (newp));
4920	}
4921
4922	/*
4923	------------------------------ memalign ------------------------------
4924	*/
4925
4926	static void *
4927	_int_memalign (mstate av, size_t alignment, size_t bytes)
4928	{
4929	INTERNAL_SIZE_T nb; / padded request size /
4930	char m; /* memory returned by malloc call /
4931	mchunkptr p; / corresponding chunk /
4932	char brk; /* alignment point within p /
4933	mchunkptr newp; / chunk to return /
4934	INTERNAL_SIZE_T newsize; / its size /
4935	INTERNAL_SIZE_T leadsize; / leading space before alignment point /
4936	mchunkptr remainder; / spare room at end to split off /
4937	unsigned long remainder_size; / its size /
4938	INTERNAL_SIZE_T size;
4939
4940
4941
4942	nb = checked_request2size (bytes);
4943	if (nb == `0`)
4944	{
4945	__set_errno (ENOMEM);
4946	return NULL;
4947	}
4948
4949	/*
4950	Strategy: find a spot within that chunk that meets the alignment
4951	request, and then possibly free the leading and trailing space.
4952	*/
4953
4954	/ Call malloc with worst case padding to hit alignment. /
4955
4956	m = (char *) (_int_malloc (av, nb + alignment + MINSIZE));
4957
4958	if (m == `0`)
4959	return `0`; / propagate failure /
4960
4961	p = mem2chunk (m);
4962
4963	if ((((unsigned long) (m)) % alignment) != `0`) / misaligned /
4964
4965	{ /*
4966	Find an aligned spot inside chunk. Since we need to give back
4967	leading space in a chunk of at least MINSIZE, if the first
4968	calculation places us at a spot with less than MINSIZE leader,
4969	we can move to the next aligned spot -- we've allocated enough
4970	total room so that this is always possible.
4971	*/
4972	brk = (char ) mem2chunk (((unsigned* long) (m + alignment - `1`)) &
4973	- ((signed long) alignment));
4974	if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
4975	brk += alignment;
4976
4977	newp = (mchunkptr) brk;
4978	leadsize = brk - (char *) (p);
4979	newsize = chunksize (p) - leadsize;
4980
4981	/ For mmapped chunks, just adjust offset /
4982	if (chunk_is_mmapped (p))
4983	{
4984	set_prev_size (newp, prev_size (p) + leadsize);
4985	set_head (newp, newsize \| IS_MMAPPED);
4986	return chunk2mem (newp);
4987	}
4988
4989	/ Otherwise, give back leader, use the rest /
4990	set_head (newp, newsize \| PREV_INUSE \|
4991	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4992	set_inuse_bit_at_offset (newp, newsize);
4993	set_head_size (p, leadsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4994	_int_free (av, p, `1`);
4995	p = newp;
4996
4997	assert (newsize >= nb &&
4998	(((unsigned long) (chunk2mem (p))) % alignment) == `0`);
4999	}
5000
5001	/ Also give back spare room at the end /
5002	if (!chunk_is_mmapped (p))
5003	{
5004	size = chunksize (p);
5005	if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
5006	{
5007	remainder_size = size - nb;
5008	remainder = chunk_at_offset (p, nb);
5009	set_head (remainder, remainder_size \| PREV_INUSE \|
5010	(av != &main_arena ? NON_MAIN_ARENA : `0`));
5011	set_head_size (p, nb);
5012	_int_free (av, remainder, `1`);
5013	}
5014	}
5015
5016	check_inuse_chunk (av, p);
5017	return chunk2mem (p);
5018	}
5019
5020
5021	/*
5022	------------------------------ malloc_trim ------------------------------
5023	*/
5024
5025	static int
5026	mtrim (mstate av, size_t pad)
5027	{
5028	/ Ensure all blocks are consolidated. /
5029	malloc_consolidate (av);
5030
5031	const size_t ps = GLRO (dl_pagesize);
5032	int psindex = bin_index (ps);
5033	const size_t psm1 = ps - `1`;
5034
5035	int result = `0`;
5036	for (int i = `1`; i < NBINS; ++i)
5037	if (i == `1` \|\| i >= psindex)
5038	{
5039	mbinptr bin = bin_at (av, i);
5040
5041	for (mchunkptr p = last (bin); p != bin; p = p->bk)
5042	{
5043	INTERNAL_SIZE_T size = chunksize (p);
5044
5045	if (size > psm1 + sizeof (struct malloc_chunk))
5046	{
5047	/ See whether the chunk contains at least one unused page. /
5048	char paligned_mem = (char* *) (((uintptr_t) p
5049	+ sizeof (struct malloc_chunk)
5050	+ psm1) & ~psm1);
5051
5052	assert ((char ) chunk2mem (p) + `2` CHUNK_HDR_SZ
5053	<= paligned_mem);
5054	assert ((char *) p + size > paligned_mem);
5055
5056	/ This is the size we could potentially free. /
5057	size -= paligned_mem - (char *) p;
5058
5059	if (size > psm1)
5060	{
5061	#if MALLOC_DEBUG
5062	/ When debugging we simulate destroying the memory*
5063	content. /*
5064	memset (paligned_mem, `0x89`, size & ~psm1);
5065	#endif
5066	__madvise (paligned_mem, size & ~psm1, MADV_DONTNEED);
5067
5068	result = `1`;
5069	}
5070	}
5071	}
5072	}
5073
5074	#ifndef MORECORE_CANNOT_TRIM
5075	return result \| (av == &main_arena ? systrim (pad, av) : `0`);
5076
5077	#else
5078	return result;
5079	#endif
5080	}
5081
5082
5083	int
5084	__malloc_trim (size_t s)
5085	{
5086	int result = `0`;
5087
5088	if (!__malloc_initialized)
5089	ptmalloc_init ();
5090
5091	mstate ar_ptr = &main_arena;
5092	do
5093	{
5094	__libc_lock_lock (ar_ptr->mutex);
5095	result \|= mtrim (ar_ptr, s);
5096	__libc_lock_unlock (ar_ptr->mutex);
5097
5098	ar_ptr = ar_ptr->next;
5099	}
5100	while (ar_ptr != &main_arena);
5101
5102	return result;
5103	}
5104
5105
5106	/*
5107	------------------------- malloc_usable_size -------------------------
5108	*/
5109
5110	static size_t
5111	musable (void *mem)
5112	{
5113	mchunkptr p = mem2chunk (mem);
5114
5115	if (chunk_is_mmapped (p))
5116	return chunksize (p) - CHUNK_HDR_SZ;
5117	else if (inuse (p))
5118	return memsize (p);
5119
5120	return `0`;
5121	}
5122
5123	#if IS_IN (libc)
5124	size_t
5125	__malloc_usable_size (void *m)
5126	{
5127	if (m == NULL)
5128	return `0`;
5129	return musable (m);
5130	}
5131	#endif
5132
5133	/*
5134	------------------------------ mallinfo ------------------------------
5135	Accumulate malloc statistics for arena AV into M.
5136	*/
5137	static void
5138	int_mallinfo (mstate av, struct mallinfo2 *m)
5139	{
5140	size_t i;
5141	mbinptr b;
5142	mchunkptr p;
5143	INTERNAL_SIZE_T avail;
5144	INTERNAL_SIZE_T fastavail;
5145	int nblocks;
5146	int nfastblocks;
5147
5148	check_malloc_state (av);
5149
5150	/ Account for top /
5151	avail = chunksize (av->top);
5152	nblocks = `1`; / top always exists /
5153
5154	/ traverse fastbins /
5155	nfastblocks = `0`;
5156	fastavail = `0`;
5157
5158	for (i = `0`; i < NFASTBINS; ++i)
5159	{
5160	for (p = fastbin (av, i);
5161	p != `0`;
5162	p = REVEAL_PTR (p->fd))
5163	{
5164	if (__glibc_unlikely (misaligned_chunk (p)))
5165	malloc_printerr ("int_mallinfo(): "
5166	"unaligned fastbin chunk detected");
5167	++nfastblocks;
5168	fastavail += chunksize (p);
5169	}
5170	}
5171
5172	avail += fastavail;
5173
5174	/ traverse regular bins /
5175	for (i = `1`; i < NBINS; ++i)
5176	{
5177	b = bin_at (av, i);
5178	for (p = last (b); p != b; p = p->bk)
5179	{
5180	++nblocks;
5181	avail += chunksize (p);
5182	}
5183	}
5184
5185	m->smblks += nfastblocks;
5186	m->ordblks += nblocks;
5187	m->fordblks += avail;
5188	m->uordblks += av->system_mem - avail;
5189	m->arena += av->system_mem;
5190	m->fsmblks += fastavail;
5191	if (av == &main_arena)
5192	{
5193	m->hblks = mp_.n_mmaps;
5194	m->hblkhd = mp_.mmapped_mem;
5195	m->usmblks = `0`;
5196	m->keepcost = chunksize (av->top);
5197	}
5198	}
5199
5200
5201	struct mallinfo2
5202	__libc_mallinfo2 (void)
5203	{
5204	struct mallinfo2 m;
5205	mstate ar_ptr;
5206
5207	if (!__malloc_initialized)
5208	ptmalloc_init ();
5209
5210	memset (&m, `0`, sizeof (m));
5211	ar_ptr = &main_arena;
5212	do
5213	{
5214	__libc_lock_lock (ar_ptr->mutex);
5215	int_mallinfo (ar_ptr, &m);
5216	__libc_lock_unlock (ar_ptr->mutex);
5217
5218	ar_ptr = ar_ptr->next;
5219	}
5220	while (ar_ptr != &main_arena);
5221
5222	return m;
5223	}
5224	libc_hidden_def (__libc_mallinfo2)
5225
5226	struct mallinfo
5227	__libc_mallinfo (void)
5228	{
5229	struct mallinfo m;
5230	struct mallinfo2 m2 = __libc_mallinfo2 ();
5231
5232	m.arena = m2.arena;
5233	m.ordblks = m2.ordblks;
5234	m.smblks = m2.smblks;
5235	m.hblks = m2.hblks;
5236	m.hblkhd = m2.hblkhd;
5237	m.usmblks = m2.usmblks;
5238	m.fsmblks = m2.fsmblks;
5239	m.uordblks = m2.uordblks;
5240	m.fordblks = m2.fordblks;
5241	m.keepcost = m2.keepcost;
5242
5243	return m;
5244	}
5245
5246
5247	/*
5248	------------------------------ malloc_stats ------------------------------
5249	*/
5250
5251	void
5252	__malloc_stats (void)
5253	{
5254	int i;
5255	mstate ar_ptr;
5256	unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
5257
5258	if (!__malloc_initialized)
5259	ptmalloc_init ();
5260	_IO_flockfile (stderr);
5261	int old_flags2 = stderr->_flags2;
5262	stderr->_flags2 \|= _IO_FLAGS2_NOTCANCEL;
5263	for (i = `0`, ar_ptr = &main_arena;; i++)
5264	{
5265	struct mallinfo2 mi;
5266
5267	memset (&mi, `0`, sizeof (mi));
5268	__libc_lock_lock (ar_ptr->mutex);
5269	int_mallinfo (ar_ptr, &mi);
5270	fprintf (stderr, "Arena %d:\n", i);
5271	fprintf (stderr, "system bytes = %10u\n", (unsigned int) mi.arena);
5272	fprintf (stderr, "in use bytes = %10u\n", (unsigned int) mi.uordblks);
5273	#if MALLOC_DEBUG > 1
5274	if (i > `0`)
5275	dump_heap (heap_for_ptr (top (ar_ptr)));
5276	#endif
5277	system_b += mi.arena;
5278	in_use_b += mi.uordblks;
5279	__libc_lock_unlock (ar_ptr->mutex);
5280	ar_ptr = ar_ptr->next;
5281	if (ar_ptr == &main_arena)
5282	break;
5283	}
5284	fprintf (stderr, "Total (incl. mmap):\n");
5285	fprintf (stderr, "system bytes = %10u\n", system_b);
5286	fprintf (stderr, "in use bytes = %10u\n", in_use_b);
5287	fprintf (stderr, "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
5288	fprintf (stderr, "max mmap bytes = %10lu\n",
5289	(unsigned long) mp_.max_mmapped_mem);
5290	stderr->_flags2 = old_flags2;
5291	_IO_funlockfile (stderr);
5292	}
5293
5294
5295	/*
5296	------------------------------ mallopt ------------------------------
5297	*/
5298	static __always_inline int
5299	do_set_trim_threshold (size_t value)
5300	{
5301	LIBC_PROBE (memory_mallopt_trim_threshold, `3`, value, mp_.trim_threshold,
5302	mp_.no_dyn_threshold);
5303	mp_.trim_threshold = value;
5304	mp_.no_dyn_threshold = `1`;
5305	return `1`;
5306	}
5307
5308	static __always_inline int
5309	do_set_top_pad (size_t value)
5310	{
5311	LIBC_PROBE (memory_mallopt_top_pad, `3`, value, mp_.top_pad,
5312	mp_.no_dyn_threshold);
5313	mp_.top_pad = value;
5314	mp_.no_dyn_threshold = `1`;
5315	return `1`;
5316	}
5317
5318	static __always_inline int
5319	do_set_mmap_threshold (size_t value)
5320	{
5321	LIBC_PROBE (memory_mallopt_mmap_threshold, `3`, value, mp_.mmap_threshold,
5322	mp_.no_dyn_threshold);
5323	mp_.mmap_threshold = value;
5324	mp_.no_dyn_threshold = `1`;
5325	return `1`;
5326	}
5327
5328	static __always_inline int
5329	do_set_mmaps_max (int32_t value)
5330	{
5331	LIBC_PROBE (memory_mallopt_mmap_max, `3`, value, mp_.n_mmaps_max,
5332	mp_.no_dyn_threshold);
5333	mp_.n_mmaps_max = value;
5334	mp_.no_dyn_threshold = `1`;
5335	return `1`;
5336	}
5337
5338	static __always_inline int
5339	do_set_mallopt_check (int32_t value)
5340	{
5341	return `1`;
5342	}
5343
5344	static __always_inline int
5345	do_set_perturb_byte (int32_t value)
5346	{
5347	LIBC_PROBE (memory_mallopt_perturb, `2`, value, perturb_byte);
5348	perturb_byte = value;
5349	return `1`;
5350	}
5351
5352	static __always_inline int
5353	do_set_arena_test (size_t value)
5354	{
5355	LIBC_PROBE (memory_mallopt_arena_test, `2`, value, mp_.arena_test);
5356	mp_.arena_test = value;
5357	return `1`;
5358	}
5359
5360	static __always_inline int
5361	do_set_arena_max (size_t value)
5362	{
5363	LIBC_PROBE (memory_mallopt_arena_max, `2`, value, mp_.arena_max);
5364	mp_.arena_max = value;
5365	return `1`;
5366	}
5367
5368	#if USE_TCACHE
5369	static __always_inline int
5370	do_set_tcache_max (size_t value)
5371	{
5372	if (value <= MAX_TCACHE_SIZE)
5373	{
5374	LIBC_PROBE (memory_tunable_tcache_max_bytes, `2`, value, mp_.tcache_max_bytes);
5375	mp_.tcache_max_bytes = value;
5376	mp_.tcache_bins = csize2tidx (request2size(value)) + `1`;
5377	return `1`;
5378	}
5379	return `0`;
5380	}
5381
5382	static __always_inline int
5383	do_set_tcache_count (size_t value)
5384	{
5385	if (value <= MAX_TCACHE_COUNT)
5386	{
5387	LIBC_PROBE (memory_tunable_tcache_count, `2`, value, mp_.tcache_count);
5388	mp_.tcache_count = value;
5389	return `1`;
5390	}
5391	return `0`;
5392	}
5393
5394	static __always_inline int
5395	do_set_tcache_unsorted_limit (size_t value)
5396	{
5397	LIBC_PROBE (memory_tunable_tcache_unsorted_limit, `2`, value, mp_.tcache_unsorted_limit);
5398	mp_.tcache_unsorted_limit = value;
5399	return `1`;
5400	}
5401	#endif
5402
5403	static __always_inline int
5404	do_set_mxfast (size_t value)
5405	{
5406	if (value <= MAX_FAST_SIZE)
5407	{
5408	LIBC_PROBE (memory_mallopt_mxfast, `2`, value, get_max_fast ());
5409	set_max_fast (value);
5410	return `1`;
5411	}
5412	return `0`;
5413	}
5414
5415	#if HAVE_TUNABLES
5416	static __always_inline int
5417	do_set_hugetlb (size_t value)
5418	{
5419	if (value == `1`)
5420	{
5421	enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
5422	/*
5423	Only enable THP madvise usage if system does support it and
5424	has 'madvise' mode. Otherwise the madvise() call is wasteful.
5425	*/
5426	if (thp_mode == malloc_thp_mode_madvise)
5427	mp_.thp_pagesize = __malloc_default_thp_pagesize ();
5428	}
5429	else if (value >= `2`)
5430	__malloc_hugepage_config (value == `2` ? `0` : value, &mp_.hp_pagesize,
5431	&mp_.hp_flags);
5432	return `0`;
5433	}
5434	#endif
5435
5436	int
5437	__libc_mallopt (int param_number, int value)
5438	{
5439	mstate av = &main_arena;
5440	int res = `1`;
5441
5442	if (!__malloc_initialized)
5443	ptmalloc_init ();
5444	__libc_lock_lock (av->mutex);
5445
5446	LIBC_PROBE (memory_mallopt, `2`, param_number, value);
5447
5448	/ We must consolidate main arena before changing max_fast*
5449	(see definition of set_max_fast). /*
5450	malloc_consolidate (av);
5451
5452	/ Many of these helper functions take a size_t. We do not worry*
5453	about overflow here, because negative int values will wrap to
5454	very large size_t values and the helpers have sufficient range
5455	checking for such conversions. Many of these helpers are also
5456	used by the tunables macros in arena.c. /*
5457
5458	switch (param_number)
5459	{
5460	case M_MXFAST:
5461	res = do_set_mxfast (value);
5462	break;
5463
5464	case M_TRIM_THRESHOLD:
5465	res = do_set_trim_threshold (value);
5466	break;
5467
5468	case M_TOP_PAD:
5469	res = do_set_top_pad (value);
5470	break;
5471
5472	case M_MMAP_THRESHOLD:
5473	res = do_set_mmap_threshold (value);
5474	break;
5475
5476	case M_MMAP_MAX:
5477	res = do_set_mmaps_max (value);
5478	break;
5479
5480	case M_CHECK_ACTION:
5481	res = do_set_mallopt_check (value);
5482	break;
5483
5484	case M_PERTURB:
5485	res = do_set_perturb_byte (value);
5486	break;
5487
5488	case M_ARENA_TEST:
5489	if (value > `0`)
5490	res = do_set_arena_test (value);
5491	break;
5492
5493	case M_ARENA_MAX:
5494	if (value > `0`)
5495	res = do_set_arena_max (value);
5496	break;
5497	}
5498	__libc_lock_unlock (av->mutex);
5499	return res;
5500	}
5501	libc_hidden_def (__libc_mallopt)
5502
5503
5504	/*
5505	-------------------- Alternative MORECORE functions --------------------
5506	*/
5507
5508
5509	/*
5510	General Requirements for MORECORE.
5511
5512	The MORECORE function must have the following properties:
5513
5514	If MORECORE_CONTIGUOUS is false:
5515
5516	* MORECORE must allocate in multiples of pagesize. It will
5517	only be called with arguments that are multiples of pagesize.
5518
5519	* MORECORE(0) must return an address that is at least
5520	MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
5521
5522	else (i.e. If MORECORE_CONTIGUOUS is true):
5523
5524	* Consecutive calls to MORECORE with positive arguments
5525	return increasing addresses, indicating that space has been
5526	contiguously extended.
5527
5528	* MORECORE need not allocate in multiples of pagesize.
5529	Calls to MORECORE need not have args of multiples of pagesize.
5530
5531	* MORECORE need not page-align.
5532
5533	In either case:
5534
5535	* MORECORE may allocate more memory than requested. (Or even less,
5536	but this will generally result in a malloc failure.)
5537
5538	* MORECORE must not allocate memory when given argument zero, but
5539	instead return one past the end address of memory from previous
5540	nonzero call. This malloc does NOT call MORECORE(0)
5541	until at least one call with positive arguments is made, so
5542	the initial value returned is not important.
5543
5544	* Even though consecutive calls to MORECORE need not return contiguous
5545	addresses, it must be OK for malloc'ed chunks to span multiple
5546	regions in those cases where they do happen to be contiguous.
5547
5548	* MORECORE need not handle negative arguments -- it may instead
5549	just return MORECORE_FAILURE when given negative arguments.
5550	Negative arguments are always multiples of pagesize. MORECORE
5551	must not misinterpret negative args as large positive unsigned
5552	args. You can suppress all such calls from even occurring by defining
5553	MORECORE_CANNOT_TRIM,
5554
5555	There is some variation across systems about the type of the
5556	argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
5557	actually be size_t, because sbrk supports negative args, so it is
5558	normally the signed type of the same width as size_t (sometimes
5559	declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
5560	matter though. Internally, we use "long" as arguments, which should
5561	work across all reasonable possibilities.
5562
5563	Additionally, if MORECORE ever returns failure for a positive
5564	request, then mmap is used as a noncontiguous system allocator. This
5565	is a useful backup strategy for systems with holes in address spaces
5566	-- in this case sbrk cannot contiguously expand the heap, but mmap
5567	may be able to map noncontiguous space.
5568
5569	If you'd like mmap to ALWAYS be used, you can define MORECORE to be
5570	a function that always returns MORECORE_FAILURE.
5571
5572	If you are using this malloc with something other than sbrk (or its
5573	emulation) to supply memory regions, you probably want to set
5574	MORECORE_CONTIGUOUS as false. As an example, here is a custom
5575	allocator kindly contributed for pre-OSX macOS. It uses virtually
5576	but not necessarily physically contiguous non-paged memory (locked
5577	in, present and won't get swapped out). You can use it by
5578	uncommenting this section, adding some #includes, and setting up the
5579	appropriate defines above:
5580
5581	*#define MORECORE osMoreCore
5582	*#define MORECORE_CONTIGUOUS 0
5583
5584	There is also a shutdown routine that should somehow be called for
5585	cleanup upon program exit.
5586
5587	*#define MAX_POOL_ENTRIES 100
5588	#define MINIMUM_MORECORE_SIZE (64 1024)
5589	static int next_os_pool;
5590	void our_os_pools[MAX_POOL_ENTRIES];*
5591
5592	void osMoreCore(int size)*
5593	{
5594	void ptr = 0;*
5595	static void sbrk_top = 0;*
5596
5597	if (size > 0)
5598	{
5599	if (size < MINIMUM_MORECORE_SIZE)
5600	size = MINIMUM_MORECORE_SIZE;
5601	if (CurrentExecutionLevel() == kTaskLevel)
5602	ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
5603	if (ptr == 0)
5604	{
5605	return (void ) MORECORE_FAILURE;*
5606	}
5607	// save ptrs so they can be freed during cleanup
5608	our_os_pools[next_os_pool] = ptr;
5609	next_os_pool++;
5610	ptr = (void ) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);*
5611	sbrk_top = (char ) ptr + size;*
5612	return ptr;
5613	}
5614	else if (size < 0)
5615	{
5616	// we don't currently support shrink behavior
5617	return (void ) MORECORE_FAILURE;*
5618	}
5619	else
5620	{
5621	return sbrk_top;
5622	}
5623	}
5624
5625	// cleanup any allocated memory pools
5626	// called as last thing before shutting down driver
5627
5628	void osCleanupMem(void)
5629	{
5630	void ptr;
5631
5632	for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
5633	if (ptr)*
5634	{
5635	PoolDeallocate(ptr);*
5636	* ptr = 0;
5637	}
5638	}
5639
5640	*/
5641
5642
5643	/ Helper code. /
5644
5645	extern char **__libc_argv attribute_hidden;
5646
5647	static void
5648	malloc_printerr (const char *str)
5649	{
5650	#if IS_IN (libc)
5651	__libc_message ("%s\n", str);
5652	#else
5653	__libc_fatal (str);
5654	#endif
5655	__builtin_unreachable ();
5656	}
5657
5658	#if IS_IN (libc)
5659	/ We need a wrapper function for one of the additions of POSIX. /
5660	int
5661	__posix_memalign (void **memptr, size_t alignment, size_t size)
5662	{
5663	void *mem;
5664
5665	if (!__malloc_initialized)
5666	ptmalloc_init ();
5667
5668	/ Test whether the SIZE argument is valid. It must be a power of*
5669	two multiple of sizeof (void ). /
5670	if (alignment % sizeof (void *) != `0`
5671	\|\| !powerof2 (alignment / sizeof (void *))
5672	\|\| alignment == `0`)
5673	return EINVAL;
5674
5675
5676	void *address = RETURN_ADDRESS (`0`);
5677	mem = _mid_memalign (alignment, size, address);
5678
5679	if (mem != NULL)
5680	{
5681	*memptr = mem;
5682	return `0`;
5683	}
5684
5685	return ENOMEM;
5686	}
5687	weak_alias (__posix_memalign, posix_memalign)
5688	#endif
5689
5690
5691	int
5692	__malloc_info (int options, FILE *fp)
5693	{
5694	/ For now, at least. /
5695	if (options != `0`)
5696	return EINVAL;
5697
5698	int n = `0`;
5699	size_t total_nblocks = `0`;
5700	size_t total_nfastblocks = `0`;
5701	size_t total_avail = `0`;
5702	size_t total_fastavail = `0`;
5703	size_t total_system = `0`;
5704	size_t total_max_system = `0`;
5705	size_t total_aspace = `0`;
5706	size_t total_aspace_mprotect = `0`;
5707
5708
5709
5710	if (!__malloc_initialized)
5711	ptmalloc_init ();
5712
5713	fputs ("<malloc version=\"1\">\n", fp);
5714
5715	/ Iterate over all arenas currently in use. /
5716	mstate ar_ptr = &main_arena;
5717	do
5718	{
5719	fprintf (fp, "<heap nr=\"%d\">\n<sizes>\n", n++);
5720
5721	size_t nblocks = `0`;
5722	size_t nfastblocks = `0`;
5723	size_t avail = `0`;
5724	size_t fastavail = `0`;
5725	struct
5726	{
5727	size_t from;
5728	size_t to;
5729	size_t total;
5730	size_t count;
5731	} sizes[NFASTBINS + NBINS - `1`];
5732	#define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5733
5734	__libc_lock_lock (ar_ptr->mutex);
5735
5736	/ Account for top chunk. The top-most available chunk is*
5737	treated specially and is never in any bin. See "initial_top"
5738	comments. /*
5739	avail = chunksize (ar_ptr->top);
5740	nblocks = `1`; / Top always exists. /
5741
5742	for (size_t i = `0`; i < NFASTBINS; ++i)
5743	{
5744	mchunkptr p = fastbin (ar_ptr, i);
5745	if (p != NULL)
5746	{
5747	size_t nthissize = `0`;
5748	size_t thissize = chunksize (p);
5749
5750	while (p != NULL)
5751	{
5752	if (__glibc_unlikely (misaligned_chunk (p)))
5753	malloc_printerr ("__malloc_info(): "
5754	"unaligned fastbin chunk detected");
5755	++nthissize;
5756	p = REVEAL_PTR (p->fd);
5757	}
5758
5759	fastavail += nthissize * thissize;
5760	nfastblocks += nthissize;
5761	sizes[i].from = thissize - (MALLOC_ALIGNMENT - `1`);
5762	sizes[i].to = thissize;
5763	sizes[i].count = nthissize;
5764	}
5765	else
5766	sizes[i].from = sizes[i].to = sizes[i].count = `0`;
5767
5768	sizes[i].total = sizes[i].count * sizes[i].to;
5769	}
5770
5771
5772	mbinptr bin;
5773	struct malloc_chunk *r;
5774
5775	for (size_t i = `1`; i < NBINS; ++i)
5776	{
5777	bin = bin_at (ar_ptr, i);
5778	r = bin->fd;
5779	sizes[NFASTBINS - `1` + i].from = ~((size_t) `0`);
5780	sizes[NFASTBINS - `1` + i].to = sizes[NFASTBINS - `1` + i].total
5781	= sizes[NFASTBINS - `1` + i].count = `0`;
5782
5783	if (r != NULL)
5784	while (r != bin)
5785	{
5786	size_t r_size = chunksize_nomask (r);
5787	++sizes[NFASTBINS - `1` + i].count;
5788	sizes[NFASTBINS - `1` + i].total += r_size;
5789	sizes[NFASTBINS - `1` + i].from
5790	= MIN (sizes[NFASTBINS - `1` + i].from, r_size);
5791	sizes[NFASTBINS - `1` + i].to = MAX (sizes[NFASTBINS - `1` + i].to,
5792	r_size);
5793
5794	r = r->fd;
5795	}
5796
5797	if (sizes[NFASTBINS - `1` + i].count == `0`)
5798	sizes[NFASTBINS - `1` + i].from = `0`;
5799	nblocks += sizes[NFASTBINS - `1` + i].count;
5800	avail += sizes[NFASTBINS - `1` + i].total;
5801	}
5802
5803	size_t heap_size = `0`;
5804	size_t heap_mprotect_size = `0`;
5805	size_t heap_count = `0`;
5806	if (ar_ptr != &main_arena)
5807	{
5808	/ Iterate over the arena heaps from back to front. /
5809	heap_info *heap = heap_for_ptr (top (ar_ptr));
5810	do
5811	{
5812	heap_size += heap->size;
5813	heap_mprotect_size += heap->mprotect_size;
5814	heap = heap->prev;
5815	++heap_count;
5816	}
5817	while (heap != NULL);
5818	}
5819
5820	__libc_lock_unlock (ar_ptr->mutex);
5821
5822	total_nfastblocks += nfastblocks;
5823	total_fastavail += fastavail;
5824
5825	total_nblocks += nblocks;
5826	total_avail += avail;
5827
5828	for (size_t i = `0`; i < nsizes; ++i)
5829	if (sizes[i].count != `0` && i != NFASTBINS)
5830	fprintf (fp, "\
5831	<size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5832	sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
5833
5834	if (sizes[NFASTBINS].count != `0`)
5835	fprintf (fp, "\
5836	<unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5837	sizes[NFASTBINS].from, sizes[NFASTBINS].to,
5838	sizes[NFASTBINS].total, sizes[NFASTBINS].count);
5839
5840	total_system += ar_ptr->system_mem;
5841	total_max_system += ar_ptr->max_system_mem;
5842
5843	fprintf (fp,
5844	"</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5845	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5846	"<system type=\"current\" size=\"%zu\"/>\n"
5847	"<system type=\"max\" size=\"%zu\"/>\n",
5848	nfastblocks, fastavail, nblocks, avail,
5849	ar_ptr->system_mem, ar_ptr->max_system_mem);
5850
5851	if (ar_ptr != &main_arena)
5852	{
5853	fprintf (fp,
5854	"<aspace type=\"total\" size=\"%zu\"/>\n"
5855	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5856	"<aspace type=\"subheaps\" size=\"%zu\"/>\n",
5857	heap_size, heap_mprotect_size, heap_count);
5858	total_aspace += heap_size;
5859	total_aspace_mprotect += heap_mprotect_size;
5860	}
5861	else
5862	{
5863	fprintf (fp,
5864	"<aspace type=\"total\" size=\"%zu\"/>\n"
5865	"<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5866	ar_ptr->system_mem, ar_ptr->system_mem);
5867	total_aspace += ar_ptr->system_mem;
5868	total_aspace_mprotect += ar_ptr->system_mem;
5869	}
5870
5871	fputs ("</heap>\n", fp);
5872	ar_ptr = ar_ptr->next;
5873	}
5874	while (ar_ptr != &main_arena);
5875
5876	fprintf (fp,
5877	"<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5878	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5879	"<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
5880	"<system type=\"current\" size=\"%zu\"/>\n"
5881	"<system type=\"max\" size=\"%zu\"/>\n"
5882	"<aspace type=\"total\" size=\"%zu\"/>\n"
5883	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5884	"</malloc>\n",
5885	total_nfastblocks, total_fastavail, total_nblocks, total_avail,
5886	mp_.n_mmaps, mp_.mmapped_mem,
5887	total_system, total_max_system,
5888	total_aspace, total_aspace_mprotect);
5889
5890	return `0`;
5891	}
5892	#if IS_IN (libc)
5893	weak_alias (__malloc_info, malloc_info)
5894
5895	strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5896	strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5897	strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5898	strong_alias (__libc_memalign, __memalign)
5899	weak_alias (__libc_memalign, memalign)
5900	strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5901	strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5902	strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5903	strong_alias (__libc_mallinfo, __mallinfo)
5904	weak_alias (__libc_mallinfo, mallinfo)
5905	strong_alias (__libc_mallinfo2, __mallinfo2)
5906	weak_alias (__libc_mallinfo2, mallinfo2)
5907	strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5908
5909	weak_alias (__malloc_stats, malloc_stats)
5910	weak_alias (__malloc_usable_size, malloc_usable_size)
5911	weak_alias (__malloc_trim, malloc_trim)
5912	#endif
5913
5914	#if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
5915	compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
5916	#endif
5917
5918	/ ------------------------------------------------------------*
5919	History:
5920
5921	[see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
5922
5923	*/
5924	/*
5925	* Local variables:
5926	* c-basic-offset: 2
5927	* End:
5928	*/
5929

Browse the source code of glibc/malloc/malloc.c