malloc.c source code [glibc/malloc/malloc.c]

1	/ Malloc implementation for multiple threads without lock contention.*
2	Copyright (C) 1996-2022 Free Software Foundation, Inc.
3	Copyright The GNU Toolchain Authors.
4	This file is part of the GNU C Library.
5
6	The GNU C Library is free software; you can redistribute it and/or
7	modify it under the terms of the GNU Lesser General Public License as
8	published by the Free Software Foundation; either version 2.1 of the
9	License, or (at your option) any later version.
10
11	The GNU C Library is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	Lesser General Public License for more details.
15
16	You should have received a copy of the GNU Lesser General Public
17	License along with the GNU C Library; see the file COPYING.LIB. If
18	not, see <https://www.gnu.org/licenses/>. /*
19
20	/*
21	This is a version (aka ptmalloc2) of malloc/free/realloc written by
22	Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
23
24	There have been substantial changes made after the integration into
25	glibc in all parts of the code. Do not look for much commonality
26	with the ptmalloc2 version.
27
28	* Version ptmalloc2-20011215
29	based on:
30	VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
31
32	* Quickstart
33
34	In order to compile this implementation, a Makefile is provided with
35	the ptmalloc2 distribution, which has pre-defined targets for some
36	popular systems (e.g. "make posix" for Posix threads). All that is
37	typically required with regard to compiler flags is the selection of
38	the thread package via defining one out of USE_PTHREADS, USE_THR or
39	USE_SPROC. Check the thread-m.h file for what effects this has.
40	Many/most systems will additionally require USE_TSD_DATA_HACK to be
41	defined, so this is the default for "make posix".
42
43	* Why use this malloc?
44
45	This is not the fastest, most space-conserving, most portable, or
46	most tunable malloc ever written. However it is among the fastest
47	while also being among the most space-conserving, portable and tunable.
48	Consistent balance across these factors results in a good general-purpose
49	allocator for malloc-intensive programs.
50
51	The main properties of the algorithms are:
52	* For large (>= 512 bytes) requests, it is a pure best-fit allocator,
53	with ties normally decided via FIFO (i.e. least recently used).
54	* For small (<= 64 bytes by default) requests, it is a caching
55	allocator, that maintains pools of quickly recycled chunks.
56	* In between, and for combinations of large and small requests, it does
57	the best it can trying to meet both goals at once.
58	* For very large requests (>= 128KB by default), it relies on system
59	memory mapping facilities, if supported.
60
61	For a longer but slightly out of date high-level description, see
62	http://gee.cs.oswego.edu/dl/html/malloc.html
63
64	You may already by default be using a C library containing a malloc
65	that is based on some version of this malloc (for example in
66	linux). You might still want to use the one in this file in order to
67	customize settings or to avoid overheads associated with library
68	versions.
69
70	* Contents, described in more detail in "description of public routines" below.
71
72	Standard (ANSI/SVID/...) functions:
73	malloc(size_t n);
74	calloc(size_t n_elements, size_t element_size);
75	free(void p);*
76	realloc(void p, size_t n);*
77	memalign(size_t alignment, size_t n);
78	valloc(size_t n);
79	mallinfo()
80	mallopt(int parameter_number, int parameter_value)
81
82	Additional functions:
83	independent_calloc(size_t n_elements, size_t size, void chunks[]);*
84	independent_comalloc(size_t n_elements, size_t sizes[], void chunks[]);*
85	pvalloc(size_t n);
86	malloc_trim(size_t pad);
87	malloc_usable_size(void p);*
88	malloc_stats();
89
90	* Vital statistics:
91
92	Supported pointer representation: 4 or 8 bytes
93	Supported size_t representation: 4 or 8 bytes
94	Note that size_t is allowed to be 4 bytes even if pointers are 8.
95	You can adjust this by defining INTERNAL_SIZE_T
96
97	Alignment: 2 sizeof(size_t) (default)*
98	(i.e., 8 byte alignment with 4byte size_t). This suffices for
99	nearly all current machines and C compilers. However, you can
100	define MALLOC_ALIGNMENT to be wider than this if necessary.
101
102	Minimum overhead per allocated chunk: 4 or 8 bytes
103	Each malloced chunk has a hidden word of overhead holding size
104	and status information.
105
106	Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
107	8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
108
109	When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
110	ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
111	needed; 4 (8) for a trailing size field and 8 (16) bytes for
112	free list pointers. Thus, the minimum allocatable size is
113	16/24/32 bytes.
114
115	Even a request for zero bytes (i.e., malloc(0)) returns a
116	pointer to something of the minimum allocatable size.
117
118	The maximum overhead wastage (i.e., number of extra bytes
119	allocated than were requested in malloc) is less than or equal
120	to the minimum size, except for requests >= mmap_threshold that
121	are serviced via mmap(), where the worst case wastage is 2 *
122	sizeof(size_t) bytes plus the remainder from a system page (the
123	minimal mmap unit); typically 4096 or 8192 bytes.
124
125	Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
126	8-byte size_t: 2^64 minus about two pages
127
128	It is assumed that (possibly signed) size_t values suffice to
129	represent chunk sizes. `Possibly signed' is due to the fact
130	that `size_t' may be defined on a system as either a signed or
131	an unsigned type. The ISO C standard says that it must be
132	unsigned, but a few systems are known not to adhere to this.
133	Additionally, even when size_t is unsigned, sbrk (which is by
134	default used to obtain memory from system) accepts signed
135	arguments, and may not be able to handle size_t-wide arguments
136	with negative sign bit. Generally, values that would
137	appear as negative after accounting for overhead and alignment
138	are supported only via mmap(), which does not have this
139	limitation.
140
141	Requests for sizes outside the allowed range will perform an optional
142	failure action and then return null. (Requests may also
143	also fail because a system is out of memory.)
144
145	Thread-safety: thread-safe
146
147	Compliance: I believe it is compliant with the 1997 Single Unix Specification
148	Also SVID/XPG, ANSI C, and probably others as well.
149
150	* Synopsis of compile-time options:
151
152	People have reported using previous versions of this malloc on all
153	versions of Unix, sometimes by tweaking some of the defines
154	below. It has been tested most extensively on Solaris and Linux.
155	People also report using it in stand-alone embedded systems.
156
157	The implementation is in straight, hand-tuned ANSI C. It is not
158	at all modular. (Sorry!) It uses a lot of macros. To be at all
159	usable, this code should be compiled using an optimizing compiler
160	(for example gcc -O3) that can simplify expressions and control
161	paths. (FAQ: some macros import variables as arguments rather than
162	declare locals because people reported that some debuggers
163	otherwise get confused.)
164
165	OPTION DEFAULT VALUE
166
167	Compilation Environment options:
168
169	HAVE_MREMAP 0
170
171	Changing default word sizes:
172
173	INTERNAL_SIZE_T size_t
174
175	Configuration and functionality options:
176
177	USE_PUBLIC_MALLOC_WRAPPERS NOT defined
178	USE_MALLOC_LOCK NOT defined
179	MALLOC_DEBUG NOT defined
180	REALLOC_ZERO_BYTES_FREES 1
181	TRIM_FASTBINS 0
182
183	Options for customizing MORECORE:
184
185	MORECORE sbrk
186	MORECORE_FAILURE -1
187	MORECORE_CONTIGUOUS 1
188	MORECORE_CANNOT_TRIM NOT defined
189	MORECORE_CLEARS 1
190	MMAP_AS_MORECORE_SIZE (1024 1024)*
191
192	Tuning options that are also dynamically changeable via mallopt:
193
194	DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
195	DEFAULT_TRIM_THRESHOLD 128 1024*
196	DEFAULT_TOP_PAD 0
197	DEFAULT_MMAP_THRESHOLD 128 1024*
198	DEFAULT_MMAP_MAX 65536
199
200	There are several other #defined constants and macros that you
201	probably don't want to touch unless you are extending or adapting malloc. /*
202
203	/*
204	void is the pointer type that malloc should say it returns*
205	*/
206
207	#ifndef void
208	#define void void
209	#endif /void/
210
211	#include <stddef.h> /* for size_t */
212	#include <stdlib.h> /* for getenv(), abort() */
213	#include <unistd.h> /* for __libc_enable_secure */
214
215	#include <atomic.h>
216	#include <_itoa.h>
217	#include <bits/wordsize.h>
218	#include <sys/sysinfo.h>
219
220	#include <ldsodefs.h>
221
222	#include <unistd.h>
223	#include <stdio.h> /* needed for malloc_stats */
224	#include <errno.h>
225	#include <assert.h>
226
227	#include <shlib-compat.h>
228
229	/ For uintptr_t. /
230	#include <stdint.h>
231
232	/ For va_arg, va_start, va_end. /
233	#include <stdarg.h>
234
235	/ For MIN, MAX, powerof2. /
236	#include <sys/param.h>
237
238	/ For ALIGN_UP et. al. /
239	#include <libc-pointer-arith.h>
240
241	/ For DIAG_PUSH/POP_NEEDS_COMMENT et al. /
242	#include <libc-diag.h>
243
244	/ For memory tagging. /
245	#include <libc-mtag.h>
246
247	#include <malloc/malloc-internal.h>
248
249	/ For SINGLE_THREAD_P. /
250	#include <sysdep-cancel.h>
251
252	#include <libc-internal.h>
253
254	/ For tcache double-free check. /
255	#include <random-bits.h>
256	#include <sys/random.h>
257
258	/*
259	Debugging:
260
261	Because freed chunks may be overwritten with bookkeeping fields, this
262	malloc will often die when freed memory is overwritten by user
263	programs. This can be very effective (albeit in an annoying way)
264	in helping track down dangling pointers.
265
266	If you compile with -DMALLOC_DEBUG, a number of assertion checks are
267	enabled that will catch more memory errors. You probably won't be
268	able to make much sense of the actual assertion errors, but they
269	should help you locate incorrectly overwritten memory. The checking
270	is fairly extensive, and will slow down execution
271	noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
272	will attempt to check every non-mmapped allocated and free chunk in
273	the course of computing the summmaries. (By nature, mmapped regions
274	cannot be checked very much automatically.)
275
276	Setting MALLOC_DEBUG may also be helpful if you are trying to modify
277	this code. The assertions in the check routines spell out in more
278	detail the assumptions and invariants underlying the algorithms.
279
280	Setting MALLOC_DEBUG does NOT provide an automated mechanism for
281	checking that all accesses to malloced memory stay within their
282	bounds. However, there are several add-ons and adaptations of this
283	or other mallocs available that do this.
284	*/
285
286	#ifndef MALLOC_DEBUG
287	#define MALLOC_DEBUG 0
288	#endif
289
290	#if IS_IN (libc)
291	#ifndef NDEBUG
292	# define __assert_fail(assertion, file, line, function) \
293	__malloc_assert(assertion, file, line, function)
294
295	_Noreturn static void
296	__malloc_assert (const char assertion, const* char file, unsigned* int line,
297	const char *function)
298	{
299	__libc_message (do_abort, "\
300	Fatal glibc error: malloc assertion failure in %s: %s\n",
301	function, assertion);
302	__builtin_unreachable ();
303	}
304	#endif
305	#endif
306
307	#if USE_TCACHE
308	/ We want 64 entries. This is an arbitrary limit, which tunables can reduce. /
309	# define TCACHE_MAX_BINS 64
310	# define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1)
311
312	/ Only used to pre-fill the tunables. /
313	# define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
314
315	/ When "x" is from chunksize(). /
316	# define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
317	/ When "x" is a user-provided size. /
318	# define usize2tidx(x) csize2tidx (request2size (x))
319
320	/ With rounding and alignment, the bins are...*
321	idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit)
322	idx 1 bytes 25..40 or 13..20
323	idx 2 bytes 41..56 or 21..28
324	etc. /*
325
326	/ This is another arbitrary limit, which tunables can change. Each*
327	tcache bin will hold at most this number of chunks. /*
328	# define TCACHE_FILL_COUNT 7
329
330	/ Maximum chunks in tcache bins for tunables. This value must fit the range*
331	of tcache->counts[] entries, else they may overflow. /*
332	# define MAX_TCACHE_COUNT UINT16_MAX
333	#endif
334
335	/ Safe-Linking:*
336	Use randomness from ASLR (mmap_base) to protect single-linked lists
337	of Fast-Bins and TCache. That is, mask the "next" pointers of the
338	lists' chunks, and also perform allocation alignment checks on them.
339	This mechanism reduces the risk of pointer hijacking, as was done with
340	Safe-Unlinking in the double-linked lists of Small-Bins.
341	It assumes a minimum page size of 4096 bytes (12 bits). Systems with
342	larger pages provide less entropy, although the pointer mangling
343	still works. /*
344	#define PROTECT_PTR(pos, ptr) \
345	((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
346	#define REVEAL_PTR(ptr) PROTECT_PTR (&ptr, ptr)
347
348	/*
349	The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
350	when p is nonnull. If the macro is nonzero, the realloc call returns NULL;
351	otherwise, the call returns what malloc (0) would. In either case,
352	p is freed. Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
353	implements common historical practice.
354
355	ISO C17 says the realloc call has implementation-defined behavior,
356	and it might not even free p.
357	*/
358
359	#ifndef REALLOC_ZERO_BYTES_FREES
360	#define REALLOC_ZERO_BYTES_FREES 1
361	#endif
362
363	/*
364	TRIM_FASTBINS controls whether free() of a very small chunk can
365	immediately lead to trimming. Setting to true (1) can reduce memory
366	footprint, but will almost always slow down programs that use a lot
367	of small chunks.
368
369	Define this only if you are willing to give up some speed to more
370	aggressively reduce system-level memory footprint when releasing
371	memory in programs that use many small chunks. You can get
372	essentially the same effect by setting MXFAST to 0, but this can
373	lead to even greater slowdowns in programs using many small chunks.
374	TRIM_FASTBINS is an in-between compile-time option, that disables
375	only those chunks bordering topmost memory from being placed in
376	fastbins.
377	*/
378
379	#ifndef TRIM_FASTBINS
380	#define TRIM_FASTBINS 0
381	#endif
382
383	/ Definition for getting more memory from the OS. /
384	#include "morecore.c"
385
386	#define MORECORE (*__glibc_morecore)
387	#define MORECORE_FAILURE 0
388
389	/ Memory tagging. /
390
391	/ Some systems support the concept of tagging (sometimes known as*
392	coloring) memory locations on a fine grained basis. Each memory
393	location is given a color (normally allocated randomly) and
394	pointers are also colored. When the pointer is dereferenced, the
395	pointer's color is checked against the memory's color and if they
396	differ the access is faulted (sometimes lazily).
397
398	We use this in glibc by maintaining a single color for the malloc
399	data structures that are interleaved with the user data and then
400	assigning separate colors for each block allocation handed out. In
401	this way simple buffer overruns will be rapidly detected. When
402	memory is freed, the memory is recolored back to the glibc default
403	so that simple use-after-free errors can also be detected.
404
405	If memory is reallocated the buffer is recolored even if the
406	address remains the same. This has a performance impact, but
407	guarantees that the old pointer cannot mistakenly be reused (code
408	that compares old against new will see a mismatch and will then
409	need to behave as though realloc moved the data to a new location).
410
411	Internal API for memory tagging support.
412
413	The aim is to keep the code for memory tagging support as close to
414	the normal APIs in glibc as possible, so that if tagging is not
415	enabled in the library, or is disabled at runtime then standard
416	operations can continue to be used. Support macros are used to do
417	this:
418
419	void tag_new_zero_region (void ptr, size_t size)
420
421	Allocates a new tag, colors the memory with that tag, zeros the
422	memory and returns a pointer that is correctly colored for that
423	location. The non-tagging version will simply call memset with 0.
424
425	void tag_region (void ptr, size_t size)
426
427	Color the region of memory pointed to by PTR and size SIZE with
428	the color of PTR. Returns the original pointer.
429
430	void tag_new_usable (void ptr)
431
432	Allocate a new random color and use it to color the user region of
433	a chunk; this may include data from the subsequent chunk's header
434	if tagging is sufficiently fine grained. Returns PTR suitably
435	recolored for accessing the memory there.
436
437	void tag_at (void ptr)
438
439	Read the current color of the memory at the address pointed to by
440	PTR (ignoring it's current color) and return PTR recolored to that
441	color. PTR must be valid address in all other respects. When
442	tagging is not enabled, it simply returns the original pointer.
443	*/
444
445	#ifdef USE_MTAG
446	static bool mtag_enabled = false;
447	static int mtag_mmap_flags = `0`;
448	#else
449	# define mtag_enabled false
450	# define mtag_mmap_flags 0
451	#endif
452
453	static __always_inline void *
454	tag_region (void *ptr, size_t size)
455	{
456	if (__glibc_unlikely (mtag_enabled))
457	return __libc_mtag_tag_region (ptr, size);
458	return ptr;
459	}
460
461	static __always_inline void *
462	tag_new_zero_region (void *ptr, size_t size)
463	{
464	if (__glibc_unlikely (mtag_enabled))
465	return __libc_mtag_tag_zero_region (__libc_mtag_new_tag (ptr), size);
466	return memset (ptr, `0`, size);
467	}
468
469	/ Defined later. /
470	static void *
471	tag_new_usable (void *ptr);
472
473	static __always_inline void *
474	tag_at (void *ptr)
475	{
476	if (__glibc_unlikely (mtag_enabled))
477	return __libc_mtag_address_get_tag (ptr);
478	return ptr;
479	}
480
481	#include <string.h>
482
483	/*
484	MORECORE-related declarations. By default, rely on sbrk
485	*/
486
487
488	/*
489	MORECORE is the name of the routine to call to obtain more memory
490	from the system. See below for general guidance on writing
491	alternative MORECORE functions, as well as a version for WIN32 and a
492	sample version for pre-OSX macos.
493	*/
494
495	#ifndef MORECORE
496	#define MORECORE sbrk
497	#endif
498
499	/*
500	MORECORE_FAILURE is the value returned upon failure of MORECORE
501	as well as mmap. Since it cannot be an otherwise valid memory address,
502	and must reflect values of standard sys calls, you probably ought not
503	try to redefine it.
504	*/
505
506	#ifndef MORECORE_FAILURE
507	#define MORECORE_FAILURE (-1)
508	#endif
509
510	/*
511	If MORECORE_CONTIGUOUS is true, take advantage of fact that
512	consecutive calls to MORECORE with positive arguments always return
513	contiguous increasing addresses. This is true of unix sbrk. Even
514	if not defined, when regions happen to be contiguous, malloc will
515	permit allocations spanning regions obtained from different
516	calls. But defining this when applicable enables some stronger
517	consistency checks and space efficiencies.
518	*/
519
520	#ifndef MORECORE_CONTIGUOUS
521	#define MORECORE_CONTIGUOUS 1
522	#endif
523
524	/*
525	Define MORECORE_CANNOT_TRIM if your version of MORECORE
526	cannot release space back to the system when given negative
527	arguments. This is generally necessary only if you are using
528	a hand-crafted MORECORE function that cannot handle negative arguments.
529	*/
530
531	/ #define MORECORE_CANNOT_TRIM /
532
533	/ MORECORE_CLEARS (default 1)*
534	The degree to which the routine mapped to MORECORE zeroes out
535	memory: never (0), only for newly allocated space (1) or always
536	(2). The distinction between (1) and (2) is necessary because on
537	some systems, if the application first decrements and then
538	increments the break value, the contents of the reallocated space
539	are unspecified.
540	*/
541
542	#ifndef MORECORE_CLEARS
543	# define MORECORE_CLEARS 1
544	#endif
545
546
547	/*
548	MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
549	sbrk fails, and mmap is used as a backup. The value must be a
550	multiple of page size. This backup strategy generally applies only
551	when systems have "holes" in address space, so sbrk cannot perform
552	contiguous expansion, but there is still space available on system.
553	On systems for which this is known to be useful (i.e. most linux
554	kernels), this occurs only when programs allocate huge amounts of
555	memory. Between this, and the fact that mmap regions tend to be
556	limited, the size should be large, to avoid too many mmap calls and
557	thus avoid running out of kernel resources. /*
558
559	#ifndef MMAP_AS_MORECORE_SIZE
560	#define MMAP_AS_MORECORE_SIZE (1024 * 1024)
561	#endif
562
563	/*
564	Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
565	large blocks.
566	*/
567
568	#ifndef HAVE_MREMAP
569	#define HAVE_MREMAP 0
570	#endif
571
572	/*
573	This version of malloc supports the standard SVID/XPG mallinfo
574	routine that returns a struct containing usage properties and
575	statistics. It should work on any SVID/XPG compliant system that has
576	a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
577	install such a thing yourself, cut out the preliminary declarations
578	as described above and below and save them in a malloc.h file. But
579	there's no compelling reason to bother to do this.)
580
581	The main declaration needed is the mallinfo struct that is returned
582	(by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
583	bunch of fields that are not even meaningful in this version of
584	malloc. These fields are are instead filled by mallinfo() with
585	other numbers that might be of interest.
586	*/
587
588
589	/ ---------- description of public routines ------------ /
590
591	#if IS_IN (libc)
592	/*
593	malloc(size_t n)
594	Returns a pointer to a newly allocated chunk of at least n bytes, or null
595	if no space is available. Additionally, on failure, errno is
596	set to ENOMEM on ANSI C systems.
597
598	If n is zero, malloc returns a minimum-sized chunk. (The minimum
599	size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
600	systems.) On most systems, size_t is an unsigned type, so calls
601	with negative arguments are interpreted as requests for huge amounts
602	of space, which will often fail. The maximum supported value of n
603	differs across systems, but is in all cases less than the maximum
604	representable value of a size_t.
605	*/
606	void* __libc_malloc(size_t);
607	libc_hidden_proto (__libc_malloc)
608
609	/*
610	free(void p)*
611	Releases the chunk of memory pointed to by p, that had been previously
612	allocated using malloc or a related routine such as realloc.
613	It has no effect if p is null. It can have arbitrary (i.e., bad!)
614	effects if p has already been freed.
615
616	Unless disabled (using mallopt), freeing very large spaces will
617	when possible, automatically trigger operations that give
618	back unused memory to the system, thus reducing program footprint.
619	*/
620	void __libc_free(void*);
621	libc_hidden_proto (__libc_free)
622
623	/*
624	calloc(size_t n_elements, size_t element_size);
625	Returns a pointer to n_elements element_size bytes, with all locations*
626	set to zero.
627	*/
628	void* __libc_calloc(size_t, size_t);
629
630	/*
631	realloc(void p, size_t n)*
632	Returns a pointer to a chunk of size n that contains the same data
633	as does chunk p up to the minimum of (n, p's size) bytes, or null
634	if no space is available.
635
636	The returned pointer may or may not be the same as p. The algorithm
637	prefers extending p when possible, otherwise it employs the
638	equivalent of a malloc-copy-free sequence.
639
640	If p is null, realloc is equivalent to malloc.
641
642	If space is not available, realloc returns null, errno is set (if on
643	ANSI) and p is NOT freed.
644
645	if n is for fewer bytes than already held by p, the newly unused
646	space is lopped off and freed if possible. Unless the #define
647	REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
648	zero (re)allocates a minimum-sized chunk.
649
650	Large chunks that were internally obtained via mmap will always be
651	grown using malloc-copy-free sequences unless the system supports
652	MREMAP (currently only linux).
653
654	The old unix realloc convention of allowing the last-free'd chunk
655	to be used as an argument to realloc is not supported.
656	*/
657	void* __libc_realloc(void*, size_t);
658	libc_hidden_proto (__libc_realloc)
659
660	/*
661	memalign(size_t alignment, size_t n);
662	Returns a pointer to a newly allocated chunk of n bytes, aligned
663	in accord with the alignment argument.
664
665	The alignment argument should be a power of two. If the argument is
666	not a power of two, the nearest greater power is used.
667	8-byte alignment is guaranteed by normal malloc calls, so don't
668	bother calling memalign with an argument of 8 or less.
669
670	Overreliance on memalign is a sure way to fragment space.
671	*/
672	void* __libc_memalign(size_t, size_t);
673	libc_hidden_proto (__libc_memalign)
674
675	/*
676	valloc(size_t n);
677	Equivalent to memalign(pagesize, n), where pagesize is the page
678	size of the system. If the pagesize is unknown, 4096 is used.
679	*/
680	void* __libc_valloc(size_t);
681
682
683
684	/*
685	mallinfo()
686	Returns (by copy) a struct containing various summary statistics:
687
688	arena: current total non-mmapped bytes allocated from system
689	ordblks: the number of free chunks
690	smblks: the number of fastbin blocks (i.e., small chunks that
691	have been freed but not use resused or consolidated)
692	hblks: current number of mmapped regions
693	hblkhd: total bytes held in mmapped regions
694	usmblks: always 0
695	fsmblks: total bytes held in fastbin blocks
696	uordblks: current total allocated space (normal or mmapped)
697	fordblks: total free space
698	keepcost: the maximum number of bytes that could ideally be released
699	back to system via malloc_trim. ("ideally" means that
700	it ignores page restrictions etc.)
701
702	Because these fields are ints, but internal bookkeeping may
703	be kept as longs, the reported values may wrap around zero and
704	thus be inaccurate.
705	*/
706	struct mallinfo2 __libc_mallinfo2(void);
707	libc_hidden_proto (__libc_mallinfo2)
708
709	struct mallinfo __libc_mallinfo(void);
710
711
712	/*
713	pvalloc(size_t n);
714	Equivalent to valloc(minimum-page-that-holds(n)), that is,
715	round up n to nearest pagesize.
716	*/
717	void* __libc_pvalloc(size_t);
718
719	/*
720	malloc_trim(size_t pad);
721
722	If possible, gives memory back to the system (via negative
723	arguments to sbrk) if there is unused memory at the `high' end of
724	the malloc pool. You can call this after freeing large blocks of
725	memory to potentially reduce the system-level memory requirements
726	of a program. However, it cannot guarantee to reduce memory. Under
727	some allocation patterns, some large free blocks of memory will be
728	locked between two used chunks, so they cannot be given back to
729	the system.
730
731	The `pad' argument to malloc_trim represents the amount of free
732	trailing space to leave untrimmed. If this argument is zero,
733	only the minimum amount of memory to maintain internal data
734	structures will be left (one page or less). Non-zero arguments
735	can be supplied to maintain enough trailing space to service
736	future expected allocations without having to re-obtain memory
737	from the system.
738
739	Malloc_trim returns 1 if it actually released any memory, else 0.
740	On systems that do not support "negative sbrks", it will always
741	return 0.
742	*/
743	int __malloc_trim(size_t);
744
745	/*
746	malloc_usable_size(void p);*
747
748	Returns the number of bytes you can actually use in
749	an allocated chunk, which may be more than you requested (although
750	often not) due to alignment and minimum size constraints.
751	You can use this many bytes without worrying about
752	overwriting other allocated objects. This is not a particularly great
753	programming practice. malloc_usable_size can be more useful in
754	debugging and assertions, for example:
755
756	p = malloc(n);
757	assert(malloc_usable_size(p) >= 256);
758
759	*/
760	size_t __malloc_usable_size(void*);
761
762	/*
763	malloc_stats();
764	Prints on stderr the amount of space obtained from the system (both
765	via sbrk and mmap), the maximum amount (which may be more than
766	current if malloc_trim and/or munmap got called), and the current
767	number of bytes allocated via malloc (or realloc, etc) but not yet
768	freed. Note that this is the number of bytes allocated, not the
769	number requested. It will be larger than the number requested
770	because of alignment and bookkeeping overhead. Because it includes
771	alignment wastage as being in use, this figure may be greater than
772	zero even when no user-level chunks are allocated.
773
774	The reported current and maximum system memory can be inaccurate if
775	a program makes other calls to system memory allocation functions
776	(normally sbrk) outside of malloc.
777
778	malloc_stats prints only the most commonly interesting statistics.
779	More information can be obtained by calling mallinfo.
780
781	*/
782	void __malloc_stats(void);
783
784	/*
785	posix_memalign(void memptr, size_t alignment, size_t size);
786
787	POSIX wrapper like memalign(), checking for validity of size.
788	*/
789	int __posix_memalign(void **, size_t, size_t);
790	#endif /* IS_IN (libc) */
791
792	/*
793	mallopt(int parameter_number, int parameter_value)
794	Sets tunable parameters The format is to provide a
795	(parameter-number, parameter-value) pair. mallopt then sets the
796	corresponding parameter to the argument value if it can (i.e., so
797	long as the value is meaningful), and returns 1 if successful else
798	0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
799	normally defined in malloc.h. Only one of these (M_MXFAST) is used
800	in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
801	so setting them has no effect. But this malloc also supports four
802	other options in mallopt. See below for details. Briefly, supported
803	parameters are as follows (listed defaults are for "typical"
804	configurations).
805
806	Symbol param # default allowed param values
807	M_MXFAST 1 64 0-80 (0 disables fastbins)
808	M_TRIM_THRESHOLD -1 1281024 any (-1U disables trimming)*
809	M_TOP_PAD -2 0 any
810	M_MMAP_THRESHOLD -3 1281024 any (or 0 if no MMAP support)*
811	M_MMAP_MAX -4 65536 any (0 disables use of mmap)
812	*/
813	int __libc_mallopt(int, int);
814	#if IS_IN (libc)
815	libc_hidden_proto (__libc_mallopt)
816	#endif
817
818	/ mallopt tuning options /
819
820	/*
821	M_MXFAST is the maximum request size used for "fastbins", special bins
822	that hold returned chunks without consolidating their spaces. This
823	enables future requests for chunks of the same size to be handled
824	very quickly, but can increase fragmentation, and thus increase the
825	overall memory footprint of a program.
826
827	This malloc manages fastbins very conservatively yet still
828	efficiently, so fragmentation is rarely a problem for values less
829	than or equal to the default. The maximum supported value of MXFAST
830	is 80. You wouldn't want it any higher than this anyway. Fastbins
831	are designed especially for use with many small structs, objects or
832	strings -- the default handles structs/objects/arrays with sizes up
833	to 8 4byte fields, or small strings representing words, tokens,
834	etc. Using fastbins for larger objects normally worsens
835	fragmentation without improving speed.
836
837	M_MXFAST is set in REQUEST size units. It is internally used in
838	chunksize units, which adds padding and alignment. You can reduce
839	M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
840	algorithm to be a closer approximation of fifo-best-fit in all cases,
841	not just for larger requests, but will generally cause it to be
842	slower.
843	*/
844
845
846	/ M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h /
847	#ifndef M_MXFAST
848	#define M_MXFAST 1
849	#endif
850
851	#ifndef DEFAULT_MXFAST
852	#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
853	#endif
854
855
856	/*
857	M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
858	to keep before releasing via malloc_trim in free().
859
860	Automatic trimming is mainly useful in long-lived programs.
861	Because trimming via sbrk can be slow on some systems, and can
862	sometimes be wasteful (in cases where programs immediately
863	afterward allocate more large chunks) the value should be high
864	enough so that your overall system performance would improve by
865	releasing this much memory.
866
867	The trim threshold and the mmap control parameters (see below)
868	can be traded off with one another. Trimming and mmapping are
869	two different ways of releasing unused memory back to the
870	system. Between these two, it is often possible to keep
871	system-level demands of a long-lived program down to a bare
872	minimum. For example, in one test suite of sessions measuring
873	the XF86 X server on Linux, using a trim threshold of 128K and a
874	mmap threshold of 192K led to near-minimal long term resource
875	consumption.
876
877	If you are using this malloc in a long-lived program, it should
878	pay to experiment with these values. As a rough guide, you
879	might set to a value close to the average size of a process
880	(program) running on your system. Releasing this much memory
881	would allow such a process to run in memory. Generally, it's
882	worth it to tune for trimming rather tham memory mapping when a
883	program undergoes phases where several large chunks are
884	allocated and released in ways that can reuse each other's
885	storage, perhaps mixed with phases where there are no such
886	chunks at all. And in well-behaved long-lived programs,
887	controlling release of large blocks via trimming versus mapping
888	is usually faster.
889
890	However, in most programs, these parameters serve mainly as
891	protection against the system-level effects of carrying around
892	massive amounts of unneeded memory. Since frequent calls to
893	sbrk, mmap, and munmap otherwise degrade performance, the default
894	parameters are set to relatively high values that serve only as
895	safeguards.
896
897	The trim value It must be greater than page size to have any useful
898	effect. To disable trimming completely, you can set to
899	(unsigned long)(-1)
900
901	Trim settings interact with fastbin (MXFAST) settings: Unless
902	TRIM_FASTBINS is defined, automatic trimming never takes place upon
903	freeing a chunk with size less than or equal to MXFAST. Trimming is
904	instead delayed until subsequent freeing of larger chunks. However,
905	you can still force an attempted trim by calling malloc_trim.
906
907	Also, trimming is not generally possible in cases where
908	the main arena is obtained via mmap.
909
910	Note that the trick some people use of mallocing a huge space and
911	then freeing it at program startup, in an attempt to reserve system
912	memory, doesn't have the intended effect under automatic trimming,
913	since that memory will immediately be returned to the system.
914	*/
915
916	#define M_TRIM_THRESHOLD -1
917
918	#ifndef DEFAULT_TRIM_THRESHOLD
919	#define DEFAULT_TRIM_THRESHOLD (128 * 1024)
920	#endif
921
922	/*
923	M_TOP_PAD is the amount of extra `padding' space to allocate or
924	retain whenever sbrk is called. It is used in two ways internally:
925
926	* When sbrk is called to extend the top of the arena to satisfy
927	a new malloc request, this much padding is added to the sbrk
928	request.
929
930	* When malloc_trim is called automatically from free(),
931	it is used as the `pad' argument.
932
933	In both cases, the actual amount of padding is rounded
934	so that the end of the arena is always a system page boundary.
935
936	The main reason for using padding is to avoid calling sbrk so
937	often. Having even a small pad greatly reduces the likelihood
938	that nearly every malloc request during program start-up (or
939	after trimming) will invoke sbrk, which needlessly wastes
940	time.
941
942	Automatic rounding-up to page-size units is normally sufficient
943	to avoid measurable overhead, so the default is 0. However, in
944	systems where sbrk is relatively slow, it can pay to increase
945	this value, at the expense of carrying around more memory than
946	the program needs.
947	*/
948
949	#define M_TOP_PAD -2
950
951	#ifndef DEFAULT_TOP_PAD
952	#define DEFAULT_TOP_PAD (0)
953	#endif
954
955	/*
956	MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
957	adjusted MMAP_THRESHOLD.
958	*/
959
960	#ifndef DEFAULT_MMAP_THRESHOLD_MIN
961	#define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
962	#endif
963
964	#ifndef DEFAULT_MMAP_THRESHOLD_MAX
965	/ For 32-bit platforms we cannot increase the maximum mmap*
966	threshold much because it is also the minimum value for the
967	maximum heap size and its alignment. Going above 512k (i.e., 1M
968	for new heaps) wastes too much address space. /*
969	# if __WORDSIZE == 32
970	# define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
971	# else
972	# define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
973	# endif
974	#endif
975
976	/*
977	M_MMAP_THRESHOLD is the request size threshold for using mmap()
978	to service a request. Requests of at least this size that cannot
979	be allocated using already-existing space will be serviced via mmap.
980	(If enough normal freed space already exists it is used instead.)
981
982	Using mmap segregates relatively large chunks of memory so that
983	they can be individually obtained and released from the host
984	system. A request serviced through mmap is never reused by any
985	other request (at least not directly; the system may just so
986	happen to remap successive requests to the same locations).
987
988	Segregating space in this way has the benefits that:
989
990	1. Mmapped space can ALWAYS be individually released back
991	to the system, which helps keep the system level memory
992	demands of a long-lived program low.
993	2. Mapped memory can never become `locked' between
994	other chunks, as can happen with normally allocated chunks, which
995	means that even trimming via malloc_trim would not release them.
996	3. On some systems with "holes" in address spaces, mmap can obtain
997	memory that sbrk cannot.
998
999	However, it has the disadvantages that:
1000
1001	1. The space cannot be reclaimed, consolidated, and then
1002	used to service later requests, as happens with normal chunks.
1003	2. It can lead to more wastage because of mmap page alignment
1004	requirements
1005	3. It causes malloc performance to be more dependent on host
1006	system memory management support routines which may vary in
1007	implementation quality and may impose arbitrary
1008	limitations. Generally, servicing a request via normal
1009	malloc steps is faster than going through a system's mmap.
1010
1011	The advantages of mmap nearly always outweigh disadvantages for
1012	"large" chunks, but the value of "large" varies across systems. The
1013	default is an empirically derived value that works well in most
1014	systems.
1015
1016
1017	Update in 2006:
1018	The above was written in 2001. Since then the world has changed a lot.
1019	Memory got bigger. Applications got bigger. The virtual address space
1020	layout in 32 bit linux changed.
1021
1022	In the new situation, brk() and mmap space is shared and there are no
1023	artificial limits on brk size imposed by the kernel. What is more,
1024	applications have started using transient allocations larger than the
1025	128Kb as was imagined in 2001.
1026
1027	The price for mmap is also high now; each time glibc mmaps from the
1028	kernel, the kernel is forced to zero out the memory it gives to the
1029	application. Zeroing memory is expensive and eats a lot of cache and
1030	memory bandwidth. This has nothing to do with the efficiency of the
1031	virtual memory system, by doing mmap the kernel just has no choice but
1032	to zero.
1033
1034	In 2001, the kernel had a maximum size for brk() which was about 800
1035	megabytes on 32 bit x86, at that point brk() would hit the first
1036	mmaped shared libaries and couldn't expand anymore. With current 2.6
1037	kernels, the VA space layout is different and brk() and mmap
1038	both can span the entire heap at will.
1039
1040	Rather than using a static threshold for the brk/mmap tradeoff,
1041	we are now using a simple dynamic one. The goal is still to avoid
1042	fragmentation. The old goals we kept are
1043	1) try to get the long lived large allocations to use mmap()
1044	2) really large allocations should always use mmap()
1045	and we're adding now:
1046	3) transient allocations should use brk() to avoid forcing the kernel
1047	having to zero memory over and over again
1048
1049	The implementation works with a sliding threshold, which is by default
1050	limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
1051	out at 128Kb as per the 2001 default.
1052
1053	This allows us to satisfy requirement 1) under the assumption that long
1054	lived allocations are made early in the process' lifespan, before it has
1055	started doing dynamic allocations of the same size (which will
1056	increase the threshold).
1057
1058	The upperbound on the threshold satisfies requirement 2)
1059
1060	The threshold goes up in value when the application frees memory that was
1061	allocated with the mmap allocator. The idea is that once the application
1062	starts freeing memory of a certain size, it's highly probable that this is
1063	a size the application uses for transient allocations. This estimator
1064	is there to satisfy the new third requirement.
1065
1066	*/
1067
1068	#define M_MMAP_THRESHOLD -3
1069
1070	#ifndef DEFAULT_MMAP_THRESHOLD
1071	#define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
1072	#endif
1073
1074	/*
1075	M_MMAP_MAX is the maximum number of requests to simultaneously
1076	service using mmap. This parameter exists because
1077	some systems have a limited number of internal tables for
1078	use by mmap, and using more than a few of them may degrade
1079	performance.
1080
1081	The default is set to a value that serves only as a safeguard.
1082	Setting to 0 disables use of mmap for servicing large requests.
1083	*/
1084
1085	#define M_MMAP_MAX -4
1086
1087	#ifndef DEFAULT_MMAP_MAX
1088	#define DEFAULT_MMAP_MAX (65536)
1089	#endif
1090
1091	#include <malloc.h>
1092
1093	#ifndef RETURN_ADDRESS
1094	#define RETURN_ADDRESS(X_) (NULL)
1095	#endif
1096
1097	/ Forward declarations. /
1098	struct malloc_chunk;
1099	typedef struct malloc_chunk* mchunkptr;
1100
1101	/ Internal routines. /
1102
1103	static void* _int_malloc(mstate, size_t);
1104	static void _int_free(mstate, mchunkptr, int);
1105	static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
1106	INTERNAL_SIZE_T);
1107	static void* _int_memalign(mstate, size_t, size_t);
1108	#if IS_IN (libc)
1109	static void* _mid_memalign(size_t, size_t, void *);
1110	#endif
1111
1112	static void malloc_printerr(const char str) __attribute__* ((noreturn));
1113
1114	static void munmap_chunk(mchunkptr p);
1115	#if HAVE_MREMAP
1116	static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
1117	#endif
1118
1119	/ ------------------ MMAP support ------------------ /
1120
1121
1122	#include <fcntl.h>
1123	#include <sys/mman.h>
1124
1125	#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1126	# define MAP_ANONYMOUS MAP_ANON
1127	#endif
1128
1129	#ifndef MAP_NORESERVE
1130	# define MAP_NORESERVE 0
1131	#endif
1132
1133	#define MMAP(addr, size, prot, flags) \
1134	__mmap((addr), (size), (prot), (flags)\|MAP_ANONYMOUS\|MAP_PRIVATE, -1, 0)
1135
1136
1137	/*
1138	----------------------- Chunk representations -----------------------
1139	*/
1140
1141
1142	/*
1143	This struct declaration is misleading (but accurate and necessary).
1144	It declares a "view" into memory allowing access to necessary
1145	fields at known offsets from a given base. See explanation below.
1146	*/
1147
1148	struct malloc_chunk {
1149
1150	INTERNAL_SIZE_T mchunk_prev_size; / Size of previous chunk (if free). /
1151	INTERNAL_SIZE_T mchunk_size; / Size in bytes, including overhead. /
1152
1153	struct malloc_chunk* fd; / double links -- used only if free. /
1154	struct malloc_chunk* bk;
1155
1156	/ Only used for large blocks: pointer to next larger size. /
1157	struct malloc_chunk* fd_nextsize; / double links -- used only if free. /
1158	struct malloc_chunk* bk_nextsize;
1159	};
1160
1161
1162	/*
1163	malloc_chunk details:
1164
1165	(The following includes lightly edited explanations by Colin Plumb.)
1166
1167	Chunks of memory are maintained using a `boundary tag' method as
1168	described in e.g., Knuth or Standish. (See the paper by Paul
1169	Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1170	survey of such techniques.) Sizes of free chunks are stored both
1171	in the front of each chunk and at the end. This makes
1172	consolidating fragmented chunks into bigger chunks very fast. The
1173	size fields also hold bits representing whether chunks are free or
1174	in use.
1175
1176	An allocated chunk looks like this:
1177
1178
1179	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1180	\| Size of previous chunk, if unallocated (P clear) \|
1181	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1182	\| Size of chunk, in bytes \|A\|M\|P\|
1183	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1184	\| User data starts here... .
1185	. .
1186	. (malloc_usable_size() bytes) .
1187	. \|
1188	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1189	\| (size of chunk, but used for application data) \|
1190	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1191	\| Size of next chunk, in bytes \|A\|0\|1\|
1192	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1193
1194	Where "chunk" is the front of the chunk for the purpose of most of
1195	the malloc code, but "mem" is the pointer that is returned to the
1196	user. "Nextchunk" is the beginning of the next contiguous chunk.
1197
1198	Chunks always begin on even word boundaries, so the mem portion
1199	(which is returned to the user) is also on an even word boundary, and
1200	thus at least double-word aligned.
1201
1202	Free chunks are stored in circular doubly-linked lists, and look like this:
1203
1204	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1205	\| Size of previous chunk, if unallocated (P clear) \|
1206	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1207	`head:' \| Size of chunk, in bytes \|A\|0\|P\|
1208	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1209	\| Forward pointer to next chunk in list \|
1210	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1211	\| Back pointer to previous chunk in list \|
1212	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1213	\| Unused space (may be 0 bytes long) .
1214	. .
1215	. \|
1216	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1217	`foot:' \| Size of chunk, in bytes \|
1218	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1219	\| Size of next chunk, in bytes \|A\|0\|0\|
1220	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1221
1222	The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1223	chunk size (which is always a multiple of two words), is an in-use
1224	bit for the previous* chunk. If that bit is clear, then the*
1225	word before the current chunk size contains the previous chunk
1226	size, and can be used to find the front of the previous chunk.
1227	The very first chunk allocated always has this bit set,
1228	preventing access to non-existent (or non-owned) memory. If
1229	prev_inuse is set for any given chunk, then you CANNOT determine
1230	the size of the previous chunk, and might even get a memory
1231	addressing fault when trying to do so.
1232
1233	The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
1234	main arena, described by the main_arena variable. When additional
1235	threads are spawned, each thread receives its own arena (up to a
1236	configurable limit, after which arenas are reused for multiple
1237	threads), and the chunks in these arenas have the A bit set. To
1238	find the arena for a chunk on such a non-main arena, heap_for_ptr
1239	performs a bit mask operation and indirection through the ar_ptr
1240	member of the per-heap header heap_info (see arena.c).
1241
1242	Note that the `foot' of the current chunk is actually represented
1243	as the prev_size of the NEXT chunk. This makes it easier to
1244	deal with alignments etc but can be very confusing when trying
1245	to extend or adapt this code.
1246
1247	The three exceptions to all this are:
1248
1249	1. The special chunk `top' doesn't bother using the
1250	trailing size field since there is no next contiguous chunk
1251	that would have to index off it. After initialization, `top'
1252	is forced to always exist. If it would become less than
1253	MINSIZE bytes long, it is replenished.
1254
1255	2. Chunks allocated via mmap, which have the second-lowest-order
1256	bit M (IS_MMAPPED) set in their size fields. Because they are
1257	allocated one-by-one, each must contain its own trailing size
1258	field. If the M bit is set, the other bits are ignored
1259	(because mmapped chunks are neither in an arena, nor adjacent
1260	to a freed chunk). The M bit is also used for chunks which
1261	originally came from a dumped heap via malloc_set_state in
1262	hooks.c.
1263
1264	3. Chunks in fastbins are treated as allocated chunks from the
1265	point of view of the chunk allocator. They are consolidated
1266	with their neighbors only in bulk, in malloc_consolidate.
1267	*/
1268
1269	/*
1270	---------- Size and alignment checks and conversions ----------
1271	*/
1272
1273	/ Conversion from malloc headers to user pointers, and back. When*
1274	using memory tagging the user data and the malloc data structure
1275	headers have distinct tags. Converting fully from one to the other
1276	involves extracting the tag at the other address and creating a
1277	suitable pointer using it. That can be quite expensive. There are
1278	cases when the pointers are not dereferenced (for example only used
1279	for alignment check) so the tags are not relevant, and there are
1280	cases when user data is not tagged distinctly from malloc headers
1281	(user data is untagged because tagging is done late in malloc and
1282	early in free). User memory tagging across internal interfaces:
1283
1284	sysmalloc: Returns untagged memory.
1285	_int_malloc: Returns untagged memory.
1286	_int_free: Takes untagged memory.
1287	_int_memalign: Returns untagged memory.
1288	_int_memalign: Returns untagged memory.
1289	_mid_memalign: Returns tagged memory.
1290	_int_realloc: Takes and returns tagged memory.
1291	*/
1292
1293	/ The chunk header is two SIZE_SZ elements, but this is used widely, so*
1294	we define it here for clarity later. /*
1295	#define CHUNK_HDR_SZ (2 * SIZE_SZ)
1296
1297	/ Convert a chunk address to a user mem pointer without correcting*
1298	the tag. /*
1299	#define chunk2mem(p) ((void)((char)(p) + CHUNK_HDR_SZ))
1300
1301	/ Convert a chunk address to a user mem pointer and extract the right tag. /
1302	#define chunk2mem_tag(p) ((void)tag_at ((char)(p) + CHUNK_HDR_SZ))
1303
1304	/ Convert a user mem pointer to a chunk address and extract the right tag. /
1305	#define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
1306
1307	/ The smallest possible chunk /
1308	#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1309
1310	/ The smallest size we can malloc is an aligned minimal chunk /
1311
1312	#define MINSIZE \
1313	(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1314
1315	/ Check if m has acceptable alignment /
1316
1317	#define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1318
1319	#define misaligned_chunk(p) \
1320	((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
1321	& MALLOC_ALIGN_MASK)
1322
1323	/ pad request bytes into a usable size -- internal version /
1324	/ Note: This must be a macro that evaluates to a compile time constant*
1325	if passed a literal constant. /*
1326	#define request2size(req) \
1327	(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1328	MINSIZE : \
1329	((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1330
1331	/ Check if REQ overflows when padded and aligned and if the resulting*
1332	value is less than PTRDIFF_T. Returns the requested size or
1333	MINSIZE in case the value is less than MINSIZE, or 0 if any of the
1334	previous checks fail. /*
1335	static inline size_t
1336	checked_request2size (size_t req) __nonnull (`1`)
1337	{
1338	if (__glibc_unlikely (req > PTRDIFF_MAX))
1339	return `0`;
1340
1341	/ When using tagged memory, we cannot share the end of the user*
1342	block with the header for the next chunk, so ensure that we
1343	allocate blocks that are rounded up to the granule size. Take
1344	care not to overflow from close to MAX_SIZE_T to a small
1345	number. Ideally, this would be part of request2size(), but that
1346	must be a macro that produces a compile time constant if passed
1347	a constant literal. /*
1348	if (__glibc_unlikely (mtag_enabled))
1349	{
1350	/ Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551. /
1351	asm ("");
1352
1353	req = (req + (__MTAG_GRANULE_SIZE - `1`)) &
1354	~(size_t)(__MTAG_GRANULE_SIZE - `1`);
1355	}
1356
1357	return request2size (req);
1358	}
1359
1360	/*
1361	--------------- Physical chunk operations ---------------
1362	*/
1363
1364
1365	/ size field is or'ed with PREV_INUSE when previous adjacent chunk in use /
1366	#define PREV_INUSE 0x1
1367
1368	/ extract inuse bit of previous chunk /
1369	#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
1370
1371
1372	/ size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() /
1373	#define IS_MMAPPED 0x2
1374
1375	/ check for mmap()'ed chunk /
1376	#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
1377
1378
1379	/ size field is or'ed with NON_MAIN_ARENA if the chunk was obtained*
1380	from a non-main arena. This is only set immediately before handing
1381	the chunk to the user, if necessary. /*
1382	#define NON_MAIN_ARENA 0x4
1383
1384	/ Check for chunk from main arena. /
1385	#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
1386
1387	/ Mark a chunk as not being on the main arena. /
1388	#define set_non_main_arena(p) ((p)->mchunk_size \|= NON_MAIN_ARENA)
1389
1390
1391	/*
1392	Bits to mask off when extracting size
1393
1394	Note: IS_MMAPPED is intentionally not masked off from size field in
1395	macros for which mmapped chunks should never be seen. This should
1396	cause helpful core dumps to occur if it is tried by accident by
1397	people extending or adapting this malloc.
1398	*/
1399	#define SIZE_BITS (PREV_INUSE \| IS_MMAPPED \| NON_MAIN_ARENA)
1400
1401	/ Get size, ignoring use bits /
1402	#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
1403
1404	/ Like chunksize, but do not mask SIZE_BITS. /
1405	#define chunksize_nomask(p) ((p)->mchunk_size)
1406
1407	/ Ptr to next physical malloc_chunk. /
1408	#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
1409
1410	/ Size of the chunk below P. Only valid if !prev_inuse (P). /
1411	#define prev_size(p) ((p)->mchunk_prev_size)
1412
1413	/ Set the size of the chunk below P. Only valid if !prev_inuse (P). /
1414	#define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
1415
1416	/ Ptr to previous physical malloc_chunk. Only valid if !prev_inuse (P). /
1417	#define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
1418
1419	/ Treat space at ptr + offset as a chunk /
1420	#define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s)))
1421
1422	/ extract p's inuse bit /
1423	#define inuse(p) \
1424	((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
1425
1426	/ set/clear chunk as being inuse without otherwise disturbing /
1427	#define set_inuse(p) \
1428	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size \|= PREV_INUSE
1429
1430	#define clear_inuse(p) \
1431	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
1432
1433
1434	/ check/set/clear inuse bits in known places /
1435	#define inuse_bit_at_offset(p, s) \
1436	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
1437
1438	#define set_inuse_bit_at_offset(p, s) \
1439	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size \|= PREV_INUSE)
1440
1441	#define clear_inuse_bit_at_offset(p, s) \
1442	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
1443
1444
1445	/ Set size at head, without disturbing its use bit /
1446	#define set_head_size(p, s) ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) \| (s)))
1447
1448	/ Set size/use field /
1449	#define set_head(p, s) ((p)->mchunk_size = (s))
1450
1451	/ Set size at footer (only when chunk is not in use) /
1452	#define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
1453
1454	#pragma GCC poison mchunk_size
1455	#pragma GCC poison mchunk_prev_size
1456
1457	/ This is the size of the real usable data in the chunk. Not valid for*
1458	dumped heap chunks. /*
1459	#define memsize(p) \
1460	(__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
1461	chunksize (p) - CHUNK_HDR_SZ : \
1462	chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
1463
1464	/ If memory tagging is enabled the layout changes to accommodate the granule*
1465	size, this is wasteful for small allocations so not done by default.
1466	Both the chunk header and user data has to be granule aligned. /*
1467	_Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
1468	"memory tagging is not supported with large granule.");
1469
1470	static __always_inline void *
1471	tag_new_usable (void *ptr)
1472	{
1473	if (__glibc_unlikely (mtag_enabled) && ptr)
1474	{
1475	mchunkptr cp = mem2chunk(ptr);
1476	ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp));
1477	}
1478	return ptr;
1479	}
1480
1481	/*
1482	-------------------- Internal data structures --------------------
1483
1484	All internal state is held in an instance of malloc_state defined
1485	below. There are no other static variables, except in two optional
1486	cases:
1487	* If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1488	* If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1489	for mmap.
1490
1491	Beware of lots of tricks that minimize the total bookkeeping space
1492	requirements. The result is a little over 1K bytes (for 4byte
1493	pointers and size_t.)
1494	*/
1495
1496	/*
1497	Bins
1498
1499	An array of bin headers for free chunks. Each bin is doubly
1500	linked. The bins are approximately proportionally (log) spaced.
1501	There are a lot of these bins (128). This may look excessive, but
1502	works very well in practice. Most bins hold sizes that are
1503	unusual as malloc request sizes, but are more usual for fragments
1504	and consolidated sets of chunks, which is what these bins hold, so
1505	they can be found quickly. All procedures maintain the invariant
1506	that no consolidated chunk physically borders another one, so each
1507	chunk in a list is known to be preceeded and followed by either
1508	inuse chunks or the ends of memory.
1509
1510	Chunks in bins are kept in size order, with ties going to the
1511	approximately least recently used chunk. Ordering isn't needed
1512	for the small bins, which all contain the same-sized chunks, but
1513	facilitates best-fit allocation for larger chunks. These lists
1514	are just sequential. Keeping them in order almost never requires
1515	enough traversal to warrant using fancier ordered data
1516	structures.
1517
1518	Chunks of the same size are linked with the most
1519	recently freed at the front, and allocations are taken from the
1520	back. This results in LRU (FIFO) allocation order, which tends
1521	to give each chunk an equal opportunity to be consolidated with
1522	adjacent freed chunks, resulting in larger free chunks and less
1523	fragmentation.
1524
1525	To simplify use in double-linked lists, each bin header acts
1526	as a malloc_chunk. This avoids special-casing for headers.
1527	But to conserve space and improve locality, we allocate
1528	only the fd/bk pointers of bins, and then use repositioning tricks
1529	to treat these as the fields of a malloc_chunk.*
1530	*/
1531
1532	typedef struct malloc_chunk *mbinptr;
1533
1534	/ addressing -- note that bin_at(0) does not exist /
1535	#define bin_at(m, i) \
1536	(mbinptr) (((char ) &((m)->bins[((i) - 1) 2])) \
1537	- offsetof (struct malloc_chunk, fd))
1538
1539	/ analog of ++bin /
1540	#define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
1541
1542	/ Reminders about list directionality within bins /
1543	#define first(b) ((b)->fd)
1544	#define last(b) ((b)->bk)
1545
1546	/*
1547	Indexing
1548
1549	Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1550	8 bytes apart. Larger bins are approximately logarithmically spaced:
1551
1552	64 bins of size 8
1553	32 bins of size 64
1554	16 bins of size 512
1555	8 bins of size 4096
1556	4 bins of size 32768
1557	2 bins of size 262144
1558	1 bin of size what's left
1559
1560	There is actually a little bit of slop in the numbers in bin_index
1561	for the sake of speed. This makes no difference elsewhere.
1562
1563	The bins top out around 1MB because we expect to service large
1564	requests via mmap.
1565
1566	Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1567	a valid chunk size the small bins are bumped up one.
1568	*/
1569
1570	#define NBINS 128
1571	#define NSMALLBINS 64
1572	#define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1573	#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
1574	#define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1575
1576	#define in_smallbin_range(sz) \
1577	((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
1578
1579	#define smallbin_index(sz) \
1580	((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
1581	+ SMALLBIN_CORRECTION)
1582
1583	#define largebin_index_32(sz) \
1584	(((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\
1585	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1586	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1587	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1588	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1589	126)
1590
1591	#define largebin_index_32_big(sz) \
1592	(((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\
1593	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1594	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1595	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1596	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1597	126)
1598
1599	// XXX It remains to be seen whether it is good to keep the widths of
1600	// XXX the buckets the same or whether it should be scaled by a factor
1601	// XXX of two as well.
1602	#define largebin_index_64(sz) \
1603	(((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\
1604	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1605	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1606	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1607	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1608	126)
1609
1610	#define largebin_index(sz) \
1611	(SIZE_SZ == 8 ? largebin_index_64 (sz) \
1612	: MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1613	: largebin_index_32 (sz))
1614
1615	#define bin_index(sz) \
1616	((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
1617
1618	/ Take a chunk off a bin list. /
1619	static void
1620	unlink_chunk (mstate av, mchunkptr p)
1621	{
1622	if (chunksize (p) != prev_size (next_chunk (p)))
1623	malloc_printerr ("corrupted size vs. prev_size");
1624
1625	mchunkptr fd = p->fd;
1626	mchunkptr bk = p->bk;
1627
1628	if (__builtin_expect (fd->bk != p \|\| bk->fd != p, `0`))
1629	malloc_printerr ("corrupted double-linked list");
1630
1631	fd->bk = bk;
1632	bk->fd = fd;
1633	if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
1634	{
1635	if (p->fd_nextsize->bk_nextsize != p
1636	\|\| p->bk_nextsize->fd_nextsize != p)
1637	malloc_printerr ("corrupted double-linked list (not small)");
1638
1639	if (fd->fd_nextsize == NULL)
1640	{
1641	if (p->fd_nextsize == p)
1642	fd->fd_nextsize = fd->bk_nextsize = fd;
1643	else
1644	{
1645	fd->fd_nextsize = p->fd_nextsize;
1646	fd->bk_nextsize = p->bk_nextsize;
1647	p->fd_nextsize->bk_nextsize = fd;
1648	p->bk_nextsize->fd_nextsize = fd;
1649	}
1650	}
1651	else
1652	{
1653	p->fd_nextsize->bk_nextsize = p->bk_nextsize;
1654	p->bk_nextsize->fd_nextsize = p->fd_nextsize;
1655	}
1656	}
1657	}
1658
1659	/*
1660	Unsorted chunks
1661
1662	All remainders from chunk splits, as well as all returned chunks,
1663	are first placed in the "unsorted" bin. They are then placed
1664	in regular bins after malloc gives them ONE chance to be used before
1665	binning. So, basically, the unsorted_chunks list acts as a queue,
1666	with chunks being placed on it in free (and malloc_consolidate),
1667	and taken off (to be either used or placed in bins) in malloc.
1668
1669	The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1670	does not have to be taken into account in size comparisons.
1671	*/
1672
1673	/ The otherwise unindexable 1-bin is used to hold unsorted chunks. /
1674	#define unsorted_chunks(M) (bin_at (M, 1))
1675
1676	/*
1677	Top
1678
1679	The top-most available chunk (i.e., the one bordering the end of
1680	available memory) is treated specially. It is never included in
1681	any bin, is used only if no other chunk is available, and is
1682	released back to the system if it is very large (see
1683	M_TRIM_THRESHOLD). Because top initially
1684	points to its own bin with initial zero size, thus forcing
1685	extension on the first malloc request, we avoid having any special
1686	code in malloc to check whether it even exists yet. But we still
1687	need to do so when getting memory from system, so we make
1688	initial_top treat the bin as a legal but unusable chunk during the
1689	interval between initialization and the first call to
1690	sysmalloc. (This is somewhat delicate, since it relies on
1691	the 2 preceding words to be zero during this interval as well.)
1692	*/
1693
1694	/ Conveniently, the unsorted bin can be used as dummy top on first call /
1695	#define initial_top(M) (unsorted_chunks (M))
1696
1697	/*
1698	Binmap
1699
1700	To help compensate for the large number of bins, a one-level index
1701	structure is used for bin-by-bin searching. `binmap' is a
1702	bitvector recording whether bins are definitely empty so they can
1703	be skipped over during during traversals. The bits are NOT always
1704	cleared as soon as bins are empty, but instead only
1705	when they are noticed to be empty during traversal in malloc.
1706	*/
1707
1708	/ Conservatively use 32 bits per map word, even if on 64bit system /
1709	#define BINMAPSHIFT 5
1710	#define BITSPERMAP (1U << BINMAPSHIFT)
1711	#define BINMAPSIZE (NBINS / BITSPERMAP)
1712
1713	#define idx2block(i) ((i) >> BINMAPSHIFT)
1714	#define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
1715
1716	#define mark_bin(m, i) ((m)->binmap[idx2block (i)] \|= idx2bit (i))
1717	#define unmark_bin(m, i) ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
1718	#define get_binmap(m, i) ((m)->binmap[idx2block (i)] & idx2bit (i))
1719
1720	/*
1721	Fastbins
1722
1723	An array of lists holding recently freed small chunks. Fastbins
1724	are not doubly linked. It is faster to single-link them, and
1725	since chunks are never removed from the middles of these lists,
1726	double linking is not necessary. Also, unlike regular bins, they
1727	are not even processed in FIFO order (they use faster LIFO) since
1728	ordering doesn't much matter in the transient contexts in which
1729	fastbins are normally used.
1730
1731	Chunks in fastbins keep their inuse bit set, so they cannot
1732	be consolidated with other free chunks. malloc_consolidate
1733	releases all chunks in fastbins and consolidates them with
1734	other free chunks.
1735	*/
1736
1737	typedef struct malloc_chunk *mfastbinptr;
1738	#define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1739
1740	/ offset 2 to use otherwise unindexable first 2 bins /
1741	#define fastbin_index(sz) \
1742	((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1743
1744
1745	/ The maximum fastbin request size we support /
1746	#define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1747
1748	#define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
1749
1750	/*
1751	FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1752	that triggers automatic consolidation of possibly-surrounding
1753	fastbin chunks. This is a heuristic, so the exact value should not
1754	matter too much. It is defined at half the default trim threshold as a
1755	compromise heuristic to only attempt consolidation if it is likely
1756	to lead to trimming. However, it is not dynamically tunable, since
1757	consolidation reduces fragmentation surrounding large chunks even
1758	if trimming is not used.
1759	*/
1760
1761	#define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1762
1763	/*
1764	NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1765	regions. Otherwise, contiguity is exploited in merging together,
1766	when possible, results from consecutive MORECORE calls.
1767
1768	The initial value comes from MORECORE_CONTIGUOUS, but is
1769	changed dynamically if mmap is ever used as an sbrk substitute.
1770	*/
1771
1772	#define NONCONTIGUOUS_BIT (2U)
1773
1774	#define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1775	#define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1776	#define set_noncontiguous(M) ((M)->flags \|= NONCONTIGUOUS_BIT)
1777	#define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1778
1779	/ Maximum size of memory handled in fastbins. /
1780	static INTERNAL_SIZE_T global_max_fast;
1781
1782	/*
1783	Set value of max_fast.
1784	Use impossibly small value if 0.
1785	Precondition: there are no existing fastbin chunks in the main arena.
1786	Since do_check_malloc_state () checks this, we call malloc_consolidate ()
1787	before changing max_fast. Note other arenas will leak their fast bin
1788	entries if max_fast is reduced.
1789	*/
1790
1791	#define set_max_fast(s) \
1792	global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \
1793	? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1794
1795	static inline INTERNAL_SIZE_T
1796	get_max_fast (void)
1797	{
1798	/ Tell the GCC optimizers that global_max_fast is never larger*
1799	than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in
1800	_int_malloc after constant propagation of the size parameter.
1801	(The code never executes because malloc preserves the
1802	global_max_fast invariant, but the optimizers may not recognize
1803	this.) /*
1804	if (global_max_fast > MAX_FAST_SIZE)
1805	__builtin_unreachable ();
1806	return global_max_fast;
1807	}
1808
1809	/*
1810	----------- Internal state representation and initialization -----------
1811	*/
1812
1813	/*
1814	have_fastchunks indicates that there are probably some fastbin chunks.
1815	It is set true on entering a chunk into any fastbin, and cleared early in
1816	malloc_consolidate. The value is approximate since it may be set when there
1817	are no fastbin chunks, or it may be clear even if there are fastbin chunks
1818	available. Given it's sole purpose is to reduce number of redundant calls to
1819	malloc_consolidate, it does not affect correctness. As a result we can safely
1820	use relaxed atomic accesses.
1821	*/
1822
1823
1824	struct malloc_state
1825	{
1826	/ Serialize access. /
1827	__libc_lock_define (, mutex);
1828
1829	/ Flags (formerly in max_fast). /
1830	int flags;
1831
1832	/ Set if the fastbin chunks contain recently inserted free blocks. /
1833	/ Note this is a bool but not all targets support atomics on booleans. /
1834	int have_fastchunks;
1835
1836	/ Fastbins /
1837	mfastbinptr fastbinsY[NFASTBINS];
1838
1839	/ Base of the topmost chunk -- not otherwise kept in a bin /
1840	mchunkptr top;
1841
1842	/ The remainder from the most recent split of a small request /
1843	mchunkptr last_remainder;
1844
1845	/ Normal bins packed as described above /
1846	mchunkptr bins[NBINS * `2` - `2`];
1847
1848	/ Bitmap of bins /
1849	unsigned int binmap[BINMAPSIZE];
1850
1851	/ Linked list /
1852	struct malloc_state *next;
1853
1854	/ Linked list for free arenas. Access to this field is serialized*
1855	by free_list_lock in arena.c. /*
1856	struct malloc_state *next_free;
1857
1858	/ Number of threads attached to this arena. 0 if the arena is on*
1859	the free list. Access to this field is serialized by
1860	free_list_lock in arena.c. /*
1861	INTERNAL_SIZE_T attached_threads;
1862
1863	/ Memory allocated from the system in this arena. /
1864	INTERNAL_SIZE_T system_mem;
1865	INTERNAL_SIZE_T max_system_mem;
1866	};
1867
1868	struct malloc_par
1869	{
1870	/ Tunable parameters /
1871	unsigned long trim_threshold;
1872	INTERNAL_SIZE_T top_pad;
1873	INTERNAL_SIZE_T mmap_threshold;
1874	INTERNAL_SIZE_T arena_test;
1875	INTERNAL_SIZE_T arena_max;
1876
1877	#if HAVE_TUNABLES
1878	/ Transparent Large Page support. /
1879	INTERNAL_SIZE_T thp_pagesize;
1880	/ A value different than 0 means to align mmap allocation to hp_pagesize*
1881	add hp_flags on flags. /*
1882	INTERNAL_SIZE_T hp_pagesize;
1883	int hp_flags;
1884	#endif
1885
1886	/ Memory map support /
1887	int n_mmaps;
1888	int n_mmaps_max;
1889	int max_n_mmaps;
1890	/ the mmap_threshold is dynamic, until the user sets*
1891	it manually, at which point we need to disable any
1892	dynamic behavior. /*
1893	int no_dyn_threshold;
1894
1895	/ Statistics /
1896	INTERNAL_SIZE_T mmapped_mem;
1897	INTERNAL_SIZE_T max_mmapped_mem;
1898
1899	/ First address handed out by MORECORE/sbrk. /
1900	char *sbrk_base;
1901
1902	#if USE_TCACHE
1903	/ Maximum number of buckets to use. /
1904	size_t tcache_bins;
1905	size_t tcache_max_bytes;
1906	/ Maximum number of chunks in each bucket. /
1907	size_t tcache_count;
1908	/ Maximum number of chunks to remove from the unsorted list, which*
1909	aren't used to prefill the cache. /*
1910	size_t tcache_unsorted_limit;
1911	#endif
1912	};
1913
1914	/ There are several instances of this struct ("arenas") in this*
1915	malloc. If you are adapting this malloc in a way that does NOT use
1916	a static or mmapped malloc_state, you MUST explicitly zero-fill it
1917	before using. This malloc relies on the property that malloc_state
1918	is initialized to all zeroes (as is true of C statics). /*
1919
1920	static struct malloc_state main_arena =
1921	{
1922	.mutex = _LIBC_LOCK_INITIALIZER,
1923	.next = &main_arena,
1924	.attached_threads = `1`
1925	};
1926
1927	/ There is only one instance of the malloc parameters. /
1928
1929	static struct malloc_par mp_ =
1930	{
1931	.top_pad = DEFAULT_TOP_PAD,
1932	.n_mmaps_max = DEFAULT_MMAP_MAX,
1933	.mmap_threshold = DEFAULT_MMAP_THRESHOLD,
1934	.trim_threshold = DEFAULT_TRIM_THRESHOLD,
1935	#define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
1936	.arena_test = NARENAS_FROM_NCORES (`1`)
1937	#if USE_TCACHE
1938	,
1939	.tcache_count = TCACHE_FILL_COUNT,
1940	.tcache_bins = TCACHE_MAX_BINS,
1941	.tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-`1`),
1942	.tcache_unsorted_limit = `0` / No limit. /
1943	#endif
1944	};
1945
1946	/*
1947	Initialize a malloc_state struct.
1948
1949	This is called from ptmalloc_init () or from _int_new_arena ()
1950	when creating a new arena.
1951	*/
1952
1953	static void
1954	malloc_init_state (mstate av)
1955	{
1956	int i;
1957	mbinptr bin;
1958
1959	/ Establish circular links for normal bins /
1960	for (i = `1`; i < NBINS; ++i)
1961	{
1962	bin = bin_at (av, i);
1963	bin->fd = bin->bk = bin;
1964	}
1965
1966	#if MORECORE_CONTIGUOUS
1967	if (av != &main_arena)
1968	#endif
1969	set_noncontiguous (av);
1970	if (av == &main_arena)
1971	set_max_fast (DEFAULT_MXFAST);
1972	atomic_store_relaxed (&av->have_fastchunks, false);
1973
1974	av->top = initial_top (av);
1975	}
1976
1977	/*
1978	Other internal utilities operating on mstates
1979	*/
1980
1981	static void *sysmalloc (INTERNAL_SIZE_T, mstate);
1982	static int systrim (size_t, mstate);
1983	static void malloc_consolidate (mstate);
1984
1985
1986	/ -------------- Early definitions for debugging hooks ---------------- /
1987
1988	/ This function is called from the arena shutdown hook, to free the*
1989	thread cache (if it exists). /*
1990	static void tcache_thread_shutdown (void);
1991
1992	/ ------------------ Testing support ----------------------------------/
1993
1994	static int perturb_byte;
1995
1996	static void
1997	alloc_perturb (char *p, size_t n)
1998	{
1999	if (__glibc_unlikely (perturb_byte))
2000	memset (p, perturb_byte ^ `0xff`, n);
2001	}
2002
2003	static void
2004	free_perturb (char *p, size_t n)
2005	{
2006	if (__glibc_unlikely (perturb_byte))
2007	memset (p, perturb_byte, n);
2008	}
2009
2010
2011
2012	#include <stap-probe.h>
2013
2014	/ ----------- Routines dealing with transparent huge pages ----------- /
2015
2016	static inline void
2017	madvise_thp (void *p, INTERNAL_SIZE_T size)
2018	{
2019	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2020	/ Do not consider areas smaller than a huge page or if the tunable is*
2021	not active. /*
2022	if (mp_.thp_pagesize == `0` \|\| size < mp_.thp_pagesize)
2023	return;
2024
2025	/ Linux requires the input address to be page-aligned, and unaligned*
2026	inputs happens only for initial data segment. /*
2027	if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
2028	{
2029	void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
2030	size += PTR_DIFF (p, q);
2031	p = q;
2032	}
2033
2034	__madvise (p, size, MADV_HUGEPAGE);
2035	#endif
2036	}
2037
2038	/ ------------------- Support for multiple arenas -------------------- /
2039	#include "arena.c"
2040
2041	/*
2042	Debugging support
2043
2044	These routines make a number of assertions about the states
2045	of data structures that should be true at all times. If any
2046	are not true, it's very likely that a user program has somehow
2047	trashed memory. (It's also possible that there is a coding error
2048	in malloc. In which case, please report it!)
2049	*/
2050
2051	#if !MALLOC_DEBUG
2052
2053	# define check_chunk(A, P)
2054	# define check_free_chunk(A, P)
2055	# define check_inuse_chunk(A, P)
2056	# define check_remalloced_chunk(A, P, N)
2057	# define check_malloced_chunk(A, P, N)
2058	# define check_malloc_state(A)
2059
2060	#else
2061
2062	# define check_chunk(A, P) do_check_chunk (A, P)
2063	# define check_free_chunk(A, P) do_check_free_chunk (A, P)
2064	# define check_inuse_chunk(A, P) do_check_inuse_chunk (A, P)
2065	# define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
2066	# define check_malloced_chunk(A, P, N) do_check_malloced_chunk (A, P, N)
2067	# define check_malloc_state(A) do_check_malloc_state (A)
2068
2069	/*
2070	Properties of all chunks
2071	*/
2072
2073	static void
2074	do_check_chunk (mstate av, mchunkptr p)
2075	{
2076	unsigned long sz = chunksize (p);
2077	/ min and max possible addresses assuming contiguous allocation /
2078	char max_address = (char* *) (av->top) + chunksize (av->top);
2079	char *min_address = max_address - av->system_mem;
2080
2081	if (!chunk_is_mmapped (p))
2082	{
2083	/ Has legal address ... /
2084	if (p != av->top)
2085	{
2086	if (contiguous (av))
2087	{
2088	assert (((char *) p) >= min_address);
2089	assert (((char ) p + sz) <= ((char* *) (av->top)));
2090	}
2091	}
2092	else
2093	{
2094	/ top size is always at least MINSIZE /
2095	assert ((unsigned long) (sz) >= MINSIZE);
2096	/ top predecessor always marked inuse /
2097	assert (prev_inuse (p));
2098	}
2099	}
2100	else
2101	{
2102	/ address is outside main heap /
2103	if (contiguous (av) && av->top != initial_top (av))
2104	{
2105	assert (((char ) p) < min_address \|\| ((char* *) p) >= max_address);
2106	}
2107	/ chunk is page-aligned /
2108	assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - `1`)) == `0`);
2109	/ mem is aligned /
2110	assert (aligned_OK (chunk2mem (p)));
2111	}
2112	}
2113
2114	/*
2115	Properties of free chunks
2116	*/
2117
2118	static void
2119	do_check_free_chunk (mstate av, mchunkptr p)
2120	{
2121	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2122	mchunkptr next = chunk_at_offset (p, sz);
2123
2124	do_check_chunk (av, p);
2125
2126	/ Chunk must claim to be free ... /
2127	assert (!inuse (p));
2128	assert (!chunk_is_mmapped (p));
2129
2130	/ Unless a special marker, must have OK fields /
2131	if ((unsigned long) (sz) >= MINSIZE)
2132	{
2133	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2134	assert (aligned_OK (chunk2mem (p)));
2135	/ ... matching footer field /
2136	assert (prev_size (next_chunk (p)) == sz);
2137	/ ... and is fully consolidated /
2138	assert (prev_inuse (p));
2139	assert (next == av->top \|\| inuse (next));
2140
2141	/ ... and has minimally sane links /
2142	assert (p->fd->bk == p);
2143	assert (p->bk->fd == p);
2144	}
2145	else / markers are always of size SIZE_SZ /
2146	assert (sz == SIZE_SZ);
2147	}
2148
2149	/*
2150	Properties of inuse chunks
2151	*/
2152
2153	static void
2154	do_check_inuse_chunk (mstate av, mchunkptr p)
2155	{
2156	mchunkptr next;
2157
2158	do_check_chunk (av, p);
2159
2160	if (chunk_is_mmapped (p))
2161	return; / mmapped chunks have no next/prev /
2162
2163	/ Check whether it claims to be in use ... /
2164	assert (inuse (p));
2165
2166	next = next_chunk (p);
2167
2168	/ ... and is surrounded by OK chunks.*
2169	Since more things can be checked with free chunks than inuse ones,
2170	if an inuse chunk borders them and debug is on, it's worth doing them.
2171	*/
2172	if (!prev_inuse (p))
2173	{
2174	/ Note that we cannot even look at prev unless it is not inuse /
2175	mchunkptr prv = prev_chunk (p);
2176	assert (next_chunk (prv) == p);
2177	do_check_free_chunk (av, prv);
2178	}
2179
2180	if (next == av->top)
2181	{
2182	assert (prev_inuse (next));
2183	assert (chunksize (next) >= MINSIZE);
2184	}
2185	else if (!inuse (next))
2186	do_check_free_chunk (av, next);
2187	}
2188
2189	/*
2190	Properties of chunks recycled from fastbins
2191	*/
2192
2193	static void
2194	do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2195	{
2196	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2197
2198	if (!chunk_is_mmapped (p))
2199	{
2200	assert (av == arena_for_chunk (p));
2201	if (chunk_main_arena (p))
2202	assert (av == &main_arena);
2203	else
2204	assert (av != &main_arena);
2205	}
2206
2207	do_check_inuse_chunk (av, p);
2208
2209	/ Legal size ... /
2210	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2211	assert ((unsigned long) (sz) >= MINSIZE);
2212	/ ... and alignment /
2213	assert (aligned_OK (chunk2mem (p)));
2214	/ chunk is less than MINSIZE more than request /
2215	assert ((long) (sz) - (long) (s) >= `0`);
2216	assert ((long) (sz) - (long) (s + MINSIZE) < `0`);
2217	}
2218
2219	/*
2220	Properties of nonrecycled chunks at the point they are malloced
2221	*/
2222
2223	static void
2224	do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2225	{
2226	/ same as recycled case ... /
2227	do_check_remalloced_chunk (av, p, s);
2228
2229	/*
2230	... plus, must obey implementation invariant that prev_inuse is
2231	always true of any allocated chunk; i.e., that each allocated
2232	chunk borders either a previously allocated and still in-use
2233	chunk, or the base of its memory arena. This is ensured
2234	by making all allocations from the `lowest' part of any found
2235	chunk. This does not necessarily hold however for chunks
2236	recycled via fastbins.
2237	*/
2238
2239	assert (prev_inuse (p));
2240	}
2241
2242
2243	/*
2244	Properties of malloc_state.
2245
2246	This may be useful for debugging malloc, as well as detecting user
2247	programmer errors that somehow write into malloc_state.
2248
2249	If you are extending or experimenting with this malloc, you can
2250	probably figure out how to hack this routine to print out or
2251	display chunk addresses, sizes, bins, and other instrumentation.
2252	*/
2253
2254	static void
2255	do_check_malloc_state (mstate av)
2256	{
2257	int i;
2258	mchunkptr p;
2259	mchunkptr q;
2260	mbinptr b;
2261	unsigned int idx;
2262	INTERNAL_SIZE_T size;
2263	unsigned long total = `0`;
2264	int max_fast_bin;
2265
2266	/ internal size_t must be no wider than pointer type /
2267	assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
2268
2269	/ alignment is a power of 2 /
2270	assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - `1`)) == `0`);
2271
2272	/ Check the arena is initialized. /
2273	assert (av->top != `0`);
2274
2275	/ No memory has been allocated yet, so doing more tests is not possible. /
2276	if (av->top == initial_top (av))
2277	return;
2278
2279	/ pagesize is a power of 2 /
2280	assert (powerof2(GLRO (dl_pagesize)));
2281
2282	/ A contiguous main_arena is consistent with sbrk_base. /
2283	if (av == &main_arena && contiguous (av))
2284	assert ((char *) mp_.sbrk_base + av->system_mem ==
2285	(char *) av->top + chunksize (av->top));
2286
2287	/ properties of fastbins /
2288
2289	/ max_fast is in allowed range /
2290	assert ((get_max_fast () & ~`1`) <= request2size (MAX_FAST_SIZE));
2291
2292	max_fast_bin = fastbin_index (get_max_fast ());
2293
2294	for (i = `0`; i < NFASTBINS; ++i)
2295	{
2296	p = fastbin (av, i);
2297
2298	/ The following test can only be performed for the main arena.*
2299	While mallopt calls malloc_consolidate to get rid of all fast
2300	bins (especially those larger than the new maximum) this does
2301	only happen for the main arena. Trying to do this for any
2302	other arena would mean those arenas have to be locked and
2303	malloc_consolidate be called for them. This is excessive. And
2304	even if this is acceptable to somebody it still cannot solve
2305	the problem completely since if the arena is locked a
2306	concurrent malloc call might create a new arena which then
2307	could use the newly invalid fast bins. /*
2308
2309	/ all bins past max_fast are empty /
2310	if (av == &main_arena && i > max_fast_bin)
2311	assert (p == `0`);
2312
2313	while (p != `0`)
2314	{
2315	if (__glibc_unlikely (misaligned_chunk (p)))
2316	malloc_printerr ("do_check_malloc_state(): "
2317	"unaligned fastbin chunk detected");
2318	/ each chunk claims to be inuse /
2319	do_check_inuse_chunk (av, p);
2320	total += chunksize (p);
2321	/ chunk belongs in this bin /
2322	assert (fastbin_index (chunksize (p)) == i);
2323	p = REVEAL_PTR (p->fd);
2324	}
2325	}
2326
2327	/ check normal bins /
2328	for (i = `1`; i < NBINS; ++i)
2329	{
2330	b = bin_at (av, i);
2331
2332	/ binmap is accurate (except for bin 1 == unsorted_chunks) /
2333	if (i >= `2`)
2334	{
2335	unsigned int binbit = get_binmap (av, i);
2336	int empty = last (b) == b;
2337	if (!binbit)
2338	assert (empty);
2339	else if (!empty)
2340	assert (binbit);
2341	}
2342
2343	for (p = last (b); p != b; p = p->bk)
2344	{
2345	/ each chunk claims to be free /
2346	do_check_free_chunk (av, p);
2347	size = chunksize (p);
2348	total += size;
2349	if (i >= `2`)
2350	{
2351	/ chunk belongs in bin /
2352	idx = bin_index (size);
2353	assert (idx == i);
2354	/ lists are sorted /
2355	assert (p->bk == b \|\|
2356	(unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
2357
2358	if (!in_smallbin_range (size))
2359	{
2360	if (p->fd_nextsize != NULL)
2361	{
2362	if (p->fd_nextsize == p)
2363	assert (p->bk_nextsize == p);
2364	else
2365	{
2366	if (p->fd_nextsize == first (b))
2367	assert (chunksize (p) < chunksize (p->fd_nextsize));
2368	else
2369	assert (chunksize (p) > chunksize (p->fd_nextsize));
2370
2371	if (p == first (b))
2372	assert (chunksize (p) > chunksize (p->bk_nextsize));
2373	else
2374	assert (chunksize (p) < chunksize (p->bk_nextsize));
2375	}
2376	}
2377	else
2378	assert (p->bk_nextsize == NULL);
2379	}
2380	}
2381	else if (!in_smallbin_range (size))
2382	assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
2383	/ chunk is followed by a legal chain of inuse chunks /
2384	for (q = next_chunk (p);
2385	(q != av->top && inuse (q) &&
2386	(unsigned long) (chunksize (q)) >= MINSIZE);
2387	q = next_chunk (q))
2388	do_check_inuse_chunk (av, q);
2389	}
2390	}
2391
2392	/ top chunk is OK /
2393	check_chunk (av, av->top);
2394	}
2395	#endif
2396
2397
2398	/ ----------------- Support for debugging hooks -------------------- /
2399	#if IS_IN (libc)
2400	#include "hooks.c"
2401	#endif
2402
2403
2404	/ ----------- Routines dealing with system allocation -------------- /
2405
2406	/*
2407	sysmalloc handles malloc cases requiring more memory from the system.
2408	On entry, it is assumed that av->top does not have enough
2409	space to service request for nb bytes, thus requiring that av->top
2410	be extended or replaced.
2411	*/
2412
2413	static void *
2414	sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av)
2415	{
2416	long int size;
2417
2418	/*
2419	Round up size to nearest page. For mmapped chunks, the overhead is one
2420	SIZE_SZ unit larger than for normal chunks, because there is no
2421	following chunk whose prev_size field could be used.
2422
2423	See the front_misalign handling below, for glibc there is no need for
2424	further alignments unless we have have high alignment.
2425	*/
2426	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2427	size = ALIGN_UP (nb + SIZE_SZ, pagesize);
2428	else
2429	size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
2430
2431	/ Don't try if size wraps around 0. /
2432	if ((unsigned long) (size) <= (unsigned long) (nb))
2433	return MAP_FAILED;
2434
2435	char mm = (char* *) MMAP (`0`, size,
2436	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2437	extra_flags);
2438	if (mm == MAP_FAILED)
2439	return mm;
2440
2441	#ifdef MAP_HUGETLB
2442	if (!(extra_flags & MAP_HUGETLB))
2443	madvise_thp (mm, size);
2444	#endif
2445
2446	/*
2447	The offset to the start of the mmapped region is stored in the prev_size
2448	field of the chunk. This allows us to adjust returned start address to
2449	meet alignment requirements here and in memalign(), and still be able to
2450	compute proper address argument for later munmap in free() and realloc().
2451	*/
2452
2453	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2454
2455	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2456	{
2457	/ For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and*
2458	MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1. Each mmap'ed area is page
2459	aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. /*
2460	assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == `0`);
2461	front_misalign = `0`;
2462	}
2463	else
2464	front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
2465
2466	mchunkptr p; / the allocated/returned chunk /
2467
2468	if (front_misalign > `0`)
2469	{
2470	ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign;
2471	p = (mchunkptr) (mm + correction);
2472	set_prev_size (p, correction);
2473	set_head (p, (size - correction) \| IS_MMAPPED);
2474	}
2475	else
2476	{
2477	p = (mchunkptr) mm;
2478	set_prev_size (p, `0`);
2479	set_head (p, size \| IS_MMAPPED);
2480	}
2481
2482	/ update statistics /
2483	int new = atomic_exchange_and_add (&mp_.n_mmaps, `1`) + `1`;
2484	atomic_max (&mp_.max_n_mmaps, new);
2485
2486	unsigned long sum;
2487	sum = atomic_exchange_and_add (&mp_.mmapped_mem, size) + size;
2488	atomic_max (&mp_.max_mmapped_mem, sum);
2489
2490	check_chunk (av, p);
2491
2492	return chunk2mem (p);
2493	}
2494
2495	/*
2496	Allocate memory using mmap() based on S and NB requested size, aligning to
2497	PAGESIZE if required. The EXTRA_FLAGS is used on mmap() call. If the call
2498	succeedes S is updated with the allocated size. This is used as a fallback
2499	if MORECORE fails.
2500	*/
2501	static void *
2502	sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb,
2503	INTERNAL_SIZE_T old_size, size_t minsize,
2504	size_t pagesize, int extra_flags, mstate av)
2505	{
2506	long int size = *s;
2507
2508	/ Cannot merge with old top, so add its size back in /
2509	if (contiguous (av))
2510	size = ALIGN_UP (size + old_size, pagesize);
2511
2512	/ If we are relying on mmap as backup, then use larger units /
2513	if ((unsigned long) (size) < minsize)
2514	size = minsize;
2515
2516	/ Don't try if size wraps around 0 /
2517	if ((unsigned long) (size) <= (unsigned long) (nb))
2518	return MORECORE_FAILURE;
2519
2520	char mbrk = (char* *) (MMAP (`0`, size,
2521	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2522	extra_flags));
2523	if (mbrk == MAP_FAILED)
2524	return MAP_FAILED;
2525
2526	#ifdef MAP_HUGETLB
2527	if (!(extra_flags & MAP_HUGETLB))
2528	madvise_thp (mbrk, size);
2529	#endif
2530
2531	/ Record that we no longer have a contiguous sbrk region. After the first*
2532	time mmap is used as backup, we do not ever rely on contiguous space
2533	since this could incorrectly bridge regions. /*
2534	set_noncontiguous (av);
2535
2536	*s = size;
2537	return mbrk;
2538	}
2539
2540	static void *
2541	sysmalloc (INTERNAL_SIZE_T nb, mstate av)
2542	{
2543	mchunkptr old_top; / incoming value of av->top /
2544	INTERNAL_SIZE_T old_size; / its size /
2545	char old_end; /* its end address /
2546
2547	long size; / arg to first MORECORE or mmap call /
2548	char brk; /* return value from MORECORE /
2549
2550	long correction; / arg to 2nd MORECORE call /
2551	char snd_brk; /* 2nd return val /
2552
2553	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2554	INTERNAL_SIZE_T end_misalign; / partial page left at end of new space /
2555	char aligned_brk; /* aligned offset into brk /
2556
2557	mchunkptr p; / the allocated/returned chunk /
2558	mchunkptr remainder; / remainder from allocation /
2559	unsigned long remainder_size; / its size /
2560
2561
2562	size_t pagesize = GLRO (dl_pagesize);
2563	bool tried_mmap = false;
2564
2565
2566	/*
2567	If have mmap, and the request size meets the mmap threshold, and
2568	the system supports mmap, and there are few enough currently
2569	allocated mmapped regions, try to directly map this request
2570	rather than expanding top.
2571	*/
2572
2573	if (av == NULL
2574	\|\| ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
2575	&& (mp_.n_mmaps < mp_.n_mmaps_max)))
2576	{
2577	char *mm;
2578	#if HAVE_TUNABLES
2579	if (mp_.hp_pagesize > `0` && nb >= mp_.hp_pagesize)
2580	{
2581	/ There is no need to isse the THP madvise call if Huge Pages are*
2582	used directly. /*
2583	mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av);
2584	if (mm != MAP_FAILED)
2585	return mm;
2586	}
2587	#endif
2588	mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2589	if (mm != MAP_FAILED)
2590	return mm;
2591	tried_mmap = true;
2592	}
2593
2594	/ There are no usable arenas and mmap also failed. /
2595	if (av == NULL)
2596	return `0`;
2597
2598	/ Record incoming configuration of top /
2599
2600	old_top = av->top;
2601	old_size = chunksize (old_top);
2602	old_end = (char *) (chunk_at_offset (old_top, old_size));
2603
2604	brk = snd_brk = (char *) (MORECORE_FAILURE);
2605
2606	/*
2607	If not the first time through, we require old_size to be
2608	at least MINSIZE and to have prev_inuse set.
2609	*/
2610
2611	assert ((old_top == initial_top (av) && old_size == `0`) \|\|
2612	((unsigned long) (old_size) >= MINSIZE &&
2613	prev_inuse (old_top) &&
2614	((unsigned long) old_end & (pagesize - `1`)) == `0`));
2615
2616	/ Precondition: not enough current space to satisfy nb request /
2617	assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
2618
2619
2620	if (av != &main_arena)
2621	{
2622	heap_info old_heap, heap;
2623	size_t old_heap_size;
2624
2625	/ First try to extend the current heap. /
2626	old_heap = heap_for_ptr (old_top);
2627	old_heap_size = old_heap->size;
2628	if ((long) (MINSIZE + nb - old_size) > `0`
2629	&& grow_heap (old_heap, MINSIZE + nb - old_size) == `0`)
2630	{
2631	av->system_mem += old_heap->size - old_heap_size;
2632	set_head (old_top, (((char ) old_heap + old_heap->size) - (char* *) old_top)
2633	\| PREV_INUSE);
2634	}
2635	else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad)))
2636	{
2637	/ Use a newly allocated heap. /
2638	heap->ar_ptr = av;
2639	heap->prev = old_heap;
2640	av->system_mem += heap->size;
2641	/ Set up the new top. /
2642	top (av) = chunk_at_offset (heap, sizeof (*heap));
2643	set_head (top (av), (heap->size - sizeof (*heap)) \| PREV_INUSE);
2644
2645	/ Setup fencepost and free the old top chunk with a multiple of*
2646	MALLOC_ALIGNMENT in size. /*
2647	/ The fencepost takes at least MINSIZE bytes, because it might*
2648	become the top chunk again later. Note that a footer is set
2649	up, too, although the chunk is marked in use. /*
2650	old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
2651	set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
2652	`0` \| PREV_INUSE);
2653	if (old_size >= MINSIZE)
2654	{
2655	set_head (chunk_at_offset (old_top, old_size),
2656	CHUNK_HDR_SZ \| PREV_INUSE);
2657	set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
2658	set_head (old_top, old_size \| PREV_INUSE \| NON_MAIN_ARENA);
2659	_int_free (av, old_top, `1`);
2660	}
2661	else
2662	{
2663	set_head (old_top, (old_size + CHUNK_HDR_SZ) \| PREV_INUSE);
2664	set_foot (old_top, (old_size + CHUNK_HDR_SZ));
2665	}
2666	}
2667	else if (!tried_mmap)
2668	{
2669	/ We can at least try to use to mmap memory. If new_heap fails*
2670	it is unlikely that trying to allocate huge pages will
2671	succeed. /*
2672	char *mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2673	if (mm != MAP_FAILED)
2674	return mm;
2675	}
2676	}
2677	else / av == main_arena /
2678
2679
2680	{ / Request enough space for nb + pad + overhead /
2681	size = nb + mp_.top_pad + MINSIZE;
2682
2683	/*
2684	If contiguous, we can subtract out existing space that we hope to
2685	combine with new space. We add it back later only if
2686	we don't actually get contiguous space.
2687	*/
2688
2689	if (contiguous (av))
2690	size -= old_size;
2691
2692	/*
2693	Round to a multiple of page size or huge page size.
2694	If MORECORE is not contiguous, this ensures that we only call it
2695	with whole-page arguments. And if MORECORE is contiguous and
2696	this is not first time through, this preserves page-alignment of
2697	previous calls. Otherwise, we correct to page-align below.
2698	*/
2699
2700	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2701	/ Defined in brk.c. /
2702	extern void *__curbrk;
2703	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2704	{
2705	uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
2706	mp_.thp_pagesize);
2707	size = top - (uintptr_t) __curbrk;
2708	}
2709	else
2710	#endif
2711	size = ALIGN_UP (size, GLRO(dl_pagesize));
2712
2713	/*
2714	Don't try to call MORECORE if argument is so big as to appear
2715	negative. Note that since mmap takes size_t arg, it may succeed
2716	below even if we cannot call MORECORE.
2717	*/
2718
2719	if (size > `0`)
2720	{
2721	brk = (char *) (MORECORE (size));
2722	if (brk != (char *) (MORECORE_FAILURE))
2723	madvise_thp (brk, size);
2724	LIBC_PROBE (memory_sbrk_more, `2`, brk, size);
2725	}
2726
2727	if (brk == (char *) (MORECORE_FAILURE))
2728	{
2729	/*
2730	If have mmap, try using it as a backup when MORECORE fails or
2731	cannot be used. This is worth doing on systems that have "holes" in
2732	address space, so sbrk cannot extend to give contiguous space, but
2733	space is available elsewhere. Note that we ignore mmap max count
2734	and threshold limits, since the space will not be used as a
2735	segregated mmap region.
2736	*/
2737
2738	char *mbrk = MAP_FAILED;
2739	#if HAVE_TUNABLES
2740	if (mp_.hp_pagesize > `0`)
2741	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size,
2742	mp_.hp_pagesize, mp_.hp_pagesize,
2743	mp_.hp_flags, av);
2744	#endif
2745	if (mbrk == MAP_FAILED)
2746	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, pagesize,
2747	MMAP_AS_MORECORE_SIZE, `0`, av);
2748	if (mbrk != MAP_FAILED)
2749	{
2750	/ We do not need, and cannot use, another sbrk call to find end /
2751	brk = mbrk;
2752	snd_brk = brk + size;
2753	}
2754	}
2755
2756	if (brk != (char *) (MORECORE_FAILURE))
2757	{
2758	if (mp_.sbrk_base == `0`)
2759	mp_.sbrk_base = brk;
2760	av->system_mem += size;
2761
2762	/*
2763	If MORECORE extends previous space, we can likewise extend top size.
2764	*/
2765
2766	if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
2767	set_head (old_top, (size + old_size) \| PREV_INUSE);
2768
2769	else if (contiguous (av) && old_size && brk < old_end)
2770	/ Oops! Someone else killed our space.. Can't touch anything. /
2771	malloc_printerr ("break adjusted to free malloc space");
2772
2773	/*
2774	Otherwise, make adjustments:
2775
2776	* If the first time through or noncontiguous, we need to call sbrk
2777	just to find out where the end of memory lies.
2778
2779	* We need to ensure that all returned chunks from malloc will meet
2780	MALLOC_ALIGNMENT
2781
2782	* If there was an intervening foreign sbrk, we need to adjust sbrk
2783	request size to account for fact that we will not be able to
2784	combine new space with existing space in old_top.
2785
2786	* Almost all systems internally allocate whole pages at a time, in
2787	which case we might as well use the whole last page of request.
2788	So we allocate enough more memory to hit a page boundary now,
2789	which in turn causes future contiguous calls to page-align.
2790	*/
2791
2792	else
2793	{
2794	front_misalign = `0`;
2795	end_misalign = `0`;
2796	correction = `0`;
2797	aligned_brk = brk;
2798
2799	/ handle contiguous cases /
2800	if (contiguous (av))
2801	{
2802	/ Count foreign sbrk as system_mem. /
2803	if (old_size)
2804	av->system_mem += brk - old_end;
2805
2806	/ Guarantee alignment of first new chunk made from this space /
2807
2808	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2809	if (front_misalign > `0`)
2810	{
2811	/*
2812	Skip over some bytes to arrive at an aligned position.
2813	We don't need to specially mark these wasted front bytes.
2814	They will never be accessed anyway because
2815	prev_inuse of av->top (and any chunk created from its start)
2816	is always true after initialization.
2817	*/
2818
2819	correction = MALLOC_ALIGNMENT - front_misalign;
2820	aligned_brk += correction;
2821	}
2822
2823	/*
2824	If this isn't adjacent to existing space, then we will not
2825	be able to merge with old_top space, so must add to 2nd request.
2826	*/
2827
2828	correction += old_size;
2829
2830	/ Extend the end address to hit a page boundary /
2831	end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
2832	correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
2833
2834	assert (correction >= `0`);
2835	snd_brk = (char *) (MORECORE (correction));
2836
2837	/*
2838	If can't allocate correction, try to at least find out current
2839	brk. It might be enough to proceed without failing.
2840
2841	Note that if second sbrk did NOT fail, we assume that space
2842	is contiguous with first sbrk. This is a safe assumption unless
2843	program is multithreaded but doesn't use locks and a foreign sbrk
2844	occurred between our first and second calls.
2845	*/
2846
2847	if (snd_brk == (char *) (MORECORE_FAILURE))
2848	{
2849	correction = `0`;
2850	snd_brk = (char *) (MORECORE (`0`));
2851	}
2852	else
2853	madvise_thp (snd_brk, correction);
2854	}
2855
2856	/ handle non-contiguous cases /
2857	else
2858	{
2859	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2860	/ MORECORE/mmap must correctly align /
2861	assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == `0`);
2862	else
2863	{
2864	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2865	if (front_misalign > `0`)
2866	{
2867	/*
2868	Skip over some bytes to arrive at an aligned position.
2869	We don't need to specially mark these wasted front bytes.
2870	They will never be accessed anyway because
2871	prev_inuse of av->top (and any chunk created from its start)
2872	is always true after initialization.
2873	*/
2874
2875	aligned_brk += MALLOC_ALIGNMENT - front_misalign;
2876	}
2877	}
2878
2879	/ Find out current end of memory /
2880	if (snd_brk == (char *) (MORECORE_FAILURE))
2881	{
2882	snd_brk = (char *) (MORECORE (`0`));
2883	}
2884	}
2885
2886	/ Adjust top based on results of second sbrk /
2887	if (snd_brk != (char *) (MORECORE_FAILURE))
2888	{
2889	av->top = (mchunkptr) aligned_brk;
2890	set_head (av->top, (snd_brk - aligned_brk + correction) \| PREV_INUSE);
2891	av->system_mem += correction;
2892
2893	/*
2894	If not the first time through, we either have a
2895	gap due to foreign sbrk or a non-contiguous region. Insert a
2896	double fencepost at old_top to prevent consolidation with space
2897	we don't own. These fenceposts are artificial chunks that are
2898	marked as inuse and are in any case too small to use. We need
2899	two to make sizes and alignments work out.
2900	*/
2901
2902	if (old_size != `0`)
2903	{
2904	/*
2905	Shrink old_top to insert fenceposts, keeping size a
2906	multiple of MALLOC_ALIGNMENT. We know there is at least
2907	enough space in old_top to do this.
2908	*/
2909	old_size = (old_size - `2` * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
2910	set_head (old_top, old_size \| PREV_INUSE);
2911
2912	/*
2913	Note that the following assignments completely overwrite
2914	old_top when old_size was previously MINSIZE. This is
2915	intentional. We need the fencepost, even if old_top otherwise gets
2916	lost.
2917	*/
2918	set_head (chunk_at_offset (old_top, old_size),
2919	CHUNK_HDR_SZ \| PREV_INUSE);
2920	set_head (chunk_at_offset (old_top,
2921	old_size + CHUNK_HDR_SZ),
2922	CHUNK_HDR_SZ \| PREV_INUSE);
2923
2924	/ If possible, release the rest. /
2925	if (old_size >= MINSIZE)
2926	{
2927	_int_free (av, old_top, `1`);
2928	}
2929	}
2930	}
2931	}
2932	}
2933	} / if (av != &main_arena) /
2934
2935	if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
2936	av->max_system_mem = av->system_mem;
2937	check_malloc_state (av);
2938
2939	/ finally, do the allocation /
2940	p = av->top;
2941	size = chunksize (p);
2942
2943	/ check that one of the above allocation paths succeeded /
2944	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
2945	{
2946	remainder_size = size - nb;
2947	remainder = chunk_at_offset (p, nb);
2948	av->top = remainder;
2949	set_head (p, nb \| PREV_INUSE \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
2950	set_head (remainder, remainder_size \| PREV_INUSE);
2951	check_malloced_chunk (av, p, nb);
2952	return chunk2mem (p);
2953	}
2954
2955	/ catch all failure paths /
2956	__set_errno (ENOMEM);
2957	return `0`;
2958	}
2959
2960
2961	/*
2962	systrim is an inverse of sorts to sysmalloc. It gives memory back
2963	to the system (via negative arguments to sbrk) if there is unused
2964	memory at the `high' end of the malloc pool. It is called
2965	automatically by free() when top space exceeds the trim
2966	threshold. It is also called by the public malloc_trim routine. It
2967	returns 1 if it actually released any memory, else 0.
2968	*/
2969
2970	static int
2971	systrim (size_t pad, mstate av)
2972	{
2973	long top_size; / Amount of top-most memory /
2974	long extra; / Amount to release /
2975	long released; / Amount actually released /
2976	char current_brk; /* address returned by pre-check sbrk call /
2977	char new_brk; /* address returned by post-check sbrk call /
2978	long top_area;
2979
2980	top_size = chunksize (av->top);
2981
2982	top_area = top_size - MINSIZE - `1`;
2983	if (top_area <= pad)
2984	return `0`;
2985
2986	/ Release in pagesize units and round down to the nearest page. /
2987	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2988	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2989	extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
2990	else
2991	#endif
2992	extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
2993
2994	if (extra == `0`)
2995	return `0`;
2996
2997	/*
2998	Only proceed if end of memory is where we last set it.
2999	This avoids problems if there were foreign sbrk calls.
3000	*/
3001	current_brk = (char *) (MORECORE (`0`));
3002	if (current_brk == (char *) (av->top) + top_size)
3003	{
3004	/*
3005	Attempt to release memory. We ignore MORECORE return value,
3006	and instead call again to find out where new end of memory is.
3007	This avoids problems if first call releases less than we asked,
3008	of if failure somehow altered brk value. (We could still
3009	encounter problems if it altered brk in some very bad way,
3010	but the only thing we can do is adjust anyway, which will cause
3011	some downstream failure.)
3012	*/
3013
3014	MORECORE (-extra);
3015	new_brk = (char *) (MORECORE (`0`));
3016
3017	LIBC_PROBE (memory_sbrk_less, `2`, new_brk, extra);
3018
3019	if (new_brk != (char *) MORECORE_FAILURE)
3020	{
3021	released = (long) (current_brk - new_brk);
3022
3023	if (released != `0`)
3024	{
3025	/ Success. Adjust top. /
3026	av->system_mem -= released;
3027	set_head (av->top, (top_size - released) \| PREV_INUSE);
3028	check_malloc_state (av);
3029	return `1`;
3030	}
3031	}
3032	}
3033	return `0`;
3034	}
3035
3036	static void
3037	munmap_chunk (mchunkptr p)
3038	{
3039	size_t pagesize = GLRO (dl_pagesize);
3040	INTERNAL_SIZE_T size = chunksize (p);
3041
3042	assert (chunk_is_mmapped (p));
3043
3044	uintptr_t mem = (uintptr_t) chunk2mem (p);
3045	uintptr_t block = (uintptr_t) p - prev_size (p);
3046	size_t total_size = prev_size (p) + size;
3047	/ Unfortunately we have to do the compilers job by hand here. Normally*
3048	we would test BLOCK and TOTAL-SIZE separately for compliance with the
3049	page size. But gcc does not recognize the optimization possibility
3050	(in the moment at least) so we combine the two values into one before
3051	the bit test. /*
3052	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3053	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3054	malloc_printerr ("munmap_chunk(): invalid pointer");
3055
3056	atomic_decrement (&mp_.n_mmaps);
3057	atomic_add (&mp_.mmapped_mem, -total_size);
3058
3059	/ If munmap failed the process virtual memory address space is in a*
3060	bad shape. Just leave the block hanging around, the process will
3061	terminate shortly anyway since not much can be done. /*
3062	__munmap ((char *) block, total_size);
3063	}
3064
3065	#if HAVE_MREMAP
3066
3067	static mchunkptr
3068	mremap_chunk (mchunkptr p, size_t new_size)
3069	{
3070	size_t pagesize = GLRO (dl_pagesize);
3071	INTERNAL_SIZE_T offset = prev_size (p);
3072	INTERNAL_SIZE_T size = chunksize (p);
3073	char *cp;
3074
3075	assert (chunk_is_mmapped (p));
3076
3077	uintptr_t block = (uintptr_t) p - offset;
3078	uintptr_t mem = (uintptr_t) chunk2mem(p);
3079	size_t total_size = offset + size;
3080	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3081	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3082	malloc_printerr("mremap_chunk(): invalid pointer");
3083
3084	/ Note the extra SIZE_SZ overhead as in mmap_chunk(). /
3085	new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
3086
3087	/ No need to remap if the number of pages does not change. /
3088	if (total_size == new_size)
3089	return p;
3090
3091	cp = (char ) __mremap ((char* *) block, total_size, new_size,
3092	MREMAP_MAYMOVE);
3093
3094	if (cp == MAP_FAILED)
3095	return `0`;
3096
3097	madvise_thp (cp, new_size);
3098
3099	p = (mchunkptr) (cp + offset);
3100
3101	assert (aligned_OK (chunk2mem (p)));
3102
3103	assert (prev_size (p) == offset);
3104	set_head (p, (new_size - offset) \| IS_MMAPPED);
3105
3106	INTERNAL_SIZE_T new;
3107	new = atomic_exchange_and_add (&mp_.mmapped_mem, new_size - size - offset)
3108	+ new_size - size - offset;
3109	atomic_max (&mp_.max_mmapped_mem, new);
3110	return p;
3111	}
3112	#endif /* HAVE_MREMAP */
3113
3114	/------------------------ Public wrappers. --------------------------------/
3115
3116	#if USE_TCACHE
3117
3118	/ We overlay this structure on the user-data portion of a chunk when*
3119	the chunk is stored in the per-thread cache. /*
3120	typedef struct tcache_entry
3121	{
3122	struct tcache_entry *next;
3123	/ This field exists to detect double frees. /
3124	uintptr_t key;
3125	} tcache_entry;
3126
3127	/ There is one of these for each thread, which contains the*
3128	per-thread cache (hence "tcache_perthread_struct"). Keeping
3129	overall size low is mildly important. Note that COUNTS and ENTRIES
3130	are redundant (we could have just counted the linked list each
3131	time), this is for performance reasons. /*
3132	typedef struct tcache_perthread_struct
3133	{
3134	uint16_t counts[TCACHE_MAX_BINS];
3135	tcache_entry *entries[TCACHE_MAX_BINS];
3136	} tcache_perthread_struct;
3137
3138	static __thread bool tcache_shutting_down = false;
3139	static __thread tcache_perthread_struct *tcache = NULL;
3140
3141	/ Process-wide key to try and catch a double-free in the same thread. /
3142	static uintptr_t tcache_key;
3143
3144	/ The value of tcache_key does not really have to be a cryptographically*
3145	secure random number. It only needs to be arbitrary enough so that it does
3146	not collide with values present in applications. If a collision does happen
3147	consistently enough, it could cause a degradation in performance since the
3148	entire list is checked to check if the block indeed has been freed the
3149	second time. The odds of this happening are exceedingly low though, about 1
3150	in 2^wordsize. There is probably a higher chance of the performance
3151	degradation being due to a double free where the first free happened in a
3152	different thread; that's a case this check does not cover. /*
3153	static void
3154	tcache_key_initialize (void)
3155	{
3156	if (__getrandom (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
3157	!= sizeof (tcache_key))
3158	{
3159	tcache_key = random_bits ();
3160	#if __WORDSIZE == 64
3161	tcache_key = (tcache_key << `32`) \| random_bits ();
3162	#endif
3163	}
3164	}
3165
3166	/ Caller must ensure that we know tc_idx is valid and there's room*
3167	for more chunks. /*
3168	static __always_inline void
3169	tcache_put (mchunkptr chunk, size_t tc_idx)
3170	{
3171	tcache_entry e = (tcache_entry ) chunk2mem (chunk);
3172
3173	/ Mark this chunk as "in the tcache" so the test in _int_free will*
3174	detect a double free. /*
3175	e->key = tcache_key;
3176
3177	e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
3178	tcache->entries[tc_idx] = e;
3179	++(tcache->counts[tc_idx]);
3180	}
3181
3182	/ Caller must ensure that we know tc_idx is valid and there's*
3183	available chunks to remove. /*
3184	static __always_inline void *
3185	tcache_get (size_t tc_idx)
3186	{
3187	tcache_entry *e = tcache->entries[tc_idx];
3188	if (__glibc_unlikely (!aligned_OK (e)))
3189	malloc_printerr ("malloc(): unaligned tcache chunk detected");
3190	tcache->entries[tc_idx] = REVEAL_PTR (e->next);
3191	--(tcache->counts[tc_idx]);
3192	e->key = `0`;
3193	return (void *) e;
3194	}
3195
3196	static void
3197	tcache_thread_shutdown (void)
3198	{
3199	int i;
3200	tcache_perthread_struct *tcache_tmp = tcache;
3201
3202	tcache_shutting_down = true;
3203
3204	if (!tcache)
3205	return;
3206
3207	/ Disable the tcache and prevent it from being reinitialized. /
3208	tcache = NULL;
3209
3210	/ Free all of the entries and the tcache itself back to the arena*
3211	heap for coalescing. /*
3212	for (i = `0`; i < TCACHE_MAX_BINS; ++i)
3213	{
3214	while (tcache_tmp->entries[i])
3215	{
3216	tcache_entry *e = tcache_tmp->entries[i];
3217	if (__glibc_unlikely (!aligned_OK (e)))
3218	malloc_printerr ("tcache_thread_shutdown(): "
3219	"unaligned tcache chunk detected");
3220	tcache_tmp->entries[i] = REVEAL_PTR (e->next);
3221	__libc_free (e);
3222	}
3223	}
3224
3225	__libc_free (tcache_tmp);
3226	}
3227
3228	static void
3229	tcache_init(void)
3230	{
3231	mstate ar_ptr;
3232	void *victim = `0`;
3233	const size_t bytes = sizeof (tcache_perthread_struct);
3234
3235	if (tcache_shutting_down)
3236	return;
3237
3238	arena_get (ar_ptr, bytes);
3239	victim = _int_malloc (ar_ptr, bytes);
3240	if (!victim && ar_ptr != NULL)
3241	{
3242	ar_ptr = arena_get_retry (ar_ptr, bytes);
3243	victim = _int_malloc (ar_ptr, bytes);
3244	}
3245
3246
3247	if (ar_ptr != NULL)
3248	__libc_lock_unlock (ar_ptr->mutex);
3249
3250	/ In a low memory situation, we may not be able to allocate memory*
3251	- in which case, we just keep trying later. However, we
3252	typically do this very early, so either there is sufficient
3253	memory, or there isn't enough memory to do non-trivial
3254	allocations anyway. /*
3255	if (victim)
3256	{
3257	tcache = (tcache_perthread_struct *) victim;
3258	memset (tcache, `0`, sizeof (tcache_perthread_struct));
3259	}
3260
3261	}
3262
3263	# define MAYBE_INIT_TCACHE() \
3264	if (__glibc_unlikely (tcache == NULL)) \
3265	tcache_init();
3266
3267	#else /* !USE_TCACHE */
3268	# define MAYBE_INIT_TCACHE()
3269
3270	static void
3271	tcache_thread_shutdown (void)
3272	{
3273	/ Nothing to do if there is no thread cache. /
3274	}
3275
3276	#endif /* !USE_TCACHE */
3277
3278	#if IS_IN (libc)
3279	void *
3280	__libc_malloc (size_t bytes)
3281	{
3282	mstate ar_ptr;
3283	void *victim;
3284
3285	_Static_assert (PTRDIFF_MAX <= SIZE_MAX / `2`,
3286	"PTRDIFF_MAX is not more than half of SIZE_MAX");
3287
3288	if (!__malloc_initialized)
3289	ptmalloc_init ();
3290	#if USE_TCACHE
3291	/ int_free also calls request2size, be careful to not pad twice. /
3292	size_t tbytes = checked_request2size (bytes);
3293	if (tbytes == `0`)
3294	{
3295	__set_errno (ENOMEM);
3296	return NULL;
3297	}
3298	size_t tc_idx = csize2tidx (tbytes);
3299
3300	MAYBE_INIT_TCACHE ();
3301
3302	DIAG_PUSH_NEEDS_COMMENT;
3303	if (tc_idx < mp_.tcache_bins
3304	&& tcache
3305	&& tcache->counts[tc_idx] > `0`)
3306	{
3307	victim = tcache_get (tc_idx);
3308	return tag_new_usable (victim);
3309	}
3310	DIAG_POP_NEEDS_COMMENT;
3311	#endif
3312
3313	if (SINGLE_THREAD_P)
3314	{
3315	victim = tag_new_usable (_int_malloc (&main_arena, bytes));
3316	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3317	&main_arena == arena_for_chunk (mem2chunk (victim)));
3318	return victim;
3319	}
3320
3321	arena_get (ar_ptr, bytes);
3322
3323	victim = _int_malloc (ar_ptr, bytes);
3324	/ Retry with another arena only if we were able to find a usable arena*
3325	before. /*
3326	if (!victim && ar_ptr != NULL)
3327	{
3328	LIBC_PROBE (memory_malloc_retry, `1`, bytes);
3329	ar_ptr = arena_get_retry (ar_ptr, bytes);
3330	victim = _int_malloc (ar_ptr, bytes);
3331	}
3332
3333	if (ar_ptr != NULL)
3334	__libc_lock_unlock (ar_ptr->mutex);
3335
3336	victim = tag_new_usable (victim);
3337
3338	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3339	ar_ptr == arena_for_chunk (mem2chunk (victim)));
3340	return victim;
3341	}
3342	libc_hidden_def (__libc_malloc)
3343
3344	void
3345	__libc_free (void *mem)
3346	{
3347	mstate ar_ptr;
3348	mchunkptr p; / chunk corresponding to mem /
3349
3350	if (mem == `0`) / free(0) has no effect /
3351	return;
3352
3353	/ Quickly check that the freed pointer matches the tag for the memory.*
3354	This gives a useful double-free detection. /*
3355	if (__glibc_unlikely (mtag_enabled))
3356	(volatile* char *)mem;
3357
3358	int err = errno;
3359
3360	p = mem2chunk (mem);
3361
3362	if (chunk_is_mmapped (p)) / release mmapped memory. /
3363	{
3364	/ See if the dynamic brk/mmap threshold needs adjusting.*
3365	Dumped fake mmapped chunks do not affect the threshold. /*
3366	if (!mp_.no_dyn_threshold
3367	&& chunksize_nomask (p) > mp_.mmap_threshold
3368	&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
3369	{
3370	mp_.mmap_threshold = chunksize (p);
3371	mp_.trim_threshold = `2` * mp_.mmap_threshold;
3372	LIBC_PROBE (memory_mallopt_free_dyn_thresholds, `2`,
3373	mp_.mmap_threshold, mp_.trim_threshold);
3374	}
3375	munmap_chunk (p);
3376	}
3377	else
3378	{
3379	MAYBE_INIT_TCACHE ();
3380
3381	/ Mark the chunk as belonging to the library again. /
3382	(void)tag_region (chunk2mem (p), memsize (p));
3383
3384	ar_ptr = arena_for_chunk (p);
3385	_int_free (ar_ptr, p, `0`);
3386	}
3387
3388	__set_errno (err);
3389	}
3390	libc_hidden_def (__libc_free)
3391
3392	void *
3393	__libc_realloc (void *oldmem, size_t bytes)
3394	{
3395	mstate ar_ptr;
3396	INTERNAL_SIZE_T nb; / padded request size /
3397
3398	void newp; /* chunk to return /
3399
3400	if (!__malloc_initialized)
3401	ptmalloc_init ();
3402
3403	#if REALLOC_ZERO_BYTES_FREES
3404	if (bytes == `0` && oldmem != NULL)
3405	{
3406	__libc_free (oldmem); return `0`;
3407	}
3408	#endif
3409
3410	/ realloc of null is supposed to be same as malloc /
3411	if (oldmem == `0`)
3412	return __libc_malloc (bytes);
3413
3414	/ Perform a quick check to ensure that the pointer's tag matches the*
3415	memory's tag. /*
3416	if (__glibc_unlikely (mtag_enabled))
3417	(volatile* char*) oldmem;
3418
3419	/ chunk corresponding to oldmem /
3420	const mchunkptr oldp = mem2chunk (oldmem);
3421	/ its size /
3422	const INTERNAL_SIZE_T oldsize = chunksize (oldp);
3423
3424	if (chunk_is_mmapped (oldp))
3425	ar_ptr = NULL;
3426	else
3427	{
3428	MAYBE_INIT_TCACHE ();
3429	ar_ptr = arena_for_chunk (oldp);
3430	}
3431
3432	/ Little security check which won't hurt performance: the allocator*
3433	never wrapps around at the end of the address space. Therefore
3434	we can exclude some size values which might appear here by
3435	accident or by "design" from some intruder. /*
3436	if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, `0`)
3437	\|\| __builtin_expect (misaligned_chunk (oldp), `0`)))
3438	malloc_printerr ("realloc(): invalid pointer");
3439
3440	nb = checked_request2size (bytes);
3441	if (nb == `0`)
3442	{
3443	__set_errno (ENOMEM);
3444	return NULL;
3445	}
3446
3447	if (chunk_is_mmapped (oldp))
3448	{
3449	void *newmem;
3450
3451	#if HAVE_MREMAP
3452	newp = mremap_chunk (oldp, nb);
3453	if (newp)
3454	{
3455	void *newmem = chunk2mem_tag (newp);
3456	/ Give the new block a different tag. This helps to ensure*
3457	that stale handles to the previous mapping are not
3458	reused. There's a performance hit for both us and the
3459	caller for doing this, so we might want to
3460	reconsider. /*
3461	return tag_new_usable (newmem);
3462	}
3463	#endif
3464	/ Note the extra SIZE_SZ overhead. /
3465	if (oldsize - SIZE_SZ >= nb)
3466	return oldmem; / do nothing /
3467
3468	/ Must alloc, copy, free. /
3469	newmem = __libc_malloc (bytes);
3470	if (newmem == `0`)
3471	return `0`; / propagate failure /
3472
3473	memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
3474	munmap_chunk (oldp);
3475	return newmem;
3476	}
3477
3478	if (SINGLE_THREAD_P)
3479	{
3480	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3481	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3482	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3483
3484	return newp;
3485	}
3486
3487	__libc_lock_lock (ar_ptr->mutex);
3488
3489	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3490
3491	__libc_lock_unlock (ar_ptr->mutex);
3492	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3493	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3494
3495	if (newp == NULL)
3496	{
3497	/ Try harder to allocate memory in other arenas. /
3498	LIBC_PROBE (memory_realloc_retry, `2`, bytes, oldmem);
3499	newp = __libc_malloc (bytes);
3500	if (newp != NULL)
3501	{
3502	size_t sz = memsize (oldp);
3503	memcpy (newp, oldmem, sz);
3504	(void) tag_region (chunk2mem (oldp), sz);
3505	_int_free (ar_ptr, oldp, `0`);
3506	}
3507	}
3508
3509	return newp;
3510	}
3511	libc_hidden_def (__libc_realloc)
3512
3513	void *
3514	__libc_memalign (size_t alignment, size_t bytes)
3515	{
3516	if (!__malloc_initialized)
3517	ptmalloc_init ();
3518
3519	void *address = RETURN_ADDRESS (`0`);
3520	return _mid_memalign (alignment, bytes, address);
3521	}
3522
3523	static void *
3524	_mid_memalign (size_t alignment, size_t bytes, void *address)
3525	{
3526	mstate ar_ptr;
3527	void *p;
3528
3529	/ If we need less alignment than we give anyway, just relay to malloc. /
3530	if (alignment <= MALLOC_ALIGNMENT)
3531	return __libc_malloc (bytes);
3532
3533	/ Otherwise, ensure that it is at least a minimum chunk size /
3534	if (alignment < MINSIZE)
3535	alignment = MINSIZE;
3536
3537	/ If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a*
3538	power of 2 and will cause overflow in the check below. /*
3539	if (alignment > SIZE_MAX / `2` + `1`)
3540	{
3541	__set_errno (EINVAL);
3542	return `0`;
3543	}
3544
3545
3546	/ Make sure alignment is power of 2. /
3547	if (!powerof2 (alignment))
3548	{
3549	size_t a = MALLOC_ALIGNMENT * `2`;
3550	while (a < alignment)
3551	a <<= `1`;
3552	alignment = a;
3553	}
3554
3555	if (SINGLE_THREAD_P)
3556	{
3557	p = _int_memalign (&main_arena, alignment, bytes);
3558	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3559	&main_arena == arena_for_chunk (mem2chunk (p)));
3560	return tag_new_usable (p);
3561	}
3562
3563	arena_get (ar_ptr, bytes + alignment + MINSIZE);
3564
3565	p = _int_memalign (ar_ptr, alignment, bytes);
3566	if (!p && ar_ptr != NULL)
3567	{
3568	LIBC_PROBE (memory_memalign_retry, `2`, bytes, alignment);
3569	ar_ptr = arena_get_retry (ar_ptr, bytes);
3570	p = _int_memalign (ar_ptr, alignment, bytes);
3571	}
3572
3573	if (ar_ptr != NULL)
3574	__libc_lock_unlock (ar_ptr->mutex);
3575
3576	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3577	ar_ptr == arena_for_chunk (mem2chunk (p)));
3578	return tag_new_usable (p);
3579	}
3580	/ For ISO C11. /
3581	weak_alias (__libc_memalign, aligned_alloc)
3582	libc_hidden_def (__libc_memalign)
3583
3584	void *
3585	__libc_valloc (size_t bytes)
3586	{
3587	if (!__malloc_initialized)
3588	ptmalloc_init ();
3589
3590	void *address = RETURN_ADDRESS (`0`);
3591	size_t pagesize = GLRO (dl_pagesize);
3592	return _mid_memalign (pagesize, bytes, address);
3593	}
3594
3595	void *
3596	__libc_pvalloc (size_t bytes)
3597	{
3598	if (!__malloc_initialized)
3599	ptmalloc_init ();
3600
3601	void *address = RETURN_ADDRESS (`0`);
3602	size_t pagesize = GLRO (dl_pagesize);
3603	size_t rounded_bytes;
3604	/ ALIGN_UP with overflow check. /
3605	if (__glibc_unlikely (__builtin_add_overflow (bytes,
3606	pagesize - `1`,
3607	&rounded_bytes)))
3608	{
3609	__set_errno (ENOMEM);
3610	return `0`;
3611	}
3612	rounded_bytes = rounded_bytes & -(pagesize - `1`);
3613
3614	return _mid_memalign (pagesize, rounded_bytes, address);
3615	}
3616
3617	void *
3618	__libc_calloc (size_t n, size_t elem_size)
3619	{
3620	mstate av;
3621	mchunkptr oldtop;
3622	INTERNAL_SIZE_T sz, oldtopsize;
3623	void *mem;
3624	unsigned long clearsize;
3625	unsigned long nclears;
3626	INTERNAL_SIZE_T *d;
3627	ptrdiff_t bytes;
3628
3629	if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
3630	{
3631	__set_errno (ENOMEM);
3632	return NULL;
3633	}
3634
3635	sz = bytes;
3636
3637	if (!__malloc_initialized)
3638	ptmalloc_init ();
3639
3640	MAYBE_INIT_TCACHE ();
3641
3642	if (SINGLE_THREAD_P)
3643	av = &main_arena;
3644	else
3645	arena_get (av, sz);
3646
3647	if (av)
3648	{
3649	/ Check if we hand out the top chunk, in which case there may be no*
3650	need to clear. /*
3651	#if MORECORE_CLEARS
3652	oldtop = top (av);
3653	oldtopsize = chunksize (top (av));
3654	# if MORECORE_CLEARS < 2
3655	/ Only newly allocated memory is guaranteed to be cleared. /
3656	if (av == &main_arena &&
3657	oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
3658	oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
3659	# endif
3660	if (av != &main_arena)
3661	{
3662	heap_info *heap = heap_for_ptr (oldtop);
3663	if (oldtopsize < (char ) heap + heap->mprotect_size - (char* *) oldtop)
3664	oldtopsize = (char ) heap + heap->mprotect_size - (char* *) oldtop;
3665	}
3666	#endif
3667	}
3668	else
3669	{
3670	/ No usable arenas. /
3671	oldtop = `0`;
3672	oldtopsize = `0`;
3673	}
3674	mem = _int_malloc (av, sz);
3675
3676	assert (!mem \|\| chunk_is_mmapped (mem2chunk (mem)) \|\|
3677	av == arena_for_chunk (mem2chunk (mem)));
3678
3679	if (!SINGLE_THREAD_P)
3680	{
3681	if (mem == `0` && av != NULL)
3682	{
3683	LIBC_PROBE (memory_calloc_retry, `1`, sz);
3684	av = arena_get_retry (av, sz);
3685	mem = _int_malloc (av, sz);
3686	}
3687
3688	if (av != NULL)
3689	__libc_lock_unlock (av->mutex);
3690	}
3691
3692	/ Allocation failed even after a retry. /
3693	if (mem == `0`)
3694	return `0`;
3695
3696	mchunkptr p = mem2chunk (mem);
3697
3698	/ If we are using memory tagging, then we need to set the tags*
3699	regardless of MORECORE_CLEARS, so we zero the whole block while
3700	doing so. /*
3701	if (__glibc_unlikely (mtag_enabled))
3702	return tag_new_zero_region (mem, memsize (p));
3703
3704	INTERNAL_SIZE_T csz = chunksize (p);
3705
3706	/ Two optional cases in which clearing not necessary /
3707	if (chunk_is_mmapped (p))
3708	{
3709	if (__builtin_expect (perturb_byte, `0`))
3710	return memset (mem, `0`, sz);
3711
3712	return mem;
3713	}
3714
3715	#if MORECORE_CLEARS
3716	if (perturb_byte == `0` && (p == oldtop && csz > oldtopsize))
3717	{
3718	/ clear only the bytes from non-freshly-sbrked memory /
3719	csz = oldtopsize;
3720	}
3721	#endif
3722
3723	/ Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that*
3724	contents have an odd number of INTERNAL_SIZE_T-sized words;
3725	minimally 3. /*
3726	d = (INTERNAL_SIZE_T *) mem;
3727	clearsize = csz - SIZE_SZ;
3728	nclears = clearsize / sizeof (INTERNAL_SIZE_T);
3729	assert (nclears >= `3`);
3730
3731	if (nclears > `9`)
3732	return memset (d, `0`, clearsize);
3733
3734	else
3735	{
3736	*(d + `0`) = `0`;
3737	*(d + `1`) = `0`;
3738	*(d + `2`) = `0`;
3739	if (nclears > `4`)
3740	{
3741	*(d + `3`) = `0`;
3742	*(d + `4`) = `0`;
3743	if (nclears > `6`)
3744	{
3745	*(d + `5`) = `0`;
3746	*(d + `6`) = `0`;
3747	if (nclears > `8`)
3748	{
3749	*(d + `7`) = `0`;
3750	*(d + `8`) = `0`;
3751	}
3752	}
3753	}
3754	}
3755
3756	return mem;
3757	}
3758	#endif /* IS_IN (libc) */
3759
3760	/*
3761	------------------------------ malloc ------------------------------
3762	*/
3763
3764	static void *
3765	_int_malloc (mstate av, size_t bytes)
3766	{
3767	INTERNAL_SIZE_T nb; / normalized request size /
3768	unsigned int idx; / associated bin index /
3769	mbinptr bin; / associated bin /
3770
3771	mchunkptr victim; / inspected/selected chunk /
3772	INTERNAL_SIZE_T size; / its size /
3773	int victim_index; / its bin index /
3774
3775	mchunkptr remainder; / remainder from a split /
3776	unsigned long remainder_size; / its size /
3777
3778	unsigned int block; / bit map traverser /
3779	unsigned int bit; / bit map traverser /
3780	unsigned int map; / current word of binmap /
3781
3782	mchunkptr fwd; / misc temp for linking /
3783	mchunkptr bck; / misc temp for linking /
3784
3785	#if USE_TCACHE
3786	size_t tcache_unsorted_count; / count of unsorted chunks processed /
3787	#endif
3788
3789	/*
3790	Convert request size to internal form by adding SIZE_SZ bytes
3791	overhead plus possibly more to obtain necessary alignment and/or
3792	to obtain a size of at least MINSIZE, the smallest allocatable
3793	size. Also, checked_request2size returns false for request sizes
3794	that are so large that they wrap around zero when padded and
3795	aligned.
3796	*/
3797
3798	nb = checked_request2size (bytes);
3799	if (nb == `0`)
3800	{
3801	__set_errno (ENOMEM);
3802	return NULL;
3803	}
3804
3805	/ There are no usable arenas. Fall back to sysmalloc to get a chunk from*
3806	mmap. /*
3807	if (__glibc_unlikely (av == NULL))
3808	{
3809	void *p = sysmalloc (nb, av);
3810	if (p != NULL)
3811	alloc_perturb (p, bytes);
3812	return p;
3813	}
3814
3815	/*
3816	If the size qualifies as a fastbin, first check corresponding bin.
3817	This code is safe to execute even if av is not yet initialized, so we
3818	can try it without checking, which saves some time on this fast path.
3819	*/
3820
3821	#define REMOVE_FB(fb, victim, pp) \
3822	do \
3823	{ \
3824	victim = pp; \
3825	if (victim == NULL) \
3826	break; \
3827	pp = REVEAL_PTR (victim->fd); \
3828	if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp))) \
3829	malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
3830	} \
3831	while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
3832	!= victim); \
3833
3834	if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
3835	{
3836	idx = fastbin_index (nb);
3837	mfastbinptr *fb = &fastbin (av, idx);
3838	mchunkptr pp;
3839	victim = *fb;
3840
3841	if (victim != NULL)
3842	{
3843	if (__glibc_unlikely (misaligned_chunk (victim)))
3844	malloc_printerr ("malloc(): unaligned fastbin chunk detected 2");
3845
3846	if (SINGLE_THREAD_P)
3847	*fb = REVEAL_PTR (victim->fd);
3848	else
3849	REMOVE_FB (fb, pp, victim);
3850	if (__glibc_likely (victim != NULL))
3851	{
3852	size_t victim_idx = fastbin_index (chunksize (victim));
3853	if (__builtin_expect (victim_idx != idx, `0`))
3854	malloc_printerr ("malloc(): memory corruption (fast)");
3855	check_remalloced_chunk (av, victim, nb);
3856	#if USE_TCACHE
3857	/ While we're here, if we see other chunks of the same size,*
3858	stash them in the tcache. /*
3859	size_t tc_idx = csize2tidx (nb);
3860	if (tcache && tc_idx < mp_.tcache_bins)
3861	{
3862	mchunkptr tc_victim;
3863
3864	/ While bin not empty and tcache not full, copy chunks. /
3865	while (tcache->counts[tc_idx] < mp_.tcache_count
3866	&& (tc_victim = *fb) != NULL)
3867	{
3868	if (__glibc_unlikely (misaligned_chunk (tc_victim)))
3869	malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
3870	if (SINGLE_THREAD_P)
3871	*fb = REVEAL_PTR (tc_victim->fd);
3872	else
3873	{
3874	REMOVE_FB (fb, pp, tc_victim);
3875	if (__glibc_unlikely (tc_victim == NULL))
3876	break;
3877	}
3878	tcache_put (tc_victim, tc_idx);
3879	}
3880	}
3881	#endif
3882	void *p = chunk2mem (victim);
3883	alloc_perturb (p, bytes);
3884	return p;
3885	}
3886	}
3887	}
3888
3889	/*
3890	If a small request, check regular bin. Since these "smallbins"
3891	hold one size each, no searching within bins is necessary.
3892	(For a large request, we need to wait until unsorted chunks are
3893	processed to find best fit. But for small ones, fits are exact
3894	anyway, so we can check now, which is faster.)
3895	*/
3896
3897	if (in_smallbin_range (nb))
3898	{
3899	idx = smallbin_index (nb);
3900	bin = bin_at (av, idx);
3901
3902	if ((victim = last (bin)) != bin)
3903	{
3904	bck = victim->bk;
3905	if (__glibc_unlikely (bck->fd != victim))
3906	malloc_printerr ("malloc(): smallbin double linked list corrupted");
3907	set_inuse_bit_at_offset (victim, nb);
3908	bin->bk = bck;
3909	bck->fd = bin;
3910
3911	if (av != &main_arena)
3912	set_non_main_arena (victim);
3913	check_malloced_chunk (av, victim, nb);
3914	#if USE_TCACHE
3915	/ While we're here, if we see other chunks of the same size,*
3916	stash them in the tcache. /*
3917	size_t tc_idx = csize2tidx (nb);
3918	if (tcache && tc_idx < mp_.tcache_bins)
3919	{
3920	mchunkptr tc_victim;
3921
3922	/ While bin not empty and tcache not full, copy chunks over. /
3923	while (tcache->counts[tc_idx] < mp_.tcache_count
3924	&& (tc_victim = last (bin)) != bin)
3925	{
3926	if (tc_victim != `0`)
3927	{
3928	bck = tc_victim->bk;
3929	set_inuse_bit_at_offset (tc_victim, nb);
3930	if (av != &main_arena)
3931	set_non_main_arena (tc_victim);
3932	bin->bk = bck;
3933	bck->fd = bin;
3934
3935	tcache_put (tc_victim, tc_idx);
3936	}
3937	}
3938	}
3939	#endif
3940	void *p = chunk2mem (victim);
3941	alloc_perturb (p, bytes);
3942	return p;
3943	}
3944	}
3945
3946	/*
3947	If this is a large request, consolidate fastbins before continuing.
3948	While it might look excessive to kill all fastbins before
3949	even seeing if there is space available, this avoids
3950	fragmentation problems normally associated with fastbins.
3951	Also, in practice, programs tend to have runs of either small or
3952	large requests, but less often mixtures, so consolidation is not
3953	invoked all that often in most programs. And the programs that
3954	it is called frequently in otherwise tend to fragment.
3955	*/
3956
3957	else
3958	{
3959	idx = largebin_index (nb);
3960	if (atomic_load_relaxed (&av->have_fastchunks))
3961	malloc_consolidate (av);
3962	}
3963
3964	/*
3965	Process recently freed or remaindered chunks, taking one only if
3966	it is exact fit, or, if this a small request, the chunk is remainder from
3967	the most recent non-exact fit. Place other traversed chunks in
3968	bins. Note that this step is the only place in any routine where
3969	chunks are placed in bins.
3970
3971	The outer loop here is needed because we might not realize until
3972	near the end of malloc that we should have consolidated, so must
3973	do so and retry. This happens at most once, and only when we would
3974	otherwise need to expand memory to service a "small" request.
3975	*/
3976
3977	#if USE_TCACHE
3978	INTERNAL_SIZE_T tcache_nb = `0`;
3979	size_t tc_idx = csize2tidx (nb);
3980	if (tcache && tc_idx < mp_.tcache_bins)
3981	tcache_nb = nb;
3982	int return_cached = `0`;
3983
3984	tcache_unsorted_count = `0`;
3985	#endif
3986
3987	for (;; )
3988	{
3989	int iters = `0`;
3990	while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
3991	{
3992	bck = victim->bk;
3993	size = chunksize (victim);
3994	mchunkptr next = chunk_at_offset (victim, size);
3995
3996	if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
3997	\|\| __glibc_unlikely (size > av->system_mem))
3998	malloc_printerr ("malloc(): invalid size (unsorted)");
3999	if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
4000	\|\| __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
4001	malloc_printerr ("malloc(): invalid next size (unsorted)");
4002	if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
4003	malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)");
4004	if (__glibc_unlikely (bck->fd != victim)
4005	\|\| __glibc_unlikely (victim->fd != unsorted_chunks (av)))
4006	malloc_printerr ("malloc(): unsorted double linked list corrupted");
4007	if (__glibc_unlikely (prev_inuse (next)))
4008	malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)");
4009
4010	/*
4011	If a small request, try to use last remainder if it is the
4012	only chunk in unsorted bin. This helps promote locality for
4013	runs of consecutive small requests. This is the only
4014	exception to best-fit, and applies only when there is
4015	no exact fit for a small chunk.
4016	*/
4017
4018	if (in_smallbin_range (nb) &&
4019	bck == unsorted_chunks (av) &&
4020	victim == av->last_remainder &&
4021	(unsigned long) (size) > (unsigned long) (nb + MINSIZE))
4022	{
4023	/ split and reattach remainder /
4024	remainder_size = size - nb;
4025	remainder = chunk_at_offset (victim, nb);
4026	unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
4027	av->last_remainder = remainder;
4028	remainder->bk = remainder->fd = unsorted_chunks (av);
4029	if (!in_smallbin_range (remainder_size))
4030	{
4031	remainder->fd_nextsize = NULL;
4032	remainder->bk_nextsize = NULL;
4033	}
4034
4035	set_head (victim, nb \| PREV_INUSE \|
4036	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4037	set_head (remainder, remainder_size \| PREV_INUSE);
4038	set_foot (remainder, remainder_size);
4039
4040	check_malloced_chunk (av, victim, nb);
4041	void *p = chunk2mem (victim);
4042	alloc_perturb (p, bytes);
4043	return p;
4044	}
4045
4046	/ remove from unsorted list /
4047	if (__glibc_unlikely (bck->fd != victim))
4048	malloc_printerr ("malloc(): corrupted unsorted chunks 3");
4049	unsorted_chunks (av)->bk = bck;
4050	bck->fd = unsorted_chunks (av);
4051
4052	/ Take now instead of binning if exact fit /
4053
4054	if (size == nb)
4055	{
4056	set_inuse_bit_at_offset (victim, size);
4057	if (av != &main_arena)
4058	set_non_main_arena (victim);
4059	#if USE_TCACHE
4060	/ Fill cache first, return to user only if cache fills.*
4061	We may return one of these chunks later. /*
4062	if (tcache_nb
4063	&& tcache->counts[tc_idx] < mp_.tcache_count)
4064	{
4065	tcache_put (victim, tc_idx);
4066	return_cached = `1`;
4067	continue;
4068	}
4069	else
4070	{
4071	#endif
4072	check_malloced_chunk (av, victim, nb);
4073	void *p = chunk2mem (victim);
4074	alloc_perturb (p, bytes);
4075	return p;
4076	#if USE_TCACHE
4077	}
4078	#endif
4079	}
4080
4081	/ place chunk in bin /
4082
4083	if (in_smallbin_range (size))
4084	{
4085	victim_index = smallbin_index (size);
4086	bck = bin_at (av, victim_index);
4087	fwd = bck->fd;
4088	}
4089	else
4090	{
4091	victim_index = largebin_index (size);
4092	bck = bin_at (av, victim_index);
4093	fwd = bck->fd;
4094
4095	/ maintain large bins in sorted order /
4096	if (fwd != bck)
4097	{
4098	/ Or with inuse bit to speed comparisons /
4099	size \|= PREV_INUSE;
4100	/ if smaller than smallest, bypass loop below /
4101	assert (chunk_main_arena (bck->bk));
4102	if ((unsigned long) (size)
4103	< (unsigned long) chunksize_nomask (bck->bk))
4104	{
4105	fwd = bck;
4106	bck = bck->bk;
4107
4108	victim->fd_nextsize = fwd->fd;
4109	victim->bk_nextsize = fwd->fd->bk_nextsize;
4110	fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
4111	}
4112	else
4113	{
4114	assert (chunk_main_arena (fwd));
4115	while ((unsigned long) size < chunksize_nomask (fwd))
4116	{
4117	fwd = fwd->fd_nextsize;
4118	assert (chunk_main_arena (fwd));
4119	}
4120
4121	if ((unsigned long) size
4122	== (unsigned long) chunksize_nomask (fwd))
4123	/ Always insert in the second position. /
4124	fwd = fwd->fd;
4125	else
4126	{
4127	victim->fd_nextsize = fwd;
4128	victim->bk_nextsize = fwd->bk_nextsize;
4129	if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
4130	malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)");
4131	fwd->bk_nextsize = victim;
4132	victim->bk_nextsize->fd_nextsize = victim;
4133	}
4134	bck = fwd->bk;
4135	if (bck->fd != fwd)
4136	malloc_printerr ("malloc(): largebin double linked list corrupted (bk)");
4137	}
4138	}
4139	else
4140	victim->fd_nextsize = victim->bk_nextsize = victim;
4141	}
4142
4143	mark_bin (av, victim_index);
4144	victim->bk = bck;
4145	victim->fd = fwd;
4146	fwd->bk = victim;
4147	bck->fd = victim;
4148
4149	#if USE_TCACHE
4150	/ If we've processed as many chunks as we're allowed while*
4151	filling the cache, return one of the cached ones. /*
4152	++tcache_unsorted_count;
4153	if (return_cached
4154	&& mp_.tcache_unsorted_limit > `0`
4155	&& tcache_unsorted_count > mp_.tcache_unsorted_limit)
4156	{
4157	return tcache_get (tc_idx);
4158	}
4159	#endif
4160
4161	#define MAX_ITERS 10000
4162	if (++iters >= MAX_ITERS)
4163	break;
4164	}
4165
4166	#if USE_TCACHE
4167	/ If all the small chunks we found ended up cached, return one now. /
4168	if (return_cached)
4169	{
4170	return tcache_get (tc_idx);
4171	}
4172	#endif
4173
4174	/*
4175	If a large request, scan through the chunks of current bin in
4176	sorted order to find smallest that fits. Use the skip list for this.
4177	*/
4178
4179	if (!in_smallbin_range (nb))
4180	{
4181	bin = bin_at (av, idx);
4182
4183	/ skip scan if empty or largest chunk is too small /
4184	if ((victim = first (bin)) != bin
4185	&& (unsigned long) chunksize_nomask (victim)
4186	>= (unsigned long) (nb))
4187	{
4188	victim = victim->bk_nextsize;
4189	while (((unsigned long) (size = chunksize (victim)) <
4190	(unsigned long) (nb)))
4191	victim = victim->bk_nextsize;
4192
4193	/ Avoid removing the first entry for a size so that the skip*
4194	list does not have to be rerouted. /*
4195	if (victim != last (bin)
4196	&& chunksize_nomask (victim)
4197	== chunksize_nomask (victim->fd))
4198	victim = victim->fd;
4199
4200	remainder_size = size - nb;
4201	unlink_chunk (av, victim);
4202
4203	/ Exhaust /
4204	if (remainder_size < MINSIZE)
4205	{
4206	set_inuse_bit_at_offset (victim, size);
4207	if (av != &main_arena)
4208	set_non_main_arena (victim);
4209	}
4210	/ Split /
4211	else
4212	{
4213	remainder = chunk_at_offset (victim, nb);
4214	/ We cannot assume the unsorted list is empty and therefore*
4215	have to perform a complete insert here. /*
4216	bck = unsorted_chunks (av);
4217	fwd = bck->fd;
4218	if (__glibc_unlikely (fwd->bk != bck))
4219	malloc_printerr ("malloc(): corrupted unsorted chunks");
4220	remainder->bk = bck;
4221	remainder->fd = fwd;
4222	bck->fd = remainder;
4223	fwd->bk = remainder;
4224	if (!in_smallbin_range (remainder_size))
4225	{
4226	remainder->fd_nextsize = NULL;
4227	remainder->bk_nextsize = NULL;
4228	}
4229	set_head (victim, nb \| PREV_INUSE \|
4230	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4231	set_head (remainder, remainder_size \| PREV_INUSE);
4232	set_foot (remainder, remainder_size);
4233	}
4234	check_malloced_chunk (av, victim, nb);
4235	void *p = chunk2mem (victim);
4236	alloc_perturb (p, bytes);
4237	return p;
4238	}
4239	}
4240
4241	/*
4242	Search for a chunk by scanning bins, starting with next largest
4243	bin. This search is strictly by best-fit; i.e., the smallest
4244	(with ties going to approximately the least recently used) chunk
4245	that fits is selected.
4246
4247	The bitmap avoids needing to check that most blocks are nonempty.
4248	The particular case of skipping all bins during warm-up phases
4249	when no chunks have been returned yet is faster than it might look.
4250	*/
4251
4252	++idx;
4253	bin = bin_at (av, idx);
4254	block = idx2block (idx);
4255	map = av->binmap[block];
4256	bit = idx2bit (idx);
4257
4258	for (;; )
4259	{
4260	/ Skip rest of block if there are no more set bits in this block. /
4261	if (bit > map \|\| bit == `0`)
4262	{
4263	do
4264	{
4265	if (++block >= BINMAPSIZE) / out of bins /
4266	goto use_top;
4267	}
4268	while ((map = av->binmap[block]) == `0`);
4269
4270	bin = bin_at (av, (block << BINMAPSHIFT));
4271	bit = `1`;
4272	}
4273
4274	/ Advance to bin with set bit. There must be one. /
4275	while ((bit & map) == `0`)
4276	{
4277	bin = next_bin (bin);
4278	bit <<= `1`;
4279	assert (bit != `0`);
4280	}
4281
4282	/ Inspect the bin. It is likely to be non-empty /
4283	victim = last (bin);
4284
4285	/ If a false alarm (empty bin), clear the bit. /
4286	if (victim == bin)
4287	{
4288	av->binmap[block] = map &= ~bit; / Write through /
4289	bin = next_bin (bin);
4290	bit <<= `1`;
4291	}
4292
4293	else
4294	{
4295	size = chunksize (victim);
4296
4297	/ We know the first chunk in this bin is big enough to use. /
4298	assert ((unsigned long) (size) >= (unsigned long) (nb));
4299
4300	remainder_size = size - nb;
4301
4302	/ unlink /
4303	unlink_chunk (av, victim);
4304
4305	/ Exhaust /
4306	if (remainder_size < MINSIZE)
4307	{
4308	set_inuse_bit_at_offset (victim, size);
4309	if (av != &main_arena)
4310	set_non_main_arena (victim);
4311	}
4312
4313	/ Split /
4314	else
4315	{
4316	remainder = chunk_at_offset (victim, nb);
4317
4318	/ We cannot assume the unsorted list is empty and therefore*
4319	have to perform a complete insert here. /*
4320	bck = unsorted_chunks (av);
4321	fwd = bck->fd;
4322	if (__glibc_unlikely (fwd->bk != bck))
4323	malloc_printerr ("malloc(): corrupted unsorted chunks 2");
4324	remainder->bk = bck;
4325	remainder->fd = fwd;
4326	bck->fd = remainder;
4327	fwd->bk = remainder;
4328
4329	/ advertise as last remainder /
4330	if (in_smallbin_range (nb))
4331	av->last_remainder = remainder;
4332	if (!in_smallbin_range (remainder_size))
4333	{
4334	remainder->fd_nextsize = NULL;
4335	remainder->bk_nextsize = NULL;
4336	}
4337	set_head (victim, nb \| PREV_INUSE \|
4338	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4339	set_head (remainder, remainder_size \| PREV_INUSE);
4340	set_foot (remainder, remainder_size);
4341	}
4342	check_malloced_chunk (av, victim, nb);
4343	void *p = chunk2mem (victim);
4344	alloc_perturb (p, bytes);
4345	return p;
4346	}
4347	}
4348
4349	use_top:
4350	/*
4351	If large enough, split off the chunk bordering the end of memory
4352	(held in av->top). Note that this is in accord with the best-fit
4353	search rule. In effect, av->top is treated as larger (and thus
4354	less well fitting) than any other available chunk since it can
4355	be extended to be as large as necessary (up to system
4356	limitations).
4357
4358	We require that av->top always exists (i.e., has size >=
4359	MINSIZE) after initialization, so if it would otherwise be
4360	exhausted by current request, it is replenished. (The main
4361	reason for ensuring it exists is that we may need MINSIZE space
4362	to put in fenceposts in sysmalloc.)
4363	*/
4364
4365	victim = av->top;
4366	size = chunksize (victim);
4367
4368	if (__glibc_unlikely (size > av->system_mem))
4369	malloc_printerr ("malloc(): corrupted top size");
4370
4371	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
4372	{
4373	remainder_size = size - nb;
4374	remainder = chunk_at_offset (victim, nb);
4375	av->top = remainder;
4376	set_head (victim, nb \| PREV_INUSE \|
4377	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4378	set_head (remainder, remainder_size \| PREV_INUSE);
4379
4380	check_malloced_chunk (av, victim, nb);
4381	void *p = chunk2mem (victim);
4382	alloc_perturb (p, bytes);
4383	return p;
4384	}
4385
4386	/ When we are using atomic ops to free fast chunks we can get*
4387	here for all block sizes. /*
4388	else if (atomic_load_relaxed (&av->have_fastchunks))
4389	{
4390	malloc_consolidate (av);
4391	/ restore original bin index /
4392	if (in_smallbin_range (nb))
4393	idx = smallbin_index (nb);
4394	else
4395	idx = largebin_index (nb);
4396	}
4397
4398	/*
4399	Otherwise, relay to handle system-dependent cases
4400	*/
4401	else
4402	{
4403	void *p = sysmalloc (nb, av);
4404	if (p != NULL)
4405	alloc_perturb (p, bytes);
4406	return p;
4407	}
4408	}
4409	}
4410
4411	/*
4412	------------------------------ free ------------------------------
4413	*/
4414
4415	static void
4416	_int_free (mstate av, mchunkptr p, int have_lock)
4417	{
4418	INTERNAL_SIZE_T size; / its size /
4419	mfastbinptr fb; /* associated fastbin /
4420	mchunkptr nextchunk; / next contiguous chunk /
4421	INTERNAL_SIZE_T nextsize; / its size /
4422	int nextinuse; / true if nextchunk is used /
4423	INTERNAL_SIZE_T prevsize; / size of previous contiguous chunk /
4424	mchunkptr bck; / misc temp for linking /
4425	mchunkptr fwd; / misc temp for linking /
4426
4427	size = chunksize (p);
4428
4429	/ Little security check which won't hurt performance: the*
4430	allocator never wrapps around at the end of the address space.
4431	Therefore we can exclude some size values which might appear
4432	here by accident or by "design" from some intruder. /*
4433	if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, `0`)
4434	\|\| __builtin_expect (misaligned_chunk (p), `0`))
4435	malloc_printerr ("free(): invalid pointer");
4436	/ We know that each chunk is at least MINSIZE bytes in size or a*
4437	multiple of MALLOC_ALIGNMENT. /*
4438	if (__glibc_unlikely (size < MINSIZE \|\| !aligned_OK (size)))
4439	malloc_printerr ("free(): invalid size");
4440
4441	check_inuse_chunk(av, p);
4442
4443	#if USE_TCACHE
4444	{
4445	size_t tc_idx = csize2tidx (size);
4446	if (tcache != NULL && tc_idx < mp_.tcache_bins)
4447	{
4448	/ Check to see if it's already in the tcache. /
4449	tcache_entry e = (tcache_entry ) chunk2mem (p);
4450
4451	/ This test succeeds on double free. However, we don't 100%*
4452	trust it (it also matches random payload data at a 1 in
4453	2^<size_t> chance), so verify it's not an unlikely
4454	coincidence before aborting. /*
4455	if (__glibc_unlikely (e->key == tcache_key))
4456	{
4457	tcache_entry *tmp;
4458	size_t cnt = `0`;
4459	LIBC_PROBE (memory_tcache_double_free, `2`, e, tc_idx);
4460	for (tmp = tcache->entries[tc_idx];
4461	tmp;
4462	tmp = REVEAL_PTR (tmp->next), ++cnt)
4463	{
4464	if (cnt >= mp_.tcache_count)
4465	malloc_printerr ("free(): too many chunks detected in tcache");
4466	if (__glibc_unlikely (!aligned_OK (tmp)))
4467	malloc_printerr ("free(): unaligned chunk detected in tcache 2");
4468	if (tmp == e)
4469	malloc_printerr ("free(): double free detected in tcache 2");
4470	/ If we get here, it was a coincidence. We've wasted a*
4471	few cycles, but don't abort. /*
4472	}
4473	}
4474
4475	if (tcache->counts[tc_idx] < mp_.tcache_count)
4476	{
4477	tcache_put (p, tc_idx);
4478	return;
4479	}
4480	}
4481	}
4482	#endif
4483
4484	/*
4485	If eligible, place chunk on a fastbin so it can be found
4486	and used quickly in malloc.
4487	*/
4488
4489	if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
4490
4491	#if TRIM_FASTBINS
4492	/*
4493	If TRIM_FASTBINS set, don't place chunks
4494	bordering top into fastbins
4495	*/
4496	&& (chunk_at_offset(p, size) != av->top)
4497	#endif
4498	) {
4499
4500	if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
4501	<= CHUNK_HDR_SZ, `0`)
4502	\|\| __builtin_expect (chunksize (chunk_at_offset (p, size))
4503	>= av->system_mem, `0`))
4504	{
4505	bool fail = true;
4506	/ We might not have a lock at this point and concurrent modifications*
4507	of system_mem might result in a false positive. Redo the test after
4508	getting the lock. /*
4509	if (!have_lock)
4510	{
4511	__libc_lock_lock (av->mutex);
4512	fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
4513	\|\| chunksize (chunk_at_offset (p, size)) >= av->system_mem);
4514	__libc_lock_unlock (av->mutex);
4515	}
4516
4517	if (fail)
4518	malloc_printerr ("free(): invalid next size (fast)");
4519	}
4520
4521	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4522
4523	atomic_store_relaxed (&av->have_fastchunks, true);
4524	unsigned int idx = fastbin_index(size);
4525	fb = &fastbin (av, idx);
4526
4527	/ Atomically link P to its fastbin: P->FD = FB; FB = P; /
4528	mchunkptr old = *fb, old2;
4529
4530	if (SINGLE_THREAD_P)
4531	{
4532	/ Check that the top of the bin is not the record we are going to*
4533	add (i.e., double free). /*
4534	if (__builtin_expect (old == p, `0`))
4535	malloc_printerr ("double free or corruption (fasttop)");
4536	p->fd = PROTECT_PTR (&p->fd, old);
4537	*fb = p;
4538	}
4539	else
4540	do
4541	{
4542	/ Check that the top of the bin is not the record we are going to*
4543	add (i.e., double free). /*
4544	if (__builtin_expect (old == p, `0`))
4545	malloc_printerr ("double free or corruption (fasttop)");
4546	old2 = old;
4547	p->fd = PROTECT_PTR (&p->fd, old);
4548	}
4549	while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
4550	!= old2);
4551
4552	/ Check that size of fastbin chunk at the top is the same as*
4553	size of the chunk that we are adding. We can dereference OLD
4554	only if we have the lock, otherwise it might have already been
4555	allocated again. /*
4556	if (have_lock && old != NULL
4557	&& __builtin_expect (fastbin_index (chunksize (old)) != idx, `0`))
4558	malloc_printerr ("invalid fastbin entry (free)");
4559	}
4560
4561	/*
4562	Consolidate other non-mmapped chunks as they arrive.
4563	*/
4564
4565	else if (!chunk_is_mmapped(p)) {
4566
4567	/ If we're single-threaded, don't lock the arena. /
4568	if (SINGLE_THREAD_P)
4569	have_lock = true;
4570
4571	if (!have_lock)
4572	__libc_lock_lock (av->mutex);
4573
4574	nextchunk = chunk_at_offset(p, size);
4575
4576	/ Lightweight tests: check whether the block is already the*
4577	top block. /*
4578	if (__glibc_unlikely (p == av->top))
4579	malloc_printerr ("double free or corruption (top)");
4580	/ Or whether the next chunk is beyond the boundaries of the arena. /
4581	if (__builtin_expect (contiguous (av)
4582	&& (char *) nextchunk
4583	>= ((char *) av->top + chunksize(av->top)), `0`))
4584	malloc_printerr ("double free or corruption (out)");
4585	/ Or whether the block is actually not marked used. /
4586	if (__glibc_unlikely (!prev_inuse(nextchunk)))
4587	malloc_printerr ("double free or corruption (!prev)");
4588
4589	nextsize = chunksize(nextchunk);
4590	if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, `0`)
4591	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4592	malloc_printerr ("free(): invalid next size (normal)");
4593
4594	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4595
4596	/ consolidate backward /
4597	if (!prev_inuse(p)) {
4598	prevsize = prev_size (p);
4599	size += prevsize;
4600	p = chunk_at_offset(p, -((long) prevsize));
4601	if (__glibc_unlikely (chunksize(p) != prevsize))
4602	malloc_printerr ("corrupted size vs. prev_size while consolidating");
4603	unlink_chunk (av, p);
4604	}
4605
4606	if (nextchunk != av->top) {
4607	/ get and clear inuse bit /
4608	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4609
4610	/ consolidate forward /
4611	if (!nextinuse) {
4612	unlink_chunk (av, nextchunk);
4613	size += nextsize;
4614	} else
4615	clear_inuse_bit_at_offset(nextchunk, `0`);
4616
4617	/*
4618	Place the chunk in unsorted chunk list. Chunks are
4619	not placed into regular bins until after they have
4620	been given one chance to be used in malloc.
4621	*/
4622
4623	bck = unsorted_chunks(av);
4624	fwd = bck->fd;
4625	if (__glibc_unlikely (fwd->bk != bck))
4626	malloc_printerr ("free(): corrupted unsorted chunks");
4627	p->fd = fwd;
4628	p->bk = bck;
4629	if (!in_smallbin_range(size))
4630	{
4631	p->fd_nextsize = NULL;
4632	p->bk_nextsize = NULL;
4633	}
4634	bck->fd = p;
4635	fwd->bk = p;
4636
4637	set_head(p, size \| PREV_INUSE);
4638	set_foot(p, size);
4639
4640	check_free_chunk(av, p);
4641	}
4642
4643	/*
4644	If the chunk borders the current high end of memory,
4645	consolidate into top
4646	*/
4647
4648	else {
4649	size += nextsize;
4650	set_head(p, size \| PREV_INUSE);
4651	av->top = p;
4652	check_chunk(av, p);
4653	}
4654
4655	/*
4656	If freeing a large space, consolidate possibly-surrounding
4657	chunks. Then, if the total unused topmost memory exceeds trim
4658	threshold, ask malloc_trim to reduce top.
4659
4660	Unless max_fast is 0, we don't know if there are fastbins
4661	bordering top, so we cannot tell for sure whether threshold
4662	has been reached unless fastbins are consolidated. But we
4663	don't want to consolidate on each free. As a compromise,
4664	consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
4665	is reached.
4666	*/
4667
4668	if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
4669	if (atomic_load_relaxed (&av->have_fastchunks))
4670	malloc_consolidate(av);
4671
4672	if (av == &main_arena) {
4673	#ifndef MORECORE_CANNOT_TRIM
4674	if ((unsigned long)(chunksize(av->top)) >=
4675	(unsigned long)(mp_.trim_threshold))
4676	systrim(mp_.top_pad, av);
4677	#endif
4678	} else {
4679	/ Always try heap_trim(), even if the top chunk is not*
4680	large, because the corresponding heap might go away. /*
4681	heap_info *heap = heap_for_ptr(top(av));
4682
4683	assert(heap->ar_ptr == av);
4684	heap_trim(heap, mp_.top_pad);
4685	}
4686	}
4687
4688	if (!have_lock)
4689	__libc_lock_unlock (av->mutex);
4690	}
4691	/*
4692	If the chunk was allocated via mmap, release via munmap().
4693	*/
4694
4695	else {
4696	munmap_chunk (p);
4697	}
4698	}
4699
4700	/*
4701	------------------------- malloc_consolidate -------------------------
4702
4703	malloc_consolidate is a specialized version of free() that tears
4704	down chunks held in fastbins. Free itself cannot be used for this
4705	purpose since, among other things, it might place chunks back onto
4706	fastbins. So, instead, we need to use a minor variant of the same
4707	code.
4708	*/
4709
4710	static void malloc_consolidate(mstate av)
4711	{
4712	mfastbinptr* fb; / current fastbin being consolidated /
4713	mfastbinptr* maxfb; / last fastbin (for loop control) /
4714	mchunkptr p; / current chunk being consolidated /
4715	mchunkptr nextp; / next chunk to consolidate /
4716	mchunkptr unsorted_bin; / bin header /
4717	mchunkptr first_unsorted; / chunk to link to /
4718
4719	/ These have same use as in free() /
4720	mchunkptr nextchunk;
4721	INTERNAL_SIZE_T size;
4722	INTERNAL_SIZE_T nextsize;
4723	INTERNAL_SIZE_T prevsize;
4724	int nextinuse;
4725
4726	atomic_store_relaxed (&av->have_fastchunks, false);
4727
4728	unsorted_bin = unsorted_chunks(av);
4729
4730	/*
4731	Remove each chunk from fast bin and consolidate it, placing it
4732	then in unsorted bin. Among other reasons for doing this,
4733	placing in unsorted bin avoids needing to calculate actual bins
4734	until malloc is sure that chunks aren't immediately going to be
4735	reused anyway.
4736	*/
4737
4738	maxfb = &fastbin (av, NFASTBINS - `1`);
4739	fb = &fastbin (av, `0`);
4740	do {
4741	p = atomic_exchange_acq (fb, NULL);
4742	if (p != `0`) {
4743	do {
4744	{
4745	if (__glibc_unlikely (misaligned_chunk (p)))
4746	malloc_printerr ("malloc_consolidate(): "
4747	"unaligned fastbin chunk detected");
4748
4749	unsigned int idx = fastbin_index (chunksize (p));
4750	if ((&fastbin (av, idx)) != fb)
4751	malloc_printerr ("malloc_consolidate(): invalid chunk size");
4752	}
4753
4754	check_inuse_chunk(av, p);
4755	nextp = REVEAL_PTR (p->fd);
4756
4757	/ Slightly streamlined version of consolidation code in free() /
4758	size = chunksize (p);
4759	nextchunk = chunk_at_offset(p, size);
4760	nextsize = chunksize(nextchunk);
4761
4762	if (!prev_inuse(p)) {
4763	prevsize = prev_size (p);
4764	size += prevsize;
4765	p = chunk_at_offset(p, -((long) prevsize));
4766	if (__glibc_unlikely (chunksize(p) != prevsize))
4767	malloc_printerr ("corrupted size vs. prev_size in fastbins");
4768	unlink_chunk (av, p);
4769	}
4770
4771	if (nextchunk != av->top) {
4772	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4773
4774	if (!nextinuse) {
4775	size += nextsize;
4776	unlink_chunk (av, nextchunk);
4777	} else
4778	clear_inuse_bit_at_offset(nextchunk, `0`);
4779
4780	first_unsorted = unsorted_bin->fd;
4781	unsorted_bin->fd = p;
4782	first_unsorted->bk = p;
4783
4784	if (!in_smallbin_range (size)) {
4785	p->fd_nextsize = NULL;
4786	p->bk_nextsize = NULL;
4787	}
4788
4789	set_head(p, size \| PREV_INUSE);
4790	p->bk = unsorted_bin;
4791	p->fd = first_unsorted;
4792	set_foot(p, size);
4793	}
4794
4795	else {
4796	size += nextsize;
4797	set_head(p, size \| PREV_INUSE);
4798	av->top = p;
4799	}
4800
4801	} while ( (p = nextp) != `0`);
4802
4803	}
4804	} while (fb++ != maxfb);
4805	}
4806
4807	/*
4808	------------------------------ realloc ------------------------------
4809	*/
4810
4811	static void *
4812	_int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
4813	INTERNAL_SIZE_T nb)
4814	{
4815	mchunkptr newp; / chunk to return /
4816	INTERNAL_SIZE_T newsize; / its size /
4817	void* newmem; / corresponding user mem /
4818
4819	mchunkptr next; / next contiguous chunk after oldp /
4820
4821	mchunkptr remainder; / extra space at end of newp /
4822	unsigned long remainder_size; / its size /
4823
4824	/ oldmem size /
4825	if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, `0`)
4826	\|\| __builtin_expect (oldsize >= av->system_mem, `0`))
4827	malloc_printerr ("realloc(): invalid old size");
4828
4829	check_inuse_chunk (av, oldp);
4830
4831	/ All callers already filter out mmap'ed chunks. /
4832	assert (!chunk_is_mmapped (oldp));
4833
4834	next = chunk_at_offset (oldp, oldsize);
4835	INTERNAL_SIZE_T nextsize = chunksize (next);
4836	if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, `0`)
4837	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4838	malloc_printerr ("realloc(): invalid next size");
4839
4840	if ((unsigned long) (oldsize) >= (unsigned long) (nb))
4841	{
4842	/ already big enough; split below /
4843	newp = oldp;
4844	newsize = oldsize;
4845	}
4846
4847	else
4848	{
4849	/ Try to expand forward into top /
4850	if (next == av->top &&
4851	(unsigned long) (newsize = oldsize + nextsize) >=
4852	(unsigned long) (nb + MINSIZE))
4853	{
4854	set_head_size (oldp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4855	av->top = chunk_at_offset (oldp, nb);
4856	set_head (av->top, (newsize - nb) \| PREV_INUSE);
4857	check_inuse_chunk (av, oldp);
4858	return tag_new_usable (chunk2mem (oldp));
4859	}
4860
4861	/ Try to expand forward into next chunk; split off remainder below /
4862	else if (next != av->top &&
4863	!inuse (next) &&
4864	(unsigned long) (newsize = oldsize + nextsize) >=
4865	(unsigned long) (nb))
4866	{
4867	newp = oldp;
4868	unlink_chunk (av, next);
4869	}
4870
4871	/ allocate, copy, free /
4872	else
4873	{
4874	newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK);
4875	if (newmem == `0`)
4876	return `0`; / propagate failure /
4877
4878	newp = mem2chunk (newmem);
4879	newsize = chunksize (newp);
4880
4881	/*
4882	Avoid copy if newp is next chunk after oldp.
4883	*/
4884	if (newp == next)
4885	{
4886	newsize += oldsize;
4887	newp = oldp;
4888	}
4889	else
4890	{
4891	void *oldmem = chunk2mem (oldp);
4892	size_t sz = memsize (oldp);
4893	(void) tag_region (oldmem, sz);
4894	newmem = tag_new_usable (newmem);
4895	memcpy (newmem, oldmem, sz);
4896	_int_free (av, oldp, `1`);
4897	check_inuse_chunk (av, newp);
4898	return newmem;
4899	}
4900	}
4901	}
4902
4903	/ If possible, free extra space in old or extended chunk /
4904
4905	assert ((unsigned long) (newsize) >= (unsigned long) (nb));
4906
4907	remainder_size = newsize - nb;
4908
4909	if (remainder_size < MINSIZE) / not enough extra to split off /
4910	{
4911	set_head_size (newp, newsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4912	set_inuse_bit_at_offset (newp, newsize);
4913	}
4914	else / split remainder /
4915	{
4916	remainder = chunk_at_offset (newp, nb);
4917	/ Clear any user-space tags before writing the header. /
4918	remainder = tag_region (remainder, remainder_size);
4919	set_head_size (newp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4920	set_head (remainder, remainder_size \| PREV_INUSE \|
4921	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4922	/ Mark remainder as inuse so free() won't complain /
4923	set_inuse_bit_at_offset (remainder, remainder_size);
4924	_int_free (av, remainder, `1`);
4925	}
4926
4927	check_inuse_chunk (av, newp);
4928	return tag_new_usable (chunk2mem (newp));
4929	}
4930
4931	/*
4932	------------------------------ memalign ------------------------------
4933	*/
4934
4935	static void *
4936	_int_memalign (mstate av, size_t alignment, size_t bytes)
4937	{
4938	INTERNAL_SIZE_T nb; / padded request size /
4939	char m; /* memory returned by malloc call /
4940	mchunkptr p; / corresponding chunk /
4941	char brk; /* alignment point within p /
4942	mchunkptr newp; / chunk to return /
4943	INTERNAL_SIZE_T newsize; / its size /
4944	INTERNAL_SIZE_T leadsize; / leading space before alignment point /
4945	mchunkptr remainder; / spare room at end to split off /
4946	unsigned long remainder_size; / its size /
4947	INTERNAL_SIZE_T size;
4948
4949
4950
4951	nb = checked_request2size (bytes);
4952	if (nb == `0`)
4953	{
4954	__set_errno (ENOMEM);
4955	return NULL;
4956	}
4957
4958	/*
4959	Strategy: find a spot within that chunk that meets the alignment
4960	request, and then possibly free the leading and trailing space.
4961	*/
4962
4963	/ Call malloc with worst case padding to hit alignment. /
4964
4965	m = (char *) (_int_malloc (av, nb + alignment + MINSIZE));
4966
4967	if (m == `0`)
4968	return `0`; / propagate failure /
4969
4970	p = mem2chunk (m);
4971
4972	if ((((unsigned long) (m)) % alignment) != `0`) / misaligned /
4973
4974	{ /*
4975	Find an aligned spot inside chunk. Since we need to give back
4976	leading space in a chunk of at least MINSIZE, if the first
4977	calculation places us at a spot with less than MINSIZE leader,
4978	we can move to the next aligned spot -- we've allocated enough
4979	total room so that this is always possible.
4980	*/
4981	brk = (char ) mem2chunk (((unsigned* long) (m + alignment - `1`)) &
4982	- ((signed long) alignment));
4983	if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
4984	brk += alignment;
4985
4986	newp = (mchunkptr) brk;
4987	leadsize = brk - (char *) (p);
4988	newsize = chunksize (p) - leadsize;
4989
4990	/ For mmapped chunks, just adjust offset /
4991	if (chunk_is_mmapped (p))
4992	{
4993	set_prev_size (newp, prev_size (p) + leadsize);
4994	set_head (newp, newsize \| IS_MMAPPED);
4995	return chunk2mem (newp);
4996	}
4997
4998	/ Otherwise, give back leader, use the rest /
4999	set_head (newp, newsize \| PREV_INUSE \|
5000	(av != &main_arena ? NON_MAIN_ARENA : `0`));
5001	set_inuse_bit_at_offset (newp, newsize);
5002	set_head_size (p, leadsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
5003	_int_free (av, p, `1`);
5004	p = newp;
5005
5006	assert (newsize >= nb &&
5007	(((unsigned long) (chunk2mem (p))) % alignment) == `0`);
5008	}
5009
5010	/ Also give back spare room at the end /
5011	if (!chunk_is_mmapped (p))
5012	{
5013	size = chunksize (p);
5014	if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
5015	{
5016	remainder_size = size - nb;
5017	remainder = chunk_at_offset (p, nb);
5018	set_head (remainder, remainder_size \| PREV_INUSE \|
5019	(av != &main_arena ? NON_MAIN_ARENA : `0`));
5020	set_head_size (p, nb);
5021	_int_free (av, remainder, `1`);
5022	}
5023	}
5024
5025	check_inuse_chunk (av, p);
5026	return chunk2mem (p);
5027	}
5028
5029
5030	/*
5031	------------------------------ malloc_trim ------------------------------
5032	*/
5033
5034	static int
5035	mtrim (mstate av, size_t pad)
5036	{
5037	/ Ensure all blocks are consolidated. /
5038	malloc_consolidate (av);
5039
5040	const size_t ps = GLRO (dl_pagesize);
5041	int psindex = bin_index (ps);
5042	const size_t psm1 = ps - `1`;
5043
5044	int result = `0`;
5045	for (int i = `1`; i < NBINS; ++i)
5046	if (i == `1` \|\| i >= psindex)
5047	{
5048	mbinptr bin = bin_at (av, i);
5049
5050	for (mchunkptr p = last (bin); p != bin; p = p->bk)
5051	{
5052	INTERNAL_SIZE_T size = chunksize (p);
5053
5054	if (size > psm1 + sizeof (struct malloc_chunk))
5055	{
5056	/ See whether the chunk contains at least one unused page. /
5057	char paligned_mem = (char* *) (((uintptr_t) p
5058	+ sizeof (struct malloc_chunk)
5059	+ psm1) & ~psm1);
5060
5061	assert ((char ) chunk2mem (p) + `2` CHUNK_HDR_SZ
5062	<= paligned_mem);
5063	assert ((char *) p + size > paligned_mem);
5064
5065	/ This is the size we could potentially free. /
5066	size -= paligned_mem - (char *) p;
5067
5068	if (size > psm1)
5069	{
5070	#if MALLOC_DEBUG
5071	/ When debugging we simulate destroying the memory*
5072	content. /*
5073	memset (paligned_mem, `0x89`, size & ~psm1);
5074	#endif
5075	__madvise (paligned_mem, size & ~psm1, MADV_DONTNEED);
5076
5077	result = `1`;
5078	}
5079	}
5080	}
5081	}
5082
5083	#ifndef MORECORE_CANNOT_TRIM
5084	return result \| (av == &main_arena ? systrim (pad, av) : `0`);
5085
5086	#else
5087	return result;
5088	#endif
5089	}
5090
5091
5092	int
5093	__malloc_trim (size_t s)
5094	{
5095	int result = `0`;
5096
5097	if (!__malloc_initialized)
5098	ptmalloc_init ();
5099
5100	mstate ar_ptr = &main_arena;
5101	do
5102	{
5103	__libc_lock_lock (ar_ptr->mutex);
5104	result \|= mtrim (ar_ptr, s);
5105	__libc_lock_unlock (ar_ptr->mutex);
5106
5107	ar_ptr = ar_ptr->next;
5108	}
5109	while (ar_ptr != &main_arena);
5110
5111	return result;
5112	}
5113
5114
5115	/*
5116	------------------------- malloc_usable_size -------------------------
5117	*/
5118
5119	static size_t
5120	musable (void *mem)
5121	{
5122	mchunkptr p = mem2chunk (mem);
5123
5124	if (chunk_is_mmapped (p))
5125	return chunksize (p) - CHUNK_HDR_SZ;
5126	else if (inuse (p))
5127	return memsize (p);
5128
5129	return `0`;
5130	}
5131
5132	#if IS_IN (libc)
5133	size_t
5134	__malloc_usable_size (void *m)
5135	{
5136	if (m == NULL)
5137	return `0`;
5138	return musable (m);
5139	}
5140	#endif
5141
5142	/*
5143	------------------------------ mallinfo ------------------------------
5144	Accumulate malloc statistics for arena AV into M.
5145	*/
5146	static void
5147	int_mallinfo (mstate av, struct mallinfo2 *m)
5148	{
5149	size_t i;
5150	mbinptr b;
5151	mchunkptr p;
5152	INTERNAL_SIZE_T avail;
5153	INTERNAL_SIZE_T fastavail;
5154	int nblocks;
5155	int nfastblocks;
5156
5157	check_malloc_state (av);
5158
5159	/ Account for top /
5160	avail = chunksize (av->top);
5161	nblocks = `1`; / top always exists /
5162
5163	/ traverse fastbins /
5164	nfastblocks = `0`;
5165	fastavail = `0`;
5166
5167	for (i = `0`; i < NFASTBINS; ++i)
5168	{
5169	for (p = fastbin (av, i);
5170	p != `0`;
5171	p = REVEAL_PTR (p->fd))
5172	{
5173	if (__glibc_unlikely (misaligned_chunk (p)))
5174	malloc_printerr ("int_mallinfo(): "
5175	"unaligned fastbin chunk detected");
5176	++nfastblocks;
5177	fastavail += chunksize (p);
5178	}
5179	}
5180
5181	avail += fastavail;
5182
5183	/ traverse regular bins /
5184	for (i = `1`; i < NBINS; ++i)
5185	{
5186	b = bin_at (av, i);
5187	for (p = last (b); p != b; p = p->bk)
5188	{
5189	++nblocks;
5190	avail += chunksize (p);
5191	}
5192	}
5193
5194	m->smblks += nfastblocks;
5195	m->ordblks += nblocks;
5196	m->fordblks += avail;
5197	m->uordblks += av->system_mem - avail;
5198	m->arena += av->system_mem;
5199	m->fsmblks += fastavail;
5200	if (av == &main_arena)
5201	{
5202	m->hblks = mp_.n_mmaps;
5203	m->hblkhd = mp_.mmapped_mem;
5204	m->usmblks = `0`;
5205	m->keepcost = chunksize (av->top);
5206	}
5207	}
5208
5209
5210	struct mallinfo2
5211	__libc_mallinfo2 (void)
5212	{
5213	struct mallinfo2 m;
5214	mstate ar_ptr;
5215
5216	if (!__malloc_initialized)
5217	ptmalloc_init ();
5218
5219	memset (&m, `0`, sizeof (m));
5220	ar_ptr = &main_arena;
5221	do
5222	{
5223	__libc_lock_lock (ar_ptr->mutex);
5224	int_mallinfo (ar_ptr, &m);
5225	__libc_lock_unlock (ar_ptr->mutex);
5226
5227	ar_ptr = ar_ptr->next;
5228	}
5229	while (ar_ptr != &main_arena);
5230
5231	return m;
5232	}
5233	libc_hidden_def (__libc_mallinfo2)
5234
5235	struct mallinfo
5236	__libc_mallinfo (void)
5237	{
5238	struct mallinfo m;
5239	struct mallinfo2 m2 = __libc_mallinfo2 ();
5240
5241	m.arena = m2.arena;
5242	m.ordblks = m2.ordblks;
5243	m.smblks = m2.smblks;
5244	m.hblks = m2.hblks;
5245	m.hblkhd = m2.hblkhd;
5246	m.usmblks = m2.usmblks;
5247	m.fsmblks = m2.fsmblks;
5248	m.uordblks = m2.uordblks;
5249	m.fordblks = m2.fordblks;
5250	m.keepcost = m2.keepcost;
5251
5252	return m;
5253	}
5254
5255
5256	/*
5257	------------------------------ malloc_stats ------------------------------
5258	*/
5259
5260	void
5261	__malloc_stats (void)
5262	{
5263	int i;
5264	mstate ar_ptr;
5265	unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
5266
5267	if (!__malloc_initialized)
5268	ptmalloc_init ();
5269	_IO_flockfile (stderr);
5270	int old_flags2 = stderr->_flags2;
5271	stderr->_flags2 \|= _IO_FLAGS2_NOTCANCEL;
5272	for (i = `0`, ar_ptr = &main_arena;; i++)
5273	{
5274	struct mallinfo2 mi;
5275
5276	memset (&mi, `0`, sizeof (mi));
5277	__libc_lock_lock (ar_ptr->mutex);
5278	int_mallinfo (ar_ptr, &mi);
5279	fprintf (stderr, "Arena %d:\n", i);
5280	fprintf (stderr, "system bytes = %10u\n", (unsigned int) mi.arena);
5281	fprintf (stderr, "in use bytes = %10u\n", (unsigned int) mi.uordblks);
5282	#if MALLOC_DEBUG > 1
5283	if (i > `0`)
5284	dump_heap (heap_for_ptr (top (ar_ptr)));
5285	#endif
5286	system_b += mi.arena;
5287	in_use_b += mi.uordblks;
5288	__libc_lock_unlock (ar_ptr->mutex);
5289	ar_ptr = ar_ptr->next;
5290	if (ar_ptr == &main_arena)
5291	break;
5292	}
5293	fprintf (stderr, "Total (incl. mmap):\n");
5294	fprintf (stderr, "system bytes = %10u\n", system_b);
5295	fprintf (stderr, "in use bytes = %10u\n", in_use_b);
5296	fprintf (stderr, "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
5297	fprintf (stderr, "max mmap bytes = %10lu\n",
5298	(unsigned long) mp_.max_mmapped_mem);
5299	stderr->_flags2 = old_flags2;
5300	_IO_funlockfile (stderr);
5301	}
5302
5303
5304	/*
5305	------------------------------ mallopt ------------------------------
5306	*/
5307	static __always_inline int
5308	do_set_trim_threshold (size_t value)
5309	{
5310	LIBC_PROBE (memory_mallopt_trim_threshold, `3`, value, mp_.trim_threshold,
5311	mp_.no_dyn_threshold);
5312	mp_.trim_threshold = value;
5313	mp_.no_dyn_threshold = `1`;
5314	return `1`;
5315	}
5316
5317	static __always_inline int
5318	do_set_top_pad (size_t value)
5319	{
5320	LIBC_PROBE (memory_mallopt_top_pad, `3`, value, mp_.top_pad,
5321	mp_.no_dyn_threshold);
5322	mp_.top_pad = value;
5323	mp_.no_dyn_threshold = `1`;
5324	return `1`;
5325	}
5326
5327	static __always_inline int
5328	do_set_mmap_threshold (size_t value)
5329	{
5330	LIBC_PROBE (memory_mallopt_mmap_threshold, `3`, value, mp_.mmap_threshold,
5331	mp_.no_dyn_threshold);
5332	mp_.mmap_threshold = value;
5333	mp_.no_dyn_threshold = `1`;
5334	return `1`;
5335	}
5336
5337	static __always_inline int
5338	do_set_mmaps_max (int32_t value)
5339	{
5340	LIBC_PROBE (memory_mallopt_mmap_max, `3`, value, mp_.n_mmaps_max,
5341	mp_.no_dyn_threshold);
5342	mp_.n_mmaps_max = value;
5343	mp_.no_dyn_threshold = `1`;
5344	return `1`;
5345	}
5346
5347	static __always_inline int
5348	do_set_mallopt_check (int32_t value)
5349	{
5350	return `1`;
5351	}
5352
5353	static __always_inline int
5354	do_set_perturb_byte (int32_t value)
5355	{
5356	LIBC_PROBE (memory_mallopt_perturb, `2`, value, perturb_byte);
5357	perturb_byte = value;
5358	return `1`;
5359	}
5360
5361	static __always_inline int
5362	do_set_arena_test (size_t value)
5363	{
5364	LIBC_PROBE (memory_mallopt_arena_test, `2`, value, mp_.arena_test);
5365	mp_.arena_test = value;
5366	return `1`;
5367	}
5368
5369	static __always_inline int
5370	do_set_arena_max (size_t value)
5371	{
5372	LIBC_PROBE (memory_mallopt_arena_max, `2`, value, mp_.arena_max);
5373	mp_.arena_max = value;
5374	return `1`;
5375	}
5376
5377	#if USE_TCACHE
5378	static __always_inline int
5379	do_set_tcache_max (size_t value)
5380	{
5381	if (value <= MAX_TCACHE_SIZE)
5382	{
5383	LIBC_PROBE (memory_tunable_tcache_max_bytes, `2`, value, mp_.tcache_max_bytes);
5384	mp_.tcache_max_bytes = value;
5385	mp_.tcache_bins = csize2tidx (request2size(value)) + `1`;
5386	return `1`;
5387	}
5388	return `0`;
5389	}
5390
5391	static __always_inline int
5392	do_set_tcache_count (size_t value)
5393	{
5394	if (value <= MAX_TCACHE_COUNT)
5395	{
5396	LIBC_PROBE (memory_tunable_tcache_count, `2`, value, mp_.tcache_count);
5397	mp_.tcache_count = value;
5398	return `1`;
5399	}
5400	return `0`;
5401	}
5402
5403	static __always_inline int
5404	do_set_tcache_unsorted_limit (size_t value)
5405	{
5406	LIBC_PROBE (memory_tunable_tcache_unsorted_limit, `2`, value, mp_.tcache_unsorted_limit);
5407	mp_.tcache_unsorted_limit = value;
5408	return `1`;
5409	}
5410	#endif
5411
5412	static __always_inline int
5413	do_set_mxfast (size_t value)
5414	{
5415	if (value <= MAX_FAST_SIZE)
5416	{
5417	LIBC_PROBE (memory_mallopt_mxfast, `2`, value, get_max_fast ());
5418	set_max_fast (value);
5419	return `1`;
5420	}
5421	return `0`;
5422	}
5423
5424	#if HAVE_TUNABLES
5425	static __always_inline int
5426	do_set_hugetlb (size_t value)
5427	{
5428	if (value == `1`)
5429	{
5430	enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
5431	/*
5432	Only enable THP madvise usage if system does support it and
5433	has 'madvise' mode. Otherwise the madvise() call is wasteful.
5434	*/
5435	if (thp_mode == malloc_thp_mode_madvise)
5436	mp_.thp_pagesize = __malloc_default_thp_pagesize ();
5437	}
5438	else if (value >= `2`)
5439	__malloc_hugepage_config (value == `2` ? `0` : value, &mp_.hp_pagesize,
5440	&mp_.hp_flags);
5441	return `0`;
5442	}
5443	#endif
5444
5445	int
5446	__libc_mallopt (int param_number, int value)
5447	{
5448	mstate av = &main_arena;
5449	int res = `1`;
5450
5451	if (!__malloc_initialized)
5452	ptmalloc_init ();
5453	__libc_lock_lock (av->mutex);
5454
5455	LIBC_PROBE (memory_mallopt, `2`, param_number, value);
5456
5457	/ We must consolidate main arena before changing max_fast*
5458	(see definition of set_max_fast). /*
5459	malloc_consolidate (av);
5460
5461	/ Many of these helper functions take a size_t. We do not worry*
5462	about overflow here, because negative int values will wrap to
5463	very large size_t values and the helpers have sufficient range
5464	checking for such conversions. Many of these helpers are also
5465	used by the tunables macros in arena.c. /*
5466
5467	switch (param_number)
5468	{
5469	case M_MXFAST:
5470	res = do_set_mxfast (value);
5471	break;
5472
5473	case M_TRIM_THRESHOLD:
5474	res = do_set_trim_threshold (value);
5475	break;
5476
5477	case M_TOP_PAD:
5478	res = do_set_top_pad (value);
5479	break;
5480
5481	case M_MMAP_THRESHOLD:
5482	res = do_set_mmap_threshold (value);
5483	break;
5484
5485	case M_MMAP_MAX:
5486	res = do_set_mmaps_max (value);
5487	break;
5488
5489	case M_CHECK_ACTION:
5490	res = do_set_mallopt_check (value);
5491	break;
5492
5493	case M_PERTURB:
5494	res = do_set_perturb_byte (value);
5495	break;
5496
5497	case M_ARENA_TEST:
5498	if (value > `0`)
5499	res = do_set_arena_test (value);
5500	break;
5501
5502	case M_ARENA_MAX:
5503	if (value > `0`)
5504	res = do_set_arena_max (value);
5505	break;
5506	}
5507	__libc_lock_unlock (av->mutex);
5508	return res;
5509	}
5510	libc_hidden_def (__libc_mallopt)
5511
5512
5513	/*
5514	-------------------- Alternative MORECORE functions --------------------
5515	*/
5516
5517
5518	/*
5519	General Requirements for MORECORE.
5520
5521	The MORECORE function must have the following properties:
5522
5523	If MORECORE_CONTIGUOUS is false:
5524
5525	* MORECORE must allocate in multiples of pagesize. It will
5526	only be called with arguments that are multiples of pagesize.
5527
5528	* MORECORE(0) must return an address that is at least
5529	MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
5530
5531	else (i.e. If MORECORE_CONTIGUOUS is true):
5532
5533	* Consecutive calls to MORECORE with positive arguments
5534	return increasing addresses, indicating that space has been
5535	contiguously extended.
5536
5537	* MORECORE need not allocate in multiples of pagesize.
5538	Calls to MORECORE need not have args of multiples of pagesize.
5539
5540	* MORECORE need not page-align.
5541
5542	In either case:
5543
5544	* MORECORE may allocate more memory than requested. (Or even less,
5545	but this will generally result in a malloc failure.)
5546
5547	* MORECORE must not allocate memory when given argument zero, but
5548	instead return one past the end address of memory from previous
5549	nonzero call. This malloc does NOT call MORECORE(0)
5550	until at least one call with positive arguments is made, so
5551	the initial value returned is not important.
5552
5553	* Even though consecutive calls to MORECORE need not return contiguous
5554	addresses, it must be OK for malloc'ed chunks to span multiple
5555	regions in those cases where they do happen to be contiguous.
5556
5557	* MORECORE need not handle negative arguments -- it may instead
5558	just return MORECORE_FAILURE when given negative arguments.
5559	Negative arguments are always multiples of pagesize. MORECORE
5560	must not misinterpret negative args as large positive unsigned
5561	args. You can suppress all such calls from even occurring by defining
5562	MORECORE_CANNOT_TRIM,
5563
5564	There is some variation across systems about the type of the
5565	argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
5566	actually be size_t, because sbrk supports negative args, so it is
5567	normally the signed type of the same width as size_t (sometimes
5568	declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
5569	matter though. Internally, we use "long" as arguments, which should
5570	work across all reasonable possibilities.
5571
5572	Additionally, if MORECORE ever returns failure for a positive
5573	request, then mmap is used as a noncontiguous system allocator. This
5574	is a useful backup strategy for systems with holes in address spaces
5575	-- in this case sbrk cannot contiguously expand the heap, but mmap
5576	may be able to map noncontiguous space.
5577
5578	If you'd like mmap to ALWAYS be used, you can define MORECORE to be
5579	a function that always returns MORECORE_FAILURE.
5580
5581	If you are using this malloc with something other than sbrk (or its
5582	emulation) to supply memory regions, you probably want to set
5583	MORECORE_CONTIGUOUS as false. As an example, here is a custom
5584	allocator kindly contributed for pre-OSX macOS. It uses virtually
5585	but not necessarily physically contiguous non-paged memory (locked
5586	in, present and won't get swapped out). You can use it by
5587	uncommenting this section, adding some #includes, and setting up the
5588	appropriate defines above:
5589
5590	*#define MORECORE osMoreCore
5591	*#define MORECORE_CONTIGUOUS 0
5592
5593	There is also a shutdown routine that should somehow be called for
5594	cleanup upon program exit.
5595
5596	*#define MAX_POOL_ENTRIES 100
5597	#define MINIMUM_MORECORE_SIZE (64 1024)
5598	static int next_os_pool;
5599	void our_os_pools[MAX_POOL_ENTRIES];*
5600
5601	void osMoreCore(int size)*
5602	{
5603	void ptr = 0;*
5604	static void sbrk_top = 0;*
5605
5606	if (size > 0)
5607	{
5608	if (size < MINIMUM_MORECORE_SIZE)
5609	size = MINIMUM_MORECORE_SIZE;
5610	if (CurrentExecutionLevel() == kTaskLevel)
5611	ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
5612	if (ptr == 0)
5613	{
5614	return (void ) MORECORE_FAILURE;*
5615	}
5616	// save ptrs so they can be freed during cleanup
5617	our_os_pools[next_os_pool] = ptr;
5618	next_os_pool++;
5619	ptr = (void ) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);*
5620	sbrk_top = (char ) ptr + size;*
5621	return ptr;
5622	}
5623	else if (size < 0)
5624	{
5625	// we don't currently support shrink behavior
5626	return (void ) MORECORE_FAILURE;*
5627	}
5628	else
5629	{
5630	return sbrk_top;
5631	}
5632	}
5633
5634	// cleanup any allocated memory pools
5635	// called as last thing before shutting down driver
5636
5637	void osCleanupMem(void)
5638	{
5639	void ptr;
5640
5641	for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
5642	if (ptr)*
5643	{
5644	PoolDeallocate(ptr);*
5645	* ptr = 0;
5646	}
5647	}
5648
5649	*/
5650
5651
5652	/ Helper code. /
5653
5654	extern char **__libc_argv attribute_hidden;
5655
5656	static void
5657	malloc_printerr (const char *str)
5658	{
5659	#if IS_IN (libc)
5660	__libc_message (do_abort, "%s\n", str);
5661	#else
5662	__libc_fatal (str);
5663	#endif
5664	__builtin_unreachable ();
5665	}
5666
5667	#if IS_IN (libc)
5668	/ We need a wrapper function for one of the additions of POSIX. /
5669	int
5670	__posix_memalign (void **memptr, size_t alignment, size_t size)
5671	{
5672	void *mem;
5673
5674	if (!__malloc_initialized)
5675	ptmalloc_init ();
5676
5677	/ Test whether the SIZE argument is valid. It must be a power of*
5678	two multiple of sizeof (void ). /
5679	if (alignment % sizeof (void *) != `0`
5680	\|\| !powerof2 (alignment / sizeof (void *))
5681	\|\| alignment == `0`)
5682	return EINVAL;
5683
5684
5685	void *address = RETURN_ADDRESS (`0`);
5686	mem = _mid_memalign (alignment, size, address);
5687
5688	if (mem != NULL)
5689	{
5690	*memptr = mem;
5691	return `0`;
5692	}
5693
5694	return ENOMEM;
5695	}
5696	weak_alias (__posix_memalign, posix_memalign)
5697	#endif
5698
5699
5700	int
5701	__malloc_info (int options, FILE *fp)
5702	{
5703	/ For now, at least. /
5704	if (options != `0`)
5705	return EINVAL;
5706
5707	int n = `0`;
5708	size_t total_nblocks = `0`;
5709	size_t total_nfastblocks = `0`;
5710	size_t total_avail = `0`;
5711	size_t total_fastavail = `0`;
5712	size_t total_system = `0`;
5713	size_t total_max_system = `0`;
5714	size_t total_aspace = `0`;
5715	size_t total_aspace_mprotect = `0`;
5716
5717
5718
5719	if (!__malloc_initialized)
5720	ptmalloc_init ();
5721
5722	fputs ("<malloc version=\"1\">\n", fp);
5723
5724	/ Iterate over all arenas currently in use. /
5725	mstate ar_ptr = &main_arena;
5726	do
5727	{
5728	fprintf (fp, "<heap nr=\"%d\">\n<sizes>\n", n++);
5729
5730	size_t nblocks = `0`;
5731	size_t nfastblocks = `0`;
5732	size_t avail = `0`;
5733	size_t fastavail = `0`;
5734	struct
5735	{
5736	size_t from;
5737	size_t to;
5738	size_t total;
5739	size_t count;
5740	} sizes[NFASTBINS + NBINS - `1`];
5741	#define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5742
5743	__libc_lock_lock (ar_ptr->mutex);
5744
5745	/ Account for top chunk. The top-most available chunk is*
5746	treated specially and is never in any bin. See "initial_top"
5747	comments. /*
5748	avail = chunksize (ar_ptr->top);
5749	nblocks = `1`; / Top always exists. /
5750
5751	for (size_t i = `0`; i < NFASTBINS; ++i)
5752	{
5753	mchunkptr p = fastbin (ar_ptr, i);
5754	if (p != NULL)
5755	{
5756	size_t nthissize = `0`;
5757	size_t thissize = chunksize (p);
5758
5759	while (p != NULL)
5760	{
5761	if (__glibc_unlikely (misaligned_chunk (p)))
5762	malloc_printerr ("__malloc_info(): "
5763	"unaligned fastbin chunk detected");
5764	++nthissize;
5765	p = REVEAL_PTR (p->fd);
5766	}
5767
5768	fastavail += nthissize * thissize;
5769	nfastblocks += nthissize;
5770	sizes[i].from = thissize - (MALLOC_ALIGNMENT - `1`);
5771	sizes[i].to = thissize;
5772	sizes[i].count = nthissize;
5773	}
5774	else
5775	sizes[i].from = sizes[i].to = sizes[i].count = `0`;
5776
5777	sizes[i].total = sizes[i].count * sizes[i].to;
5778	}
5779
5780
5781	mbinptr bin;
5782	struct malloc_chunk *r;
5783
5784	for (size_t i = `1`; i < NBINS; ++i)
5785	{
5786	bin = bin_at (ar_ptr, i);
5787	r = bin->fd;
5788	sizes[NFASTBINS - `1` + i].from = ~((size_t) `0`);
5789	sizes[NFASTBINS - `1` + i].to = sizes[NFASTBINS - `1` + i].total
5790	= sizes[NFASTBINS - `1` + i].count = `0`;
5791
5792	if (r != NULL)
5793	while (r != bin)
5794	{
5795	size_t r_size = chunksize_nomask (r);
5796	++sizes[NFASTBINS - `1` + i].count;
5797	sizes[NFASTBINS - `1` + i].total += r_size;
5798	sizes[NFASTBINS - `1` + i].from
5799	= MIN (sizes[NFASTBINS - `1` + i].from, r_size);
5800	sizes[NFASTBINS - `1` + i].to = MAX (sizes[NFASTBINS - `1` + i].to,
5801	r_size);
5802
5803	r = r->fd;
5804	}
5805
5806	if (sizes[NFASTBINS - `1` + i].count == `0`)
5807	sizes[NFASTBINS - `1` + i].from = `0`;
5808	nblocks += sizes[NFASTBINS - `1` + i].count;
5809	avail += sizes[NFASTBINS - `1` + i].total;
5810	}
5811
5812	size_t heap_size = `0`;
5813	size_t heap_mprotect_size = `0`;
5814	size_t heap_count = `0`;
5815	if (ar_ptr != &main_arena)
5816	{
5817	/ Iterate over the arena heaps from back to front. /
5818	heap_info *heap = heap_for_ptr (top (ar_ptr));
5819	do
5820	{
5821	heap_size += heap->size;
5822	heap_mprotect_size += heap->mprotect_size;
5823	heap = heap->prev;
5824	++heap_count;
5825	}
5826	while (heap != NULL);
5827	}
5828
5829	__libc_lock_unlock (ar_ptr->mutex);
5830
5831	total_nfastblocks += nfastblocks;
5832	total_fastavail += fastavail;
5833
5834	total_nblocks += nblocks;
5835	total_avail += avail;
5836
5837	for (size_t i = `0`; i < nsizes; ++i)
5838	if (sizes[i].count != `0` && i != NFASTBINS)
5839	fprintf (fp, "\
5840	<size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5841	sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
5842
5843	if (sizes[NFASTBINS].count != `0`)
5844	fprintf (fp, "\
5845	<unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5846	sizes[NFASTBINS].from, sizes[NFASTBINS].to,
5847	sizes[NFASTBINS].total, sizes[NFASTBINS].count);
5848
5849	total_system += ar_ptr->system_mem;
5850	total_max_system += ar_ptr->max_system_mem;
5851
5852	fprintf (fp,
5853	"</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5854	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5855	"<system type=\"current\" size=\"%zu\"/>\n"
5856	"<system type=\"max\" size=\"%zu\"/>\n",
5857	nfastblocks, fastavail, nblocks, avail,
5858	ar_ptr->system_mem, ar_ptr->max_system_mem);
5859
5860	if (ar_ptr != &main_arena)
5861	{
5862	fprintf (fp,
5863	"<aspace type=\"total\" size=\"%zu\"/>\n"
5864	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5865	"<aspace type=\"subheaps\" size=\"%zu\"/>\n",
5866	heap_size, heap_mprotect_size, heap_count);
5867	total_aspace += heap_size;
5868	total_aspace_mprotect += heap_mprotect_size;
5869	}
5870	else
5871	{
5872	fprintf (fp,
5873	"<aspace type=\"total\" size=\"%zu\"/>\n"
5874	"<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5875	ar_ptr->system_mem, ar_ptr->system_mem);
5876	total_aspace += ar_ptr->system_mem;
5877	total_aspace_mprotect += ar_ptr->system_mem;
5878	}
5879
5880	fputs ("</heap>\n", fp);
5881	ar_ptr = ar_ptr->next;
5882	}
5883	while (ar_ptr != &main_arena);
5884
5885	fprintf (fp,
5886	"<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5887	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5888	"<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
5889	"<system type=\"current\" size=\"%zu\"/>\n"
5890	"<system type=\"max\" size=\"%zu\"/>\n"
5891	"<aspace type=\"total\" size=\"%zu\"/>\n"
5892	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5893	"</malloc>\n",
5894	total_nfastblocks, total_fastavail, total_nblocks, total_avail,
5895	mp_.n_mmaps, mp_.mmapped_mem,
5896	total_system, total_max_system,
5897	total_aspace, total_aspace_mprotect);
5898
5899	return `0`;
5900	}
5901	#if IS_IN (libc)
5902	weak_alias (__malloc_info, malloc_info)
5903
5904	strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5905	strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5906	strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5907	strong_alias (__libc_memalign, __memalign)
5908	weak_alias (__libc_memalign, memalign)
5909	strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5910	strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5911	strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5912	strong_alias (__libc_mallinfo, __mallinfo)
5913	weak_alias (__libc_mallinfo, mallinfo)
5914	strong_alias (__libc_mallinfo2, __mallinfo2)
5915	weak_alias (__libc_mallinfo2, mallinfo2)
5916	strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5917
5918	weak_alias (__malloc_stats, malloc_stats)
5919	weak_alias (__malloc_usable_size, malloc_usable_size)
5920	weak_alias (__malloc_trim, malloc_trim)
5921	#endif
5922
5923	#if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
5924	compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
5925	#endif
5926
5927	/ ------------------------------------------------------------*
5928	History:
5929
5930	[see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
5931
5932	*/
5933	/*
5934	* Local variables:
5935	* c-basic-offset: 2
5936	* End:
5937	*/
5938

Browse the source code of glibc/malloc/malloc.c