malloc.c source code [glibc/malloc/malloc.c]

1	/ Malloc implementation for multiple threads without lock contention.*
2	Copyright (C) 1996-2022 Free Software Foundation, Inc.
3	Copyright The GNU Toolchain Authors.
4	This file is part of the GNU C Library.
5
6	The GNU C Library is free software; you can redistribute it and/or
7	modify it under the terms of the GNU Lesser General Public License as
8	published by the Free Software Foundation; either version 2.1 of the
9	License, or (at your option) any later version.
10
11	The GNU C Library is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	Lesser General Public License for more details.
15
16	You should have received a copy of the GNU Lesser General Public
17	License along with the GNU C Library; see the file COPYING.LIB. If
18	not, see <https://www.gnu.org/licenses/>. /*
19
20	/*
21	This is a version (aka ptmalloc2) of malloc/free/realloc written by
22	Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
23
24	There have been substantial changes made after the integration into
25	glibc in all parts of the code. Do not look for much commonality
26	with the ptmalloc2 version.
27
28	* Version ptmalloc2-20011215
29	based on:
30	VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
31
32	* Quickstart
33
34	In order to compile this implementation, a Makefile is provided with
35	the ptmalloc2 distribution, which has pre-defined targets for some
36	popular systems (e.g. "make posix" for Posix threads). All that is
37	typically required with regard to compiler flags is the selection of
38	the thread package via defining one out of USE_PTHREADS, USE_THR or
39	USE_SPROC. Check the thread-m.h file for what effects this has.
40	Many/most systems will additionally require USE_TSD_DATA_HACK to be
41	defined, so this is the default for "make posix".
42
43	* Why use this malloc?
44
45	This is not the fastest, most space-conserving, most portable, or
46	most tunable malloc ever written. However it is among the fastest
47	while also being among the most space-conserving, portable and tunable.
48	Consistent balance across these factors results in a good general-purpose
49	allocator for malloc-intensive programs.
50
51	The main properties of the algorithms are:
52	* For large (>= 512 bytes) requests, it is a pure best-fit allocator,
53	with ties normally decided via FIFO (i.e. least recently used).
54	* For small (<= 64 bytes by default) requests, it is a caching
55	allocator, that maintains pools of quickly recycled chunks.
56	* In between, and for combinations of large and small requests, it does
57	the best it can trying to meet both goals at once.
58	* For very large requests (>= 128KB by default), it relies on system
59	memory mapping facilities, if supported.
60
61	For a longer but slightly out of date high-level description, see
62	http://gee.cs.oswego.edu/dl/html/malloc.html
63
64	You may already by default be using a C library containing a malloc
65	that is based on some version of this malloc (for example in
66	linux). You might still want to use the one in this file in order to
67	customize settings or to avoid overheads associated with library
68	versions.
69
70	* Contents, described in more detail in "description of public routines" below.
71
72	Standard (ANSI/SVID/...) functions:
73	malloc(size_t n);
74	calloc(size_t n_elements, size_t element_size);
75	free(void p);*
76	realloc(void p, size_t n);*
77	memalign(size_t alignment, size_t n);
78	valloc(size_t n);
79	mallinfo()
80	mallopt(int parameter_number, int parameter_value)
81
82	Additional functions:
83	independent_calloc(size_t n_elements, size_t size, void chunks[]);*
84	independent_comalloc(size_t n_elements, size_t sizes[], void chunks[]);*
85	pvalloc(size_t n);
86	malloc_trim(size_t pad);
87	malloc_usable_size(void p);*
88	malloc_stats();
89
90	* Vital statistics:
91
92	Supported pointer representation: 4 or 8 bytes
93	Supported size_t representation: 4 or 8 bytes
94	Note that size_t is allowed to be 4 bytes even if pointers are 8.
95	You can adjust this by defining INTERNAL_SIZE_T
96
97	Alignment: 2 sizeof(size_t) (default)*
98	(i.e., 8 byte alignment with 4byte size_t). This suffices for
99	nearly all current machines and C compilers. However, you can
100	define MALLOC_ALIGNMENT to be wider than this if necessary.
101
102	Minimum overhead per allocated chunk: 4 or 8 bytes
103	Each malloced chunk has a hidden word of overhead holding size
104	and status information.
105
106	Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
107	8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
108
109	When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
110	ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
111	needed; 4 (8) for a trailing size field and 8 (16) bytes for
112	free list pointers. Thus, the minimum allocatable size is
113	16/24/32 bytes.
114
115	Even a request for zero bytes (i.e., malloc(0)) returns a
116	pointer to something of the minimum allocatable size.
117
118	The maximum overhead wastage (i.e., number of extra bytes
119	allocated than were requested in malloc) is less than or equal
120	to the minimum size, except for requests >= mmap_threshold that
121	are serviced via mmap(), where the worst case wastage is 2 *
122	sizeof(size_t) bytes plus the remainder from a system page (the
123	minimal mmap unit); typically 4096 or 8192 bytes.
124
125	Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
126	8-byte size_t: 2^64 minus about two pages
127
128	It is assumed that (possibly signed) size_t values suffice to
129	represent chunk sizes. `Possibly signed' is due to the fact
130	that `size_t' may be defined on a system as either a signed or
131	an unsigned type. The ISO C standard says that it must be
132	unsigned, but a few systems are known not to adhere to this.
133	Additionally, even when size_t is unsigned, sbrk (which is by
134	default used to obtain memory from system) accepts signed
135	arguments, and may not be able to handle size_t-wide arguments
136	with negative sign bit. Generally, values that would
137	appear as negative after accounting for overhead and alignment
138	are supported only via mmap(), which does not have this
139	limitation.
140
141	Requests for sizes outside the allowed range will perform an optional
142	failure action and then return null. (Requests may also
143	also fail because a system is out of memory.)
144
145	Thread-safety: thread-safe
146
147	Compliance: I believe it is compliant with the 1997 Single Unix Specification
148	Also SVID/XPG, ANSI C, and probably others as well.
149
150	* Synopsis of compile-time options:
151
152	People have reported using previous versions of this malloc on all
153	versions of Unix, sometimes by tweaking some of the defines
154	below. It has been tested most extensively on Solaris and Linux.
155	People also report using it in stand-alone embedded systems.
156
157	The implementation is in straight, hand-tuned ANSI C. It is not
158	at all modular. (Sorry!) It uses a lot of macros. To be at all
159	usable, this code should be compiled using an optimizing compiler
160	(for example gcc -O3) that can simplify expressions and control
161	paths. (FAQ: some macros import variables as arguments rather than
162	declare locals because people reported that some debuggers
163	otherwise get confused.)
164
165	OPTION DEFAULT VALUE
166
167	Compilation Environment options:
168
169	HAVE_MREMAP 0
170
171	Changing default word sizes:
172
173	INTERNAL_SIZE_T size_t
174
175	Configuration and functionality options:
176
177	USE_PUBLIC_MALLOC_WRAPPERS NOT defined
178	USE_MALLOC_LOCK NOT defined
179	MALLOC_DEBUG NOT defined
180	REALLOC_ZERO_BYTES_FREES 1
181	TRIM_FASTBINS 0
182
183	Options for customizing MORECORE:
184
185	MORECORE sbrk
186	MORECORE_FAILURE -1
187	MORECORE_CONTIGUOUS 1
188	MORECORE_CANNOT_TRIM NOT defined
189	MORECORE_CLEARS 1
190	MMAP_AS_MORECORE_SIZE (1024 1024)*
191
192	Tuning options that are also dynamically changeable via mallopt:
193
194	DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
195	DEFAULT_TRIM_THRESHOLD 128 1024*
196	DEFAULT_TOP_PAD 0
197	DEFAULT_MMAP_THRESHOLD 128 1024*
198	DEFAULT_MMAP_MAX 65536
199
200	There are several other #defined constants and macros that you
201	probably don't want to touch unless you are extending or adapting malloc. /*
202
203	/*
204	void is the pointer type that malloc should say it returns*
205	*/
206
207	#ifndef void
208	#define void void
209	#endif /void/
210
211	#include <stddef.h> /* for size_t */
212	#include <stdlib.h> /* for getenv(), abort() */
213	#include <unistd.h> /* for __libc_enable_secure */
214
215	#include <atomic.h>
216	#include <_itoa.h>
217	#include <bits/wordsize.h>
218	#include <sys/sysinfo.h>
219
220	#include <ldsodefs.h>
221
222	#include <unistd.h>
223	#include <stdio.h> /* needed for malloc_stats */
224	#include <errno.h>
225	#include <assert.h>
226
227	#include <shlib-compat.h>
228
229	/ For uintptr_t. /
230	#include <stdint.h>
231
232	/ For va_arg, va_start, va_end. /
233	#include <stdarg.h>
234
235	/ For MIN, MAX, powerof2. /
236	#include <sys/param.h>
237
238	/ For ALIGN_UP et. al. /
239	#include <libc-pointer-arith.h>
240
241	/ For DIAG_PUSH/POP_NEEDS_COMMENT et al. /
242	#include <libc-diag.h>
243
244	/ For memory tagging. /
245	#include <libc-mtag.h>
246
247	#include <malloc/malloc-internal.h>
248
249	/ For SINGLE_THREAD_P. /
250	#include <sysdep-cancel.h>
251
252	#include <libc-internal.h>
253
254	/ For tcache double-free check. /
255	#include <random-bits.h>
256	#include <sys/random.h>
257
258	/*
259	Debugging:
260
261	Because freed chunks may be overwritten with bookkeeping fields, this
262	malloc will often die when freed memory is overwritten by user
263	programs. This can be very effective (albeit in an annoying way)
264	in helping track down dangling pointers.
265
266	If you compile with -DMALLOC_DEBUG, a number of assertion checks are
267	enabled that will catch more memory errors. You probably won't be
268	able to make much sense of the actual assertion errors, but they
269	should help you locate incorrectly overwritten memory. The checking
270	is fairly extensive, and will slow down execution
271	noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
272	will attempt to check every non-mmapped allocated and free chunk in
273	the course of computing the summmaries. (By nature, mmapped regions
274	cannot be checked very much automatically.)
275
276	Setting MALLOC_DEBUG may also be helpful if you are trying to modify
277	this code. The assertions in the check routines spell out in more
278	detail the assumptions and invariants underlying the algorithms.
279
280	Setting MALLOC_DEBUG does NOT provide an automated mechanism for
281	checking that all accesses to malloced memory stay within their
282	bounds. However, there are several add-ons and adaptations of this
283	or other mallocs available that do this.
284	*/
285
286	#ifndef MALLOC_DEBUG
287	#define MALLOC_DEBUG 0
288	#endif
289
290	#if IS_IN (libc)
291	#ifndef NDEBUG
292	# define __assert_fail(assertion, file, line, function) \
293	__malloc_assert(assertion, file, line, function)
294
295	extern const char *__progname;
296
297	static void
298	__malloc_assert (const char assertion, const* char file, unsigned* int line,
299	const char *function)
300	{
301	(void) __fxprintf (NULL, "%s%s%s:%u: %s%sAssertion `%s' failed.\n",
302	__progname, __progname[`0`] ? ": " : "",
303	file, line,
304	function ? function : "", function ? ": " : "",
305	assertion);
306	fflush (stderr);
307	abort ();
308	}
309	#endif
310	#endif
311
312	#if USE_TCACHE
313	/ We want 64 entries. This is an arbitrary limit, which tunables can reduce. /
314	# define TCACHE_MAX_BINS 64
315	# define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1)
316
317	/ Only used to pre-fill the tunables. /
318	# define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
319
320	/ When "x" is from chunksize(). /
321	# define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
322	/ When "x" is a user-provided size. /
323	# define usize2tidx(x) csize2tidx (request2size (x))
324
325	/ With rounding and alignment, the bins are...*
326	idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit)
327	idx 1 bytes 25..40 or 13..20
328	idx 2 bytes 41..56 or 21..28
329	etc. /*
330
331	/ This is another arbitrary limit, which tunables can change. Each*
332	tcache bin will hold at most this number of chunks. /*
333	# define TCACHE_FILL_COUNT 7
334
335	/ Maximum chunks in tcache bins for tunables. This value must fit the range*
336	of tcache->counts[] entries, else they may overflow. /*
337	# define MAX_TCACHE_COUNT UINT16_MAX
338	#endif
339
340	/ Safe-Linking:*
341	Use randomness from ASLR (mmap_base) to protect single-linked lists
342	of Fast-Bins and TCache. That is, mask the "next" pointers of the
343	lists' chunks, and also perform allocation alignment checks on them.
344	This mechanism reduces the risk of pointer hijacking, as was done with
345	Safe-Unlinking in the double-linked lists of Small-Bins.
346	It assumes a minimum page size of 4096 bytes (12 bits). Systems with
347	larger pages provide less entropy, although the pointer mangling
348	still works. /*
349	#define PROTECT_PTR(pos, ptr) \
350	((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
351	#define REVEAL_PTR(ptr) PROTECT_PTR (&ptr, ptr)
352
353	/*
354	The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
355	when p is nonnull. If the macro is nonzero, the realloc call returns NULL;
356	otherwise, the call returns what malloc (0) would. In either case,
357	p is freed. Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
358	implements common historical practice.
359
360	ISO C17 says the realloc call has implementation-defined behavior,
361	and it might not even free p.
362	*/
363
364	#ifndef REALLOC_ZERO_BYTES_FREES
365	#define REALLOC_ZERO_BYTES_FREES 1
366	#endif
367
368	/*
369	TRIM_FASTBINS controls whether free() of a very small chunk can
370	immediately lead to trimming. Setting to true (1) can reduce memory
371	footprint, but will almost always slow down programs that use a lot
372	of small chunks.
373
374	Define this only if you are willing to give up some speed to more
375	aggressively reduce system-level memory footprint when releasing
376	memory in programs that use many small chunks. You can get
377	essentially the same effect by setting MXFAST to 0, but this can
378	lead to even greater slowdowns in programs using many small chunks.
379	TRIM_FASTBINS is an in-between compile-time option, that disables
380	only those chunks bordering topmost memory from being placed in
381	fastbins.
382	*/
383
384	#ifndef TRIM_FASTBINS
385	#define TRIM_FASTBINS 0
386	#endif
387
388	/ Definition for getting more memory from the OS. /
389	#include "morecore.c"
390
391	#define MORECORE (*__glibc_morecore)
392	#define MORECORE_FAILURE 0
393
394	/ Memory tagging. /
395
396	/ Some systems support the concept of tagging (sometimes known as*
397	coloring) memory locations on a fine grained basis. Each memory
398	location is given a color (normally allocated randomly) and
399	pointers are also colored. When the pointer is dereferenced, the
400	pointer's color is checked against the memory's color and if they
401	differ the access is faulted (sometimes lazily).
402
403	We use this in glibc by maintaining a single color for the malloc
404	data structures that are interleaved with the user data and then
405	assigning separate colors for each block allocation handed out. In
406	this way simple buffer overruns will be rapidly detected. When
407	memory is freed, the memory is recolored back to the glibc default
408	so that simple use-after-free errors can also be detected.
409
410	If memory is reallocated the buffer is recolored even if the
411	address remains the same. This has a performance impact, but
412	guarantees that the old pointer cannot mistakenly be reused (code
413	that compares old against new will see a mismatch and will then
414	need to behave as though realloc moved the data to a new location).
415
416	Internal API for memory tagging support.
417
418	The aim is to keep the code for memory tagging support as close to
419	the normal APIs in glibc as possible, so that if tagging is not
420	enabled in the library, or is disabled at runtime then standard
421	operations can continue to be used. Support macros are used to do
422	this:
423
424	void tag_new_zero_region (void ptr, size_t size)
425
426	Allocates a new tag, colors the memory with that tag, zeros the
427	memory and returns a pointer that is correctly colored for that
428	location. The non-tagging version will simply call memset with 0.
429
430	void tag_region (void ptr, size_t size)
431
432	Color the region of memory pointed to by PTR and size SIZE with
433	the color of PTR. Returns the original pointer.
434
435	void tag_new_usable (void ptr)
436
437	Allocate a new random color and use it to color the user region of
438	a chunk; this may include data from the subsequent chunk's header
439	if tagging is sufficiently fine grained. Returns PTR suitably
440	recolored for accessing the memory there.
441
442	void tag_at (void ptr)
443
444	Read the current color of the memory at the address pointed to by
445	PTR (ignoring it's current color) and return PTR recolored to that
446	color. PTR must be valid address in all other respects. When
447	tagging is not enabled, it simply returns the original pointer.
448	*/
449
450	#ifdef USE_MTAG
451	static bool mtag_enabled = false;
452	static int mtag_mmap_flags = `0`;
453	#else
454	# define mtag_enabled false
455	# define mtag_mmap_flags 0
456	#endif
457
458	static __always_inline void *
459	tag_region (void *ptr, size_t size)
460	{
461	if (__glibc_unlikely (mtag_enabled))
462	return __libc_mtag_tag_region (ptr, size);
463	return ptr;
464	}
465
466	static __always_inline void *
467	tag_new_zero_region (void *ptr, size_t size)
468	{
469	if (__glibc_unlikely (mtag_enabled))
470	return __libc_mtag_tag_zero_region (__libc_mtag_new_tag (ptr), size);
471	return memset (ptr, `0`, size);
472	}
473
474	/ Defined later. /
475	static void *
476	tag_new_usable (void *ptr);
477
478	static __always_inline void *
479	tag_at (void *ptr)
480	{
481	if (__glibc_unlikely (mtag_enabled))
482	return __libc_mtag_address_get_tag (ptr);
483	return ptr;
484	}
485
486	#include <string.h>
487
488	/*
489	MORECORE-related declarations. By default, rely on sbrk
490	*/
491
492
493	/*
494	MORECORE is the name of the routine to call to obtain more memory
495	from the system. See below for general guidance on writing
496	alternative MORECORE functions, as well as a version for WIN32 and a
497	sample version for pre-OSX macos.
498	*/
499
500	#ifndef MORECORE
501	#define MORECORE sbrk
502	#endif
503
504	/*
505	MORECORE_FAILURE is the value returned upon failure of MORECORE
506	as well as mmap. Since it cannot be an otherwise valid memory address,
507	and must reflect values of standard sys calls, you probably ought not
508	try to redefine it.
509	*/
510
511	#ifndef MORECORE_FAILURE
512	#define MORECORE_FAILURE (-1)
513	#endif
514
515	/*
516	If MORECORE_CONTIGUOUS is true, take advantage of fact that
517	consecutive calls to MORECORE with positive arguments always return
518	contiguous increasing addresses. This is true of unix sbrk. Even
519	if not defined, when regions happen to be contiguous, malloc will
520	permit allocations spanning regions obtained from different
521	calls. But defining this when applicable enables some stronger
522	consistency checks and space efficiencies.
523	*/
524
525	#ifndef MORECORE_CONTIGUOUS
526	#define MORECORE_CONTIGUOUS 1
527	#endif
528
529	/*
530	Define MORECORE_CANNOT_TRIM if your version of MORECORE
531	cannot release space back to the system when given negative
532	arguments. This is generally necessary only if you are using
533	a hand-crafted MORECORE function that cannot handle negative arguments.
534	*/
535
536	/ #define MORECORE_CANNOT_TRIM /
537
538	/ MORECORE_CLEARS (default 1)*
539	The degree to which the routine mapped to MORECORE zeroes out
540	memory: never (0), only for newly allocated space (1) or always
541	(2). The distinction between (1) and (2) is necessary because on
542	some systems, if the application first decrements and then
543	increments the break value, the contents of the reallocated space
544	are unspecified.
545	*/
546
547	#ifndef MORECORE_CLEARS
548	# define MORECORE_CLEARS 1
549	#endif
550
551
552	/*
553	MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
554	sbrk fails, and mmap is used as a backup. The value must be a
555	multiple of page size. This backup strategy generally applies only
556	when systems have "holes" in address space, so sbrk cannot perform
557	contiguous expansion, but there is still space available on system.
558	On systems for which this is known to be useful (i.e. most linux
559	kernels), this occurs only when programs allocate huge amounts of
560	memory. Between this, and the fact that mmap regions tend to be
561	limited, the size should be large, to avoid too many mmap calls and
562	thus avoid running out of kernel resources. /*
563
564	#ifndef MMAP_AS_MORECORE_SIZE
565	#define MMAP_AS_MORECORE_SIZE (1024 * 1024)
566	#endif
567
568	/*
569	Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
570	large blocks.
571	*/
572
573	#ifndef HAVE_MREMAP
574	#define HAVE_MREMAP 0
575	#endif
576
577	/*
578	This version of malloc supports the standard SVID/XPG mallinfo
579	routine that returns a struct containing usage properties and
580	statistics. It should work on any SVID/XPG compliant system that has
581	a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
582	install such a thing yourself, cut out the preliminary declarations
583	as described above and below and save them in a malloc.h file. But
584	there's no compelling reason to bother to do this.)
585
586	The main declaration needed is the mallinfo struct that is returned
587	(by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
588	bunch of fields that are not even meaningful in this version of
589	malloc. These fields are are instead filled by mallinfo() with
590	other numbers that might be of interest.
591	*/
592
593
594	/ ---------- description of public routines ------------ /
595
596	#if IS_IN (libc)
597	/*
598	malloc(size_t n)
599	Returns a pointer to a newly allocated chunk of at least n bytes, or null
600	if no space is available. Additionally, on failure, errno is
601	set to ENOMEM on ANSI C systems.
602
603	If n is zero, malloc returns a minimum-sized chunk. (The minimum
604	size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
605	systems.) On most systems, size_t is an unsigned type, so calls
606	with negative arguments are interpreted as requests for huge amounts
607	of space, which will often fail. The maximum supported value of n
608	differs across systems, but is in all cases less than the maximum
609	representable value of a size_t.
610	*/
611	void* __libc_malloc(size_t);
612	libc_hidden_proto (__libc_malloc)
613
614	/*
615	free(void p)*
616	Releases the chunk of memory pointed to by p, that had been previously
617	allocated using malloc or a related routine such as realloc.
618	It has no effect if p is null. It can have arbitrary (i.e., bad!)
619	effects if p has already been freed.
620
621	Unless disabled (using mallopt), freeing very large spaces will
622	when possible, automatically trigger operations that give
623	back unused memory to the system, thus reducing program footprint.
624	*/
625	void __libc_free(void*);
626	libc_hidden_proto (__libc_free)
627
628	/*
629	calloc(size_t n_elements, size_t element_size);
630	Returns a pointer to n_elements element_size bytes, with all locations*
631	set to zero.
632	*/
633	void* __libc_calloc(size_t, size_t);
634
635	/*
636	realloc(void p, size_t n)*
637	Returns a pointer to a chunk of size n that contains the same data
638	as does chunk p up to the minimum of (n, p's size) bytes, or null
639	if no space is available.
640
641	The returned pointer may or may not be the same as p. The algorithm
642	prefers extending p when possible, otherwise it employs the
643	equivalent of a malloc-copy-free sequence.
644
645	If p is null, realloc is equivalent to malloc.
646
647	If space is not available, realloc returns null, errno is set (if on
648	ANSI) and p is NOT freed.
649
650	if n is for fewer bytes than already held by p, the newly unused
651	space is lopped off and freed if possible. Unless the #define
652	REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
653	zero (re)allocates a minimum-sized chunk.
654
655	Large chunks that were internally obtained via mmap will always be
656	grown using malloc-copy-free sequences unless the system supports
657	MREMAP (currently only linux).
658
659	The old unix realloc convention of allowing the last-free'd chunk
660	to be used as an argument to realloc is not supported.
661	*/
662	void* __libc_realloc(void*, size_t);
663	libc_hidden_proto (__libc_realloc)
664
665	/*
666	memalign(size_t alignment, size_t n);
667	Returns a pointer to a newly allocated chunk of n bytes, aligned
668	in accord with the alignment argument.
669
670	The alignment argument should be a power of two. If the argument is
671	not a power of two, the nearest greater power is used.
672	8-byte alignment is guaranteed by normal malloc calls, so don't
673	bother calling memalign with an argument of 8 or less.
674
675	Overreliance on memalign is a sure way to fragment space.
676	*/
677	void* __libc_memalign(size_t, size_t);
678	libc_hidden_proto (__libc_memalign)
679
680	/*
681	valloc(size_t n);
682	Equivalent to memalign(pagesize, n), where pagesize is the page
683	size of the system. If the pagesize is unknown, 4096 is used.
684	*/
685	void* __libc_valloc(size_t);
686
687
688
689	/*
690	mallinfo()
691	Returns (by copy) a struct containing various summary statistics:
692
693	arena: current total non-mmapped bytes allocated from system
694	ordblks: the number of free chunks
695	smblks: the number of fastbin blocks (i.e., small chunks that
696	have been freed but not use resused or consolidated)
697	hblks: current number of mmapped regions
698	hblkhd: total bytes held in mmapped regions
699	usmblks: always 0
700	fsmblks: total bytes held in fastbin blocks
701	uordblks: current total allocated space (normal or mmapped)
702	fordblks: total free space
703	keepcost: the maximum number of bytes that could ideally be released
704	back to system via malloc_trim. ("ideally" means that
705	it ignores page restrictions etc.)
706
707	Because these fields are ints, but internal bookkeeping may
708	be kept as longs, the reported values may wrap around zero and
709	thus be inaccurate.
710	*/
711	struct mallinfo2 __libc_mallinfo2(void);
712	libc_hidden_proto (__libc_mallinfo2)
713
714	struct mallinfo __libc_mallinfo(void);
715
716
717	/*
718	pvalloc(size_t n);
719	Equivalent to valloc(minimum-page-that-holds(n)), that is,
720	round up n to nearest pagesize.
721	*/
722	void* __libc_pvalloc(size_t);
723
724	/*
725	malloc_trim(size_t pad);
726
727	If possible, gives memory back to the system (via negative
728	arguments to sbrk) if there is unused memory at the `high' end of
729	the malloc pool. You can call this after freeing large blocks of
730	memory to potentially reduce the system-level memory requirements
731	of a program. However, it cannot guarantee to reduce memory. Under
732	some allocation patterns, some large free blocks of memory will be
733	locked between two used chunks, so they cannot be given back to
734	the system.
735
736	The `pad' argument to malloc_trim represents the amount of free
737	trailing space to leave untrimmed. If this argument is zero,
738	only the minimum amount of memory to maintain internal data
739	structures will be left (one page or less). Non-zero arguments
740	can be supplied to maintain enough trailing space to service
741	future expected allocations without having to re-obtain memory
742	from the system.
743
744	Malloc_trim returns 1 if it actually released any memory, else 0.
745	On systems that do not support "negative sbrks", it will always
746	return 0.
747	*/
748	int __malloc_trim(size_t);
749
750	/*
751	malloc_usable_size(void p);*
752
753	Returns the number of bytes you can actually use in
754	an allocated chunk, which may be more than you requested (although
755	often not) due to alignment and minimum size constraints.
756	You can use this many bytes without worrying about
757	overwriting other allocated objects. This is not a particularly great
758	programming practice. malloc_usable_size can be more useful in
759	debugging and assertions, for example:
760
761	p = malloc(n);
762	assert(malloc_usable_size(p) >= 256);
763
764	*/
765	size_t __malloc_usable_size(void*);
766
767	/*
768	malloc_stats();
769	Prints on stderr the amount of space obtained from the system (both
770	via sbrk and mmap), the maximum amount (which may be more than
771	current if malloc_trim and/or munmap got called), and the current
772	number of bytes allocated via malloc (or realloc, etc) but not yet
773	freed. Note that this is the number of bytes allocated, not the
774	number requested. It will be larger than the number requested
775	because of alignment and bookkeeping overhead. Because it includes
776	alignment wastage as being in use, this figure may be greater than
777	zero even when no user-level chunks are allocated.
778
779	The reported current and maximum system memory can be inaccurate if
780	a program makes other calls to system memory allocation functions
781	(normally sbrk) outside of malloc.
782
783	malloc_stats prints only the most commonly interesting statistics.
784	More information can be obtained by calling mallinfo.
785
786	*/
787	void __malloc_stats(void);
788
789	/*
790	posix_memalign(void memptr, size_t alignment, size_t size);
791
792	POSIX wrapper like memalign(), checking for validity of size.
793	*/
794	int __posix_memalign(void **, size_t, size_t);
795	#endif /* IS_IN (libc) */
796
797	/*
798	mallopt(int parameter_number, int parameter_value)
799	Sets tunable parameters The format is to provide a
800	(parameter-number, parameter-value) pair. mallopt then sets the
801	corresponding parameter to the argument value if it can (i.e., so
802	long as the value is meaningful), and returns 1 if successful else
803	0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
804	normally defined in malloc.h. Only one of these (M_MXFAST) is used
805	in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
806	so setting them has no effect. But this malloc also supports four
807	other options in mallopt. See below for details. Briefly, supported
808	parameters are as follows (listed defaults are for "typical"
809	configurations).
810
811	Symbol param # default allowed param values
812	M_MXFAST 1 64 0-80 (0 disables fastbins)
813	M_TRIM_THRESHOLD -1 1281024 any (-1U disables trimming)*
814	M_TOP_PAD -2 0 any
815	M_MMAP_THRESHOLD -3 1281024 any (or 0 if no MMAP support)*
816	M_MMAP_MAX -4 65536 any (0 disables use of mmap)
817	*/
818	int __libc_mallopt(int, int);
819	#if IS_IN (libc)
820	libc_hidden_proto (__libc_mallopt)
821	#endif
822
823	/ mallopt tuning options /
824
825	/*
826	M_MXFAST is the maximum request size used for "fastbins", special bins
827	that hold returned chunks without consolidating their spaces. This
828	enables future requests for chunks of the same size to be handled
829	very quickly, but can increase fragmentation, and thus increase the
830	overall memory footprint of a program.
831
832	This malloc manages fastbins very conservatively yet still
833	efficiently, so fragmentation is rarely a problem for values less
834	than or equal to the default. The maximum supported value of MXFAST
835	is 80. You wouldn't want it any higher than this anyway. Fastbins
836	are designed especially for use with many small structs, objects or
837	strings -- the default handles structs/objects/arrays with sizes up
838	to 8 4byte fields, or small strings representing words, tokens,
839	etc. Using fastbins for larger objects normally worsens
840	fragmentation without improving speed.
841
842	M_MXFAST is set in REQUEST size units. It is internally used in
843	chunksize units, which adds padding and alignment. You can reduce
844	M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
845	algorithm to be a closer approximation of fifo-best-fit in all cases,
846	not just for larger requests, but will generally cause it to be
847	slower.
848	*/
849
850
851	/ M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h /
852	#ifndef M_MXFAST
853	#define M_MXFAST 1
854	#endif
855
856	#ifndef DEFAULT_MXFAST
857	#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
858	#endif
859
860
861	/*
862	M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
863	to keep before releasing via malloc_trim in free().
864
865	Automatic trimming is mainly useful in long-lived programs.
866	Because trimming via sbrk can be slow on some systems, and can
867	sometimes be wasteful (in cases where programs immediately
868	afterward allocate more large chunks) the value should be high
869	enough so that your overall system performance would improve by
870	releasing this much memory.
871
872	The trim threshold and the mmap control parameters (see below)
873	can be traded off with one another. Trimming and mmapping are
874	two different ways of releasing unused memory back to the
875	system. Between these two, it is often possible to keep
876	system-level demands of a long-lived program down to a bare
877	minimum. For example, in one test suite of sessions measuring
878	the XF86 X server on Linux, using a trim threshold of 128K and a
879	mmap threshold of 192K led to near-minimal long term resource
880	consumption.
881
882	If you are using this malloc in a long-lived program, it should
883	pay to experiment with these values. As a rough guide, you
884	might set to a value close to the average size of a process
885	(program) running on your system. Releasing this much memory
886	would allow such a process to run in memory. Generally, it's
887	worth it to tune for trimming rather tham memory mapping when a
888	program undergoes phases where several large chunks are
889	allocated and released in ways that can reuse each other's
890	storage, perhaps mixed with phases where there are no such
891	chunks at all. And in well-behaved long-lived programs,
892	controlling release of large blocks via trimming versus mapping
893	is usually faster.
894
895	However, in most programs, these parameters serve mainly as
896	protection against the system-level effects of carrying around
897	massive amounts of unneeded memory. Since frequent calls to
898	sbrk, mmap, and munmap otherwise degrade performance, the default
899	parameters are set to relatively high values that serve only as
900	safeguards.
901
902	The trim value It must be greater than page size to have any useful
903	effect. To disable trimming completely, you can set to
904	(unsigned long)(-1)
905
906	Trim settings interact with fastbin (MXFAST) settings: Unless
907	TRIM_FASTBINS is defined, automatic trimming never takes place upon
908	freeing a chunk with size less than or equal to MXFAST. Trimming is
909	instead delayed until subsequent freeing of larger chunks. However,
910	you can still force an attempted trim by calling malloc_trim.
911
912	Also, trimming is not generally possible in cases where
913	the main arena is obtained via mmap.
914
915	Note that the trick some people use of mallocing a huge space and
916	then freeing it at program startup, in an attempt to reserve system
917	memory, doesn't have the intended effect under automatic trimming,
918	since that memory will immediately be returned to the system.
919	*/
920
921	#define M_TRIM_THRESHOLD -1
922
923	#ifndef DEFAULT_TRIM_THRESHOLD
924	#define DEFAULT_TRIM_THRESHOLD (128 * 1024)
925	#endif
926
927	/*
928	M_TOP_PAD is the amount of extra `padding' space to allocate or
929	retain whenever sbrk is called. It is used in two ways internally:
930
931	* When sbrk is called to extend the top of the arena to satisfy
932	a new malloc request, this much padding is added to the sbrk
933	request.
934
935	* When malloc_trim is called automatically from free(),
936	it is used as the `pad' argument.
937
938	In both cases, the actual amount of padding is rounded
939	so that the end of the arena is always a system page boundary.
940
941	The main reason for using padding is to avoid calling sbrk so
942	often. Having even a small pad greatly reduces the likelihood
943	that nearly every malloc request during program start-up (or
944	after trimming) will invoke sbrk, which needlessly wastes
945	time.
946
947	Automatic rounding-up to page-size units is normally sufficient
948	to avoid measurable overhead, so the default is 0. However, in
949	systems where sbrk is relatively slow, it can pay to increase
950	this value, at the expense of carrying around more memory than
951	the program needs.
952	*/
953
954	#define M_TOP_PAD -2
955
956	#ifndef DEFAULT_TOP_PAD
957	#define DEFAULT_TOP_PAD (0)
958	#endif
959
960	/*
961	MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
962	adjusted MMAP_THRESHOLD.
963	*/
964
965	#ifndef DEFAULT_MMAP_THRESHOLD_MIN
966	#define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
967	#endif
968
969	#ifndef DEFAULT_MMAP_THRESHOLD_MAX
970	/ For 32-bit platforms we cannot increase the maximum mmap*
971	threshold much because it is also the minimum value for the
972	maximum heap size and its alignment. Going above 512k (i.e., 1M
973	for new heaps) wastes too much address space. /*
974	# if __WORDSIZE == 32
975	# define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
976	# else
977	# define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
978	# endif
979	#endif
980
981	/*
982	M_MMAP_THRESHOLD is the request size threshold for using mmap()
983	to service a request. Requests of at least this size that cannot
984	be allocated using already-existing space will be serviced via mmap.
985	(If enough normal freed space already exists it is used instead.)
986
987	Using mmap segregates relatively large chunks of memory so that
988	they can be individually obtained and released from the host
989	system. A request serviced through mmap is never reused by any
990	other request (at least not directly; the system may just so
991	happen to remap successive requests to the same locations).
992
993	Segregating space in this way has the benefits that:
994
995	1. Mmapped space can ALWAYS be individually released back
996	to the system, which helps keep the system level memory
997	demands of a long-lived program low.
998	2. Mapped memory can never become `locked' between
999	other chunks, as can happen with normally allocated chunks, which
1000	means that even trimming via malloc_trim would not release them.
1001	3. On some systems with "holes" in address spaces, mmap can obtain
1002	memory that sbrk cannot.
1003
1004	However, it has the disadvantages that:
1005
1006	1. The space cannot be reclaimed, consolidated, and then
1007	used to service later requests, as happens with normal chunks.
1008	2. It can lead to more wastage because of mmap page alignment
1009	requirements
1010	3. It causes malloc performance to be more dependent on host
1011	system memory management support routines which may vary in
1012	implementation quality and may impose arbitrary
1013	limitations. Generally, servicing a request via normal
1014	malloc steps is faster than going through a system's mmap.
1015
1016	The advantages of mmap nearly always outweigh disadvantages for
1017	"large" chunks, but the value of "large" varies across systems. The
1018	default is an empirically derived value that works well in most
1019	systems.
1020
1021
1022	Update in 2006:
1023	The above was written in 2001. Since then the world has changed a lot.
1024	Memory got bigger. Applications got bigger. The virtual address space
1025	layout in 32 bit linux changed.
1026
1027	In the new situation, brk() and mmap space is shared and there are no
1028	artificial limits on brk size imposed by the kernel. What is more,
1029	applications have started using transient allocations larger than the
1030	128Kb as was imagined in 2001.
1031
1032	The price for mmap is also high now; each time glibc mmaps from the
1033	kernel, the kernel is forced to zero out the memory it gives to the
1034	application. Zeroing memory is expensive and eats a lot of cache and
1035	memory bandwidth. This has nothing to do with the efficiency of the
1036	virtual memory system, by doing mmap the kernel just has no choice but
1037	to zero.
1038
1039	In 2001, the kernel had a maximum size for brk() which was about 800
1040	megabytes on 32 bit x86, at that point brk() would hit the first
1041	mmaped shared libaries and couldn't expand anymore. With current 2.6
1042	kernels, the VA space layout is different and brk() and mmap
1043	both can span the entire heap at will.
1044
1045	Rather than using a static threshold for the brk/mmap tradeoff,
1046	we are now using a simple dynamic one. The goal is still to avoid
1047	fragmentation. The old goals we kept are
1048	1) try to get the long lived large allocations to use mmap()
1049	2) really large allocations should always use mmap()
1050	and we're adding now:
1051	3) transient allocations should use brk() to avoid forcing the kernel
1052	having to zero memory over and over again
1053
1054	The implementation works with a sliding threshold, which is by default
1055	limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
1056	out at 128Kb as per the 2001 default.
1057
1058	This allows us to satisfy requirement 1) under the assumption that long
1059	lived allocations are made early in the process' lifespan, before it has
1060	started doing dynamic allocations of the same size (which will
1061	increase the threshold).
1062
1063	The upperbound on the threshold satisfies requirement 2)
1064
1065	The threshold goes up in value when the application frees memory that was
1066	allocated with the mmap allocator. The idea is that once the application
1067	starts freeing memory of a certain size, it's highly probable that this is
1068	a size the application uses for transient allocations. This estimator
1069	is there to satisfy the new third requirement.
1070
1071	*/
1072
1073	#define M_MMAP_THRESHOLD -3
1074
1075	#ifndef DEFAULT_MMAP_THRESHOLD
1076	#define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
1077	#endif
1078
1079	/*
1080	M_MMAP_MAX is the maximum number of requests to simultaneously
1081	service using mmap. This parameter exists because
1082	some systems have a limited number of internal tables for
1083	use by mmap, and using more than a few of them may degrade
1084	performance.
1085
1086	The default is set to a value that serves only as a safeguard.
1087	Setting to 0 disables use of mmap for servicing large requests.
1088	*/
1089
1090	#define M_MMAP_MAX -4
1091
1092	#ifndef DEFAULT_MMAP_MAX
1093	#define DEFAULT_MMAP_MAX (65536)
1094	#endif
1095
1096	#include <malloc.h>
1097
1098	#ifndef RETURN_ADDRESS
1099	#define RETURN_ADDRESS(X_) (NULL)
1100	#endif
1101
1102	/ Forward declarations. /
1103	struct malloc_chunk;
1104	typedef struct malloc_chunk* mchunkptr;
1105
1106	/ Internal routines. /
1107
1108	static void* _int_malloc(mstate, size_t);
1109	static void _int_free(mstate, mchunkptr, int);
1110	static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
1111	INTERNAL_SIZE_T);
1112	static void* _int_memalign(mstate, size_t, size_t);
1113	#if IS_IN (libc)
1114	static void* _mid_memalign(size_t, size_t, void *);
1115	#endif
1116
1117	static void malloc_printerr(const char str) __attribute__* ((noreturn));
1118
1119	static void munmap_chunk(mchunkptr p);
1120	#if HAVE_MREMAP
1121	static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
1122	#endif
1123
1124	/ ------------------ MMAP support ------------------ /
1125
1126
1127	#include <fcntl.h>
1128	#include <sys/mman.h>
1129
1130	#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1131	# define MAP_ANONYMOUS MAP_ANON
1132	#endif
1133
1134	#ifndef MAP_NORESERVE
1135	# define MAP_NORESERVE 0
1136	#endif
1137
1138	#define MMAP(addr, size, prot, flags) \
1139	__mmap((addr), (size), (prot), (flags)\|MAP_ANONYMOUS\|MAP_PRIVATE, -1, 0)
1140
1141
1142	/*
1143	----------------------- Chunk representations -----------------------
1144	*/
1145
1146
1147	/*
1148	This struct declaration is misleading (but accurate and necessary).
1149	It declares a "view" into memory allowing access to necessary
1150	fields at known offsets from a given base. See explanation below.
1151	*/
1152
1153	struct malloc_chunk {
1154
1155	INTERNAL_SIZE_T mchunk_prev_size; / Size of previous chunk (if free). /
1156	INTERNAL_SIZE_T mchunk_size; / Size in bytes, including overhead. /
1157
1158	struct malloc_chunk* fd; / double links -- used only if free. /
1159	struct malloc_chunk* bk;
1160
1161	/ Only used for large blocks: pointer to next larger size. /
1162	struct malloc_chunk* fd_nextsize; / double links -- used only if free. /
1163	struct malloc_chunk* bk_nextsize;
1164	};
1165
1166
1167	/*
1168	malloc_chunk details:
1169
1170	(The following includes lightly edited explanations by Colin Plumb.)
1171
1172	Chunks of memory are maintained using a `boundary tag' method as
1173	described in e.g., Knuth or Standish. (See the paper by Paul
1174	Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1175	survey of such techniques.) Sizes of free chunks are stored both
1176	in the front of each chunk and at the end. This makes
1177	consolidating fragmented chunks into bigger chunks very fast. The
1178	size fields also hold bits representing whether chunks are free or
1179	in use.
1180
1181	An allocated chunk looks like this:
1182
1183
1184	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1185	\| Size of previous chunk, if unallocated (P clear) \|
1186	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1187	\| Size of chunk, in bytes \|A\|M\|P\|
1188	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1189	\| User data starts here... .
1190	. .
1191	. (malloc_usable_size() bytes) .
1192	. \|
1193	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1194	\| (size of chunk, but used for application data) \|
1195	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1196	\| Size of next chunk, in bytes \|A\|0\|1\|
1197	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1198
1199	Where "chunk" is the front of the chunk for the purpose of most of
1200	the malloc code, but "mem" is the pointer that is returned to the
1201	user. "Nextchunk" is the beginning of the next contiguous chunk.
1202
1203	Chunks always begin on even word boundaries, so the mem portion
1204	(which is returned to the user) is also on an even word boundary, and
1205	thus at least double-word aligned.
1206
1207	Free chunks are stored in circular doubly-linked lists, and look like this:
1208
1209	chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1210	\| Size of previous chunk, if unallocated (P clear) \|
1211	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1212	`head:' \| Size of chunk, in bytes \|A\|0\|P\|
1213	mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1214	\| Forward pointer to next chunk in list \|
1215	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1216	\| Back pointer to previous chunk in list \|
1217	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1218	\| Unused space (may be 0 bytes long) .
1219	. .
1220	. \|
1221	nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1222	`foot:' \| Size of chunk, in bytes \|
1223	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1224	\| Size of next chunk, in bytes \|A\|0\|0\|
1225	+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1226
1227	The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1228	chunk size (which is always a multiple of two words), is an in-use
1229	bit for the previous* chunk. If that bit is clear, then the*
1230	word before the current chunk size contains the previous chunk
1231	size, and can be used to find the front of the previous chunk.
1232	The very first chunk allocated always has this bit set,
1233	preventing access to non-existent (or non-owned) memory. If
1234	prev_inuse is set for any given chunk, then you CANNOT determine
1235	the size of the previous chunk, and might even get a memory
1236	addressing fault when trying to do so.
1237
1238	The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
1239	main arena, described by the main_arena variable. When additional
1240	threads are spawned, each thread receives its own arena (up to a
1241	configurable limit, after which arenas are reused for multiple
1242	threads), and the chunks in these arenas have the A bit set. To
1243	find the arena for a chunk on such a non-main arena, heap_for_ptr
1244	performs a bit mask operation and indirection through the ar_ptr
1245	member of the per-heap header heap_info (see arena.c).
1246
1247	Note that the `foot' of the current chunk is actually represented
1248	as the prev_size of the NEXT chunk. This makes it easier to
1249	deal with alignments etc but can be very confusing when trying
1250	to extend or adapt this code.
1251
1252	The three exceptions to all this are:
1253
1254	1. The special chunk `top' doesn't bother using the
1255	trailing size field since there is no next contiguous chunk
1256	that would have to index off it. After initialization, `top'
1257	is forced to always exist. If it would become less than
1258	MINSIZE bytes long, it is replenished.
1259
1260	2. Chunks allocated via mmap, which have the second-lowest-order
1261	bit M (IS_MMAPPED) set in their size fields. Because they are
1262	allocated one-by-one, each must contain its own trailing size
1263	field. If the M bit is set, the other bits are ignored
1264	(because mmapped chunks are neither in an arena, nor adjacent
1265	to a freed chunk). The M bit is also used for chunks which
1266	originally came from a dumped heap via malloc_set_state in
1267	hooks.c.
1268
1269	3. Chunks in fastbins are treated as allocated chunks from the
1270	point of view of the chunk allocator. They are consolidated
1271	with their neighbors only in bulk, in malloc_consolidate.
1272	*/
1273
1274	/*
1275	---------- Size and alignment checks and conversions ----------
1276	*/
1277
1278	/ Conversion from malloc headers to user pointers, and back. When*
1279	using memory tagging the user data and the malloc data structure
1280	headers have distinct tags. Converting fully from one to the other
1281	involves extracting the tag at the other address and creating a
1282	suitable pointer using it. That can be quite expensive. There are
1283	cases when the pointers are not dereferenced (for example only used
1284	for alignment check) so the tags are not relevant, and there are
1285	cases when user data is not tagged distinctly from malloc headers
1286	(user data is untagged because tagging is done late in malloc and
1287	early in free). User memory tagging across internal interfaces:
1288
1289	sysmalloc: Returns untagged memory.
1290	_int_malloc: Returns untagged memory.
1291	_int_free: Takes untagged memory.
1292	_int_memalign: Returns untagged memory.
1293	_int_memalign: Returns untagged memory.
1294	_mid_memalign: Returns tagged memory.
1295	_int_realloc: Takes and returns tagged memory.
1296	*/
1297
1298	/ The chunk header is two SIZE_SZ elements, but this is used widely, so*
1299	we define it here for clarity later. /*
1300	#define CHUNK_HDR_SZ (2 * SIZE_SZ)
1301
1302	/ Convert a chunk address to a user mem pointer without correcting*
1303	the tag. /*
1304	#define chunk2mem(p) ((void)((char)(p) + CHUNK_HDR_SZ))
1305
1306	/ Convert a chunk address to a user mem pointer and extract the right tag. /
1307	#define chunk2mem_tag(p) ((void)tag_at ((char)(p) + CHUNK_HDR_SZ))
1308
1309	/ Convert a user mem pointer to a chunk address and extract the right tag. /
1310	#define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
1311
1312	/ The smallest possible chunk /
1313	#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1314
1315	/ The smallest size we can malloc is an aligned minimal chunk /
1316
1317	#define MINSIZE \
1318	(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1319
1320	/ Check if m has acceptable alignment /
1321
1322	#define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1323
1324	#define misaligned_chunk(p) \
1325	((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
1326	& MALLOC_ALIGN_MASK)
1327
1328	/ pad request bytes into a usable size -- internal version /
1329	/ Note: This must be a macro that evaluates to a compile time constant*
1330	if passed a literal constant. /*
1331	#define request2size(req) \
1332	(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1333	MINSIZE : \
1334	((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1335
1336	/ Check if REQ overflows when padded and aligned and if the resulting value*
1337	is less than PTRDIFF_T. Returns TRUE and the requested size or MINSIZE in
1338	case the value is less than MINSIZE on SZ or false if any of the previous
1339	check fail. /*
1340	static inline bool
1341	checked_request2size (size_t req, size_t *sz) __nonnull (`1`)
1342	{
1343	if (__glibc_unlikely (req > PTRDIFF_MAX))
1344	return false;
1345
1346	/ When using tagged memory, we cannot share the end of the user*
1347	block with the header for the next chunk, so ensure that we
1348	allocate blocks that are rounded up to the granule size. Take
1349	care not to overflow from close to MAX_SIZE_T to a small
1350	number. Ideally, this would be part of request2size(), but that
1351	must be a macro that produces a compile time constant if passed
1352	a constant literal. /*
1353	if (__glibc_unlikely (mtag_enabled))
1354	{
1355	/ Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551. /
1356	asm ("");
1357
1358	req = (req + (__MTAG_GRANULE_SIZE - `1`)) &
1359	~(size_t)(__MTAG_GRANULE_SIZE - `1`);
1360	}
1361
1362	*sz = request2size (req);
1363	return true;
1364	}
1365
1366	/*
1367	--------------- Physical chunk operations ---------------
1368	*/
1369
1370
1371	/ size field is or'ed with PREV_INUSE when previous adjacent chunk in use /
1372	#define PREV_INUSE 0x1
1373
1374	/ extract inuse bit of previous chunk /
1375	#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
1376
1377
1378	/ size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() /
1379	#define IS_MMAPPED 0x2
1380
1381	/ check for mmap()'ed chunk /
1382	#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
1383
1384
1385	/ size field is or'ed with NON_MAIN_ARENA if the chunk was obtained*
1386	from a non-main arena. This is only set immediately before handing
1387	the chunk to the user, if necessary. /*
1388	#define NON_MAIN_ARENA 0x4
1389
1390	/ Check for chunk from main arena. /
1391	#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
1392
1393	/ Mark a chunk as not being on the main arena. /
1394	#define set_non_main_arena(p) ((p)->mchunk_size \|= NON_MAIN_ARENA)
1395
1396
1397	/*
1398	Bits to mask off when extracting size
1399
1400	Note: IS_MMAPPED is intentionally not masked off from size field in
1401	macros for which mmapped chunks should never be seen. This should
1402	cause helpful core dumps to occur if it is tried by accident by
1403	people extending or adapting this malloc.
1404	*/
1405	#define SIZE_BITS (PREV_INUSE \| IS_MMAPPED \| NON_MAIN_ARENA)
1406
1407	/ Get size, ignoring use bits /
1408	#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
1409
1410	/ Like chunksize, but do not mask SIZE_BITS. /
1411	#define chunksize_nomask(p) ((p)->mchunk_size)
1412
1413	/ Ptr to next physical malloc_chunk. /
1414	#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
1415
1416	/ Size of the chunk below P. Only valid if !prev_inuse (P). /
1417	#define prev_size(p) ((p)->mchunk_prev_size)
1418
1419	/ Set the size of the chunk below P. Only valid if !prev_inuse (P). /
1420	#define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
1421
1422	/ Ptr to previous physical malloc_chunk. Only valid if !prev_inuse (P). /
1423	#define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
1424
1425	/ Treat space at ptr + offset as a chunk /
1426	#define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s)))
1427
1428	/ extract p's inuse bit /
1429	#define inuse(p) \
1430	((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
1431
1432	/ set/clear chunk as being inuse without otherwise disturbing /
1433	#define set_inuse(p) \
1434	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size \|= PREV_INUSE
1435
1436	#define clear_inuse(p) \
1437	((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
1438
1439
1440	/ check/set/clear inuse bits in known places /
1441	#define inuse_bit_at_offset(p, s) \
1442	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
1443
1444	#define set_inuse_bit_at_offset(p, s) \
1445	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size \|= PREV_INUSE)
1446
1447	#define clear_inuse_bit_at_offset(p, s) \
1448	(((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
1449
1450
1451	/ Set size at head, without disturbing its use bit /
1452	#define set_head_size(p, s) ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) \| (s)))
1453
1454	/ Set size/use field /
1455	#define set_head(p, s) ((p)->mchunk_size = (s))
1456
1457	/ Set size at footer (only when chunk is not in use) /
1458	#define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
1459
1460	#pragma GCC poison mchunk_size
1461	#pragma GCC poison mchunk_prev_size
1462
1463	/ This is the size of the real usable data in the chunk. Not valid for*
1464	dumped heap chunks. /*
1465	#define memsize(p) \
1466	(__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
1467	chunksize (p) - CHUNK_HDR_SZ : \
1468	chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
1469
1470	/ If memory tagging is enabled the layout changes to accommodate the granule*
1471	size, this is wasteful for small allocations so not done by default.
1472	Both the chunk header and user data has to be granule aligned. /*
1473	_Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
1474	"memory tagging is not supported with large granule.");
1475
1476	static __always_inline void *
1477	tag_new_usable (void *ptr)
1478	{
1479	if (__glibc_unlikely (mtag_enabled) && ptr)
1480	{
1481	mchunkptr cp = mem2chunk(ptr);
1482	ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp));
1483	}
1484	return ptr;
1485	}
1486
1487	/*
1488	-------------------- Internal data structures --------------------
1489
1490	All internal state is held in an instance of malloc_state defined
1491	below. There are no other static variables, except in two optional
1492	cases:
1493	* If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1494	* If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1495	for mmap.
1496
1497	Beware of lots of tricks that minimize the total bookkeeping space
1498	requirements. The result is a little over 1K bytes (for 4byte
1499	pointers and size_t.)
1500	*/
1501
1502	/*
1503	Bins
1504
1505	An array of bin headers for free chunks. Each bin is doubly
1506	linked. The bins are approximately proportionally (log) spaced.
1507	There are a lot of these bins (128). This may look excessive, but
1508	works very well in practice. Most bins hold sizes that are
1509	unusual as malloc request sizes, but are more usual for fragments
1510	and consolidated sets of chunks, which is what these bins hold, so
1511	they can be found quickly. All procedures maintain the invariant
1512	that no consolidated chunk physically borders another one, so each
1513	chunk in a list is known to be preceeded and followed by either
1514	inuse chunks or the ends of memory.
1515
1516	Chunks in bins are kept in size order, with ties going to the
1517	approximately least recently used chunk. Ordering isn't needed
1518	for the small bins, which all contain the same-sized chunks, but
1519	facilitates best-fit allocation for larger chunks. These lists
1520	are just sequential. Keeping them in order almost never requires
1521	enough traversal to warrant using fancier ordered data
1522	structures.
1523
1524	Chunks of the same size are linked with the most
1525	recently freed at the front, and allocations are taken from the
1526	back. This results in LRU (FIFO) allocation order, which tends
1527	to give each chunk an equal opportunity to be consolidated with
1528	adjacent freed chunks, resulting in larger free chunks and less
1529	fragmentation.
1530
1531	To simplify use in double-linked lists, each bin header acts
1532	as a malloc_chunk. This avoids special-casing for headers.
1533	But to conserve space and improve locality, we allocate
1534	only the fd/bk pointers of bins, and then use repositioning tricks
1535	to treat these as the fields of a malloc_chunk.*
1536	*/
1537
1538	typedef struct malloc_chunk *mbinptr;
1539
1540	/ addressing -- note that bin_at(0) does not exist /
1541	#define bin_at(m, i) \
1542	(mbinptr) (((char ) &((m)->bins[((i) - 1) 2])) \
1543	- offsetof (struct malloc_chunk, fd))
1544
1545	/ analog of ++bin /
1546	#define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
1547
1548	/ Reminders about list directionality within bins /
1549	#define first(b) ((b)->fd)
1550	#define last(b) ((b)->bk)
1551
1552	/*
1553	Indexing
1554
1555	Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1556	8 bytes apart. Larger bins are approximately logarithmically spaced:
1557
1558	64 bins of size 8
1559	32 bins of size 64
1560	16 bins of size 512
1561	8 bins of size 4096
1562	4 bins of size 32768
1563	2 bins of size 262144
1564	1 bin of size what's left
1565
1566	There is actually a little bit of slop in the numbers in bin_index
1567	for the sake of speed. This makes no difference elsewhere.
1568
1569	The bins top out around 1MB because we expect to service large
1570	requests via mmap.
1571
1572	Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1573	a valid chunk size the small bins are bumped up one.
1574	*/
1575
1576	#define NBINS 128
1577	#define NSMALLBINS 64
1578	#define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1579	#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
1580	#define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1581
1582	#define in_smallbin_range(sz) \
1583	((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
1584
1585	#define smallbin_index(sz) \
1586	((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
1587	+ SMALLBIN_CORRECTION)
1588
1589	#define largebin_index_32(sz) \
1590	(((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\
1591	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1592	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1593	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1594	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1595	126)
1596
1597	#define largebin_index_32_big(sz) \
1598	(((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\
1599	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1600	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1601	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1602	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1603	126)
1604
1605	// XXX It remains to be seen whether it is good to keep the widths of
1606	// XXX the buckets the same or whether it should be scaled by a factor
1607	// XXX of two as well.
1608	#define largebin_index_64(sz) \
1609	(((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\
1610	((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1611	((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1612	((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1613	((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1614	126)
1615
1616	#define largebin_index(sz) \
1617	(SIZE_SZ == 8 ? largebin_index_64 (sz) \
1618	: MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1619	: largebin_index_32 (sz))
1620
1621	#define bin_index(sz) \
1622	((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
1623
1624	/ Take a chunk off a bin list. /
1625	static void
1626	unlink_chunk (mstate av, mchunkptr p)
1627	{
1628	if (chunksize (p) != prev_size (next_chunk (p)))
1629	malloc_printerr ("corrupted size vs. prev_size");
1630
1631	mchunkptr fd = p->fd;
1632	mchunkptr bk = p->bk;
1633
1634	if (__builtin_expect (fd->bk != p \|\| bk->fd != p, `0`))
1635	malloc_printerr ("corrupted double-linked list");
1636
1637	fd->bk = bk;
1638	bk->fd = fd;
1639	if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
1640	{
1641	if (p->fd_nextsize->bk_nextsize != p
1642	\|\| p->bk_nextsize->fd_nextsize != p)
1643	malloc_printerr ("corrupted double-linked list (not small)");
1644
1645	if (fd->fd_nextsize == NULL)
1646	{
1647	if (p->fd_nextsize == p)
1648	fd->fd_nextsize = fd->bk_nextsize = fd;
1649	else
1650	{
1651	fd->fd_nextsize = p->fd_nextsize;
1652	fd->bk_nextsize = p->bk_nextsize;
1653	p->fd_nextsize->bk_nextsize = fd;
1654	p->bk_nextsize->fd_nextsize = fd;
1655	}
1656	}
1657	else
1658	{
1659	p->fd_nextsize->bk_nextsize = p->bk_nextsize;
1660	p->bk_nextsize->fd_nextsize = p->fd_nextsize;
1661	}
1662	}
1663	}
1664
1665	/*
1666	Unsorted chunks
1667
1668	All remainders from chunk splits, as well as all returned chunks,
1669	are first placed in the "unsorted" bin. They are then placed
1670	in regular bins after malloc gives them ONE chance to be used before
1671	binning. So, basically, the unsorted_chunks list acts as a queue,
1672	with chunks being placed on it in free (and malloc_consolidate),
1673	and taken off (to be either used or placed in bins) in malloc.
1674
1675	The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1676	does not have to be taken into account in size comparisons.
1677	*/
1678
1679	/ The otherwise unindexable 1-bin is used to hold unsorted chunks. /
1680	#define unsorted_chunks(M) (bin_at (M, 1))
1681
1682	/*
1683	Top
1684
1685	The top-most available chunk (i.e., the one bordering the end of
1686	available memory) is treated specially. It is never included in
1687	any bin, is used only if no other chunk is available, and is
1688	released back to the system if it is very large (see
1689	M_TRIM_THRESHOLD). Because top initially
1690	points to its own bin with initial zero size, thus forcing
1691	extension on the first malloc request, we avoid having any special
1692	code in malloc to check whether it even exists yet. But we still
1693	need to do so when getting memory from system, so we make
1694	initial_top treat the bin as a legal but unusable chunk during the
1695	interval between initialization and the first call to
1696	sysmalloc. (This is somewhat delicate, since it relies on
1697	the 2 preceding words to be zero during this interval as well.)
1698	*/
1699
1700	/ Conveniently, the unsorted bin can be used as dummy top on first call /
1701	#define initial_top(M) (unsorted_chunks (M))
1702
1703	/*
1704	Binmap
1705
1706	To help compensate for the large number of bins, a one-level index
1707	structure is used for bin-by-bin searching. `binmap' is a
1708	bitvector recording whether bins are definitely empty so they can
1709	be skipped over during during traversals. The bits are NOT always
1710	cleared as soon as bins are empty, but instead only
1711	when they are noticed to be empty during traversal in malloc.
1712	*/
1713
1714	/ Conservatively use 32 bits per map word, even if on 64bit system /
1715	#define BINMAPSHIFT 5
1716	#define BITSPERMAP (1U << BINMAPSHIFT)
1717	#define BINMAPSIZE (NBINS / BITSPERMAP)
1718
1719	#define idx2block(i) ((i) >> BINMAPSHIFT)
1720	#define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
1721
1722	#define mark_bin(m, i) ((m)->binmap[idx2block (i)] \|= idx2bit (i))
1723	#define unmark_bin(m, i) ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
1724	#define get_binmap(m, i) ((m)->binmap[idx2block (i)] & idx2bit (i))
1725
1726	/*
1727	Fastbins
1728
1729	An array of lists holding recently freed small chunks. Fastbins
1730	are not doubly linked. It is faster to single-link them, and
1731	since chunks are never removed from the middles of these lists,
1732	double linking is not necessary. Also, unlike regular bins, they
1733	are not even processed in FIFO order (they use faster LIFO) since
1734	ordering doesn't much matter in the transient contexts in which
1735	fastbins are normally used.
1736
1737	Chunks in fastbins keep their inuse bit set, so they cannot
1738	be consolidated with other free chunks. malloc_consolidate
1739	releases all chunks in fastbins and consolidates them with
1740	other free chunks.
1741	*/
1742
1743	typedef struct malloc_chunk *mfastbinptr;
1744	#define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1745
1746	/ offset 2 to use otherwise unindexable first 2 bins /
1747	#define fastbin_index(sz) \
1748	((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1749
1750
1751	/ The maximum fastbin request size we support /
1752	#define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1753
1754	#define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
1755
1756	/*
1757	FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1758	that triggers automatic consolidation of possibly-surrounding
1759	fastbin chunks. This is a heuristic, so the exact value should not
1760	matter too much. It is defined at half the default trim threshold as a
1761	compromise heuristic to only attempt consolidation if it is likely
1762	to lead to trimming. However, it is not dynamically tunable, since
1763	consolidation reduces fragmentation surrounding large chunks even
1764	if trimming is not used.
1765	*/
1766
1767	#define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1768
1769	/*
1770	NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1771	regions. Otherwise, contiguity is exploited in merging together,
1772	when possible, results from consecutive MORECORE calls.
1773
1774	The initial value comes from MORECORE_CONTIGUOUS, but is
1775	changed dynamically if mmap is ever used as an sbrk substitute.
1776	*/
1777
1778	#define NONCONTIGUOUS_BIT (2U)
1779
1780	#define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1781	#define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1782	#define set_noncontiguous(M) ((M)->flags \|= NONCONTIGUOUS_BIT)
1783	#define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1784
1785	/ Maximum size of memory handled in fastbins. /
1786	static INTERNAL_SIZE_T global_max_fast;
1787
1788	/*
1789	Set value of max_fast.
1790	Use impossibly small value if 0.
1791	Precondition: there are no existing fastbin chunks in the main arena.
1792	Since do_check_malloc_state () checks this, we call malloc_consolidate ()
1793	before changing max_fast. Note other arenas will leak their fast bin
1794	entries if max_fast is reduced.
1795	*/
1796
1797	#define set_max_fast(s) \
1798	global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \
1799	? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1800
1801	static inline INTERNAL_SIZE_T
1802	get_max_fast (void)
1803	{
1804	/ Tell the GCC optimizers that global_max_fast is never larger*
1805	than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in
1806	_int_malloc after constant propagation of the size parameter.
1807	(The code never executes because malloc preserves the
1808	global_max_fast invariant, but the optimizers may not recognize
1809	this.) /*
1810	if (global_max_fast > MAX_FAST_SIZE)
1811	__builtin_unreachable ();
1812	return global_max_fast;
1813	}
1814
1815	/*
1816	----------- Internal state representation and initialization -----------
1817	*/
1818
1819	/*
1820	have_fastchunks indicates that there are probably some fastbin chunks.
1821	It is set true on entering a chunk into any fastbin, and cleared early in
1822	malloc_consolidate. The value is approximate since it may be set when there
1823	are no fastbin chunks, or it may be clear even if there are fastbin chunks
1824	available. Given it's sole purpose is to reduce number of redundant calls to
1825	malloc_consolidate, it does not affect correctness. As a result we can safely
1826	use relaxed atomic accesses.
1827	*/
1828
1829
1830	struct malloc_state
1831	{
1832	/ Serialize access. /
1833	__libc_lock_define (, mutex);
1834
1835	/ Flags (formerly in max_fast). /
1836	int flags;
1837
1838	/ Set if the fastbin chunks contain recently inserted free blocks. /
1839	/ Note this is a bool but not all targets support atomics on booleans. /
1840	int have_fastchunks;
1841
1842	/ Fastbins /
1843	mfastbinptr fastbinsY[NFASTBINS];
1844
1845	/ Base of the topmost chunk -- not otherwise kept in a bin /
1846	mchunkptr top;
1847
1848	/ The remainder from the most recent split of a small request /
1849	mchunkptr last_remainder;
1850
1851	/ Normal bins packed as described above /
1852	mchunkptr bins[NBINS * `2` - `2`];
1853
1854	/ Bitmap of bins /
1855	unsigned int binmap[BINMAPSIZE];
1856
1857	/ Linked list /
1858	struct malloc_state *next;
1859
1860	/ Linked list for free arenas. Access to this field is serialized*
1861	by free_list_lock in arena.c. /*
1862	struct malloc_state *next_free;
1863
1864	/ Number of threads attached to this arena. 0 if the arena is on*
1865	the free list. Access to this field is serialized by
1866	free_list_lock in arena.c. /*
1867	INTERNAL_SIZE_T attached_threads;
1868
1869	/ Memory allocated from the system in this arena. /
1870	INTERNAL_SIZE_T system_mem;
1871	INTERNAL_SIZE_T max_system_mem;
1872	};
1873
1874	struct malloc_par
1875	{
1876	/ Tunable parameters /
1877	unsigned long trim_threshold;
1878	INTERNAL_SIZE_T top_pad;
1879	INTERNAL_SIZE_T mmap_threshold;
1880	INTERNAL_SIZE_T arena_test;
1881	INTERNAL_SIZE_T arena_max;
1882
1883	#if HAVE_TUNABLES
1884	/ Transparent Large Page support. /
1885	INTERNAL_SIZE_T thp_pagesize;
1886	/ A value different than 0 means to align mmap allocation to hp_pagesize*
1887	add hp_flags on flags. /*
1888	INTERNAL_SIZE_T hp_pagesize;
1889	int hp_flags;
1890	#endif
1891
1892	/ Memory map support /
1893	int n_mmaps;
1894	int n_mmaps_max;
1895	int max_n_mmaps;
1896	/ the mmap_threshold is dynamic, until the user sets*
1897	it manually, at which point we need to disable any
1898	dynamic behavior. /*
1899	int no_dyn_threshold;
1900
1901	/ Statistics /
1902	INTERNAL_SIZE_T mmapped_mem;
1903	INTERNAL_SIZE_T max_mmapped_mem;
1904
1905	/ First address handed out by MORECORE/sbrk. /
1906	char *sbrk_base;
1907
1908	#if USE_TCACHE
1909	/ Maximum number of buckets to use. /
1910	size_t tcache_bins;
1911	size_t tcache_max_bytes;
1912	/ Maximum number of chunks in each bucket. /
1913	size_t tcache_count;
1914	/ Maximum number of chunks to remove from the unsorted list, which*
1915	aren't used to prefill the cache. /*
1916	size_t tcache_unsorted_limit;
1917	#endif
1918	};
1919
1920	/ There are several instances of this struct ("arenas") in this*
1921	malloc. If you are adapting this malloc in a way that does NOT use
1922	a static or mmapped malloc_state, you MUST explicitly zero-fill it
1923	before using. This malloc relies on the property that malloc_state
1924	is initialized to all zeroes (as is true of C statics). /*
1925
1926	static struct malloc_state main_arena =
1927	{
1928	.mutex = _LIBC_LOCK_INITIALIZER,
1929	.next = &main_arena,
1930	.attached_threads = `1`
1931	};
1932
1933	/ There is only one instance of the malloc parameters. /
1934
1935	static struct malloc_par mp_ =
1936	{
1937	.top_pad = DEFAULT_TOP_PAD,
1938	.n_mmaps_max = DEFAULT_MMAP_MAX,
1939	.mmap_threshold = DEFAULT_MMAP_THRESHOLD,
1940	.trim_threshold = DEFAULT_TRIM_THRESHOLD,
1941	#define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
1942	.arena_test = NARENAS_FROM_NCORES (`1`)
1943	#if USE_TCACHE
1944	,
1945	.tcache_count = TCACHE_FILL_COUNT,
1946	.tcache_bins = TCACHE_MAX_BINS,
1947	.tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-`1`),
1948	.tcache_unsorted_limit = `0` / No limit. /
1949	#endif
1950	};
1951
1952	/*
1953	Initialize a malloc_state struct.
1954
1955	This is called from ptmalloc_init () or from _int_new_arena ()
1956	when creating a new arena.
1957	*/
1958
1959	static void
1960	malloc_init_state (mstate av)
1961	{
1962	int i;
1963	mbinptr bin;
1964
1965	/ Establish circular links for normal bins /
1966	for (i = `1`; i < NBINS; ++i)
1967	{
1968	bin = bin_at (av, i);
1969	bin->fd = bin->bk = bin;
1970	}
1971
1972	#if MORECORE_CONTIGUOUS
1973	if (av != &main_arena)
1974	#endif
1975	set_noncontiguous (av);
1976	if (av == &main_arena)
1977	set_max_fast (DEFAULT_MXFAST);
1978	atomic_store_relaxed (&av->have_fastchunks, false);
1979
1980	av->top = initial_top (av);
1981	}
1982
1983	/*
1984	Other internal utilities operating on mstates
1985	*/
1986
1987	static void *sysmalloc (INTERNAL_SIZE_T, mstate);
1988	static int systrim (size_t, mstate);
1989	static void malloc_consolidate (mstate);
1990
1991
1992	/ -------------- Early definitions for debugging hooks ---------------- /
1993
1994	/ This function is called from the arena shutdown hook, to free the*
1995	thread cache (if it exists). /*
1996	static void tcache_thread_shutdown (void);
1997
1998	/ ------------------ Testing support ----------------------------------/
1999
2000	static int perturb_byte;
2001
2002	static void
2003	alloc_perturb (char *p, size_t n)
2004	{
2005	if (__glibc_unlikely (perturb_byte))
2006	memset (p, perturb_byte ^ `0xff`, n);
2007	}
2008
2009	static void
2010	free_perturb (char *p, size_t n)
2011	{
2012	if (__glibc_unlikely (perturb_byte))
2013	memset (p, perturb_byte, n);
2014	}
2015
2016
2017
2018	#include <stap-probe.h>
2019
2020	/ ----------- Routines dealing with transparent huge pages ----------- /
2021
2022	static inline void
2023	madvise_thp (void *p, INTERNAL_SIZE_T size)
2024	{
2025	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2026	/ Do not consider areas smaller than a huge page or if the tunable is*
2027	not active. /*
2028	if (mp_.thp_pagesize == `0` \|\| size < mp_.thp_pagesize)
2029	return;
2030
2031	/ Linux requires the input address to be page-aligned, and unaligned*
2032	inputs happens only for initial data segment. /*
2033	if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
2034	{
2035	void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
2036	size += PTR_DIFF (p, q);
2037	p = q;
2038	}
2039
2040	__madvise (p, size, MADV_HUGEPAGE);
2041	#endif
2042	}
2043
2044	/ ------------------- Support for multiple arenas -------------------- /
2045	#include "arena.c"
2046
2047	/*
2048	Debugging support
2049
2050	These routines make a number of assertions about the states
2051	of data structures that should be true at all times. If any
2052	are not true, it's very likely that a user program has somehow
2053	trashed memory. (It's also possible that there is a coding error
2054	in malloc. In which case, please report it!)
2055	*/
2056
2057	#if !MALLOC_DEBUG
2058
2059	# define check_chunk(A, P)
2060	# define check_free_chunk(A, P)
2061	# define check_inuse_chunk(A, P)
2062	# define check_remalloced_chunk(A, P, N)
2063	# define check_malloced_chunk(A, P, N)
2064	# define check_malloc_state(A)
2065
2066	#else
2067
2068	# define check_chunk(A, P) do_check_chunk (A, P)
2069	# define check_free_chunk(A, P) do_check_free_chunk (A, P)
2070	# define check_inuse_chunk(A, P) do_check_inuse_chunk (A, P)
2071	# define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
2072	# define check_malloced_chunk(A, P, N) do_check_malloced_chunk (A, P, N)
2073	# define check_malloc_state(A) do_check_malloc_state (A)
2074
2075	/*
2076	Properties of all chunks
2077	*/
2078
2079	static void
2080	do_check_chunk (mstate av, mchunkptr p)
2081	{
2082	unsigned long sz = chunksize (p);
2083	/ min and max possible addresses assuming contiguous allocation /
2084	char max_address = (char* *) (av->top) + chunksize (av->top);
2085	char *min_address = max_address - av->system_mem;
2086
2087	if (!chunk_is_mmapped (p))
2088	{
2089	/ Has legal address ... /
2090	if (p != av->top)
2091	{
2092	if (contiguous (av))
2093	{
2094	assert (((char *) p) >= min_address);
2095	assert (((char ) p + sz) <= ((char* *) (av->top)));
2096	}
2097	}
2098	else
2099	{
2100	/ top size is always at least MINSIZE /
2101	assert ((unsigned long) (sz) >= MINSIZE);
2102	/ top predecessor always marked inuse /
2103	assert (prev_inuse (p));
2104	}
2105	}
2106	else
2107	{
2108	/ address is outside main heap /
2109	if (contiguous (av) && av->top != initial_top (av))
2110	{
2111	assert (((char ) p) < min_address \|\| ((char* *) p) >= max_address);
2112	}
2113	/ chunk is page-aligned /
2114	assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - `1`)) == `0`);
2115	/ mem is aligned /
2116	assert (aligned_OK (chunk2mem (p)));
2117	}
2118	}
2119
2120	/*
2121	Properties of free chunks
2122	*/
2123
2124	static void
2125	do_check_free_chunk (mstate av, mchunkptr p)
2126	{
2127	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2128	mchunkptr next = chunk_at_offset (p, sz);
2129
2130	do_check_chunk (av, p);
2131
2132	/ Chunk must claim to be free ... /
2133	assert (!inuse (p));
2134	assert (!chunk_is_mmapped (p));
2135
2136	/ Unless a special marker, must have OK fields /
2137	if ((unsigned long) (sz) >= MINSIZE)
2138	{
2139	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2140	assert (aligned_OK (chunk2mem (p)));
2141	/ ... matching footer field /
2142	assert (prev_size (next_chunk (p)) == sz);
2143	/ ... and is fully consolidated /
2144	assert (prev_inuse (p));
2145	assert (next == av->top \|\| inuse (next));
2146
2147	/ ... and has minimally sane links /
2148	assert (p->fd->bk == p);
2149	assert (p->bk->fd == p);
2150	}
2151	else / markers are always of size SIZE_SZ /
2152	assert (sz == SIZE_SZ);
2153	}
2154
2155	/*
2156	Properties of inuse chunks
2157	*/
2158
2159	static void
2160	do_check_inuse_chunk (mstate av, mchunkptr p)
2161	{
2162	mchunkptr next;
2163
2164	do_check_chunk (av, p);
2165
2166	if (chunk_is_mmapped (p))
2167	return; / mmapped chunks have no next/prev /
2168
2169	/ Check whether it claims to be in use ... /
2170	assert (inuse (p));
2171
2172	next = next_chunk (p);
2173
2174	/ ... and is surrounded by OK chunks.*
2175	Since more things can be checked with free chunks than inuse ones,
2176	if an inuse chunk borders them and debug is on, it's worth doing them.
2177	*/
2178	if (!prev_inuse (p))
2179	{
2180	/ Note that we cannot even look at prev unless it is not inuse /
2181	mchunkptr prv = prev_chunk (p);
2182	assert (next_chunk (prv) == p);
2183	do_check_free_chunk (av, prv);
2184	}
2185
2186	if (next == av->top)
2187	{
2188	assert (prev_inuse (next));
2189	assert (chunksize (next) >= MINSIZE);
2190	}
2191	else if (!inuse (next))
2192	do_check_free_chunk (av, next);
2193	}
2194
2195	/*
2196	Properties of chunks recycled from fastbins
2197	*/
2198
2199	static void
2200	do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2201	{
2202	INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE \| NON_MAIN_ARENA);
2203
2204	if (!chunk_is_mmapped (p))
2205	{
2206	assert (av == arena_for_chunk (p));
2207	if (chunk_main_arena (p))
2208	assert (av == &main_arena);
2209	else
2210	assert (av != &main_arena);
2211	}
2212
2213	do_check_inuse_chunk (av, p);
2214
2215	/ Legal size ... /
2216	assert ((sz & MALLOC_ALIGN_MASK) == `0`);
2217	assert ((unsigned long) (sz) >= MINSIZE);
2218	/ ... and alignment /
2219	assert (aligned_OK (chunk2mem (p)));
2220	/ chunk is less than MINSIZE more than request /
2221	assert ((long) (sz) - (long) (s) >= `0`);
2222	assert ((long) (sz) - (long) (s + MINSIZE) < `0`);
2223	}
2224
2225	/*
2226	Properties of nonrecycled chunks at the point they are malloced
2227	*/
2228
2229	static void
2230	do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2231	{
2232	/ same as recycled case ... /
2233	do_check_remalloced_chunk (av, p, s);
2234
2235	/*
2236	... plus, must obey implementation invariant that prev_inuse is
2237	always true of any allocated chunk; i.e., that each allocated
2238	chunk borders either a previously allocated and still in-use
2239	chunk, or the base of its memory arena. This is ensured
2240	by making all allocations from the `lowest' part of any found
2241	chunk. This does not necessarily hold however for chunks
2242	recycled via fastbins.
2243	*/
2244
2245	assert (prev_inuse (p));
2246	}
2247
2248
2249	/*
2250	Properties of malloc_state.
2251
2252	This may be useful for debugging malloc, as well as detecting user
2253	programmer errors that somehow write into malloc_state.
2254
2255	If you are extending or experimenting with this malloc, you can
2256	probably figure out how to hack this routine to print out or
2257	display chunk addresses, sizes, bins, and other instrumentation.
2258	*/
2259
2260	static void
2261	do_check_malloc_state (mstate av)
2262	{
2263	int i;
2264	mchunkptr p;
2265	mchunkptr q;
2266	mbinptr b;
2267	unsigned int idx;
2268	INTERNAL_SIZE_T size;
2269	unsigned long total = `0`;
2270	int max_fast_bin;
2271
2272	/ internal size_t must be no wider than pointer type /
2273	assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
2274
2275	/ alignment is a power of 2 /
2276	assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - `1`)) == `0`);
2277
2278	/ Check the arena is initialized. /
2279	assert (av->top != `0`);
2280
2281	/ No memory has been allocated yet, so doing more tests is not possible. /
2282	if (av->top == initial_top (av))
2283	return;
2284
2285	/ pagesize is a power of 2 /
2286	assert (powerof2(GLRO (dl_pagesize)));
2287
2288	/ A contiguous main_arena is consistent with sbrk_base. /
2289	if (av == &main_arena && contiguous (av))
2290	assert ((char *) mp_.sbrk_base + av->system_mem ==
2291	(char *) av->top + chunksize (av->top));
2292
2293	/ properties of fastbins /
2294
2295	/ max_fast is in allowed range /
2296	assert ((get_max_fast () & ~`1`) <= request2size (MAX_FAST_SIZE));
2297
2298	max_fast_bin = fastbin_index (get_max_fast ());
2299
2300	for (i = `0`; i < NFASTBINS; ++i)
2301	{
2302	p = fastbin (av, i);
2303
2304	/ The following test can only be performed for the main arena.*
2305	While mallopt calls malloc_consolidate to get rid of all fast
2306	bins (especially those larger than the new maximum) this does
2307	only happen for the main arena. Trying to do this for any
2308	other arena would mean those arenas have to be locked and
2309	malloc_consolidate be called for them. This is excessive. And
2310	even if this is acceptable to somebody it still cannot solve
2311	the problem completely since if the arena is locked a
2312	concurrent malloc call might create a new arena which then
2313	could use the newly invalid fast bins. /*
2314
2315	/ all bins past max_fast are empty /
2316	if (av == &main_arena && i > max_fast_bin)
2317	assert (p == `0`);
2318
2319	while (p != `0`)
2320	{
2321	if (__glibc_unlikely (misaligned_chunk (p)))
2322	malloc_printerr ("do_check_malloc_state(): "
2323	"unaligned fastbin chunk detected");
2324	/ each chunk claims to be inuse /
2325	do_check_inuse_chunk (av, p);
2326	total += chunksize (p);
2327	/ chunk belongs in this bin /
2328	assert (fastbin_index (chunksize (p)) == i);
2329	p = REVEAL_PTR (p->fd);
2330	}
2331	}
2332
2333	/ check normal bins /
2334	for (i = `1`; i < NBINS; ++i)
2335	{
2336	b = bin_at (av, i);
2337
2338	/ binmap is accurate (except for bin 1 == unsorted_chunks) /
2339	if (i >= `2`)
2340	{
2341	unsigned int binbit = get_binmap (av, i);
2342	int empty = last (b) == b;
2343	if (!binbit)
2344	assert (empty);
2345	else if (!empty)
2346	assert (binbit);
2347	}
2348
2349	for (p = last (b); p != b; p = p->bk)
2350	{
2351	/ each chunk claims to be free /
2352	do_check_free_chunk (av, p);
2353	size = chunksize (p);
2354	total += size;
2355	if (i >= `2`)
2356	{
2357	/ chunk belongs in bin /
2358	idx = bin_index (size);
2359	assert (idx == i);
2360	/ lists are sorted /
2361	assert (p->bk == b \|\|
2362	(unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
2363
2364	if (!in_smallbin_range (size))
2365	{
2366	if (p->fd_nextsize != NULL)
2367	{
2368	if (p->fd_nextsize == p)
2369	assert (p->bk_nextsize == p);
2370	else
2371	{
2372	if (p->fd_nextsize == first (b))
2373	assert (chunksize (p) < chunksize (p->fd_nextsize));
2374	else
2375	assert (chunksize (p) > chunksize (p->fd_nextsize));
2376
2377	if (p == first (b))
2378	assert (chunksize (p) > chunksize (p->bk_nextsize));
2379	else
2380	assert (chunksize (p) < chunksize (p->bk_nextsize));
2381	}
2382	}
2383	else
2384	assert (p->bk_nextsize == NULL);
2385	}
2386	}
2387	else if (!in_smallbin_range (size))
2388	assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
2389	/ chunk is followed by a legal chain of inuse chunks /
2390	for (q = next_chunk (p);
2391	(q != av->top && inuse (q) &&
2392	(unsigned long) (chunksize (q)) >= MINSIZE);
2393	q = next_chunk (q))
2394	do_check_inuse_chunk (av, q);
2395	}
2396	}
2397
2398	/ top chunk is OK /
2399	check_chunk (av, av->top);
2400	}
2401	#endif
2402
2403
2404	/ ----------------- Support for debugging hooks -------------------- /
2405	#if IS_IN (libc)
2406	#include "hooks.c"
2407	#endif
2408
2409
2410	/ ----------- Routines dealing with system allocation -------------- /
2411
2412	/*
2413	sysmalloc handles malloc cases requiring more memory from the system.
2414	On entry, it is assumed that av->top does not have enough
2415	space to service request for nb bytes, thus requiring that av->top
2416	be extended or replaced.
2417	*/
2418
2419	static void *
2420	sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av)
2421	{
2422	long int size;
2423
2424	/*
2425	Round up size to nearest page. For mmapped chunks, the overhead is one
2426	SIZE_SZ unit larger than for normal chunks, because there is no
2427	following chunk whose prev_size field could be used.
2428
2429	See the front_misalign handling below, for glibc there is no need for
2430	further alignments unless we have have high alignment.
2431	*/
2432	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2433	size = ALIGN_UP (nb + SIZE_SZ, pagesize);
2434	else
2435	size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
2436
2437	/ Don't try if size wraps around 0. /
2438	if ((unsigned long) (size) <= (unsigned long) (nb))
2439	return MAP_FAILED;
2440
2441	char mm = (char* *) MMAP (`0`, size,
2442	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2443	extra_flags);
2444	if (mm == MAP_FAILED)
2445	return mm;
2446
2447	#ifdef MAP_HUGETLB
2448	if (!(extra_flags & MAP_HUGETLB))
2449	madvise_thp (mm, size);
2450	#endif
2451
2452	/*
2453	The offset to the start of the mmapped region is stored in the prev_size
2454	field of the chunk. This allows us to adjust returned start address to
2455	meet alignment requirements here and in memalign(), and still be able to
2456	compute proper address argument for later munmap in free() and realloc().
2457	*/
2458
2459	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2460
2461	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2462	{
2463	/ For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and*
2464	MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1. Each mmap'ed area is page
2465	aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. /*
2466	assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == `0`);
2467	front_misalign = `0`;
2468	}
2469	else
2470	front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
2471
2472	mchunkptr p; / the allocated/returned chunk /
2473
2474	if (front_misalign > `0`)
2475	{
2476	ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign;
2477	p = (mchunkptr) (mm + correction);
2478	set_prev_size (p, correction);
2479	set_head (p, (size - correction) \| IS_MMAPPED);
2480	}
2481	else
2482	{
2483	p = (mchunkptr) mm;
2484	set_prev_size (p, `0`);
2485	set_head (p, size \| IS_MMAPPED);
2486	}
2487
2488	/ update statistics /
2489	int new = atomic_exchange_and_add (&mp_.n_mmaps, `1`) + `1`;
2490	atomic_max (&mp_.max_n_mmaps, new);
2491
2492	unsigned long sum;
2493	sum = atomic_exchange_and_add (&mp_.mmapped_mem, size) + size;
2494	atomic_max (&mp_.max_mmapped_mem, sum);
2495
2496	check_chunk (av, p);
2497
2498	return chunk2mem (p);
2499	}
2500
2501	/*
2502	Allocate memory using mmap() based on S and NB requested size, aligning to
2503	PAGESIZE if required. The EXTRA_FLAGS is used on mmap() call. If the call
2504	succeedes S is updated with the allocated size. This is used as a fallback
2505	if MORECORE fails.
2506	*/
2507	static void *
2508	sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb,
2509	INTERNAL_SIZE_T old_size, size_t minsize,
2510	size_t pagesize, int extra_flags, mstate av)
2511	{
2512	long int size = *s;
2513
2514	/ Cannot merge with old top, so add its size back in /
2515	if (contiguous (av))
2516	size = ALIGN_UP (size + old_size, pagesize);
2517
2518	/ If we are relying on mmap as backup, then use larger units /
2519	if ((unsigned long) (size) < minsize)
2520	size = minsize;
2521
2522	/ Don't try if size wraps around 0 /
2523	if ((unsigned long) (size) <= (unsigned long) (nb))
2524	return MORECORE_FAILURE;
2525
2526	char mbrk = (char* *) (MMAP (`0`, size,
2527	mtag_mmap_flags \| PROT_READ \| PROT_WRITE,
2528	extra_flags));
2529	if (mbrk == MAP_FAILED)
2530	return MAP_FAILED;
2531
2532	#ifdef MAP_HUGETLB
2533	if (!(extra_flags & MAP_HUGETLB))
2534	madvise_thp (mbrk, size);
2535	#endif
2536
2537	/ Record that we no longer have a contiguous sbrk region. After the first*
2538	time mmap is used as backup, we do not ever rely on contiguous space
2539	since this could incorrectly bridge regions. /*
2540	set_noncontiguous (av);
2541
2542	*s = size;
2543	return mbrk;
2544	}
2545
2546	static void *
2547	sysmalloc (INTERNAL_SIZE_T nb, mstate av)
2548	{
2549	mchunkptr old_top; / incoming value of av->top /
2550	INTERNAL_SIZE_T old_size; / its size /
2551	char old_end; /* its end address /
2552
2553	long size; / arg to first MORECORE or mmap call /
2554	char brk; /* return value from MORECORE /
2555
2556	long correction; / arg to 2nd MORECORE call /
2557	char snd_brk; /* 2nd return val /
2558
2559	INTERNAL_SIZE_T front_misalign; / unusable bytes at front of new space /
2560	INTERNAL_SIZE_T end_misalign; / partial page left at end of new space /
2561	char aligned_brk; /* aligned offset into brk /
2562
2563	mchunkptr p; / the allocated/returned chunk /
2564	mchunkptr remainder; / remainder from allocation /
2565	unsigned long remainder_size; / its size /
2566
2567
2568	size_t pagesize = GLRO (dl_pagesize);
2569	bool tried_mmap = false;
2570
2571
2572	/*
2573	If have mmap, and the request size meets the mmap threshold, and
2574	the system supports mmap, and there are few enough currently
2575	allocated mmapped regions, try to directly map this request
2576	rather than expanding top.
2577	*/
2578
2579	if (av == NULL
2580	\|\| ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
2581	&& (mp_.n_mmaps < mp_.n_mmaps_max)))
2582	{
2583	char *mm;
2584	#if HAVE_TUNABLES
2585	if (mp_.hp_pagesize > `0` && nb >= mp_.hp_pagesize)
2586	{
2587	/ There is no need to isse the THP madvise call if Huge Pages are*
2588	used directly. /*
2589	mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av);
2590	if (mm != MAP_FAILED)
2591	return mm;
2592	}
2593	#endif
2594	mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2595	if (mm != MAP_FAILED)
2596	return mm;
2597	tried_mmap = true;
2598	}
2599
2600	/ There are no usable arenas and mmap also failed. /
2601	if (av == NULL)
2602	return `0`;
2603
2604	/ Record incoming configuration of top /
2605
2606	old_top = av->top;
2607	old_size = chunksize (old_top);
2608	old_end = (char *) (chunk_at_offset (old_top, old_size));
2609
2610	brk = snd_brk = (char *) (MORECORE_FAILURE);
2611
2612	/*
2613	If not the first time through, we require old_size to be
2614	at least MINSIZE and to have prev_inuse set.
2615	*/
2616
2617	assert ((old_top == initial_top (av) && old_size == `0`) \|\|
2618	((unsigned long) (old_size) >= MINSIZE &&
2619	prev_inuse (old_top) &&
2620	((unsigned long) old_end & (pagesize - `1`)) == `0`));
2621
2622	/ Precondition: not enough current space to satisfy nb request /
2623	assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
2624
2625
2626	if (av != &main_arena)
2627	{
2628	heap_info old_heap, heap;
2629	size_t old_heap_size;
2630
2631	/ First try to extend the current heap. /
2632	old_heap = heap_for_ptr (old_top);
2633	old_heap_size = old_heap->size;
2634	if ((long) (MINSIZE + nb - old_size) > `0`
2635	&& grow_heap (old_heap, MINSIZE + nb - old_size) == `0`)
2636	{
2637	av->system_mem += old_heap->size - old_heap_size;
2638	set_head (old_top, (((char ) old_heap + old_heap->size) - (char* *) old_top)
2639	\| PREV_INUSE);
2640	}
2641	else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad)))
2642	{
2643	/ Use a newly allocated heap. /
2644	heap->ar_ptr = av;
2645	heap->prev = old_heap;
2646	av->system_mem += heap->size;
2647	/ Set up the new top. /
2648	top (av) = chunk_at_offset (heap, sizeof (*heap));
2649	set_head (top (av), (heap->size - sizeof (*heap)) \| PREV_INUSE);
2650
2651	/ Setup fencepost and free the old top chunk with a multiple of*
2652	MALLOC_ALIGNMENT in size. /*
2653	/ The fencepost takes at least MINSIZE bytes, because it might*
2654	become the top chunk again later. Note that a footer is set
2655	up, too, although the chunk is marked in use. /*
2656	old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
2657	set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
2658	`0` \| PREV_INUSE);
2659	if (old_size >= MINSIZE)
2660	{
2661	set_head (chunk_at_offset (old_top, old_size),
2662	CHUNK_HDR_SZ \| PREV_INUSE);
2663	set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
2664	set_head (old_top, old_size \| PREV_INUSE \| NON_MAIN_ARENA);
2665	_int_free (av, old_top, `1`);
2666	}
2667	else
2668	{
2669	set_head (old_top, (old_size + CHUNK_HDR_SZ) \| PREV_INUSE);
2670	set_foot (old_top, (old_size + CHUNK_HDR_SZ));
2671	}
2672	}
2673	else if (!tried_mmap)
2674	{
2675	/ We can at least try to use to mmap memory. If new_heap fails*
2676	it is unlikely that trying to allocate huge pages will
2677	succeed. /*
2678	char *mm = sysmalloc_mmap (nb, pagesize, `0`, av);
2679	if (mm != MAP_FAILED)
2680	return mm;
2681	}
2682	}
2683	else / av == main_arena /
2684
2685
2686	{ / Request enough space for nb + pad + overhead /
2687	size = nb + mp_.top_pad + MINSIZE;
2688
2689	/*
2690	If contiguous, we can subtract out existing space that we hope to
2691	combine with new space. We add it back later only if
2692	we don't actually get contiguous space.
2693	*/
2694
2695	if (contiguous (av))
2696	size -= old_size;
2697
2698	/*
2699	Round to a multiple of page size or huge page size.
2700	If MORECORE is not contiguous, this ensures that we only call it
2701	with whole-page arguments. And if MORECORE is contiguous and
2702	this is not first time through, this preserves page-alignment of
2703	previous calls. Otherwise, we correct to page-align below.
2704	*/
2705
2706	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2707	/ Defined in brk.c. /
2708	extern void *__curbrk;
2709	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2710	{
2711	uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
2712	mp_.thp_pagesize);
2713	size = top - (uintptr_t) __curbrk;
2714	}
2715	else
2716	#endif
2717	size = ALIGN_UP (size, GLRO(dl_pagesize));
2718
2719	/*
2720	Don't try to call MORECORE if argument is so big as to appear
2721	negative. Note that since mmap takes size_t arg, it may succeed
2722	below even if we cannot call MORECORE.
2723	*/
2724
2725	if (size > `0`)
2726	{
2727	brk = (char *) (MORECORE (size));
2728	if (brk != (char *) (MORECORE_FAILURE))
2729	madvise_thp (brk, size);
2730	LIBC_PROBE (memory_sbrk_more, `2`, brk, size);
2731	}
2732
2733	if (brk == (char *) (MORECORE_FAILURE))
2734	{
2735	/*
2736	If have mmap, try using it as a backup when MORECORE fails or
2737	cannot be used. This is worth doing on systems that have "holes" in
2738	address space, so sbrk cannot extend to give contiguous space, but
2739	space is available elsewhere. Note that we ignore mmap max count
2740	and threshold limits, since the space will not be used as a
2741	segregated mmap region.
2742	*/
2743
2744	char *mbrk = MAP_FAILED;
2745	#if HAVE_TUNABLES
2746	if (mp_.hp_pagesize > `0`)
2747	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size,
2748	mp_.hp_pagesize, mp_.hp_pagesize,
2749	mp_.hp_flags, av);
2750	#endif
2751	if (mbrk == MAP_FAILED)
2752	mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, pagesize,
2753	MMAP_AS_MORECORE_SIZE, `0`, av);
2754	if (mbrk != MAP_FAILED)
2755	{
2756	/ We do not need, and cannot use, another sbrk call to find end /
2757	brk = mbrk;
2758	snd_brk = brk + size;
2759	}
2760	}
2761
2762	if (brk != (char *) (MORECORE_FAILURE))
2763	{
2764	if (mp_.sbrk_base == `0`)
2765	mp_.sbrk_base = brk;
2766	av->system_mem += size;
2767
2768	/*
2769	If MORECORE extends previous space, we can likewise extend top size.
2770	*/
2771
2772	if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
2773	set_head (old_top, (size + old_size) \| PREV_INUSE);
2774
2775	else if (contiguous (av) && old_size && brk < old_end)
2776	/ Oops! Someone else killed our space.. Can't touch anything. /
2777	malloc_printerr ("break adjusted to free malloc space");
2778
2779	/*
2780	Otherwise, make adjustments:
2781
2782	* If the first time through or noncontiguous, we need to call sbrk
2783	just to find out where the end of memory lies.
2784
2785	* We need to ensure that all returned chunks from malloc will meet
2786	MALLOC_ALIGNMENT
2787
2788	* If there was an intervening foreign sbrk, we need to adjust sbrk
2789	request size to account for fact that we will not be able to
2790	combine new space with existing space in old_top.
2791
2792	* Almost all systems internally allocate whole pages at a time, in
2793	which case we might as well use the whole last page of request.
2794	So we allocate enough more memory to hit a page boundary now,
2795	which in turn causes future contiguous calls to page-align.
2796	*/
2797
2798	else
2799	{
2800	front_misalign = `0`;
2801	end_misalign = `0`;
2802	correction = `0`;
2803	aligned_brk = brk;
2804
2805	/ handle contiguous cases /
2806	if (contiguous (av))
2807	{
2808	/ Count foreign sbrk as system_mem. /
2809	if (old_size)
2810	av->system_mem += brk - old_end;
2811
2812	/ Guarantee alignment of first new chunk made from this space /
2813
2814	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2815	if (front_misalign > `0`)
2816	{
2817	/*
2818	Skip over some bytes to arrive at an aligned position.
2819	We don't need to specially mark these wasted front bytes.
2820	They will never be accessed anyway because
2821	prev_inuse of av->top (and any chunk created from its start)
2822	is always true after initialization.
2823	*/
2824
2825	correction = MALLOC_ALIGNMENT - front_misalign;
2826	aligned_brk += correction;
2827	}
2828
2829	/*
2830	If this isn't adjacent to existing space, then we will not
2831	be able to merge with old_top space, so must add to 2nd request.
2832	*/
2833
2834	correction += old_size;
2835
2836	/ Extend the end address to hit a page boundary /
2837	end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
2838	correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
2839
2840	assert (correction >= `0`);
2841	snd_brk = (char *) (MORECORE (correction));
2842
2843	/*
2844	If can't allocate correction, try to at least find out current
2845	brk. It might be enough to proceed without failing.
2846
2847	Note that if second sbrk did NOT fail, we assume that space
2848	is contiguous with first sbrk. This is a safe assumption unless
2849	program is multithreaded but doesn't use locks and a foreign sbrk
2850	occurred between our first and second calls.
2851	*/
2852
2853	if (snd_brk == (char *) (MORECORE_FAILURE))
2854	{
2855	correction = `0`;
2856	snd_brk = (char *) (MORECORE (`0`));
2857	}
2858	else
2859	madvise_thp (snd_brk, correction);
2860	}
2861
2862	/ handle non-contiguous cases /
2863	else
2864	{
2865	if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2866	/ MORECORE/mmap must correctly align /
2867	assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == `0`);
2868	else
2869	{
2870	front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2871	if (front_misalign > `0`)
2872	{
2873	/*
2874	Skip over some bytes to arrive at an aligned position.
2875	We don't need to specially mark these wasted front bytes.
2876	They will never be accessed anyway because
2877	prev_inuse of av->top (and any chunk created from its start)
2878	is always true after initialization.
2879	*/
2880
2881	aligned_brk += MALLOC_ALIGNMENT - front_misalign;
2882	}
2883	}
2884
2885	/ Find out current end of memory /
2886	if (snd_brk == (char *) (MORECORE_FAILURE))
2887	{
2888	snd_brk = (char *) (MORECORE (`0`));
2889	}
2890	}
2891
2892	/ Adjust top based on results of second sbrk /
2893	if (snd_brk != (char *) (MORECORE_FAILURE))
2894	{
2895	av->top = (mchunkptr) aligned_brk;
2896	set_head (av->top, (snd_brk - aligned_brk + correction) \| PREV_INUSE);
2897	av->system_mem += correction;
2898
2899	/*
2900	If not the first time through, we either have a
2901	gap due to foreign sbrk or a non-contiguous region. Insert a
2902	double fencepost at old_top to prevent consolidation with space
2903	we don't own. These fenceposts are artificial chunks that are
2904	marked as inuse and are in any case too small to use. We need
2905	two to make sizes and alignments work out.
2906	*/
2907
2908	if (old_size != `0`)
2909	{
2910	/*
2911	Shrink old_top to insert fenceposts, keeping size a
2912	multiple of MALLOC_ALIGNMENT. We know there is at least
2913	enough space in old_top to do this.
2914	*/
2915	old_size = (old_size - `2` * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
2916	set_head (old_top, old_size \| PREV_INUSE);
2917
2918	/*
2919	Note that the following assignments completely overwrite
2920	old_top when old_size was previously MINSIZE. This is
2921	intentional. We need the fencepost, even if old_top otherwise gets
2922	lost.
2923	*/
2924	set_head (chunk_at_offset (old_top, old_size),
2925	CHUNK_HDR_SZ \| PREV_INUSE);
2926	set_head (chunk_at_offset (old_top,
2927	old_size + CHUNK_HDR_SZ),
2928	CHUNK_HDR_SZ \| PREV_INUSE);
2929
2930	/ If possible, release the rest. /
2931	if (old_size >= MINSIZE)
2932	{
2933	_int_free (av, old_top, `1`);
2934	}
2935	}
2936	}
2937	}
2938	}
2939	} / if (av != &main_arena) /
2940
2941	if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
2942	av->max_system_mem = av->system_mem;
2943	check_malloc_state (av);
2944
2945	/ finally, do the allocation /
2946	p = av->top;
2947	size = chunksize (p);
2948
2949	/ check that one of the above allocation paths succeeded /
2950	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
2951	{
2952	remainder_size = size - nb;
2953	remainder = chunk_at_offset (p, nb);
2954	av->top = remainder;
2955	set_head (p, nb \| PREV_INUSE \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
2956	set_head (remainder, remainder_size \| PREV_INUSE);
2957	check_malloced_chunk (av, p, nb);
2958	return chunk2mem (p);
2959	}
2960
2961	/ catch all failure paths /
2962	__set_errno (ENOMEM);
2963	return `0`;
2964	}
2965
2966
2967	/*
2968	systrim is an inverse of sorts to sysmalloc. It gives memory back
2969	to the system (via negative arguments to sbrk) if there is unused
2970	memory at the `high' end of the malloc pool. It is called
2971	automatically by free() when top space exceeds the trim
2972	threshold. It is also called by the public malloc_trim routine. It
2973	returns 1 if it actually released any memory, else 0.
2974	*/
2975
2976	static int
2977	systrim (size_t pad, mstate av)
2978	{
2979	long top_size; / Amount of top-most memory /
2980	long extra; / Amount to release /
2981	long released; / Amount actually released /
2982	char current_brk; /* address returned by pre-check sbrk call /
2983	char new_brk; /* address returned by post-check sbrk call /
2984	long top_area;
2985
2986	top_size = chunksize (av->top);
2987
2988	top_area = top_size - MINSIZE - `1`;
2989	if (top_area <= pad)
2990	return `0`;
2991
2992	/ Release in pagesize units and round down to the nearest page. /
2993	#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2994	if (__glibc_unlikely (mp_.thp_pagesize != `0`))
2995	extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
2996	else
2997	#endif
2998	extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
2999
3000	if (extra == `0`)
3001	return `0`;
3002
3003	/*
3004	Only proceed if end of memory is where we last set it.
3005	This avoids problems if there were foreign sbrk calls.
3006	*/
3007	current_brk = (char *) (MORECORE (`0`));
3008	if (current_brk == (char *) (av->top) + top_size)
3009	{
3010	/*
3011	Attempt to release memory. We ignore MORECORE return value,
3012	and instead call again to find out where new end of memory is.
3013	This avoids problems if first call releases less than we asked,
3014	of if failure somehow altered brk value. (We could still
3015	encounter problems if it altered brk in some very bad way,
3016	but the only thing we can do is adjust anyway, which will cause
3017	some downstream failure.)
3018	*/
3019
3020	MORECORE (-extra);
3021	new_brk = (char *) (MORECORE (`0`));
3022
3023	LIBC_PROBE (memory_sbrk_less, `2`, new_brk, extra);
3024
3025	if (new_brk != (char *) MORECORE_FAILURE)
3026	{
3027	released = (long) (current_brk - new_brk);
3028
3029	if (released != `0`)
3030	{
3031	/ Success. Adjust top. /
3032	av->system_mem -= released;
3033	set_head (av->top, (top_size - released) \| PREV_INUSE);
3034	check_malloc_state (av);
3035	return `1`;
3036	}
3037	}
3038	}
3039	return `0`;
3040	}
3041
3042	static void
3043	munmap_chunk (mchunkptr p)
3044	{
3045	size_t pagesize = GLRO (dl_pagesize);
3046	INTERNAL_SIZE_T size = chunksize (p);
3047
3048	assert (chunk_is_mmapped (p));
3049
3050	uintptr_t mem = (uintptr_t) chunk2mem (p);
3051	uintptr_t block = (uintptr_t) p - prev_size (p);
3052	size_t total_size = prev_size (p) + size;
3053	/ Unfortunately we have to do the compilers job by hand here. Normally*
3054	we would test BLOCK and TOTAL-SIZE separately for compliance with the
3055	page size. But gcc does not recognize the optimization possibility
3056	(in the moment at least) so we combine the two values into one before
3057	the bit test. /*
3058	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3059	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3060	malloc_printerr ("munmap_chunk(): invalid pointer");
3061
3062	atomic_decrement (&mp_.n_mmaps);
3063	atomic_add (&mp_.mmapped_mem, -total_size);
3064
3065	/ If munmap failed the process virtual memory address space is in a*
3066	bad shape. Just leave the block hanging around, the process will
3067	terminate shortly anyway since not much can be done. /*
3068	__munmap ((char *) block, total_size);
3069	}
3070
3071	#if HAVE_MREMAP
3072
3073	static mchunkptr
3074	mremap_chunk (mchunkptr p, size_t new_size)
3075	{
3076	size_t pagesize = GLRO (dl_pagesize);
3077	INTERNAL_SIZE_T offset = prev_size (p);
3078	INTERNAL_SIZE_T size = chunksize (p);
3079	char *cp;
3080
3081	assert (chunk_is_mmapped (p));
3082
3083	uintptr_t block = (uintptr_t) p - offset;
3084	uintptr_t mem = (uintptr_t) chunk2mem(p);
3085	size_t total_size = offset + size;
3086	if (__glibc_unlikely ((block \| total_size) & (pagesize - `1`)) != `0`
3087	\|\| __glibc_unlikely (!powerof2 (mem & (pagesize - `1`))))
3088	malloc_printerr("mremap_chunk(): invalid pointer");
3089
3090	/ Note the extra SIZE_SZ overhead as in mmap_chunk(). /
3091	new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
3092
3093	/ No need to remap if the number of pages does not change. /
3094	if (total_size == new_size)
3095	return p;
3096
3097	cp = (char ) __mremap ((char* *) block, total_size, new_size,
3098	MREMAP_MAYMOVE);
3099
3100	if (cp == MAP_FAILED)
3101	return `0`;
3102
3103	madvise_thp (cp, new_size);
3104
3105	p = (mchunkptr) (cp + offset);
3106
3107	assert (aligned_OK (chunk2mem (p)));
3108
3109	assert (prev_size (p) == offset);
3110	set_head (p, (new_size - offset) \| IS_MMAPPED);
3111
3112	INTERNAL_SIZE_T new;
3113	new = atomic_exchange_and_add (&mp_.mmapped_mem, new_size - size - offset)
3114	+ new_size - size - offset;
3115	atomic_max (&mp_.max_mmapped_mem, new);
3116	return p;
3117	}
3118	#endif /* HAVE_MREMAP */
3119
3120	/------------------------ Public wrappers. --------------------------------/
3121
3122	#if USE_TCACHE
3123
3124	/ We overlay this structure on the user-data portion of a chunk when*
3125	the chunk is stored in the per-thread cache. /*
3126	typedef struct tcache_entry
3127	{
3128	struct tcache_entry *next;
3129	/ This field exists to detect double frees. /
3130	uintptr_t key;
3131	} tcache_entry;
3132
3133	/ There is one of these for each thread, which contains the*
3134	per-thread cache (hence "tcache_perthread_struct"). Keeping
3135	overall size low is mildly important. Note that COUNTS and ENTRIES
3136	are redundant (we could have just counted the linked list each
3137	time), this is for performance reasons. /*
3138	typedef struct tcache_perthread_struct
3139	{
3140	uint16_t counts[TCACHE_MAX_BINS];
3141	tcache_entry *entries[TCACHE_MAX_BINS];
3142	} tcache_perthread_struct;
3143
3144	static __thread bool tcache_shutting_down = false;
3145	static __thread tcache_perthread_struct *tcache = NULL;
3146
3147	/ Process-wide key to try and catch a double-free in the same thread. /
3148	static uintptr_t tcache_key;
3149
3150	/ The value of tcache_key does not really have to be a cryptographically*
3151	secure random number. It only needs to be arbitrary enough so that it does
3152	not collide with values present in applications. If a collision does happen
3153	consistently enough, it could cause a degradation in performance since the
3154	entire list is checked to check if the block indeed has been freed the
3155	second time. The odds of this happening are exceedingly low though, about 1
3156	in 2^wordsize. There is probably a higher chance of the performance
3157	degradation being due to a double free where the first free happened in a
3158	different thread; that's a case this check does not cover. /*
3159	static void
3160	tcache_key_initialize (void)
3161	{
3162	if (__getrandom (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
3163	!= sizeof (tcache_key))
3164	{
3165	tcache_key = random_bits ();
3166	#if __WORDSIZE == 64
3167	tcache_key = (tcache_key << `32`) \| random_bits ();
3168	#endif
3169	}
3170	}
3171
3172	/ Caller must ensure that we know tc_idx is valid and there's room*
3173	for more chunks. /*
3174	static __always_inline void
3175	tcache_put (mchunkptr chunk, size_t tc_idx)
3176	{
3177	tcache_entry e = (tcache_entry ) chunk2mem (chunk);
3178
3179	/ Mark this chunk as "in the tcache" so the test in _int_free will*
3180	detect a double free. /*
3181	e->key = tcache_key;
3182
3183	e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
3184	tcache->entries[tc_idx] = e;
3185	++(tcache->counts[tc_idx]);
3186	}
3187
3188	/ Caller must ensure that we know tc_idx is valid and there's*
3189	available chunks to remove. /*
3190	static __always_inline void *
3191	tcache_get (size_t tc_idx)
3192	{
3193	tcache_entry *e = tcache->entries[tc_idx];
3194	if (__glibc_unlikely (!aligned_OK (e)))
3195	malloc_printerr ("malloc(): unaligned tcache chunk detected");
3196	tcache->entries[tc_idx] = REVEAL_PTR (e->next);
3197	--(tcache->counts[tc_idx]);
3198	e->key = `0`;
3199	return (void *) e;
3200	}
3201
3202	static void
3203	tcache_thread_shutdown (void)
3204	{
3205	int i;
3206	tcache_perthread_struct *tcache_tmp = tcache;
3207
3208	tcache_shutting_down = true;
3209
3210	if (!tcache)
3211	return;
3212
3213	/ Disable the tcache and prevent it from being reinitialized. /
3214	tcache = NULL;
3215
3216	/ Free all of the entries and the tcache itself back to the arena*
3217	heap for coalescing. /*
3218	for (i = `0`; i < TCACHE_MAX_BINS; ++i)
3219	{
3220	while (tcache_tmp->entries[i])
3221	{
3222	tcache_entry *e = tcache_tmp->entries[i];
3223	if (__glibc_unlikely (!aligned_OK (e)))
3224	malloc_printerr ("tcache_thread_shutdown(): "
3225	"unaligned tcache chunk detected");
3226	tcache_tmp->entries[i] = REVEAL_PTR (e->next);
3227	__libc_free (e);
3228	}
3229	}
3230
3231	__libc_free (tcache_tmp);
3232	}
3233
3234	static void
3235	tcache_init(void)
3236	{
3237	mstate ar_ptr;
3238	void *victim = `0`;
3239	const size_t bytes = sizeof (tcache_perthread_struct);
3240
3241	if (tcache_shutting_down)
3242	return;
3243
3244	arena_get (ar_ptr, bytes);
3245	victim = _int_malloc (ar_ptr, bytes);
3246	if (!victim && ar_ptr != NULL)
3247	{
3248	ar_ptr = arena_get_retry (ar_ptr, bytes);
3249	victim = _int_malloc (ar_ptr, bytes);
3250	}
3251
3252
3253	if (ar_ptr != NULL)
3254	__libc_lock_unlock (ar_ptr->mutex);
3255
3256	/ In a low memory situation, we may not be able to allocate memory*
3257	- in which case, we just keep trying later. However, we
3258	typically do this very early, so either there is sufficient
3259	memory, or there isn't enough memory to do non-trivial
3260	allocations anyway. /*
3261	if (victim)
3262	{
3263	tcache = (tcache_perthread_struct *) victim;
3264	memset (tcache, `0`, sizeof (tcache_perthread_struct));
3265	}
3266
3267	}
3268
3269	# define MAYBE_INIT_TCACHE() \
3270	if (__glibc_unlikely (tcache == NULL)) \
3271	tcache_init();
3272
3273	#else /* !USE_TCACHE */
3274	# define MAYBE_INIT_TCACHE()
3275
3276	static void
3277	tcache_thread_shutdown (void)
3278	{
3279	/ Nothing to do if there is no thread cache. /
3280	}
3281
3282	#endif /* !USE_TCACHE */
3283
3284	#if IS_IN (libc)
3285	void *
3286	__libc_malloc (size_t bytes)
3287	{
3288	mstate ar_ptr;
3289	void *victim;
3290
3291	_Static_assert (PTRDIFF_MAX <= SIZE_MAX / `2`,
3292	"PTRDIFF_MAX is not more than half of SIZE_MAX");
3293
3294	if (!__malloc_initialized)
3295	ptmalloc_init ();
3296	#if USE_TCACHE
3297	/ int_free also calls request2size, be careful to not pad twice. /
3298	size_t tbytes;
3299	if (!checked_request2size (bytes, &tbytes))
3300	{
3301	__set_errno (ENOMEM);
3302	return NULL;
3303	}
3304	size_t tc_idx = csize2tidx (tbytes);
3305
3306	MAYBE_INIT_TCACHE ();
3307
3308	DIAG_PUSH_NEEDS_COMMENT;
3309	if (tc_idx < mp_.tcache_bins
3310	&& tcache
3311	&& tcache->counts[tc_idx] > `0`)
3312	{
3313	victim = tcache_get (tc_idx);
3314	return tag_new_usable (victim);
3315	}
3316	DIAG_POP_NEEDS_COMMENT;
3317	#endif
3318
3319	if (SINGLE_THREAD_P)
3320	{
3321	victim = tag_new_usable (_int_malloc (&main_arena, bytes));
3322	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3323	&main_arena == arena_for_chunk (mem2chunk (victim)));
3324	return victim;
3325	}
3326
3327	arena_get (ar_ptr, bytes);
3328
3329	victim = _int_malloc (ar_ptr, bytes);
3330	/ Retry with another arena only if we were able to find a usable arena*
3331	before. /*
3332	if (!victim && ar_ptr != NULL)
3333	{
3334	LIBC_PROBE (memory_malloc_retry, `1`, bytes);
3335	ar_ptr = arena_get_retry (ar_ptr, bytes);
3336	victim = _int_malloc (ar_ptr, bytes);
3337	}
3338
3339	if (ar_ptr != NULL)
3340	__libc_lock_unlock (ar_ptr->mutex);
3341
3342	victim = tag_new_usable (victim);
3343
3344	assert (!victim \|\| chunk_is_mmapped (mem2chunk (victim)) \|\|
3345	ar_ptr == arena_for_chunk (mem2chunk (victim)));
3346	return victim;
3347	}
3348	libc_hidden_def (__libc_malloc)
3349
3350	void
3351	__libc_free (void *mem)
3352	{
3353	mstate ar_ptr;
3354	mchunkptr p; / chunk corresponding to mem /
3355
3356	if (mem == `0`) / free(0) has no effect /
3357	return;
3358
3359	/ Quickly check that the freed pointer matches the tag for the memory.*
3360	This gives a useful double-free detection. /*
3361	if (__glibc_unlikely (mtag_enabled))
3362	(volatile* char *)mem;
3363
3364	int err = errno;
3365
3366	p = mem2chunk (mem);
3367
3368	if (chunk_is_mmapped (p)) / release mmapped memory. /
3369	{
3370	/ See if the dynamic brk/mmap threshold needs adjusting.*
3371	Dumped fake mmapped chunks do not affect the threshold. /*
3372	if (!mp_.no_dyn_threshold
3373	&& chunksize_nomask (p) > mp_.mmap_threshold
3374	&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
3375	{
3376	mp_.mmap_threshold = chunksize (p);
3377	mp_.trim_threshold = `2` * mp_.mmap_threshold;
3378	LIBC_PROBE (memory_mallopt_free_dyn_thresholds, `2`,
3379	mp_.mmap_threshold, mp_.trim_threshold);
3380	}
3381	munmap_chunk (p);
3382	}
3383	else
3384	{
3385	MAYBE_INIT_TCACHE ();
3386
3387	/ Mark the chunk as belonging to the library again. /
3388	(void)tag_region (chunk2mem (p), memsize (p));
3389
3390	ar_ptr = arena_for_chunk (p);
3391	_int_free (ar_ptr, p, `0`);
3392	}
3393
3394	__set_errno (err);
3395	}
3396	libc_hidden_def (__libc_free)
3397
3398	void *
3399	__libc_realloc (void *oldmem, size_t bytes)
3400	{
3401	mstate ar_ptr;
3402	INTERNAL_SIZE_T nb; / padded request size /
3403
3404	void newp; /* chunk to return /
3405
3406	if (!__malloc_initialized)
3407	ptmalloc_init ();
3408
3409	#if REALLOC_ZERO_BYTES_FREES
3410	if (bytes == `0` && oldmem != NULL)
3411	{
3412	__libc_free (oldmem); return `0`;
3413	}
3414	#endif
3415
3416	/ realloc of null is supposed to be same as malloc /
3417	if (oldmem == `0`)
3418	return __libc_malloc (bytes);
3419
3420	/ Perform a quick check to ensure that the pointer's tag matches the*
3421	memory's tag. /*
3422	if (__glibc_unlikely (mtag_enabled))
3423	(volatile* char*) oldmem;
3424
3425	/ chunk corresponding to oldmem /
3426	const mchunkptr oldp = mem2chunk (oldmem);
3427	/ its size /
3428	const INTERNAL_SIZE_T oldsize = chunksize (oldp);
3429
3430	if (chunk_is_mmapped (oldp))
3431	ar_ptr = NULL;
3432	else
3433	{
3434	MAYBE_INIT_TCACHE ();
3435	ar_ptr = arena_for_chunk (oldp);
3436	}
3437
3438	/ Little security check which won't hurt performance: the allocator*
3439	never wrapps around at the end of the address space. Therefore
3440	we can exclude some size values which might appear here by
3441	accident or by "design" from some intruder. /*
3442	if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, `0`)
3443	\|\| __builtin_expect (misaligned_chunk (oldp), `0`)))
3444	malloc_printerr ("realloc(): invalid pointer");
3445
3446	if (!checked_request2size (bytes, &nb))
3447	{
3448	__set_errno (ENOMEM);
3449	return NULL;
3450	}
3451
3452	if (chunk_is_mmapped (oldp))
3453	{
3454	void *newmem;
3455
3456	#if HAVE_MREMAP
3457	newp = mremap_chunk (oldp, nb);
3458	if (newp)
3459	{
3460	void *newmem = chunk2mem_tag (newp);
3461	/ Give the new block a different tag. This helps to ensure*
3462	that stale handles to the previous mapping are not
3463	reused. There's a performance hit for both us and the
3464	caller for doing this, so we might want to
3465	reconsider. /*
3466	return tag_new_usable (newmem);
3467	}
3468	#endif
3469	/ Note the extra SIZE_SZ overhead. /
3470	if (oldsize - SIZE_SZ >= nb)
3471	return oldmem; / do nothing /
3472
3473	/ Must alloc, copy, free. /
3474	newmem = __libc_malloc (bytes);
3475	if (newmem == `0`)
3476	return `0`; / propagate failure /
3477
3478	memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
3479	munmap_chunk (oldp);
3480	return newmem;
3481	}
3482
3483	if (SINGLE_THREAD_P)
3484	{
3485	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3486	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3487	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3488
3489	return newp;
3490	}
3491
3492	__libc_lock_lock (ar_ptr->mutex);
3493
3494	newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3495
3496	__libc_lock_unlock (ar_ptr->mutex);
3497	assert (!newp \|\| chunk_is_mmapped (mem2chunk (newp)) \|\|
3498	ar_ptr == arena_for_chunk (mem2chunk (newp)));
3499
3500	if (newp == NULL)
3501	{
3502	/ Try harder to allocate memory in other arenas. /
3503	LIBC_PROBE (memory_realloc_retry, `2`, bytes, oldmem);
3504	newp = __libc_malloc (bytes);
3505	if (newp != NULL)
3506	{
3507	size_t sz = memsize (oldp);
3508	memcpy (newp, oldmem, sz);
3509	(void) tag_region (chunk2mem (oldp), sz);
3510	_int_free (ar_ptr, oldp, `0`);
3511	}
3512	}
3513
3514	return newp;
3515	}
3516	libc_hidden_def (__libc_realloc)
3517
3518	void *
3519	__libc_memalign (size_t alignment, size_t bytes)
3520	{
3521	if (!__malloc_initialized)
3522	ptmalloc_init ();
3523
3524	void *address = RETURN_ADDRESS (`0`);
3525	return _mid_memalign (alignment, bytes, address);
3526	}
3527
3528	static void *
3529	_mid_memalign (size_t alignment, size_t bytes, void *address)
3530	{
3531	mstate ar_ptr;
3532	void *p;
3533
3534	/ If we need less alignment than we give anyway, just relay to malloc. /
3535	if (alignment <= MALLOC_ALIGNMENT)
3536	return __libc_malloc (bytes);
3537
3538	/ Otherwise, ensure that it is at least a minimum chunk size /
3539	if (alignment < MINSIZE)
3540	alignment = MINSIZE;
3541
3542	/ If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a*
3543	power of 2 and will cause overflow in the check below. /*
3544	if (alignment > SIZE_MAX / `2` + `1`)
3545	{
3546	__set_errno (EINVAL);
3547	return `0`;
3548	}
3549
3550
3551	/ Make sure alignment is power of 2. /
3552	if (!powerof2 (alignment))
3553	{
3554	size_t a = MALLOC_ALIGNMENT * `2`;
3555	while (a < alignment)
3556	a <<= `1`;
3557	alignment = a;
3558	}
3559
3560	if (SINGLE_THREAD_P)
3561	{
3562	p = _int_memalign (&main_arena, alignment, bytes);
3563	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3564	&main_arena == arena_for_chunk (mem2chunk (p)));
3565	return tag_new_usable (p);
3566	}
3567
3568	arena_get (ar_ptr, bytes + alignment + MINSIZE);
3569
3570	p = _int_memalign (ar_ptr, alignment, bytes);
3571	if (!p && ar_ptr != NULL)
3572	{
3573	LIBC_PROBE (memory_memalign_retry, `2`, bytes, alignment);
3574	ar_ptr = arena_get_retry (ar_ptr, bytes);
3575	p = _int_memalign (ar_ptr, alignment, bytes);
3576	}
3577
3578	if (ar_ptr != NULL)
3579	__libc_lock_unlock (ar_ptr->mutex);
3580
3581	assert (!p \|\| chunk_is_mmapped (mem2chunk (p)) \|\|
3582	ar_ptr == arena_for_chunk (mem2chunk (p)));
3583	return tag_new_usable (p);
3584	}
3585	/ For ISO C11. /
3586	weak_alias (__libc_memalign, aligned_alloc)
3587	libc_hidden_def (__libc_memalign)
3588
3589	void *
3590	__libc_valloc (size_t bytes)
3591	{
3592	if (!__malloc_initialized)
3593	ptmalloc_init ();
3594
3595	void *address = RETURN_ADDRESS (`0`);
3596	size_t pagesize = GLRO (dl_pagesize);
3597	return _mid_memalign (pagesize, bytes, address);
3598	}
3599
3600	void *
3601	__libc_pvalloc (size_t bytes)
3602	{
3603	if (!__malloc_initialized)
3604	ptmalloc_init ();
3605
3606	void *address = RETURN_ADDRESS (`0`);
3607	size_t pagesize = GLRO (dl_pagesize);
3608	size_t rounded_bytes;
3609	/ ALIGN_UP with overflow check. /
3610	if (__glibc_unlikely (__builtin_add_overflow (bytes,
3611	pagesize - `1`,
3612	&rounded_bytes)))
3613	{
3614	__set_errno (ENOMEM);
3615	return `0`;
3616	}
3617	rounded_bytes = rounded_bytes & -(pagesize - `1`);
3618
3619	return _mid_memalign (pagesize, rounded_bytes, address);
3620	}
3621
3622	void *
3623	__libc_calloc (size_t n, size_t elem_size)
3624	{
3625	mstate av;
3626	mchunkptr oldtop;
3627	INTERNAL_SIZE_T sz, oldtopsize;
3628	void *mem;
3629	unsigned long clearsize;
3630	unsigned long nclears;
3631	INTERNAL_SIZE_T *d;
3632	ptrdiff_t bytes;
3633
3634	if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
3635	{
3636	__set_errno (ENOMEM);
3637	return NULL;
3638	}
3639
3640	sz = bytes;
3641
3642	if (!__malloc_initialized)
3643	ptmalloc_init ();
3644
3645	MAYBE_INIT_TCACHE ();
3646
3647	if (SINGLE_THREAD_P)
3648	av = &main_arena;
3649	else
3650	arena_get (av, sz);
3651
3652	if (av)
3653	{
3654	/ Check if we hand out the top chunk, in which case there may be no*
3655	need to clear. /*
3656	#if MORECORE_CLEARS
3657	oldtop = top (av);
3658	oldtopsize = chunksize (top (av));
3659	# if MORECORE_CLEARS < 2
3660	/ Only newly allocated memory is guaranteed to be cleared. /
3661	if (av == &main_arena &&
3662	oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
3663	oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
3664	# endif
3665	if (av != &main_arena)
3666	{
3667	heap_info *heap = heap_for_ptr (oldtop);
3668	if (oldtopsize < (char ) heap + heap->mprotect_size - (char* *) oldtop)
3669	oldtopsize = (char ) heap + heap->mprotect_size - (char* *) oldtop;
3670	}
3671	#endif
3672	}
3673	else
3674	{
3675	/ No usable arenas. /
3676	oldtop = `0`;
3677	oldtopsize = `0`;
3678	}
3679	mem = _int_malloc (av, sz);
3680
3681	assert (!mem \|\| chunk_is_mmapped (mem2chunk (mem)) \|\|
3682	av == arena_for_chunk (mem2chunk (mem)));
3683
3684	if (!SINGLE_THREAD_P)
3685	{
3686	if (mem == `0` && av != NULL)
3687	{
3688	LIBC_PROBE (memory_calloc_retry, `1`, sz);
3689	av = arena_get_retry (av, sz);
3690	mem = _int_malloc (av, sz);
3691	}
3692
3693	if (av != NULL)
3694	__libc_lock_unlock (av->mutex);
3695	}
3696
3697	/ Allocation failed even after a retry. /
3698	if (mem == `0`)
3699	return `0`;
3700
3701	mchunkptr p = mem2chunk (mem);
3702
3703	/ If we are using memory tagging, then we need to set the tags*
3704	regardless of MORECORE_CLEARS, so we zero the whole block while
3705	doing so. /*
3706	if (__glibc_unlikely (mtag_enabled))
3707	return tag_new_zero_region (mem, memsize (p));
3708
3709	INTERNAL_SIZE_T csz = chunksize (p);
3710
3711	/ Two optional cases in which clearing not necessary /
3712	if (chunk_is_mmapped (p))
3713	{
3714	if (__builtin_expect (perturb_byte, `0`))
3715	return memset (mem, `0`, sz);
3716
3717	return mem;
3718	}
3719
3720	#if MORECORE_CLEARS
3721	if (perturb_byte == `0` && (p == oldtop && csz > oldtopsize))
3722	{
3723	/ clear only the bytes from non-freshly-sbrked memory /
3724	csz = oldtopsize;
3725	}
3726	#endif
3727
3728	/ Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that*
3729	contents have an odd number of INTERNAL_SIZE_T-sized words;
3730	minimally 3. /*
3731	d = (INTERNAL_SIZE_T *) mem;
3732	clearsize = csz - SIZE_SZ;
3733	nclears = clearsize / sizeof (INTERNAL_SIZE_T);
3734	assert (nclears >= `3`);
3735
3736	if (nclears > `9`)
3737	return memset (d, `0`, clearsize);
3738
3739	else
3740	{
3741	*(d + `0`) = `0`;
3742	*(d + `1`) = `0`;
3743	*(d + `2`) = `0`;
3744	if (nclears > `4`)
3745	{
3746	*(d + `3`) = `0`;
3747	*(d + `4`) = `0`;
3748	if (nclears > `6`)
3749	{
3750	*(d + `5`) = `0`;
3751	*(d + `6`) = `0`;
3752	if (nclears > `8`)
3753	{
3754	*(d + `7`) = `0`;
3755	*(d + `8`) = `0`;
3756	}
3757	}
3758	}
3759	}
3760
3761	return mem;
3762	}
3763	#endif /* IS_IN (libc) */
3764
3765	/*
3766	------------------------------ malloc ------------------------------
3767	*/
3768
3769	static void *
3770	_int_malloc (mstate av, size_t bytes)
3771	{
3772	INTERNAL_SIZE_T nb; / normalized request size /
3773	unsigned int idx; / associated bin index /
3774	mbinptr bin; / associated bin /
3775
3776	mchunkptr victim; / inspected/selected chunk /
3777	INTERNAL_SIZE_T size; / its size /
3778	int victim_index; / its bin index /
3779
3780	mchunkptr remainder; / remainder from a split /
3781	unsigned long remainder_size; / its size /
3782
3783	unsigned int block; / bit map traverser /
3784	unsigned int bit; / bit map traverser /
3785	unsigned int map; / current word of binmap /
3786
3787	mchunkptr fwd; / misc temp for linking /
3788	mchunkptr bck; / misc temp for linking /
3789
3790	#if USE_TCACHE
3791	size_t tcache_unsorted_count; / count of unsorted chunks processed /
3792	#endif
3793
3794	/*
3795	Convert request size to internal form by adding SIZE_SZ bytes
3796	overhead plus possibly more to obtain necessary alignment and/or
3797	to obtain a size of at least MINSIZE, the smallest allocatable
3798	size. Also, checked_request2size returns false for request sizes
3799	that are so large that they wrap around zero when padded and
3800	aligned.
3801	*/
3802
3803	if (!checked_request2size (bytes, &nb))
3804	{
3805	__set_errno (ENOMEM);
3806	return NULL;
3807	}
3808
3809	/ There are no usable arenas. Fall back to sysmalloc to get a chunk from*
3810	mmap. /*
3811	if (__glibc_unlikely (av == NULL))
3812	{
3813	void *p = sysmalloc (nb, av);
3814	if (p != NULL)
3815	alloc_perturb (p, bytes);
3816	return p;
3817	}
3818
3819	/*
3820	If the size qualifies as a fastbin, first check corresponding bin.
3821	This code is safe to execute even if av is not yet initialized, so we
3822	can try it without checking, which saves some time on this fast path.
3823	*/
3824
3825	#define REMOVE_FB(fb, victim, pp) \
3826	do \
3827	{ \
3828	victim = pp; \
3829	if (victim == NULL) \
3830	break; \
3831	pp = REVEAL_PTR (victim->fd); \
3832	if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp))) \
3833	malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
3834	} \
3835	while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
3836	!= victim); \
3837
3838	if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
3839	{
3840	idx = fastbin_index (nb);
3841	mfastbinptr *fb = &fastbin (av, idx);
3842	mchunkptr pp;
3843	victim = *fb;
3844
3845	if (victim != NULL)
3846	{
3847	if (__glibc_unlikely (misaligned_chunk (victim)))
3848	malloc_printerr ("malloc(): unaligned fastbin chunk detected 2");
3849
3850	if (SINGLE_THREAD_P)
3851	*fb = REVEAL_PTR (victim->fd);
3852	else
3853	REMOVE_FB (fb, pp, victim);
3854	if (__glibc_likely (victim != NULL))
3855	{
3856	size_t victim_idx = fastbin_index (chunksize (victim));
3857	if (__builtin_expect (victim_idx != idx, `0`))
3858	malloc_printerr ("malloc(): memory corruption (fast)");
3859	check_remalloced_chunk (av, victim, nb);
3860	#if USE_TCACHE
3861	/ While we're here, if we see other chunks of the same size,*
3862	stash them in the tcache. /*
3863	size_t tc_idx = csize2tidx (nb);
3864	if (tcache && tc_idx < mp_.tcache_bins)
3865	{
3866	mchunkptr tc_victim;
3867
3868	/ While bin not empty and tcache not full, copy chunks. /
3869	while (tcache->counts[tc_idx] < mp_.tcache_count
3870	&& (tc_victim = *fb) != NULL)
3871	{
3872	if (__glibc_unlikely (misaligned_chunk (tc_victim)))
3873	malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
3874	if (SINGLE_THREAD_P)
3875	*fb = REVEAL_PTR (tc_victim->fd);
3876	else
3877	{
3878	REMOVE_FB (fb, pp, tc_victim);
3879	if (__glibc_unlikely (tc_victim == NULL))
3880	break;
3881	}
3882	tcache_put (tc_victim, tc_idx);
3883	}
3884	}
3885	#endif
3886	void *p = chunk2mem (victim);
3887	alloc_perturb (p, bytes);
3888	return p;
3889	}
3890	}
3891	}
3892
3893	/*
3894	If a small request, check regular bin. Since these "smallbins"
3895	hold one size each, no searching within bins is necessary.
3896	(For a large request, we need to wait until unsorted chunks are
3897	processed to find best fit. But for small ones, fits are exact
3898	anyway, so we can check now, which is faster.)
3899	*/
3900
3901	if (in_smallbin_range (nb))
3902	{
3903	idx = smallbin_index (nb);
3904	bin = bin_at (av, idx);
3905
3906	if ((victim = last (bin)) != bin)
3907	{
3908	bck = victim->bk;
3909	if (__glibc_unlikely (bck->fd != victim))
3910	malloc_printerr ("malloc(): smallbin double linked list corrupted");
3911	set_inuse_bit_at_offset (victim, nb);
3912	bin->bk = bck;
3913	bck->fd = bin;
3914
3915	if (av != &main_arena)
3916	set_non_main_arena (victim);
3917	check_malloced_chunk (av, victim, nb);
3918	#if USE_TCACHE
3919	/ While we're here, if we see other chunks of the same size,*
3920	stash them in the tcache. /*
3921	size_t tc_idx = csize2tidx (nb);
3922	if (tcache && tc_idx < mp_.tcache_bins)
3923	{
3924	mchunkptr tc_victim;
3925
3926	/ While bin not empty and tcache not full, copy chunks over. /
3927	while (tcache->counts[tc_idx] < mp_.tcache_count
3928	&& (tc_victim = last (bin)) != bin)
3929	{
3930	if (tc_victim != `0`)
3931	{
3932	bck = tc_victim->bk;
3933	set_inuse_bit_at_offset (tc_victim, nb);
3934	if (av != &main_arena)
3935	set_non_main_arena (tc_victim);
3936	bin->bk = bck;
3937	bck->fd = bin;
3938
3939	tcache_put (tc_victim, tc_idx);
3940	}
3941	}
3942	}
3943	#endif
3944	void *p = chunk2mem (victim);
3945	alloc_perturb (p, bytes);
3946	return p;
3947	}
3948	}
3949
3950	/*
3951	If this is a large request, consolidate fastbins before continuing.
3952	While it might look excessive to kill all fastbins before
3953	even seeing if there is space available, this avoids
3954	fragmentation problems normally associated with fastbins.
3955	Also, in practice, programs tend to have runs of either small or
3956	large requests, but less often mixtures, so consolidation is not
3957	invoked all that often in most programs. And the programs that
3958	it is called frequently in otherwise tend to fragment.
3959	*/
3960
3961	else
3962	{
3963	idx = largebin_index (nb);
3964	if (atomic_load_relaxed (&av->have_fastchunks))
3965	malloc_consolidate (av);
3966	}
3967
3968	/*
3969	Process recently freed or remaindered chunks, taking one only if
3970	it is exact fit, or, if this a small request, the chunk is remainder from
3971	the most recent non-exact fit. Place other traversed chunks in
3972	bins. Note that this step is the only place in any routine where
3973	chunks are placed in bins.
3974
3975	The outer loop here is needed because we might not realize until
3976	near the end of malloc that we should have consolidated, so must
3977	do so and retry. This happens at most once, and only when we would
3978	otherwise need to expand memory to service a "small" request.
3979	*/
3980
3981	#if USE_TCACHE
3982	INTERNAL_SIZE_T tcache_nb = `0`;
3983	size_t tc_idx = csize2tidx (nb);
3984	if (tcache && tc_idx < mp_.tcache_bins)
3985	tcache_nb = nb;
3986	int return_cached = `0`;
3987
3988	tcache_unsorted_count = `0`;
3989	#endif
3990
3991	for (;; )
3992	{
3993	int iters = `0`;
3994	while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
3995	{
3996	bck = victim->bk;
3997	size = chunksize (victim);
3998	mchunkptr next = chunk_at_offset (victim, size);
3999
4000	if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
4001	\|\| __glibc_unlikely (size > av->system_mem))
4002	malloc_printerr ("malloc(): invalid size (unsorted)");
4003	if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
4004	\|\| __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
4005	malloc_printerr ("malloc(): invalid next size (unsorted)");
4006	if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
4007	malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)");
4008	if (__glibc_unlikely (bck->fd != victim)
4009	\|\| __glibc_unlikely (victim->fd != unsorted_chunks (av)))
4010	malloc_printerr ("malloc(): unsorted double linked list corrupted");
4011	if (__glibc_unlikely (prev_inuse (next)))
4012	malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)");
4013
4014	/*
4015	If a small request, try to use last remainder if it is the
4016	only chunk in unsorted bin. This helps promote locality for
4017	runs of consecutive small requests. This is the only
4018	exception to best-fit, and applies only when there is
4019	no exact fit for a small chunk.
4020	*/
4021
4022	if (in_smallbin_range (nb) &&
4023	bck == unsorted_chunks (av) &&
4024	victim == av->last_remainder &&
4025	(unsigned long) (size) > (unsigned long) (nb + MINSIZE))
4026	{
4027	/ split and reattach remainder /
4028	remainder_size = size - nb;
4029	remainder = chunk_at_offset (victim, nb);
4030	unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
4031	av->last_remainder = remainder;
4032	remainder->bk = remainder->fd = unsorted_chunks (av);
4033	if (!in_smallbin_range (remainder_size))
4034	{
4035	remainder->fd_nextsize = NULL;
4036	remainder->bk_nextsize = NULL;
4037	}
4038
4039	set_head (victim, nb \| PREV_INUSE \|
4040	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4041	set_head (remainder, remainder_size \| PREV_INUSE);
4042	set_foot (remainder, remainder_size);
4043
4044	check_malloced_chunk (av, victim, nb);
4045	void *p = chunk2mem (victim);
4046	alloc_perturb (p, bytes);
4047	return p;
4048	}
4049
4050	/ remove from unsorted list /
4051	if (__glibc_unlikely (bck->fd != victim))
4052	malloc_printerr ("malloc(): corrupted unsorted chunks 3");
4053	unsorted_chunks (av)->bk = bck;
4054	bck->fd = unsorted_chunks (av);
4055
4056	/ Take now instead of binning if exact fit /
4057
4058	if (size == nb)
4059	{
4060	set_inuse_bit_at_offset (victim, size);
4061	if (av != &main_arena)
4062	set_non_main_arena (victim);
4063	#if USE_TCACHE
4064	/ Fill cache first, return to user only if cache fills.*
4065	We may return one of these chunks later. /*
4066	if (tcache_nb
4067	&& tcache->counts[tc_idx] < mp_.tcache_count)
4068	{
4069	tcache_put (victim, tc_idx);
4070	return_cached = `1`;
4071	continue;
4072	}
4073	else
4074	{
4075	#endif
4076	check_malloced_chunk (av, victim, nb);
4077	void *p = chunk2mem (victim);
4078	alloc_perturb (p, bytes);
4079	return p;
4080	#if USE_TCACHE
4081	}
4082	#endif
4083	}
4084
4085	/ place chunk in bin /
4086
4087	if (in_smallbin_range (size))
4088	{
4089	victim_index = smallbin_index (size);
4090	bck = bin_at (av, victim_index);
4091	fwd = bck->fd;
4092	}
4093	else
4094	{
4095	victim_index = largebin_index (size);
4096	bck = bin_at (av, victim_index);
4097	fwd = bck->fd;
4098
4099	/ maintain large bins in sorted order /
4100	if (fwd != bck)
4101	{
4102	/ Or with inuse bit to speed comparisons /
4103	size \|= PREV_INUSE;
4104	/ if smaller than smallest, bypass loop below /
4105	assert (chunk_main_arena (bck->bk));
4106	if ((unsigned long) (size)
4107	< (unsigned long) chunksize_nomask (bck->bk))
4108	{
4109	fwd = bck;
4110	bck = bck->bk;
4111
4112	victim->fd_nextsize = fwd->fd;
4113	victim->bk_nextsize = fwd->fd->bk_nextsize;
4114	fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
4115	}
4116	else
4117	{
4118	assert (chunk_main_arena (fwd));
4119	while ((unsigned long) size < chunksize_nomask (fwd))
4120	{
4121	fwd = fwd->fd_nextsize;
4122	assert (chunk_main_arena (fwd));
4123	}
4124
4125	if ((unsigned long) size
4126	== (unsigned long) chunksize_nomask (fwd))
4127	/ Always insert in the second position. /
4128	fwd = fwd->fd;
4129	else
4130	{
4131	victim->fd_nextsize = fwd;
4132	victim->bk_nextsize = fwd->bk_nextsize;
4133	if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
4134	malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)");
4135	fwd->bk_nextsize = victim;
4136	victim->bk_nextsize->fd_nextsize = victim;
4137	}
4138	bck = fwd->bk;
4139	if (bck->fd != fwd)
4140	malloc_printerr ("malloc(): largebin double linked list corrupted (bk)");
4141	}
4142	}
4143	else
4144	victim->fd_nextsize = victim->bk_nextsize = victim;
4145	}
4146
4147	mark_bin (av, victim_index);
4148	victim->bk = bck;
4149	victim->fd = fwd;
4150	fwd->bk = victim;
4151	bck->fd = victim;
4152
4153	#if USE_TCACHE
4154	/ If we've processed as many chunks as we're allowed while*
4155	filling the cache, return one of the cached ones. /*
4156	++tcache_unsorted_count;
4157	if (return_cached
4158	&& mp_.tcache_unsorted_limit > `0`
4159	&& tcache_unsorted_count > mp_.tcache_unsorted_limit)
4160	{
4161	return tcache_get (tc_idx);
4162	}
4163	#endif
4164
4165	#define MAX_ITERS 10000
4166	if (++iters >= MAX_ITERS)
4167	break;
4168	}
4169
4170	#if USE_TCACHE
4171	/ If all the small chunks we found ended up cached, return one now. /
4172	if (return_cached)
4173	{
4174	return tcache_get (tc_idx);
4175	}
4176	#endif
4177
4178	/*
4179	If a large request, scan through the chunks of current bin in
4180	sorted order to find smallest that fits. Use the skip list for this.
4181	*/
4182
4183	if (!in_smallbin_range (nb))
4184	{
4185	bin = bin_at (av, idx);
4186
4187	/ skip scan if empty or largest chunk is too small /
4188	if ((victim = first (bin)) != bin
4189	&& (unsigned long) chunksize_nomask (victim)
4190	>= (unsigned long) (nb))
4191	{
4192	victim = victim->bk_nextsize;
4193	while (((unsigned long) (size = chunksize (victim)) <
4194	(unsigned long) (nb)))
4195	victim = victim->bk_nextsize;
4196
4197	/ Avoid removing the first entry for a size so that the skip*
4198	list does not have to be rerouted. /*
4199	if (victim != last (bin)
4200	&& chunksize_nomask (victim)
4201	== chunksize_nomask (victim->fd))
4202	victim = victim->fd;
4203
4204	remainder_size = size - nb;
4205	unlink_chunk (av, victim);
4206
4207	/ Exhaust /
4208	if (remainder_size < MINSIZE)
4209	{
4210	set_inuse_bit_at_offset (victim, size);
4211	if (av != &main_arena)
4212	set_non_main_arena (victim);
4213	}
4214	/ Split /
4215	else
4216	{
4217	remainder = chunk_at_offset (victim, nb);
4218	/ We cannot assume the unsorted list is empty and therefore*
4219	have to perform a complete insert here. /*
4220	bck = unsorted_chunks (av);
4221	fwd = bck->fd;
4222	if (__glibc_unlikely (fwd->bk != bck))
4223	malloc_printerr ("malloc(): corrupted unsorted chunks");
4224	remainder->bk = bck;
4225	remainder->fd = fwd;
4226	bck->fd = remainder;
4227	fwd->bk = remainder;
4228	if (!in_smallbin_range (remainder_size))
4229	{
4230	remainder->fd_nextsize = NULL;
4231	remainder->bk_nextsize = NULL;
4232	}
4233	set_head (victim, nb \| PREV_INUSE \|
4234	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4235	set_head (remainder, remainder_size \| PREV_INUSE);
4236	set_foot (remainder, remainder_size);
4237	}
4238	check_malloced_chunk (av, victim, nb);
4239	void *p = chunk2mem (victim);
4240	alloc_perturb (p, bytes);
4241	return p;
4242	}
4243	}
4244
4245	/*
4246	Search for a chunk by scanning bins, starting with next largest
4247	bin. This search is strictly by best-fit; i.e., the smallest
4248	(with ties going to approximately the least recently used) chunk
4249	that fits is selected.
4250
4251	The bitmap avoids needing to check that most blocks are nonempty.
4252	The particular case of skipping all bins during warm-up phases
4253	when no chunks have been returned yet is faster than it might look.
4254	*/
4255
4256	++idx;
4257	bin = bin_at (av, idx);
4258	block = idx2block (idx);
4259	map = av->binmap[block];
4260	bit = idx2bit (idx);
4261
4262	for (;; )
4263	{
4264	/ Skip rest of block if there are no more set bits in this block. /
4265	if (bit > map \|\| bit == `0`)
4266	{
4267	do
4268	{
4269	if (++block >= BINMAPSIZE) / out of bins /
4270	goto use_top;
4271	}
4272	while ((map = av->binmap[block]) == `0`);
4273
4274	bin = bin_at (av, (block << BINMAPSHIFT));
4275	bit = `1`;
4276	}
4277
4278	/ Advance to bin with set bit. There must be one. /
4279	while ((bit & map) == `0`)
4280	{
4281	bin = next_bin (bin);
4282	bit <<= `1`;
4283	assert (bit != `0`);
4284	}
4285
4286	/ Inspect the bin. It is likely to be non-empty /
4287	victim = last (bin);
4288
4289	/ If a false alarm (empty bin), clear the bit. /
4290	if (victim == bin)
4291	{
4292	av->binmap[block] = map &= ~bit; / Write through /
4293	bin = next_bin (bin);
4294	bit <<= `1`;
4295	}
4296
4297	else
4298	{
4299	size = chunksize (victim);
4300
4301	/ We know the first chunk in this bin is big enough to use. /
4302	assert ((unsigned long) (size) >= (unsigned long) (nb));
4303
4304	remainder_size = size - nb;
4305
4306	/ unlink /
4307	unlink_chunk (av, victim);
4308
4309	/ Exhaust /
4310	if (remainder_size < MINSIZE)
4311	{
4312	set_inuse_bit_at_offset (victim, size);
4313	if (av != &main_arena)
4314	set_non_main_arena (victim);
4315	}
4316
4317	/ Split /
4318	else
4319	{
4320	remainder = chunk_at_offset (victim, nb);
4321
4322	/ We cannot assume the unsorted list is empty and therefore*
4323	have to perform a complete insert here. /*
4324	bck = unsorted_chunks (av);
4325	fwd = bck->fd;
4326	if (__glibc_unlikely (fwd->bk != bck))
4327	malloc_printerr ("malloc(): corrupted unsorted chunks 2");
4328	remainder->bk = bck;
4329	remainder->fd = fwd;
4330	bck->fd = remainder;
4331	fwd->bk = remainder;
4332
4333	/ advertise as last remainder /
4334	if (in_smallbin_range (nb))
4335	av->last_remainder = remainder;
4336	if (!in_smallbin_range (remainder_size))
4337	{
4338	remainder->fd_nextsize = NULL;
4339	remainder->bk_nextsize = NULL;
4340	}
4341	set_head (victim, nb \| PREV_INUSE \|
4342	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4343	set_head (remainder, remainder_size \| PREV_INUSE);
4344	set_foot (remainder, remainder_size);
4345	}
4346	check_malloced_chunk (av, victim, nb);
4347	void *p = chunk2mem (victim);
4348	alloc_perturb (p, bytes);
4349	return p;
4350	}
4351	}
4352
4353	use_top:
4354	/*
4355	If large enough, split off the chunk bordering the end of memory
4356	(held in av->top). Note that this is in accord with the best-fit
4357	search rule. In effect, av->top is treated as larger (and thus
4358	less well fitting) than any other available chunk since it can
4359	be extended to be as large as necessary (up to system
4360	limitations).
4361
4362	We require that av->top always exists (i.e., has size >=
4363	MINSIZE) after initialization, so if it would otherwise be
4364	exhausted by current request, it is replenished. (The main
4365	reason for ensuring it exists is that we may need MINSIZE space
4366	to put in fenceposts in sysmalloc.)
4367	*/
4368
4369	victim = av->top;
4370	size = chunksize (victim);
4371
4372	if (__glibc_unlikely (size > av->system_mem))
4373	malloc_printerr ("malloc(): corrupted top size");
4374
4375	if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
4376	{
4377	remainder_size = size - nb;
4378	remainder = chunk_at_offset (victim, nb);
4379	av->top = remainder;
4380	set_head (victim, nb \| PREV_INUSE \|
4381	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4382	set_head (remainder, remainder_size \| PREV_INUSE);
4383
4384	check_malloced_chunk (av, victim, nb);
4385	void *p = chunk2mem (victim);
4386	alloc_perturb (p, bytes);
4387	return p;
4388	}
4389
4390	/ When we are using atomic ops to free fast chunks we can get*
4391	here for all block sizes. /*
4392	else if (atomic_load_relaxed (&av->have_fastchunks))
4393	{
4394	malloc_consolidate (av);
4395	/ restore original bin index /
4396	if (in_smallbin_range (nb))
4397	idx = smallbin_index (nb);
4398	else
4399	idx = largebin_index (nb);
4400	}
4401
4402	/*
4403	Otherwise, relay to handle system-dependent cases
4404	*/
4405	else
4406	{
4407	void *p = sysmalloc (nb, av);
4408	if (p != NULL)
4409	alloc_perturb (p, bytes);
4410	return p;
4411	}
4412	}
4413	}
4414
4415	/*
4416	------------------------------ free ------------------------------
4417	*/
4418
4419	static void
4420	_int_free (mstate av, mchunkptr p, int have_lock)
4421	{
4422	INTERNAL_SIZE_T size; / its size /
4423	mfastbinptr fb; /* associated fastbin /
4424	mchunkptr nextchunk; / next contiguous chunk /
4425	INTERNAL_SIZE_T nextsize; / its size /
4426	int nextinuse; / true if nextchunk is used /
4427	INTERNAL_SIZE_T prevsize; / size of previous contiguous chunk /
4428	mchunkptr bck; / misc temp for linking /
4429	mchunkptr fwd; / misc temp for linking /
4430
4431	size = chunksize (p);
4432
4433	/ Little security check which won't hurt performance: the*
4434	allocator never wrapps around at the end of the address space.
4435	Therefore we can exclude some size values which might appear
4436	here by accident or by "design" from some intruder. /*
4437	if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, `0`)
4438	\|\| __builtin_expect (misaligned_chunk (p), `0`))
4439	malloc_printerr ("free(): invalid pointer");
4440	/ We know that each chunk is at least MINSIZE bytes in size or a*
4441	multiple of MALLOC_ALIGNMENT. /*
4442	if (__glibc_unlikely (size < MINSIZE \|\| !aligned_OK (size)))
4443	malloc_printerr ("free(): invalid size");
4444
4445	check_inuse_chunk(av, p);
4446
4447	#if USE_TCACHE
4448	{
4449	size_t tc_idx = csize2tidx (size);
4450	if (tcache != NULL && tc_idx < mp_.tcache_bins)
4451	{
4452	/ Check to see if it's already in the tcache. /
4453	tcache_entry e = (tcache_entry ) chunk2mem (p);
4454
4455	/ This test succeeds on double free. However, we don't 100%*
4456	trust it (it also matches random payload data at a 1 in
4457	2^<size_t> chance), so verify it's not an unlikely
4458	coincidence before aborting. /*
4459	if (__glibc_unlikely (e->key == tcache_key))
4460	{
4461	tcache_entry *tmp;
4462	size_t cnt = `0`;
4463	LIBC_PROBE (memory_tcache_double_free, `2`, e, tc_idx);
4464	for (tmp = tcache->entries[tc_idx];
4465	tmp;
4466	tmp = REVEAL_PTR (tmp->next), ++cnt)
4467	{
4468	if (cnt >= mp_.tcache_count)
4469	malloc_printerr ("free(): too many chunks detected in tcache");
4470	if (__glibc_unlikely (!aligned_OK (tmp)))
4471	malloc_printerr ("free(): unaligned chunk detected in tcache 2");
4472	if (tmp == e)
4473	malloc_printerr ("free(): double free detected in tcache 2");
4474	/ If we get here, it was a coincidence. We've wasted a*
4475	few cycles, but don't abort. /*
4476	}
4477	}
4478
4479	if (tcache->counts[tc_idx] < mp_.tcache_count)
4480	{
4481	tcache_put (p, tc_idx);
4482	return;
4483	}
4484	}
4485	}
4486	#endif
4487
4488	/*
4489	If eligible, place chunk on a fastbin so it can be found
4490	and used quickly in malloc.
4491	*/
4492
4493	if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
4494
4495	#if TRIM_FASTBINS
4496	/*
4497	If TRIM_FASTBINS set, don't place chunks
4498	bordering top into fastbins
4499	*/
4500	&& (chunk_at_offset(p, size) != av->top)
4501	#endif
4502	) {
4503
4504	if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
4505	<= CHUNK_HDR_SZ, `0`)
4506	\|\| __builtin_expect (chunksize (chunk_at_offset (p, size))
4507	>= av->system_mem, `0`))
4508	{
4509	bool fail = true;
4510	/ We might not have a lock at this point and concurrent modifications*
4511	of system_mem might result in a false positive. Redo the test after
4512	getting the lock. /*
4513	if (!have_lock)
4514	{
4515	__libc_lock_lock (av->mutex);
4516	fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
4517	\|\| chunksize (chunk_at_offset (p, size)) >= av->system_mem);
4518	__libc_lock_unlock (av->mutex);
4519	}
4520
4521	if (fail)
4522	malloc_printerr ("free(): invalid next size (fast)");
4523	}
4524
4525	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4526
4527	atomic_store_relaxed (&av->have_fastchunks, true);
4528	unsigned int idx = fastbin_index(size);
4529	fb = &fastbin (av, idx);
4530
4531	/ Atomically link P to its fastbin: P->FD = FB; FB = P; /
4532	mchunkptr old = *fb, old2;
4533
4534	if (SINGLE_THREAD_P)
4535	{
4536	/ Check that the top of the bin is not the record we are going to*
4537	add (i.e., double free). /*
4538	if (__builtin_expect (old == p, `0`))
4539	malloc_printerr ("double free or corruption (fasttop)");
4540	p->fd = PROTECT_PTR (&p->fd, old);
4541	*fb = p;
4542	}
4543	else
4544	do
4545	{
4546	/ Check that the top of the bin is not the record we are going to*
4547	add (i.e., double free). /*
4548	if (__builtin_expect (old == p, `0`))
4549	malloc_printerr ("double free or corruption (fasttop)");
4550	old2 = old;
4551	p->fd = PROTECT_PTR (&p->fd, old);
4552	}
4553	while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
4554	!= old2);
4555
4556	/ Check that size of fastbin chunk at the top is the same as*
4557	size of the chunk that we are adding. We can dereference OLD
4558	only if we have the lock, otherwise it might have already been
4559	allocated again. /*
4560	if (have_lock && old != NULL
4561	&& __builtin_expect (fastbin_index (chunksize (old)) != idx, `0`))
4562	malloc_printerr ("invalid fastbin entry (free)");
4563	}
4564
4565	/*
4566	Consolidate other non-mmapped chunks as they arrive.
4567	*/
4568
4569	else if (!chunk_is_mmapped(p)) {
4570
4571	/ If we're single-threaded, don't lock the arena. /
4572	if (SINGLE_THREAD_P)
4573	have_lock = true;
4574
4575	if (!have_lock)
4576	__libc_lock_lock (av->mutex);
4577
4578	nextchunk = chunk_at_offset(p, size);
4579
4580	/ Lightweight tests: check whether the block is already the*
4581	top block. /*
4582	if (__glibc_unlikely (p == av->top))
4583	malloc_printerr ("double free or corruption (top)");
4584	/ Or whether the next chunk is beyond the boundaries of the arena. /
4585	if (__builtin_expect (contiguous (av)
4586	&& (char *) nextchunk
4587	>= ((char *) av->top + chunksize(av->top)), `0`))
4588	malloc_printerr ("double free or corruption (out)");
4589	/ Or whether the block is actually not marked used. /
4590	if (__glibc_unlikely (!prev_inuse(nextchunk)))
4591	malloc_printerr ("double free or corruption (!prev)");
4592
4593	nextsize = chunksize(nextchunk);
4594	if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, `0`)
4595	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4596	malloc_printerr ("free(): invalid next size (normal)");
4597
4598	free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
4599
4600	/ consolidate backward /
4601	if (!prev_inuse(p)) {
4602	prevsize = prev_size (p);
4603	size += prevsize;
4604	p = chunk_at_offset(p, -((long) prevsize));
4605	if (__glibc_unlikely (chunksize(p) != prevsize))
4606	malloc_printerr ("corrupted size vs. prev_size while consolidating");
4607	unlink_chunk (av, p);
4608	}
4609
4610	if (nextchunk != av->top) {
4611	/ get and clear inuse bit /
4612	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4613
4614	/ consolidate forward /
4615	if (!nextinuse) {
4616	unlink_chunk (av, nextchunk);
4617	size += nextsize;
4618	} else
4619	clear_inuse_bit_at_offset(nextchunk, `0`);
4620
4621	/*
4622	Place the chunk in unsorted chunk list. Chunks are
4623	not placed into regular bins until after they have
4624	been given one chance to be used in malloc.
4625	*/
4626
4627	bck = unsorted_chunks(av);
4628	fwd = bck->fd;
4629	if (__glibc_unlikely (fwd->bk != bck))
4630	malloc_printerr ("free(): corrupted unsorted chunks");
4631	p->fd = fwd;
4632	p->bk = bck;
4633	if (!in_smallbin_range(size))
4634	{
4635	p->fd_nextsize = NULL;
4636	p->bk_nextsize = NULL;
4637	}
4638	bck->fd = p;
4639	fwd->bk = p;
4640
4641	set_head(p, size \| PREV_INUSE);
4642	set_foot(p, size);
4643
4644	check_free_chunk(av, p);
4645	}
4646
4647	/*
4648	If the chunk borders the current high end of memory,
4649	consolidate into top
4650	*/
4651
4652	else {
4653	size += nextsize;
4654	set_head(p, size \| PREV_INUSE);
4655	av->top = p;
4656	check_chunk(av, p);
4657	}
4658
4659	/*
4660	If freeing a large space, consolidate possibly-surrounding
4661	chunks. Then, if the total unused topmost memory exceeds trim
4662	threshold, ask malloc_trim to reduce top.
4663
4664	Unless max_fast is 0, we don't know if there are fastbins
4665	bordering top, so we cannot tell for sure whether threshold
4666	has been reached unless fastbins are consolidated. But we
4667	don't want to consolidate on each free. As a compromise,
4668	consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
4669	is reached.
4670	*/
4671
4672	if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
4673	if (atomic_load_relaxed (&av->have_fastchunks))
4674	malloc_consolidate(av);
4675
4676	if (av == &main_arena) {
4677	#ifndef MORECORE_CANNOT_TRIM
4678	if ((unsigned long)(chunksize(av->top)) >=
4679	(unsigned long)(mp_.trim_threshold))
4680	systrim(mp_.top_pad, av);
4681	#endif
4682	} else {
4683	/ Always try heap_trim(), even if the top chunk is not*
4684	large, because the corresponding heap might go away. /*
4685	heap_info *heap = heap_for_ptr(top(av));
4686
4687	assert(heap->ar_ptr == av);
4688	heap_trim(heap, mp_.top_pad);
4689	}
4690	}
4691
4692	if (!have_lock)
4693	__libc_lock_unlock (av->mutex);
4694	}
4695	/*
4696	If the chunk was allocated via mmap, release via munmap().
4697	*/
4698
4699	else {
4700	munmap_chunk (p);
4701	}
4702	}
4703
4704	/*
4705	------------------------- malloc_consolidate -------------------------
4706
4707	malloc_consolidate is a specialized version of free() that tears
4708	down chunks held in fastbins. Free itself cannot be used for this
4709	purpose since, among other things, it might place chunks back onto
4710	fastbins. So, instead, we need to use a minor variant of the same
4711	code.
4712	*/
4713
4714	static void malloc_consolidate(mstate av)
4715	{
4716	mfastbinptr* fb; / current fastbin being consolidated /
4717	mfastbinptr* maxfb; / last fastbin (for loop control) /
4718	mchunkptr p; / current chunk being consolidated /
4719	mchunkptr nextp; / next chunk to consolidate /
4720	mchunkptr unsorted_bin; / bin header /
4721	mchunkptr first_unsorted; / chunk to link to /
4722
4723	/ These have same use as in free() /
4724	mchunkptr nextchunk;
4725	INTERNAL_SIZE_T size;
4726	INTERNAL_SIZE_T nextsize;
4727	INTERNAL_SIZE_T prevsize;
4728	int nextinuse;
4729
4730	atomic_store_relaxed (&av->have_fastchunks, false);
4731
4732	unsorted_bin = unsorted_chunks(av);
4733
4734	/*
4735	Remove each chunk from fast bin and consolidate it, placing it
4736	then in unsorted bin. Among other reasons for doing this,
4737	placing in unsorted bin avoids needing to calculate actual bins
4738	until malloc is sure that chunks aren't immediately going to be
4739	reused anyway.
4740	*/
4741
4742	maxfb = &fastbin (av, NFASTBINS - `1`);
4743	fb = &fastbin (av, `0`);
4744	do {
4745	p = atomic_exchange_acq (fb, NULL);
4746	if (p != `0`) {
4747	do {
4748	{
4749	if (__glibc_unlikely (misaligned_chunk (p)))
4750	malloc_printerr ("malloc_consolidate(): "
4751	"unaligned fastbin chunk detected");
4752
4753	unsigned int idx = fastbin_index (chunksize (p));
4754	if ((&fastbin (av, idx)) != fb)
4755	malloc_printerr ("malloc_consolidate(): invalid chunk size");
4756	}
4757
4758	check_inuse_chunk(av, p);
4759	nextp = REVEAL_PTR (p->fd);
4760
4761	/ Slightly streamlined version of consolidation code in free() /
4762	size = chunksize (p);
4763	nextchunk = chunk_at_offset(p, size);
4764	nextsize = chunksize(nextchunk);
4765
4766	if (!prev_inuse(p)) {
4767	prevsize = prev_size (p);
4768	size += prevsize;
4769	p = chunk_at_offset(p, -((long) prevsize));
4770	if (__glibc_unlikely (chunksize(p) != prevsize))
4771	malloc_printerr ("corrupted size vs. prev_size in fastbins");
4772	unlink_chunk (av, p);
4773	}
4774
4775	if (nextchunk != av->top) {
4776	nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4777
4778	if (!nextinuse) {
4779	size += nextsize;
4780	unlink_chunk (av, nextchunk);
4781	} else
4782	clear_inuse_bit_at_offset(nextchunk, `0`);
4783
4784	first_unsorted = unsorted_bin->fd;
4785	unsorted_bin->fd = p;
4786	first_unsorted->bk = p;
4787
4788	if (!in_smallbin_range (size)) {
4789	p->fd_nextsize = NULL;
4790	p->bk_nextsize = NULL;
4791	}
4792
4793	set_head(p, size \| PREV_INUSE);
4794	p->bk = unsorted_bin;
4795	p->fd = first_unsorted;
4796	set_foot(p, size);
4797	}
4798
4799	else {
4800	size += nextsize;
4801	set_head(p, size \| PREV_INUSE);
4802	av->top = p;
4803	}
4804
4805	} while ( (p = nextp) != `0`);
4806
4807	}
4808	} while (fb++ != maxfb);
4809	}
4810
4811	/*
4812	------------------------------ realloc ------------------------------
4813	*/
4814
4815	static void *
4816	_int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
4817	INTERNAL_SIZE_T nb)
4818	{
4819	mchunkptr newp; / chunk to return /
4820	INTERNAL_SIZE_T newsize; / its size /
4821	void* newmem; / corresponding user mem /
4822
4823	mchunkptr next; / next contiguous chunk after oldp /
4824
4825	mchunkptr remainder; / extra space at end of newp /
4826	unsigned long remainder_size; / its size /
4827
4828	/ oldmem size /
4829	if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, `0`)
4830	\|\| __builtin_expect (oldsize >= av->system_mem, `0`))
4831	malloc_printerr ("realloc(): invalid old size");
4832
4833	check_inuse_chunk (av, oldp);
4834
4835	/ All callers already filter out mmap'ed chunks. /
4836	assert (!chunk_is_mmapped (oldp));
4837
4838	next = chunk_at_offset (oldp, oldsize);
4839	INTERNAL_SIZE_T nextsize = chunksize (next);
4840	if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, `0`)
4841	\|\| __builtin_expect (nextsize >= av->system_mem, `0`))
4842	malloc_printerr ("realloc(): invalid next size");
4843
4844	if ((unsigned long) (oldsize) >= (unsigned long) (nb))
4845	{
4846	/ already big enough; split below /
4847	newp = oldp;
4848	newsize = oldsize;
4849	}
4850
4851	else
4852	{
4853	/ Try to expand forward into top /
4854	if (next == av->top &&
4855	(unsigned long) (newsize = oldsize + nextsize) >=
4856	(unsigned long) (nb + MINSIZE))
4857	{
4858	set_head_size (oldp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4859	av->top = chunk_at_offset (oldp, nb);
4860	set_head (av->top, (newsize - nb) \| PREV_INUSE);
4861	check_inuse_chunk (av, oldp);
4862	return tag_new_usable (chunk2mem (oldp));
4863	}
4864
4865	/ Try to expand forward into next chunk; split off remainder below /
4866	else if (next != av->top &&
4867	!inuse (next) &&
4868	(unsigned long) (newsize = oldsize + nextsize) >=
4869	(unsigned long) (nb))
4870	{
4871	newp = oldp;
4872	unlink_chunk (av, next);
4873	}
4874
4875	/ allocate, copy, free /
4876	else
4877	{
4878	newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK);
4879	if (newmem == `0`)
4880	return `0`; / propagate failure /
4881
4882	newp = mem2chunk (newmem);
4883	newsize = chunksize (newp);
4884
4885	/*
4886	Avoid copy if newp is next chunk after oldp.
4887	*/
4888	if (newp == next)
4889	{
4890	newsize += oldsize;
4891	newp = oldp;
4892	}
4893	else
4894	{
4895	void *oldmem = chunk2mem (oldp);
4896	size_t sz = memsize (oldp);
4897	(void) tag_region (oldmem, sz);
4898	newmem = tag_new_usable (newmem);
4899	memcpy (newmem, oldmem, sz);
4900	_int_free (av, oldp, `1`);
4901	check_inuse_chunk (av, newp);
4902	return newmem;
4903	}
4904	}
4905	}
4906
4907	/ If possible, free extra space in old or extended chunk /
4908
4909	assert ((unsigned long) (newsize) >= (unsigned long) (nb));
4910
4911	remainder_size = newsize - nb;
4912
4913	if (remainder_size < MINSIZE) / not enough extra to split off /
4914	{
4915	set_head_size (newp, newsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4916	set_inuse_bit_at_offset (newp, newsize);
4917	}
4918	else / split remainder /
4919	{
4920	remainder = chunk_at_offset (newp, nb);
4921	/ Clear any user-space tags before writing the header. /
4922	remainder = tag_region (remainder, remainder_size);
4923	set_head_size (newp, nb \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
4924	set_head (remainder, remainder_size \| PREV_INUSE \|
4925	(av != &main_arena ? NON_MAIN_ARENA : `0`));
4926	/ Mark remainder as inuse so free() won't complain /
4927	set_inuse_bit_at_offset (remainder, remainder_size);
4928	_int_free (av, remainder, `1`);
4929	}
4930
4931	check_inuse_chunk (av, newp);
4932	return tag_new_usable (chunk2mem (newp));
4933	}
4934
4935	/*
4936	------------------------------ memalign ------------------------------
4937	*/
4938
4939	static void *
4940	_int_memalign (mstate av, size_t alignment, size_t bytes)
4941	{
4942	INTERNAL_SIZE_T nb; / padded request size /
4943	char m; /* memory returned by malloc call /
4944	mchunkptr p; / corresponding chunk /
4945	char brk; /* alignment point within p /
4946	mchunkptr newp; / chunk to return /
4947	INTERNAL_SIZE_T newsize; / its size /
4948	INTERNAL_SIZE_T leadsize; / leading space before alignment point /
4949	mchunkptr remainder; / spare room at end to split off /
4950	unsigned long remainder_size; / its size /
4951	INTERNAL_SIZE_T size;
4952
4953
4954
4955	if (!checked_request2size (bytes, &nb))
4956	{
4957	__set_errno (ENOMEM);
4958	return NULL;
4959	}
4960
4961	/*
4962	Strategy: find a spot within that chunk that meets the alignment
4963	request, and then possibly free the leading and trailing space.
4964	*/
4965
4966	/ Call malloc with worst case padding to hit alignment. /
4967
4968	m = (char *) (_int_malloc (av, nb + alignment + MINSIZE));
4969
4970	if (m == `0`)
4971	return `0`; / propagate failure /
4972
4973	p = mem2chunk (m);
4974
4975	if ((((unsigned long) (m)) % alignment) != `0`) / misaligned /
4976
4977	{ /*
4978	Find an aligned spot inside chunk. Since we need to give back
4979	leading space in a chunk of at least MINSIZE, if the first
4980	calculation places us at a spot with less than MINSIZE leader,
4981	we can move to the next aligned spot -- we've allocated enough
4982	total room so that this is always possible.
4983	*/
4984	brk = (char ) mem2chunk (((unsigned* long) (m + alignment - `1`)) &
4985	- ((signed long) alignment));
4986	if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
4987	brk += alignment;
4988
4989	newp = (mchunkptr) brk;
4990	leadsize = brk - (char *) (p);
4991	newsize = chunksize (p) - leadsize;
4992
4993	/ For mmapped chunks, just adjust offset /
4994	if (chunk_is_mmapped (p))
4995	{
4996	set_prev_size (newp, prev_size (p) + leadsize);
4997	set_head (newp, newsize \| IS_MMAPPED);
4998	return chunk2mem (newp);
4999	}
5000
5001	/ Otherwise, give back leader, use the rest /
5002	set_head (newp, newsize \| PREV_INUSE \|
5003	(av != &main_arena ? NON_MAIN_ARENA : `0`));
5004	set_inuse_bit_at_offset (newp, newsize);
5005	set_head_size (p, leadsize \| (av != &main_arena ? NON_MAIN_ARENA : `0`));
5006	_int_free (av, p, `1`);
5007	p = newp;
5008
5009	assert (newsize >= nb &&
5010	(((unsigned long) (chunk2mem (p))) % alignment) == `0`);
5011	}
5012
5013	/ Also give back spare room at the end /
5014	if (!chunk_is_mmapped (p))
5015	{
5016	size = chunksize (p);
5017	if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
5018	{
5019	remainder_size = size - nb;
5020	remainder = chunk_at_offset (p, nb);
5021	set_head (remainder, remainder_size \| PREV_INUSE \|
5022	(av != &main_arena ? NON_MAIN_ARENA : `0`));
5023	set_head_size (p, nb);
5024	_int_free (av, remainder, `1`);
5025	}
5026	}
5027
5028	check_inuse_chunk (av, p);
5029	return chunk2mem (p);
5030	}
5031
5032
5033	/*
5034	------------------------------ malloc_trim ------------------------------
5035	*/
5036
5037	static int
5038	mtrim (mstate av, size_t pad)
5039	{
5040	/ Ensure all blocks are consolidated. /
5041	malloc_consolidate (av);
5042
5043	const size_t ps = GLRO (dl_pagesize);
5044	int psindex = bin_index (ps);
5045	const size_t psm1 = ps - `1`;
5046
5047	int result = `0`;
5048	for (int i = `1`; i < NBINS; ++i)
5049	if (i == `1` \|\| i >= psindex)
5050	{
5051	mbinptr bin = bin_at (av, i);
5052
5053	for (mchunkptr p = last (bin); p != bin; p = p->bk)
5054	{
5055	INTERNAL_SIZE_T size = chunksize (p);
5056
5057	if (size > psm1 + sizeof (struct malloc_chunk))
5058	{
5059	/ See whether the chunk contains at least one unused page. /
5060	char paligned_mem = (char* *) (((uintptr_t) p
5061	+ sizeof (struct malloc_chunk)
5062	+ psm1) & ~psm1);
5063
5064	assert ((char ) chunk2mem (p) + `2` CHUNK_HDR_SZ
5065	<= paligned_mem);
5066	assert ((char *) p + size > paligned_mem);
5067
5068	/ This is the size we could potentially free. /
5069	size -= paligned_mem - (char *) p;
5070
5071	if (size > psm1)
5072	{
5073	#if MALLOC_DEBUG
5074	/ When debugging we simulate destroying the memory*
5075	content. /*
5076	memset (paligned_mem, `0x89`, size & ~psm1);
5077	#endif
5078	__madvise (paligned_mem, size & ~psm1, MADV_DONTNEED);
5079
5080	result = `1`;
5081	}
5082	}
5083	}
5084	}
5085
5086	#ifndef MORECORE_CANNOT_TRIM
5087	return result \| (av == &main_arena ? systrim (pad, av) : `0`);
5088
5089	#else
5090	return result;
5091	#endif
5092	}
5093
5094
5095	int
5096	__malloc_trim (size_t s)
5097	{
5098	int result = `0`;
5099
5100	if (!__malloc_initialized)
5101	ptmalloc_init ();
5102
5103	mstate ar_ptr = &main_arena;
5104	do
5105	{
5106	__libc_lock_lock (ar_ptr->mutex);
5107	result \|= mtrim (ar_ptr, s);
5108	__libc_lock_unlock (ar_ptr->mutex);
5109
5110	ar_ptr = ar_ptr->next;
5111	}
5112	while (ar_ptr != &main_arena);
5113
5114	return result;
5115	}
5116
5117
5118	/*
5119	------------------------- malloc_usable_size -------------------------
5120	*/
5121
5122	static size_t
5123	musable (void *mem)
5124	{
5125	mchunkptr p = mem2chunk (mem);
5126
5127	if (chunk_is_mmapped (p))
5128	return chunksize (p) - CHUNK_HDR_SZ;
5129	else if (inuse (p))
5130	return memsize (p);
5131
5132	return `0`;
5133	}
5134
5135	#if IS_IN (libc)
5136	size_t
5137	__malloc_usable_size (void *m)
5138	{
5139	if (m == NULL)
5140	return `0`;
5141	return musable (m);
5142	}
5143	#endif
5144
5145	/*
5146	------------------------------ mallinfo ------------------------------
5147	Accumulate malloc statistics for arena AV into M.
5148	*/
5149	static void
5150	int_mallinfo (mstate av, struct mallinfo2 *m)
5151	{
5152	size_t i;
5153	mbinptr b;
5154	mchunkptr p;
5155	INTERNAL_SIZE_T avail;
5156	INTERNAL_SIZE_T fastavail;
5157	int nblocks;
5158	int nfastblocks;
5159
5160	check_malloc_state (av);
5161
5162	/ Account for top /
5163	avail = chunksize (av->top);
5164	nblocks = `1`; / top always exists /
5165
5166	/ traverse fastbins /
5167	nfastblocks = `0`;
5168	fastavail = `0`;
5169
5170	for (i = `0`; i < NFASTBINS; ++i)
5171	{
5172	for (p = fastbin (av, i);
5173	p != `0`;
5174	p = REVEAL_PTR (p->fd))
5175	{
5176	if (__glibc_unlikely (misaligned_chunk (p)))
5177	malloc_printerr ("int_mallinfo(): "
5178	"unaligned fastbin chunk detected");
5179	++nfastblocks;
5180	fastavail += chunksize (p);
5181	}
5182	}
5183
5184	avail += fastavail;
5185
5186	/ traverse regular bins /
5187	for (i = `1`; i < NBINS; ++i)
5188	{
5189	b = bin_at (av, i);
5190	for (p = last (b); p != b; p = p->bk)
5191	{
5192	++nblocks;
5193	avail += chunksize (p);
5194	}
5195	}
5196
5197	m->smblks += nfastblocks;
5198	m->ordblks += nblocks;
5199	m->fordblks += avail;
5200	m->uordblks += av->system_mem - avail;
5201	m->arena += av->system_mem;
5202	m->fsmblks += fastavail;
5203	if (av == &main_arena)
5204	{
5205	m->hblks = mp_.n_mmaps;
5206	m->hblkhd = mp_.mmapped_mem;
5207	m->usmblks = `0`;
5208	m->keepcost = chunksize (av->top);
5209	}
5210	}
5211
5212
5213	struct mallinfo2
5214	__libc_mallinfo2 (void)
5215	{
5216	struct mallinfo2 m;
5217	mstate ar_ptr;
5218
5219	if (!__malloc_initialized)
5220	ptmalloc_init ();
5221
5222	memset (&m, `0`, sizeof (m));
5223	ar_ptr = &main_arena;
5224	do
5225	{
5226	__libc_lock_lock (ar_ptr->mutex);
5227	int_mallinfo (ar_ptr, &m);
5228	__libc_lock_unlock (ar_ptr->mutex);
5229
5230	ar_ptr = ar_ptr->next;
5231	}
5232	while (ar_ptr != &main_arena);
5233
5234	return m;
5235	}
5236	libc_hidden_def (__libc_mallinfo2)
5237
5238	struct mallinfo
5239	__libc_mallinfo (void)
5240	{
5241	struct mallinfo m;
5242	struct mallinfo2 m2 = __libc_mallinfo2 ();
5243
5244	m.arena = m2.arena;
5245	m.ordblks = m2.ordblks;
5246	m.smblks = m2.smblks;
5247	m.hblks = m2.hblks;
5248	m.hblkhd = m2.hblkhd;
5249	m.usmblks = m2.usmblks;
5250	m.fsmblks = m2.fsmblks;
5251	m.uordblks = m2.uordblks;
5252	m.fordblks = m2.fordblks;
5253	m.keepcost = m2.keepcost;
5254
5255	return m;
5256	}
5257
5258
5259	/*
5260	------------------------------ malloc_stats ------------------------------
5261	*/
5262
5263	void
5264	__malloc_stats (void)
5265	{
5266	int i;
5267	mstate ar_ptr;
5268	unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
5269
5270	if (!__malloc_initialized)
5271	ptmalloc_init ();
5272	_IO_flockfile (stderr);
5273	int old_flags2 = stderr->_flags2;
5274	stderr->_flags2 \|= _IO_FLAGS2_NOTCANCEL;
5275	for (i = `0`, ar_ptr = &main_arena;; i++)
5276	{
5277	struct mallinfo2 mi;
5278
5279	memset (&mi, `0`, sizeof (mi));
5280	__libc_lock_lock (ar_ptr->mutex);
5281	int_mallinfo (ar_ptr, &mi);
5282	fprintf (stderr, "Arena %d:\n", i);
5283	fprintf (stderr, "system bytes = %10u\n", (unsigned int) mi.arena);
5284	fprintf (stderr, "in use bytes = %10u\n", (unsigned int) mi.uordblks);
5285	#if MALLOC_DEBUG > 1
5286	if (i > `0`)
5287	dump_heap (heap_for_ptr (top (ar_ptr)));
5288	#endif
5289	system_b += mi.arena;
5290	in_use_b += mi.uordblks;
5291	__libc_lock_unlock (ar_ptr->mutex);
5292	ar_ptr = ar_ptr->next;
5293	if (ar_ptr == &main_arena)
5294	break;
5295	}
5296	fprintf (stderr, "Total (incl. mmap):\n");
5297	fprintf (stderr, "system bytes = %10u\n", system_b);
5298	fprintf (stderr, "in use bytes = %10u\n", in_use_b);
5299	fprintf (stderr, "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
5300	fprintf (stderr, "max mmap bytes = %10lu\n",
5301	(unsigned long) mp_.max_mmapped_mem);
5302	stderr->_flags2 = old_flags2;
5303	_IO_funlockfile (stderr);
5304	}
5305
5306
5307	/*
5308	------------------------------ mallopt ------------------------------
5309	*/
5310	static __always_inline int
5311	do_set_trim_threshold (size_t value)
5312	{
5313	LIBC_PROBE (memory_mallopt_trim_threshold, `3`, value, mp_.trim_threshold,
5314	mp_.no_dyn_threshold);
5315	mp_.trim_threshold = value;
5316	mp_.no_dyn_threshold = `1`;
5317	return `1`;
5318	}
5319
5320	static __always_inline int
5321	do_set_top_pad (size_t value)
5322	{
5323	LIBC_PROBE (memory_mallopt_top_pad, `3`, value, mp_.top_pad,
5324	mp_.no_dyn_threshold);
5325	mp_.top_pad = value;
5326	mp_.no_dyn_threshold = `1`;
5327	return `1`;
5328	}
5329
5330	static __always_inline int
5331	do_set_mmap_threshold (size_t value)
5332	{
5333	LIBC_PROBE (memory_mallopt_mmap_threshold, `3`, value, mp_.mmap_threshold,
5334	mp_.no_dyn_threshold);
5335	mp_.mmap_threshold = value;
5336	mp_.no_dyn_threshold = `1`;
5337	return `1`;
5338	}
5339
5340	static __always_inline int
5341	do_set_mmaps_max (int32_t value)
5342	{
5343	LIBC_PROBE (memory_mallopt_mmap_max, `3`, value, mp_.n_mmaps_max,
5344	mp_.no_dyn_threshold);
5345	mp_.n_mmaps_max = value;
5346	mp_.no_dyn_threshold = `1`;
5347	return `1`;
5348	}
5349
5350	static __always_inline int
5351	do_set_mallopt_check (int32_t value)
5352	{
5353	return `1`;
5354	}
5355
5356	static __always_inline int
5357	do_set_perturb_byte (int32_t value)
5358	{
5359	LIBC_PROBE (memory_mallopt_perturb, `2`, value, perturb_byte);
5360	perturb_byte = value;
5361	return `1`;
5362	}
5363
5364	static __always_inline int
5365	do_set_arena_test (size_t value)
5366	{
5367	LIBC_PROBE (memory_mallopt_arena_test, `2`, value, mp_.arena_test);
5368	mp_.arena_test = value;
5369	return `1`;
5370	}
5371
5372	static __always_inline int
5373	do_set_arena_max (size_t value)
5374	{
5375	LIBC_PROBE (memory_mallopt_arena_max, `2`, value, mp_.arena_max);
5376	mp_.arena_max = value;
5377	return `1`;
5378	}
5379
5380	#if USE_TCACHE
5381	static __always_inline int
5382	do_set_tcache_max (size_t value)
5383	{
5384	if (value <= MAX_TCACHE_SIZE)
5385	{
5386	LIBC_PROBE (memory_tunable_tcache_max_bytes, `2`, value, mp_.tcache_max_bytes);
5387	mp_.tcache_max_bytes = value;
5388	mp_.tcache_bins = csize2tidx (request2size(value)) + `1`;
5389	return `1`;
5390	}
5391	return `0`;
5392	}
5393
5394	static __always_inline int
5395	do_set_tcache_count (size_t value)
5396	{
5397	if (value <= MAX_TCACHE_COUNT)
5398	{
5399	LIBC_PROBE (memory_tunable_tcache_count, `2`, value, mp_.tcache_count);
5400	mp_.tcache_count = value;
5401	return `1`;
5402	}
5403	return `0`;
5404	}
5405
5406	static __always_inline int
5407	do_set_tcache_unsorted_limit (size_t value)
5408	{
5409	LIBC_PROBE (memory_tunable_tcache_unsorted_limit, `2`, value, mp_.tcache_unsorted_limit);
5410	mp_.tcache_unsorted_limit = value;
5411	return `1`;
5412	}
5413	#endif
5414
5415	static inline int
5416	__always_inline
5417	do_set_mxfast (size_t value)
5418	{
5419	if (value <= MAX_FAST_SIZE)
5420	{
5421	LIBC_PROBE (memory_mallopt_mxfast, `2`, value, get_max_fast ());
5422	set_max_fast (value);
5423	return `1`;
5424	}
5425	return `0`;
5426	}
5427
5428	#if HAVE_TUNABLES
5429	static __always_inline int
5430	do_set_hugetlb (size_t value)
5431	{
5432	if (value == `1`)
5433	{
5434	enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
5435	/*
5436	Only enable THP madvise usage if system does support it and
5437	has 'madvise' mode. Otherwise the madvise() call is wasteful.
5438	*/
5439	if (thp_mode == malloc_thp_mode_madvise)
5440	mp_.thp_pagesize = __malloc_default_thp_pagesize ();
5441	}
5442	else if (value >= `2`)
5443	__malloc_hugepage_config (value == `2` ? `0` : value, &mp_.hp_pagesize,
5444	&mp_.hp_flags);
5445	return `0`;
5446	}
5447	#endif
5448
5449	int
5450	__libc_mallopt (int param_number, int value)
5451	{
5452	mstate av = &main_arena;
5453	int res = `1`;
5454
5455	if (!__malloc_initialized)
5456	ptmalloc_init ();
5457	__libc_lock_lock (av->mutex);
5458
5459	LIBC_PROBE (memory_mallopt, `2`, param_number, value);
5460
5461	/ We must consolidate main arena before changing max_fast*
5462	(see definition of set_max_fast). /*
5463	malloc_consolidate (av);
5464
5465	/ Many of these helper functions take a size_t. We do not worry*
5466	about overflow here, because negative int values will wrap to
5467	very large size_t values and the helpers have sufficient range
5468	checking for such conversions. Many of these helpers are also
5469	used by the tunables macros in arena.c. /*
5470
5471	switch (param_number)
5472	{
5473	case M_MXFAST:
5474	res = do_set_mxfast (value);
5475	break;
5476
5477	case M_TRIM_THRESHOLD:
5478	res = do_set_trim_threshold (value);
5479	break;
5480
5481	case M_TOP_PAD:
5482	res = do_set_top_pad (value);
5483	break;
5484
5485	case M_MMAP_THRESHOLD:
5486	res = do_set_mmap_threshold (value);
5487	break;
5488
5489	case M_MMAP_MAX:
5490	res = do_set_mmaps_max (value);
5491	break;
5492
5493	case M_CHECK_ACTION:
5494	res = do_set_mallopt_check (value);
5495	break;
5496
5497	case M_PERTURB:
5498	res = do_set_perturb_byte (value);
5499	break;
5500
5501	case M_ARENA_TEST:
5502	if (value > `0`)
5503	res = do_set_arena_test (value);
5504	break;
5505
5506	case M_ARENA_MAX:
5507	if (value > `0`)
5508	res = do_set_arena_max (value);
5509	break;
5510	}
5511	__libc_lock_unlock (av->mutex);
5512	return res;
5513	}
5514	libc_hidden_def (__libc_mallopt)
5515
5516
5517	/*
5518	-------------------- Alternative MORECORE functions --------------------
5519	*/
5520
5521
5522	/*
5523	General Requirements for MORECORE.
5524
5525	The MORECORE function must have the following properties:
5526
5527	If MORECORE_CONTIGUOUS is false:
5528
5529	* MORECORE must allocate in multiples of pagesize. It will
5530	only be called with arguments that are multiples of pagesize.
5531
5532	* MORECORE(0) must return an address that is at least
5533	MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
5534
5535	else (i.e. If MORECORE_CONTIGUOUS is true):
5536
5537	* Consecutive calls to MORECORE with positive arguments
5538	return increasing addresses, indicating that space has been
5539	contiguously extended.
5540
5541	* MORECORE need not allocate in multiples of pagesize.
5542	Calls to MORECORE need not have args of multiples of pagesize.
5543
5544	* MORECORE need not page-align.
5545
5546	In either case:
5547
5548	* MORECORE may allocate more memory than requested. (Or even less,
5549	but this will generally result in a malloc failure.)
5550
5551	* MORECORE must not allocate memory when given argument zero, but
5552	instead return one past the end address of memory from previous
5553	nonzero call. This malloc does NOT call MORECORE(0)
5554	until at least one call with positive arguments is made, so
5555	the initial value returned is not important.
5556
5557	* Even though consecutive calls to MORECORE need not return contiguous
5558	addresses, it must be OK for malloc'ed chunks to span multiple
5559	regions in those cases where they do happen to be contiguous.
5560
5561	* MORECORE need not handle negative arguments -- it may instead
5562	just return MORECORE_FAILURE when given negative arguments.
5563	Negative arguments are always multiples of pagesize. MORECORE
5564	must not misinterpret negative args as large positive unsigned
5565	args. You can suppress all such calls from even occurring by defining
5566	MORECORE_CANNOT_TRIM,
5567
5568	There is some variation across systems about the type of the
5569	argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
5570	actually be size_t, because sbrk supports negative args, so it is
5571	normally the signed type of the same width as size_t (sometimes
5572	declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
5573	matter though. Internally, we use "long" as arguments, which should
5574	work across all reasonable possibilities.
5575
5576	Additionally, if MORECORE ever returns failure for a positive
5577	request, then mmap is used as a noncontiguous system allocator. This
5578	is a useful backup strategy for systems with holes in address spaces
5579	-- in this case sbrk cannot contiguously expand the heap, but mmap
5580	may be able to map noncontiguous space.
5581
5582	If you'd like mmap to ALWAYS be used, you can define MORECORE to be
5583	a function that always returns MORECORE_FAILURE.
5584
5585	If you are using this malloc with something other than sbrk (or its
5586	emulation) to supply memory regions, you probably want to set
5587	MORECORE_CONTIGUOUS as false. As an example, here is a custom
5588	allocator kindly contributed for pre-OSX macOS. It uses virtually
5589	but not necessarily physically contiguous non-paged memory (locked
5590	in, present and won't get swapped out). You can use it by
5591	uncommenting this section, adding some #includes, and setting up the
5592	appropriate defines above:
5593
5594	*#define MORECORE osMoreCore
5595	*#define MORECORE_CONTIGUOUS 0
5596
5597	There is also a shutdown routine that should somehow be called for
5598	cleanup upon program exit.
5599
5600	*#define MAX_POOL_ENTRIES 100
5601	#define MINIMUM_MORECORE_SIZE (64 1024)
5602	static int next_os_pool;
5603	void our_os_pools[MAX_POOL_ENTRIES];*
5604
5605	void osMoreCore(int size)*
5606	{
5607	void ptr = 0;*
5608	static void sbrk_top = 0;*
5609
5610	if (size > 0)
5611	{
5612	if (size < MINIMUM_MORECORE_SIZE)
5613	size = MINIMUM_MORECORE_SIZE;
5614	if (CurrentExecutionLevel() == kTaskLevel)
5615	ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
5616	if (ptr == 0)
5617	{
5618	return (void ) MORECORE_FAILURE;*
5619	}
5620	// save ptrs so they can be freed during cleanup
5621	our_os_pools[next_os_pool] = ptr;
5622	next_os_pool++;
5623	ptr = (void ) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);*
5624	sbrk_top = (char ) ptr + size;*
5625	return ptr;
5626	}
5627	else if (size < 0)
5628	{
5629	// we don't currently support shrink behavior
5630	return (void ) MORECORE_FAILURE;*
5631	}
5632	else
5633	{
5634	return sbrk_top;
5635	}
5636	}
5637
5638	// cleanup any allocated memory pools
5639	// called as last thing before shutting down driver
5640
5641	void osCleanupMem(void)
5642	{
5643	void ptr;
5644
5645	for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
5646	if (ptr)*
5647	{
5648	PoolDeallocate(ptr);*
5649	* ptr = 0;
5650	}
5651	}
5652
5653	*/
5654
5655
5656	/ Helper code. /
5657
5658	extern char **__libc_argv attribute_hidden;
5659
5660	static void
5661	malloc_printerr (const char *str)
5662	{
5663	#if IS_IN (libc)
5664	__libc_message (do_abort, "%s\n", str);
5665	#else
5666	__libc_fatal (str);
5667	#endif
5668	__builtin_unreachable ();
5669	}
5670
5671	#if IS_IN (libc)
5672	/ We need a wrapper function for one of the additions of POSIX. /
5673	int
5674	__posix_memalign (void **memptr, size_t alignment, size_t size)
5675	{
5676	void *mem;
5677
5678	if (!__malloc_initialized)
5679	ptmalloc_init ();
5680
5681	/ Test whether the SIZE argument is valid. It must be a power of*
5682	two multiple of sizeof (void ). /
5683	if (alignment % sizeof (void *) != `0`
5684	\|\| !powerof2 (alignment / sizeof (void *))
5685	\|\| alignment == `0`)
5686	return EINVAL;
5687
5688
5689	void *address = RETURN_ADDRESS (`0`);
5690	mem = _mid_memalign (alignment, size, address);
5691
5692	if (mem != NULL)
5693	{
5694	*memptr = mem;
5695	return `0`;
5696	}
5697
5698	return ENOMEM;
5699	}
5700	weak_alias (__posix_memalign, posix_memalign)
5701	#endif
5702
5703
5704	int
5705	__malloc_info (int options, FILE *fp)
5706	{
5707	/ For now, at least. /
5708	if (options != `0`)
5709	return EINVAL;
5710
5711	int n = `0`;
5712	size_t total_nblocks = `0`;
5713	size_t total_nfastblocks = `0`;
5714	size_t total_avail = `0`;
5715	size_t total_fastavail = `0`;
5716	size_t total_system = `0`;
5717	size_t total_max_system = `0`;
5718	size_t total_aspace = `0`;
5719	size_t total_aspace_mprotect = `0`;
5720
5721
5722
5723	if (!__malloc_initialized)
5724	ptmalloc_init ();
5725
5726	fputs ("<malloc version=\"1\">\n", fp);
5727
5728	/ Iterate over all arenas currently in use. /
5729	mstate ar_ptr = &main_arena;
5730	do
5731	{
5732	fprintf (fp, "<heap nr=\"%d\">\n<sizes>\n", n++);
5733
5734	size_t nblocks = `0`;
5735	size_t nfastblocks = `0`;
5736	size_t avail = `0`;
5737	size_t fastavail = `0`;
5738	struct
5739	{
5740	size_t from;
5741	size_t to;
5742	size_t total;
5743	size_t count;
5744	} sizes[NFASTBINS + NBINS - `1`];
5745	#define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5746
5747	__libc_lock_lock (ar_ptr->mutex);
5748
5749	/ Account for top chunk. The top-most available chunk is*
5750	treated specially and is never in any bin. See "initial_top"
5751	comments. /*
5752	avail = chunksize (ar_ptr->top);
5753	nblocks = `1`; / Top always exists. /
5754
5755	for (size_t i = `0`; i < NFASTBINS; ++i)
5756	{
5757	mchunkptr p = fastbin (ar_ptr, i);
5758	if (p != NULL)
5759	{
5760	size_t nthissize = `0`;
5761	size_t thissize = chunksize (p);
5762
5763	while (p != NULL)
5764	{
5765	if (__glibc_unlikely (misaligned_chunk (p)))
5766	malloc_printerr ("__malloc_info(): "
5767	"unaligned fastbin chunk detected");
5768	++nthissize;
5769	p = REVEAL_PTR (p->fd);
5770	}
5771
5772	fastavail += nthissize * thissize;
5773	nfastblocks += nthissize;
5774	sizes[i].from = thissize - (MALLOC_ALIGNMENT - `1`);
5775	sizes[i].to = thissize;
5776	sizes[i].count = nthissize;
5777	}
5778	else
5779	sizes[i].from = sizes[i].to = sizes[i].count = `0`;
5780
5781	sizes[i].total = sizes[i].count * sizes[i].to;
5782	}
5783
5784
5785	mbinptr bin;
5786	struct malloc_chunk *r;
5787
5788	for (size_t i = `1`; i < NBINS; ++i)
5789	{
5790	bin = bin_at (ar_ptr, i);
5791	r = bin->fd;
5792	sizes[NFASTBINS - `1` + i].from = ~((size_t) `0`);
5793	sizes[NFASTBINS - `1` + i].to = sizes[NFASTBINS - `1` + i].total
5794	= sizes[NFASTBINS - `1` + i].count = `0`;
5795
5796	if (r != NULL)
5797	while (r != bin)
5798	{
5799	size_t r_size = chunksize_nomask (r);
5800	++sizes[NFASTBINS - `1` + i].count;
5801	sizes[NFASTBINS - `1` + i].total += r_size;
5802	sizes[NFASTBINS - `1` + i].from
5803	= MIN (sizes[NFASTBINS - `1` + i].from, r_size);
5804	sizes[NFASTBINS - `1` + i].to = MAX (sizes[NFASTBINS - `1` + i].to,
5805	r_size);
5806
5807	r = r->fd;
5808	}
5809
5810	if (sizes[NFASTBINS - `1` + i].count == `0`)
5811	sizes[NFASTBINS - `1` + i].from = `0`;
5812	nblocks += sizes[NFASTBINS - `1` + i].count;
5813	avail += sizes[NFASTBINS - `1` + i].total;
5814	}
5815
5816	size_t heap_size = `0`;
5817	size_t heap_mprotect_size = `0`;
5818	size_t heap_count = `0`;
5819	if (ar_ptr != &main_arena)
5820	{
5821	/ Iterate over the arena heaps from back to front. /
5822	heap_info *heap = heap_for_ptr (top (ar_ptr));
5823	do
5824	{
5825	heap_size += heap->size;
5826	heap_mprotect_size += heap->mprotect_size;
5827	heap = heap->prev;
5828	++heap_count;
5829	}
5830	while (heap != NULL);
5831	}
5832
5833	__libc_lock_unlock (ar_ptr->mutex);
5834
5835	total_nfastblocks += nfastblocks;
5836	total_fastavail += fastavail;
5837
5838	total_nblocks += nblocks;
5839	total_avail += avail;
5840
5841	for (size_t i = `0`; i < nsizes; ++i)
5842	if (sizes[i].count != `0` && i != NFASTBINS)
5843	fprintf (fp, "\
5844	<size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5845	sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
5846
5847	if (sizes[NFASTBINS].count != `0`)
5848	fprintf (fp, "\
5849	<unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5850	sizes[NFASTBINS].from, sizes[NFASTBINS].to,
5851	sizes[NFASTBINS].total, sizes[NFASTBINS].count);
5852
5853	total_system += ar_ptr->system_mem;
5854	total_max_system += ar_ptr->max_system_mem;
5855
5856	fprintf (fp,
5857	"</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5858	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5859	"<system type=\"current\" size=\"%zu\"/>\n"
5860	"<system type=\"max\" size=\"%zu\"/>\n",
5861	nfastblocks, fastavail, nblocks, avail,
5862	ar_ptr->system_mem, ar_ptr->max_system_mem);
5863
5864	if (ar_ptr != &main_arena)
5865	{
5866	fprintf (fp,
5867	"<aspace type=\"total\" size=\"%zu\"/>\n"
5868	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5869	"<aspace type=\"subheaps\" size=\"%zu\"/>\n",
5870	heap_size, heap_mprotect_size, heap_count);
5871	total_aspace += heap_size;
5872	total_aspace_mprotect += heap_mprotect_size;
5873	}
5874	else
5875	{
5876	fprintf (fp,
5877	"<aspace type=\"total\" size=\"%zu\"/>\n"
5878	"<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5879	ar_ptr->system_mem, ar_ptr->system_mem);
5880	total_aspace += ar_ptr->system_mem;
5881	total_aspace_mprotect += ar_ptr->system_mem;
5882	}
5883
5884	fputs ("</heap>\n", fp);
5885	ar_ptr = ar_ptr->next;
5886	}
5887	while (ar_ptr != &main_arena);
5888
5889	fprintf (fp,
5890	"<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5891	"<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5892	"<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
5893	"<system type=\"current\" size=\"%zu\"/>\n"
5894	"<system type=\"max\" size=\"%zu\"/>\n"
5895	"<aspace type=\"total\" size=\"%zu\"/>\n"
5896	"<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5897	"</malloc>\n",
5898	total_nfastblocks, total_fastavail, total_nblocks, total_avail,
5899	mp_.n_mmaps, mp_.mmapped_mem,
5900	total_system, total_max_system,
5901	total_aspace, total_aspace_mprotect);
5902
5903	return `0`;
5904	}
5905	#if IS_IN (libc)
5906	weak_alias (__malloc_info, malloc_info)
5907
5908	strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5909	strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5910	strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5911	strong_alias (__libc_memalign, __memalign)
5912	weak_alias (__libc_memalign, memalign)
5913	strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5914	strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5915	strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5916	strong_alias (__libc_mallinfo, __mallinfo)
5917	weak_alias (__libc_mallinfo, mallinfo)
5918	strong_alias (__libc_mallinfo2, __mallinfo2)
5919	weak_alias (__libc_mallinfo2, mallinfo2)
5920	strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5921
5922	weak_alias (__malloc_stats, malloc_stats)
5923	weak_alias (__malloc_usable_size, malloc_usable_size)
5924	weak_alias (__malloc_trim, malloc_trim)
5925	#endif
5926
5927	#if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
5928	compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
5929	#endif
5930
5931	/ ------------------------------------------------------------*
5932	History:
5933
5934	[see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
5935
5936	*/
5937	/*
5938	* Local variables:
5939	* c-basic-offset: 2
5940	* End:
5941	*/
5942

Browse the source code of glibc/malloc/malloc.c