1/*
2 * Copyright (c) 1998-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29/*
30 * Copyright (c) 1982, 1986, 1988, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
62 */
63/*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70#include <sys/param.h>
71#include <sys/systm.h>
72#include <sys/malloc.h>
73#include <sys/mbuf.h>
74#include <sys/kernel.h>
75#include <sys/sysctl.h>
76#include <sys/syslog.h>
77#include <sys/protosw.h>
78#include <sys/domain.h>
79#include <sys/queue.h>
80#include <sys/proc.h>
81
82#include <dev/random/randomdev.h>
83
84#include <kern/kern_types.h>
85#include <kern/simple_lock.h>
86#include <kern/queue.h>
87#include <kern/sched_prim.h>
88#include <kern/backtrace.h>
89#include <kern/cpu_number.h>
90#include <kern/zalloc.h>
91
92#include <libkern/OSAtomic.h>
93#include <libkern/OSDebug.h>
94#include <libkern/libkern.h>
95
96#include <os/log.h>
97
98#include <IOKit/IOMapper.h>
99
100#include <machine/limits.h>
101#include <machine/machine_routines.h>
102
103#if CONFIG_MACF_NET
104#include <security/mac_framework.h>
105#endif /* MAC_NET */
106
107#include <sys/mcache.h>
108#include <net/ntstat.h>
109
110/*
111 * MBUF IMPLEMENTATION NOTES.
112 *
113 * There is a total of 5 per-CPU caches:
114 *
115 * MC_MBUF:
116 * This is a cache of rudimentary objects of MSIZE in size; each
117 * object represents an mbuf structure. This cache preserves only
118 * the m_type field of the mbuf during its transactions.
119 *
120 * MC_CL:
121 * This is a cache of rudimentary objects of MCLBYTES in size; each
122 * object represents a mcluster structure. This cache does not
123 * preserve the contents of the objects during its transactions.
124 *
125 * MC_BIGCL:
126 * This is a cache of rudimentary objects of MBIGCLBYTES in size; each
127 * object represents a mbigcluster structure. This cache does not
128 * preserve the contents of the objects during its transaction.
129 *
130 * MC_MBUF_CL:
131 * This is a cache of mbufs each having a cluster attached to it.
132 * It is backed by MC_MBUF and MC_CL rudimentary caches. Several
133 * fields of the mbuf related to the external cluster are preserved
134 * during transactions.
135 *
136 * MC_MBUF_BIGCL:
137 * This is a cache of mbufs each having a big cluster attached to it.
138 * It is backed by MC_MBUF and MC_BIGCL rudimentary caches. Several
139 * fields of the mbuf related to the external cluster are preserved
140 * during transactions.
141 *
142 * OBJECT ALLOCATION:
143 *
144 * Allocation requests are handled first at the per-CPU (mcache) layer
145 * before falling back to the slab layer. Performance is optimal when
146 * the request is satisfied at the CPU layer because global data/lock
147 * never gets accessed. When the slab layer is entered for allocation,
148 * the slab freelist will be checked first for available objects before
149 * the VM backing store is invoked. Slab layer operations are serialized
150 * for all of the caches as the mbuf global lock is held most of the time.
151 * Allocation paths are different depending on the class of objects:
152 *
153 * a. Rudimentary object:
154 *
155 * { m_get_common(), m_clattach(), m_mclget(),
156 * m_mclalloc(), m_bigalloc(), m_copym_with_hdrs(),
157 * composite object allocation }
158 * | ^
159 * | |
160 * | +-----------------------+
161 * v |
162 * mcache_alloc/mcache_alloc_ext() mbuf_slab_audit()
163 * | ^
164 * v |
165 * [CPU cache] -------> (found?) -------+
166 * | |
167 * v |
168 * mbuf_slab_alloc() |
169 * | |
170 * v |
171 * +---------> [freelist] -------> (found?) -------+
172 * | |
173 * | v
174 * | m_clalloc()
175 * | |
176 * | v
177 * +---<<---- kmem_mb_alloc()
178 *
179 * b. Composite object:
180 *
181 * { m_getpackets_internal(), m_allocpacket_internal() }
182 * | ^
183 * | |
184 * | +------ (done) ---------+
185 * v |
186 * mcache_alloc/mcache_alloc_ext() mbuf_cslab_audit()
187 * | ^
188 * v |
189 * [CPU cache] -------> (found?) -------+
190 * | |
191 * v |
192 * mbuf_cslab_alloc() |
193 * | |
194 * v |
195 * [freelist] -------> (found?) -------+
196 * | |
197 * v |
198 * (rudimentary object) |
199 * mcache_alloc/mcache_alloc_ext() ------>>-----+
200 *
201 * Auditing notes: If auditing is enabled, buffers will be subjected to
202 * integrity checks by the audit routine. This is done by verifying their
203 * contents against DEADBEEF (free) pattern before returning them to caller.
204 * As part of this step, the routine will also record the transaction and
205 * pattern-fill the buffers with BADDCAFE (uninitialized) pattern. It will
206 * also restore any constructed data structure fields if necessary.
207 *
208 * OBJECT DEALLOCATION:
209 *
210 * Freeing an object simply involves placing it into the CPU cache; this
211 * pollutes the cache to benefit subsequent allocations. The slab layer
212 * will only be entered if the object is to be purged out of the cache.
213 * During normal operations, this happens only when the CPU layer resizes
214 * its bucket while it's adjusting to the allocation load. Deallocation
215 * paths are different depending on the class of objects:
216 *
217 * a. Rudimentary object:
218 *
219 * { m_free(), m_freem_list(), composite object deallocation }
220 * | ^
221 * | |
222 * | +------ (done) ---------+
223 * v |
224 * mcache_free/mcache_free_ext() |
225 * | |
226 * v |
227 * mbuf_slab_audit() |
228 * | |
229 * v |
230 * [CPU cache] ---> (not purging?) -----+
231 * | |
232 * v |
233 * mbuf_slab_free() |
234 * | |
235 * v |
236 * [freelist] ----------->>------------+
237 * (objects get purged to VM only on demand)
238 *
239 * b. Composite object:
240 *
241 * { m_free(), m_freem_list() }
242 * | ^
243 * | |
244 * | +------ (done) ---------+
245 * v |
246 * mcache_free/mcache_free_ext() |
247 * | |
248 * v |
249 * mbuf_cslab_audit() |
250 * | |
251 * v |
252 * [CPU cache] ---> (not purging?) -----+
253 * | |
254 * v |
255 * mbuf_cslab_free() |
256 * | |
257 * v |
258 * [freelist] ---> (not purging?) -----+
259 * | |
260 * v |
261 * (rudimentary object) |
262 * mcache_free/mcache_free_ext() ------->>------+
263 *
264 * Auditing notes: If auditing is enabled, the audit routine will save
265 * any constructed data structure fields (if necessary) before filling the
266 * contents of the buffers with DEADBEEF (free) pattern and recording the
267 * transaction. Buffers that are freed (whether at CPU or slab layer) are
268 * expected to contain the free pattern.
269 *
270 * DEBUGGING:
271 *
272 * Debugging can be enabled by adding "mbuf_debug=0x3" to boot-args; this
273 * translates to the mcache flags (MCF_VERIFY | MCF_AUDIT). Additionally,
274 * the CPU layer cache can be disabled by setting the MCF_NOCPUCACHE flag,
275 * i.e. modify the boot argument parameter to "mbuf_debug=0x13". Leak
276 * detection may also be disabled by setting the MCF_NOLEAKLOG flag, e.g.
277 * "mbuf_debug=0x113". Note that debugging consumes more CPU and memory.
278 *
279 * Each object is associated with exactly one mcache_audit_t structure that
280 * contains the information related to its last buffer transaction. Given
281 * an address of an object, the audit structure can be retrieved by finding
282 * the position of the object relevant to the base address of the cluster:
283 *
284 * +------------+ +=============+
285 * | mbuf addr | | mclaudit[i] |
286 * +------------+ +=============+
287 * | | cl_audit[0] |
288 * i = MTOBG(addr) +-------------+
289 * | +-----> | cl_audit[1] | -----> mcache_audit_t
290 * b = BGTOM(i) | +-------------+
291 * | | | ... |
292 * x = MCLIDX(b, addr) | +-------------+
293 * | | | cl_audit[7] |
294 * +-----------------+ +-------------+
295 * (e.g. x == 1)
296 *
297 * The mclaudit[] array is allocated at initialization time, but its contents
298 * get populated when the corresponding cluster is created. Because a page
299 * can be turned into NMBPG number of mbufs, we preserve enough space for the
300 * mbufs so that there is a 1-to-1 mapping between them. A page that never
301 * gets (or has not yet) turned into mbufs will use only cl_audit[0] with the
302 * remaining entries unused. For 16KB cluster, only one entry from the first
303 * page is allocated and used for the entire object.
304 */
305
306/* TODO: should be in header file */
307/* kernel translater */
308extern vm_offset_t kmem_mb_alloc(vm_map_t, int, int, kern_return_t *);
309extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
310extern vm_map_t mb_map; /* special map */
311
312static uint32_t mb_kmem_contig_failed;
313static uint32_t mb_kmem_failed;
314static uint32_t mb_kmem_one_failed;
315/* Timestamp of allocation failures. */
316static uint64_t mb_kmem_contig_failed_ts;
317static uint64_t mb_kmem_failed_ts;
318static uint64_t mb_kmem_one_failed_ts;
319static uint64_t mb_kmem_contig_failed_size;
320static uint64_t mb_kmem_failed_size;
321static uint32_t mb_kmem_stats[6];
322static const char *mb_kmem_stats_labels[] = { "INVALID_ARGUMENT",
323 "INVALID_ADDRESS",
324 "RESOURCE_SHORTAGE",
325 "NO_SPACE",
326 "KERN_FAILURE",
327 "OTHERS" };
328
329/* Global lock */
330decl_lck_mtx_data(static, mbuf_mlock_data);
331static lck_mtx_t *mbuf_mlock = &mbuf_mlock_data;
332static lck_attr_t *mbuf_mlock_attr;
333static lck_grp_t *mbuf_mlock_grp;
334static lck_grp_attr_t *mbuf_mlock_grp_attr;
335
336/* Back-end (common) layer */
337static uint64_t mb_expand_cnt;
338static uint64_t mb_expand_cl_cnt;
339static uint64_t mb_expand_cl_total;
340static uint64_t mb_expand_bigcl_cnt;
341static uint64_t mb_expand_bigcl_total;
342static uint64_t mb_expand_16kcl_cnt;
343static uint64_t mb_expand_16kcl_total;
344static boolean_t mbuf_worker_needs_wakeup; /* wait channel for mbuf worker */
345static uint32_t mbuf_worker_run_cnt;
346static uint64_t mbuf_worker_last_runtime;
347static uint64_t mbuf_drain_last_runtime;
348static int mbuf_worker_ready; /* worker thread is runnable */
349static int ncpu; /* number of CPUs */
350static ppnum_t *mcl_paddr; /* Array of cluster physical addresses */
351static ppnum_t mcl_pages; /* Size of array (# physical pages) */
352static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
353static mcache_t *ref_cache; /* Cache of cluster reference & flags */
354static mcache_t *mcl_audit_con_cache; /* Audit contents cache */
355static unsigned int mbuf_debug; /* patchable mbuf mcache flags */
356static unsigned int mb_normalized; /* number of packets "normalized" */
357
358#define MB_GROWTH_AGGRESSIVE 1 /* Threshold: 1/2 of total */
359#define MB_GROWTH_NORMAL 2 /* Threshold: 3/4 of total */
360
361typedef enum {
362 MC_MBUF = 0, /* Regular mbuf */
363 MC_CL, /* Cluster */
364 MC_BIGCL, /* Large (4KB) cluster */
365 MC_16KCL, /* Jumbo (16KB) cluster */
366 MC_MBUF_CL, /* mbuf + cluster */
367 MC_MBUF_BIGCL, /* mbuf + large (4KB) cluster */
368 MC_MBUF_16KCL /* mbuf + jumbo (16KB) cluster */
369} mbuf_class_t;
370
371#define MBUF_CLASS_MIN MC_MBUF
372#define MBUF_CLASS_MAX MC_MBUF_16KCL
373#define MBUF_CLASS_LAST MC_16KCL
374#define MBUF_CLASS_VALID(c) \
375 ((int)(c) >= MBUF_CLASS_MIN && (int)(c) <= MBUF_CLASS_MAX)
376#define MBUF_CLASS_COMPOSITE(c) \
377 ((int)(c) > MBUF_CLASS_LAST)
378
379
380/*
381 * mbuf specific mcache allocation request flags.
382 */
383#define MCR_COMP MCR_USR1 /* for MC_MBUF_{CL,BIGCL,16KCL} caches */
384
385/*
386 * Per-cluster slab structure.
387 *
388 * A slab is a cluster control structure that contains one or more object
389 * chunks; the available chunks are chained in the slab's freelist (sl_head).
390 * Each time a chunk is taken out of the slab, the slab's reference count
391 * gets incremented. When all chunks have been taken out, the empty slab
392 * gets removed (SLF_DETACHED) from the class's slab list. A chunk that is
393 * returned to a slab causes the slab's reference count to be decremented;
394 * it also causes the slab to be reinserted back to class's slab list, if
395 * it's not already done.
396 *
397 * Compartmentalizing of the object chunks into slabs allows us to easily
398 * merge one or more slabs together when the adjacent slabs are idle, as
399 * well as to convert or move a slab from one class to another; e.g. the
400 * mbuf cluster slab can be converted to a regular cluster slab when all
401 * mbufs in the slab have been freed.
402 *
403 * A slab may also span across multiple clusters for chunks larger than
404 * a cluster's size. In this case, only the slab of the first cluster is
405 * used. The rest of the slabs are marked with SLF_PARTIAL to indicate
406 * that they are part of the larger slab.
407 *
408 * Each slab controls a page of memory.
409 */
410typedef struct mcl_slab {
411 struct mcl_slab *sl_next; /* neighboring slab */
412 u_int8_t sl_class; /* controlling mbuf class */
413 int8_t sl_refcnt; /* outstanding allocations */
414 int8_t sl_chunks; /* chunks (bufs) in this slab */
415 u_int16_t sl_flags; /* slab flags (see below) */
416 u_int16_t sl_len; /* slab length */
417 void *sl_base; /* base of allocated memory */
418 void *sl_head; /* first free buffer */
419 TAILQ_ENTRY(mcl_slab) sl_link; /* next/prev slab on freelist */
420} mcl_slab_t;
421
422#define SLF_MAPPED 0x0001 /* backed by a mapped page */
423#define SLF_PARTIAL 0x0002 /* part of another slab */
424#define SLF_DETACHED 0x0004 /* not in slab freelist */
425
426/*
427 * The array of slabs are broken into groups of arrays per 1MB of kernel
428 * memory to reduce the footprint. Each group is allocated on demand
429 * whenever a new piece of memory mapped in from the VM crosses the 1MB
430 * boundary.
431 */
432#define NSLABSPMB ((1 << MBSHIFT) >> PAGE_SHIFT)
433
434typedef struct mcl_slabg {
435 mcl_slab_t *slg_slab; /* group of slabs */
436} mcl_slabg_t;
437
438/*
439 * Number of slabs needed to control a 16KB cluster object.
440 */
441#define NSLABSP16KB (M16KCLBYTES >> PAGE_SHIFT)
442
443/*
444 * Per-cluster audit structure.
445 */
446typedef struct {
447 mcache_audit_t **cl_audit; /* array of audits */
448} mcl_audit_t;
449
450typedef struct {
451 struct thread *msa_thread; /* thread doing transaction */
452 struct thread *msa_pthread; /* previous transaction thread */
453 uint32_t msa_tstamp; /* transaction timestamp (ms) */
454 uint32_t msa_ptstamp; /* prev transaction timestamp (ms) */
455 uint16_t msa_depth; /* pc stack depth */
456 uint16_t msa_pdepth; /* previous transaction pc stack */
457 void *msa_stack[MCACHE_STACK_DEPTH];
458 void *msa_pstack[MCACHE_STACK_DEPTH];
459} mcl_scratch_audit_t;
460
461typedef struct {
462 /*
463 * Size of data from the beginning of an mbuf that covers m_hdr,
464 * pkthdr and m_ext structures. If auditing is enabled, we allocate
465 * a shadow mbuf structure of this size inside each audit structure,
466 * and the contents of the real mbuf gets copied into it when the mbuf
467 * is freed. This allows us to pattern-fill the mbuf for integrity
468 * check, and to preserve any constructed mbuf fields (e.g. mbuf +
469 * cluster cache case). Note that we don't save the contents of
470 * clusters when they are freed; we simply pattern-fill them.
471 */
472 u_int8_t sc_mbuf[(MSIZE - _MHLEN) + sizeof (_m_ext_t)];
473 mcl_scratch_audit_t sc_scratch __attribute__((aligned(8)));
474} mcl_saved_contents_t;
475
476#define AUDIT_CONTENTS_SIZE (sizeof (mcl_saved_contents_t))
477
478#define MCA_SAVED_MBUF_PTR(_mca) \
479 ((struct mbuf *)(void *)((mcl_saved_contents_t *) \
480 (_mca)->mca_contents)->sc_mbuf)
481#define MCA_SAVED_MBUF_SIZE \
482 (sizeof (((mcl_saved_contents_t *)0)->sc_mbuf))
483#define MCA_SAVED_SCRATCH_PTR(_mca) \
484 (&((mcl_saved_contents_t *)(_mca)->mca_contents)->sc_scratch)
485
486/*
487 * mbuf specific mcache audit flags
488 */
489#define MB_INUSE 0x01 /* object has not been returned to slab */
490#define MB_COMP_INUSE 0x02 /* object has not been returned to cslab */
491#define MB_SCVALID 0x04 /* object has valid saved contents */
492
493/*
494 * Each of the following two arrays hold up to nmbclusters elements.
495 */
496static mcl_audit_t *mclaudit; /* array of cluster audit information */
497static unsigned int maxclaudit; /* max # of entries in audit table */
498static mcl_slabg_t **slabstbl; /* cluster slabs table */
499static unsigned int maxslabgrp; /* max # of entries in slabs table */
500static unsigned int slabgrp; /* # of entries in slabs table */
501
502/* Globals */
503int nclusters; /* # of clusters for non-jumbo (legacy) sizes */
504int njcl; /* # of clusters for jumbo sizes */
505int njclbytes; /* size of a jumbo cluster */
506unsigned char *mbutl; /* first mapped cluster address */
507unsigned char *embutl; /* ending virtual address of mclusters */
508int _max_linkhdr; /* largest link-level header */
509int _max_protohdr; /* largest protocol header */
510int max_hdr; /* largest link+protocol header */
511int max_datalen; /* MHLEN - max_hdr */
512
513static boolean_t mclverify; /* debug: pattern-checking */
514static boolean_t mcltrace; /* debug: stack tracing */
515static boolean_t mclfindleak; /* debug: leak detection */
516static boolean_t mclexpleak; /* debug: expose leak info to user space */
517
518static struct timeval mb_start; /* beginning of time */
519
520/* mbuf leak detection variables */
521static struct mleak_table mleak_table;
522static mleak_stat_t *mleak_stat;
523
524#define MLEAK_STAT_SIZE(n) \
525 __builtin_offsetof(mleak_stat_t, ml_trace[n])
526
527struct mallocation {
528 mcache_obj_t *element; /* the alloc'ed element, NULL if unused */
529 u_int32_t trace_index; /* mtrace index for corresponding backtrace */
530 u_int32_t count; /* How many objects were requested */
531 u_int64_t hitcount; /* for determining hash effectiveness */
532};
533
534struct mtrace {
535 u_int64_t collisions;
536 u_int64_t hitcount;
537 u_int64_t allocs;
538 u_int64_t depth;
539 uintptr_t addr[MLEAK_STACK_DEPTH];
540};
541
542/* Size must be a power of two for the zhash to be able to just mask off bits */
543#define MLEAK_ALLOCATION_MAP_NUM 512
544#define MLEAK_TRACE_MAP_NUM 256
545
546/*
547 * Sample factor for how often to record a trace. This is overwritable
548 * by the boot-arg mleak_sample_factor.
549 */
550#define MLEAK_SAMPLE_FACTOR 500
551
552/*
553 * Number of top leakers recorded.
554 */
555#define MLEAK_NUM_TRACES 5
556
557#define MB_LEAK_SPACING_64 " "
558#define MB_LEAK_SPACING_32 " "
559
560
561#define MB_LEAK_HDR_32 "\n\
562 trace [1] trace [2] trace [3] trace [4] trace [5] \n\
563 ---------- ---------- ---------- ---------- ---------- \n\
564"
565
566#define MB_LEAK_HDR_64 "\n\
567 trace [1] trace [2] trace [3] \
568 trace [4] trace [5] \n\
569 ------------------ ------------------ ------------------ \
570 ------------------ ------------------ \n\
571"
572
573static uint32_t mleak_alloc_buckets = MLEAK_ALLOCATION_MAP_NUM;
574static uint32_t mleak_trace_buckets = MLEAK_TRACE_MAP_NUM;
575
576/* Hashmaps of allocations and their corresponding traces */
577static struct mallocation *mleak_allocations;
578static struct mtrace *mleak_traces;
579static struct mtrace *mleak_top_trace[MLEAK_NUM_TRACES];
580
581/* Lock to protect mleak tables from concurrent modification */
582decl_lck_mtx_data(static, mleak_lock_data);
583static lck_mtx_t *mleak_lock = &mleak_lock_data;
584static lck_attr_t *mleak_lock_attr;
585static lck_grp_t *mleak_lock_grp;
586static lck_grp_attr_t *mleak_lock_grp_attr;
587
588/* *Failed* large allocations. */
589struct mtracelarge {
590 uint64_t size;
591 uint64_t depth;
592 uintptr_t addr[MLEAK_STACK_DEPTH];
593};
594
595#define MTRACELARGE_NUM_TRACES 5
596static struct mtracelarge mtracelarge_table[MTRACELARGE_NUM_TRACES];
597
598static void mtracelarge_register(size_t size);
599
600/* Lock to protect the completion callback table */
601static lck_grp_attr_t *mbuf_tx_compl_tbl_lck_grp_attr = NULL;
602static lck_attr_t *mbuf_tx_compl_tbl_lck_attr = NULL;
603static lck_grp_t *mbuf_tx_compl_tbl_lck_grp = NULL;
604decl_lck_rw_data(, mbuf_tx_compl_tbl_lck_rw_data);
605lck_rw_t *mbuf_tx_compl_tbl_lock = &mbuf_tx_compl_tbl_lck_rw_data;
606
607extern u_int32_t high_sb_max;
608
609/* The minimum number of objects that are allocated, to start. */
610#define MINCL 32
611#define MINBIGCL (MINCL >> 1)
612#define MIN16KCL (MINCL >> 2)
613
614/* Low watermarks (only map in pages once free counts go below) */
615#define MBIGCL_LOWAT MINBIGCL
616#define M16KCL_LOWAT MIN16KCL
617
618typedef struct {
619 mbuf_class_t mtbl_class; /* class type */
620 mcache_t *mtbl_cache; /* mcache for this buffer class */
621 TAILQ_HEAD(mcl_slhead, mcl_slab) mtbl_slablist; /* slab list */
622 mcache_obj_t *mtbl_cobjlist; /* composite objects freelist */
623 mb_class_stat_t *mtbl_stats; /* statistics fetchable via sysctl */
624 u_int32_t mtbl_maxsize; /* maximum buffer size */
625 int mtbl_minlimit; /* minimum allowed */
626 int mtbl_maxlimit; /* maximum allowed */
627 u_int32_t mtbl_wantpurge; /* purge during next reclaim */
628 uint32_t mtbl_avgtotal; /* average total on iOS */
629 u_int32_t mtbl_expand; /* worker should expand the class */
630} mbuf_table_t;
631
632#define m_class(c) mbuf_table[c].mtbl_class
633#define m_cache(c) mbuf_table[c].mtbl_cache
634#define m_slablist(c) mbuf_table[c].mtbl_slablist
635#define m_cobjlist(c) mbuf_table[c].mtbl_cobjlist
636#define m_maxsize(c) mbuf_table[c].mtbl_maxsize
637#define m_minlimit(c) mbuf_table[c].mtbl_minlimit
638#define m_maxlimit(c) mbuf_table[c].mtbl_maxlimit
639#define m_wantpurge(c) mbuf_table[c].mtbl_wantpurge
640#define m_cname(c) mbuf_table[c].mtbl_stats->mbcl_cname
641#define m_size(c) mbuf_table[c].mtbl_stats->mbcl_size
642#define m_total(c) mbuf_table[c].mtbl_stats->mbcl_total
643#define m_active(c) mbuf_table[c].mtbl_stats->mbcl_active
644#define m_infree(c) mbuf_table[c].mtbl_stats->mbcl_infree
645#define m_slab_cnt(c) mbuf_table[c].mtbl_stats->mbcl_slab_cnt
646#define m_alloc_cnt(c) mbuf_table[c].mtbl_stats->mbcl_alloc_cnt
647#define m_free_cnt(c) mbuf_table[c].mtbl_stats->mbcl_free_cnt
648#define m_notified(c) mbuf_table[c].mtbl_stats->mbcl_notified
649#define m_purge_cnt(c) mbuf_table[c].mtbl_stats->mbcl_purge_cnt
650#define m_fail_cnt(c) mbuf_table[c].mtbl_stats->mbcl_fail_cnt
651#define m_ctotal(c) mbuf_table[c].mtbl_stats->mbcl_ctotal
652#define m_peak(c) mbuf_table[c].mtbl_stats->mbcl_peak_reported
653#define m_release_cnt(c) mbuf_table[c].mtbl_stats->mbcl_release_cnt
654#define m_region_expand(c) mbuf_table[c].mtbl_expand
655
656static mbuf_table_t mbuf_table[] = {
657 /*
658 * The caches for mbufs, regular clusters and big clusters.
659 * The average total values were based on data gathered by actual
660 * usage patterns on iOS.
661 */
662 { MC_MBUF, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_MBUF)),
663 NULL, NULL, 0, 0, 0, 0, 3000, 0 },
664 { MC_CL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_CL)),
665 NULL, NULL, 0, 0, 0, 0, 2000, 0 },
666 { MC_BIGCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_BIGCL)),
667 NULL, NULL, 0, 0, 0, 0, 1000, 0 },
668 { MC_16KCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_16KCL)),
669 NULL, NULL, 0, 0, 0, 0, 200, 0 },
670 /*
671 * The following are special caches; they serve as intermediate
672 * caches backed by the above rudimentary caches. Each object
673 * in the cache is an mbuf with a cluster attached to it. Unlike
674 * the above caches, these intermediate caches do not directly
675 * deal with the slab structures; instead, the constructed
676 * cached elements are simply stored in the freelists.
677 */
678 { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000, 0 },
679 { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000, 0 },
680 { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 200, 0 },
681};
682
683#define NELEM(a) (sizeof (a) / sizeof ((a)[0]))
684
685
686static uint32_t
687m_avgtotal(mbuf_class_t c)
688{
689 return (mbuf_table[c].mtbl_avgtotal);
690}
691
692static void *mb_waitchan = &mbuf_table; /* wait channel for all caches */
693static int mb_waiters; /* number of waiters */
694
695boolean_t mb_peak_newreport = FALSE;
696boolean_t mb_peak_firstreport = FALSE;
697
698/* generate a report by default after 1 week of uptime */
699#define MBUF_PEAK_FIRST_REPORT_THRESHOLD 604800
700
701#define MB_WDT_MAXTIME 10 /* # of secs before watchdog panic */
702static struct timeval mb_wdtstart; /* watchdog start timestamp */
703static char *mbuf_dump_buf;
704
705#define MBUF_DUMP_BUF_SIZE 4096
706
707/*
708 * mbuf watchdog is enabled by default on embedded platforms. It is
709 * also toggeable via the kern.ipc.mb_watchdog sysctl.
710 * Garbage collection is also enabled by default on embedded platforms.
711 * mb_drain_maxint controls the amount of time to wait (in seconds) before
712 * consecutive calls to mbuf_drain().
713 */
714#if CONFIG_EMBEDDED
715static unsigned int mb_watchdog = 1;
716static unsigned int mb_drain_maxint = 60;
717#else
718static unsigned int mb_watchdog = 0;
719static unsigned int mb_drain_maxint = 0;
720#endif /* CONFIG_EMBEDDED */
721
722uintptr_t mb_obscure_extfree __attribute__((visibility("hidden")));
723uintptr_t mb_obscure_extref __attribute__((visibility("hidden")));
724
725/* Red zone */
726static u_int32_t mb_redzone_cookie;
727static void m_redzone_init(struct mbuf *);
728static void m_redzone_verify(struct mbuf *m);
729
730/* The following are used to serialize m_clalloc() */
731static boolean_t mb_clalloc_busy;
732static void *mb_clalloc_waitchan = &mb_clalloc_busy;
733static int mb_clalloc_waiters;
734
735static void mbuf_mtypes_sync(boolean_t);
736static int mbstat_sysctl SYSCTL_HANDLER_ARGS;
737static void mbuf_stat_sync(void);
738static int mb_stat_sysctl SYSCTL_HANDLER_ARGS;
739static int mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS;
740static int mleak_table_sysctl SYSCTL_HANDLER_ARGS;
741static char *mbuf_dump(void);
742static void mbuf_table_init(void);
743static inline void m_incref(struct mbuf *);
744static inline u_int16_t m_decref(struct mbuf *);
745static int m_clalloc(const u_int32_t, const int, const u_int32_t);
746static void mbuf_worker_thread_init(void);
747static mcache_obj_t *slab_alloc(mbuf_class_t, int);
748static void slab_free(mbuf_class_t, mcache_obj_t *);
749static unsigned int mbuf_slab_alloc(void *, mcache_obj_t ***,
750 unsigned int, int);
751static void mbuf_slab_free(void *, mcache_obj_t *, int);
752static void mbuf_slab_audit(void *, mcache_obj_t *, boolean_t);
753static void mbuf_slab_notify(void *, u_int32_t);
754static unsigned int cslab_alloc(mbuf_class_t, mcache_obj_t ***,
755 unsigned int);
756static unsigned int cslab_free(mbuf_class_t, mcache_obj_t *, int);
757static unsigned int mbuf_cslab_alloc(void *, mcache_obj_t ***,
758 unsigned int, int);
759static void mbuf_cslab_free(void *, mcache_obj_t *, int);
760static void mbuf_cslab_audit(void *, mcache_obj_t *, boolean_t);
761static int freelist_populate(mbuf_class_t, unsigned int, int);
762static void freelist_init(mbuf_class_t);
763static boolean_t mbuf_cached_above(mbuf_class_t, int);
764static boolean_t mbuf_steal(mbuf_class_t, unsigned int);
765static void m_reclaim(mbuf_class_t, unsigned int, boolean_t);
766static int m_howmany(int, size_t);
767static void mbuf_worker_thread(void);
768static void mbuf_watchdog(void);
769static boolean_t mbuf_sleep(mbuf_class_t, unsigned int, int);
770
771static void mcl_audit_init(void *, mcache_audit_t **, mcache_obj_t **,
772 size_t, unsigned int);
773static void mcl_audit_free(void *, unsigned int);
774static mcache_audit_t *mcl_audit_buf2mca(mbuf_class_t, mcache_obj_t *);
775static void mcl_audit_mbuf(mcache_audit_t *, void *, boolean_t, boolean_t);
776static void mcl_audit_cluster(mcache_audit_t *, void *, size_t, boolean_t,
777 boolean_t);
778static void mcl_audit_restore_mbuf(struct mbuf *, mcache_audit_t *, boolean_t);
779static void mcl_audit_save_mbuf(struct mbuf *, mcache_audit_t *);
780static void mcl_audit_scratch(mcache_audit_t *);
781static void mcl_audit_mcheck_panic(struct mbuf *);
782static void mcl_audit_verify_nextptr(void *, mcache_audit_t *);
783
784static void mleak_activate(void);
785static void mleak_logger(u_int32_t, mcache_obj_t *, boolean_t);
786static boolean_t mleak_log(uintptr_t *, mcache_obj_t *, uint32_t, int);
787static void mleak_free(mcache_obj_t *);
788static void mleak_sort_traces(void);
789static void mleak_update_stats(void);
790
791static mcl_slab_t *slab_get(void *);
792static void slab_init(mcl_slab_t *, mbuf_class_t, u_int32_t,
793 void *, void *, unsigned int, int, int);
794static void slab_insert(mcl_slab_t *, mbuf_class_t);
795static void slab_remove(mcl_slab_t *, mbuf_class_t);
796static boolean_t slab_inrange(mcl_slab_t *, void *);
797static void slab_nextptr_panic(mcl_slab_t *, void *);
798static void slab_detach(mcl_slab_t *);
799static boolean_t slab_is_detached(mcl_slab_t *);
800
801static int m_copyback0(struct mbuf **, int, int, const void *, int, int);
802static struct mbuf *m_split0(struct mbuf *, int, int, int);
803__private_extern__ void mbuf_report_peak_usage(void);
804static boolean_t mbuf_report_usage(mbuf_class_t);
805#if DEBUG || DEVELOPMENT
806#define mbwdog_logger(fmt, ...) _mbwdog_logger(__func__, __LINE__, fmt, ## __VA_ARGS__)
807static void _mbwdog_logger(const char *func, const int line, const char *fmt, ...);
808static char *mbwdog_logging;
809const unsigned mbwdog_logging_size = 4096;
810static size_t mbwdog_logging_used;
811#else
812#define mbwdog_logger(fmt, ...) do { } while (0)
813#endif
814static void mbuf_drain_locked(boolean_t);
815
816/* flags for m_copyback0 */
817#define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */
818#define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */
819#define M_COPYBACK0_COW 0x0004 /* do copy-on-write */
820#define M_COPYBACK0_EXTEND 0x0008 /* extend chain */
821
822/*
823 * This flag is set for all mbufs that come out of and into the composite
824 * mbuf + cluster caches, i.e. MC_MBUF_CL and MC_MBUF_BIGCL. mbufs that
825 * are marked with such a flag have clusters attached to them, and will be
826 * treated differently when they are freed; instead of being placed back
827 * into the mbuf and cluster freelists, the composite mbuf + cluster objects
828 * are placed back into the appropriate composite cache's freelist, and the
829 * actual freeing is deferred until the composite objects are purged. At
830 * such a time, this flag will be cleared from the mbufs and the objects
831 * will be freed into their own separate freelists.
832 */
833#define EXTF_COMPOSITE 0x1
834
835/*
836 * This flag indicates that the external cluster is read-only, i.e. it is
837 * or was referred to by more than one mbufs. Once set, this flag is never
838 * cleared.
839 */
840#define EXTF_READONLY 0x2
841/*
842 * This flag indicates that the external cluster is paired with the mbuf.
843 * Pairing implies an external free routine defined which will be invoked
844 * when the reference count drops to the minimum at m_free time. This
845 * flag is never cleared.
846 */
847#define EXTF_PAIRED 0x4
848
849#define EXTF_MASK \
850 (EXTF_COMPOSITE | EXTF_READONLY | EXTF_PAIRED)
851
852#define MEXT_MINREF(m) ((m_get_rfa(m))->minref)
853#define MEXT_REF(m) ((m_get_rfa(m))->refcnt)
854#define MEXT_PREF(m) ((m_get_rfa(m))->prefcnt)
855#define MEXT_FLAGS(m) ((m_get_rfa(m))->flags)
856#define MEXT_PRIV(m) ((m_get_rfa(m))->priv)
857#define MEXT_PMBUF(m) ((m_get_rfa(m))->paired)
858#define MEXT_TOKEN(m) ((m_get_rfa(m))->ext_token)
859#define MBUF_IS_COMPOSITE(m) \
860 (MEXT_REF(m) == MEXT_MINREF(m) && \
861 (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_COMPOSITE)
862/*
863 * This macro can be used to test if the mbuf is paired to an external
864 * cluster. The test for MEXT_PMBUF being equal to the mbuf in subject
865 * is important, as EXTF_PAIRED alone is insufficient since it is immutable,
866 * and thus survives calls to m_free_paired.
867 */
868#define MBUF_IS_PAIRED(m) \
869 (((m)->m_flags & M_EXT) && \
870 (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_PAIRED && \
871 MEXT_PMBUF(m) == (m))
872
873/*
874 * Macros used to verify the integrity of the mbuf.
875 */
876#define _MCHECK(m) { \
877 if ((m)->m_type != MT_FREE && !MBUF_IS_PAIRED(m)) { \
878 if (mclaudit == NULL) \
879 panic("MCHECK: m_type=%d m=%p", \
880 (u_int16_t)(m)->m_type, m); \
881 else \
882 mcl_audit_mcheck_panic(m); \
883 } \
884}
885
886#define MBUF_IN_MAP(addr) \
887 ((unsigned char *)(addr) >= mbutl && \
888 (unsigned char *)(addr) < embutl)
889
890#define MRANGE(addr) { \
891 if (!MBUF_IN_MAP(addr)) \
892 panic("MRANGE: address out of range 0x%p", addr); \
893}
894
895/*
896 * Macro version of mtod.
897 */
898#define MTOD(m, t) ((t)((m)->m_data))
899
900/*
901 * Macros to obtain page index given a base cluster address
902 */
903#define MTOPG(x) (((unsigned char *)x - mbutl) >> PAGE_SHIFT)
904#define PGTOM(x) (mbutl + (x << PAGE_SHIFT))
905
906/*
907 * Macro to find the mbuf index relative to a base.
908 */
909#define MBPAGEIDX(c, m) \
910 (((unsigned char *)(m) - (unsigned char *)(c)) >> MSIZESHIFT)
911
912/*
913 * Same thing for 2KB cluster index.
914 */
915#define CLPAGEIDX(c, m) \
916 (((unsigned char *)(m) - (unsigned char *)(c)) >> MCLSHIFT)
917
918/*
919 * Macro to find 4KB cluster index relative to a base
920 */
921#define BCLPAGEIDX(c, m) \
922 (((unsigned char *)(m) - (unsigned char *)(c)) >> MBIGCLSHIFT)
923
924/*
925 * Macros used during mbuf and cluster initialization.
926 */
927#define MBUF_INIT_PKTHDR(m) { \
928 (m)->m_pkthdr.rcvif = NULL; \
929 (m)->m_pkthdr.pkt_hdr = NULL; \
930 (m)->m_pkthdr.len = 0; \
931 (m)->m_pkthdr.csum_flags = 0; \
932 (m)->m_pkthdr.csum_data = 0; \
933 (m)->m_pkthdr.vlan_tag = 0; \
934 m_classifier_init(m, 0); \
935 m_tag_init(m, 1); \
936 m_scratch_init(m); \
937 m_redzone_init(m); \
938}
939
940#define MBUF_INIT(m, pkthdr, type) { \
941 _MCHECK(m); \
942 (m)->m_next = (m)->m_nextpkt = NULL; \
943 (m)->m_len = 0; \
944 (m)->m_type = type; \
945 if ((pkthdr) == 0) { \
946 (m)->m_data = (m)->m_dat; \
947 (m)->m_flags = 0; \
948 } else { \
949 (m)->m_data = (m)->m_pktdat; \
950 (m)->m_flags = M_PKTHDR; \
951 MBUF_INIT_PKTHDR(m); \
952 } \
953}
954
955#define MEXT_INIT(m, buf, size, free, arg, rfa, min, ref, pref, flag, \
956 priv, pm) { \
957 (m)->m_data = (m)->m_ext.ext_buf = (buf); \
958 (m)->m_flags |= M_EXT; \
959 m_set_ext((m), (rfa), (free), (arg)); \
960 (m)->m_ext.ext_size = (size); \
961 MEXT_MINREF(m) = (min); \
962 MEXT_REF(m) = (ref); \
963 MEXT_PREF(m) = (pref); \
964 MEXT_FLAGS(m) = (flag); \
965 MEXT_PRIV(m) = (priv); \
966 MEXT_PMBUF(m) = (pm); \
967}
968
969#define MBUF_CL_INIT(m, buf, rfa, ref, flag) \
970 MEXT_INIT(m, buf, m_maxsize(MC_CL), NULL, NULL, rfa, 0, \
971 ref, 0, flag, 0, NULL)
972
973#define MBUF_BIGCL_INIT(m, buf, rfa, ref, flag) \
974 MEXT_INIT(m, buf, m_maxsize(MC_BIGCL), m_bigfree, NULL, rfa, 0, \
975 ref, 0, flag, 0, NULL)
976
977#define MBUF_16KCL_INIT(m, buf, rfa, ref, flag) \
978 MEXT_INIT(m, buf, m_maxsize(MC_16KCL), m_16kfree, NULL, rfa, 0, \
979 ref, 0, flag, 0, NULL)
980
981/*
982 * Macro to convert BSD malloc sleep flag to mcache's
983 */
984#define MSLEEPF(f) ((!((f) & M_DONTWAIT)) ? MCR_SLEEP : MCR_NOSLEEP)
985
986/*
987 * The structure that holds all mbuf class statistics exportable via sysctl.
988 * Similar to mbstat structure, the mb_stat structure is protected by the
989 * global mbuf lock. It contains additional information about the classes
990 * that allows for a more accurate view of the state of the allocator.
991 */
992struct mb_stat *mb_stat;
993struct omb_stat *omb_stat; /* For backwards compatibility */
994
995#define MB_STAT_SIZE(n) \
996 __builtin_offsetof(mb_stat_t, mbs_class[n])
997#define OMB_STAT_SIZE(n) \
998 ((size_t)(&((struct omb_stat *)0)->mbs_class[n]))
999
1000/*
1001 * The legacy structure holding all of the mbuf allocation statistics.
1002 * The actual statistics used by the kernel are stored in the mbuf_table
1003 * instead, and are updated atomically while the global mbuf lock is held.
1004 * They are mirrored in mbstat to support legacy applications (e.g. netstat).
1005 * Unlike before, the kernel no longer relies on the contents of mbstat for
1006 * its operations (e.g. cluster expansion) because the structure is exposed
1007 * to outside and could possibly be modified, therefore making it unsafe.
1008 * With the exception of the mbstat.m_mtypes array (see below), all of the
1009 * statistics are updated as they change.
1010 */
1011struct mbstat mbstat;
1012
1013#define MBSTAT_MTYPES_MAX \
1014 (sizeof (mbstat.m_mtypes) / sizeof (mbstat.m_mtypes[0]))
1015
1016/*
1017 * Allocation statistics related to mbuf types (up to MT_MAX-1) are updated
1018 * atomically and stored in a per-CPU structure which is lock-free; this is
1019 * done in order to avoid writing to the global mbstat data structure which
1020 * would cause false sharing. During sysctl request for kern.ipc.mbstat,
1021 * the statistics across all CPUs will be converged into the mbstat.m_mtypes
1022 * array and returned to the application. Any updates for types greater or
1023 * equal than MT_MAX would be done atomically to the mbstat; this slows down
1024 * performance but is okay since the kernel uses only up to MT_MAX-1 while
1025 * anything beyond that (up to type 255) is considered a corner case.
1026 */
1027typedef struct {
1028 unsigned int cpu_mtypes[MT_MAX];
1029} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE), packed)) mtypes_cpu_t;
1030
1031typedef struct {
1032 mtypes_cpu_t mbs_cpu[1];
1033} mbuf_mtypes_t;
1034
1035static mbuf_mtypes_t *mbuf_mtypes; /* per-CPU statistics */
1036
1037#define MBUF_MTYPES_SIZE(n) \
1038 ((size_t)(&((mbuf_mtypes_t *)0)->mbs_cpu[n]))
1039
1040#define MTYPES_CPU(p) \
1041 ((mtypes_cpu_t *)(void *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number())))
1042
1043#define mtype_stat_add(type, n) { \
1044 if ((unsigned)(type) < MT_MAX) { \
1045 mtypes_cpu_t *mbs = MTYPES_CPU(mbuf_mtypes); \
1046 atomic_add_32(&mbs->cpu_mtypes[type], n); \
1047 } else if ((unsigned)(type) < (unsigned)MBSTAT_MTYPES_MAX) { \
1048 atomic_add_16((int16_t *)&mbstat.m_mtypes[type], n); \
1049 } \
1050}
1051
1052#define mtype_stat_sub(t, n) mtype_stat_add(t, -(n))
1053#define mtype_stat_inc(t) mtype_stat_add(t, 1)
1054#define mtype_stat_dec(t) mtype_stat_sub(t, 1)
1055
1056static void
1057mbuf_mtypes_sync(boolean_t locked)
1058{
1059 int m, n;
1060 mtypes_cpu_t mtc;
1061
1062 if (locked)
1063 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1064
1065 bzero(&mtc, sizeof (mtc));
1066 for (m = 0; m < ncpu; m++) {
1067 mtypes_cpu_t *scp = &mbuf_mtypes->mbs_cpu[m];
1068 mtypes_cpu_t temp;
1069
1070 bcopy(&scp->cpu_mtypes, &temp.cpu_mtypes,
1071 sizeof (temp.cpu_mtypes));
1072
1073 for (n = 0; n < MT_MAX; n++)
1074 mtc.cpu_mtypes[n] += temp.cpu_mtypes[n];
1075 }
1076 if (!locked)
1077 lck_mtx_lock(mbuf_mlock);
1078 for (n = 0; n < MT_MAX; n++)
1079 mbstat.m_mtypes[n] = mtc.cpu_mtypes[n];
1080 if (!locked)
1081 lck_mtx_unlock(mbuf_mlock);
1082}
1083
1084static int
1085mbstat_sysctl SYSCTL_HANDLER_ARGS
1086{
1087#pragma unused(oidp, arg1, arg2)
1088 mbuf_mtypes_sync(FALSE);
1089
1090 return (SYSCTL_OUT(req, &mbstat, sizeof (mbstat)));
1091}
1092
1093static void
1094mbuf_stat_sync(void)
1095{
1096 mb_class_stat_t *sp;
1097 mcache_cpu_t *ccp;
1098 mcache_t *cp;
1099 int k, m, bktsize;
1100
1101 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1102
1103 for (k = 0; k < NELEM(mbuf_table); k++) {
1104 cp = m_cache(k);
1105 ccp = &cp->mc_cpu[0];
1106 bktsize = ccp->cc_bktsize;
1107 sp = mbuf_table[k].mtbl_stats;
1108
1109 if (cp->mc_flags & MCF_NOCPUCACHE)
1110 sp->mbcl_mc_state = MCS_DISABLED;
1111 else if (cp->mc_purge_cnt > 0)
1112 sp->mbcl_mc_state = MCS_PURGING;
1113 else if (bktsize == 0)
1114 sp->mbcl_mc_state = MCS_OFFLINE;
1115 else
1116 sp->mbcl_mc_state = MCS_ONLINE;
1117
1118 sp->mbcl_mc_cached = 0;
1119 for (m = 0; m < ncpu; m++) {
1120 ccp = &cp->mc_cpu[m];
1121 if (ccp->cc_objs > 0)
1122 sp->mbcl_mc_cached += ccp->cc_objs;
1123 if (ccp->cc_pobjs > 0)
1124 sp->mbcl_mc_cached += ccp->cc_pobjs;
1125 }
1126 sp->mbcl_mc_cached += (cp->mc_full.bl_total * bktsize);
1127 sp->mbcl_active = sp->mbcl_total - sp->mbcl_mc_cached -
1128 sp->mbcl_infree;
1129
1130 sp->mbcl_mc_waiter_cnt = cp->mc_waiter_cnt;
1131 sp->mbcl_mc_wretry_cnt = cp->mc_wretry_cnt;
1132 sp->mbcl_mc_nwretry_cnt = cp->mc_nwretry_cnt;
1133
1134 /* Calculate total count specific to each class */
1135 sp->mbcl_ctotal = sp->mbcl_total;
1136 switch (m_class(k)) {
1137 case MC_MBUF:
1138 /* Deduct mbufs used in composite caches */
1139 sp->mbcl_ctotal -= (m_total(MC_MBUF_CL) +
1140 m_total(MC_MBUF_BIGCL));
1141 break;
1142
1143 case MC_CL:
1144 /* Deduct clusters used in composite cache */
1145 sp->mbcl_ctotal -= m_total(MC_MBUF_CL);
1146 break;
1147
1148 case MC_BIGCL:
1149 /* Deduct clusters used in composite cache */
1150 sp->mbcl_ctotal -= m_total(MC_MBUF_BIGCL);
1151 break;
1152
1153 case MC_16KCL:
1154 /* Deduct clusters used in composite cache */
1155 sp->mbcl_ctotal -= m_total(MC_MBUF_16KCL);
1156 break;
1157
1158 default:
1159 break;
1160 }
1161 }
1162}
1163
1164static int
1165mb_stat_sysctl SYSCTL_HANDLER_ARGS
1166{
1167#pragma unused(oidp, arg1, arg2)
1168 void *statp;
1169 int k, statsz, proc64 = proc_is64bit(req->p);
1170
1171 lck_mtx_lock(mbuf_mlock);
1172 mbuf_stat_sync();
1173
1174 if (!proc64) {
1175 struct omb_class_stat *oc;
1176 struct mb_class_stat *c;
1177
1178 omb_stat->mbs_cnt = mb_stat->mbs_cnt;
1179 oc = &omb_stat->mbs_class[0];
1180 c = &mb_stat->mbs_class[0];
1181 for (k = 0; k < omb_stat->mbs_cnt; k++, oc++, c++) {
1182 (void) snprintf(oc->mbcl_cname, sizeof (oc->mbcl_cname),
1183 "%s", c->mbcl_cname);
1184 oc->mbcl_size = c->mbcl_size;
1185 oc->mbcl_total = c->mbcl_total;
1186 oc->mbcl_active = c->mbcl_active;
1187 oc->mbcl_infree = c->mbcl_infree;
1188 oc->mbcl_slab_cnt = c->mbcl_slab_cnt;
1189 oc->mbcl_alloc_cnt = c->mbcl_alloc_cnt;
1190 oc->mbcl_free_cnt = c->mbcl_free_cnt;
1191 oc->mbcl_notified = c->mbcl_notified;
1192 oc->mbcl_purge_cnt = c->mbcl_purge_cnt;
1193 oc->mbcl_fail_cnt = c->mbcl_fail_cnt;
1194 oc->mbcl_ctotal = c->mbcl_ctotal;
1195 oc->mbcl_release_cnt = c->mbcl_release_cnt;
1196 oc->mbcl_mc_state = c->mbcl_mc_state;
1197 oc->mbcl_mc_cached = c->mbcl_mc_cached;
1198 oc->mbcl_mc_waiter_cnt = c->mbcl_mc_waiter_cnt;
1199 oc->mbcl_mc_wretry_cnt = c->mbcl_mc_wretry_cnt;
1200 oc->mbcl_mc_nwretry_cnt = c->mbcl_mc_nwretry_cnt;
1201 }
1202 statp = omb_stat;
1203 statsz = OMB_STAT_SIZE(NELEM(mbuf_table));
1204 } else {
1205 statp = mb_stat;
1206 statsz = MB_STAT_SIZE(NELEM(mbuf_table));
1207 }
1208
1209 lck_mtx_unlock(mbuf_mlock);
1210
1211 return (SYSCTL_OUT(req, statp, statsz));
1212}
1213
1214static int
1215mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS
1216{
1217#pragma unused(oidp, arg1, arg2)
1218 int i;
1219
1220 /* Ensure leak tracing turned on */
1221 if (!mclfindleak || !mclexpleak)
1222 return (ENXIO);
1223
1224 lck_mtx_lock(mleak_lock);
1225 mleak_update_stats();
1226 i = SYSCTL_OUT(req, mleak_stat, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES));
1227 lck_mtx_unlock(mleak_lock);
1228
1229 return (i);
1230}
1231
1232static int
1233mleak_table_sysctl SYSCTL_HANDLER_ARGS
1234{
1235#pragma unused(oidp, arg1, arg2)
1236 int i = 0;
1237
1238 /* Ensure leak tracing turned on */
1239 if (!mclfindleak || !mclexpleak)
1240 return (ENXIO);
1241
1242 lck_mtx_lock(mleak_lock);
1243 i = SYSCTL_OUT(req, &mleak_table, sizeof (mleak_table));
1244 lck_mtx_unlock(mleak_lock);
1245
1246 return (i);
1247}
1248
1249static inline void
1250m_incref(struct mbuf *m)
1251{
1252 UInt16 old, new;
1253 volatile UInt16 *addr = (volatile UInt16 *)&MEXT_REF(m);
1254
1255 do {
1256 old = *addr;
1257 new = old + 1;
1258 ASSERT(new != 0);
1259 } while (!OSCompareAndSwap16(old, new, addr));
1260
1261 /*
1262 * If cluster is shared, mark it with (sticky) EXTF_READONLY;
1263 * we don't clear the flag when the refcount goes back to the
1264 * minimum, to simplify code calling m_mclhasreference().
1265 */
1266 if (new > (MEXT_MINREF(m) + 1) && !(MEXT_FLAGS(m) & EXTF_READONLY))
1267 (void) OSBitOrAtomic16(EXTF_READONLY, &MEXT_FLAGS(m));
1268}
1269
1270static inline u_int16_t
1271m_decref(struct mbuf *m)
1272{
1273 UInt16 old, new;
1274 volatile UInt16 *addr = (volatile UInt16 *)&MEXT_REF(m);
1275
1276 do {
1277 old = *addr;
1278 new = old - 1;
1279 ASSERT(old != 0);
1280 } while (!OSCompareAndSwap16(old, new, addr));
1281
1282 return (new);
1283}
1284
1285static void
1286mbuf_table_init(void)
1287{
1288 unsigned int b, c, s;
1289 int m, config_mbuf_jumbo = 0;
1290
1291 MALLOC(omb_stat, struct omb_stat *, OMB_STAT_SIZE(NELEM(mbuf_table)),
1292 M_TEMP, M_WAITOK | M_ZERO);
1293 VERIFY(omb_stat != NULL);
1294
1295 MALLOC(mb_stat, mb_stat_t *, MB_STAT_SIZE(NELEM(mbuf_table)),
1296 M_TEMP, M_WAITOK | M_ZERO);
1297 VERIFY(mb_stat != NULL);
1298
1299 mb_stat->mbs_cnt = NELEM(mbuf_table);
1300 for (m = 0; m < NELEM(mbuf_table); m++)
1301 mbuf_table[m].mtbl_stats = &mb_stat->mbs_class[m];
1302
1303#if CONFIG_MBUF_JUMBO
1304 config_mbuf_jumbo = 1;
1305#endif /* CONFIG_MBUF_JUMBO */
1306
1307 if (config_mbuf_jumbo == 1 || PAGE_SIZE == M16KCLBYTES) {
1308 /*
1309 * Set aside 1/3 of the mbuf cluster map for jumbo
1310 * clusters; we do this only on platforms where jumbo
1311 * cluster pool is enabled.
1312 */
1313 njcl = nmbclusters / 3;
1314 njclbytes = M16KCLBYTES;
1315 }
1316
1317 /*
1318 * nclusters holds both the 2KB and 4KB pools, so ensure it's
1319 * a multiple of 4KB clusters.
1320 */
1321 nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPG);
1322 if (njcl > 0) {
1323 /*
1324 * Each jumbo cluster takes 8 2KB clusters, so make
1325 * sure that the pool size is evenly divisible by 8;
1326 * njcl is in 2KB unit, hence treated as such.
1327 */
1328 njcl = P2ROUNDDOWN(nmbclusters - nclusters, NCLPJCL);
1329
1330 /* Update nclusters with rounded down value of njcl */
1331 nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPG);
1332 }
1333
1334 /*
1335 * njcl is valid only on platforms with 16KB jumbo clusters or
1336 * with 16KB pages, where it is configured to 1/3 of the pool
1337 * size. On these platforms, the remaining is used for 2KB
1338 * and 4KB clusters. On platforms without 16KB jumbo clusters,
1339 * the entire pool is used for both 2KB and 4KB clusters. A 4KB
1340 * cluster can either be splitted into 16 mbufs, or into 2 2KB
1341 * clusters.
1342 *
1343 * +---+---+------------ ... -----------+------- ... -------+
1344 * | c | b | s | njcl |
1345 * +---+---+------------ ... -----------+------- ... -------+
1346 *
1347 * 1/32th of the shared region is reserved for pure 2KB and 4KB
1348 * clusters (1/64th each.)
1349 */
1350 c = P2ROUNDDOWN((nclusters >> 6), NCLPG); /* in 2KB unit */
1351 b = P2ROUNDDOWN((nclusters >> (6 + NCLPBGSHIFT)), NBCLPG); /* in 4KB unit */
1352 s = nclusters - (c + (b << NCLPBGSHIFT)); /* in 2KB unit */
1353
1354 /*
1355 * 1/64th (c) is reserved for 2KB clusters.
1356 */
1357 m_minlimit(MC_CL) = c;
1358 m_maxlimit(MC_CL) = s + c; /* in 2KB unit */
1359 m_maxsize(MC_CL) = m_size(MC_CL) = MCLBYTES;
1360 (void) snprintf(m_cname(MC_CL), MAX_MBUF_CNAME, "cl");
1361
1362 /*
1363 * Another 1/64th (b) of the map is reserved for 4KB clusters.
1364 * It cannot be turned into 2KB clusters or mbufs.
1365 */
1366 m_minlimit(MC_BIGCL) = b;
1367 m_maxlimit(MC_BIGCL) = (s >> NCLPBGSHIFT) + b; /* in 4KB unit */
1368 m_maxsize(MC_BIGCL) = m_size(MC_BIGCL) = MBIGCLBYTES;
1369 (void) snprintf(m_cname(MC_BIGCL), MAX_MBUF_CNAME, "bigcl");
1370
1371 /*
1372 * The remaining 31/32ths (s) are all-purpose (mbufs, 2KB, or 4KB)
1373 */
1374 m_minlimit(MC_MBUF) = 0;
1375 m_maxlimit(MC_MBUF) = (s << NMBPCLSHIFT); /* in mbuf unit */
1376 m_maxsize(MC_MBUF) = m_size(MC_MBUF) = MSIZE;
1377 (void) snprintf(m_cname(MC_MBUF), MAX_MBUF_CNAME, "mbuf");
1378
1379 /*
1380 * Set limits for the composite classes.
1381 */
1382 m_minlimit(MC_MBUF_CL) = 0;
1383 m_maxlimit(MC_MBUF_CL) = m_maxlimit(MC_CL);
1384 m_maxsize(MC_MBUF_CL) = MCLBYTES;
1385 m_size(MC_MBUF_CL) = m_size(MC_MBUF) + m_size(MC_CL);
1386 (void) snprintf(m_cname(MC_MBUF_CL), MAX_MBUF_CNAME, "mbuf_cl");
1387
1388 m_minlimit(MC_MBUF_BIGCL) = 0;
1389 m_maxlimit(MC_MBUF_BIGCL) = m_maxlimit(MC_BIGCL);
1390 m_maxsize(MC_MBUF_BIGCL) = MBIGCLBYTES;
1391 m_size(MC_MBUF_BIGCL) = m_size(MC_MBUF) + m_size(MC_BIGCL);
1392 (void) snprintf(m_cname(MC_MBUF_BIGCL), MAX_MBUF_CNAME, "mbuf_bigcl");
1393
1394 /*
1395 * And for jumbo classes.
1396 */
1397 m_minlimit(MC_16KCL) = 0;
1398 m_maxlimit(MC_16KCL) = (njcl >> NCLPJCLSHIFT); /* in 16KB unit */
1399 m_maxsize(MC_16KCL) = m_size(MC_16KCL) = M16KCLBYTES;
1400 (void) snprintf(m_cname(MC_16KCL), MAX_MBUF_CNAME, "16kcl");
1401
1402 m_minlimit(MC_MBUF_16KCL) = 0;
1403 m_maxlimit(MC_MBUF_16KCL) = m_maxlimit(MC_16KCL);
1404 m_maxsize(MC_MBUF_16KCL) = M16KCLBYTES;
1405 m_size(MC_MBUF_16KCL) = m_size(MC_MBUF) + m_size(MC_16KCL);
1406 (void) snprintf(m_cname(MC_MBUF_16KCL), MAX_MBUF_CNAME, "mbuf_16kcl");
1407
1408 /*
1409 * Initialize the legacy mbstat structure.
1410 */
1411 bzero(&mbstat, sizeof (mbstat));
1412 mbstat.m_msize = m_maxsize(MC_MBUF);
1413 mbstat.m_mclbytes = m_maxsize(MC_CL);
1414 mbstat.m_minclsize = MINCLSIZE;
1415 mbstat.m_mlen = MLEN;
1416 mbstat.m_mhlen = MHLEN;
1417 mbstat.m_bigmclbytes = m_maxsize(MC_BIGCL);
1418}
1419
1420#if defined(__LP64__)
1421typedef struct ncl_tbl {
1422 uint64_t nt_maxmem; /* memory (sane) size */
1423 uint32_t nt_mbpool; /* mbuf pool size */
1424} ncl_tbl_t;
1425
1426/* Non-server */
1427static ncl_tbl_t ncl_table[] = {
1428 { (1ULL << GBSHIFT) /* 1 GB */, (64 << MBSHIFT) /* 64 MB */ },
1429 { (1ULL << (GBSHIFT + 3)) /* 8 GB */, (96 << MBSHIFT) /* 96 MB */ },
1430 { (1ULL << (GBSHIFT + 4)) /* 16 GB */, (128 << MBSHIFT) /* 128 MB */ },
1431 { 0, 0 }
1432};
1433
1434/* Server */
1435static ncl_tbl_t ncl_table_srv[] = {
1436 { (1ULL << GBSHIFT) /* 1 GB */, (96 << MBSHIFT) /* 96 MB */ },
1437 { (1ULL << (GBSHIFT + 2)) /* 4 GB */, (128 << MBSHIFT) /* 128 MB */ },
1438 { (1ULL << (GBSHIFT + 3)) /* 8 GB */, (160 << MBSHIFT) /* 160 MB */ },
1439 { (1ULL << (GBSHIFT + 4)) /* 16 GB */, (192 << MBSHIFT) /* 192 MB */ },
1440 { (1ULL << (GBSHIFT + 5)) /* 32 GB */, (256 << MBSHIFT) /* 256 MB */ },
1441 { (1ULL << (GBSHIFT + 6)) /* 64 GB */, (384 << MBSHIFT) /* 384 MB */ },
1442 { 0, 0 }
1443};
1444#endif /* __LP64__ */
1445
1446__private_extern__ unsigned int
1447mbuf_default_ncl(int server, uint64_t mem)
1448{
1449#if !defined(__LP64__)
1450#pragma unused(server)
1451 unsigned int n;
1452 /*
1453 * 32-bit kernel (default to 64MB of mbuf pool for >= 1GB RAM).
1454 */
1455 if ((n = ((mem / 16) / MCLBYTES)) > 32768)
1456 n = 32768;
1457#else
1458 unsigned int n, i;
1459 ncl_tbl_t *tbl = (server ? ncl_table_srv : ncl_table);
1460 /*
1461 * 64-bit kernel (mbuf pool size based on table).
1462 */
1463 n = tbl[0].nt_mbpool;
1464 for (i = 0; tbl[i].nt_mbpool != 0; i++) {
1465 if (mem < tbl[i].nt_maxmem)
1466 break;
1467 n = tbl[i].nt_mbpool;
1468 }
1469 n >>= MCLSHIFT;
1470#endif /* !__LP64__ */
1471 return (n);
1472}
1473
1474__private_extern__ void
1475mbinit(void)
1476{
1477 unsigned int m;
1478 unsigned int initmcl = 0;
1479 void *buf;
1480 thread_t thread = THREAD_NULL;
1481
1482 microuptime(&mb_start);
1483
1484 /*
1485 * These MBUF_ values must be equal to their private counterparts.
1486 */
1487 _CASSERT(MBUF_EXT == M_EXT);
1488 _CASSERT(MBUF_PKTHDR == M_PKTHDR);
1489 _CASSERT(MBUF_EOR == M_EOR);
1490 _CASSERT(MBUF_LOOP == M_LOOP);
1491 _CASSERT(MBUF_BCAST == M_BCAST);
1492 _CASSERT(MBUF_MCAST == M_MCAST);
1493 _CASSERT(MBUF_FRAG == M_FRAG);
1494 _CASSERT(MBUF_FIRSTFRAG == M_FIRSTFRAG);
1495 _CASSERT(MBUF_LASTFRAG == M_LASTFRAG);
1496 _CASSERT(MBUF_PROMISC == M_PROMISC);
1497 _CASSERT(MBUF_HASFCS == M_HASFCS);
1498
1499 _CASSERT(MBUF_TYPE_FREE == MT_FREE);
1500 _CASSERT(MBUF_TYPE_DATA == MT_DATA);
1501 _CASSERT(MBUF_TYPE_HEADER == MT_HEADER);
1502 _CASSERT(MBUF_TYPE_SOCKET == MT_SOCKET);
1503 _CASSERT(MBUF_TYPE_PCB == MT_PCB);
1504 _CASSERT(MBUF_TYPE_RTABLE == MT_RTABLE);
1505 _CASSERT(MBUF_TYPE_HTABLE == MT_HTABLE);
1506 _CASSERT(MBUF_TYPE_ATABLE == MT_ATABLE);
1507 _CASSERT(MBUF_TYPE_SONAME == MT_SONAME);
1508 _CASSERT(MBUF_TYPE_SOOPTS == MT_SOOPTS);
1509 _CASSERT(MBUF_TYPE_FTABLE == MT_FTABLE);
1510 _CASSERT(MBUF_TYPE_RIGHTS == MT_RIGHTS);
1511 _CASSERT(MBUF_TYPE_IFADDR == MT_IFADDR);
1512 _CASSERT(MBUF_TYPE_CONTROL == MT_CONTROL);
1513 _CASSERT(MBUF_TYPE_OOBDATA == MT_OOBDATA);
1514
1515 _CASSERT(MBUF_TSO_IPV4 == CSUM_TSO_IPV4);
1516 _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6);
1517 _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_PARTIAL);
1518 _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16);
1519 _CASSERT(MBUF_CSUM_REQ_ZERO_INVERT == CSUM_ZERO_INVERT);
1520 _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP);
1521 _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP);
1522 _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP);
1523 _CASSERT(MBUF_CSUM_REQ_TCPIPV6 == CSUM_TCPIPV6);
1524 _CASSERT(MBUF_CSUM_REQ_UDPIPV6 == CSUM_UDPIPV6);
1525 _CASSERT(MBUF_CSUM_DID_IP == CSUM_IP_CHECKED);
1526 _CASSERT(MBUF_CSUM_IP_GOOD == CSUM_IP_VALID);
1527 _CASSERT(MBUF_CSUM_DID_DATA == CSUM_DATA_VALID);
1528 _CASSERT(MBUF_CSUM_PSEUDO_HDR == CSUM_PSEUDO_HDR);
1529
1530 _CASSERT(MBUF_WAITOK == M_WAIT);
1531 _CASSERT(MBUF_DONTWAIT == M_DONTWAIT);
1532 _CASSERT(MBUF_COPYALL == M_COPYALL);
1533
1534 _CASSERT(MBUF_SC2TC(MBUF_SC_BK_SYS) == MBUF_TC_BK);
1535 _CASSERT(MBUF_SC2TC(MBUF_SC_BK) == MBUF_TC_BK);
1536 _CASSERT(MBUF_SC2TC(MBUF_SC_BE) == MBUF_TC_BE);
1537 _CASSERT(MBUF_SC2TC(MBUF_SC_RD) == MBUF_TC_BE);
1538 _CASSERT(MBUF_SC2TC(MBUF_SC_OAM) == MBUF_TC_BE);
1539 _CASSERT(MBUF_SC2TC(MBUF_SC_AV) == MBUF_TC_VI);
1540 _CASSERT(MBUF_SC2TC(MBUF_SC_RV) == MBUF_TC_VI);
1541 _CASSERT(MBUF_SC2TC(MBUF_SC_VI) == MBUF_TC_VI);
1542 _CASSERT(MBUF_SC2TC(MBUF_SC_SIG) == MBUF_TC_VI);
1543 _CASSERT(MBUF_SC2TC(MBUF_SC_VO) == MBUF_TC_VO);
1544 _CASSERT(MBUF_SC2TC(MBUF_SC_CTL) == MBUF_TC_VO);
1545
1546 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BK) == SCVAL_BK);
1547 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BE) == SCVAL_BE);
1548 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VI) == SCVAL_VI);
1549 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VO) == SCVAL_VO);
1550
1551 /* Module specific scratch space (32-bit alignment requirement) */
1552 _CASSERT(!(offsetof(struct mbuf, m_pkthdr.pkt_mpriv) %
1553 sizeof (uint32_t)));
1554
1555 /* Initialize random red zone cookie value */
1556 _CASSERT(sizeof (mb_redzone_cookie) ==
1557 sizeof (((struct pkthdr *)0)->redzone));
1558 read_random(&mb_redzone_cookie, sizeof (mb_redzone_cookie));
1559 read_random(&mb_obscure_extref, sizeof (mb_obscure_extref));
1560 read_random(&mb_obscure_extfree, sizeof (mb_obscure_extfree));
1561 mb_obscure_extref |= 0x3;
1562 mb_obscure_extfree |= 0x3;
1563
1564 /* Make sure we don't save more than we should */
1565 _CASSERT(MCA_SAVED_MBUF_SIZE <= sizeof (struct mbuf));
1566
1567 if (nmbclusters == 0)
1568 nmbclusters = NMBCLUSTERS;
1569
1570 /* This should be a sane (at least even) value by now */
1571 VERIFY(nmbclusters != 0 && !(nmbclusters & 0x1));
1572
1573 /* Setup the mbuf table */
1574 mbuf_table_init();
1575
1576 /* Global lock for common layer */
1577 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
1578 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
1579 mbuf_mlock_attr = lck_attr_alloc_init();
1580 lck_mtx_init(mbuf_mlock, mbuf_mlock_grp, mbuf_mlock_attr);
1581
1582 /*
1583 * Allocate cluster slabs table:
1584 *
1585 * maxslabgrp = (N * 2048) / (1024 * 1024)
1586 *
1587 * Where N is nmbclusters rounded up to the nearest 512. This yields
1588 * mcl_slab_g_t units, each one representing a MB of memory.
1589 */
1590 maxslabgrp =
1591 (P2ROUNDUP(nmbclusters, (MBSIZE >> MCLSHIFT)) << MCLSHIFT) >> MBSHIFT;
1592 MALLOC(slabstbl, mcl_slabg_t **, maxslabgrp * sizeof (mcl_slabg_t *),
1593 M_TEMP, M_WAITOK | M_ZERO);
1594 VERIFY(slabstbl != NULL);
1595
1596 /*
1597 * Allocate audit structures, if needed:
1598 *
1599 * maxclaudit = (maxslabgrp * 1024 * 1024) / PAGE_SIZE
1600 *
1601 * This yields mcl_audit_t units, each one representing a page.
1602 */
1603 PE_parse_boot_argn("mbuf_debug", &mbuf_debug, sizeof (mbuf_debug));
1604 mbuf_debug |= mcache_getflags();
1605 if (mbuf_debug & MCF_DEBUG) {
1606 int l;
1607 mcl_audit_t *mclad;
1608 maxclaudit = ((maxslabgrp << MBSHIFT) >> PAGE_SHIFT);
1609 MALLOC(mclaudit, mcl_audit_t *, maxclaudit * sizeof (*mclaudit),
1610 M_TEMP, M_WAITOK | M_ZERO);
1611 VERIFY(mclaudit != NULL);
1612 for (l = 0, mclad = mclaudit; l < maxclaudit; l++) {
1613 MALLOC(mclad[l].cl_audit, mcache_audit_t **,
1614 NMBPG * sizeof(mcache_audit_t *),
1615 M_TEMP, M_WAITOK | M_ZERO);
1616 VERIFY(mclad[l].cl_audit != NULL);
1617 }
1618
1619 mcl_audit_con_cache = mcache_create("mcl_audit_contents",
1620 AUDIT_CONTENTS_SIZE, sizeof (u_int64_t), 0, MCR_SLEEP);
1621 VERIFY(mcl_audit_con_cache != NULL);
1622 }
1623 mclverify = (mbuf_debug & MCF_VERIFY);
1624 mcltrace = (mbuf_debug & MCF_TRACE);
1625 mclfindleak = !(mbuf_debug & MCF_NOLEAKLOG);
1626 mclexpleak = mclfindleak && (mbuf_debug & MCF_EXPLEAKLOG);
1627
1628 /* Enable mbuf leak logging, with a lock to protect the tables */
1629
1630 mleak_lock_grp_attr = lck_grp_attr_alloc_init();
1631 mleak_lock_grp = lck_grp_alloc_init("mleak_lock", mleak_lock_grp_attr);
1632 mleak_lock_attr = lck_attr_alloc_init();
1633 lck_mtx_init(mleak_lock, mleak_lock_grp, mleak_lock_attr);
1634
1635 mleak_activate();
1636
1637 /*
1638 * Allocate structure for per-CPU statistics that's aligned
1639 * on the CPU cache boundary; this code assumes that we never
1640 * uninitialize this framework, since the original address
1641 * before alignment is not saved.
1642 */
1643 ncpu = ml_get_max_cpus();
1644 MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE,
1645 M_TEMP, M_WAITOK);
1646 VERIFY(buf != NULL);
1647
1648 mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf,
1649 CPU_CACHE_LINE_SIZE);
1650 bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu));
1651
1652 /* Calculate the number of pages assigned to the cluster pool */
1653 mcl_pages = (nmbclusters << MCLSHIFT) / PAGE_SIZE;
1654 MALLOC(mcl_paddr, ppnum_t *, mcl_pages * sizeof (ppnum_t),
1655 M_TEMP, M_WAITOK);
1656 VERIFY(mcl_paddr != NULL);
1657
1658 /* Register with the I/O Bus mapper */
1659 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
1660 bzero((char *)mcl_paddr, mcl_pages * sizeof (ppnum_t));
1661
1662 embutl = (mbutl + (nmbclusters * MCLBYTES));
1663 VERIFY(((embutl - mbutl) % MBIGCLBYTES) == 0);
1664
1665 /* Prime up the freelist */
1666 PE_parse_boot_argn("initmcl", &initmcl, sizeof (initmcl));
1667 if (initmcl != 0) {
1668 initmcl >>= NCLPBGSHIFT; /* become a 4K unit */
1669 if (initmcl > m_maxlimit(MC_BIGCL))
1670 initmcl = m_maxlimit(MC_BIGCL);
1671 }
1672 if (initmcl < m_minlimit(MC_BIGCL))
1673 initmcl = m_minlimit(MC_BIGCL);
1674
1675 lck_mtx_lock(mbuf_mlock);
1676
1677 /*
1678 * For classes with non-zero minimum limits, populate their freelists
1679 * so that m_total(class) is at least m_minlimit(class).
1680 */
1681 VERIFY(m_total(MC_BIGCL) == 0 && m_minlimit(MC_BIGCL) != 0);
1682 freelist_populate(m_class(MC_BIGCL), initmcl, M_WAIT);
1683 VERIFY(m_total(MC_BIGCL) >= m_minlimit(MC_BIGCL));
1684 freelist_init(m_class(MC_CL));
1685
1686 for (m = 0; m < NELEM(mbuf_table); m++) {
1687 /* Make sure we didn't miss any */
1688 VERIFY(m_minlimit(m_class(m)) == 0 ||
1689 m_total(m_class(m)) >= m_minlimit(m_class(m)));
1690
1691 /* populate the initial sizes and report from there on */
1692 m_peak(m_class(m)) = m_total(m_class(m));
1693 }
1694 mb_peak_newreport = FALSE;
1695
1696 lck_mtx_unlock(mbuf_mlock);
1697
1698 (void) kernel_thread_start((thread_continue_t)mbuf_worker_thread_init,
1699 NULL, &thread);
1700 thread_deallocate(thread);
1701
1702 ref_cache = mcache_create("mext_ref", sizeof (struct ext_ref),
1703 0, 0, MCR_SLEEP);
1704
1705 /* Create the cache for each class */
1706 for (m = 0; m < NELEM(mbuf_table); m++) {
1707 void *allocfunc, *freefunc, *auditfunc, *logfunc;
1708 u_int32_t flags;
1709
1710 flags = mbuf_debug;
1711 if (m_class(m) == MC_MBUF_CL || m_class(m) == MC_MBUF_BIGCL ||
1712 m_class(m) == MC_MBUF_16KCL) {
1713 allocfunc = mbuf_cslab_alloc;
1714 freefunc = mbuf_cslab_free;
1715 auditfunc = mbuf_cslab_audit;
1716 logfunc = mleak_logger;
1717 } else {
1718 allocfunc = mbuf_slab_alloc;
1719 freefunc = mbuf_slab_free;
1720 auditfunc = mbuf_slab_audit;
1721 logfunc = mleak_logger;
1722 }
1723
1724 /*
1725 * Disable per-CPU caches for jumbo classes if there
1726 * is no jumbo cluster pool available in the system.
1727 * The cache itself is still created (but will never
1728 * be populated) since it simplifies the code.
1729 */
1730 if ((m_class(m) == MC_MBUF_16KCL || m_class(m) == MC_16KCL) &&
1731 njcl == 0)
1732 flags |= MCF_NOCPUCACHE;
1733
1734 if (!mclfindleak)
1735 flags |= MCF_NOLEAKLOG;
1736
1737 m_cache(m) = mcache_create_ext(m_cname(m), m_maxsize(m),
1738 allocfunc, freefunc, auditfunc, logfunc, mbuf_slab_notify,
1739 (void *)(uintptr_t)m, flags, MCR_SLEEP);
1740 }
1741
1742 /*
1743 * Set the max limit on sb_max to be 1/16 th of the size of
1744 * memory allocated for mbuf clusters.
1745 */
1746 high_sb_max = (nmbclusters << (MCLSHIFT - 4));
1747 if (high_sb_max < sb_max) {
1748 /* sb_max is too large for this configuration, scale it down */
1749 if (high_sb_max > (1 << MBSHIFT)) {
1750 /* We have atleast 16 M of mbuf pool */
1751 sb_max = high_sb_max;
1752 } else if ((nmbclusters << MCLSHIFT) > (1 << MBSHIFT)) {
1753 /*
1754 * If we have more than 1M of mbufpool, cap the size of
1755 * max sock buf at 1M
1756 */
1757 sb_max = high_sb_max = (1 << MBSHIFT);
1758 } else {
1759 sb_max = high_sb_max;
1760 }
1761 }
1762
1763 /* allocate space for mbuf_dump_buf */
1764 MALLOC(mbuf_dump_buf, char *, MBUF_DUMP_BUF_SIZE, M_TEMP, M_WAITOK);
1765 VERIFY(mbuf_dump_buf != NULL);
1766
1767 if (mbuf_debug & MCF_DEBUG) {
1768 printf("%s: MLEN %d, MHLEN %d\n", __func__,
1769 (int)_MLEN, (int)_MHLEN);
1770 }
1771
1772 printf("%s: done [%d MB total pool size, (%d/%d) split]\n", __func__,
1773 (nmbclusters << MCLSHIFT) >> MBSHIFT,
1774 (nclusters << MCLSHIFT) >> MBSHIFT,
1775 (njcl << MCLSHIFT) >> MBSHIFT);
1776
1777 /* initialize lock form tx completion callback table */
1778 mbuf_tx_compl_tbl_lck_grp_attr = lck_grp_attr_alloc_init();
1779 if (mbuf_tx_compl_tbl_lck_grp_attr == NULL) {
1780 panic("%s: lck_grp_attr_alloc_init failed", __func__);
1781 /* NOTREACHED */
1782 }
1783 mbuf_tx_compl_tbl_lck_grp = lck_grp_alloc_init("mbuf_tx_compl_tbl",
1784 mbuf_tx_compl_tbl_lck_grp_attr);
1785 if (mbuf_tx_compl_tbl_lck_grp == NULL) {
1786 panic("%s: lck_grp_alloc_init failed", __func__);
1787 /* NOTREACHED */
1788 }
1789 mbuf_tx_compl_tbl_lck_attr = lck_attr_alloc_init();
1790 if (mbuf_tx_compl_tbl_lck_attr == NULL) {
1791 panic("%s: lck_attr_alloc_init failed", __func__);
1792 /* NOTREACHED */
1793 }
1794 lck_rw_init(mbuf_tx_compl_tbl_lock, mbuf_tx_compl_tbl_lck_grp,
1795 mbuf_tx_compl_tbl_lck_attr);
1796
1797}
1798
1799/*
1800 * Obtain a slab of object(s) from the class's freelist.
1801 */
1802static mcache_obj_t *
1803slab_alloc(mbuf_class_t class, int wait)
1804{
1805 mcl_slab_t *sp;
1806 mcache_obj_t *buf;
1807
1808 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1809
1810 /* This should always be NULL for us */
1811 VERIFY(m_cobjlist(class) == NULL);
1812
1813 /*
1814 * Treat composite objects as having longer lifespan by using
1815 * a slab from the reverse direction, in hoping that this could
1816 * reduce the probability of fragmentation for slabs that hold
1817 * more than one buffer chunks (e.g. mbuf slabs). For other
1818 * slabs, this probably doesn't make much of a difference.
1819 */
1820 if ((class == MC_MBUF || class == MC_CL || class == MC_BIGCL)
1821 && (wait & MCR_COMP))
1822 sp = (mcl_slab_t *)TAILQ_LAST(&m_slablist(class), mcl_slhead);
1823 else
1824 sp = (mcl_slab_t *)TAILQ_FIRST(&m_slablist(class));
1825
1826 if (sp == NULL) {
1827 VERIFY(m_infree(class) == 0 && m_slab_cnt(class) == 0);
1828 /* The slab list for this class is empty */
1829 return (NULL);
1830 }
1831
1832 VERIFY(m_infree(class) > 0);
1833 VERIFY(!slab_is_detached(sp));
1834 VERIFY(sp->sl_class == class &&
1835 (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
1836 buf = sp->sl_head;
1837 VERIFY(slab_inrange(sp, buf) && sp == slab_get(buf));
1838 sp->sl_head = buf->obj_next;
1839 /* Increment slab reference */
1840 sp->sl_refcnt++;
1841
1842 VERIFY(sp->sl_head != NULL || sp->sl_refcnt == sp->sl_chunks);
1843
1844 if (sp->sl_head != NULL && !slab_inrange(sp, sp->sl_head)) {
1845 slab_nextptr_panic(sp, sp->sl_head);
1846 /* In case sl_head is in the map but not in the slab */
1847 VERIFY(slab_inrange(sp, sp->sl_head));
1848 /* NOTREACHED */
1849 }
1850
1851 if (mclaudit != NULL) {
1852 mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
1853 mca->mca_uflags = 0;
1854 /* Save contents on mbuf objects only */
1855 if (class == MC_MBUF)
1856 mca->mca_uflags |= MB_SCVALID;
1857 }
1858
1859 if (class == MC_CL) {
1860 mbstat.m_clfree = (--m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
1861 /*
1862 * A 2K cluster slab can have at most NCLPG references.
1863 */
1864 VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NCLPG &&
1865 sp->sl_chunks == NCLPG && sp->sl_len == PAGE_SIZE);
1866 VERIFY(sp->sl_refcnt < NCLPG || sp->sl_head == NULL);
1867 } else if (class == MC_BIGCL) {
1868 mbstat.m_bigclfree = (--m_infree(MC_BIGCL)) +
1869 m_infree(MC_MBUF_BIGCL);
1870 /*
1871 * A 4K cluster slab can have NBCLPG references.
1872 */
1873 VERIFY(sp->sl_refcnt >= 1 && sp->sl_chunks == NBCLPG &&
1874 sp->sl_len == PAGE_SIZE &&
1875 (sp->sl_refcnt < NBCLPG || sp->sl_head == NULL));
1876 } else if (class == MC_16KCL) {
1877 mcl_slab_t *nsp;
1878 int k;
1879
1880 --m_infree(MC_16KCL);
1881 VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 &&
1882 sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1883 /*
1884 * Increment 2nd-Nth slab reference, where N is NSLABSP16KB.
1885 * A 16KB big cluster takes NSLABSP16KB slabs, each having at
1886 * most 1 reference.
1887 */
1888 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
1889 nsp = nsp->sl_next;
1890 /* Next slab must already be present */
1891 VERIFY(nsp != NULL);
1892 nsp->sl_refcnt++;
1893 VERIFY(!slab_is_detached(nsp));
1894 VERIFY(nsp->sl_class == MC_16KCL &&
1895 nsp->sl_flags == (SLF_MAPPED | SLF_PARTIAL) &&
1896 nsp->sl_refcnt == 1 && nsp->sl_chunks == 0 &&
1897 nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
1898 nsp->sl_head == NULL);
1899 }
1900 } else {
1901 VERIFY(class == MC_MBUF);
1902 --m_infree(MC_MBUF);
1903 /*
1904 * If auditing is turned on, this check is
1905 * deferred until later in mbuf_slab_audit().
1906 */
1907 if (mclaudit == NULL)
1908 _MCHECK((struct mbuf *)buf);
1909 /*
1910 * Since we have incremented the reference count above,
1911 * an mbuf slab (formerly a 4KB cluster slab that was cut
1912 * up into mbufs) must have a reference count between 1
1913 * and NMBPG at this point.
1914 */
1915 VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NMBPG &&
1916 sp->sl_chunks == NMBPG &&
1917 sp->sl_len == PAGE_SIZE);
1918 VERIFY(sp->sl_refcnt < NMBPG || sp->sl_head == NULL);
1919 }
1920
1921 /* If empty, remove this slab from the class's freelist */
1922 if (sp->sl_head == NULL) {
1923 VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPG);
1924 VERIFY(class != MC_CL || sp->sl_refcnt == NCLPG);
1925 VERIFY(class != MC_BIGCL || sp->sl_refcnt == NBCLPG);
1926 slab_remove(sp, class);
1927 }
1928
1929 return (buf);
1930}
1931
1932/*
1933 * Place a slab of object(s) back into a class's slab list.
1934 */
1935static void
1936slab_free(mbuf_class_t class, mcache_obj_t *buf)
1937{
1938 mcl_slab_t *sp;
1939 boolean_t reinit_supercl = false;
1940 mbuf_class_t super_class;
1941
1942 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1943
1944 VERIFY(class != MC_16KCL || njcl > 0);
1945 VERIFY(buf->obj_next == NULL);
1946
1947 /*
1948 * Synchronizing with m_clalloc, as it reads m_total, while we here
1949 * are modifying m_total.
1950 */
1951 while (mb_clalloc_busy) {
1952 mb_clalloc_waiters++;
1953 (void) msleep(mb_clalloc_waitchan, mbuf_mlock,
1954 (PZERO-1), "m_clalloc", NULL);
1955 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1956 }
1957
1958 /* We are busy now; tell everyone else to go away */
1959 mb_clalloc_busy = TRUE;
1960
1961 sp = slab_get(buf);
1962 VERIFY(sp->sl_class == class && slab_inrange(sp, buf) &&
1963 (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
1964
1965 /* Decrement slab reference */
1966 sp->sl_refcnt--;
1967
1968 if (class == MC_CL) {
1969 VERIFY(IS_P2ALIGNED(buf, MCLBYTES));
1970 /*
1971 * A slab that has been splitted for 2KB clusters can have
1972 * at most 1 outstanding reference at this point.
1973 */
1974 VERIFY(sp->sl_refcnt >= 0 && sp->sl_refcnt <= (NCLPG - 1) &&
1975 sp->sl_chunks == NCLPG && sp->sl_len == PAGE_SIZE);
1976 VERIFY(sp->sl_refcnt < (NCLPG - 1) ||
1977 (slab_is_detached(sp) && sp->sl_head == NULL));
1978 } else if (class == MC_BIGCL) {
1979 VERIFY(IS_P2ALIGNED(buf, MBIGCLBYTES));
1980
1981 /* A 4KB cluster slab can have NBCLPG references at most */
1982 VERIFY(sp->sl_refcnt >= 0 && sp->sl_chunks == NBCLPG);
1983 VERIFY(sp->sl_refcnt < (NBCLPG - 1) ||
1984 (slab_is_detached(sp) && sp->sl_head == NULL));
1985 } else if (class == MC_16KCL) {
1986 mcl_slab_t *nsp;
1987 int k;
1988 /*
1989 * A 16KB cluster takes NSLABSP16KB slabs, all must
1990 * now have 0 reference.
1991 */
1992 VERIFY(IS_P2ALIGNED(buf, PAGE_SIZE));
1993 VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 &&
1994 sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
1995 VERIFY(slab_is_detached(sp));
1996 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
1997 nsp = nsp->sl_next;
1998 /* Next slab must already be present */
1999 VERIFY(nsp != NULL);
2000 nsp->sl_refcnt--;
2001 VERIFY(slab_is_detached(nsp));
2002 VERIFY(nsp->sl_class == MC_16KCL &&
2003 (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) &&
2004 nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 &&
2005 nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
2006 nsp->sl_head == NULL);
2007 }
2008 } else {
2009 /*
2010 * A slab that has been splitted for mbufs has at most
2011 * NMBPG reference counts. Since we have decremented
2012 * one reference above, it must now be between 0 and
2013 * NMBPG-1.
2014 */
2015 VERIFY(class == MC_MBUF);
2016 VERIFY(sp->sl_refcnt >= 0 &&
2017 sp->sl_refcnt <= (NMBPG - 1) &&
2018 sp->sl_chunks == NMBPG &&
2019 sp->sl_len == PAGE_SIZE);
2020 VERIFY(sp->sl_refcnt < (NMBPG - 1) ||
2021 (slab_is_detached(sp) && sp->sl_head == NULL));
2022 }
2023
2024 /*
2025 * When auditing is enabled, ensure that the buffer still
2026 * contains the free pattern. Otherwise it got corrupted
2027 * while at the CPU cache layer.
2028 */
2029 if (mclaudit != NULL) {
2030 mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
2031 if (mclverify) {
2032 mcache_audit_free_verify(mca, buf, 0,
2033 m_maxsize(class));
2034 }
2035 mca->mca_uflags &= ~MB_SCVALID;
2036 }
2037
2038 if (class == MC_CL) {
2039 mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
2040 buf->obj_next = sp->sl_head;
2041 } else if (class == MC_BIGCL) {
2042 mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) +
2043 m_infree(MC_MBUF_BIGCL);
2044 buf->obj_next = sp->sl_head;
2045 } else if (class == MC_16KCL) {
2046 ++m_infree(MC_16KCL);
2047 } else {
2048 ++m_infree(MC_MBUF);
2049 buf->obj_next = sp->sl_head;
2050 }
2051 sp->sl_head = buf;
2052
2053 /*
2054 * If a slab has been split to either one which holds 2KB clusters,
2055 * or one which holds mbufs, turn it back to one which holds a
2056 * 4 or 16 KB cluster depending on the page size.
2057 */
2058 if (m_maxsize(MC_BIGCL) == PAGE_SIZE) {
2059 super_class = MC_BIGCL;
2060 } else {
2061 VERIFY(PAGE_SIZE == m_maxsize(MC_16KCL));
2062 super_class = MC_16KCL;
2063 }
2064 if (class == MC_MBUF && sp->sl_refcnt == 0 &&
2065 m_total(class) >= (m_minlimit(class) + NMBPG) &&
2066 m_total(super_class) < m_maxlimit(super_class)) {
2067 int i = NMBPG;
2068
2069 m_total(MC_MBUF) -= NMBPG;
2070 mbstat.m_mbufs = m_total(MC_MBUF);
2071 m_infree(MC_MBUF) -= NMBPG;
2072 mtype_stat_add(MT_FREE, -((unsigned)NMBPG));
2073
2074 while (i--) {
2075 struct mbuf *m = sp->sl_head;
2076 VERIFY(m != NULL);
2077 sp->sl_head = m->m_next;
2078 m->m_next = NULL;
2079 }
2080 reinit_supercl = true;
2081 } else if (class == MC_CL && sp->sl_refcnt == 0 &&
2082 m_total(class) >= (m_minlimit(class) + NCLPG) &&
2083 m_total(super_class) < m_maxlimit(super_class)) {
2084 int i = NCLPG;
2085
2086 m_total(MC_CL) -= NCLPG;
2087 mbstat.m_clusters = m_total(MC_CL);
2088 m_infree(MC_CL) -= NCLPG;
2089
2090 while (i--) {
2091 union mcluster *c = sp->sl_head;
2092 VERIFY(c != NULL);
2093 sp->sl_head = c->mcl_next;
2094 c->mcl_next = NULL;
2095 }
2096 reinit_supercl = true;
2097 } else if (class == MC_BIGCL && super_class != MC_BIGCL &&
2098 sp->sl_refcnt == 0 &&
2099 m_total(class) >= (m_minlimit(class) + NBCLPG) &&
2100 m_total(super_class) < m_maxlimit(super_class)) {
2101 int i = NBCLPG;
2102
2103 VERIFY(super_class == MC_16KCL);
2104 m_total(MC_BIGCL) -= NBCLPG;
2105 mbstat.m_bigclusters = m_total(MC_BIGCL);
2106 m_infree(MC_BIGCL) -= NBCLPG;
2107
2108 while (i--) {
2109 union mbigcluster *bc = sp->sl_head;
2110 VERIFY(bc != NULL);
2111 sp->sl_head = bc->mbc_next;
2112 bc->mbc_next = NULL;
2113 }
2114 reinit_supercl = true;
2115 }
2116
2117 if (reinit_supercl) {
2118 VERIFY(sp->sl_head == NULL);
2119 VERIFY(m_total(class) >= m_minlimit(class));
2120 slab_remove(sp, class);
2121
2122 /* Reinitialize it as a cluster for the super class */
2123 m_total(super_class)++;
2124 m_infree(super_class)++;
2125 VERIFY(sp->sl_flags == (SLF_MAPPED | SLF_DETACHED) &&
2126 sp->sl_len == PAGE_SIZE && sp->sl_refcnt == 0);
2127
2128 slab_init(sp, super_class, SLF_MAPPED, sp->sl_base,
2129 sp->sl_base, PAGE_SIZE, 0, 1);
2130 if (mclverify)
2131 mcache_set_pattern(MCACHE_FREE_PATTERN,
2132 (caddr_t)sp->sl_base, sp->sl_len);
2133 ((mcache_obj_t *)(sp->sl_base))->obj_next = NULL;
2134
2135 if (super_class == MC_BIGCL) {
2136 mbstat.m_bigclusters = m_total(MC_BIGCL);
2137 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
2138 m_infree(MC_MBUF_BIGCL);
2139 }
2140
2141 VERIFY(slab_is_detached(sp));
2142 VERIFY(m_total(super_class) <= m_maxlimit(super_class));
2143
2144 /* And finally switch class */
2145 class = super_class;
2146 }
2147
2148 /* Reinsert the slab to the class's slab list */
2149 if (slab_is_detached(sp))
2150 slab_insert(sp, class);
2151
2152 /* We're done; let others enter */
2153 mb_clalloc_busy = FALSE;
2154 if (mb_clalloc_waiters > 0) {
2155 mb_clalloc_waiters = 0;
2156 wakeup(mb_clalloc_waitchan);
2157 }
2158}
2159
2160/*
2161 * Common allocator for rudimentary objects called by the CPU cache layer
2162 * during an allocation request whenever there is no available element in the
2163 * bucket layer. It returns one or more elements from the appropriate global
2164 * freelist. If the freelist is empty, it will attempt to populate it and
2165 * retry the allocation.
2166 */
2167static unsigned int
2168mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
2169{
2170 mbuf_class_t class = (mbuf_class_t)arg;
2171 unsigned int need = num;
2172 mcache_obj_t **list = *plist;
2173
2174 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2175 ASSERT(need > 0);
2176
2177 lck_mtx_lock(mbuf_mlock);
2178
2179 for (;;) {
2180 if ((*list = slab_alloc(class, wait)) != NULL) {
2181 (*list)->obj_next = NULL;
2182 list = *plist = &(*list)->obj_next;
2183
2184 if (--need == 0) {
2185 /*
2186 * If the number of elements in freelist has
2187 * dropped below low watermark, asynchronously
2188 * populate the freelist now rather than doing
2189 * it later when we run out of elements.
2190 */
2191 if (!mbuf_cached_above(class, wait) &&
2192 m_infree(class) < (m_total(class) >> 5)) {
2193 (void) freelist_populate(class, 1,
2194 M_DONTWAIT);
2195 }
2196 break;
2197 }
2198 } else {
2199 VERIFY(m_infree(class) == 0 || class == MC_CL);
2200
2201 (void) freelist_populate(class, 1,
2202 (wait & MCR_NOSLEEP) ? M_DONTWAIT : M_WAIT);
2203
2204 if (m_infree(class) > 0)
2205 continue;
2206
2207 /* Check if there's anything at the cache layer */
2208 if (mbuf_cached_above(class, wait))
2209 break;
2210
2211 /* watchdog checkpoint */
2212 mbuf_watchdog();
2213
2214 /* We have nothing and cannot block; give up */
2215 if (wait & MCR_NOSLEEP) {
2216 if (!(wait & MCR_TRYHARD)) {
2217 m_fail_cnt(class)++;
2218 mbstat.m_drops++;
2219 break;
2220 }
2221 }
2222
2223 /*
2224 * If the freelist is still empty and the caller is
2225 * willing to be blocked, sleep on the wait channel
2226 * until an element is available. Otherwise, if
2227 * MCR_TRYHARD is set, do our best to satisfy the
2228 * request without having to go to sleep.
2229 */
2230 if (mbuf_worker_ready &&
2231 mbuf_sleep(class, need, wait))
2232 break;
2233
2234 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2235 }
2236 }
2237
2238 m_alloc_cnt(class) += num - need;
2239 lck_mtx_unlock(mbuf_mlock);
2240
2241 return (num - need);
2242}
2243
2244/*
2245 * Common de-allocator for rudimentary objects called by the CPU cache
2246 * layer when one or more elements need to be returned to the appropriate
2247 * global freelist.
2248 */
2249static void
2250mbuf_slab_free(void *arg, mcache_obj_t *list, __unused int purged)
2251{
2252 mbuf_class_t class = (mbuf_class_t)arg;
2253 mcache_obj_t *nlist;
2254 unsigned int num = 0;
2255 int w;
2256
2257 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2258
2259 lck_mtx_lock(mbuf_mlock);
2260
2261 for (;;) {
2262 nlist = list->obj_next;
2263 list->obj_next = NULL;
2264 slab_free(class, list);
2265 ++num;
2266 if ((list = nlist) == NULL)
2267 break;
2268 }
2269 m_free_cnt(class) += num;
2270
2271 if ((w = mb_waiters) > 0)
2272 mb_waiters = 0;
2273 if (w) {
2274 mbwdog_logger("waking up all threads");
2275 }
2276 lck_mtx_unlock(mbuf_mlock);
2277
2278 if (w != 0)
2279 wakeup(mb_waitchan);
2280}
2281
2282/*
2283 * Common auditor for rudimentary objects called by the CPU cache layer
2284 * during an allocation or free request. For the former, this is called
2285 * after the objects are obtained from either the bucket or slab layer
2286 * and before they are returned to the caller. For the latter, this is
2287 * called immediately during free and before placing the objects into
2288 * the bucket or slab layer.
2289 */
2290static void
2291mbuf_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
2292{
2293 mbuf_class_t class = (mbuf_class_t)arg;
2294 mcache_audit_t *mca;
2295
2296 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2297
2298 while (list != NULL) {
2299 lck_mtx_lock(mbuf_mlock);
2300 mca = mcl_audit_buf2mca(class, list);
2301
2302 /* Do the sanity checks */
2303 if (class == MC_MBUF) {
2304 mcl_audit_mbuf(mca, list, FALSE, alloc);
2305 ASSERT(mca->mca_uflags & MB_SCVALID);
2306 } else {
2307 mcl_audit_cluster(mca, list, m_maxsize(class),
2308 alloc, TRUE);
2309 ASSERT(!(mca->mca_uflags & MB_SCVALID));
2310 }
2311 /* Record this transaction */
2312 if (mcltrace)
2313 mcache_buffer_log(mca, list, m_cache(class), &mb_start);
2314
2315 if (alloc)
2316 mca->mca_uflags |= MB_INUSE;
2317 else
2318 mca->mca_uflags &= ~MB_INUSE;
2319 /* Unpair the object (unconditionally) */
2320 mca->mca_uptr = NULL;
2321 lck_mtx_unlock(mbuf_mlock);
2322
2323 list = list->obj_next;
2324 }
2325}
2326
2327/*
2328 * Common notify routine for all caches. It is called by mcache when
2329 * one or more objects get freed. We use this indication to trigger
2330 * the wakeup of any sleeping threads so that they can retry their
2331 * allocation requests.
2332 */
2333static void
2334mbuf_slab_notify(void *arg, u_int32_t reason)
2335{
2336 mbuf_class_t class = (mbuf_class_t)arg;
2337 int w;
2338
2339 ASSERT(MBUF_CLASS_VALID(class));
2340
2341 if (reason != MCN_RETRYALLOC)
2342 return;
2343
2344 lck_mtx_lock(mbuf_mlock);
2345 if ((w = mb_waiters) > 0) {
2346 m_notified(class)++;
2347 mb_waiters = 0;
2348 }
2349 if (w) {
2350 mbwdog_logger("waking up all threads");
2351 }
2352 lck_mtx_unlock(mbuf_mlock);
2353
2354 if (w != 0)
2355 wakeup(mb_waitchan);
2356}
2357
2358/*
2359 * Obtain object(s) from the composite class's freelist.
2360 */
2361static unsigned int
2362cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num)
2363{
2364 unsigned int need = num;
2365 mcl_slab_t *sp, *clsp, *nsp;
2366 struct mbuf *m;
2367 mcache_obj_t **list = *plist;
2368 void *cl;
2369
2370 VERIFY(need > 0);
2371 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2372 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2373
2374 /* Get what we can from the freelist */
2375 while ((*list = m_cobjlist(class)) != NULL) {
2376 MRANGE(*list);
2377
2378 m = (struct mbuf *)*list;
2379 sp = slab_get(m);
2380 cl = m->m_ext.ext_buf;
2381 clsp = slab_get(cl);
2382 VERIFY(m->m_flags == M_EXT && cl != NULL);
2383 VERIFY(m_get_rfa(m) != NULL && MBUF_IS_COMPOSITE(m));
2384
2385 if (class == MC_MBUF_CL) {
2386 VERIFY(clsp->sl_refcnt >= 1 &&
2387 clsp->sl_refcnt <= NCLPG);
2388 } else {
2389 VERIFY(clsp->sl_refcnt >= 1 &&
2390 clsp->sl_refcnt <= NBCLPG);
2391 }
2392
2393 if (class == MC_MBUF_16KCL) {
2394 int k;
2395 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
2396 nsp = nsp->sl_next;
2397 /* Next slab must already be present */
2398 VERIFY(nsp != NULL);
2399 VERIFY(nsp->sl_refcnt == 1);
2400 }
2401 }
2402
2403 if ((m_cobjlist(class) = (*list)->obj_next) != NULL &&
2404 !MBUF_IN_MAP(m_cobjlist(class))) {
2405 slab_nextptr_panic(sp, m_cobjlist(class));
2406 /* NOTREACHED */
2407 }
2408 (*list)->obj_next = NULL;
2409 list = *plist = &(*list)->obj_next;
2410
2411 if (--need == 0)
2412 break;
2413 }
2414 m_infree(class) -= (num - need);
2415
2416 return (num - need);
2417}
2418
2419/*
2420 * Place object(s) back into a composite class's freelist.
2421 */
2422static unsigned int
2423cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged)
2424{
2425 mcache_obj_t *o, *tail;
2426 unsigned int num = 0;
2427 struct mbuf *m, *ms;
2428 mcache_audit_t *mca = NULL;
2429 mcache_obj_t *ref_list = NULL;
2430 mcl_slab_t *clsp, *nsp;
2431 void *cl;
2432 mbuf_class_t cl_class;
2433
2434 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2435 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2436 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2437
2438 if (class == MC_MBUF_CL) {
2439 cl_class = MC_CL;
2440 } else if (class == MC_MBUF_BIGCL) {
2441 cl_class = MC_BIGCL;
2442 } else {
2443 VERIFY(class == MC_MBUF_16KCL);
2444 cl_class = MC_16KCL;
2445 }
2446
2447 o = tail = list;
2448
2449 while ((m = ms = (struct mbuf *)o) != NULL) {
2450 mcache_obj_t *rfa, *nexto = o->obj_next;
2451
2452 /* Do the mbuf sanity checks */
2453 if (mclaudit != NULL) {
2454 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
2455 if (mclverify) {
2456 mcache_audit_free_verify(mca, m, 0,
2457 m_maxsize(MC_MBUF));
2458 }
2459 ms = MCA_SAVED_MBUF_PTR(mca);
2460 }
2461
2462 /* Do the cluster sanity checks */
2463 cl = ms->m_ext.ext_buf;
2464 clsp = slab_get(cl);
2465 if (mclverify) {
2466 size_t size = m_maxsize(cl_class);
2467 mcache_audit_free_verify(mcl_audit_buf2mca(cl_class,
2468 (mcache_obj_t *)cl), cl, 0, size);
2469 }
2470 VERIFY(ms->m_type == MT_FREE);
2471 VERIFY(ms->m_flags == M_EXT);
2472 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
2473 if (cl_class == MC_CL) {
2474 VERIFY(clsp->sl_refcnt >= 1 &&
2475 clsp->sl_refcnt <= NCLPG);
2476 } else {
2477 VERIFY(clsp->sl_refcnt >= 1 &&
2478 clsp->sl_refcnt <= NBCLPG);
2479 }
2480 if (cl_class == MC_16KCL) {
2481 int k;
2482 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
2483 nsp = nsp->sl_next;
2484 /* Next slab must already be present */
2485 VERIFY(nsp != NULL);
2486 VERIFY(nsp->sl_refcnt == 1);
2487 }
2488 }
2489
2490 /*
2491 * If we're asked to purge, restore the actual mbuf using
2492 * contents of the shadow structure (if auditing is enabled)
2493 * and clear EXTF_COMPOSITE flag from the mbuf, as we are
2494 * about to free it and the attached cluster into their caches.
2495 */
2496 if (purged) {
2497 /* Restore constructed mbuf fields */
2498 if (mclaudit != NULL)
2499 mcl_audit_restore_mbuf(m, mca, TRUE);
2500
2501 MEXT_MINREF(m) = 0;
2502 MEXT_REF(m) = 0;
2503 MEXT_PREF(m) = 0;
2504 MEXT_FLAGS(m) = 0;
2505 MEXT_PRIV(m) = 0;
2506 MEXT_PMBUF(m) = NULL;
2507 MEXT_TOKEN(m) = 0;
2508
2509 rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
2510 m_set_ext(m, NULL, NULL, NULL);
2511 rfa->obj_next = ref_list;
2512 ref_list = rfa;
2513
2514 m->m_type = MT_FREE;
2515 m->m_flags = m->m_len = 0;
2516 m->m_next = m->m_nextpkt = NULL;
2517
2518 /* Save mbuf fields and make auditing happy */
2519 if (mclaudit != NULL)
2520 mcl_audit_mbuf(mca, o, FALSE, FALSE);
2521
2522 VERIFY(m_total(class) > 0);
2523 m_total(class)--;
2524
2525 /* Free the mbuf */
2526 o->obj_next = NULL;
2527 slab_free(MC_MBUF, o);
2528
2529 /* And free the cluster */
2530 ((mcache_obj_t *)cl)->obj_next = NULL;
2531 if (class == MC_MBUF_CL)
2532 slab_free(MC_CL, cl);
2533 else if (class == MC_MBUF_BIGCL)
2534 slab_free(MC_BIGCL, cl);
2535 else
2536 slab_free(MC_16KCL, cl);
2537 }
2538
2539 ++num;
2540 tail = o;
2541 o = nexto;
2542 }
2543
2544 if (!purged) {
2545 tail->obj_next = m_cobjlist(class);
2546 m_cobjlist(class) = list;
2547 m_infree(class) += num;
2548 } else if (ref_list != NULL) {
2549 mcache_free_ext(ref_cache, ref_list);
2550 }
2551
2552 return (num);
2553}
2554
2555/*
2556 * Common allocator for composite objects called by the CPU cache layer
2557 * during an allocation request whenever there is no available element in
2558 * the bucket layer. It returns one or more composite elements from the
2559 * appropriate global freelist. If the freelist is empty, it will attempt
2560 * to obtain the rudimentary objects from their caches and construct them
2561 * into composite mbuf + cluster objects.
2562 */
2563static unsigned int
2564mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed,
2565 int wait)
2566{
2567 mbuf_class_t class = (mbuf_class_t)arg;
2568 mbuf_class_t cl_class = 0;
2569 unsigned int num = 0, cnum = 0, want = needed;
2570 mcache_obj_t *ref_list = NULL;
2571 mcache_obj_t *mp_list = NULL;
2572 mcache_obj_t *clp_list = NULL;
2573 mcache_obj_t **list;
2574 struct ext_ref *rfa;
2575 struct mbuf *m;
2576 void *cl;
2577
2578 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2579 ASSERT(needed > 0);
2580
2581 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
2582
2583 /* There should not be any slab for this class */
2584 VERIFY(m_slab_cnt(class) == 0 &&
2585 m_slablist(class).tqh_first == NULL &&
2586 m_slablist(class).tqh_last == NULL);
2587
2588 lck_mtx_lock(mbuf_mlock);
2589
2590 /* Try using the freelist first */
2591 num = cslab_alloc(class, plist, needed);
2592 list = *plist;
2593 if (num == needed) {
2594 m_alloc_cnt(class) += num;
2595 lck_mtx_unlock(mbuf_mlock);
2596 return (needed);
2597 }
2598
2599 lck_mtx_unlock(mbuf_mlock);
2600
2601 /*
2602 * We could not satisfy the request using the freelist alone;
2603 * allocate from the appropriate rudimentary caches and use
2604 * whatever we can get to construct the composite objects.
2605 */
2606 needed -= num;
2607
2608 /*
2609 * Mark these allocation requests as coming from a composite cache.
2610 * Also, if the caller is willing to be blocked, mark the request
2611 * with MCR_FAILOK such that we don't end up sleeping at the mbuf
2612 * slab layer waiting for the individual object when one or more
2613 * of the already-constructed composite objects are available.
2614 */
2615 wait |= MCR_COMP;
2616 if (!(wait & MCR_NOSLEEP))
2617 wait |= MCR_FAILOK;
2618
2619 /* allocate mbufs */
2620 needed = mcache_alloc_ext(m_cache(MC_MBUF), &mp_list, needed, wait);
2621 if (needed == 0) {
2622 ASSERT(mp_list == NULL);
2623 goto fail;
2624 }
2625
2626 /* allocate clusters */
2627 if (class == MC_MBUF_CL) {
2628 cl_class = MC_CL;
2629 } else if (class == MC_MBUF_BIGCL) {
2630 cl_class = MC_BIGCL;
2631 } else {
2632 VERIFY(class == MC_MBUF_16KCL);
2633 cl_class = MC_16KCL;
2634 }
2635 needed = mcache_alloc_ext(m_cache(cl_class), &clp_list, needed, wait);
2636 if (needed == 0) {
2637 ASSERT(clp_list == NULL);
2638 goto fail;
2639 }
2640
2641 needed = mcache_alloc_ext(ref_cache, &ref_list, needed, wait);
2642 if (needed == 0) {
2643 ASSERT(ref_list == NULL);
2644 goto fail;
2645 }
2646
2647 /*
2648 * By this time "needed" is MIN(mbuf, cluster, ref). Any left
2649 * overs will get freed accordingly before we return to caller.
2650 */
2651 for (cnum = 0; cnum < needed; cnum++) {
2652 struct mbuf *ms;
2653
2654 m = ms = (struct mbuf *)mp_list;
2655 mp_list = mp_list->obj_next;
2656
2657 cl = clp_list;
2658 clp_list = clp_list->obj_next;
2659 ((mcache_obj_t *)cl)->obj_next = NULL;
2660
2661 rfa = (struct ext_ref *)ref_list;
2662 ref_list = ref_list->obj_next;
2663 ((mcache_obj_t *)(void *)rfa)->obj_next = NULL;
2664
2665 /*
2666 * If auditing is enabled, construct the shadow mbuf
2667 * in the audit structure instead of in the actual one.
2668 * mbuf_cslab_audit() will take care of restoring the
2669 * contents after the integrity check.
2670 */
2671 if (mclaudit != NULL) {
2672 mcache_audit_t *mca, *cl_mca;
2673
2674 lck_mtx_lock(mbuf_mlock);
2675 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
2676 ms = MCA_SAVED_MBUF_PTR(mca);
2677 cl_mca = mcl_audit_buf2mca(cl_class,
2678 (mcache_obj_t *)cl);
2679
2680 /*
2681 * Pair them up. Note that this is done at the time
2682 * the mbuf+cluster objects are constructed. This
2683 * information should be treated as "best effort"
2684 * debugging hint since more than one mbufs can refer
2685 * to a cluster. In that case, the cluster might not
2686 * be freed along with the mbuf it was paired with.
2687 */
2688 mca->mca_uptr = cl_mca;
2689 cl_mca->mca_uptr = mca;
2690
2691 ASSERT(mca->mca_uflags & MB_SCVALID);
2692 ASSERT(!(cl_mca->mca_uflags & MB_SCVALID));
2693 lck_mtx_unlock(mbuf_mlock);
2694
2695 /* Technically, they are in the freelist */
2696 if (mclverify) {
2697 size_t size;
2698
2699 mcache_set_pattern(MCACHE_FREE_PATTERN, m,
2700 m_maxsize(MC_MBUF));
2701
2702 if (class == MC_MBUF_CL)
2703 size = m_maxsize(MC_CL);
2704 else if (class == MC_MBUF_BIGCL)
2705 size = m_maxsize(MC_BIGCL);
2706 else
2707 size = m_maxsize(MC_16KCL);
2708
2709 mcache_set_pattern(MCACHE_FREE_PATTERN, cl,
2710 size);
2711 }
2712 }
2713
2714 MBUF_INIT(ms, 0, MT_FREE);
2715 if (class == MC_MBUF_16KCL) {
2716 MBUF_16KCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
2717 } else if (class == MC_MBUF_BIGCL) {
2718 MBUF_BIGCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
2719 } else {
2720 MBUF_CL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
2721 }
2722 VERIFY(ms->m_flags == M_EXT);
2723 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
2724
2725 *list = (mcache_obj_t *)m;
2726 (*list)->obj_next = NULL;
2727 list = *plist = &(*list)->obj_next;
2728 }
2729
2730fail:
2731 /*
2732 * Free up what's left of the above.
2733 */
2734 if (mp_list != NULL)
2735 mcache_free_ext(m_cache(MC_MBUF), mp_list);
2736 if (clp_list != NULL)
2737 mcache_free_ext(m_cache(cl_class), clp_list);
2738 if (ref_list != NULL)
2739 mcache_free_ext(ref_cache, ref_list);
2740
2741 lck_mtx_lock(mbuf_mlock);
2742 if (num > 0 || cnum > 0) {
2743 m_total(class) += cnum;
2744 VERIFY(m_total(class) <= m_maxlimit(class));
2745 m_alloc_cnt(class) += num + cnum;
2746 }
2747 if ((num + cnum) < want)
2748 m_fail_cnt(class) += (want - (num + cnum));
2749 lck_mtx_unlock(mbuf_mlock);
2750
2751 return (num + cnum);
2752}
2753
2754/*
2755 * Common de-allocator for composite objects called by the CPU cache
2756 * layer when one or more elements need to be returned to the appropriate
2757 * global freelist.
2758 */
2759static void
2760mbuf_cslab_free(void *arg, mcache_obj_t *list, int purged)
2761{
2762 mbuf_class_t class = (mbuf_class_t)arg;
2763 unsigned int num;
2764 int w;
2765
2766 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2767
2768 lck_mtx_lock(mbuf_mlock);
2769
2770 num = cslab_free(class, list, purged);
2771 m_free_cnt(class) += num;
2772
2773 if ((w = mb_waiters) > 0)
2774 mb_waiters = 0;
2775 if (w) {
2776 mbwdog_logger("waking up all threads");
2777 }
2778
2779 lck_mtx_unlock(mbuf_mlock);
2780
2781 if (w != 0)
2782 wakeup(mb_waitchan);
2783}
2784
2785/*
2786 * Common auditor for composite objects called by the CPU cache layer
2787 * during an allocation or free request. For the former, this is called
2788 * after the objects are obtained from either the bucket or slab layer
2789 * and before they are returned to the caller. For the latter, this is
2790 * called immediately during free and before placing the objects into
2791 * the bucket or slab layer.
2792 */
2793static void
2794mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
2795{
2796 mbuf_class_t class = (mbuf_class_t)arg, cl_class;
2797 mcache_audit_t *mca;
2798 struct mbuf *m, *ms;
2799 mcl_slab_t *clsp, *nsp;
2800 size_t cl_size;
2801 void *cl;
2802
2803 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
2804 if (class == MC_MBUF_CL)
2805 cl_class = MC_CL;
2806 else if (class == MC_MBUF_BIGCL)
2807 cl_class = MC_BIGCL;
2808 else
2809 cl_class = MC_16KCL;
2810 cl_size = m_maxsize(cl_class);
2811
2812 while ((m = ms = (struct mbuf *)list) != NULL) {
2813 lck_mtx_lock(mbuf_mlock);
2814 /* Do the mbuf sanity checks and record its transaction */
2815 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
2816 mcl_audit_mbuf(mca, m, TRUE, alloc);
2817 if (mcltrace)
2818 mcache_buffer_log(mca, m, m_cache(class), &mb_start);
2819
2820 if (alloc)
2821 mca->mca_uflags |= MB_COMP_INUSE;
2822 else
2823 mca->mca_uflags &= ~MB_COMP_INUSE;
2824
2825 /*
2826 * Use the shadow mbuf in the audit structure if we are
2827 * freeing, since the contents of the actual mbuf has been
2828 * pattern-filled by the above call to mcl_audit_mbuf().
2829 */
2830 if (!alloc && mclverify)
2831 ms = MCA_SAVED_MBUF_PTR(mca);
2832
2833 /* Do the cluster sanity checks and record its transaction */
2834 cl = ms->m_ext.ext_buf;
2835 clsp = slab_get(cl);
2836 VERIFY(ms->m_flags == M_EXT && cl != NULL);
2837 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
2838 if (class == MC_MBUF_CL)
2839 VERIFY(clsp->sl_refcnt >= 1 &&
2840 clsp->sl_refcnt <= NCLPG);
2841 else
2842 VERIFY(clsp->sl_refcnt >= 1 &&
2843 clsp->sl_refcnt <= NBCLPG);
2844
2845 if (class == MC_MBUF_16KCL) {
2846 int k;
2847 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
2848 nsp = nsp->sl_next;
2849 /* Next slab must already be present */
2850 VERIFY(nsp != NULL);
2851 VERIFY(nsp->sl_refcnt == 1);
2852 }
2853 }
2854
2855
2856 mca = mcl_audit_buf2mca(cl_class, cl);
2857 mcl_audit_cluster(mca, cl, cl_size, alloc, FALSE);
2858 if (mcltrace)
2859 mcache_buffer_log(mca, cl, m_cache(class), &mb_start);
2860
2861 if (alloc)
2862 mca->mca_uflags |= MB_COMP_INUSE;
2863 else
2864 mca->mca_uflags &= ~MB_COMP_INUSE;
2865 lck_mtx_unlock(mbuf_mlock);
2866
2867 list = list->obj_next;
2868 }
2869}
2870
2871static void
2872m_vm_error_stats(uint32_t *cnt, uint64_t *ts, uint64_t *size,
2873 uint64_t alloc_size, kern_return_t error)
2874{
2875
2876 *cnt = *cnt + 1;
2877 *ts = net_uptime();
2878 if (size) {
2879 *size = alloc_size;
2880 }
2881 _CASSERT(sizeof(mb_kmem_stats) / sizeof(mb_kmem_stats[0]) ==
2882 sizeof(mb_kmem_stats_labels) / sizeof(mb_kmem_stats_labels[0]));
2883 switch (error) {
2884 case KERN_SUCCESS:
2885 break;
2886 case KERN_INVALID_ARGUMENT:
2887 mb_kmem_stats[0]++;
2888 break;
2889 case KERN_INVALID_ADDRESS:
2890 mb_kmem_stats[1]++;
2891 break;
2892 case KERN_RESOURCE_SHORTAGE:
2893 mb_kmem_stats[2]++;
2894 break;
2895 case KERN_NO_SPACE:
2896 mb_kmem_stats[3]++;
2897 break;
2898 case KERN_FAILURE:
2899 mb_kmem_stats[4]++;
2900 break;
2901 default:
2902 mb_kmem_stats[5]++;
2903 break;
2904 }
2905}
2906
2907/*
2908 * Allocate some number of mbuf clusters and place on cluster freelist.
2909 */
2910static int
2911m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
2912{
2913 int i, count = 0;
2914 vm_size_t size = 0;
2915 int numpages = 0, large_buffer;
2916 vm_offset_t page = 0;
2917 mcache_audit_t *mca_list = NULL;
2918 mcache_obj_t *con_list = NULL;
2919 mcl_slab_t *sp;
2920 mbuf_class_t class;
2921 kern_return_t error;
2922
2923 /* Set if a buffer allocation needs allocation of multiple pages */
2924 large_buffer = ((bufsize == m_maxsize(MC_16KCL)) &&
2925 PAGE_SIZE < M16KCLBYTES);
2926 VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
2927 bufsize == m_maxsize(MC_16KCL));
2928
2929 VERIFY((bufsize == PAGE_SIZE) ||
2930 (bufsize > PAGE_SIZE && bufsize == m_maxsize(MC_16KCL)));
2931
2932 if (bufsize == m_size(MC_BIGCL))
2933 class = MC_BIGCL;
2934 else
2935 class = MC_16KCL;
2936
2937 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2938
2939 /*
2940 * Multiple threads may attempt to populate the cluster map one
2941 * after another. Since we drop the lock below prior to acquiring
2942 * the physical page(s), our view of the cluster map may no longer
2943 * be accurate, and we could end up over-committing the pages beyond
2944 * the maximum allowed for each class. To prevent it, this entire
2945 * operation (including the page mapping) is serialized.
2946 */
2947 while (mb_clalloc_busy) {
2948 mb_clalloc_waiters++;
2949 (void) msleep(mb_clalloc_waitchan, mbuf_mlock,
2950 (PZERO-1), "m_clalloc", NULL);
2951 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2952 }
2953
2954 /* We are busy now; tell everyone else to go away */
2955 mb_clalloc_busy = TRUE;
2956
2957 /*
2958 * Honor the caller's wish to block or not block. We have a way
2959 * to grow the pool asynchronously using the mbuf worker thread.
2960 */
2961 i = m_howmany(num, bufsize);
2962 if (i <= 0 || (wait & M_DONTWAIT))
2963 goto out;
2964
2965 lck_mtx_unlock(mbuf_mlock);
2966
2967 size = round_page(i * bufsize);
2968 page = kmem_mb_alloc(mb_map, size, large_buffer, &error);
2969
2970 /*
2971 * If we did ask for "n" 16KB physically contiguous chunks
2972 * and didn't get them, then please try again without this
2973 * restriction.
2974 */
2975 net_update_uptime();
2976 if (large_buffer && page == 0) {
2977 m_vm_error_stats(&mb_kmem_contig_failed,
2978 &mb_kmem_contig_failed_ts,
2979 &mb_kmem_contig_failed_size,
2980 size, error);
2981 page = kmem_mb_alloc(mb_map, size, 0, &error);
2982 }
2983
2984 if (page == 0) {
2985 m_vm_error_stats(&mb_kmem_failed,
2986 &mb_kmem_failed_ts,
2987 &mb_kmem_failed_size,
2988 size, error);
2989#if PAGE_SIZE == 4096
2990 if (bufsize == m_maxsize(MC_BIGCL)) {
2991#else
2992 if (bufsize >= m_maxsize(MC_BIGCL)) {
2993#endif
2994 /* Try for 1 page if failed */
2995 size = PAGE_SIZE;
2996 page = kmem_mb_alloc(mb_map, size, 0, &error);
2997 if (page == 0) {
2998 m_vm_error_stats(&mb_kmem_one_failed,
2999 &mb_kmem_one_failed_ts,
3000 NULL, size, error);
3001 }
3002 }
3003
3004 if (page == 0) {
3005 lck_mtx_lock(mbuf_mlock);
3006 goto out;
3007 }
3008 }
3009
3010 VERIFY(IS_P2ALIGNED(page, PAGE_SIZE));
3011 numpages = size / PAGE_SIZE;
3012
3013 /* If auditing is enabled, allocate the audit structures now */
3014 if (mclaudit != NULL) {
3015 int needed;
3016
3017 /*
3018 * Yes, I realize this is a waste of memory for clusters
3019 * that never get transformed into mbufs, as we may end
3020 * up with NMBPG-1 unused audit structures per cluster.
3021 * But doing so tremendously simplifies the allocation
3022 * strategy, since at this point we are not holding the
3023 * mbuf lock and the caller is okay to be blocked.
3024 */
3025 if (bufsize == PAGE_SIZE) {
3026 needed = numpages * NMBPG;
3027
3028 i = mcache_alloc_ext(mcl_audit_con_cache,
3029 &con_list, needed, MCR_SLEEP);
3030
3031 VERIFY(con_list != NULL && i == needed);
3032 } else {
3033 /*
3034 * if multiple 4K pages are being used for a
3035 * 16K cluster
3036 */
3037 needed = numpages / NSLABSP16KB;
3038 }
3039
3040 i = mcache_alloc_ext(mcache_audit_cache,
3041 (mcache_obj_t **)&mca_list, needed, MCR_SLEEP);
3042
3043 VERIFY(mca_list != NULL && i == needed);
3044 }
3045
3046 lck_mtx_lock(mbuf_mlock);
3047
3048 for (i = 0; i < numpages; i++, page += PAGE_SIZE) {
3049 ppnum_t offset =
3050 ((unsigned char *)page - mbutl) >> PAGE_SHIFT;
3051 ppnum_t new_page = pmap_find_phys(kernel_pmap, page);
3052
3053 /*
3054 * If there is a mapper the appropriate I/O page is
3055 * returned; zero out the page to discard its past
3056 * contents to prevent exposing leftover kernel memory.
3057 */
3058 VERIFY(offset < mcl_pages);
3059 if (mcl_paddr_base != 0) {
3060 bzero((void *)(uintptr_t) page, PAGE_SIZE);
3061 new_page = IOMapperInsertPage(mcl_paddr_base,
3062 offset, new_page);
3063 }
3064 mcl_paddr[offset] = new_page;
3065
3066 /* Pattern-fill this fresh page */
3067 if (mclverify) {
3068 mcache_set_pattern(MCACHE_FREE_PATTERN,
3069 (caddr_t)page, PAGE_SIZE);
3070 }
3071 if (bufsize == PAGE_SIZE) {
3072 mcache_obj_t *buf;
3073 /* One for the entire page */
3074 sp = slab_get((void *)page);
3075 if (mclaudit != NULL) {
3076 mcl_audit_init((void *)page,
3077 &mca_list, &con_list,
3078 AUDIT_CONTENTS_SIZE, NMBPG);
3079 }
3080 VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
3081 slab_init(sp, class, SLF_MAPPED, (void *)page,
3082 (void *)page, PAGE_SIZE, 0, 1);
3083 buf = (mcache_obj_t *)page;
3084 buf->obj_next = NULL;
3085
3086 /* Insert this slab */
3087 slab_insert(sp, class);
3088
3089 /* Update stats now since slab_get drops the lock */
3090 ++m_infree(class);
3091 ++m_total(class);
3092 VERIFY(m_total(class) <= m_maxlimit(class));
3093 if (class == MC_BIGCL) {
3094 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
3095 m_infree(MC_MBUF_BIGCL);
3096 mbstat.m_bigclusters = m_total(MC_BIGCL);
3097 }
3098 ++count;
3099 } else if ((bufsize > PAGE_SIZE) &&
3100 (i % NSLABSP16KB) == 0) {
3101 union m16kcluster *m16kcl = (union m16kcluster *)page;
3102 mcl_slab_t *nsp;
3103 int k;
3104
3105 /* One for the entire 16KB */
3106 sp = slab_get(m16kcl);
3107 if (mclaudit != NULL)
3108 mcl_audit_init(m16kcl, &mca_list, NULL, 0, 1);
3109
3110 VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
3111 slab_init(sp, MC_16KCL, SLF_MAPPED,
3112 m16kcl, m16kcl, bufsize, 0, 1);
3113 m16kcl->m16kcl_next = NULL;
3114
3115 /*
3116 * 2nd-Nth page's slab is part of the first one,
3117 * where N is NSLABSP16KB.
3118 */
3119 for (k = 1; k < NSLABSP16KB; k++) {
3120 nsp = slab_get(((union mbigcluster *)page) + k);
3121 VERIFY(nsp->sl_refcnt == 0 &&
3122 nsp->sl_flags == 0);
3123 slab_init(nsp, MC_16KCL,
3124 SLF_MAPPED | SLF_PARTIAL,
3125 m16kcl, NULL, 0, 0, 0);
3126 }
3127 /* Insert this slab */
3128 slab_insert(sp, MC_16KCL);
3129
3130 /* Update stats now since slab_get drops the lock */
3131 ++m_infree(MC_16KCL);
3132 ++m_total(MC_16KCL);
3133 VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
3134 ++count;
3135 }
3136 }
3137 VERIFY(mca_list == NULL && con_list == NULL);
3138
3139 if (!mb_peak_newreport && mbuf_report_usage(class))
3140 mb_peak_newreport = TRUE;
3141
3142 /* We're done; let others enter */
3143 mb_clalloc_busy = FALSE;
3144 if (mb_clalloc_waiters > 0) {
3145 mb_clalloc_waiters = 0;
3146 wakeup(mb_clalloc_waitchan);
3147 }
3148
3149 return (count);
3150out:
3151 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3152
3153 mtracelarge_register(size);
3154
3155 /* We're done; let others enter */
3156 mb_clalloc_busy = FALSE;
3157 if (mb_clalloc_waiters > 0) {
3158 mb_clalloc_waiters = 0;
3159 wakeup(mb_clalloc_waitchan);
3160 }
3161
3162 /*
3163 * When non-blocking we kick a thread if we have to grow the
3164 * pool or if the number of free clusters is less than requested.
3165 */
3166 if (i > 0 && mbuf_worker_ready && mbuf_worker_needs_wakeup) {
3167 mbwdog_logger("waking up the worker thread to to grow %s by %d",
3168 m_cname(class), i);
3169 wakeup((caddr_t)&mbuf_worker_needs_wakeup);
3170 mbuf_worker_needs_wakeup = FALSE;
3171 }
3172 if (class == MC_BIGCL) {
3173 if (i > 0) {
3174 /*
3175 * Remember total number of 4KB clusters needed
3176 * at this time.
3177 */
3178 i += m_total(MC_BIGCL);
3179 if (i > m_region_expand(MC_BIGCL)) {
3180 m_region_expand(MC_BIGCL) = i;
3181 }
3182 }
3183 if (m_infree(MC_BIGCL) >= num)
3184 return (1);
3185 } else {
3186 if (i > 0) {
3187 /*
3188 * Remember total number of 16KB clusters needed
3189 * at this time.
3190 */
3191 i += m_total(MC_16KCL);
3192 if (i > m_region_expand(MC_16KCL)) {
3193 m_region_expand(MC_16KCL) = i;
3194 }
3195 }
3196 if (m_infree(MC_16KCL) >= num)
3197 return (1);
3198 }
3199 return (0);
3200}
3201
3202/*
3203 * Populate the global freelist of the corresponding buffer class.
3204 */
3205static int
3206freelist_populate(mbuf_class_t class, unsigned int num, int wait)
3207{
3208 mcache_obj_t *o = NULL;
3209 int i, numpages = 0, count;
3210 mbuf_class_t super_class;
3211
3212 VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL ||
3213 class == MC_16KCL);
3214
3215 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3216
3217 VERIFY(PAGE_SIZE == m_maxsize(MC_BIGCL) ||
3218 PAGE_SIZE == m_maxsize(MC_16KCL));
3219
3220 if (m_maxsize(class) >= PAGE_SIZE)
3221 return(m_clalloc(num, wait, m_maxsize(class)) != 0);
3222
3223 /*
3224 * The rest of the function will allocate pages and will slice
3225 * them up into the right size
3226 */
3227
3228 numpages = (num * m_size(class) + PAGE_SIZE - 1) / PAGE_SIZE;
3229
3230 /* Currently assume that pages are 4K or 16K */
3231 if (PAGE_SIZE == m_maxsize(MC_BIGCL))
3232 super_class = MC_BIGCL;
3233 else
3234 super_class = MC_16KCL;
3235
3236 i = m_clalloc(numpages, wait, m_maxsize(super_class));
3237
3238 /* how many objects will we cut the page into? */
3239 int numobj = PAGE_SIZE / m_maxsize(class);
3240
3241 for (count = 0; count < numpages; count++) {
3242 /* respect totals, minlimit, maxlimit */
3243 if (m_total(super_class) <= m_minlimit(super_class) ||
3244 m_total(class) >= m_maxlimit(class))
3245 break;
3246
3247 if ((o = slab_alloc(super_class, wait)) == NULL)
3248 break;
3249
3250 struct mbuf *m = (struct mbuf *)o;
3251 union mcluster *c = (union mcluster *)o;
3252 union mbigcluster *mbc = (union mbigcluster *)o;
3253 mcl_slab_t *sp = slab_get(o);
3254 mcache_audit_t *mca = NULL;
3255
3256 /*
3257 * since one full page will be converted to MC_MBUF or
3258 * MC_CL, verify that the reference count will match that
3259 * assumption
3260 */
3261 VERIFY(sp->sl_refcnt == 1 && slab_is_detached(sp));
3262 VERIFY((sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
3263 /*
3264 * Make sure that the cluster is unmolested
3265 * while in freelist
3266 */
3267 if (mclverify) {
3268 mca = mcl_audit_buf2mca(super_class,
3269 (mcache_obj_t *)o);
3270 mcache_audit_free_verify(mca,
3271 (mcache_obj_t *)o, 0, m_maxsize(super_class));
3272 }
3273
3274 /* Reinitialize it as an mbuf or 2K or 4K slab */
3275 slab_init(sp, class, sp->sl_flags,
3276 sp->sl_base, NULL, PAGE_SIZE, 0, numobj);
3277
3278 VERIFY(sp->sl_head == NULL);
3279
3280 VERIFY(m_total(super_class) >= 1);
3281 m_total(super_class)--;
3282
3283 if (super_class == MC_BIGCL)
3284 mbstat.m_bigclusters = m_total(MC_BIGCL);
3285
3286 m_total(class) += numobj;
3287 VERIFY(m_total(class) <= m_maxlimit(class));
3288 m_infree(class) += numobj;
3289
3290 if (!mb_peak_newreport && mbuf_report_usage(class))
3291 mb_peak_newreport = TRUE;
3292
3293 i = numobj;
3294 if (class == MC_MBUF) {
3295 mbstat.m_mbufs = m_total(MC_MBUF);
3296 mtype_stat_add(MT_FREE, NMBPG);
3297 while (i--) {
3298 /*
3299 * If auditing is enabled, construct the
3300 * shadow mbuf in the audit structure
3301 * instead of the actual one.
3302 * mbuf_slab_audit() will take care of
3303 * restoring the contents after the
3304 * integrity check.
3305 */
3306 if (mclaudit != NULL) {
3307 struct mbuf *ms;
3308 mca = mcl_audit_buf2mca(MC_MBUF,
3309 (mcache_obj_t *)m);
3310 ms = MCA_SAVED_MBUF_PTR(mca);
3311 ms->m_type = MT_FREE;
3312 } else {
3313 m->m_type = MT_FREE;
3314 }
3315 m->m_next = sp->sl_head;
3316 sp->sl_head = (void *)m++;
3317 }
3318 } else if (class == MC_CL) { /* MC_CL */
3319 mbstat.m_clfree =
3320 m_infree(MC_CL) + m_infree(MC_MBUF_CL);
3321 mbstat.m_clusters = m_total(MC_CL);
3322 while (i--) {
3323 c->mcl_next = sp->sl_head;
3324 sp->sl_head = (void *)c++;
3325 }
3326 } else {
3327 VERIFY(class == MC_BIGCL);
3328 mbstat.m_bigclusters = m_total(MC_BIGCL);
3329 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
3330 m_infree(MC_MBUF_BIGCL);
3331 while (i--) {
3332 mbc->mbc_next = sp->sl_head;
3333 sp->sl_head = (void *)mbc++;
3334 }
3335 }
3336
3337 /* Insert into the mbuf or 2k or 4k slab list */
3338 slab_insert(sp, class);
3339
3340 if ((i = mb_waiters) > 0)
3341 mb_waiters = 0;
3342 if (i != 0) {
3343 mbwdog_logger("waking up all threads");
3344 wakeup(mb_waitchan);
3345 }
3346 }
3347 return (count != 0);
3348}
3349
3350/*
3351 * For each class, initialize the freelist to hold m_minlimit() objects.
3352 */
3353static void
3354freelist_init(mbuf_class_t class)
3355{
3356 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3357
3358 VERIFY(class == MC_CL || class == MC_BIGCL);
3359 VERIFY(m_total(class) == 0);
3360 VERIFY(m_minlimit(class) > 0);
3361
3362 while (m_total(class) < m_minlimit(class))
3363 (void) freelist_populate(class, m_minlimit(class), M_WAIT);
3364
3365 VERIFY(m_total(class) >= m_minlimit(class));
3366}
3367
3368/*
3369 * (Inaccurately) check if it might be worth a trip back to the
3370 * mcache layer due the availability of objects there. We'll
3371 * end up back here if there's nothing up there.
3372 */
3373static boolean_t
3374mbuf_cached_above(mbuf_class_t class, int wait)
3375{
3376 switch (class) {
3377 case MC_MBUF:
3378 if (wait & MCR_COMP)
3379 return (!mcache_bkt_isempty(m_cache(MC_MBUF_CL)) ||
3380 !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL)));
3381 break;
3382
3383 case MC_CL:
3384 if (wait & MCR_COMP)
3385 return (!mcache_bkt_isempty(m_cache(MC_MBUF_CL)));
3386 break;
3387
3388 case MC_BIGCL:
3389 if (wait & MCR_COMP)
3390 return (!mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL)));
3391 break;
3392
3393 case MC_16KCL:
3394 if (wait & MCR_COMP)
3395 return (!mcache_bkt_isempty(m_cache(MC_MBUF_16KCL)));
3396 break;
3397
3398 case MC_MBUF_CL:
3399 case MC_MBUF_BIGCL:
3400 case MC_MBUF_16KCL:
3401 break;
3402
3403 default:
3404 VERIFY(0);
3405 /* NOTREACHED */
3406 }
3407
3408 return (!mcache_bkt_isempty(m_cache(class)));
3409}
3410
3411/*
3412 * If possible, convert constructed objects to raw ones.
3413 */
3414static boolean_t
3415mbuf_steal(mbuf_class_t class, unsigned int num)
3416{
3417 mcache_obj_t *top = NULL;
3418 mcache_obj_t **list = &top;
3419 unsigned int tot = 0;
3420
3421 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3422
3423 switch (class) {
3424 case MC_MBUF:
3425 case MC_CL:
3426 case MC_BIGCL:
3427 case MC_16KCL:
3428 return (FALSE);
3429
3430 case MC_MBUF_CL:
3431 case MC_MBUF_BIGCL:
3432 case MC_MBUF_16KCL:
3433 /* Get the required number of constructed objects if possible */
3434 if (m_infree(class) > m_minlimit(class)) {
3435 tot = cslab_alloc(class, &list,
3436 MIN(num, m_infree(class)));
3437 }
3438
3439 /* And destroy them to get back the raw objects */
3440 if (top != NULL)
3441 (void) cslab_free(class, top, 1);
3442 break;
3443
3444 default:
3445 VERIFY(0);
3446 /* NOTREACHED */
3447 }
3448
3449 return (tot == num);
3450}
3451
3452static void
3453m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp)
3454{
3455 int m, bmap = 0;
3456
3457 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3458
3459 VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
3460 VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL));
3461 VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
3462
3463 /*
3464 * This logic can be made smarter; for now, simply mark
3465 * all other related classes as potential victims.
3466 */
3467 switch (class) {
3468 case MC_MBUF:
3469 m_wantpurge(MC_CL)++;
3470 m_wantpurge(MC_BIGCL)++;
3471 m_wantpurge(MC_MBUF_CL)++;
3472 m_wantpurge(MC_MBUF_BIGCL)++;
3473 break;
3474
3475 case MC_CL:
3476 m_wantpurge(MC_MBUF)++;
3477 m_wantpurge(MC_BIGCL)++;
3478 m_wantpurge(MC_MBUF_BIGCL)++;
3479 if (!comp)
3480 m_wantpurge(MC_MBUF_CL)++;
3481 break;
3482
3483 case MC_BIGCL:
3484 m_wantpurge(MC_MBUF)++;
3485 m_wantpurge(MC_CL)++;
3486 m_wantpurge(MC_MBUF_CL)++;
3487 if (!comp)
3488 m_wantpurge(MC_MBUF_BIGCL)++;
3489 break;
3490
3491 case MC_16KCL:
3492 if (!comp)
3493 m_wantpurge(MC_MBUF_16KCL)++;
3494 break;
3495
3496 default:
3497 VERIFY(0);
3498 /* NOTREACHED */
3499 }
3500
3501 /*
3502 * Run through each marked class and check if we really need to
3503 * purge (and therefore temporarily disable) the per-CPU caches
3504 * layer used by the class. If so, remember the classes since
3505 * we are going to drop the lock below prior to purging.
3506 */
3507 for (m = 0; m < NELEM(mbuf_table); m++) {
3508 if (m_wantpurge(m) > 0) {
3509 m_wantpurge(m) = 0;
3510 /*
3511 * Try hard to steal the required number of objects
3512 * from the freelist of other mbuf classes. Only
3513 * purge and disable the per-CPU caches layer when
3514 * we don't have enough; it's the last resort.
3515 */
3516 if (!mbuf_steal(m, num))
3517 bmap |= (1 << m);
3518 }
3519 }
3520
3521 lck_mtx_unlock(mbuf_mlock);
3522
3523 if (bmap != 0) {
3524 /* signal the domains to drain */
3525 net_drain_domains();
3526
3527 /* Sigh; we have no other choices but to ask mcache to purge */
3528 for (m = 0; m < NELEM(mbuf_table); m++) {
3529 if ((bmap & (1 << m)) &&
3530 mcache_purge_cache(m_cache(m), TRUE)) {
3531 lck_mtx_lock(mbuf_mlock);
3532 m_purge_cnt(m)++;
3533 mbstat.m_drain++;
3534 lck_mtx_unlock(mbuf_mlock);
3535 }
3536 }
3537 } else {
3538 /*
3539 * Request mcache to reap extra elements from all of its caches;
3540 * note that all reaps are serialized and happen only at a fixed
3541 * interval.
3542 */
3543 mcache_reap();
3544 }
3545 lck_mtx_lock(mbuf_mlock);
3546}
3547
3548static inline struct mbuf *
3549m_get_common(int wait, short type, int hdr)
3550{
3551 struct mbuf *m;
3552 int mcflags = MSLEEPF(wait);
3553
3554 /* Is this due to a non-blocking retry? If so, then try harder */
3555 if (mcflags & MCR_NOSLEEP)
3556 mcflags |= MCR_TRYHARD;
3557
3558 m = mcache_alloc(m_cache(MC_MBUF), mcflags);
3559 if (m != NULL) {
3560 MBUF_INIT(m, hdr, type);
3561 mtype_stat_inc(type);
3562 mtype_stat_dec(MT_FREE);
3563#if CONFIG_MACF_NET
3564 if (hdr && mac_init_mbuf(m, wait) != 0) {
3565 m_free(m);
3566 return (NULL);
3567 }
3568#endif /* MAC_NET */
3569 }
3570 return (m);
3571}
3572
3573/*
3574 * Space allocation routines; these are also available as macros
3575 * for critical paths.
3576 */
3577#define _M_GET(wait, type) m_get_common(wait, type, 0)
3578#define _M_GETHDR(wait, type) m_get_common(wait, type, 1)
3579#define _M_RETRY(wait, type) _M_GET(wait, type)
3580#define _M_RETRYHDR(wait, type) _M_GETHDR(wait, type)
3581#define _MGET(m, how, type) ((m) = _M_GET(how, type))
3582#define _MGETHDR(m, how, type) ((m) = _M_GETHDR(how, type))
3583
3584struct mbuf *
3585m_get(int wait, int type)
3586{
3587 return (_M_GET(wait, type));
3588}
3589
3590struct mbuf *
3591m_gethdr(int wait, int type)
3592{
3593 return (_M_GETHDR(wait, type));
3594}
3595
3596struct mbuf *
3597m_retry(int wait, int type)
3598{
3599 return (_M_RETRY(wait, type));
3600}
3601
3602struct mbuf *
3603m_retryhdr(int wait, int type)
3604{
3605 return (_M_RETRYHDR(wait, type));
3606}
3607
3608struct mbuf *
3609m_getclr(int wait, int type)
3610{
3611 struct mbuf *m;
3612
3613 _MGET(m, wait, type);
3614 if (m != NULL)
3615 bzero(MTOD(m, caddr_t), MLEN);
3616 return (m);
3617}
3618
3619static int
3620m_free_paired(struct mbuf *m)
3621{
3622 VERIFY((m->m_flags & M_EXT) && (MEXT_FLAGS(m) & EXTF_PAIRED));
3623
3624 membar_sync();
3625 if (MEXT_PMBUF(m) == m) {
3626 volatile UInt16 *addr = (volatile UInt16 *)&MEXT_PREF(m);
3627 int16_t oprefcnt, prefcnt;
3628
3629 /*
3630 * Paired ref count might be negative in case we lose
3631 * against another thread clearing MEXT_PMBUF, in the
3632 * event it occurs after the above memory barrier sync.
3633 * In that case just ignore as things have been unpaired.
3634 */
3635 do {
3636 oprefcnt = *addr;
3637 prefcnt = oprefcnt - 1;
3638 } while (!OSCompareAndSwap16(oprefcnt, prefcnt, addr));
3639
3640 if (prefcnt > 1) {
3641 return (1);
3642 } else if (prefcnt == 1) {
3643 (*(m_get_ext_free(m)))(m->m_ext.ext_buf,
3644 m->m_ext.ext_size, m_get_ext_arg(m));
3645 return (1);
3646 } else if (prefcnt == 0) {
3647 VERIFY(MBUF_IS_PAIRED(m));
3648
3649 /*
3650 * Restore minref to its natural value, so that
3651 * the caller will be able to free the cluster
3652 * as appropriate.
3653 */
3654 MEXT_MINREF(m) = 0;
3655
3656 /*
3657 * Clear MEXT_PMBUF, but leave EXTF_PAIRED intact
3658 * as it is immutable. atomic_set_ptr also causes
3659 * memory barrier sync.
3660 */
3661 atomic_set_ptr(&MEXT_PMBUF(m), NULL);
3662
3663 switch (m->m_ext.ext_size) {
3664 case MCLBYTES:
3665 m_set_ext(m, m_get_rfa(m), NULL, NULL);
3666 break;
3667
3668 case MBIGCLBYTES:
3669 m_set_ext(m, m_get_rfa(m), m_bigfree, NULL);
3670 break;
3671
3672 case M16KCLBYTES:
3673 m_set_ext(m, m_get_rfa(m), m_16kfree, NULL);
3674 break;
3675
3676 default:
3677 VERIFY(0);
3678 /* NOTREACHED */
3679 }
3680 }
3681 }
3682
3683 /*
3684 * Tell caller the unpair has occurred, and that the reference
3685 * count on the external cluster held for the paired mbuf should
3686 * now be dropped.
3687 */
3688 return (0);
3689}
3690
3691struct mbuf *
3692m_free(struct mbuf *m)
3693{
3694 struct mbuf *n = m->m_next;
3695
3696 if (m->m_type == MT_FREE)
3697 panic("m_free: freeing an already freed mbuf");
3698
3699 if (m->m_flags & M_PKTHDR) {
3700 /* Check for scratch area overflow */
3701 m_redzone_verify(m);
3702 /* Free the aux data and tags if there is any */
3703 m_tag_delete_chain(m, NULL);
3704
3705 m_do_tx_compl_callback(m, NULL);
3706 }
3707
3708 if (m->m_flags & M_EXT) {
3709 u_int16_t refcnt;
3710 u_int32_t composite;
3711 m_ext_free_func_t m_free_func;
3712
3713 if (MBUF_IS_PAIRED(m) && m_free_paired(m))
3714 return (n);
3715
3716 refcnt = m_decref(m);
3717 composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
3718 m_free_func = m_get_ext_free(m);
3719
3720 if (refcnt == MEXT_MINREF(m) && !composite) {
3721 if (m_free_func == NULL) {
3722 mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
3723 } else if (m_free_func == m_bigfree) {
3724 mcache_free(m_cache(MC_BIGCL),
3725 m->m_ext.ext_buf);
3726 } else if (m_free_func == m_16kfree) {
3727 mcache_free(m_cache(MC_16KCL),
3728 m->m_ext.ext_buf);
3729 } else {
3730 (*m_free_func)(m->m_ext.ext_buf,
3731 m->m_ext.ext_size, m_get_ext_arg(m));
3732 }
3733 mcache_free(ref_cache, m_get_rfa(m));
3734 m_set_ext(m, NULL, NULL, NULL);
3735 } else if (refcnt == MEXT_MINREF(m) && composite) {
3736 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
3737 VERIFY(m->m_type != MT_FREE);
3738
3739 mtype_stat_dec(m->m_type);
3740 mtype_stat_inc(MT_FREE);
3741
3742 m->m_type = MT_FREE;
3743 m->m_flags = M_EXT;
3744 m->m_len = 0;
3745 m->m_next = m->m_nextpkt = NULL;
3746
3747 MEXT_FLAGS(m) &= ~EXTF_READONLY;
3748
3749 /* "Free" into the intermediate cache */
3750 if (m_free_func == NULL) {
3751 mcache_free(m_cache(MC_MBUF_CL), m);
3752 } else if (m_free_func == m_bigfree) {
3753 mcache_free(m_cache(MC_MBUF_BIGCL), m);
3754 } else {
3755 VERIFY(m_free_func == m_16kfree);
3756 mcache_free(m_cache(MC_MBUF_16KCL), m);
3757 }
3758 return (n);
3759 }
3760 }
3761
3762 if (m->m_type != MT_FREE) {
3763 mtype_stat_dec(m->m_type);
3764 mtype_stat_inc(MT_FREE);
3765 }
3766
3767 m->m_type = MT_FREE;
3768 m->m_flags = m->m_len = 0;
3769 m->m_next = m->m_nextpkt = NULL;
3770
3771 mcache_free(m_cache(MC_MBUF), m);
3772
3773 return (n);
3774}
3775
3776__private_extern__ struct mbuf *
3777m_clattach(struct mbuf *m, int type, caddr_t extbuf,
3778 void (*extfree)(caddr_t, u_int, caddr_t), u_int extsize, caddr_t extarg,
3779 int wait, int pair)
3780{
3781 struct ext_ref *rfa = NULL;
3782
3783 /*
3784 * If pairing is requested and an existing mbuf is provided, reject
3785 * it if it's already been paired to another cluster. Otherwise,
3786 * allocate a new one or free any existing below.
3787 */
3788 if ((m != NULL && MBUF_IS_PAIRED(m)) ||
3789 (m == NULL && (m = _M_GETHDR(wait, type)) == NULL))
3790 return (NULL);
3791
3792 if (m->m_flags & M_EXT) {
3793 u_int16_t refcnt;
3794 u_int32_t composite;
3795 m_ext_free_func_t m_free_func;
3796
3797 refcnt = m_decref(m);
3798 composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
3799 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED) && MEXT_PMBUF(m) == NULL);
3800 m_free_func = m_get_ext_free(m);
3801 if (refcnt == MEXT_MINREF(m) && !composite) {
3802 if (m_free_func == NULL) {
3803 mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
3804 } else if (m_free_func == m_bigfree) {
3805 mcache_free(m_cache(MC_BIGCL),
3806 m->m_ext.ext_buf);
3807 } else if (m_free_func == m_16kfree) {
3808 mcache_free(m_cache(MC_16KCL),
3809 m->m_ext.ext_buf);
3810 } else {
3811 (*m_free_func)(m->m_ext.ext_buf,
3812 m->m_ext.ext_size, m_get_ext_arg(m));
3813 }
3814 /* Re-use the reference structure */
3815 rfa = m_get_rfa(m);
3816 } else if (refcnt == MEXT_MINREF(m) && composite) {
3817 VERIFY(m->m_type != MT_FREE);
3818
3819 mtype_stat_dec(m->m_type);
3820 mtype_stat_inc(MT_FREE);
3821
3822 m->m_type = MT_FREE;
3823 m->m_flags = M_EXT;
3824 m->m_len = 0;
3825 m->m_next = m->m_nextpkt = NULL;
3826
3827 MEXT_FLAGS(m) &= ~EXTF_READONLY;
3828
3829 /* "Free" into the intermediate cache */
3830 if (m_free_func == NULL) {
3831 mcache_free(m_cache(MC_MBUF_CL), m);
3832 } else if (m_free_func == m_bigfree) {
3833 mcache_free(m_cache(MC_MBUF_BIGCL), m);
3834 } else {
3835 VERIFY(m_free_func == m_16kfree);
3836 mcache_free(m_cache(MC_MBUF_16KCL), m);
3837 }
3838 /*
3839 * Allocate a new mbuf, since we didn't divorce
3840 * the composite mbuf + cluster pair above.
3841 */
3842 if ((m = _M_GETHDR(wait, type)) == NULL)
3843 return (NULL);
3844 }
3845 }
3846
3847 if (rfa == NULL &&
3848 (rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
3849 m_free(m);
3850 return (NULL);
3851 }
3852
3853 if (!pair) {
3854 MEXT_INIT(m, extbuf, extsize, extfree, extarg, rfa,
3855 0, 1, 0, 0, 0, NULL);
3856 } else {
3857 MEXT_INIT(m, extbuf, extsize, extfree, (caddr_t)m, rfa,
3858 1, 1, 1, EXTF_PAIRED, 0, m);
3859 }
3860
3861 return (m);
3862}
3863
3864/*
3865 * Perform `fast' allocation mbuf clusters from a cache of recently-freed
3866 * clusters. (If the cache is empty, new clusters are allocated en-masse.)
3867 */
3868struct mbuf *
3869m_getcl(int wait, int type, int flags)
3870{
3871 struct mbuf *m;
3872 int mcflags = MSLEEPF(wait);
3873 int hdr = (flags & M_PKTHDR);
3874
3875 /* Is this due to a non-blocking retry? If so, then try harder */
3876 if (mcflags & MCR_NOSLEEP)
3877 mcflags |= MCR_TRYHARD;
3878
3879 m = mcache_alloc(m_cache(MC_MBUF_CL), mcflags);
3880 if (m != NULL) {
3881 u_int16_t flag;
3882 struct ext_ref *rfa;
3883 void *cl;
3884
3885 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
3886 cl = m->m_ext.ext_buf;
3887 rfa = m_get_rfa(m);
3888
3889 ASSERT(cl != NULL && rfa != NULL);
3890 VERIFY(MBUF_IS_COMPOSITE(m) && m_get_ext_free(m) == NULL);
3891
3892 flag = MEXT_FLAGS(m);
3893
3894 MBUF_INIT(m, hdr, type);
3895 MBUF_CL_INIT(m, cl, rfa, 1, flag);
3896
3897 mtype_stat_inc(type);
3898 mtype_stat_dec(MT_FREE);
3899#if CONFIG_MACF_NET
3900 if (hdr && mac_init_mbuf(m, wait) != 0) {
3901 m_freem(m);
3902 return (NULL);
3903 }
3904#endif /* MAC_NET */
3905 }
3906 return (m);
3907}
3908
3909/* m_mclget() add an mbuf cluster to a normal mbuf */
3910struct mbuf *
3911m_mclget(struct mbuf *m, int wait)
3912{
3913 struct ext_ref *rfa;
3914
3915 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
3916 return (m);
3917
3918 m->m_ext.ext_buf = m_mclalloc(wait);
3919 if (m->m_ext.ext_buf != NULL) {
3920 MBUF_CL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
3921 } else {
3922 mcache_free(ref_cache, rfa);
3923 }
3924 return (m);
3925}
3926
3927/* Allocate an mbuf cluster */
3928caddr_t
3929m_mclalloc(int wait)
3930{
3931 int mcflags = MSLEEPF(wait);
3932
3933 /* Is this due to a non-blocking retry? If so, then try harder */
3934 if (mcflags & MCR_NOSLEEP)
3935 mcflags |= MCR_TRYHARD;
3936
3937 return (mcache_alloc(m_cache(MC_CL), mcflags));
3938}
3939
3940/* Free an mbuf cluster */
3941void
3942m_mclfree(caddr_t p)
3943{
3944 mcache_free(m_cache(MC_CL), p);
3945}
3946
3947/*
3948 * mcl_hasreference() checks if a cluster of an mbuf is referenced by
3949 * another mbuf; see comments in m_incref() regarding EXTF_READONLY.
3950 */
3951int
3952m_mclhasreference(struct mbuf *m)
3953{
3954 if (!(m->m_flags & M_EXT))
3955 return (0);
3956
3957 ASSERT(m_get_rfa(m) != NULL);
3958
3959 return ((MEXT_FLAGS(m) & EXTF_READONLY) ? 1 : 0);
3960}
3961
3962__private_extern__ caddr_t
3963m_bigalloc(int wait)
3964{
3965 int mcflags = MSLEEPF(wait);
3966
3967 /* Is this due to a non-blocking retry? If so, then try harder */
3968 if (mcflags & MCR_NOSLEEP)
3969 mcflags |= MCR_TRYHARD;
3970
3971 return (mcache_alloc(m_cache(MC_BIGCL), mcflags));
3972}
3973
3974__private_extern__ void
3975m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
3976{
3977 mcache_free(m_cache(MC_BIGCL), p);
3978}
3979
3980/* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
3981__private_extern__ struct mbuf *
3982m_mbigget(struct mbuf *m, int wait)
3983{
3984 struct ext_ref *rfa;
3985
3986 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
3987 return (m);
3988
3989 m->m_ext.ext_buf = m_bigalloc(wait);
3990 if (m->m_ext.ext_buf != NULL) {
3991 MBUF_BIGCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
3992 } else {
3993 mcache_free(ref_cache, rfa);
3994 }
3995 return (m);
3996}
3997
3998__private_extern__ caddr_t
3999m_16kalloc(int wait)
4000{
4001 int mcflags = MSLEEPF(wait);
4002
4003 /* Is this due to a non-blocking retry? If so, then try harder */
4004 if (mcflags & MCR_NOSLEEP)
4005 mcflags |= MCR_TRYHARD;
4006
4007 return (mcache_alloc(m_cache(MC_16KCL), mcflags));
4008}
4009
4010__private_extern__ void
4011m_16kfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
4012{
4013 mcache_free(m_cache(MC_16KCL), p);
4014}
4015
4016/* m_m16kget() add a 16KB mbuf cluster to a normal mbuf */
4017__private_extern__ struct mbuf *
4018m_m16kget(struct mbuf *m, int wait)
4019{
4020 struct ext_ref *rfa;
4021
4022 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
4023 return (m);
4024
4025 m->m_ext.ext_buf = m_16kalloc(wait);
4026 if (m->m_ext.ext_buf != NULL) {
4027 MBUF_16KCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
4028 } else {
4029 mcache_free(ref_cache, rfa);
4030 }
4031 return (m);
4032}
4033
4034/*
4035 * "Move" mbuf pkthdr from "from" to "to".
4036 * "from" must have M_PKTHDR set, and "to" must be empty.
4037 */
4038void
4039m_copy_pkthdr(struct mbuf *to, struct mbuf *from)
4040{
4041 VERIFY(from->m_flags & M_PKTHDR);
4042
4043 /* Check for scratch area overflow */
4044 m_redzone_verify(from);
4045
4046 if (to->m_flags & M_PKTHDR) {
4047 /* Check for scratch area overflow */
4048 m_redzone_verify(to);
4049 /* We will be taking over the tags of 'to' */
4050 m_tag_delete_chain(to, NULL);
4051 }
4052 to->m_pkthdr = from->m_pkthdr; /* especially tags */
4053 m_classifier_init(from, 0); /* purge classifier info */
4054 m_tag_init(from, 1); /* purge all tags from src */
4055 m_scratch_init(from); /* clear src scratch area */
4056 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
4057 if ((to->m_flags & M_EXT) == 0)
4058 to->m_data = to->m_pktdat;
4059 m_redzone_init(to); /* setup red zone on dst */
4060}
4061
4062/*
4063 * Duplicate "from"'s mbuf pkthdr in "to".
4064 * "from" must have M_PKTHDR set, and "to" must be empty.
4065 * In particular, this does a deep copy of the packet tags.
4066 */
4067static int
4068m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
4069{
4070 VERIFY(from->m_flags & M_PKTHDR);
4071
4072 /* Check for scratch area overflow */
4073 m_redzone_verify(from);
4074
4075 if (to->m_flags & M_PKTHDR) {
4076 /* Check for scratch area overflow */
4077 m_redzone_verify(to);
4078 /* We will be taking over the tags of 'to' */
4079 m_tag_delete_chain(to, NULL);
4080 }
4081 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
4082 if ((to->m_flags & M_EXT) == 0)
4083 to->m_data = to->m_pktdat;
4084 to->m_pkthdr = from->m_pkthdr;
4085 m_redzone_init(to); /* setup red zone on dst */
4086 m_tag_init(to, 0); /* preserve dst static tags */
4087 return (m_tag_copy_chain(to, from, how));
4088}
4089
4090void
4091m_copy_pftag(struct mbuf *to, struct mbuf *from)
4092{
4093 memcpy(m_pftag(to), m_pftag(from), sizeof(struct pf_mtag));
4094#if PF_ECN
4095 m_pftag(to)->pftag_hdr = NULL;
4096 m_pftag(to)->pftag_flags &= ~(PF_TAG_HDR_INET|PF_TAG_HDR_INET6);
4097#endif /* PF_ECN */
4098}
4099
4100void
4101m_classifier_init(struct mbuf *m, uint32_t pktf_mask)
4102{
4103 VERIFY(m->m_flags & M_PKTHDR);
4104
4105 m->m_pkthdr.pkt_proto = 0;
4106 m->m_pkthdr.pkt_flowsrc = 0;
4107 m->m_pkthdr.pkt_flowid = 0;
4108 m->m_pkthdr.pkt_flags &= pktf_mask; /* caller-defined mask */
4109 /* preserve service class and interface info for loopback packets */
4110 if (!(m->m_pkthdr.pkt_flags & PKTF_LOOP))
4111 (void) m_set_service_class(m, MBUF_SC_BE);
4112 if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO))
4113 m->m_pkthdr.pkt_ifainfo = 0;
4114 /*
4115 * Preserve timestamp if requested
4116 */
4117 if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID))
4118 m->m_pkthdr.pkt_timestamp = 0;
4119}
4120
4121void
4122m_copy_classifier(struct mbuf *to, struct mbuf *from)
4123{
4124 VERIFY(to->m_flags & M_PKTHDR);
4125 VERIFY(from->m_flags & M_PKTHDR);
4126
4127 to->m_pkthdr.pkt_proto = from->m_pkthdr.pkt_proto;
4128 to->m_pkthdr.pkt_flowsrc = from->m_pkthdr.pkt_flowsrc;
4129 to->m_pkthdr.pkt_flowid = from->m_pkthdr.pkt_flowid;
4130 to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags;
4131 (void) m_set_service_class(to, from->m_pkthdr.pkt_svc);
4132 to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo;
4133}
4134
4135/*
4136 * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
4137 * if wantall is not set, return whatever number were available. Set up the
4138 * first num_with_pkthdrs with mbuf hdrs configured as packet headers; these
4139 * are chained on the m_nextpkt field. Any packets requested beyond this
4140 * are chained onto the last packet header's m_next field. The size of
4141 * the cluster is controlled by the parameter bufsize.
4142 */
4143__private_extern__ struct mbuf *
4144m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs,
4145 int wait, int wantall, size_t bufsize)
4146{
4147 struct mbuf *m;
4148 struct mbuf **np, *top;
4149 unsigned int pnum, needed = *num_needed;
4150 mcache_obj_t *mp_list = NULL;
4151 int mcflags = MSLEEPF(wait);
4152 u_int16_t flag;
4153 struct ext_ref *rfa;
4154 mcache_t *cp;
4155 void *cl;
4156
4157 ASSERT(bufsize == m_maxsize(MC_CL) ||
4158 bufsize == m_maxsize(MC_BIGCL) ||
4159 bufsize == m_maxsize(MC_16KCL));
4160
4161 /*
4162 * Caller must first check for njcl because this
4163 * routine is internal and not exposed/used via KPI.
4164 */
4165 VERIFY(bufsize != m_maxsize(MC_16KCL) || njcl > 0);
4166
4167 top = NULL;
4168 np = &top;
4169 pnum = 0;
4170
4171 /*
4172 * The caller doesn't want all the requested buffers; only some.
4173 * Try hard to get what we can, but don't block. This effectively
4174 * overrides MCR_SLEEP, since this thread will not go to sleep
4175 * if we can't get all the buffers.
4176 */
4177 if (!wantall || (mcflags & MCR_NOSLEEP))
4178 mcflags |= MCR_TRYHARD;
4179
4180 /* Allocate the composite mbuf + cluster elements from the cache */
4181 if (bufsize == m_maxsize(MC_CL))
4182 cp = m_cache(MC_MBUF_CL);
4183 else if (bufsize == m_maxsize(MC_BIGCL))
4184 cp = m_cache(MC_MBUF_BIGCL);
4185 else
4186 cp = m_cache(MC_MBUF_16KCL);
4187 needed = mcache_alloc_ext(cp, &mp_list, needed, mcflags);
4188
4189 for (pnum = 0; pnum < needed; pnum++) {
4190 m = (struct mbuf *)mp_list;
4191 mp_list = mp_list->obj_next;
4192
4193 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
4194 cl = m->m_ext.ext_buf;
4195 rfa = m_get_rfa(m);
4196
4197 ASSERT(cl != NULL && rfa != NULL);
4198 VERIFY(MBUF_IS_COMPOSITE(m));
4199
4200 flag = MEXT_FLAGS(m);
4201
4202 MBUF_INIT(m, num_with_pkthdrs, MT_DATA);
4203 if (bufsize == m_maxsize(MC_16KCL)) {
4204 MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
4205 } else if (bufsize == m_maxsize(MC_BIGCL)) {
4206 MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
4207 } else {
4208 MBUF_CL_INIT(m, cl, rfa, 1, flag);
4209 }
4210
4211 if (num_with_pkthdrs > 0) {
4212 --num_with_pkthdrs;
4213#if CONFIG_MACF_NET
4214 if (mac_mbuf_label_init(m, wait) != 0) {
4215 m_freem(m);
4216 break;
4217 }
4218#endif /* MAC_NET */
4219 }
4220
4221 *np = m;
4222 if (num_with_pkthdrs > 0)
4223 np = &m->m_nextpkt;
4224 else
4225 np = &m->m_next;
4226 }
4227 ASSERT(pnum != *num_needed || mp_list == NULL);
4228 if (mp_list != NULL)
4229 mcache_free_ext(cp, mp_list);
4230
4231 if (pnum > 0) {
4232 mtype_stat_add(MT_DATA, pnum);
4233 mtype_stat_sub(MT_FREE, pnum);
4234 }
4235
4236 if (wantall && (pnum != *num_needed)) {
4237 if (top != NULL)
4238 m_freem_list(top);
4239 return (NULL);
4240 }
4241
4242 if (pnum > *num_needed) {
4243 printf("%s: File a radar related to <rdar://10146739>. \
4244 needed = %u, pnum = %u, num_needed = %u \n",
4245 __func__, needed, pnum, *num_needed);
4246 }
4247
4248 *num_needed = pnum;
4249 return (top);
4250}
4251
4252/*
4253 * Return list of mbuf linked by m_nextpkt. Try for numlist, and if
4254 * wantall is not set, return whatever number were available. The size of
4255 * each mbuf in the list is controlled by the parameter packetlen. Each
4256 * mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf
4257 * in the chain is called a segment. If maxsegments is not null and the
4258 * value pointed to is not null, this specify the maximum number of segments
4259 * for a chain of mbufs. If maxsegments is zero or the value pointed to
4260 * is zero the caller does not have any restriction on the number of segments.
4261 * The actual number of segments of a mbuf chain is return in the value
4262 * pointed to by maxsegments.
4263 */
4264__private_extern__ struct mbuf *
4265m_allocpacket_internal(unsigned int *numlist, size_t packetlen,
4266 unsigned int *maxsegments, int wait, int wantall, size_t wantsize)
4267{
4268 struct mbuf **np, *top, *first = NULL;
4269 size_t bufsize, r_bufsize;
4270 unsigned int num = 0;
4271 unsigned int nsegs = 0;
4272 unsigned int needed, resid;
4273 int mcflags = MSLEEPF(wait);
4274 mcache_obj_t *mp_list = NULL, *rmp_list = NULL;
4275 mcache_t *cp = NULL, *rcp = NULL;
4276
4277 if (*numlist == 0)
4278 return (NULL);
4279
4280 top = NULL;
4281 np = &top;
4282
4283 if (wantsize == 0) {
4284 if (packetlen <= MINCLSIZE) {
4285 bufsize = packetlen;
4286 } else if (packetlen > m_maxsize(MC_CL)) {
4287 /* Use 4KB if jumbo cluster pool isn't available */
4288 if (packetlen <= m_maxsize(MC_BIGCL) || njcl == 0)
4289 bufsize = m_maxsize(MC_BIGCL);
4290 else
4291 bufsize = m_maxsize(MC_16KCL);
4292 } else {
4293 bufsize = m_maxsize(MC_CL);
4294 }
4295 } else if (wantsize == m_maxsize(MC_CL) ||
4296 wantsize == m_maxsize(MC_BIGCL) ||
4297 (wantsize == m_maxsize(MC_16KCL) && njcl > 0)) {
4298 bufsize = wantsize;
4299 } else {
4300 return (NULL);
4301 }
4302
4303 if (bufsize <= MHLEN) {
4304 nsegs = 1;
4305 } else if (bufsize <= MINCLSIZE) {
4306 if (maxsegments != NULL && *maxsegments == 1) {
4307 bufsize = m_maxsize(MC_CL);
4308 nsegs = 1;
4309 } else {
4310 nsegs = 2;
4311 }
4312 } else if (bufsize == m_maxsize(MC_16KCL)) {
4313 VERIFY(njcl > 0);
4314 nsegs = ((packetlen - 1) >> M16KCLSHIFT) + 1;
4315 } else if (bufsize == m_maxsize(MC_BIGCL)) {
4316 nsegs = ((packetlen - 1) >> MBIGCLSHIFT) + 1;
4317 } else {
4318 nsegs = ((packetlen - 1) >> MCLSHIFT) + 1;
4319 }
4320 if (maxsegments != NULL) {
4321 if (*maxsegments && nsegs > *maxsegments) {
4322 *maxsegments = nsegs;
4323 return (NULL);
4324 }
4325 *maxsegments = nsegs;
4326 }
4327
4328 /*
4329 * The caller doesn't want all the requested buffers; only some.
4330 * Try hard to get what we can, but don't block. This effectively
4331 * overrides MCR_SLEEP, since this thread will not go to sleep
4332 * if we can't get all the buffers.
4333 */
4334 if (!wantall || (mcflags & MCR_NOSLEEP))
4335 mcflags |= MCR_TRYHARD;
4336
4337 /*
4338 * Simple case where all elements in the lists/chains are mbufs.
4339 * Unless bufsize is greater than MHLEN, each segment chain is made
4340 * up of exactly 1 mbuf. Otherwise, each segment chain is made up
4341 * of 2 mbufs; the second one is used for the residual data, i.e.
4342 * the remaining data that cannot fit into the first mbuf.
4343 */
4344 if (bufsize <= MINCLSIZE) {
4345 /* Allocate the elements in one shot from the mbuf cache */
4346 ASSERT(bufsize <= MHLEN || nsegs == 2);
4347 cp = m_cache(MC_MBUF);
4348 needed = mcache_alloc_ext(cp, &mp_list,
4349 (*numlist) * nsegs, mcflags);
4350
4351 /*
4352 * The number of elements must be even if we are to use an
4353 * mbuf (instead of a cluster) to store the residual data.
4354 * If we couldn't allocate the requested number of mbufs,
4355 * trim the number down (if it's odd) in order to avoid
4356 * creating a partial segment chain.
4357 */
4358 if (bufsize > MHLEN && (needed & 0x1))
4359 needed--;
4360
4361 while (num < needed) {
4362 struct mbuf *m;
4363
4364 m = (struct mbuf *)mp_list;
4365 mp_list = mp_list->obj_next;
4366 ASSERT(m != NULL);
4367
4368 MBUF_INIT(m, 1, MT_DATA);
4369#if CONFIG_MACF_NET
4370 if (mac_init_mbuf(m, wait) != 0) {
4371 m_free(m);
4372 break;
4373 }
4374#endif /* MAC_NET */
4375 num++;
4376 if (bufsize > MHLEN) {
4377 /* A second mbuf for this segment chain */
4378 m->m_next = (struct mbuf *)mp_list;
4379 mp_list = mp_list->obj_next;
4380 ASSERT(m->m_next != NULL);
4381
4382 MBUF_INIT(m->m_next, 0, MT_DATA);
4383 num++;
4384 }
4385 *np = m;
4386 np = &m->m_nextpkt;
4387 }
4388 ASSERT(num != *numlist || mp_list == NULL);
4389
4390 if (num > 0) {
4391 mtype_stat_add(MT_DATA, num);
4392 mtype_stat_sub(MT_FREE, num);
4393 }
4394 num /= nsegs;
4395
4396 /* We've got them all; return to caller */
4397 if (num == *numlist)
4398 return (top);
4399
4400 goto fail;
4401 }
4402
4403 /*
4404 * Complex cases where elements are made up of one or more composite
4405 * mbufs + cluster, depending on packetlen. Each N-segment chain can
4406 * be illustrated as follows:
4407 *
4408 * [mbuf + cluster 1] [mbuf + cluster 2] ... [mbuf + cluster N]
4409 *
4410 * Every composite mbuf + cluster element comes from the intermediate
4411 * cache (either MC_MBUF_CL or MC_MBUF_BIGCL). For space efficiency,
4412 * the last composite element will come from the MC_MBUF_CL cache,
4413 * unless the residual data is larger than 2KB where we use the
4414 * big cluster composite cache (MC_MBUF_BIGCL) instead. Residual
4415 * data is defined as extra data beyond the first element that cannot
4416 * fit into the previous element, i.e. there is no residual data if
4417 * the chain only has 1 segment.
4418 */
4419 r_bufsize = bufsize;
4420 resid = packetlen > bufsize ? packetlen % bufsize : 0;
4421 if (resid > 0) {
4422 /* There is residual data; figure out the cluster size */
4423 if (wantsize == 0 && packetlen > MINCLSIZE) {
4424 /*
4425 * Caller didn't request that all of the segments
4426 * in the chain use the same cluster size; use the
4427 * smaller of the cluster sizes.
4428 */
4429 if (njcl > 0 && resid > m_maxsize(MC_BIGCL))
4430 r_bufsize = m_maxsize(MC_16KCL);
4431 else if (resid > m_maxsize(MC_CL))
4432 r_bufsize = m_maxsize(MC_BIGCL);
4433 else
4434 r_bufsize = m_maxsize(MC_CL);
4435 } else {
4436 /* Use the same cluster size as the other segments */
4437 resid = 0;
4438 }
4439 }
4440
4441 needed = *numlist;
4442 if (resid > 0) {
4443 /*
4444 * Attempt to allocate composite mbuf + cluster elements for
4445 * the residual data in each chain; record the number of such
4446 * elements that can be allocated so that we know how many
4447 * segment chains we can afford to create.
4448 */
4449 if (r_bufsize <= m_maxsize(MC_CL))
4450 rcp = m_cache(MC_MBUF_CL);
4451 else if (r_bufsize <= m_maxsize(MC_BIGCL))
4452 rcp = m_cache(MC_MBUF_BIGCL);
4453 else
4454 rcp = m_cache(MC_MBUF_16KCL);
4455 needed = mcache_alloc_ext(rcp, &rmp_list, *numlist, mcflags);
4456
4457 if (needed == 0)
4458 goto fail;
4459
4460 /* This is temporarily reduced for calculation */
4461 ASSERT(nsegs > 1);
4462 nsegs--;
4463 }
4464
4465 /*
4466 * Attempt to allocate the rest of the composite mbuf + cluster
4467 * elements for the number of segment chains that we need.
4468 */
4469 if (bufsize <= m_maxsize(MC_CL))
4470 cp = m_cache(MC_MBUF_CL);
4471 else if (bufsize <= m_maxsize(MC_BIGCL))
4472 cp = m_cache(MC_MBUF_BIGCL);
4473 else
4474 cp = m_cache(MC_MBUF_16KCL);
4475 needed = mcache_alloc_ext(cp, &mp_list, needed * nsegs, mcflags);
4476
4477 /* Round it down to avoid creating a partial segment chain */
4478 needed = (needed / nsegs) * nsegs;
4479 if (needed == 0)
4480 goto fail;
4481
4482 if (resid > 0) {
4483 /*
4484 * We're about to construct the chain(s); take into account
4485 * the number of segments we have created above to hold the
4486 * residual data for each chain, as well as restore the
4487 * original count of segments per chain.
4488 */
4489 ASSERT(nsegs > 0);
4490 needed += needed / nsegs;
4491 nsegs++;
4492 }
4493
4494 for (;;) {
4495 struct mbuf *m;
4496 u_int16_t flag;
4497 struct ext_ref *rfa;
4498 void *cl;
4499 int pkthdr;
4500 m_ext_free_func_t m_free_func;
4501
4502 ++num;
4503 if (nsegs == 1 || (num % nsegs) != 0 || resid == 0) {
4504 m = (struct mbuf *)mp_list;
4505 mp_list = mp_list->obj_next;
4506 } else {
4507 m = (struct mbuf *)rmp_list;
4508 rmp_list = rmp_list->obj_next;
4509 }
4510 m_free_func = m_get_ext_free(m);
4511 ASSERT(m != NULL);
4512 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
4513 VERIFY(m_free_func == NULL || m_free_func == m_bigfree ||
4514 m_free_func == m_16kfree);
4515
4516 cl = m->m_ext.ext_buf;
4517 rfa = m_get_rfa(m);
4518
4519 ASSERT(cl != NULL && rfa != NULL);
4520 VERIFY(MBUF_IS_COMPOSITE(m));
4521
4522 flag = MEXT_FLAGS(m);
4523
4524 pkthdr = (nsegs == 1 || (num % nsegs) == 1);
4525 if (pkthdr)
4526 first = m;
4527 MBUF_INIT(m, pkthdr, MT_DATA);
4528 if (m_free_func == m_16kfree) {
4529 MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
4530 } else if (m_free_func == m_bigfree) {
4531 MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
4532 } else {
4533 MBUF_CL_INIT(m, cl, rfa, 1, flag);
4534 }
4535#if CONFIG_MACF_NET
4536 if (pkthdr && mac_init_mbuf(m, wait) != 0) {
4537 --num;
4538 m_freem(m);
4539 break;
4540 }
4541#endif /* MAC_NET */
4542
4543 *np = m;
4544 if ((num % nsegs) == 0)
4545 np = &first->m_nextpkt;
4546 else
4547 np = &m->m_next;
4548
4549 if (num == needed)
4550 break;
4551 }
4552
4553 if (num > 0) {
4554 mtype_stat_add(MT_DATA, num);
4555 mtype_stat_sub(MT_FREE, num);
4556 }
4557
4558 num /= nsegs;
4559
4560 /* We've got them all; return to caller */
4561 if (num == *numlist) {
4562 ASSERT(mp_list == NULL && rmp_list == NULL);
4563 return (top);
4564 }
4565
4566fail:
4567 /* Free up what's left of the above */
4568 if (mp_list != NULL)
4569 mcache_free_ext(cp, mp_list);
4570 if (rmp_list != NULL)
4571 mcache_free_ext(rcp, rmp_list);
4572 if (wantall && top != NULL) {
4573 m_freem(top);
4574 return (NULL);
4575 }
4576 *numlist = num;
4577 return (top);
4578}
4579
4580/*
4581 * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
4582 * packets on receive ring.
4583 */
4584__private_extern__ struct mbuf *
4585m_getpacket_how(int wait)
4586{
4587 unsigned int num_needed = 1;
4588
4589 return (m_getpackets_internal(&num_needed, 1, wait, 1,
4590 m_maxsize(MC_CL)));
4591}
4592
4593/*
4594 * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
4595 * packets on receive ring.
4596 */
4597struct mbuf *
4598m_getpacket(void)
4599{
4600 unsigned int num_needed = 1;
4601
4602 return (m_getpackets_internal(&num_needed, 1, M_WAIT, 1,
4603 m_maxsize(MC_CL)));
4604}
4605
4606/*
4607 * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
4608 * if this can't be met, return whatever number were available. Set up the
4609 * first num_with_pkthdrs with mbuf hdrs configured as packet headers. These
4610 * are chained on the m_nextpkt field. Any packets requested beyond this are
4611 * chained onto the last packet header's m_next field.
4612 */
4613struct mbuf *
4614m_getpackets(int num_needed, int num_with_pkthdrs, int how)
4615{
4616 unsigned int n = num_needed;
4617
4618 return (m_getpackets_internal(&n, num_with_pkthdrs, how, 0,
4619 m_maxsize(MC_CL)));
4620}
4621
4622/*
4623 * Return a list of mbuf hdrs set up as packet hdrs chained together
4624 * on the m_nextpkt field
4625 */
4626struct mbuf *
4627m_getpackethdrs(int num_needed, int how)
4628{
4629 struct mbuf *m;
4630 struct mbuf **np, *top;
4631
4632 top = NULL;
4633 np = &top;
4634
4635 while (num_needed--) {
4636 m = _M_RETRYHDR(how, MT_DATA);
4637 if (m == NULL)
4638 break;
4639
4640 *np = m;
4641 np = &m->m_nextpkt;
4642 }
4643
4644 return (top);
4645}
4646
4647/*
4648 * Free an mbuf list (m_nextpkt) while following m_next. Returns the count
4649 * for mbufs packets freed. Used by the drivers.
4650 */
4651int
4652m_freem_list(struct mbuf *m)
4653{
4654 struct mbuf *nextpkt;
4655 mcache_obj_t *mp_list = NULL;
4656 mcache_obj_t *mcl_list = NULL;
4657 mcache_obj_t *mbc_list = NULL;
4658 mcache_obj_t *m16k_list = NULL;
4659 mcache_obj_t *m_mcl_list = NULL;
4660 mcache_obj_t *m_mbc_list = NULL;
4661 mcache_obj_t *m_m16k_list = NULL;
4662 mcache_obj_t *ref_list = NULL;
4663 int pktcount = 0;
4664 int mt_free = 0, mt_data = 0, mt_header = 0, mt_soname = 0, mt_tag = 0;
4665
4666 while (m != NULL) {
4667 pktcount++;
4668
4669 nextpkt = m->m_nextpkt;
4670 m->m_nextpkt = NULL;
4671
4672 while (m != NULL) {
4673 struct mbuf *next = m->m_next;
4674 mcache_obj_t *o, *rfa;
4675 u_int32_t composite;
4676 u_int16_t refcnt;
4677 m_ext_free_func_t m_free_func;
4678
4679 if (m->m_type == MT_FREE)
4680 panic("m_free: freeing an already freed mbuf");
4681
4682 if (m->m_flags & M_PKTHDR) {
4683 /* Check for scratch area overflow */
4684 m_redzone_verify(m);
4685 /* Free the aux data and tags if there is any */
4686 m_tag_delete_chain(m, NULL);
4687 }
4688
4689 if (!(m->m_flags & M_EXT)) {
4690 mt_free++;
4691 goto simple_free;
4692 }
4693
4694 if (MBUF_IS_PAIRED(m) && m_free_paired(m)) {
4695 m = next;
4696 continue;
4697 }
4698
4699 mt_free++;
4700
4701 o = (mcache_obj_t *)(void *)m->m_ext.ext_buf;
4702 refcnt = m_decref(m);
4703 composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
4704 m_free_func = m_get_ext_free(m);
4705 if (refcnt == MEXT_MINREF(m) && !composite) {
4706 if (m_free_func == NULL) {
4707 o->obj_next = mcl_list;
4708 mcl_list = o;
4709 } else if (m_free_func == m_bigfree) {
4710 o->obj_next = mbc_list;
4711 mbc_list = o;
4712 } else if (m_free_func == m_16kfree) {
4713 o->obj_next = m16k_list;
4714 m16k_list = o;
4715 } else {
4716 (*(m_free_func))((caddr_t)o,
4717 m->m_ext.ext_size,
4718 m_get_ext_arg(m));
4719 }
4720 rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
4721 rfa->obj_next = ref_list;
4722 ref_list = rfa;
4723 m_set_ext(m, NULL, NULL, NULL);
4724 } else if (refcnt == MEXT_MINREF(m) && composite) {
4725 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
4726 VERIFY(m->m_type != MT_FREE);
4727 /*
4728 * Amortize the costs of atomic operations
4729 * by doing them at the end, if possible.
4730 */
4731 if (m->m_type == MT_DATA)
4732 mt_data++;
4733 else if (m->m_type == MT_HEADER)
4734 mt_header++;
4735 else if (m->m_type == MT_SONAME)
4736 mt_soname++;
4737 else if (m->m_type == MT_TAG)
4738 mt_tag++;
4739 else
4740 mtype_stat_dec(m->m_type);
4741
4742 m->m_type = MT_FREE;
4743 m->m_flags = M_EXT;
4744 m->m_len = 0;
4745 m->m_next = m->m_nextpkt = NULL;
4746
4747 MEXT_FLAGS(m) &= ~EXTF_READONLY;
4748
4749 /* "Free" into the intermediate cache */
4750 o = (mcache_obj_t *)m;
4751 if (m_free_func == NULL) {
4752 o->obj_next = m_mcl_list;
4753 m_mcl_list = o;
4754 } else if (m_free_func == m_bigfree) {
4755 o->obj_next = m_mbc_list;
4756 m_mbc_list = o;
4757 } else {
4758 VERIFY(m_free_func == m_16kfree);
4759 o->obj_next = m_m16k_list;
4760 m_m16k_list = o;
4761 }
4762 m = next;
4763 continue;
4764 }
4765simple_free:
4766 /*
4767 * Amortize the costs of atomic operations
4768 * by doing them at the end, if possible.
4769 */
4770 if (m->m_type == MT_DATA)
4771 mt_data++;
4772 else if (m->m_type == MT_HEADER)
4773 mt_header++;
4774 else if (m->m_type == MT_SONAME)
4775 mt_soname++;
4776 else if (m->m_type == MT_TAG)
4777 mt_tag++;
4778 else if (m->m_type != MT_FREE)
4779 mtype_stat_dec(m->m_type);
4780
4781 m->m_type = MT_FREE;
4782 m->m_flags = m->m_len = 0;
4783 m->m_next = m->m_nextpkt = NULL;
4784
4785 ((mcache_obj_t *)m)->obj_next = mp_list;
4786 mp_list = (mcache_obj_t *)m;
4787
4788 m = next;
4789 }
4790
4791 m = nextpkt;
4792 }
4793
4794 if (mt_free > 0)
4795 mtype_stat_add(MT_FREE, mt_free);
4796 if (mt_data > 0)
4797 mtype_stat_sub(MT_DATA, mt_data);
4798 if (mt_header > 0)
4799 mtype_stat_sub(MT_HEADER, mt_header);
4800 if (mt_soname > 0)
4801 mtype_stat_sub(MT_SONAME, mt_soname);
4802 if (mt_tag > 0)
4803 mtype_stat_sub(MT_TAG, mt_tag);
4804
4805 if (mp_list != NULL)
4806 mcache_free_ext(m_cache(MC_MBUF), mp_list);
4807 if (mcl_list != NULL)
4808 mcache_free_ext(m_cache(MC_CL), mcl_list);
4809 if (mbc_list != NULL)
4810 mcache_free_ext(m_cache(MC_BIGCL), mbc_list);
4811 if (m16k_list != NULL)
4812 mcache_free_ext(m_cache(MC_16KCL), m16k_list);
4813 if (m_mcl_list != NULL)
4814 mcache_free_ext(m_cache(MC_MBUF_CL), m_mcl_list);
4815 if (m_mbc_list != NULL)
4816 mcache_free_ext(m_cache(MC_MBUF_BIGCL), m_mbc_list);
4817 if (m_m16k_list != NULL)
4818 mcache_free_ext(m_cache(MC_MBUF_16KCL), m_m16k_list);
4819 if (ref_list != NULL)
4820 mcache_free_ext(ref_cache, ref_list);
4821
4822 return (pktcount);
4823}
4824
4825void
4826m_freem(struct mbuf *m)
4827{
4828 while (m != NULL)
4829 m = m_free(m);
4830}
4831
4832/*
4833 * Mbuffer utility routines.
4834 */
4835/*
4836 * Set the m_data pointer of a newly allocated mbuf to place an object of the
4837 * specified size at the end of the mbuf, longword aligned.
4838 *
4839 * NB: Historically, we had M_ALIGN(), MH_ALIGN(), and MEXT_ALIGN() as
4840 * separate macros, each asserting that it was called at the proper moment.
4841 * This required callers to themselves test the storage type and call the
4842 * right one. Rather than require callers to be aware of those layout
4843 * decisions, we centralize here.
4844 */
4845void
4846m_align(struct mbuf *m, int len)
4847{
4848 int adjust = 0;
4849
4850 /* At this point data must point to start */
4851 VERIFY(m->m_data == M_START(m));
4852 VERIFY(len >= 0);
4853 VERIFY(len <= M_SIZE(m));
4854 adjust = M_SIZE(m) - len;
4855 m->m_data += adjust &~ (sizeof(long) - 1);
4856}
4857
4858/*
4859 * Lesser-used path for M_PREPEND: allocate new mbuf to prepend to chain,
4860 * copy junk along. Does not adjust packet header length.
4861 */
4862struct mbuf *
4863m_prepend(struct mbuf *m, int len, int how)
4864{
4865 struct mbuf *mn;
4866
4867 _MGET(mn, how, m->m_type);
4868 if (mn == NULL) {
4869 m_freem(m);
4870 return (NULL);
4871 }
4872 if (m->m_flags & M_PKTHDR) {
4873 M_COPY_PKTHDR(mn, m);
4874 m->m_flags &= ~M_PKTHDR;
4875 }
4876 mn->m_next = m;
4877 m = mn;
4878 if (m->m_flags & M_PKTHDR) {
4879 VERIFY(len <= MHLEN);
4880 MH_ALIGN(m, len);
4881 } else {
4882 VERIFY(len <= MLEN);
4883 M_ALIGN(m, len);
4884 }
4885 m->m_len = len;
4886 return (m);
4887}
4888
4889/*
4890 * Replacement for old M_PREPEND macro: allocate new mbuf to prepend to
4891 * chain, copy junk along, and adjust length.
4892 */
4893struct mbuf *
4894m_prepend_2(struct mbuf *m, int len, int how, int align)
4895{
4896 if (M_LEADINGSPACE(m) >= len &&
4897 (!align || IS_P2ALIGNED((m->m_data - len), sizeof(u_int32_t)))) {
4898 m->m_data -= len;
4899 m->m_len += len;
4900 } else {
4901 m = m_prepend(m, len, how);
4902 }
4903 if ((m) && (m->m_flags & M_PKTHDR))
4904 m->m_pkthdr.len += len;
4905 return (m);
4906}
4907
4908/*
4909 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
4910 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
4911 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
4912 */
4913int MCFail;
4914
4915struct mbuf *
4916m_copym_mode(struct mbuf *m, int off0, int len, int wait, uint32_t mode)
4917{
4918 struct mbuf *n, *mhdr = NULL, **np;
4919 int off = off0;
4920 struct mbuf *top;
4921 int copyhdr = 0;
4922
4923 if (off < 0 || len < 0)
4924 panic("m_copym: invalid offset %d or len %d", off, len);
4925
4926 VERIFY((mode != M_COPYM_MUST_COPY_HDR &&
4927 mode != M_COPYM_MUST_MOVE_HDR) || (m->m_flags & M_PKTHDR));
4928
4929 if ((off == 0 && (m->m_flags & M_PKTHDR)) ||
4930 mode == M_COPYM_MUST_COPY_HDR || mode == M_COPYM_MUST_MOVE_HDR) {
4931 mhdr = m;
4932 copyhdr = 1;
4933 }
4934
4935 while (off >= m->m_len) {
4936 if (m->m_next == NULL)
4937 panic("m_copym: invalid mbuf chain");
4938 off -= m->m_len;
4939 m = m->m_next;
4940 }
4941 np = &top;
4942 top = NULL;
4943
4944 while (len > 0) {
4945 if (m == NULL) {
4946 if (len != M_COPYALL)
4947 panic("m_copym: len != M_COPYALL");
4948 break;
4949 }
4950
4951 if (copyhdr)
4952 n = _M_RETRYHDR(wait, m->m_type);
4953 else
4954 n = _M_RETRY(wait, m->m_type);
4955 *np = n;
4956
4957 if (n == NULL)
4958 goto nospace;
4959
4960 if (copyhdr != 0) {
4961 if ((mode == M_COPYM_MOVE_HDR) ||
4962 (mode == M_COPYM_MUST_MOVE_HDR)) {
4963 M_COPY_PKTHDR(n, mhdr);
4964 } else if ((mode == M_COPYM_COPY_HDR) ||
4965 (mode == M_COPYM_MUST_COPY_HDR)) {
4966 if (m_dup_pkthdr(n, mhdr, wait) == 0)
4967 goto nospace;
4968 }
4969 if (len == M_COPYALL)
4970 n->m_pkthdr.len -= off0;
4971 else
4972 n->m_pkthdr.len = len;
4973 copyhdr = 0;
4974 /*
4975 * There is data to copy from the packet header mbuf
4976 * if it is empty or it is before the starting offset
4977 */
4978 if (mhdr != m) {
4979 np = &n->m_next;
4980 continue;
4981 }
4982 }
4983 n->m_len = MIN(len, (m->m_len - off));
4984 if (m->m_flags & M_EXT) {
4985 n->m_ext = m->m_ext;
4986 m_incref(m);
4987 n->m_data = m->m_data + off;
4988 n->m_flags |= M_EXT;
4989 } else {
4990 /*
4991 * Limit to the capacity of the destination
4992 */
4993 if (n->m_flags & M_PKTHDR)
4994 n->m_len = MIN(n->m_len, MHLEN);
4995 else
4996 n->m_len = MIN(n->m_len, MLEN);
4997
4998 if (MTOD(n, char *) + n->m_len > ((char *)n) + MSIZE)
4999 panic("%s n %p copy overflow",
5000 __func__, n);
5001
5002 bcopy(MTOD(m, caddr_t)+off, MTOD(n, caddr_t),
5003 (unsigned)n->m_len);
5004 }
5005 if (len != M_COPYALL)
5006 len -= n->m_len;
5007 off = 0;
5008 m = m->m_next;
5009 np = &n->m_next;
5010 }
5011
5012 if (top == NULL)
5013 MCFail++;
5014
5015 return (top);
5016nospace:
5017
5018 m_freem(top);
5019 MCFail++;
5020 return (NULL);
5021}
5022
5023
5024struct mbuf *
5025m_copym(struct mbuf *m, int off0, int len, int wait)
5026{
5027 return (m_copym_mode(m, off0, len, wait, M_COPYM_MOVE_HDR));
5028}
5029
5030/*
5031 * Equivalent to m_copym except that all necessary mbuf hdrs are allocated
5032 * within this routine also, the last mbuf and offset accessed are passed
5033 * out and can be passed back in to avoid having to rescan the entire mbuf
5034 * list (normally hung off of the socket)
5035 */
5036struct mbuf *
5037m_copym_with_hdrs(struct mbuf *m0, int off0, int len0, int wait,
5038 struct mbuf **m_lastm, int *m_off, uint32_t mode)
5039{
5040 struct mbuf *m = m0, *n, **np = NULL;
5041 int off = off0, len = len0;
5042 struct mbuf *top = NULL;
5043 int mcflags = MSLEEPF(wait);
5044 int copyhdr = 0;
5045 int type = 0;
5046 mcache_obj_t *list = NULL;
5047 int needed = 0;
5048
5049 if (off == 0 && (m->m_flags & M_PKTHDR))
5050 copyhdr = 1;
5051
5052 if (m_lastm != NULL && *m_lastm != NULL) {
5053 m = *m_lastm;
5054 off = *m_off;
5055 } else {
5056 while (off >= m->m_len) {
5057 off -= m->m_len;
5058 m = m->m_next;
5059 }
5060 }
5061
5062 n = m;
5063 while (len > 0) {
5064 needed++;
5065 ASSERT(n != NULL);
5066 len -= MIN(len, (n->m_len - ((needed == 1) ? off : 0)));
5067 n = n->m_next;
5068 }
5069 needed++;
5070 len = len0;
5071
5072 /*
5073 * If the caller doesn't want to be put to sleep, mark it with
5074 * MCR_TRYHARD so that we may reclaim buffers from other places
5075 * before giving up.
5076 */
5077 if (mcflags & MCR_NOSLEEP)
5078 mcflags |= MCR_TRYHARD;
5079
5080 if (mcache_alloc_ext(m_cache(MC_MBUF), &list, needed,
5081 mcflags) != needed)
5082 goto nospace;
5083
5084 needed = 0;
5085 while (len > 0) {
5086 n = (struct mbuf *)list;
5087 list = list->obj_next;
5088 ASSERT(n != NULL && m != NULL);
5089
5090 type = (top == NULL) ? MT_HEADER : m->m_type;
5091 MBUF_INIT(n, (top == NULL), type);
5092#if CONFIG_MACF_NET
5093 if (top == NULL && mac_mbuf_label_init(n, wait) != 0) {
5094 mtype_stat_inc(MT_HEADER);
5095 mtype_stat_dec(MT_FREE);
5096 m_free(n);
5097 goto nospace;
5098 }
5099#endif /* MAC_NET */
5100
5101 if (top == NULL) {
5102 top = n;
5103 np = &top->m_next;
5104 continue;
5105 } else {
5106 needed++;
5107 *np = n;
5108 }
5109
5110 if (copyhdr) {
5111 if ((mode == M_COPYM_MOVE_HDR) ||
5112 (mode == M_COPYM_MUST_MOVE_HDR)) {
5113 M_COPY_PKTHDR(n, m);
5114 } else if ((mode == M_COPYM_COPY_HDR) ||
5115 (mode == M_COPYM_MUST_COPY_HDR)) {
5116 if (m_dup_pkthdr(n, m, wait) == 0)
5117 goto nospace;
5118 }
5119 n->m_pkthdr.len = len;
5120 copyhdr = 0;
5121 }
5122 n->m_len = MIN(len, (m->m_len - off));
5123
5124 if (m->m_flags & M_EXT) {
5125 n->m_ext = m->m_ext;
5126 m_incref(m);
5127 n->m_data = m->m_data + off;
5128 n->m_flags |= M_EXT;
5129 } else {
5130 if (MTOD(n, char *) + n->m_len > ((char *)n) + MSIZE)
5131 panic("%s n %p copy overflow",
5132 __func__, n);
5133
5134 bcopy(MTOD(m, caddr_t)+off, MTOD(n, caddr_t),
5135 (unsigned)n->m_len);
5136 }
5137 len -= n->m_len;
5138
5139 if (len == 0) {
5140 if (m_lastm != NULL && m_off != NULL) {
5141 if ((off + n->m_len) == m->m_len) {
5142 *m_lastm = m->m_next;
5143 *m_off = 0;
5144 } else {
5145 *m_lastm = m;
5146 *m_off = off + n->m_len;
5147 }
5148 }
5149 break;
5150 }
5151 off = 0;
5152 m = m->m_next;
5153 np = &n->m_next;
5154 }
5155
5156 mtype_stat_inc(MT_HEADER);
5157 mtype_stat_add(type, needed);
5158 mtype_stat_sub(MT_FREE, needed + 1);
5159
5160 ASSERT(list == NULL);
5161 return (top);
5162
5163nospace:
5164 if (list != NULL)
5165 mcache_free_ext(m_cache(MC_MBUF), list);
5166 if (top != NULL)
5167 m_freem(top);
5168 MCFail++;
5169 return (NULL);
5170}
5171
5172/*
5173 * Copy data from an mbuf chain starting "off" bytes from the beginning,
5174 * continuing for "len" bytes, into the indicated buffer.
5175 */
5176void
5177m_copydata(struct mbuf *m, int off, int len, void *vp)
5178{
5179 int off0 = off, len0 = len;
5180 struct mbuf *m0 = m;
5181 unsigned count;
5182 char *cp = vp;
5183
5184 if (__improbable(off < 0 || len < 0)) {
5185 panic("%s: invalid offset %d or len %d", __func__, off, len);
5186 /* NOTREACHED */
5187 }
5188
5189 while (off > 0) {
5190 if (__improbable(m == NULL)) {
5191 panic("%s: invalid mbuf chain %p [off %d, len %d]",
5192 __func__, m0, off0, len0);
5193 /* NOTREACHED */
5194 }
5195 if (off < m->m_len)
5196 break;
5197 off -= m->m_len;
5198 m = m->m_next;
5199 }
5200 while (len > 0) {
5201 if (__improbable(m == NULL)) {
5202 panic("%s: invalid mbuf chain %p [off %d, len %d]",
5203 __func__, m0, off0, len0);
5204 /* NOTREACHED */
5205 }
5206 count = MIN(m->m_len - off, len);
5207 bcopy(MTOD(m, caddr_t) + off, cp, count);
5208 len -= count;
5209 cp += count;
5210 off = 0;
5211 m = m->m_next;
5212 }
5213}
5214
5215/*
5216 * Concatenate mbuf chain n to m. Both chains must be of the same type
5217 * (e.g. MT_DATA). Any m_pkthdr is not updated.
5218 */
5219void
5220m_cat(struct mbuf *m, struct mbuf *n)
5221{
5222 while (m->m_next)
5223 m = m->m_next;
5224 while (n) {
5225 if ((m->m_flags & M_EXT) ||
5226 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
5227 /* just join the two chains */
5228 m->m_next = n;
5229 return;
5230 }
5231 /* splat the data from one into the other */
5232 bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
5233 (u_int)n->m_len);
5234 m->m_len += n->m_len;
5235 n = m_free(n);
5236 }
5237}
5238
5239void
5240m_adj(struct mbuf *mp, int req_len)
5241{
5242 int len = req_len;
5243 struct mbuf *m;
5244 int count;
5245
5246 if ((m = mp) == NULL)
5247 return;
5248 if (len >= 0) {
5249 /*
5250 * Trim from head.
5251 */
5252 while (m != NULL && len > 0) {
5253 if (m->m_len <= len) {
5254 len -= m->m_len;
5255 m->m_len = 0;
5256 m = m->m_next;
5257 } else {
5258 m->m_len -= len;
5259 m->m_data += len;
5260 len = 0;
5261 }
5262 }
5263 m = mp;
5264 if (m->m_flags & M_PKTHDR)
5265 m->m_pkthdr.len -= (req_len - len);
5266 } else {
5267 /*
5268 * Trim from tail. Scan the mbuf chain,
5269 * calculating its length and finding the last mbuf.
5270 * If the adjustment only affects this mbuf, then just
5271 * adjust and return. Otherwise, rescan and truncate
5272 * after the remaining size.
5273 */
5274 len = -len;
5275 count = 0;
5276 for (;;) {
5277 count += m->m_len;
5278 if (m->m_next == (struct mbuf *)0)
5279 break;
5280 m = m->m_next;
5281 }
5282 if (m->m_len >= len) {
5283 m->m_len -= len;
5284 m = mp;
5285 if (m->m_flags & M_PKTHDR)
5286 m->m_pkthdr.len -= len;
5287 return;
5288 }
5289 count -= len;
5290 if (count < 0)
5291 count = 0;
5292 /*
5293 * Correct length for chain is "count".
5294 * Find the mbuf with last data, adjust its length,
5295 * and toss data from remaining mbufs on chain.
5296 */
5297 m = mp;
5298 if (m->m_flags & M_PKTHDR)
5299 m->m_pkthdr.len = count;
5300 for (; m; m = m->m_next) {
5301 if (m->m_len >= count) {
5302 m->m_len = count;
5303 break;
5304 }
5305 count -= m->m_len;
5306 }
5307 while ((m = m->m_next))
5308 m->m_len = 0;
5309 }
5310}
5311
5312/*
5313 * Rearange an mbuf chain so that len bytes are contiguous
5314 * and in the data area of an mbuf (so that mtod and dtom
5315 * will work for a structure of size len). Returns the resulting
5316 * mbuf chain on success, frees it and returns null on failure.
5317 * If there is room, it will add up to max_protohdr-len extra bytes to the
5318 * contiguous region in an attempt to avoid being called next time.
5319 */
5320int MPFail;
5321
5322struct mbuf *
5323m_pullup(struct mbuf *n, int len)
5324{
5325 struct mbuf *m;
5326 int count;
5327 int space;
5328
5329 /* check invalid arguments */
5330 if (n == NULL) {
5331 panic("%s: n == NULL", __func__);
5332 }
5333 if (len < 0) {
5334 os_log_info(OS_LOG_DEFAULT, "%s: failed negative len %d",
5335 __func__, len);
5336 goto bad;
5337 }
5338 if (len > MLEN) {
5339 os_log_info(OS_LOG_DEFAULT, "%s: failed len %d too big",
5340 __func__, len);
5341 goto bad;
5342 }
5343 if ((n->m_flags & M_EXT) == 0 &&
5344 n->m_data >= &n->m_dat[MLEN]) {
5345 os_log_info(OS_LOG_DEFAULT, "%s: m_data out of bounds",
5346 __func__);
5347 goto bad;
5348 }
5349
5350 /*
5351 * If first mbuf has no cluster, and has room for len bytes
5352 * without shifting current data, pullup into it,
5353 * otherwise allocate a new mbuf to prepend to the chain.
5354 */
5355 if ((n->m_flags & M_EXT) == 0 &&
5356 len < &n->m_dat[MLEN] - n->m_data && n->m_next != NULL) {
5357 if (n->m_len >= len)
5358 return (n);
5359 m = n;
5360 n = n->m_next;
5361 len -= m->m_len;
5362 } else {
5363 if (len > MHLEN)
5364 goto bad;
5365 _MGET(m, M_DONTWAIT, n->m_type);
5366 if (m == 0)
5367 goto bad;
5368 m->m_len = 0;
5369 if (n->m_flags & M_PKTHDR) {
5370 M_COPY_PKTHDR(m, n);
5371 n->m_flags &= ~M_PKTHDR;
5372 }
5373 }
5374 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
5375 do {
5376 count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len);
5377 bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
5378 (unsigned)count);
5379 len -= count;
5380 m->m_len += count;
5381 n->m_len -= count;
5382 space -= count;
5383 if (n->m_len != 0)
5384 n->m_data += count;
5385 else
5386 n = m_free(n);
5387 } while (len > 0 && n != NULL);
5388 if (len > 0) {
5389 (void) m_free(m);
5390 goto bad;
5391 }
5392 m->m_next = n;
5393 return (m);
5394bad:
5395 m_freem(n);
5396 MPFail++;
5397 return (0);
5398}
5399
5400/*
5401 * Like m_pullup(), except a new mbuf is always allocated, and we allow
5402 * the amount of empty space before the data in the new mbuf to be specified
5403 * (in the event that the caller expects to prepend later).
5404 */
5405__private_extern__ int MSFail = 0;
5406
5407__private_extern__ struct mbuf *
5408m_copyup(struct mbuf *n, int len, int dstoff)
5409{
5410 struct mbuf *m;
5411 int count, space;
5412
5413 if (len > (MHLEN - dstoff))
5414 goto bad;
5415 MGET(m, M_DONTWAIT, n->m_type);
5416 if (m == NULL)
5417 goto bad;
5418 m->m_len = 0;
5419 if (n->m_flags & M_PKTHDR) {
5420 m_copy_pkthdr(m, n);
5421 n->m_flags &= ~M_PKTHDR;
5422 }
5423 m->m_data += dstoff;
5424 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
5425 do {
5426 count = min(min(max(len, max_protohdr), space), n->m_len);
5427 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
5428 (unsigned)count);
5429 len -= count;
5430 m->m_len += count;
5431 n->m_len -= count;
5432 space -= count;
5433 if (n->m_len)
5434 n->m_data += count;
5435 else
5436 n = m_free(n);
5437 } while (len > 0 && n);
5438 if (len > 0) {
5439 (void) m_free(m);
5440 goto bad;
5441 }
5442 m->m_next = n;
5443 return (m);
5444bad:
5445 m_freem(n);
5446 MSFail++;
5447 return (NULL);
5448}
5449
5450/*
5451 * Partition an mbuf chain in two pieces, returning the tail --
5452 * all but the first len0 bytes. In case of failure, it returns NULL and
5453 * attempts to restore the chain to its original state.
5454 */
5455struct mbuf *
5456m_split(struct mbuf *m0, int len0, int wait)
5457{
5458 return (m_split0(m0, len0, wait, 1));
5459}
5460
5461static struct mbuf *
5462m_split0(struct mbuf *m0, int len0, int wait, int copyhdr)
5463{
5464 struct mbuf *m, *n;
5465 unsigned len = len0, remain;
5466
5467 /*
5468 * First iterate to the mbuf which contains the first byte of
5469 * data at offset len0
5470 */
5471 for (m = m0; m && len > m->m_len; m = m->m_next)
5472 len -= m->m_len;
5473 if (m == NULL)
5474 return (NULL);
5475 /*
5476 * len effectively is now the offset in the current
5477 * mbuf where we have to perform split.
5478 *
5479 * remain becomes the tail length.
5480 * Note that len can also be == m->m_len
5481 */
5482 remain = m->m_len - len;
5483
5484 /*
5485 * If current mbuf len contains the entire remaining offset len,
5486 * just make the second mbuf chain pointing to next mbuf onwards
5487 * and return after making necessary adjustments
5488 */
5489 if (copyhdr && (m0->m_flags & M_PKTHDR) && remain == 0) {
5490 _MGETHDR(n, wait, m0->m_type);
5491 if (n == NULL)
5492 return (NULL);
5493 n->m_next = m->m_next;
5494 m->m_next = NULL;
5495 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
5496 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
5497 m0->m_pkthdr.len = len0;
5498 return (n);
5499 } if (copyhdr && (m0->m_flags & M_PKTHDR)) {
5500 _MGETHDR(n, wait, m0->m_type);
5501 if (n == NULL)
5502 return (NULL);
5503 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
5504 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
5505 m0->m_pkthdr.len = len0;
5506
5507 /*
5508 * If current points to external storage
5509 * then it can be shared by making last mbuf
5510 * of head chain and first mbuf of current chain
5511 * pointing to different data offsets
5512 */
5513 if (m->m_flags & M_EXT)
5514 goto extpacket;
5515 if (remain > MHLEN) {
5516 /* m can't be the lead packet */
5517 MH_ALIGN(n, 0);
5518 n->m_next = m_split(m, len, wait);
5519 if (n->m_next == NULL) {
5520 (void) m_free(n);
5521 return (NULL);
5522 } else
5523 return (n);
5524 } else
5525 MH_ALIGN(n, remain);
5526 } else if (remain == 0) {
5527 n = m->m_next;
5528 m->m_next = NULL;
5529 return (n);
5530 } else {
5531 _MGET(n, wait, m->m_type);
5532 if (n == NULL)
5533 return (NULL);
5534
5535 if ((m->m_flags & M_EXT) == 0) {
5536 VERIFY(remain <= MLEN);
5537 M_ALIGN(n, remain);
5538 }
5539 }
5540extpacket:
5541 if (m->m_flags & M_EXT) {
5542 n->m_flags |= M_EXT;
5543 n->m_ext = m->m_ext;
5544 m_incref(m);
5545 n->m_data = m->m_data + len;
5546 } else {
5547 bcopy(MTOD(m, caddr_t) + len, MTOD(n, caddr_t), remain);
5548 }
5549 n->m_len = remain;
5550 m->m_len = len;
5551 n->m_next = m->m_next;
5552 m->m_next = NULL;
5553 return (n);
5554}
5555
5556/*
5557 * Routine to copy from device local memory into mbufs.
5558 */
5559struct mbuf *
5560m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
5561 void (*copy)(const void *, void *, size_t))
5562{
5563 struct mbuf *m;
5564 struct mbuf *top = NULL, **mp = &top;
5565 int off = off0, len;
5566 char *cp;
5567 char *epkt;
5568
5569 cp = buf;
5570 epkt = cp + totlen;
5571 if (off) {
5572 /*
5573 * If 'off' is non-zero, packet is trailer-encapsulated,
5574 * so we have to skip the type and length fields.
5575 */
5576 cp += off + 2 * sizeof (u_int16_t);
5577 totlen -= 2 * sizeof (u_int16_t);
5578 }
5579 _MGETHDR(m, M_DONTWAIT, MT_DATA);
5580 if (m == NULL)
5581 return (NULL);
5582 m->m_pkthdr.rcvif = ifp;
5583 m->m_pkthdr.len = totlen;
5584 m->m_len = MHLEN;
5585
5586 while (totlen > 0) {
5587 if (top != NULL) {
5588 _MGET(m, M_DONTWAIT, MT_DATA);
5589 if (m == NULL) {
5590 m_freem(top);
5591 return (NULL);
5592 }
5593 m->m_len = MLEN;
5594 }
5595 len = MIN(totlen, epkt - cp);
5596 if (len >= MINCLSIZE) {
5597 MCLGET(m, M_DONTWAIT);
5598 if (m->m_flags & M_EXT) {
5599 m->m_len = len = MIN(len, m_maxsize(MC_CL));
5600 } else {
5601 /* give up when it's out of cluster mbufs */
5602 if (top != NULL)
5603 m_freem(top);
5604 m_freem(m);
5605 return (NULL);
5606 }
5607 } else {
5608 /*
5609 * Place initial small packet/header at end of mbuf.
5610 */
5611 if (len < m->m_len) {
5612 if (top == NULL &&
5613 len + max_linkhdr <= m->m_len)
5614 m->m_data += max_linkhdr;
5615 m->m_len = len;
5616 } else {
5617 len = m->m_len;
5618 }
5619 }
5620 if (copy)
5621 copy(cp, MTOD(m, caddr_t), (unsigned)len);
5622 else
5623 bcopy(cp, MTOD(m, caddr_t), (unsigned)len);
5624 cp += len;
5625 *mp = m;
5626 mp = &m->m_next;
5627 totlen -= len;
5628 if (cp == epkt)
5629 cp = buf;
5630 }
5631 return (top);
5632}
5633
5634#ifndef MBUF_GROWTH_NORMAL_THRESH
5635#define MBUF_GROWTH_NORMAL_THRESH 25
5636#endif
5637
5638/*
5639 * Cluster freelist allocation check.
5640 */
5641static int
5642m_howmany(int num, size_t bufsize)
5643{
5644 int i = 0, j = 0;
5645 u_int32_t m_mbclusters, m_clusters, m_bigclusters, m_16kclusters;
5646 u_int32_t m_mbfree, m_clfree, m_bigclfree, m_16kclfree;
5647 u_int32_t sumclusters, freeclusters;
5648 u_int32_t percent_pool, percent_kmem;
5649 u_int32_t mb_growth, mb_growth_thresh;
5650
5651 VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
5652 bufsize == m_maxsize(MC_16KCL));
5653
5654 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
5655
5656 /* Numbers in 2K cluster units */
5657 m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT;
5658 m_clusters = m_total(MC_CL);
5659 m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT;
5660 m_16kclusters = m_total(MC_16KCL);
5661 sumclusters = m_mbclusters + m_clusters + m_bigclusters;
5662
5663 m_mbfree = m_infree(MC_MBUF) >> NMBPCLSHIFT;
5664 m_clfree = m_infree(MC_CL);
5665 m_bigclfree = m_infree(MC_BIGCL) << NCLPBGSHIFT;
5666 m_16kclfree = m_infree(MC_16KCL);
5667 freeclusters = m_mbfree + m_clfree + m_bigclfree;
5668
5669 /* Bail if we've maxed out the mbuf memory map */
5670 if ((bufsize == m_maxsize(MC_BIGCL) && sumclusters >= nclusters) ||
5671 (njcl > 0 && bufsize == m_maxsize(MC_16KCL) &&
5672 (m_16kclusters << NCLPJCLSHIFT) >= njcl)) {
5673 mbwdog_logger("maxed out nclusters (%u >= %u) or njcl (%u >= %u)",
5674 sumclusters, nclusters,
5675 (m_16kclusters << NCLPJCLSHIFT), njcl);
5676 return (0);
5677 }
5678
5679 if (bufsize == m_maxsize(MC_BIGCL)) {
5680 /* Under minimum */
5681 if (m_bigclusters < m_minlimit(MC_BIGCL))
5682 return (m_minlimit(MC_BIGCL) - m_bigclusters);
5683
5684 percent_pool =
5685 ((sumclusters - freeclusters) * 100) / sumclusters;
5686 percent_kmem = (sumclusters * 100) / nclusters;
5687
5688 /*
5689 * If a light/normal user, grow conservatively (75%)
5690 * If a heavy user, grow aggressively (50%)
5691 */
5692 if (percent_kmem < MBUF_GROWTH_NORMAL_THRESH)
5693 mb_growth = MB_GROWTH_NORMAL;
5694 else
5695 mb_growth = MB_GROWTH_AGGRESSIVE;
5696
5697 if (percent_kmem < 5) {
5698 /* For initial allocations */
5699 i = num;
5700 } else {
5701 /* Return if >= MBIGCL_LOWAT clusters available */
5702 if (m_infree(MC_BIGCL) >= MBIGCL_LOWAT &&
5703 m_total(MC_BIGCL) >=
5704 MBIGCL_LOWAT + m_minlimit(MC_BIGCL))
5705 return (0);
5706
5707 /* Ensure at least num clusters are accessible */
5708 if (num >= m_infree(MC_BIGCL))
5709 i = num - m_infree(MC_BIGCL);
5710 if (num > m_total(MC_BIGCL) - m_minlimit(MC_BIGCL))
5711 j = num - (m_total(MC_BIGCL) -
5712 m_minlimit(MC_BIGCL));
5713
5714 i = MAX(i, j);
5715
5716 /*
5717 * Grow pool if percent_pool > 75 (normal growth)
5718 * or percent_pool > 50 (aggressive growth).
5719 */
5720 mb_growth_thresh = 100 - (100 / (1 << mb_growth));
5721 if (percent_pool > mb_growth_thresh)
5722 j = ((sumclusters + num) >> mb_growth) -
5723 freeclusters;
5724 i = MAX(i, j);
5725 }
5726
5727 /* Check to ensure we didn't go over limits */
5728 if (i + m_bigclusters >= m_maxlimit(MC_BIGCL))
5729 i = m_maxlimit(MC_BIGCL) - m_bigclusters;
5730 if ((i << 1) + sumclusters >= nclusters)
5731 i = (nclusters - sumclusters) >> 1;
5732 VERIFY((m_total(MC_BIGCL) + i) <= m_maxlimit(MC_BIGCL));
5733 VERIFY(sumclusters + (i << 1) <= nclusters);
5734
5735 } else { /* 16K CL */
5736 VERIFY(njcl > 0);
5737 /* Ensure at least num clusters are available */
5738 if (num >= m_16kclfree)
5739 i = num - m_16kclfree;
5740
5741 /* Always grow 16KCL pool aggressively */
5742 if (((m_16kclusters + num) >> 1) > m_16kclfree)
5743 j = ((m_16kclusters + num) >> 1) - m_16kclfree;
5744 i = MAX(i, j);
5745
5746 /* Check to ensure we don't go over limit */
5747 if ((i + m_total(MC_16KCL)) >= m_maxlimit(MC_16KCL))
5748 i = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
5749 }
5750 return (i);
5751}
5752/*
5753 * Return the number of bytes in the mbuf chain, m.
5754 */
5755unsigned int
5756m_length(struct mbuf *m)
5757{
5758 struct mbuf *m0;
5759 unsigned int pktlen;
5760
5761 if (m->m_flags & M_PKTHDR)
5762 return (m->m_pkthdr.len);
5763
5764 pktlen = 0;
5765 for (m0 = m; m0 != NULL; m0 = m0->m_next)
5766 pktlen += m0->m_len;
5767 return (pktlen);
5768}
5769
5770/*
5771 * Copy data from a buffer back into the indicated mbuf chain,
5772 * starting "off" bytes from the beginning, extending the mbuf
5773 * chain if necessary.
5774 */
5775void
5776m_copyback(struct mbuf *m0, int off, int len, const void *cp)
5777{
5778#if DEBUG
5779 struct mbuf *origm = m0;
5780 int error;
5781#endif /* DEBUG */
5782
5783 if (m0 == NULL)
5784 return;
5785
5786#if DEBUG
5787 error =
5788#endif /* DEBUG */
5789 m_copyback0(&m0, off, len, cp,
5790 M_COPYBACK0_COPYBACK | M_COPYBACK0_EXTEND, M_DONTWAIT);
5791
5792#if DEBUG
5793 if (error != 0 || (m0 != NULL && origm != m0))
5794 panic("m_copyback");
5795#endif /* DEBUG */
5796}
5797
5798struct mbuf *
5799m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how)
5800{
5801 int error;
5802
5803 /* don't support chain expansion */
5804 VERIFY(off + len <= m_length(m0));
5805
5806 error = m_copyback0(&m0, off, len, cp,
5807 M_COPYBACK0_COPYBACK | M_COPYBACK0_COW, how);
5808 if (error) {
5809 /*
5810 * no way to recover from partial success.
5811 * just free the chain.
5812 */
5813 m_freem(m0);
5814 return (NULL);
5815 }
5816 return (m0);
5817}
5818
5819/*
5820 * m_makewritable: ensure the specified range writable.
5821 */
5822int
5823m_makewritable(struct mbuf **mp, int off, int len, int how)
5824{
5825 int error;
5826#if DEBUG
5827 struct mbuf *n;
5828 int origlen, reslen;
5829
5830 origlen = m_length(*mp);
5831#endif /* DEBUG */
5832
5833#if 0 /* M_COPYALL is large enough */
5834 if (len == M_COPYALL)
5835 len = m_length(*mp) - off; /* XXX */
5836#endif
5837
5838 error = m_copyback0(mp, off, len, NULL,
5839 M_COPYBACK0_PRESERVE | M_COPYBACK0_COW, how);
5840
5841#if DEBUG
5842 reslen = 0;
5843 for (n = *mp; n; n = n->m_next)
5844 reslen += n->m_len;
5845 if (origlen != reslen)
5846 panic("m_makewritable: length changed");
5847 if (((*mp)->m_flags & M_PKTHDR) && reslen != (*mp)->m_pkthdr.len)
5848 panic("m_makewritable: inconsist");
5849#endif /* DEBUG */
5850
5851 return (error);
5852}
5853
5854static int
5855m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags,
5856 int how)
5857{
5858 int mlen;
5859 struct mbuf *m, *n;
5860 struct mbuf **mp;
5861 int totlen = 0;
5862 const char *cp = vp;
5863
5864 VERIFY(mp0 != NULL);
5865 VERIFY(*mp0 != NULL);
5866 VERIFY((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL);
5867 VERIFY((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL);
5868
5869 /*
5870 * we don't bother to update "totlen" in the case of M_COPYBACK0_COW,
5871 * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive.
5872 */
5873
5874 VERIFY((~flags & (M_COPYBACK0_EXTEND|M_COPYBACK0_COW)) != 0);
5875
5876 mp = mp0;
5877 m = *mp;
5878 while (off > (mlen = m->m_len)) {
5879 off -= mlen;
5880 totlen += mlen;
5881 if (m->m_next == NULL) {
5882 int tspace;
5883extend:
5884 if (!(flags & M_COPYBACK0_EXTEND))
5885 goto out;
5886
5887 /*
5888 * try to make some space at the end of "m".
5889 */
5890
5891 mlen = m->m_len;
5892 if (off + len >= MINCLSIZE &&
5893 !(m->m_flags & M_EXT) && m->m_len == 0) {
5894 MCLGET(m, how);
5895 }
5896 tspace = M_TRAILINGSPACE(m);
5897 if (tspace > 0) {
5898 tspace = MIN(tspace, off + len);
5899 VERIFY(tspace > 0);
5900 bzero(mtod(m, char *) + m->m_len,
5901 MIN(off, tspace));
5902 m->m_len += tspace;
5903 off += mlen;
5904 totlen -= mlen;
5905 continue;
5906 }
5907
5908 /*
5909 * need to allocate an mbuf.
5910 */
5911
5912 if (off + len >= MINCLSIZE) {
5913 n = m_getcl(how, m->m_type, 0);
5914 } else {
5915 n = _M_GET(how, m->m_type);
5916 }
5917 if (n == NULL) {
5918 goto out;
5919 }
5920 n->m_len = 0;
5921 n->m_len = MIN(M_TRAILINGSPACE(n), off + len);
5922 bzero(mtod(n, char *), MIN(n->m_len, off));
5923 m->m_next = n;
5924 }
5925 mp = &m->m_next;
5926 m = m->m_next;
5927 }
5928 while (len > 0) {
5929 mlen = m->m_len - off;
5930 if (mlen != 0 && m_mclhasreference(m)) {
5931 char *datap;
5932 int eatlen;
5933
5934 /*
5935 * this mbuf is read-only.
5936 * allocate a new writable mbuf and try again.
5937 */
5938
5939#if DIAGNOSTIC
5940 if (!(flags & M_COPYBACK0_COW))
5941 panic("m_copyback0: read-only");
5942#endif /* DIAGNOSTIC */
5943
5944 /*
5945 * if we're going to write into the middle of
5946 * a mbuf, split it first.
5947 */
5948 if (off > 0 && len < mlen) {
5949 n = m_split0(m, off, how, 0);
5950 if (n == NULL)
5951 goto enobufs;
5952 m->m_next = n;
5953 mp = &m->m_next;
5954 m = n;
5955 off = 0;
5956 continue;
5957 }
5958
5959 /*
5960 * XXX TODO coalesce into the trailingspace of
5961 * the previous mbuf when possible.
5962 */
5963
5964 /*
5965 * allocate a new mbuf. copy packet header if needed.
5966 */
5967 n = _M_GET(how, m->m_type);
5968 if (n == NULL)
5969 goto enobufs;
5970 if (off == 0 && (m->m_flags & M_PKTHDR)) {
5971 M_COPY_PKTHDR(n, m);
5972 n->m_len = MHLEN;
5973 } else {
5974 if (len >= MINCLSIZE)
5975 MCLGET(n, M_DONTWAIT);
5976 n->m_len =
5977 (n->m_flags & M_EXT) ? MCLBYTES : MLEN;
5978 }
5979 if (n->m_len > len)
5980 n->m_len = len;
5981
5982 /*
5983 * free the region which has been overwritten.
5984 * copying data from old mbufs if requested.
5985 */
5986 if (flags & M_COPYBACK0_PRESERVE)
5987 datap = mtod(n, char *);
5988 else
5989 datap = NULL;
5990 eatlen = n->m_len;
5991 VERIFY(off == 0 || eatlen >= mlen);
5992 if (off > 0) {
5993 VERIFY(len >= mlen);
5994 m->m_len = off;
5995 m->m_next = n;
5996 if (datap) {
5997 m_copydata(m, off, mlen, datap);
5998 datap += mlen;
5999 }
6000 eatlen -= mlen;
6001 mp = &m->m_next;
6002 m = m->m_next;
6003 }
6004 while (m != NULL && m_mclhasreference(m) &&
6005 n->m_type == m->m_type && eatlen > 0) {
6006 mlen = MIN(eatlen, m->m_len);
6007 if (datap) {
6008 m_copydata(m, 0, mlen, datap);
6009 datap += mlen;
6010 }
6011 m->m_data += mlen;
6012 m->m_len -= mlen;
6013 eatlen -= mlen;
6014 if (m->m_len == 0)
6015 *mp = m = m_free(m);
6016 }
6017 if (eatlen > 0)
6018 n->m_len -= eatlen;
6019 n->m_next = m;
6020 *mp = m = n;
6021 continue;
6022 }
6023 mlen = MIN(mlen, len);
6024 if (flags & M_COPYBACK0_COPYBACK) {
6025 bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen);
6026 cp += mlen;
6027 }
6028 len -= mlen;
6029 mlen += off;
6030 off = 0;
6031 totlen += mlen;
6032 if (len == 0)
6033 break;
6034 if (m->m_next == NULL) {
6035 goto extend;
6036 }
6037 mp = &m->m_next;
6038 m = m->m_next;
6039 }
6040out:
6041 if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) {
6042 VERIFY(flags & M_COPYBACK0_EXTEND);
6043 m->m_pkthdr.len = totlen;
6044 }
6045
6046 return (0);
6047
6048enobufs:
6049 return (ENOBUFS);
6050}
6051
6052uint64_t
6053mcl_to_paddr(char *addr)
6054{
6055 vm_offset_t base_phys;
6056
6057 if (!MBUF_IN_MAP(addr))
6058 return (0);
6059 base_phys = mcl_paddr[atop_64(addr - (char *)mbutl)];
6060
6061 if (base_phys == 0)
6062 return (0);
6063 return ((uint64_t)(ptoa_64(base_phys) | ((uint64_t)addr & PAGE_MASK)));
6064}
6065
6066/*
6067 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
6068 * And really copy the thing. That way, we don't "precompute" checksums
6069 * for unsuspecting consumers. Assumption: m->m_nextpkt == 0. Trick: for
6070 * small packets, don't dup into a cluster. That way received packets
6071 * don't take up too much room in the sockbuf (cf. sbspace()).
6072 */
6073int MDFail;
6074
6075struct mbuf *
6076m_dup(struct mbuf *m, int how)
6077{
6078 struct mbuf *n, **np;
6079 struct mbuf *top;
6080 int copyhdr = 0;
6081
6082 np = &top;
6083 top = NULL;
6084 if (m->m_flags & M_PKTHDR)
6085 copyhdr = 1;
6086
6087 /*
6088 * Quick check: if we have one mbuf and its data fits in an
6089 * mbuf with packet header, just copy and go.
6090 */
6091 if (m->m_next == NULL) {
6092 /* Then just move the data into an mbuf and be done... */
6093 if (copyhdr) {
6094 if (m->m_pkthdr.len <= MHLEN && m->m_len <= MHLEN) {
6095 if ((n = _M_GETHDR(how, m->m_type)) == NULL)
6096 return (NULL);
6097 n->m_len = m->m_len;
6098 m_dup_pkthdr(n, m, how);
6099 bcopy(m->m_data, n->m_data, m->m_len);
6100 return (n);
6101 }
6102 } else if (m->m_len <= MLEN) {
6103 if ((n = _M_GET(how, m->m_type)) == NULL)
6104 return (NULL);
6105 bcopy(m->m_data, n->m_data, m->m_len);
6106 n->m_len = m->m_len;
6107 return (n);
6108 }
6109 }
6110 while (m != NULL) {
6111#if BLUE_DEBUG
6112 printf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
6113 m->m_data);
6114#endif
6115 if (copyhdr)
6116 n = _M_GETHDR(how, m->m_type);
6117 else
6118 n = _M_GET(how, m->m_type);
6119 if (n == NULL)
6120 goto nospace;
6121 if (m->m_flags & M_EXT) {
6122 if (m->m_len <= m_maxsize(MC_CL))
6123 MCLGET(n, how);
6124 else if (m->m_len <= m_maxsize(MC_BIGCL))
6125 n = m_mbigget(n, how);
6126 else if (m->m_len <= m_maxsize(MC_16KCL) && njcl > 0)
6127 n = m_m16kget(n, how);
6128 if (!(n->m_flags & M_EXT)) {
6129 (void) m_free(n);
6130 goto nospace;
6131 }
6132 }
6133 *np = n;
6134 if (copyhdr) {
6135 /* Don't use M_COPY_PKTHDR: preserve m_data */
6136 m_dup_pkthdr(n, m, how);
6137 copyhdr = 0;
6138 if (!(n->m_flags & M_EXT))
6139 n->m_data = n->m_pktdat;
6140 }
6141 n->m_len = m->m_len;
6142 /*
6143 * Get the dup on the same bdry as the original
6144 * Assume that the two mbufs have the same offset to data area
6145 * (up to word boundaries)
6146 */
6147 bcopy(MTOD(m, caddr_t), MTOD(n, caddr_t), (unsigned)n->m_len);
6148 m = m->m_next;
6149 np = &n->m_next;
6150#if BLUE_DEBUG
6151 printf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
6152 n->m_data);
6153#endif
6154 }
6155
6156 if (top == NULL)
6157 MDFail++;
6158 return (top);
6159
6160nospace:
6161 m_freem(top);
6162 MDFail++;
6163 return (NULL);
6164}
6165
6166#define MBUF_MULTIPAGES(m) \
6167 (((m)->m_flags & M_EXT) && \
6168 ((IS_P2ALIGNED((m)->m_data, PAGE_SIZE) \
6169 && (m)->m_len > PAGE_SIZE) || \
6170 (!IS_P2ALIGNED((m)->m_data, PAGE_SIZE) && \
6171 P2ROUNDUP((m)->m_data, PAGE_SIZE) < ((uintptr_t)(m)->m_data + (m)->m_len))))
6172
6173static struct mbuf *
6174m_expand(struct mbuf *m, struct mbuf **last)
6175{
6176 struct mbuf *top = NULL;
6177 struct mbuf **nm = &top;
6178 uintptr_t data0, data;
6179 unsigned int len0, len;
6180
6181 VERIFY(MBUF_MULTIPAGES(m));
6182 VERIFY(m->m_next == NULL);
6183 data0 = (uintptr_t)m->m_data;
6184 len0 = m->m_len;
6185 *last = top;
6186
6187 for (;;) {
6188 struct mbuf *n;
6189
6190 data = data0;
6191 if (IS_P2ALIGNED(data, PAGE_SIZE) && len0 > PAGE_SIZE)
6192 len = PAGE_SIZE;
6193 else if (!IS_P2ALIGNED(data, PAGE_SIZE) &&
6194 P2ROUNDUP(data, PAGE_SIZE) < (data + len0))
6195 len = P2ROUNDUP(data, PAGE_SIZE) - data;
6196 else
6197 len = len0;
6198
6199 VERIFY(len > 0);
6200 VERIFY(m->m_flags & M_EXT);
6201 m->m_data = (void *)data;
6202 m->m_len = len;
6203
6204 *nm = *last = m;
6205 nm = &m->m_next;
6206 m->m_next = NULL;
6207
6208 data0 += len;
6209 len0 -= len;
6210 if (len0 == 0)
6211 break;
6212
6213 n = _M_RETRY(M_DONTWAIT, MT_DATA);
6214 if (n == NULL) {
6215 m_freem(top);
6216 top = *last = NULL;
6217 break;
6218 }
6219
6220 n->m_ext = m->m_ext;
6221 m_incref(m);
6222 n->m_flags |= M_EXT;
6223 m = n;
6224 }
6225 return (top);
6226}
6227
6228struct mbuf *
6229m_normalize(struct mbuf *m)
6230{
6231 struct mbuf *top = NULL;
6232 struct mbuf **nm = &top;
6233 boolean_t expanded = FALSE;
6234
6235 while (m != NULL) {
6236 struct mbuf *n;
6237
6238 n = m->m_next;
6239 m->m_next = NULL;
6240
6241 /* Does the data cross one or more page boundaries? */
6242 if (MBUF_MULTIPAGES(m)) {
6243 struct mbuf *last;
6244 if ((m = m_expand(m, &last)) == NULL) {
6245 m_freem(n);
6246 m_freem(top);
6247 top = NULL;
6248 break;
6249 }
6250 *nm = m;
6251 nm = &last->m_next;
6252 expanded = TRUE;
6253 } else {
6254 *nm = m;
6255 nm = &m->m_next;
6256 }
6257 m = n;
6258 }
6259 if (expanded)
6260 atomic_add_32(&mb_normalized, 1);
6261 return (top);
6262}
6263
6264/*
6265 * Append the specified data to the indicated mbuf chain,
6266 * Extend the mbuf chain if the new data does not fit in
6267 * existing space.
6268 *
6269 * Return 1 if able to complete the job; otherwise 0.
6270 */
6271int
6272m_append(struct mbuf *m0, int len, caddr_t cp)
6273{
6274 struct mbuf *m, *n;
6275 int remainder, space;
6276
6277 for (m = m0; m->m_next != NULL; m = m->m_next)
6278 ;
6279 remainder = len;
6280 space = M_TRAILINGSPACE(m);
6281 if (space > 0) {
6282 /*
6283 * Copy into available space.
6284 */
6285 if (space > remainder)
6286 space = remainder;
6287 bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
6288 m->m_len += space;
6289 cp += space;
6290 remainder -= space;
6291 }
6292 while (remainder > 0) {
6293 /*
6294 * Allocate a new mbuf; could check space
6295 * and allocate a cluster instead.
6296 */
6297 n = m_get(M_WAITOK, m->m_type);
6298 if (n == NULL)
6299 break;
6300 n->m_len = min(MLEN, remainder);
6301 bcopy(cp, mtod(n, caddr_t), n->m_len);
6302 cp += n->m_len;
6303 remainder -= n->m_len;
6304 m->m_next = n;
6305 m = n;
6306 }
6307 if (m0->m_flags & M_PKTHDR)
6308 m0->m_pkthdr.len += len - remainder;
6309 return (remainder == 0);
6310}
6311
6312struct mbuf *
6313m_last(struct mbuf *m)
6314{
6315 while (m->m_next != NULL)
6316 m = m->m_next;
6317 return (m);
6318}
6319
6320unsigned int
6321m_fixhdr(struct mbuf *m0)
6322{
6323 u_int len;
6324
6325 VERIFY(m0->m_flags & M_PKTHDR);
6326
6327 len = m_length2(m0, NULL);
6328 m0->m_pkthdr.len = len;
6329 return (len);
6330}
6331
6332unsigned int
6333m_length2(struct mbuf *m0, struct mbuf **last)
6334{
6335 struct mbuf *m;
6336 u_int len;
6337
6338 len = 0;
6339 for (m = m0; m != NULL; m = m->m_next) {
6340 len += m->m_len;
6341 if (m->m_next == NULL)
6342 break;
6343 }
6344 if (last != NULL)
6345 *last = m;
6346 return (len);
6347}
6348
6349/*
6350 * Defragment a mbuf chain, returning the shortest possible chain of mbufs
6351 * and clusters. If allocation fails and this cannot be completed, NULL will
6352 * be returned, but the passed in chain will be unchanged. Upon success,
6353 * the original chain will be freed, and the new chain will be returned.
6354 *
6355 * If a non-packet header is passed in, the original mbuf (chain?) will
6356 * be returned unharmed.
6357 *
6358 * If offset is specfied, the first mbuf in the chain will have a leading
6359 * space of the amount stated by the "off" parameter.
6360 *
6361 * This routine requires that the m_pkthdr.header field of the original
6362 * mbuf chain is cleared by the caller.
6363 */
6364struct mbuf *
6365m_defrag_offset(struct mbuf *m0, u_int32_t off, int how)
6366{
6367 struct mbuf *m_new = NULL, *m_final = NULL;
6368 int progress = 0, length, pktlen;
6369
6370 if (!(m0->m_flags & M_PKTHDR))
6371 return (m0);
6372
6373 VERIFY(off < MHLEN);
6374 m_fixhdr(m0); /* Needed sanity check */
6375
6376 pktlen = m0->m_pkthdr.len + off;
6377 if (pktlen > MHLEN)
6378 m_final = m_getcl(how, MT_DATA, M_PKTHDR);
6379 else
6380 m_final = m_gethdr(how, MT_DATA);
6381
6382 if (m_final == NULL)
6383 goto nospace;
6384
6385 if (off > 0) {
6386 pktlen -= off;
6387 m_final->m_data += off;
6388 }
6389
6390 /*
6391 * Caller must have handled the contents pointed to by this
6392 * pointer before coming here, as otherwise it will point to
6393 * the original mbuf which will get freed upon success.
6394 */
6395 VERIFY(m0->m_pkthdr.pkt_hdr == NULL);
6396
6397 if (m_dup_pkthdr(m_final, m0, how) == 0)
6398 goto nospace;
6399
6400 m_new = m_final;
6401
6402 while (progress < pktlen) {
6403 length = pktlen - progress;
6404 if (length > MCLBYTES)
6405 length = MCLBYTES;
6406 length -= ((m_new == m_final) ? off : 0);
6407 if (length < 0)
6408 goto nospace;
6409
6410 if (m_new == NULL) {
6411 if (length > MLEN)
6412 m_new = m_getcl(how, MT_DATA, 0);
6413 else
6414 m_new = m_get(how, MT_DATA);
6415 if (m_new == NULL)
6416 goto nospace;
6417 }
6418
6419 m_copydata(m0, progress, length, mtod(m_new, caddr_t));
6420 progress += length;
6421 m_new->m_len = length;
6422 if (m_new != m_final)
6423 m_cat(m_final, m_new);
6424 m_new = NULL;
6425 }
6426 m_freem(m0);
6427 m0 = m_final;
6428 return (m0);
6429nospace:
6430 if (m_final)
6431 m_freem(m_final);
6432 return (NULL);
6433}
6434
6435struct mbuf *
6436m_defrag(struct mbuf *m0, int how)
6437{
6438 return (m_defrag_offset(m0, 0, how));
6439}
6440
6441void
6442m_mchtype(struct mbuf *m, int t)
6443{
6444 mtype_stat_inc(t);
6445 mtype_stat_dec(m->m_type);
6446 (m)->m_type = t;
6447}
6448
6449void *
6450m_mtod(struct mbuf *m)
6451{
6452 return (MTOD(m, void *));
6453}
6454
6455struct mbuf *
6456m_dtom(void *x)
6457{
6458 return ((struct mbuf *)((uintptr_t)(x) & ~(MSIZE-1)));
6459}
6460
6461void
6462m_mcheck(struct mbuf *m)
6463{
6464 _MCHECK(m);
6465}
6466
6467/*
6468 * Return a pointer to mbuf/offset of location in mbuf chain.
6469 */
6470struct mbuf *
6471m_getptr(struct mbuf *m, int loc, int *off)
6472{
6473
6474 while (loc >= 0) {
6475 /* Normal end of search. */
6476 if (m->m_len > loc) {
6477 *off = loc;
6478 return (m);
6479 } else {
6480 loc -= m->m_len;
6481 if (m->m_next == NULL) {
6482 if (loc == 0) {
6483 /* Point at the end of valid data. */
6484 *off = m->m_len;
6485 return (m);
6486 }
6487 return (NULL);
6488 }
6489 m = m->m_next;
6490 }
6491 }
6492 return (NULL);
6493}
6494
6495/*
6496 * Inform the corresponding mcache(s) that there's a waiter below.
6497 */
6498static void
6499mbuf_waiter_inc(mbuf_class_t class, boolean_t comp)
6500{
6501 mcache_waiter_inc(m_cache(class));
6502 if (comp) {
6503 if (class == MC_CL) {
6504 mcache_waiter_inc(m_cache(MC_MBUF_CL));
6505 } else if (class == MC_BIGCL) {
6506 mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
6507 } else if (class == MC_16KCL) {
6508 mcache_waiter_inc(m_cache(MC_MBUF_16KCL));
6509 } else {
6510 mcache_waiter_inc(m_cache(MC_MBUF_CL));
6511 mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
6512 }
6513 }
6514}
6515
6516/*
6517 * Inform the corresponding mcache(s) that there's no more waiter below.
6518 */
6519static void
6520mbuf_waiter_dec(mbuf_class_t class, boolean_t comp)
6521{
6522 mcache_waiter_dec(m_cache(class));
6523 if (comp) {
6524 if (class == MC_CL) {
6525 mcache_waiter_dec(m_cache(MC_MBUF_CL));
6526 } else if (class == MC_BIGCL) {
6527 mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
6528 } else if (class == MC_16KCL) {
6529 mcache_waiter_dec(m_cache(MC_MBUF_16KCL));
6530 } else {
6531 mcache_waiter_dec(m_cache(MC_MBUF_CL));
6532 mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
6533 }
6534 }
6535}
6536
6537/*
6538 * Called during slab (blocking and non-blocking) allocation. If there
6539 * is at least one waiter, and the time since the first waiter is blocked
6540 * is greater than the watchdog timeout, panic the system.
6541 */
6542static void
6543mbuf_watchdog(void)
6544{
6545 struct timeval now;
6546 unsigned int since;
6547
6548 if (mb_waiters == 0 || !mb_watchdog)
6549 return;
6550
6551 microuptime(&now);
6552 since = now.tv_sec - mb_wdtstart.tv_sec;
6553 if (since >= MB_WDT_MAXTIME) {
6554 panic_plain("%s: %d waiters stuck for %u secs\n%s", __func__,
6555 mb_waiters, since, mbuf_dump());
6556 /* NOTREACHED */
6557 }
6558}
6559
6560/*
6561 * Called during blocking allocation. Returns TRUE if one or more objects
6562 * are available at the per-CPU caches layer and that allocation should be
6563 * retried at that level.
6564 */
6565static boolean_t
6566mbuf_sleep(mbuf_class_t class, unsigned int num, int wait)
6567{
6568 boolean_t mcache_retry = FALSE;
6569
6570 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
6571
6572 /* Check if there's anything at the cache layer */
6573 if (mbuf_cached_above(class, wait)) {
6574 mcache_retry = TRUE;
6575 goto done;
6576 }
6577
6578 /* Nothing? Then try hard to get it from somewhere */
6579 m_reclaim(class, num, (wait & MCR_COMP));
6580
6581 /* We tried hard and got something? */
6582 if (m_infree(class) > 0) {
6583 mbstat.m_wait++;
6584 goto done;
6585 } else if (mbuf_cached_above(class, wait)) {
6586 mbstat.m_wait++;
6587 mcache_retry = TRUE;
6588 goto done;
6589 } else if (wait & MCR_TRYHARD) {
6590 mcache_retry = TRUE;
6591 goto done;
6592 }
6593
6594 /*
6595 * There's really nothing for us right now; inform the
6596 * cache(s) that there is a waiter below and go to sleep.
6597 */
6598 mbuf_waiter_inc(class, (wait & MCR_COMP));
6599
6600 VERIFY(!(wait & MCR_NOSLEEP));
6601
6602 /*
6603 * If this is the first waiter, arm the watchdog timer. Otherwise
6604 * check if we need to panic the system due to watchdog timeout.
6605 */
6606 if (mb_waiters == 0)
6607 microuptime(&mb_wdtstart);
6608 else
6609 mbuf_watchdog();
6610
6611 mb_waiters++;
6612 m_region_expand(class) += m_total(class) + num;
6613 /* wake up the worker thread */
6614 if (mbuf_worker_ready &&
6615 mbuf_worker_needs_wakeup) {
6616 wakeup((caddr_t)&mbuf_worker_needs_wakeup);
6617 mbuf_worker_needs_wakeup = FALSE;
6618 }
6619 mbwdog_logger("waiting (%d mbufs in class %s)", num, m_cname(class));
6620 (void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL);
6621 mbwdog_logger("woke up (%d mbufs in class %s) ", num, m_cname(class));
6622
6623 /* We are now up; stop getting notified until next round */
6624 mbuf_waiter_dec(class, (wait & MCR_COMP));
6625
6626 /* We waited and got something */
6627 if (m_infree(class) > 0) {
6628 mbstat.m_wait++;
6629 goto done;
6630 } else if (mbuf_cached_above(class, wait)) {
6631 mbstat.m_wait++;
6632 mcache_retry = TRUE;
6633 }
6634done:
6635 return (mcache_retry);
6636}
6637
6638__attribute__((noreturn))
6639static void
6640mbuf_worker_thread(void)
6641{
6642 int mbuf_expand;
6643
6644 while (1) {
6645 lck_mtx_lock(mbuf_mlock);
6646 mbwdog_logger("worker thread running");
6647 mbuf_worker_run_cnt++;
6648 mbuf_expand = 0;
6649 /*
6650 * Allocations are based on page size, so if we have depleted
6651 * the reserved spaces, try to free mbufs from the major classes.
6652 */
6653#if PAGE_SIZE == 4096
6654 uint32_t m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT;
6655 uint32_t m_clusters = m_total(MC_CL);
6656 uint32_t m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT;
6657 uint32_t sumclusters = m_mbclusters + m_clusters + m_bigclusters;
6658 if (sumclusters >= nclusters) {
6659 mbwdog_logger("reclaiming bigcl");
6660 mbuf_drain_locked(TRUE);
6661 m_reclaim(MC_BIGCL, 4, FALSE);
6662 }
6663#else
6664 uint32_t m_16kclusters = m_total(MC_16KCL);
6665 if (njcl > 0 && (m_16kclusters << NCLPJCLSHIFT) >= njcl) {
6666 mbwdog_logger("reclaiming 16kcl");
6667 mbuf_drain_locked(TRUE);
6668 m_reclaim(MC_16KCL, 4, FALSE);
6669 }
6670#endif
6671 if (m_region_expand(MC_CL) > 0) {
6672 int n;
6673 mb_expand_cl_cnt++;
6674 /* Adjust to current number of cluster in use */
6675 n = m_region_expand(MC_CL) -
6676 (m_total(MC_CL) - m_infree(MC_CL));
6677 if ((n + m_total(MC_CL)) > m_maxlimit(MC_CL))
6678 n = m_maxlimit(MC_CL) - m_total(MC_CL);
6679 if (n > 0) {
6680 mb_expand_cl_total += n;
6681 }
6682 m_region_expand(MC_CL) = 0;
6683
6684 if (n > 0) {
6685 mbwdog_logger("expanding MC_CL by %d", n);
6686 freelist_populate(MC_CL, n, M_WAIT);
6687 }
6688 }
6689 if (m_region_expand(MC_BIGCL) > 0) {
6690 int n;
6691 mb_expand_bigcl_cnt++;
6692 /* Adjust to current number of 4 KB cluster in use */
6693 n = m_region_expand(MC_BIGCL) -
6694 (m_total(MC_BIGCL) - m_infree(MC_BIGCL));
6695 if ((n + m_total(MC_BIGCL)) > m_maxlimit(MC_BIGCL))
6696 n = m_maxlimit(MC_BIGCL) - m_total(MC_BIGCL);
6697 if (n > 0) {
6698 mb_expand_bigcl_total += n;
6699 }
6700 m_region_expand(MC_BIGCL) = 0;
6701
6702 if (n > 0) {
6703 mbwdog_logger("expanding MC_BIGCL by %d", n);
6704 freelist_populate(MC_BIGCL, n, M_WAIT);
6705 }
6706 }
6707 if (m_region_expand(MC_16KCL) > 0) {
6708 int n;
6709 mb_expand_16kcl_cnt++;
6710 /* Adjust to current number of 16 KB cluster in use */
6711 n = m_region_expand(MC_16KCL) -
6712 (m_total(MC_16KCL) - m_infree(MC_16KCL));
6713 if ((n + m_total(MC_16KCL)) > m_maxlimit(MC_16KCL))
6714 n = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
6715 if (n > 0) {
6716 mb_expand_16kcl_total += n;
6717 }
6718 m_region_expand(MC_16KCL) = 0;
6719
6720 if (n > 0) {
6721 mbwdog_logger("expanding MC_16KCL by %d", n);
6722 (void) freelist_populate(MC_16KCL, n, M_WAIT);
6723 }
6724 }
6725
6726 /*
6727 * Because we can run out of memory before filling the mbuf
6728 * map, we should not allocate more clusters than they are
6729 * mbufs -- otherwise we could have a large number of useless
6730 * clusters allocated.
6731 */
6732 mbwdog_logger("totals: MC_MBUF %d MC_BIGCL %d MC_CL %d MC_16KCL %d",
6733 m_total(MC_MBUF), m_total(MC_BIGCL), m_total(MC_CL),
6734 m_total(MC_16KCL));
6735 uint32_t total_mbufs = m_total(MC_MBUF);
6736 uint32_t total_clusters = m_total(MC_BIGCL) + m_total(MC_CL) +
6737 m_total(MC_16KCL);
6738 if (total_mbufs < total_clusters) {
6739 mbwdog_logger("expanding MC_MBUF by %d",
6740 total_clusters - total_mbufs);
6741 }
6742 while (total_mbufs < total_clusters) {
6743 mb_expand_cnt++;
6744 if (freelist_populate(MC_MBUF, 1, M_WAIT) == 0)
6745 break;
6746 total_mbufs = m_total(MC_MBUF);
6747 total_clusters = m_total(MC_BIGCL) + m_total(MC_CL) +
6748 m_total(MC_16KCL);
6749 }
6750
6751 mbuf_worker_needs_wakeup = TRUE;
6752 /*
6753 * If there's a deadlock and we're not sending / receiving
6754 * packets, net_uptime() won't be updated. Update it here
6755 * so we are sure it's correct.
6756 */
6757 net_update_uptime();
6758 mbuf_worker_last_runtime = net_uptime();
6759 assert_wait((caddr_t)&mbuf_worker_needs_wakeup,
6760 THREAD_UNINT);
6761 mbwdog_logger("worker thread sleeping");
6762 lck_mtx_unlock(mbuf_mlock);
6763 (void) thread_block((thread_continue_t)mbuf_worker_thread);
6764 }
6765}
6766
6767__attribute__((noreturn))
6768static void
6769mbuf_worker_thread_init(void)
6770{
6771 mbuf_worker_ready++;
6772 mbuf_worker_thread();
6773}
6774
6775static mcl_slab_t *
6776slab_get(void *buf)
6777{
6778 mcl_slabg_t *slg;
6779 unsigned int ix, k;
6780
6781 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
6782
6783 VERIFY(MBUF_IN_MAP(buf));
6784 ix = ((unsigned char *)buf - mbutl) >> MBSHIFT;
6785 VERIFY(ix < maxslabgrp);
6786
6787 if ((slg = slabstbl[ix]) == NULL) {
6788 /*
6789 * In the current implementation, we never shrink the slabs
6790 * table; if we attempt to reallocate a cluster group when
6791 * it's already allocated, panic since this is a sign of a
6792 * memory corruption (slabstbl[ix] got nullified).
6793 */
6794 ++slabgrp;
6795 VERIFY(ix < slabgrp);
6796 /*
6797 * Slabs expansion can only be done single threaded; when
6798 * we get here, it must be as a result of m_clalloc() which
6799 * is serialized and therefore mb_clalloc_busy must be set.
6800 */
6801 VERIFY(mb_clalloc_busy);
6802 lck_mtx_unlock(mbuf_mlock);
6803
6804 /* This is a new buffer; create the slabs group for it */
6805 MALLOC(slg, mcl_slabg_t *, sizeof (*slg), M_TEMP,
6806 M_WAITOK | M_ZERO);
6807 MALLOC(slg->slg_slab, mcl_slab_t *, sizeof(mcl_slab_t) * NSLABSPMB,
6808 M_TEMP, M_WAITOK | M_ZERO);
6809 VERIFY(slg != NULL && slg->slg_slab != NULL);
6810
6811 lck_mtx_lock(mbuf_mlock);
6812 /*
6813 * No other thread could have gone into m_clalloc() after
6814 * we dropped the lock above, so verify that it's true.
6815 */
6816 VERIFY(mb_clalloc_busy);
6817
6818 slabstbl[ix] = slg;
6819
6820 /* Chain each slab in the group to its forward neighbor */
6821 for (k = 1; k < NSLABSPMB; k++)
6822 slg->slg_slab[k - 1].sl_next = &slg->slg_slab[k];
6823 VERIFY(slg->slg_slab[NSLABSPMB - 1].sl_next == NULL);
6824
6825 /* And chain the last slab in the previous group to this */
6826 if (ix > 0) {
6827 VERIFY(slabstbl[ix - 1]->
6828 slg_slab[NSLABSPMB - 1].sl_next == NULL);
6829 slabstbl[ix - 1]->slg_slab[NSLABSPMB - 1].sl_next =
6830 &slg->slg_slab[0];
6831 }
6832 }
6833
6834 ix = MTOPG(buf) % NSLABSPMB;
6835 VERIFY(ix < NSLABSPMB);
6836
6837 return (&slg->slg_slab[ix]);
6838}
6839
6840static void
6841slab_init(mcl_slab_t *sp, mbuf_class_t class, u_int32_t flags,
6842 void *base, void *head, unsigned int len, int refcnt, int chunks)
6843{
6844 sp->sl_class = class;
6845 sp->sl_flags = flags;
6846 sp->sl_base = base;
6847 sp->sl_head = head;
6848 sp->sl_len = len;
6849 sp->sl_refcnt = refcnt;
6850 sp->sl_chunks = chunks;
6851 slab_detach(sp);
6852}
6853
6854static void
6855slab_insert(mcl_slab_t *sp, mbuf_class_t class)
6856{
6857 VERIFY(slab_is_detached(sp));
6858 m_slab_cnt(class)++;
6859 TAILQ_INSERT_TAIL(&m_slablist(class), sp, sl_link);
6860 sp->sl_flags &= ~SLF_DETACHED;
6861
6862 /*
6863 * If a buffer spans multiple contiguous pages then mark them as
6864 * detached too
6865 */
6866 if (class == MC_16KCL) {
6867 int k;
6868 for (k = 1; k < NSLABSP16KB; k++) {
6869 sp = sp->sl_next;
6870 /* Next slab must already be present */
6871 VERIFY(sp != NULL && slab_is_detached(sp));
6872 sp->sl_flags &= ~SLF_DETACHED;
6873 }
6874 }
6875}
6876
6877static void
6878slab_remove(mcl_slab_t *sp, mbuf_class_t class)
6879{
6880 int k;
6881 VERIFY(!slab_is_detached(sp));
6882 VERIFY(m_slab_cnt(class) > 0);
6883 m_slab_cnt(class)--;
6884 TAILQ_REMOVE(&m_slablist(class), sp, sl_link);
6885 slab_detach(sp);
6886 if (class == MC_16KCL) {
6887 for (k = 1; k < NSLABSP16KB; k++) {
6888 sp = sp->sl_next;
6889 /* Next slab must already be present */
6890 VERIFY(sp != NULL);
6891 VERIFY(!slab_is_detached(sp));
6892 slab_detach(sp);
6893 }
6894 }
6895}
6896
6897static boolean_t
6898slab_inrange(mcl_slab_t *sp, void *buf)
6899{
6900 return ((uintptr_t)buf >= (uintptr_t)sp->sl_base &&
6901 (uintptr_t)buf < ((uintptr_t)sp->sl_base + sp->sl_len));
6902}
6903
6904#undef panic
6905
6906static void
6907slab_nextptr_panic(mcl_slab_t *sp, void *addr)
6908{
6909 int i;
6910 unsigned int chunk_len = sp->sl_len / sp->sl_chunks;
6911 uintptr_t buf = (uintptr_t)sp->sl_base;
6912
6913 for (i = 0; i < sp->sl_chunks; i++, buf += chunk_len) {
6914 void *next = ((mcache_obj_t *)buf)->obj_next;
6915 if (next != addr)
6916 continue;
6917 if (!mclverify) {
6918 if (next != NULL && !MBUF_IN_MAP(next)) {
6919 mcache_t *cp = m_cache(sp->sl_class);
6920 panic("%s: %s buffer %p in slab %p modified "
6921 "after free at offset 0: %p out of range "
6922 "[%p-%p)\n", __func__, cp->mc_name,
6923 (void *)buf, sp, next, mbutl, embutl);
6924 /* NOTREACHED */
6925 }
6926 } else {
6927 mcache_audit_t *mca = mcl_audit_buf2mca(sp->sl_class,
6928 (mcache_obj_t *)buf);
6929 mcl_audit_verify_nextptr(next, mca);
6930 }
6931 }
6932}
6933
6934static void
6935slab_detach(mcl_slab_t *sp)
6936{
6937 sp->sl_link.tqe_next = (mcl_slab_t *)-1;
6938 sp->sl_link.tqe_prev = (mcl_slab_t **)-1;
6939 sp->sl_flags |= SLF_DETACHED;
6940}
6941
6942static boolean_t
6943slab_is_detached(mcl_slab_t *sp)
6944{
6945 return ((intptr_t)sp->sl_link.tqe_next == -1 &&
6946 (intptr_t)sp->sl_link.tqe_prev == -1 &&
6947 (sp->sl_flags & SLF_DETACHED));
6948}
6949
6950static void
6951mcl_audit_init(void *buf, mcache_audit_t **mca_list,
6952 mcache_obj_t **con_list, size_t con_size, unsigned int num)
6953{
6954 mcache_audit_t *mca, *mca_tail;
6955 mcache_obj_t *con = NULL;
6956 boolean_t save_contents = (con_list != NULL);
6957 unsigned int i, ix;
6958
6959 ASSERT(num <= NMBPG);
6960 ASSERT(con_list == NULL || con_size != 0);
6961
6962 ix = MTOPG(buf);
6963 VERIFY(ix < maxclaudit);
6964
6965 /* Make sure we haven't been here before */
6966 for (i = 0; i < num; i++)
6967 VERIFY(mclaudit[ix].cl_audit[i] == NULL);
6968
6969 mca = mca_tail = *mca_list;
6970 if (save_contents)
6971 con = *con_list;
6972
6973 for (i = 0; i < num; i++) {
6974 mcache_audit_t *next;
6975
6976 next = mca->mca_next;
6977 bzero(mca, sizeof (*mca));
6978 mca->mca_next = next;
6979 mclaudit[ix].cl_audit[i] = mca;
6980
6981 /* Attach the contents buffer if requested */
6982 if (save_contents) {
6983 mcl_saved_contents_t *msc =
6984 (mcl_saved_contents_t *)(void *)con;
6985
6986 VERIFY(msc != NULL);
6987 VERIFY(IS_P2ALIGNED(msc, sizeof (u_int64_t)));
6988 VERIFY(con_size == sizeof (*msc));
6989 mca->mca_contents_size = con_size;
6990 mca->mca_contents = msc;
6991 con = con->obj_next;
6992 bzero(mca->mca_contents, mca->mca_contents_size);
6993 }
6994
6995 mca_tail = mca;
6996 mca = mca->mca_next;
6997 }
6998
6999 if (save_contents)
7000 *con_list = con;
7001
7002 *mca_list = mca_tail->mca_next;
7003 mca_tail->mca_next = NULL;
7004}
7005
7006static void
7007mcl_audit_free(void *buf, unsigned int num)
7008{
7009 unsigned int i, ix;
7010 mcache_audit_t *mca, *mca_list;
7011
7012 ix = MTOPG(buf);
7013 VERIFY(ix < maxclaudit);
7014
7015 if (mclaudit[ix].cl_audit[0] != NULL) {
7016 mca_list = mclaudit[ix].cl_audit[0];
7017 for (i = 0; i < num; i++) {
7018 mca = mclaudit[ix].cl_audit[i];
7019 mclaudit[ix].cl_audit[i] = NULL;
7020 if (mca->mca_contents)
7021 mcache_free(mcl_audit_con_cache,
7022 mca->mca_contents);
7023 }
7024 mcache_free_ext(mcache_audit_cache,
7025 (mcache_obj_t *)mca_list);
7026 }
7027}
7028
7029/*
7030 * Given an address of a buffer (mbuf/2KB/4KB/16KB), return
7031 * the corresponding audit structure for that buffer.
7032 */
7033static mcache_audit_t *
7034mcl_audit_buf2mca(mbuf_class_t class, mcache_obj_t *mobj)
7035{
7036 mcache_audit_t *mca = NULL;
7037 int ix = MTOPG(mobj), m_idx = 0;
7038 unsigned char *page_addr;
7039
7040 VERIFY(ix < maxclaudit);
7041 VERIFY(IS_P2ALIGNED(mobj, MIN(m_maxsize(class), PAGE_SIZE)));
7042
7043 page_addr = PGTOM(ix);
7044
7045 switch (class) {
7046 case MC_MBUF:
7047 /*
7048 * For the mbuf case, find the index of the page
7049 * used by the mbuf and use that index to locate the
7050 * base address of the page. Then find out the
7051 * mbuf index relative to the page base and use
7052 * it to locate the audit structure.
7053 */
7054 m_idx = MBPAGEIDX(page_addr, mobj);
7055 VERIFY(m_idx < (int)NMBPG);
7056 mca = mclaudit[ix].cl_audit[m_idx];
7057 break;
7058
7059 case MC_CL:
7060 /*
7061 * Same thing as above, but for 2KB clusters in a page.
7062 */
7063 m_idx = CLPAGEIDX(page_addr, mobj);
7064 VERIFY(m_idx < (int)NCLPG);
7065 mca = mclaudit[ix].cl_audit[m_idx];
7066 break;
7067
7068 case MC_BIGCL:
7069 m_idx = BCLPAGEIDX(page_addr, mobj);
7070 VERIFY(m_idx < (int)NBCLPG);
7071 mca = mclaudit[ix].cl_audit[m_idx];
7072 break;
7073 case MC_16KCL:
7074 /*
7075 * Same as above, but only return the first element.
7076 */
7077 mca = mclaudit[ix].cl_audit[0];
7078 break;
7079
7080 default:
7081 VERIFY(0);
7082 /* NOTREACHED */
7083 }
7084
7085 return (mca);
7086}
7087
7088static void
7089mcl_audit_mbuf(mcache_audit_t *mca, void *addr, boolean_t composite,
7090 boolean_t alloc)
7091{
7092 struct mbuf *m = addr;
7093 mcache_obj_t *next = ((mcache_obj_t *)m)->obj_next;
7094
7095 VERIFY(mca->mca_contents != NULL &&
7096 mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
7097
7098 if (mclverify)
7099 mcl_audit_verify_nextptr(next, mca);
7100
7101 if (!alloc) {
7102 /* Save constructed mbuf fields */
7103 mcl_audit_save_mbuf(m, mca);
7104 if (mclverify) {
7105 mcache_set_pattern(MCACHE_FREE_PATTERN, m,
7106 m_maxsize(MC_MBUF));
7107 }
7108 ((mcache_obj_t *)m)->obj_next = next;
7109 return;
7110 }
7111
7112 /* Check if the buffer has been corrupted while in freelist */
7113 if (mclverify) {
7114 mcache_audit_free_verify_set(mca, addr, 0, m_maxsize(MC_MBUF));
7115 }
7116 /* Restore constructed mbuf fields */
7117 mcl_audit_restore_mbuf(m, mca, composite);
7118}
7119
7120static void
7121mcl_audit_restore_mbuf(struct mbuf *m, mcache_audit_t *mca, boolean_t composite)
7122{
7123 struct mbuf *ms = MCA_SAVED_MBUF_PTR(mca);
7124
7125 if (composite) {
7126 struct mbuf *next = m->m_next;
7127 VERIFY(ms->m_flags == M_EXT && m_get_rfa(ms) != NULL &&
7128 MBUF_IS_COMPOSITE(ms));
7129 VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
7130 /*
7131 * We could have hand-picked the mbuf fields and restore
7132 * them individually, but that will be a maintenance
7133 * headache. Instead, restore everything that was saved;
7134 * the mbuf layer will recheck and reinitialize anyway.
7135 */
7136 bcopy(ms, m, MCA_SAVED_MBUF_SIZE);
7137 m->m_next = next;
7138 } else {
7139 /*
7140 * For a regular mbuf (no cluster attached) there's nothing
7141 * to restore other than the type field, which is expected
7142 * to be MT_FREE.
7143 */
7144 m->m_type = ms->m_type;
7145 }
7146 _MCHECK(m);
7147}
7148
7149static void
7150mcl_audit_save_mbuf(struct mbuf *m, mcache_audit_t *mca)
7151{
7152 VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
7153 _MCHECK(m);
7154 bcopy(m, MCA_SAVED_MBUF_PTR(mca), MCA_SAVED_MBUF_SIZE);
7155}
7156
7157static void
7158mcl_audit_cluster(mcache_audit_t *mca, void *addr, size_t size, boolean_t alloc,
7159 boolean_t save_next)
7160{
7161 mcache_obj_t *next = ((mcache_obj_t *)addr)->obj_next;
7162
7163 if (!alloc) {
7164 if (mclverify) {
7165 mcache_set_pattern(MCACHE_FREE_PATTERN, addr, size);
7166 }
7167 if (save_next) {
7168 mcl_audit_verify_nextptr(next, mca);
7169 ((mcache_obj_t *)addr)->obj_next = next;
7170 }
7171 } else if (mclverify) {
7172 /* Check if the buffer has been corrupted while in freelist */
7173 mcl_audit_verify_nextptr(next, mca);
7174 mcache_audit_free_verify_set(mca, addr, 0, size);
7175 }
7176}
7177
7178static void
7179mcl_audit_scratch(mcache_audit_t *mca)
7180{
7181 void *stack[MCACHE_STACK_DEPTH + 1];
7182 mcl_scratch_audit_t *msa;
7183 struct timeval now;
7184
7185 VERIFY(mca->mca_contents != NULL);
7186 msa = MCA_SAVED_SCRATCH_PTR(mca);
7187
7188 msa->msa_pthread = msa->msa_thread;
7189 msa->msa_thread = current_thread();
7190 bcopy(msa->msa_stack, msa->msa_pstack, sizeof (msa->msa_pstack));
7191 msa->msa_pdepth = msa->msa_depth;
7192 bzero(stack, sizeof (stack));
7193 msa->msa_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
7194 bcopy(&stack[1], msa->msa_stack, sizeof (msa->msa_stack));
7195
7196 msa->msa_ptstamp = msa->msa_tstamp;
7197 microuptime(&now);
7198 /* tstamp is in ms relative to base_ts */
7199 msa->msa_tstamp = ((now.tv_usec - mb_start.tv_usec) / 1000);
7200 if ((now.tv_sec - mb_start.tv_sec) > 0)
7201 msa->msa_tstamp += ((now.tv_sec - mb_start.tv_sec) * 1000);
7202}
7203
7204static void
7205mcl_audit_mcheck_panic(struct mbuf *m)
7206{
7207 mcache_audit_t *mca;
7208
7209 MRANGE(m);
7210 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
7211
7212 panic("mcl_audit: freed mbuf %p with type 0x%x (instead of 0x%x)\n%s\n",
7213 m, (u_int16_t)m->m_type, MT_FREE, mcache_dump_mca(mca));
7214 /* NOTREACHED */
7215}
7216
7217static void
7218mcl_audit_verify_nextptr(void *next, mcache_audit_t *mca)
7219{
7220 if (next != NULL && !MBUF_IN_MAP(next) &&
7221 (next != (void *)MCACHE_FREE_PATTERN || !mclverify)) {
7222 panic("mcl_audit: buffer %p modified after free at offset 0: "
7223 "%p out of range [%p-%p)\n%s\n",
7224 mca->mca_addr, next, mbutl, embutl, mcache_dump_mca(mca));
7225 /* NOTREACHED */
7226 }
7227}
7228
7229/* This function turns on mbuf leak detection */
7230static void
7231mleak_activate(void)
7232{
7233 mleak_table.mleak_sample_factor = MLEAK_SAMPLE_FACTOR;
7234 PE_parse_boot_argn("mleak_sample_factor",
7235 &mleak_table.mleak_sample_factor,
7236 sizeof (mleak_table.mleak_sample_factor));
7237
7238 if (mleak_table.mleak_sample_factor == 0)
7239 mclfindleak = 0;
7240
7241 if (mclfindleak == 0)
7242 return;
7243
7244 vm_size_t alloc_size =
7245 mleak_alloc_buckets * sizeof (struct mallocation);
7246 vm_size_t trace_size = mleak_trace_buckets * sizeof (struct mtrace);
7247
7248 MALLOC(mleak_allocations, struct mallocation *, alloc_size,
7249 M_TEMP, M_WAITOK | M_ZERO);
7250 VERIFY(mleak_allocations != NULL);
7251
7252 MALLOC(mleak_traces, struct mtrace *, trace_size,
7253 M_TEMP, M_WAITOK | M_ZERO);
7254 VERIFY(mleak_traces != NULL);
7255
7256 MALLOC(mleak_stat, mleak_stat_t *, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES),
7257 M_TEMP, M_WAITOK | M_ZERO);
7258 VERIFY(mleak_stat != NULL);
7259 mleak_stat->ml_cnt = MLEAK_NUM_TRACES;
7260#ifdef __LP64__
7261 mleak_stat->ml_isaddr64 = 1;
7262#endif /* __LP64__ */
7263}
7264
7265static void
7266mleak_logger(u_int32_t num, mcache_obj_t *addr, boolean_t alloc)
7267{
7268 int temp;
7269
7270 if (mclfindleak == 0)
7271 return;
7272
7273 if (!alloc)
7274 return (mleak_free(addr));
7275
7276 temp = atomic_add_32_ov(&mleak_table.mleak_capture, 1);
7277
7278 if ((temp % mleak_table.mleak_sample_factor) == 0 && addr != NULL) {
7279 uintptr_t bt[MLEAK_STACK_DEPTH];
7280 int logged = backtrace(bt, MLEAK_STACK_DEPTH);
7281 mleak_log(bt, addr, logged, num);
7282 }
7283}
7284
7285/*
7286 * This function records the allocation in the mleak_allocations table
7287 * and the backtrace in the mleak_traces table; if allocation slot is in use,
7288 * replace old allocation with new one if the trace slot is in use, return
7289 * (or increment refcount if same trace).
7290 */
7291static boolean_t
7292mleak_log(uintptr_t *bt, mcache_obj_t *addr, uint32_t depth, int num)
7293{
7294 struct mallocation *allocation;
7295 struct mtrace *trace;
7296 uint32_t trace_index;
7297
7298 /* Quit if someone else modifying the tables */
7299 if (!lck_mtx_try_lock_spin(mleak_lock)) {
7300 mleak_table.total_conflicts++;
7301 return (FALSE);
7302 }
7303
7304 allocation = &mleak_allocations[hashaddr((uintptr_t)addr,
7305 mleak_alloc_buckets)];
7306 trace_index = hashbacktrace(bt, depth, mleak_trace_buckets);
7307 trace = &mleak_traces[trace_index];
7308
7309 VERIFY(allocation <= &mleak_allocations[mleak_alloc_buckets - 1]);
7310 VERIFY(trace <= &mleak_traces[mleak_trace_buckets - 1]);
7311
7312 allocation->hitcount++;
7313 trace->hitcount++;
7314
7315 /*
7316 * If the allocation bucket we want is occupied
7317 * and the occupier has the same trace, just bail.
7318 */
7319 if (allocation->element != NULL &&
7320 trace_index == allocation->trace_index) {
7321 mleak_table.alloc_collisions++;
7322 lck_mtx_unlock(mleak_lock);
7323 return (TRUE);
7324 }
7325
7326 /*
7327 * Store the backtrace in the traces array;
7328 * Size of zero = trace bucket is free.
7329 */
7330 if (trace->allocs > 0 &&
7331 bcmp(trace->addr, bt, (depth * sizeof (uintptr_t))) != 0) {
7332 /* Different, unique trace, but the same hash! Bail out. */
7333 trace->collisions++;
7334 mleak_table.trace_collisions++;
7335 lck_mtx_unlock(mleak_lock);
7336 return (TRUE);
7337 } else if (trace->allocs > 0) {
7338 /* Same trace, already added, so increment refcount */
7339 trace->allocs++;
7340 } else {
7341 /* Found an unused trace bucket, so record the trace here */
7342 if (trace->depth != 0) {
7343 /* this slot previously used but not currently in use */
7344 mleak_table.trace_overwrites++;
7345 }
7346 mleak_table.trace_recorded++;
7347 trace->allocs = 1;
7348 memcpy(trace->addr, bt, (depth * sizeof (uintptr_t)));
7349 trace->depth = depth;
7350 trace->collisions = 0;
7351 }
7352
7353 /* Step 2: Store the allocation record in the allocations array */
7354 if (allocation->element != NULL) {
7355 /*
7356 * Replace an existing allocation. No need to preserve
7357 * because only a subset of the allocations are being
7358 * recorded anyway.
7359 */
7360 mleak_table.alloc_collisions++;
7361 } else if (allocation->trace_index != 0) {
7362 mleak_table.alloc_overwrites++;
7363 }
7364 allocation->element = addr;
7365 allocation->trace_index = trace_index;
7366 allocation->count = num;
7367 mleak_table.alloc_recorded++;
7368 mleak_table.outstanding_allocs++;
7369
7370 lck_mtx_unlock(mleak_lock);
7371 return (TRUE);
7372}
7373
7374static void
7375mleak_free(mcache_obj_t *addr)
7376{
7377 while (addr != NULL) {
7378 struct mallocation *allocation = &mleak_allocations
7379 [hashaddr((uintptr_t)addr, mleak_alloc_buckets)];
7380
7381 if (allocation->element == addr &&
7382 allocation->trace_index < mleak_trace_buckets) {
7383 lck_mtx_lock_spin(mleak_lock);
7384 if (allocation->element == addr &&
7385 allocation->trace_index < mleak_trace_buckets) {
7386 struct mtrace *trace;
7387 trace = &mleak_traces[allocation->trace_index];
7388 /* allocs = 0 means trace bucket is unused */
7389 if (trace->allocs > 0)
7390 trace->allocs--;
7391 if (trace->allocs == 0)
7392 trace->depth = 0;
7393 /* NULL element means alloc bucket is unused */
7394 allocation->element = NULL;
7395 mleak_table.outstanding_allocs--;
7396 }
7397 lck_mtx_unlock(mleak_lock);
7398 }
7399 addr = addr->obj_next;
7400 }
7401}
7402
7403static void
7404mleak_sort_traces()
7405{
7406 int i, j, k;
7407 struct mtrace *swap;
7408
7409 for(i = 0; i < MLEAK_NUM_TRACES; i++)
7410 mleak_top_trace[i] = NULL;
7411
7412 for(i = 0, j = 0; j < MLEAK_NUM_TRACES && i < mleak_trace_buckets; i++)
7413 {
7414 if (mleak_traces[i].allocs <= 0)
7415 continue;
7416
7417 mleak_top_trace[j] = &mleak_traces[i];
7418 for (k = j; k > 0; k--) {
7419 if (mleak_top_trace[k]->allocs <=
7420 mleak_top_trace[k-1]->allocs)
7421 break;
7422
7423 swap = mleak_top_trace[k-1];
7424 mleak_top_trace[k-1] = mleak_top_trace[k];
7425 mleak_top_trace[k] = swap;
7426 }
7427 j++;
7428 }
7429
7430 j--;
7431 for(; i < mleak_trace_buckets; i++) {
7432 if (mleak_traces[i].allocs <= mleak_top_trace[j]->allocs)
7433 continue;
7434
7435 mleak_top_trace[j] = &mleak_traces[i];
7436
7437 for (k = j; k > 0; k--) {
7438 if (mleak_top_trace[k]->allocs <=
7439 mleak_top_trace[k-1]->allocs)
7440 break;
7441
7442 swap = mleak_top_trace[k-1];
7443 mleak_top_trace[k-1] = mleak_top_trace[k];
7444 mleak_top_trace[k] = swap;
7445 }
7446 }
7447}
7448
7449static void
7450mleak_update_stats()
7451{
7452 mleak_trace_stat_t *mltr;
7453 int i;
7454
7455 VERIFY(mleak_stat != NULL);
7456#ifdef __LP64__
7457 VERIFY(mleak_stat->ml_isaddr64);
7458#else
7459 VERIFY(!mleak_stat->ml_isaddr64);
7460#endif /* !__LP64__ */
7461 VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES);
7462
7463 mleak_sort_traces();
7464
7465 mltr = &mleak_stat->ml_trace[0];
7466 bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
7467 for (i = 0; i < MLEAK_NUM_TRACES; i++) {
7468 int j;
7469
7470 if (mleak_top_trace[i] == NULL ||
7471 mleak_top_trace[i]->allocs == 0)
7472 continue;
7473
7474 mltr->mltr_collisions = mleak_top_trace[i]->collisions;
7475 mltr->mltr_hitcount = mleak_top_trace[i]->hitcount;
7476 mltr->mltr_allocs = mleak_top_trace[i]->allocs;
7477 mltr->mltr_depth = mleak_top_trace[i]->depth;
7478
7479 VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH);
7480 for (j = 0; j < mltr->mltr_depth; j++)
7481 mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j];
7482
7483 mltr++;
7484 }
7485}
7486
7487static struct mbtypes {
7488 int mt_type;
7489 const char *mt_name;
7490} mbtypes[] = {
7491 { MT_DATA, "data" },
7492 { MT_OOBDATA, "oob data" },
7493 { MT_CONTROL, "ancillary data" },
7494 { MT_HEADER, "packet headers" },
7495 { MT_SOCKET, "socket structures" },
7496 { MT_PCB, "protocol control blocks" },
7497 { MT_RTABLE, "routing table entries" },
7498 { MT_HTABLE, "IMP host table entries" },
7499 { MT_ATABLE, "address resolution tables" },
7500 { MT_FTABLE, "fragment reassembly queue headers" },
7501 { MT_SONAME, "socket names and addresses" },
7502 { MT_SOOPTS, "socket options" },
7503 { MT_RIGHTS, "access rights" },
7504 { MT_IFADDR, "interface addresses" },
7505 { MT_TAG, "packet tags" },
7506 { 0, NULL }
7507};
7508
7509#define MBUF_DUMP_BUF_CHK() { \
7510 clen -= k; \
7511 if (clen < 1) \
7512 goto done; \
7513 c += k; \
7514}
7515
7516static char *
7517mbuf_dump(void)
7518{
7519 unsigned long totmem = 0, totfree = 0, totmbufs, totused, totpct,
7520 totreturned = 0;
7521 u_int32_t m_mbufs = 0, m_clfree = 0, m_bigclfree = 0;
7522 u_int32_t m_mbufclfree = 0, m_mbufbigclfree = 0;
7523 u_int32_t m_16kclusters = 0, m_16kclfree = 0, m_mbuf16kclfree = 0;
7524 int nmbtypes = sizeof (mbstat.m_mtypes) / sizeof (short);
7525 uint8_t seen[256];
7526 struct mbtypes *mp;
7527 mb_class_stat_t *sp;
7528 mleak_trace_stat_t *mltr;
7529 char *c = mbuf_dump_buf;
7530 int i, j, k, clen = MBUF_DUMP_BUF_SIZE;
7531 bool printed_banner = false;
7532
7533 mbuf_dump_buf[0] = '\0';
7534
7535 /* synchronize all statistics in the mbuf table */
7536 mbuf_stat_sync();
7537 mbuf_mtypes_sync(TRUE);
7538
7539 sp = &mb_stat->mbs_class[0];
7540 for (i = 0; i < mb_stat->mbs_cnt; i++, sp++) {
7541 u_int32_t mem;
7542
7543 if (m_class(i) == MC_MBUF) {
7544 m_mbufs = sp->mbcl_active;
7545 } else if (m_class(i) == MC_CL) {
7546 m_clfree = sp->mbcl_total - sp->mbcl_active;
7547 } else if (m_class(i) == MC_BIGCL) {
7548 m_bigclfree = sp->mbcl_total - sp->mbcl_active;
7549 } else if (njcl > 0 && m_class(i) == MC_16KCL) {
7550 m_16kclfree = sp->mbcl_total - sp->mbcl_active;
7551 m_16kclusters = sp->mbcl_total;
7552 } else if (m_class(i) == MC_MBUF_CL) {
7553 m_mbufclfree = sp->mbcl_total - sp->mbcl_active;
7554 } else if (m_class(i) == MC_MBUF_BIGCL) {
7555 m_mbufbigclfree = sp->mbcl_total - sp->mbcl_active;
7556 } else if (njcl > 0 && m_class(i) == MC_MBUF_16KCL) {
7557 m_mbuf16kclfree = sp->mbcl_total - sp->mbcl_active;
7558 }
7559
7560 mem = sp->mbcl_ctotal * sp->mbcl_size;
7561 totmem += mem;
7562 totfree += (sp->mbcl_mc_cached + sp->mbcl_infree) *
7563 sp->mbcl_size;
7564 totreturned += sp->mbcl_release_cnt;
7565
7566 }
7567
7568 /* adjust free counts to include composite caches */
7569 m_clfree += m_mbufclfree;
7570 m_bigclfree += m_mbufbigclfree;
7571 m_16kclfree += m_mbuf16kclfree;
7572
7573 totmbufs = 0;
7574 for (mp = mbtypes; mp->mt_name != NULL; mp++)
7575 totmbufs += mbstat.m_mtypes[mp->mt_type];
7576 if (totmbufs > m_mbufs)
7577 totmbufs = m_mbufs;
7578 k = snprintf(c, clen, "%lu/%u mbufs in use:\n", totmbufs, m_mbufs);
7579 MBUF_DUMP_BUF_CHK();
7580
7581 bzero(&seen, sizeof (seen));
7582 for (mp = mbtypes; mp->mt_name != NULL; mp++) {
7583 if (mbstat.m_mtypes[mp->mt_type] != 0) {
7584 seen[mp->mt_type] = 1;
7585 k = snprintf(c, clen, "\t%u mbufs allocated to %s\n",
7586 mbstat.m_mtypes[mp->mt_type], mp->mt_name);
7587 MBUF_DUMP_BUF_CHK();
7588 }
7589 }
7590 seen[MT_FREE] = 1;
7591 for (i = 0; i < nmbtypes; i++)
7592 if (!seen[i] && mbstat.m_mtypes[i] != 0) {
7593 k = snprintf(c, clen, "\t%u mbufs allocated to "
7594 "<mbuf type %d>\n", mbstat.m_mtypes[i], i);
7595 MBUF_DUMP_BUF_CHK();
7596 }
7597 if ((m_mbufs - totmbufs) > 0) {
7598 k = snprintf(c, clen, "\t%lu mbufs allocated to caches\n",
7599 m_mbufs - totmbufs);
7600 MBUF_DUMP_BUF_CHK();
7601 }
7602 k = snprintf(c, clen, "%u/%u mbuf 2KB clusters in use\n"
7603 "%u/%u mbuf 4KB clusters in use\n",
7604 (unsigned int)(mbstat.m_clusters - m_clfree),
7605 (unsigned int)mbstat.m_clusters,
7606 (unsigned int)(mbstat.m_bigclusters - m_bigclfree),
7607 (unsigned int)mbstat.m_bigclusters);
7608 MBUF_DUMP_BUF_CHK();
7609
7610 if (njcl > 0) {
7611 k = snprintf(c, clen, "%u/%u mbuf %uKB clusters in use\n",
7612 m_16kclusters - m_16kclfree, m_16kclusters,
7613 njclbytes / 1024);
7614 MBUF_DUMP_BUF_CHK();
7615 }
7616 totused = totmem - totfree;
7617 if (totmem == 0) {
7618 totpct = 0;
7619 } else if (totused < (ULONG_MAX / 100)) {
7620 totpct = (totused * 100) / totmem;
7621 } else {
7622 u_long totmem1 = totmem / 100;
7623 u_long totused1 = totused / 100;
7624 totpct = (totused1 * 100) / totmem1;
7625 }
7626 k = snprintf(c, clen, "%lu KB allocated to network (approx. %lu%% "
7627 "in use)\n", totmem / 1024, totpct);
7628 MBUF_DUMP_BUF_CHK();
7629 k = snprintf(c, clen, "%lu KB returned to the system\n",
7630 totreturned / 1024);
7631 MBUF_DUMP_BUF_CHK();
7632
7633 net_update_uptime();
7634 k = snprintf(c, clen,
7635 "VM allocation failures: contiguous %u, normal %u, one page %u\n",
7636 mb_kmem_contig_failed, mb_kmem_failed, mb_kmem_one_failed);
7637 MBUF_DUMP_BUF_CHK();
7638 if (mb_kmem_contig_failed_ts || mb_kmem_failed_ts ||
7639 mb_kmem_one_failed_ts) {
7640 k = snprintf(c, clen,
7641 "VM allocation failure timestamps: contiguous %llu "
7642 "(size %llu), normal %llu (size %llu), one page %llu "
7643 "(now %llu)\n",
7644 mb_kmem_contig_failed_ts, mb_kmem_contig_failed_size,
7645 mb_kmem_failed_ts, mb_kmem_failed_size,
7646 mb_kmem_one_failed_ts, net_uptime());
7647 MBUF_DUMP_BUF_CHK();
7648 k = snprintf(c, clen,
7649 "VM return codes: ");
7650 MBUF_DUMP_BUF_CHK();
7651 for (i = 0;
7652 i < sizeof(mb_kmem_stats) / sizeof(mb_kmem_stats[0]);
7653 i++) {
7654 k = snprintf(c, clen, "%s: %u ", mb_kmem_stats_labels[i],
7655 mb_kmem_stats[i]);
7656 MBUF_DUMP_BUF_CHK();
7657 }
7658 k = snprintf(c, clen, "\n");
7659 MBUF_DUMP_BUF_CHK();
7660 }
7661 k = snprintf(c, clen,
7662 "worker thread runs: %u, expansions: %llu, cl %llu/%llu, "
7663 "bigcl %llu/%llu, 16k %llu/%llu\n", mbuf_worker_run_cnt,
7664 mb_expand_cnt, mb_expand_cl_cnt, mb_expand_cl_total,
7665 mb_expand_bigcl_cnt, mb_expand_bigcl_total, mb_expand_16kcl_cnt,
7666 mb_expand_16kcl_total);
7667 MBUF_DUMP_BUF_CHK();
7668 if (mbuf_worker_last_runtime != 0) {
7669 k = snprintf(c, clen, "worker thread last run time: "
7670 "%llu (%llu seconds ago)\n",
7671 mbuf_worker_last_runtime,
7672 net_uptime() - mbuf_worker_last_runtime);
7673 MBUF_DUMP_BUF_CHK();
7674 }
7675 if (mbuf_drain_last_runtime != 0) {
7676 k = snprintf(c, clen, "drain routine last run time: "
7677 "%llu (%llu seconds ago)\n",
7678 mbuf_drain_last_runtime,
7679 net_uptime() - mbuf_drain_last_runtime);
7680 MBUF_DUMP_BUF_CHK();
7681 }
7682
7683#if DEBUG || DEVELOPMENT
7684 k = snprintf(c, clen, "\nworker thread log:\n%s\n", mbwdog_logging);
7685 MBUF_DUMP_BUF_CHK();
7686#endif
7687
7688 for (j = 0; j < MTRACELARGE_NUM_TRACES; j++) {
7689 struct mtracelarge *trace = &mtracelarge_table[j];
7690 if (trace->size == 0 || trace->depth == 0)
7691 continue;
7692 if (printed_banner == false) {
7693 k = snprintf(c, clen,
7694 "\nlargest allocation failure backtraces:\n");
7695 MBUF_DUMP_BUF_CHK();
7696 printed_banner = true;
7697 }
7698 k = snprintf(c, clen, "size %llu: < ", trace->size);
7699 MBUF_DUMP_BUF_CHK();
7700 for (i = 0; i < trace->depth; i++) {
7701 if (mleak_stat->ml_isaddr64) {
7702 k = snprintf(c, clen, "0x%0llx ",
7703 (uint64_t)VM_KERNEL_UNSLIDE(
7704 trace->addr[i]));
7705 } else {
7706 k = snprintf(c, clen,
7707 "0x%08x ",
7708 (uint32_t)VM_KERNEL_UNSLIDE(
7709 trace->addr[i]));
7710 }
7711 MBUF_DUMP_BUF_CHK();
7712 }
7713 k = snprintf(c, clen, ">\n");
7714 MBUF_DUMP_BUF_CHK();
7715 }
7716
7717 /* mbuf leak detection statistics */
7718 mleak_update_stats();
7719
7720 k = snprintf(c, clen, "\nmbuf leak detection table:\n");
7721 MBUF_DUMP_BUF_CHK();
7722 k = snprintf(c, clen, "\ttotal captured: %u (one per %u)\n",
7723 mleak_table.mleak_capture / mleak_table.mleak_sample_factor,
7724 mleak_table.mleak_sample_factor);
7725 MBUF_DUMP_BUF_CHK();
7726 k = snprintf(c, clen, "\ttotal allocs outstanding: %llu\n",
7727 mleak_table.outstanding_allocs);
7728 MBUF_DUMP_BUF_CHK();
7729 k = snprintf(c, clen, "\tnew hash recorded: %llu allocs, %llu traces\n",
7730 mleak_table.alloc_recorded, mleak_table.trace_recorded);
7731 MBUF_DUMP_BUF_CHK();
7732 k = snprintf(c, clen, "\thash collisions: %llu allocs, %llu traces\n",
7733 mleak_table.alloc_collisions, mleak_table.trace_collisions);
7734 MBUF_DUMP_BUF_CHK();
7735 k = snprintf(c, clen, "\toverwrites: %llu allocs, %llu traces\n",
7736 mleak_table.alloc_overwrites, mleak_table.trace_overwrites);
7737 MBUF_DUMP_BUF_CHK();
7738 k = snprintf(c, clen, "\tlock conflicts: %llu\n\n",
7739 mleak_table.total_conflicts);
7740 MBUF_DUMP_BUF_CHK();
7741
7742 k = snprintf(c, clen, "top %d outstanding traces:\n",
7743 mleak_stat->ml_cnt);
7744 MBUF_DUMP_BUF_CHK();
7745 for (i = 0; i < mleak_stat->ml_cnt; i++) {
7746 mltr = &mleak_stat->ml_trace[i];
7747 k = snprintf(c, clen, "[%d] %llu outstanding alloc(s), "
7748 "%llu hit(s), %llu collision(s)\n", (i + 1),
7749 mltr->mltr_allocs, mltr->mltr_hitcount,
7750 mltr->mltr_collisions);
7751 MBUF_DUMP_BUF_CHK();
7752 }
7753
7754 if (mleak_stat->ml_isaddr64)
7755 k = snprintf(c, clen, MB_LEAK_HDR_64);
7756 else
7757 k = snprintf(c, clen, MB_LEAK_HDR_32);
7758 MBUF_DUMP_BUF_CHK();
7759
7760 for (i = 0; i < MLEAK_STACK_DEPTH; i++) {
7761 k = snprintf(c, clen, "%2d: ", (i + 1));
7762 MBUF_DUMP_BUF_CHK();
7763 for (j = 0; j < mleak_stat->ml_cnt; j++) {
7764 mltr = &mleak_stat->ml_trace[j];
7765 if (i < mltr->mltr_depth) {
7766 if (mleak_stat->ml_isaddr64) {
7767 k = snprintf(c, clen, "0x%0llx ",
7768 (uint64_t)VM_KERNEL_UNSLIDE(
7769 mltr->mltr_addr[i]));
7770 } else {
7771 k = snprintf(c, clen,
7772 "0x%08x ",
7773 (uint32_t)VM_KERNEL_UNSLIDE(
7774 mltr->mltr_addr[i]));
7775 }
7776 } else {
7777 if (mleak_stat->ml_isaddr64)
7778 k = snprintf(c, clen,
7779 MB_LEAK_SPACING_64);
7780 else
7781 k = snprintf(c, clen,
7782 MB_LEAK_SPACING_32);
7783 }
7784 MBUF_DUMP_BUF_CHK();
7785 }
7786 k = snprintf(c, clen, "\n");
7787 MBUF_DUMP_BUF_CHK();
7788 }
7789done:
7790 return (mbuf_dump_buf);
7791}
7792
7793#undef MBUF_DUMP_BUF_CHK
7794
7795/*
7796 * Convert between a regular and a packet header mbuf. Caller is responsible
7797 * for setting or clearing M_PKTHDR; this routine does the rest of the work.
7798 */
7799int
7800m_reinit(struct mbuf *m, int hdr)
7801{
7802 int ret = 0;
7803
7804 if (hdr) {
7805 VERIFY(!(m->m_flags & M_PKTHDR));
7806 if (!(m->m_flags & M_EXT) &&
7807 (m->m_data != m->m_dat || m->m_len > 0)) {
7808 /*
7809 * If there's no external cluster attached and the
7810 * mbuf appears to contain user data, we cannot
7811 * safely convert this to a packet header mbuf,
7812 * as the packet header structure might overlap
7813 * with the data.
7814 */
7815 printf("%s: cannot set M_PKTHDR on altered mbuf %llx, "
7816 "m_data %llx (expected %llx), "
7817 "m_len %d (expected 0)\n",
7818 __func__,
7819 (uint64_t)VM_KERNEL_ADDRPERM(m),
7820 (uint64_t)VM_KERNEL_ADDRPERM(m->m_data),
7821 (uint64_t)VM_KERNEL_ADDRPERM(m->m_dat), m->m_len);
7822 ret = EBUSY;
7823 } else {
7824 VERIFY((m->m_flags & M_EXT) || m->m_data == m->m_dat);
7825 m->m_flags |= M_PKTHDR;
7826 MBUF_INIT_PKTHDR(m);
7827 }
7828 } else {
7829 /* Check for scratch area overflow */
7830 m_redzone_verify(m);
7831 /* Free the aux data and tags if there is any */
7832 m_tag_delete_chain(m, NULL);
7833 m->m_flags &= ~M_PKTHDR;
7834 }
7835
7836 return (ret);
7837}
7838
7839int
7840m_ext_set_prop(struct mbuf *m, uint32_t o, uint32_t n)
7841{
7842 ASSERT(m->m_flags & M_EXT);
7843 return (atomic_test_set_32(&MEXT_PRIV(m), o, n));
7844}
7845
7846uint32_t
7847m_ext_get_prop(struct mbuf *m)
7848{
7849 ASSERT(m->m_flags & M_EXT);
7850 return (MEXT_PRIV(m));
7851}
7852
7853int
7854m_ext_paired_is_active(struct mbuf *m)
7855{
7856 return (MBUF_IS_PAIRED(m) ? (MEXT_PREF(m) > MEXT_MINREF(m)) : 1);
7857}
7858
7859void
7860m_ext_paired_activate(struct mbuf *m)
7861{
7862 struct ext_ref *rfa;
7863 int hdr, type;
7864 caddr_t extbuf;
7865 m_ext_free_func_t extfree;
7866 u_int extsize;
7867
7868 VERIFY(MBUF_IS_PAIRED(m));
7869 VERIFY(MEXT_REF(m) == MEXT_MINREF(m));
7870 VERIFY(MEXT_PREF(m) == MEXT_MINREF(m));
7871
7872 hdr = (m->m_flags & M_PKTHDR);
7873 type = m->m_type;
7874 extbuf = m->m_ext.ext_buf;
7875 extfree = m_get_ext_free(m);
7876 extsize = m->m_ext.ext_size;
7877 rfa = m_get_rfa(m);
7878
7879 VERIFY(extbuf != NULL && rfa != NULL);
7880
7881 /*
7882 * Safe to reinitialize packet header tags, since it's
7883 * already taken care of at m_free() time. Similar to
7884 * what's done in m_clattach() for the cluster. Bump
7885 * up MEXT_PREF to indicate activation.
7886 */
7887 MBUF_INIT(m, hdr, type);
7888 MEXT_INIT(m, extbuf, extsize, extfree, (caddr_t)m, rfa,
7889 1, 1, 2, EXTF_PAIRED, MEXT_PRIV(m), m);
7890}
7891
7892void
7893m_scratch_init(struct mbuf *m)
7894{
7895 struct pkthdr *pkt = &m->m_pkthdr;
7896
7897 VERIFY(m->m_flags & M_PKTHDR);
7898
7899 /* See comments in <rdar://problem/14040693> */
7900 if (pkt->pkt_flags & PKTF_PRIV_GUARDED) {
7901 panic_plain("Invalid attempt to modify guarded module-private "
7902 "area: mbuf %p, pkt_flags 0x%x\n", m, pkt->pkt_flags);
7903 /* NOTREACHED */
7904 }
7905
7906 bzero(&pkt->pkt_mpriv, sizeof (pkt->pkt_mpriv));
7907}
7908
7909/*
7910 * This routine is reserved for mbuf_get_driver_scratch(); clients inside
7911 * xnu that intend on utilizing the module-private area should directly
7912 * refer to the pkt_mpriv structure in the pkthdr. They are also expected
7913 * to set and clear PKTF_PRIV_GUARDED, while owning the packet and prior
7914 * to handing it off to another module, respectively.
7915 */
7916u_int32_t
7917m_scratch_get(struct mbuf *m, u_int8_t **p)
7918{
7919 struct pkthdr *pkt = &m->m_pkthdr;
7920
7921 VERIFY(m->m_flags & M_PKTHDR);
7922
7923 /* See comments in <rdar://problem/14040693> */
7924 if (pkt->pkt_flags & PKTF_PRIV_GUARDED) {
7925 panic_plain("Invalid attempt to access guarded module-private "
7926 "area: mbuf %p, pkt_flags 0x%x\n", m, pkt->pkt_flags);
7927 /* NOTREACHED */
7928 }
7929
7930 if (mcltrace) {
7931 mcache_audit_t *mca;
7932
7933 lck_mtx_lock(mbuf_mlock);
7934 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
7935 if (mca->mca_uflags & MB_SCVALID)
7936 mcl_audit_scratch(mca);
7937 lck_mtx_unlock(mbuf_mlock);
7938 }
7939
7940 *p = (u_int8_t *)&pkt->pkt_mpriv;
7941 return (sizeof (pkt->pkt_mpriv));
7942}
7943
7944static void
7945m_redzone_init(struct mbuf *m)
7946{
7947 VERIFY(m->m_flags & M_PKTHDR);
7948 /*
7949 * Each mbuf has a unique red zone pattern, which is a XOR
7950 * of the red zone cookie and the address of the mbuf.
7951 */
7952 m->m_pkthdr.redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
7953}
7954
7955static void
7956m_redzone_verify(struct mbuf *m)
7957{
7958 u_int32_t mb_redzone;
7959
7960 VERIFY(m->m_flags & M_PKTHDR);
7961
7962 mb_redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
7963 if (m->m_pkthdr.redzone != mb_redzone) {
7964 panic("mbuf %p redzone violation with value 0x%x "
7965 "(instead of 0x%x, using cookie 0x%x)\n",
7966 m, m->m_pkthdr.redzone, mb_redzone, mb_redzone_cookie);
7967 /* NOTREACHED */
7968 }
7969}
7970
7971__private_extern__ inline void
7972m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free,
7973 caddr_t ext_arg)
7974{
7975 VERIFY(m->m_flags & M_EXT);
7976 if (rfa != NULL) {
7977 m->m_ext.ext_refflags =
7978 (struct ext_ref *)(((uintptr_t)rfa) ^ mb_obscure_extref);
7979 if (ext_free != NULL) {
7980 rfa->ext_token = ((uintptr_t)&rfa->ext_token) ^
7981 mb_obscure_extfree;
7982 m->m_ext.ext_free = (m_ext_free_func_t)
7983 (((uintptr_t)ext_free) ^ rfa->ext_token);
7984 if (ext_arg != NULL) {
7985 m->m_ext.ext_arg =
7986 (caddr_t)(((uintptr_t)ext_arg) ^ rfa->ext_token);
7987 } else {
7988 m->m_ext.ext_arg = NULL;
7989 }
7990 } else {
7991 rfa->ext_token = 0;
7992 m->m_ext.ext_free = NULL;
7993 m->m_ext.ext_arg = NULL;
7994 }
7995 } else {
7996 /*
7997 * If we are going to loose the cookie in ext_token by
7998 * resetting the rfa, we should use the global cookie
7999 * to obscure the ext_free and ext_arg pointers.
8000 */
8001 if (ext_free != NULL) {
8002 m->m_ext.ext_free =
8003 (m_ext_free_func_t)((uintptr_t)ext_free ^
8004 mb_obscure_extfree);
8005 if (ext_arg != NULL) {
8006 m->m_ext.ext_arg =
8007 (caddr_t)((uintptr_t)ext_arg ^
8008 mb_obscure_extfree);
8009 } else {
8010 m->m_ext.ext_arg = NULL;
8011 }
8012 } else {
8013 m->m_ext.ext_free = NULL;
8014 m->m_ext.ext_arg = NULL;
8015 }
8016 m->m_ext.ext_refflags = NULL;
8017 }
8018}
8019
8020__private_extern__ inline struct ext_ref *
8021m_get_rfa(struct mbuf *m)
8022{
8023 if (m->m_ext.ext_refflags == NULL)
8024 return (NULL);
8025 else
8026 return ((struct ext_ref *)(((uintptr_t)m->m_ext.ext_refflags) ^ mb_obscure_extref));
8027}
8028
8029__private_extern__ inline m_ext_free_func_t
8030m_get_ext_free(struct mbuf *m)
8031{
8032 struct ext_ref *rfa;
8033 if (m->m_ext.ext_free == NULL)
8034 return (NULL);
8035
8036 rfa = m_get_rfa(m);
8037 if (rfa == NULL)
8038 return ((m_ext_free_func_t)((uintptr_t)m->m_ext.ext_free ^ mb_obscure_extfree));
8039 else
8040 return ((m_ext_free_func_t)(((uintptr_t)m->m_ext.ext_free)
8041 ^ rfa->ext_token));
8042}
8043
8044__private_extern__ inline caddr_t
8045m_get_ext_arg(struct mbuf *m)
8046{
8047 struct ext_ref *rfa;
8048 if (m->m_ext.ext_arg == NULL)
8049 return (NULL);
8050
8051 rfa = m_get_rfa(m);
8052 if (rfa == NULL) {
8053 return ((caddr_t)((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree));
8054 } else {
8055 return ((caddr_t)(((uintptr_t)m->m_ext.ext_arg) ^
8056 rfa->ext_token));
8057 }
8058}
8059
8060/*
8061 * Send a report of mbuf usage if the usage is at least 6% of max limit
8062 * or if there has been at least 3% increase since the last report.
8063 *
8064 * The values 6% and 3% are chosen so that we can do simple arithmetic
8065 * with shift operations.
8066 */
8067static boolean_t
8068mbuf_report_usage(mbuf_class_t cl)
8069{
8070 /* if a report is already in progress, nothing to do */
8071 if (mb_peak_newreport)
8072 return (TRUE);
8073
8074 if (m_total(cl) > m_peak(cl) &&
8075 m_total(cl) >= (m_maxlimit(cl) >> 4) &&
8076 (m_total(cl) - m_peak(cl)) >= (m_peak(cl) >> 5))
8077 return (TRUE);
8078 return (FALSE);
8079}
8080
8081__private_extern__ void
8082mbuf_report_peak_usage(void)
8083{
8084 int i = 0;
8085 u_int64_t uptime;
8086 struct nstat_sysinfo_data ns_data;
8087 uint32_t memreleased = 0;
8088 static uint32_t prevmemreleased;
8089
8090 uptime = net_uptime();
8091 lck_mtx_lock(mbuf_mlock);
8092
8093 /* Generate an initial report after 1 week of uptime */
8094 if (!mb_peak_firstreport &&
8095 uptime > MBUF_PEAK_FIRST_REPORT_THRESHOLD) {
8096 mb_peak_newreport = TRUE;
8097 mb_peak_firstreport = TRUE;
8098 }
8099
8100 if (!mb_peak_newreport) {
8101 lck_mtx_unlock(mbuf_mlock);
8102 return;
8103 }
8104
8105 /*
8106 * Since a report is being generated before 1 week,
8107 * we do not need to force another one later
8108 */
8109 if (uptime < MBUF_PEAK_FIRST_REPORT_THRESHOLD)
8110 mb_peak_firstreport = TRUE;
8111
8112 for (i = 0; i < NELEM(mbuf_table); i++) {
8113 m_peak(m_class(i)) = m_total(m_class(i));
8114 memreleased += m_release_cnt(i);
8115 }
8116 memreleased = memreleased - prevmemreleased;
8117 prevmemreleased = memreleased;
8118 mb_peak_newreport = FALSE;
8119 lck_mtx_unlock(mbuf_mlock);
8120
8121 bzero(&ns_data, sizeof(ns_data));
8122 ns_data.flags = NSTAT_SYSINFO_MBUF_STATS;
8123 ns_data.u.mb_stats.total_256b = m_peak(MC_MBUF);
8124 ns_data.u.mb_stats.total_2kb = m_peak(MC_CL);
8125 ns_data.u.mb_stats.total_4kb = m_peak(MC_BIGCL);
8126 ns_data.u.mb_stats.total_16kb = m_peak(MC_16KCL);
8127 ns_data.u.mb_stats.sbmb_total = total_sbmb_cnt_peak;
8128 ns_data.u.mb_stats.sb_atmbuflimit = sbmb_limreached;
8129 ns_data.u.mb_stats.draincnt = mbstat.m_drain;
8130 ns_data.u.mb_stats.memreleased = memreleased;
8131 ns_data.u.mb_stats.sbmb_floor = total_sbmb_cnt_floor;
8132
8133 nstat_sysinfo_send_data(&ns_data);
8134
8135 /*
8136 * Reset the floor whenever we report a new
8137 * peak to track the trend (increase peek usage
8138 * is not a leak if mbufs get released
8139 * between reports and the floor stays low)
8140 */
8141 total_sbmb_cnt_floor = total_sbmb_cnt_peak;
8142}
8143
8144/*
8145 * Simple routine to avoid taking the lock when we can't run the
8146 * mbuf drain.
8147 */
8148static int
8149mbuf_drain_checks(boolean_t ignore_waiters)
8150{
8151
8152 if (mb_drain_maxint == 0)
8153 return 0;
8154 if (!ignore_waiters && mb_waiters != 0)
8155 return 0;
8156
8157 return 1;
8158}
8159
8160/*
8161 * Called by the VM when there's memory pressure or when we exhausted
8162 * the 4k/16k reserved space.
8163 */
8164static void
8165mbuf_drain_locked(boolean_t ignore_waiters)
8166{
8167 mbuf_class_t mc;
8168 mcl_slab_t *sp, *sp_tmp, *nsp;
8169 unsigned int num, k, interval, released = 0;
8170 unsigned long total_mem = 0, use_mem = 0;
8171 boolean_t ret, purge_caches = FALSE;
8172 ppnum_t offset;
8173 mcache_obj_t *obj;
8174 unsigned long per;
8175 static unsigned char scratch[32];
8176 static ppnum_t scratch_pa = 0;
8177
8178 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
8179 if (!mbuf_drain_checks(ignore_waiters))
8180 return;
8181 if (scratch_pa == 0) {
8182 bzero(scratch, sizeof(scratch));
8183 scratch_pa = pmap_find_phys(kernel_pmap, (addr64_t)scratch);
8184 VERIFY(scratch_pa);
8185 } else if (mclverify) {
8186 /*
8187 * Panic if a driver wrote to our scratch memory.
8188 */
8189 for (k = 0; k < sizeof(scratch); k++)
8190 if (scratch[k])
8191 panic("suspect DMA to freed address");
8192 }
8193 /*
8194 * Don't free memory too often as that could cause excessive
8195 * waiting times for mbufs. Purge caches if we were asked to drain
8196 * in the last 5 minutes.
8197 */
8198 if (mbuf_drain_last_runtime != 0) {
8199 interval = net_uptime() - mbuf_drain_last_runtime;
8200 if (interval <= mb_drain_maxint) {
8201 return;
8202 }
8203 if (interval <= mb_drain_maxint * 5)
8204 purge_caches = TRUE;
8205 }
8206 mbuf_drain_last_runtime = net_uptime();
8207 /*
8208 * Don't free any memory if we're using 60% or more.
8209 */
8210 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
8211 total_mem += m_total(mc) * m_maxsize(mc);
8212 use_mem += m_active(mc) * m_maxsize(mc);
8213 }
8214 per = (use_mem * 100) / total_mem;
8215 if (per >= 60) {
8216 return;
8217 }
8218 /*
8219 * Purge all the caches. This effectively disables
8220 * caching for a few seconds, but the mbuf worker thread will
8221 * re-enable them again.
8222 */
8223 if (purge_caches == TRUE)
8224 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
8225 if (m_total(mc) < m_avgtotal(mc))
8226 continue;
8227 lck_mtx_unlock(mbuf_mlock);
8228 ret = mcache_purge_cache(m_cache(mc), FALSE);
8229 lck_mtx_lock(mbuf_mlock);
8230 if (ret == TRUE)
8231 m_purge_cnt(mc)++;
8232 }
8233 /*
8234 * Move the objects from the composite class freelist to
8235 * the rudimentary slabs list, but keep at least 10% of the average
8236 * total in the freelist.
8237 */
8238 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
8239 while (m_cobjlist(mc) &&
8240 m_total(mc) < m_avgtotal(mc) &&
8241 m_infree(mc) > 0.1 * m_avgtotal(mc) + m_minlimit(mc)) {
8242 obj = m_cobjlist(mc);
8243 m_cobjlist(mc) = obj->obj_next;
8244 obj->obj_next = NULL;
8245 num = cslab_free(mc, obj, 1);
8246 VERIFY(num == 1);
8247 m_free_cnt(mc)++;
8248 m_infree(mc)--;
8249 /* cslab_free() handles m_total */
8250 }
8251 }
8252 /*
8253 * Free the buffers present in the slab list up to 10% of the total
8254 * average per class.
8255 *
8256 * We walk the list backwards in an attempt to reduce fragmentation.
8257 */
8258 for (mc = NELEM(mbuf_table) - 1; (int)mc >= 0; mc--) {
8259 TAILQ_FOREACH_SAFE(sp, &m_slablist(mc), sl_link, sp_tmp) {
8260 /*
8261 * Process only unused slabs occupying memory.
8262 */
8263 if (sp->sl_refcnt != 0 || sp->sl_len == 0 ||
8264 sp->sl_base == NULL)
8265 continue;
8266 if (m_total(mc) < m_avgtotal(mc) ||
8267 m_infree(mc) < 0.1 * m_avgtotal(mc) + m_minlimit(mc))
8268 break;
8269 slab_remove(sp, mc);
8270 switch (mc) {
8271 case MC_MBUF:
8272 m_infree(mc) -= NMBPG;
8273 m_total(mc) -= NMBPG;
8274 if (mclaudit != NULL)
8275 mcl_audit_free(sp->sl_base, NMBPG);
8276 break;
8277 case MC_CL:
8278 m_infree(mc) -= NCLPG;
8279 m_total(mc) -= NCLPG;
8280 if (mclaudit != NULL)
8281 mcl_audit_free(sp->sl_base, NMBPG);
8282 break;
8283 case MC_BIGCL:
8284 {
8285 m_infree(mc) -= NBCLPG;
8286 m_total(mc) -= NBCLPG;
8287 if (mclaudit != NULL)
8288 mcl_audit_free(sp->sl_base, NMBPG);
8289 break;
8290 }
8291 case MC_16KCL:
8292 m_infree(mc)--;
8293 m_total(mc)--;
8294 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
8295 nsp = nsp->sl_next;
8296 VERIFY(nsp->sl_refcnt == 0 &&
8297 nsp->sl_base != NULL &&
8298 nsp->sl_len == 0);
8299 slab_init(nsp, 0, 0, NULL, NULL, 0, 0,
8300 0);
8301 nsp->sl_flags = 0;
8302 }
8303 if (mclaudit != NULL) {
8304 if (sp->sl_len == PAGE_SIZE) {
8305 mcl_audit_free(sp->sl_base,
8306 NMBPG);
8307 } else {
8308 mcl_audit_free(sp->sl_base, 1);
8309 }
8310 }
8311 break;
8312 default:
8313 /*
8314 * The composite classes have their own
8315 * freelist (m_cobjlist), so we only
8316 * process rudimentary classes here.
8317 */
8318 VERIFY(0);
8319 }
8320 m_release_cnt(mc) += m_size(mc);
8321 released += m_size(mc);
8322 VERIFY(sp->sl_base != NULL &&
8323 sp->sl_len >= PAGE_SIZE);
8324 offset = MTOPG(sp->sl_base);
8325 /*
8326 * Make sure the IOMapper points to a valid, but
8327 * bogus, address. This should prevent further DMA
8328 * accesses to freed memory.
8329 */
8330 IOMapperInsertPage(mcl_paddr_base, offset, scratch_pa);
8331 mcl_paddr[offset] = 0;
8332 kmem_free(mb_map, (vm_offset_t)sp->sl_base,
8333 sp->sl_len);
8334 slab_init(sp, 0, 0, NULL, NULL, 0, 0, 0);
8335 sp->sl_flags = 0;
8336 }
8337 }
8338 mbstat.m_drain++;
8339 mbstat.m_bigclusters = m_total(MC_BIGCL);
8340 mbstat.m_clusters = m_total(MC_CL);
8341 mbstat.m_mbufs = m_total(MC_MBUF);
8342 mbuf_stat_sync();
8343 mbuf_mtypes_sync(TRUE);
8344}
8345
8346__private_extern__ void
8347mbuf_drain(boolean_t ignore_waiters)
8348{
8349 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_NOTOWNED);
8350 if (!mbuf_drain_checks(ignore_waiters))
8351 return;
8352 lck_mtx_lock(mbuf_mlock);
8353 mbuf_drain_locked(ignore_waiters);
8354 lck_mtx_unlock(mbuf_mlock);
8355}
8356
8357
8358static int
8359m_drain_force_sysctl SYSCTL_HANDLER_ARGS
8360{
8361#pragma unused(arg1, arg2)
8362 int val = 0, err;
8363
8364 err = sysctl_handle_int(oidp, &val, 0, req);
8365 if (err != 0 || req->newptr == USER_ADDR_NULL)
8366 return (err);
8367 if (val) {
8368 mbuf_drain(TRUE);
8369 }
8370
8371 return (err);
8372}
8373
8374#if DEBUG || DEVELOPMENT
8375static void
8376_mbwdog_logger(const char *func, const int line, const char *fmt, ...)
8377{
8378 va_list ap;
8379 struct timeval now;
8380 char str[384], p[256];
8381 int len;
8382
8383 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
8384 if (mbwdog_logging == NULL) {
8385 mbwdog_logging = _MALLOC(mbwdog_logging_size,
8386 M_TEMP, M_ZERO|M_NOWAIT);
8387 if (mbwdog_logging == NULL)
8388 return;
8389 }
8390 va_start(ap, fmt);
8391 vsnprintf(p, sizeof(p), fmt, ap);
8392 va_end(ap);
8393 microuptime(&now);
8394 len = snprintf(str, sizeof(str),
8395 "\n%ld.%d (%d/%llx) %s:%d %s",
8396 now.tv_sec, now.tv_usec,
8397 current_proc()->p_pid,
8398 (uint64_t)VM_KERNEL_ADDRPERM(current_thread()),
8399 func, line, p);
8400 if (len < 0)
8401 return;
8402 if (mbwdog_logging_used + len > mbwdog_logging_size) {
8403 mbwdog_logging_used = mbwdog_logging_used / 2;
8404 memmove(mbwdog_logging, mbwdog_logging + mbwdog_logging_used,
8405 mbwdog_logging_size - mbwdog_logging_used);
8406 mbwdog_logging[mbwdog_logging_used] = 0;
8407 }
8408 strlcat(mbwdog_logging, str, mbwdog_logging_size);
8409 mbwdog_logging_used += len;
8410}
8411
8412static int
8413sysctl_mbwdog_log SYSCTL_HANDLER_ARGS
8414{
8415#pragma unused(oidp, arg1, arg2)
8416 return SYSCTL_OUT(req, mbwdog_logging, mbwdog_logging_used);
8417}
8418SYSCTL_DECL(_kern_ipc);
8419SYSCTL_PROC(_kern_ipc, OID_AUTO, mbwdog_log,
8420 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_LOCKED,
8421 0, 0, sysctl_mbwdog_log, "A", "");
8422
8423static int mbtest_val;
8424static int mbtest_running;
8425
8426static void mbtest_thread(__unused void *arg)
8427{
8428 int i;
8429 int scale_down = 1;
8430 int iterations = 250;
8431 int allocations = nmbclusters;
8432 iterations = iterations / scale_down;
8433 allocations = allocations / scale_down;
8434 printf("%s thread starting\n", __func__);
8435 for (i = 0; i < iterations; i++) {
8436 unsigned int needed = allocations;
8437 struct mbuf *m1, *m2, *m3;
8438
8439 if (njcl > 0) {
8440 needed = allocations;
8441 m3 = m_getpackets_internal(&needed, 0, M_DONTWAIT, 0, M16KCLBYTES);
8442 m_freem_list(m3);
8443 }
8444
8445 needed = allocations;
8446 m2 = m_getpackets_internal(&needed, 0, M_DONTWAIT, 0, MBIGCLBYTES);
8447 m_freem_list(m2);
8448
8449 m1 = m_getpackets_internal(&needed, 0, M_DONTWAIT, 0, MCLBYTES);
8450 m_freem_list(m1);
8451 }
8452
8453 printf("%s thread ending\n", __func__);
8454
8455 OSDecrementAtomic(&mbtest_running);
8456 wakeup_one((caddr_t)&mbtest_running);
8457}
8458
8459static void sysctl_mbtest(void)
8460{
8461 /* We launch three threads - wait for all of them */
8462 OSIncrementAtomic(&mbtest_running);
8463 OSIncrementAtomic(&mbtest_running);
8464 OSIncrementAtomic(&mbtest_running);
8465
8466 thread_call_func_delayed((thread_call_func_t)mbtest_thread, NULL, 10);
8467 thread_call_func_delayed((thread_call_func_t)mbtest_thread, NULL, 10);
8468 thread_call_func_delayed((thread_call_func_t)mbtest_thread, NULL, 10);
8469
8470 while (mbtest_running) {
8471 msleep((caddr_t)&mbtest_running, NULL, PUSER, "mbtest_running", NULL);
8472 }
8473}
8474
8475static int
8476mbtest SYSCTL_HANDLER_ARGS
8477{
8478#pragma unused(arg1, arg2)
8479 int error = 0, val, oldval = mbtest_val;
8480
8481 val = oldval;
8482 error = sysctl_handle_int(oidp, &val, 0, req);
8483 if (error || !req->newptr)
8484 return (error);
8485
8486 if (val != oldval)
8487 sysctl_mbtest();
8488
8489 mbtest_val = val;
8490
8491 return (error);
8492}
8493#endif // DEBUG || DEVELOPMENT
8494
8495static void
8496mtracelarge_register(size_t size)
8497{
8498 int i;
8499 struct mtracelarge *trace;
8500 uintptr_t bt[MLEAK_STACK_DEPTH];
8501 unsigned int depth;
8502
8503 depth = backtrace(bt, MLEAK_STACK_DEPTH);
8504 /* Check if this entry is already on the list. */
8505 for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
8506 trace = &mtracelarge_table[i];
8507 if (trace->size == size && trace->depth == depth &&
8508 memcmp(bt, trace->addr, depth * sizeof(uintptr_t)) == 0) {
8509 return;
8510 }
8511
8512 }
8513 for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
8514 trace = &mtracelarge_table[i];
8515 if (size > trace->size) {
8516 trace->depth = depth;
8517 memcpy(trace->addr, bt, depth * sizeof(uintptr_t));
8518 trace->size = size;
8519 break;
8520 }
8521 }
8522}
8523
8524SYSCTL_DECL(_kern_ipc);
8525#if DEBUG || DEVELOPMENT
8526SYSCTL_PROC(_kern_ipc, OID_AUTO, mbtest,
8527 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &mbtest_val, 0, &mbtest, "I",
8528 "Toggle to test mbufs");
8529#endif
8530SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat,
8531 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
8532 0, 0, mbstat_sysctl, "S,mbstat", "");
8533SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_stat,
8534 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
8535 0, 0, mb_stat_sysctl, "S,mb_stat", "");
8536SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_top_trace,
8537 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
8538 0, 0, mleak_top_trace_sysctl, "S,mb_top_trace", "");
8539SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_table,
8540 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
8541 0, 0, mleak_table_sysctl, "S,mleak_table", "");
8542SYSCTL_INT(_kern_ipc, OID_AUTO, mleak_sample_factor,
8543 CTLFLAG_RW | CTLFLAG_LOCKED, &mleak_table.mleak_sample_factor, 0, "");
8544SYSCTL_INT(_kern_ipc, OID_AUTO, mb_normalized,
8545 CTLFLAG_RD | CTLFLAG_LOCKED, &mb_normalized, 0, "");
8546SYSCTL_INT(_kern_ipc, OID_AUTO, mb_watchdog,
8547 CTLFLAG_RW | CTLFLAG_LOCKED, &mb_watchdog, 0, "");
8548SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_drain_force,
8549 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
8550 m_drain_force_sysctl, "I",
8551 "Forces the mbuf garbage collection to run");
8552SYSCTL_INT(_kern_ipc, OID_AUTO, mb_drain_maxint,
8553 CTLFLAG_RW | CTLFLAG_LOCKED, &mb_drain_maxint, 0,
8554 "Minimum time interval between garbage collection");
8555