1/*
2 * Copyright (c) 2004-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#define __KPI__
30
31#include <sys/param.h>
32#include <sys/mbuf.h>
33#include <sys/mcache.h>
34#include <sys/socket.h>
35#include <kern/debug.h>
36#include <libkern/OSAtomic.h>
37#include <kern/kalloc.h>
38#include <string.h>
39#include <net/dlil.h>
40#include <netinet/in.h>
41#include <netinet/ip_var.h>
42
43#include "net/net_str_id.h"
44
45/* mbuf flags visible to KPI clients; do not add private flags here */
46static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
47 MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
48 MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS);
49
50/* Unalterable mbuf flags */
51static const mbuf_flags_t mbuf_cflags_mask = (MBUF_EXT);
52
53#define MAX_MBUF_TX_COMPL_FUNC 32
54mbuf_tx_compl_func
55mbuf_tx_compl_table[MAX_MBUF_TX_COMPL_FUNC];
56extern lck_rw_t *mbuf_tx_compl_tbl_lock;
57u_int32_t mbuf_tx_compl_index = 0;
58
59#if (DEVELOPMENT || DEBUG)
60int mbuf_tx_compl_debug = 0;
61SInt64 mbuf_tx_compl_outstanding __attribute__((aligned(8))) = 0;
62u_int64_t mbuf_tx_compl_aborted __attribute__((aligned(8))) = 0;
63
64SYSCTL_DECL(_kern_ipc);
65SYSCTL_NODE(_kern_ipc, OID_AUTO, mbtxcf,
66 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "");
67SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, debug,
68 CTLFLAG_RW | CTLFLAG_LOCKED, &mbuf_tx_compl_debug, 0, "");
69SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, index,
70 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_index, 0, "");
71SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, oustanding,
72 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_outstanding, "");
73SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, aborted,
74 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_aborted, "");
75#endif /* (DEBUG || DEVELOPMENT) */
76
77void *
78mbuf_data(mbuf_t mbuf)
79{
80 return (mbuf->m_data);
81}
82
83void *
84mbuf_datastart(mbuf_t mbuf)
85{
86 if (mbuf->m_flags & M_EXT)
87 return (mbuf->m_ext.ext_buf);
88 if (mbuf->m_flags & M_PKTHDR)
89 return (mbuf->m_pktdat);
90 return (mbuf->m_dat);
91}
92
93errno_t
94mbuf_setdata(mbuf_t mbuf, void *data, size_t len)
95{
96 size_t start = (size_t)((char *)mbuf_datastart(mbuf));
97 size_t maxlen = mbuf_maxlen(mbuf);
98
99 if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
100 return (EINVAL);
101 mbuf->m_data = data;
102 mbuf->m_len = len;
103
104 return (0);
105}
106
107errno_t
108mbuf_align_32(mbuf_t mbuf, size_t len)
109{
110 if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
111 return (ENOTSUP);
112 mbuf->m_data = mbuf_datastart(mbuf);
113 mbuf->m_data +=
114 ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
115
116 return (0);
117}
118
119/*
120 * This function is used to provide mcl_to_paddr via symbol indirection,
121 * please avoid any change in behavior or remove the indirection in
122 * config/Unsupported*
123 */
124addr64_t
125mbuf_data_to_physical(void *ptr)
126{
127 return ((addr64_t)mcl_to_paddr(ptr));
128}
129
130errno_t
131mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
132{
133 /* Must set *mbuf to NULL in failure case */
134 *mbuf = m_get(how, type);
135
136 return (*mbuf == NULL ? ENOMEM : 0);
137}
138
139errno_t
140mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
141{
142 /* Must set *mbuf to NULL in failure case */
143 *mbuf = m_gethdr(how, type);
144
145 return (*mbuf == NULL ? ENOMEM : 0);
146}
147
148errno_t
149mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
150 caddr_t extbuf, void (*extfree)(caddr_t, u_int, caddr_t),
151 size_t extsize, caddr_t extarg)
152{
153 if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0)
154 return (EINVAL);
155
156 if ((*mbuf = m_clattach(*mbuf, type, extbuf,
157 extfree, extsize, extarg, how, 0)) == NULL)
158 return (ENOMEM);
159
160 return (0);
161}
162
163errno_t
164mbuf_ring_cluster_alloc(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
165 void (*extfree)(caddr_t, u_int, caddr_t), size_t *size)
166{
167 caddr_t extbuf = NULL;
168 errno_t err;
169
170 if (mbuf == NULL || extfree == NULL || size == NULL || *size == 0)
171 return (EINVAL);
172
173 if ((err = mbuf_alloccluster(how, size, &extbuf)) != 0)
174 return (err);
175
176 if ((*mbuf = m_clattach(*mbuf, type, extbuf,
177 extfree, *size, NULL, how, 1)) == NULL) {
178 mbuf_freecluster(extbuf, *size);
179 return (ENOMEM);
180 }
181
182 return (0);
183}
184
185int
186mbuf_ring_cluster_is_active(mbuf_t mbuf)
187{
188 return (m_ext_paired_is_active(mbuf));
189}
190
191errno_t
192mbuf_ring_cluster_activate(mbuf_t mbuf)
193{
194 if (mbuf_ring_cluster_is_active(mbuf))
195 return (EBUSY);
196
197 m_ext_paired_activate(mbuf);
198 return (0);
199}
200
201errno_t
202mbuf_cluster_set_prop(mbuf_t mbuf, u_int32_t oldprop, u_int32_t newprop)
203{
204 if (mbuf == NULL || !(mbuf->m_flags & M_EXT))
205 return (EINVAL);
206
207 return (m_ext_set_prop(mbuf, oldprop, newprop) ? 0 : EBUSY);
208}
209
210errno_t
211mbuf_cluster_get_prop(mbuf_t mbuf, u_int32_t *prop)
212{
213 if (mbuf == NULL || prop == NULL || !(mbuf->m_flags & M_EXT))
214 return (EINVAL);
215
216 *prop = m_ext_get_prop(mbuf);
217 return (0);
218}
219
220errno_t
221mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
222{
223 if (size == NULL || *size == 0 || addr == NULL)
224 return (EINVAL);
225
226 *addr = NULL;
227
228 /* Jumbo cluster pool not available? */
229 if (*size > MBIGCLBYTES && njcl == 0)
230 return (ENOTSUP);
231
232 if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
233 *size = MCLBYTES;
234 else if (*size > MCLBYTES && *size <= MBIGCLBYTES &&
235 (*addr = m_bigalloc(how)) != NULL)
236 *size = MBIGCLBYTES;
237 else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES &&
238 (*addr = m_16kalloc(how)) != NULL)
239 *size = M16KCLBYTES;
240 else
241 *size = 0;
242
243 if (*addr == NULL)
244 return (ENOMEM);
245
246 return (0);
247}
248
249void
250mbuf_freecluster(caddr_t addr, size_t size)
251{
252 if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES)
253 panic("%s: invalid size (%ld) for cluster %p", __func__,
254 size, (void *)addr);
255
256 if (size == MCLBYTES)
257 m_mclfree(addr);
258 else if (size == MBIGCLBYTES)
259 m_bigfree(addr, MBIGCLBYTES, NULL);
260 else if (njcl > 0)
261 m_16kfree(addr, M16KCLBYTES, NULL);
262 else
263 panic("%s: freeing jumbo cluster to an empty pool", __func__);
264}
265
266errno_t
267mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t *mbuf)
268{
269 /* Must set *mbuf to NULL in failure case */
270 errno_t error = 0;
271 int created = 0;
272
273 if (mbuf == NULL)
274 return (EINVAL);
275 if (*mbuf == NULL) {
276 *mbuf = m_get(how, type);
277 if (*mbuf == NULL)
278 return (ENOMEM);
279 created = 1;
280 }
281 /*
282 * At the time this code was written, m_{mclget,mbigget,m16kget}
283 * would always return the same value that was passed in to it.
284 */
285 if (size == MCLBYTES) {
286 *mbuf = m_mclget(*mbuf, how);
287 } else if (size == MBIGCLBYTES) {
288 *mbuf = m_mbigget(*mbuf, how);
289 } else if (size == M16KCLBYTES) {
290 if (njcl > 0) {
291 *mbuf = m_m16kget(*mbuf, how);
292 } else {
293 /* Jumbo cluster pool not available? */
294 error = ENOTSUP;
295 goto out;
296 }
297 } else {
298 error = EINVAL;
299 goto out;
300 }
301 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
302 error = ENOMEM;
303out:
304 if (created && error != 0) {
305 mbuf_free(*mbuf);
306 *mbuf = NULL;
307 }
308 return (error);
309}
310
311errno_t
312mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
313{
314 /* Must set *mbuf to NULL in failure case */
315 errno_t error = 0;
316 int created = 0;
317 if (mbuf == NULL)
318 return (EINVAL);
319 if (*mbuf == NULL) {
320 error = mbuf_get(how, type, mbuf);
321 if (error)
322 return (error);
323 created = 1;
324 }
325
326 /*
327 * At the time this code was written, m_mclget would always
328 * return the same value that was passed in to it.
329 */
330 *mbuf = m_mclget(*mbuf, how);
331
332 if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
333 mbuf_free(*mbuf);
334 *mbuf = NULL;
335 }
336 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
337 error = ENOMEM;
338 return (error);
339}
340
341
342errno_t
343mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
344{
345 /* Must set *mbuf to NULL in failure case */
346 errno_t error = 0;
347
348 *mbuf = m_getpacket_how(how);
349
350 if (*mbuf == NULL) {
351 if (how == MBUF_WAITOK)
352 error = ENOMEM;
353 else
354 error = EWOULDBLOCK;
355 }
356
357 return (error);
358}
359
360/*
361 * This function is used to provide m_free via symbol indirection, please avoid
362 * any change in behavior or remove the indirection in config/Unsupported*
363 */
364mbuf_t
365mbuf_free(mbuf_t mbuf)
366{
367 return (m_free(mbuf));
368}
369
370/*
371 * This function is used to provide m_freem via symbol indirection, please avoid
372 * any change in behavior or remove the indirection in config/Unsupported*
373 */
374void
375mbuf_freem(mbuf_t mbuf)
376{
377 m_freem(mbuf);
378}
379
380int
381mbuf_freem_list(mbuf_t mbuf)
382{
383 return (m_freem_list(mbuf));
384}
385
386size_t
387mbuf_leadingspace(const mbuf_t mbuf)
388{
389 return (M_LEADINGSPACE(mbuf));
390}
391
392/*
393 * This function is used to provide m_trailingspace via symbol indirection,
394 * please avoid any change in behavior or remove the indirection in
395 * config/Unsupported*
396 */
397size_t
398mbuf_trailingspace(const mbuf_t mbuf)
399{
400 return (M_TRAILINGSPACE(mbuf));
401}
402
403/* Manipulation */
404errno_t
405mbuf_copym(const mbuf_t src, size_t offset, size_t len,
406 mbuf_how_t how, mbuf_t *new_mbuf)
407{
408 /* Must set *mbuf to NULL in failure case */
409 *new_mbuf = m_copym(src, offset, len, how);
410
411 return (*new_mbuf == NULL ? ENOMEM : 0);
412}
413
414errno_t
415mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
416{
417 /* Must set *new_mbuf to NULL in failure case */
418 *new_mbuf = m_dup(src, how);
419
420 return (*new_mbuf == NULL ? ENOMEM : 0);
421}
422
423errno_t
424mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
425{
426 /* Must set *orig to NULL in failure case */
427 *orig = m_prepend_2(*orig, len, how, 0);
428
429 return (*orig == NULL ? ENOMEM : 0);
430}
431
432errno_t
433mbuf_split(mbuf_t src, size_t offset,
434 mbuf_how_t how, mbuf_t *new_mbuf)
435{
436 /* Must set *new_mbuf to NULL in failure case */
437 *new_mbuf = m_split(src, offset, how);
438
439 return (*new_mbuf == NULL ? ENOMEM : 0);
440}
441
442errno_t
443mbuf_pullup(mbuf_t *mbuf, size_t len)
444{
445 /* Must set *mbuf to NULL in failure case */
446 *mbuf = m_pullup(*mbuf, len);
447
448 return (*mbuf == NULL ? ENOMEM : 0);
449}
450
451errno_t
452mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
453{
454 /* Must set *location to NULL in failure case */
455 int new_offset;
456 *location = m_pulldown(src, *offset, len, &new_offset);
457 *offset = new_offset;
458
459 return (*location == NULL ? ENOMEM : 0);
460}
461
462/*
463 * This function is used to provide m_adj via symbol indirection, please avoid
464 * any change in behavior or remove the indirection in config/Unsupported*
465 */
466void
467mbuf_adj(mbuf_t mbuf, int len)
468{
469 m_adj(mbuf, len);
470}
471
472errno_t
473mbuf_adjustlen(mbuf_t m, int amount)
474{
475 /* Verify m_len will be valid after adding amount */
476 if (amount > 0) {
477 int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
478 m->m_len;
479
480 if ((size_t)(amount + used) > mbuf_maxlen(m))
481 return (EINVAL);
482 } else if (-amount > m->m_len) {
483 return (EINVAL);
484 }
485
486 m->m_len += amount;
487 return (0);
488}
489
490mbuf_t
491mbuf_concatenate(mbuf_t dst, mbuf_t src)
492{
493 if (dst == NULL)
494 return (NULL);
495
496 m_cat(dst, src);
497
498 /* return dst as is in the current implementation */
499 return (dst);
500}
501errno_t
502mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void *out_data)
503{
504 /* Copied m_copydata, added error handling (don't just panic) */
505 int count;
506 mbuf_t m = m0;
507
508 while (off > 0) {
509 if (m == 0)
510 return (EINVAL);
511 if (off < (size_t)m->m_len)
512 break;
513 off -= m->m_len;
514 m = m->m_next;
515 }
516 while (len > 0) {
517 if (m == 0)
518 return (EINVAL);
519 count = m->m_len - off > len ? len : m->m_len - off;
520 bcopy(mtod(m, caddr_t) + off, out_data, count);
521 len -= count;
522 out_data = ((char *)out_data) + count;
523 off = 0;
524 m = m->m_next;
525 }
526
527 return (0);
528}
529
530int
531mbuf_mclhasreference(mbuf_t mbuf)
532{
533 if ((mbuf->m_flags & M_EXT))
534 return (m_mclhasreference(mbuf));
535 else
536 return (0);
537}
538
539
540/* mbuf header */
541mbuf_t
542mbuf_next(const mbuf_t mbuf)
543{
544 return (mbuf->m_next);
545}
546
547errno_t
548mbuf_setnext(mbuf_t mbuf, mbuf_t next)
549{
550 if (next && ((next)->m_nextpkt != NULL ||
551 (next)->m_type == MT_FREE))
552 return (EINVAL);
553 mbuf->m_next = next;
554
555 return (0);
556}
557
558mbuf_t
559mbuf_nextpkt(const mbuf_t mbuf)
560{
561 return (mbuf->m_nextpkt);
562}
563
564void
565mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
566{
567 mbuf->m_nextpkt = nextpkt;
568}
569
570size_t
571mbuf_len(const mbuf_t mbuf)
572{
573 return (mbuf->m_len);
574}
575
576void
577mbuf_setlen(mbuf_t mbuf, size_t len)
578{
579 mbuf->m_len = len;
580}
581
582size_t
583mbuf_maxlen(const mbuf_t mbuf)
584{
585 if (mbuf->m_flags & M_EXT)
586 return (mbuf->m_ext.ext_size);
587 return (&mbuf->m_dat[MLEN] - ((char *)mbuf_datastart(mbuf)));
588}
589
590mbuf_type_t
591mbuf_type(const mbuf_t mbuf)
592{
593 return (mbuf->m_type);
594}
595
596errno_t
597mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
598{
599 if (new_type == MBUF_TYPE_FREE)
600 return (EINVAL);
601
602 m_mchtype(mbuf, new_type);
603
604 return (0);
605}
606
607mbuf_flags_t
608mbuf_flags(const mbuf_t mbuf)
609{
610 return (mbuf->m_flags & mbuf_flags_mask);
611}
612
613errno_t
614mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
615{
616 errno_t ret = 0;
617 mbuf_flags_t oflags = mbuf->m_flags;
618
619 /*
620 * 1. Return error if public but un-alterable flags are changed
621 * in flags argument.
622 * 2. Return error if bits other than public flags are set in passed
623 * flags argument.
624 * Please note that private flag bits must be passed as reset by
625 * kexts, as they must use mbuf_flags KPI to get current set of
626 * mbuf flags and mbuf_flags KPI does not expose private flags.
627 */
628 if ((flags ^ oflags) & mbuf_cflags_mask) {
629 ret = EINVAL;
630 } else if (flags & ~mbuf_flags_mask) {
631 ret = EINVAL;
632 } else {
633 mbuf->m_flags = flags | (mbuf->m_flags & ~mbuf_flags_mask);
634 /*
635 * If M_PKTHDR bit has changed, we have work to do;
636 * m_reinit() will take care of setting/clearing the
637 * bit, as well as the rest of bookkeeping.
638 */
639 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) {
640 mbuf->m_flags ^= M_PKTHDR; /* restore */
641 ret = m_reinit(mbuf,
642 (mbuf->m_flags & M_PKTHDR) ? 0 : 1);
643 }
644 }
645
646 return (ret);
647}
648
649errno_t
650mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
651{
652 errno_t ret = 0;
653
654 if (mask & (~mbuf_flags_mask | mbuf_cflags_mask)) {
655 ret = EINVAL;
656 } else {
657 mbuf_flags_t oflags = mbuf->m_flags;
658 mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
659 /*
660 * If M_PKTHDR bit has changed, we have work to do;
661 * m_reinit() will take care of setting/clearing the
662 * bit, as well as the rest of bookkeeping.
663 */
664 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) {
665 mbuf->m_flags ^= M_PKTHDR; /* restore */
666 ret = m_reinit(mbuf,
667 (mbuf->m_flags & M_PKTHDR) ? 0 : 1);
668 }
669 }
670
671 return (ret);
672}
673
674errno_t
675mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
676{
677 if (((src)->m_flags & M_PKTHDR) == 0)
678 return (EINVAL);
679
680 m_copy_pkthdr(dest, src);
681
682 return (0);
683}
684
685size_t
686mbuf_pkthdr_len(const mbuf_t mbuf)
687{
688 return (mbuf->m_pkthdr.len);
689}
690
691__private_extern__ size_t
692mbuf_pkthdr_maxlen(mbuf_t m)
693{
694 size_t maxlen = 0;
695 mbuf_t n = m;
696
697 while (n) {
698 maxlen += mbuf_maxlen(n);
699 n = mbuf_next(n);
700 }
701 return (maxlen);
702}
703
704void
705mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
706{
707 mbuf->m_pkthdr.len = len;
708}
709
710void
711mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
712{
713 mbuf->m_pkthdr.len += amount;
714}
715
716ifnet_t
717mbuf_pkthdr_rcvif(const mbuf_t mbuf)
718{
719 /*
720 * If we reference count ifnets, we should take a reference here
721 * before returning
722 */
723 return (mbuf->m_pkthdr.rcvif);
724}
725
726errno_t
727mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
728{
729 /* May want to walk ifnet list to determine if interface is valid */
730 mbuf->m_pkthdr.rcvif = (struct ifnet *)ifnet;
731 return (0);
732}
733
734void*
735mbuf_pkthdr_header(const mbuf_t mbuf)
736{
737 return (mbuf->m_pkthdr.pkt_hdr);
738}
739
740void
741mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
742{
743 mbuf->m_pkthdr.pkt_hdr = (void*)header;
744}
745
746void
747mbuf_inbound_modified(mbuf_t mbuf)
748{
749 /* Invalidate hardware generated checksum flags */
750 mbuf->m_pkthdr.csum_flags = 0;
751}
752
753void
754mbuf_outbound_finalize(struct mbuf *m, u_int32_t pf, size_t o)
755{
756 /* Generate the packet in software, client needs it */
757 switch (pf) {
758 case PF_INET:
759 (void) in_finalize_cksum(m, o, m->m_pkthdr.csum_flags);
760 break;
761
762 case PF_INET6:
763#if INET6
764 /*
765 * Checksum offload should not have been enabled when
766 * extension headers exist; indicate that the callee
767 * should skip such case by setting optlen to -1.
768 */
769 (void) in6_finalize_cksum(m, o, -1, -1, m->m_pkthdr.csum_flags);
770#endif /* INET6 */
771 break;
772
773 default:
774 break;
775 }
776}
777
778errno_t
779mbuf_set_vlan_tag(
780 mbuf_t mbuf,
781 u_int16_t vlan)
782{
783 mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
784 mbuf->m_pkthdr.vlan_tag = vlan;
785
786 return (0);
787}
788
789errno_t
790mbuf_get_vlan_tag(
791 mbuf_t mbuf,
792 u_int16_t *vlan)
793{
794 if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
795 return (ENXIO); // No vlan tag set
796
797 *vlan = mbuf->m_pkthdr.vlan_tag;
798
799 return (0);
800}
801
802errno_t
803mbuf_clear_vlan_tag(
804 mbuf_t mbuf)
805{
806 mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
807 mbuf->m_pkthdr.vlan_tag = 0;
808
809 return (0);
810}
811
812static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
813 MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP |
814 MBUF_CSUM_PARTIAL | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6;
815
816errno_t
817mbuf_set_csum_requested(
818 mbuf_t mbuf,
819 mbuf_csum_request_flags_t request,
820 u_int32_t value)
821{
822 request &= mbuf_valid_csum_request_flags;
823 mbuf->m_pkthdr.csum_flags =
824 (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
825 mbuf->m_pkthdr.csum_data = value;
826
827 return (0);
828}
829
830static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags =
831 MBUF_TSO_IPV4 | MBUF_TSO_IPV6;
832
833errno_t
834mbuf_get_tso_requested(
835 mbuf_t mbuf,
836 mbuf_tso_request_flags_t *request,
837 u_int32_t *value)
838{
839 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
840 request == NULL || value == NULL)
841 return (EINVAL);
842
843 *request = mbuf->m_pkthdr.csum_flags;
844 *request &= mbuf_valid_tso_request_flags;
845 if (*request && value != NULL)
846 *value = mbuf->m_pkthdr.tso_segsz;
847
848 return (0);
849}
850
851errno_t
852mbuf_get_csum_requested(
853 mbuf_t mbuf,
854 mbuf_csum_request_flags_t *request,
855 u_int32_t *value)
856{
857 *request = mbuf->m_pkthdr.csum_flags;
858 *request &= mbuf_valid_csum_request_flags;
859 if (value != NULL) {
860 *value = mbuf->m_pkthdr.csum_data;
861 }
862
863 return (0);
864}
865
866errno_t
867mbuf_clear_csum_requested(
868 mbuf_t mbuf)
869{
870 mbuf->m_pkthdr.csum_flags &= 0xffff0000;
871 mbuf->m_pkthdr.csum_data = 0;
872
873 return (0);
874}
875
876static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
877 MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
878 MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_PARTIAL;
879
880errno_t
881mbuf_set_csum_performed(
882 mbuf_t mbuf,
883 mbuf_csum_performed_flags_t performed,
884 u_int32_t value)
885{
886 performed &= mbuf_valid_csum_performed_flags;
887 mbuf->m_pkthdr.csum_flags =
888 (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
889 mbuf->m_pkthdr.csum_data = value;
890
891 return (0);
892}
893
894errno_t
895mbuf_get_csum_performed(
896 mbuf_t mbuf,
897 mbuf_csum_performed_flags_t *performed,
898 u_int32_t *value)
899{
900 *performed =
901 mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
902 *value = mbuf->m_pkthdr.csum_data;
903
904 return (0);
905}
906
907errno_t
908mbuf_clear_csum_performed(
909 mbuf_t mbuf)
910{
911 mbuf->m_pkthdr.csum_flags &= 0xffff0000;
912 mbuf->m_pkthdr.csum_data = 0;
913
914 return (0);
915}
916
917errno_t
918mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
919 u_int16_t *csum)
920{
921 if (mbuf == NULL || length == 0 || csum == NULL ||
922 (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
923 return (EINVAL);
924
925 *csum = inet_cksum(mbuf, protocol, offset, length);
926 return (0);
927}
928
929#if INET6
930errno_t
931mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
932 u_int16_t *csum)
933{
934 if (mbuf == NULL || length == 0 || csum == NULL ||
935 (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
936 return (EINVAL);
937
938 *csum = inet6_cksum(mbuf, protocol, offset, length);
939 return (0);
940}
941#else /* INET6 */
942errno_t
943mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
944 __unused u_int32_t offset, __unused u_int32_t length,
945 __unused u_int16_t *csum)
946{
947 panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
948 return (0);
949}
950
951u_int16_t
952inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
953 __unused unsigned int off, __unused unsigned int len)
954{
955 panic("inet6_cksum() doesn't exist on this platform\n");
956 return (0);
957}
958
959void nd6_lookup_ipv6(void);
960void
961nd6_lookup_ipv6(void)
962{
963 panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
964}
965
966int
967in6addr_local(__unused struct in6_addr *a)
968{
969 panic("in6addr_local() doesn't exist on this platform\n");
970 return (0);
971}
972
973void nd6_storelladdr(void);
974void
975nd6_storelladdr(void)
976{
977 panic("nd6_storelladdr() doesn't exist on this platform\n");
978}
979#endif /* INET6 */
980
981/*
982 * Mbuf tag KPIs
983 */
984
985#define MTAG_FIRST_ID FIRST_KPI_STR_ID
986
987errno_t
988mbuf_tag_id_find(
989 const char *string,
990 mbuf_tag_id_t *out_id)
991{
992 return (net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1));
993}
994
995errno_t
996mbuf_tag_allocate(
997 mbuf_t mbuf,
998 mbuf_tag_id_t id,
999 mbuf_tag_type_t type,
1000 size_t length,
1001 mbuf_how_t how,
1002 void** data_p)
1003{
1004 struct m_tag *tag;
1005 u_int32_t mtag_id_first, mtag_id_last;
1006
1007 if (data_p != NULL)
1008 *data_p = NULL;
1009
1010 /* Sanity check parameters */
1011 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1012 NSI_MBUF_TAG);
1013 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1014 id < mtag_id_first || id > mtag_id_last || length < 1 ||
1015 (length & 0xffff0000) != 0 || data_p == NULL) {
1016 return (EINVAL);
1017 }
1018
1019 /* Make sure this mtag hasn't already been allocated */
1020 tag = m_tag_locate(mbuf, id, type, NULL);
1021 if (tag != NULL) {
1022 return (EEXIST);
1023 }
1024
1025 /* Allocate an mtag */
1026 tag = m_tag_create(id, type, length, how, mbuf);
1027 if (tag == NULL) {
1028 return (how == M_WAITOK ? ENOMEM : EWOULDBLOCK);
1029 }
1030
1031 /* Attach the mtag and set *data_p */
1032 m_tag_prepend(mbuf, tag);
1033 *data_p = tag + 1;
1034
1035 return (0);
1036}
1037
1038errno_t
1039mbuf_tag_find(
1040 mbuf_t mbuf,
1041 mbuf_tag_id_t id,
1042 mbuf_tag_type_t type,
1043 size_t *length,
1044 void **data_p)
1045{
1046 struct m_tag *tag;
1047 u_int32_t mtag_id_first, mtag_id_last;
1048
1049 if (length != NULL)
1050 *length = 0;
1051 if (data_p != NULL)
1052 *data_p = NULL;
1053
1054 /* Sanity check parameters */
1055 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1056 NSI_MBUF_TAG);
1057 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1058 id < mtag_id_first || id > mtag_id_last || length == NULL ||
1059 data_p == NULL) {
1060 return (EINVAL);
1061 }
1062
1063 /* Locate an mtag */
1064 tag = m_tag_locate(mbuf, id, type, NULL);
1065 if (tag == NULL) {
1066 return (ENOENT);
1067 }
1068
1069 /* Copy out the pointer to the data and the lenght value */
1070 *length = tag->m_tag_len;
1071 *data_p = tag + 1;
1072
1073 return (0);
1074}
1075
1076void
1077mbuf_tag_free(
1078 mbuf_t mbuf,
1079 mbuf_tag_id_t id,
1080 mbuf_tag_type_t type)
1081{
1082 struct m_tag *tag;
1083 u_int32_t mtag_id_first, mtag_id_last;
1084
1085 /* Sanity check parameters */
1086 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1087 NSI_MBUF_TAG);
1088 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1089 id < mtag_id_first || id > mtag_id_last)
1090 return;
1091
1092 tag = m_tag_locate(mbuf, id, type, NULL);
1093 if (tag == NULL) {
1094 return;
1095 }
1096
1097 m_tag_delete(mbuf, tag);
1098}
1099
1100/*
1101 * Maximum length of driver auxiliary data; keep this small to
1102 * fit in a single mbuf to avoid wasting memory, rounded down to
1103 * the nearest 64-bit boundary. This takes into account mbuf
1104 * tag-related (m_taghdr + m_tag) as well m_drvaux_tag structs.
1105 */
1106#define MBUF_DRVAUX_MAXLEN \
1107 P2ROUNDDOWN(MLEN - sizeof (struct m_taghdr) - \
1108 M_TAG_ALIGN(sizeof (struct m_drvaux_tag)), sizeof (uint64_t))
1109
1110errno_t
1111mbuf_add_drvaux(mbuf_t mbuf, mbuf_how_t how, u_int32_t family,
1112 u_int32_t subfamily, size_t length, void **data_p)
1113{
1114 struct m_drvaux_tag *p;
1115 struct m_tag *tag;
1116
1117 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) ||
1118 length == 0 || length > MBUF_DRVAUX_MAXLEN)
1119 return (EINVAL);
1120
1121 if (data_p != NULL)
1122 *data_p = NULL;
1123
1124 /* Check if one is already associated */
1125 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1126 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL)
1127 return (EEXIST);
1128
1129 /* Tag is (m_drvaux_tag + module specific data) */
1130 if ((tag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_DRVAUX,
1131 sizeof (*p) + length, how, mbuf)) == NULL)
1132 return ((how == MBUF_WAITOK) ? ENOMEM : EWOULDBLOCK);
1133
1134 p = (struct m_drvaux_tag *)(tag + 1);
1135 p->da_family = family;
1136 p->da_subfamily = subfamily;
1137 p->da_length = length;
1138
1139 /* Associate the tag */
1140 m_tag_prepend(mbuf, tag);
1141
1142 if (data_p != NULL)
1143 *data_p = (p + 1);
1144
1145 return (0);
1146}
1147
1148errno_t
1149mbuf_find_drvaux(mbuf_t mbuf, u_int32_t *family_p, u_int32_t *subfamily_p,
1150 u_int32_t *length_p, void **data_p)
1151{
1152 struct m_drvaux_tag *p;
1153 struct m_tag *tag;
1154
1155 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || data_p == NULL)
1156 return (EINVAL);
1157
1158 *data_p = NULL;
1159
1160 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1161 KERNEL_TAG_TYPE_DRVAUX, NULL)) == NULL)
1162 return (ENOENT);
1163
1164 /* Must be at least size of m_drvaux_tag */
1165 VERIFY(tag->m_tag_len >= sizeof (*p));
1166
1167 p = (struct m_drvaux_tag *)(tag + 1);
1168 VERIFY(p->da_length > 0 && p->da_length <= MBUF_DRVAUX_MAXLEN);
1169
1170 if (family_p != NULL)
1171 *family_p = p->da_family;
1172 if (subfamily_p != NULL)
1173 *subfamily_p = p->da_subfamily;
1174 if (length_p != NULL)
1175 *length_p = p->da_length;
1176
1177 *data_p = (p + 1);
1178
1179 return (0);
1180}
1181
1182void
1183mbuf_del_drvaux(mbuf_t mbuf)
1184{
1185 struct m_tag *tag;
1186
1187 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR))
1188 return;
1189
1190 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1191 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL)
1192 m_tag_delete(mbuf, tag);
1193}
1194
1195/* mbuf stats */
1196void
1197mbuf_stats(struct mbuf_stat *stats)
1198{
1199 stats->mbufs = mbstat.m_mbufs;
1200 stats->clusters = mbstat.m_clusters;
1201 stats->clfree = mbstat.m_clfree;
1202 stats->drops = mbstat.m_drops;
1203 stats->wait = mbstat.m_wait;
1204 stats->drain = mbstat.m_drain;
1205 __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
1206 stats->mcfail = mbstat.m_mcfail;
1207 stats->mpfail = mbstat.m_mpfail;
1208 stats->msize = mbstat.m_msize;
1209 stats->mclbytes = mbstat.m_mclbytes;
1210 stats->minclsize = mbstat.m_minclsize;
1211 stats->mlen = mbstat.m_mlen;
1212 stats->mhlen = mbstat.m_mhlen;
1213 stats->bigclusters = mbstat.m_bigclusters;
1214 stats->bigclfree = mbstat.m_bigclfree;
1215 stats->bigmclbytes = mbstat.m_bigmclbytes;
1216}
1217
1218errno_t
1219mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks,
1220 mbuf_t *mbuf)
1221{
1222 errno_t error;
1223 struct mbuf *m;
1224 unsigned int numpkts = 1;
1225 unsigned int numchunks = maxchunks ? *maxchunks : 0;
1226
1227 if (packetlen == 0) {
1228 error = EINVAL;
1229 goto out;
1230 }
1231 m = m_allocpacket_internal(&numpkts, packetlen,
1232 maxchunks ? &numchunks : NULL, how, 1, 0);
1233 if (m == 0) {
1234 if (maxchunks && *maxchunks && numchunks > *maxchunks)
1235 error = ENOBUFS;
1236 else
1237 error = ENOMEM;
1238 } else {
1239 if (maxchunks)
1240 *maxchunks = numchunks;
1241 error = 0;
1242 *mbuf = m;
1243 }
1244out:
1245 return (error);
1246}
1247
1248errno_t
1249mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen,
1250 unsigned int *maxchunks, mbuf_t *mbuf)
1251{
1252 errno_t error;
1253 struct mbuf *m;
1254 unsigned int numchunks = maxchunks ? *maxchunks : 0;
1255
1256 if (numpkts == 0) {
1257 error = EINVAL;
1258 goto out;
1259 }
1260 if (packetlen == 0) {
1261 error = EINVAL;
1262 goto out;
1263 }
1264 m = m_allocpacket_internal(&numpkts, packetlen,
1265 maxchunks ? &numchunks : NULL, how, 1, 0);
1266 if (m == 0) {
1267 if (maxchunks && *maxchunks && numchunks > *maxchunks)
1268 error = ENOBUFS;
1269 else
1270 error = ENOMEM;
1271 } else {
1272 if (maxchunks)
1273 *maxchunks = numchunks;
1274 error = 0;
1275 *mbuf = m;
1276 }
1277out:
1278 return (error);
1279}
1280
1281__private_extern__ size_t
1282mbuf_pkt_list_len(mbuf_t m)
1283{
1284 size_t len = 0;
1285 mbuf_t n = m;
1286
1287 while (n) {
1288 len += mbuf_pkthdr_len(n);
1289 n = mbuf_nextpkt(n);
1290 }
1291 return (len);
1292}
1293
1294__private_extern__ size_t
1295mbuf_pkt_list_maxlen(mbuf_t m)
1296{
1297 size_t maxlen = 0;
1298 mbuf_t n = m;
1299
1300 while (n) {
1301 maxlen += mbuf_pkthdr_maxlen(n);
1302 n = mbuf_nextpkt(n);
1303 }
1304 return (maxlen);
1305}
1306
1307/*
1308 * mbuf_copyback differs from m_copyback in a few ways:
1309 * 1) mbuf_copyback will allocate clusters for new mbufs we append
1310 * 2) mbuf_copyback will grow the last mbuf in the chain if possible
1311 * 3) mbuf_copyback reports whether or not the operation succeeded
1312 * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
1313 */
1314errno_t
1315mbuf_copyback(
1316 mbuf_t m,
1317 size_t off,
1318 size_t len,
1319 const void *data,
1320 mbuf_how_t how)
1321{
1322 size_t mlen;
1323 mbuf_t m_start = m;
1324 mbuf_t n;
1325 int totlen = 0;
1326 errno_t result = 0;
1327 const char *cp = data;
1328
1329 if (m == NULL || len == 0 || data == NULL)
1330 return (EINVAL);
1331
1332 while (off > (mlen = m->m_len)) {
1333 off -= mlen;
1334 totlen += mlen;
1335 if (m->m_next == 0) {
1336 n = m_getclr(how, m->m_type);
1337 if (n == 0) {
1338 result = ENOBUFS;
1339 goto out;
1340 }
1341 n->m_len = MIN(MLEN, len + off);
1342 m->m_next = n;
1343 }
1344 m = m->m_next;
1345 }
1346
1347 while (len > 0) {
1348 mlen = MIN(m->m_len - off, len);
1349 if (mlen < len && m->m_next == NULL &&
1350 mbuf_trailingspace(m) > 0) {
1351 size_t grow = MIN(mbuf_trailingspace(m), len - mlen);
1352 mlen += grow;
1353 m->m_len += grow;
1354 }
1355 bcopy(cp, off + (char *)mbuf_data(m), (unsigned)mlen);
1356 cp += mlen;
1357 len -= mlen;
1358 mlen += off;
1359 off = 0;
1360 totlen += mlen;
1361 if (len == 0)
1362 break;
1363 if (m->m_next == 0) {
1364 n = m_get(how, m->m_type);
1365 if (n == NULL) {
1366 result = ENOBUFS;
1367 goto out;
1368 }
1369 if (len > MINCLSIZE) {
1370 /*
1371 * cluster allocation failure is okay,
1372 * we can grow chain
1373 */
1374 mbuf_mclget(how, m->m_type, &n);
1375 }
1376 n->m_len = MIN(mbuf_maxlen(n), len);
1377 m->m_next = n;
1378 }
1379 m = m->m_next;
1380 }
1381
1382out:
1383 if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
1384 m_start->m_pkthdr.len = totlen;
1385
1386 return (result);
1387}
1388
1389u_int32_t
1390mbuf_get_mlen(void)
1391{
1392 return (_MLEN);
1393}
1394
1395u_int32_t
1396mbuf_get_mhlen(void)
1397{
1398 return (_MHLEN);
1399}
1400
1401u_int32_t
1402mbuf_get_minclsize(void)
1403{
1404 return (MHLEN + MLEN);
1405}
1406
1407u_int32_t
1408mbuf_get_traffic_class_max_count(void)
1409{
1410 return (MBUF_TC_MAX);
1411}
1412
1413errno_t
1414mbuf_get_traffic_class_index(mbuf_traffic_class_t tc, u_int32_t *index)
1415{
1416 if (index == NULL || (u_int32_t)tc >= MBUF_TC_MAX)
1417 return (EINVAL);
1418
1419 *index = MBUF_SCIDX(m_service_class_from_val(MBUF_TC2SCVAL(tc)));
1420 return (0);
1421}
1422
1423mbuf_traffic_class_t
1424mbuf_get_traffic_class(mbuf_t m)
1425{
1426 if (m == NULL || !(m->m_flags & M_PKTHDR))
1427 return (MBUF_TC_BE);
1428
1429 return (m_get_traffic_class(m));
1430}
1431
1432errno_t
1433mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc)
1434{
1435 if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1436 ((u_int32_t)tc >= MBUF_TC_MAX))
1437 return (EINVAL);
1438
1439 return (m_set_traffic_class(m, tc));
1440}
1441
1442int
1443mbuf_is_traffic_class_privileged(mbuf_t m)
1444{
1445 if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1446 !MBUF_VALID_SC(m->m_pkthdr.pkt_svc))
1447 return (0);
1448
1449 return ((m->m_pkthdr.pkt_flags & PKTF_PRIO_PRIVILEGED) ? 1 : 0);
1450}
1451
1452u_int32_t
1453mbuf_get_service_class_max_count(void)
1454{
1455 return (MBUF_SC_MAX_CLASSES);
1456}
1457
1458errno_t
1459mbuf_get_service_class_index(mbuf_svc_class_t sc, u_int32_t *index)
1460{
1461 if (index == NULL || !MBUF_VALID_SC(sc))
1462 return (EINVAL);
1463
1464 *index = MBUF_SCIDX(sc);
1465 return (0);
1466}
1467
1468mbuf_svc_class_t
1469mbuf_get_service_class(mbuf_t m)
1470{
1471 if (m == NULL || !(m->m_flags & M_PKTHDR))
1472 return (MBUF_SC_BE);
1473
1474 return (m_get_service_class(m));
1475}
1476
1477errno_t
1478mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc)
1479{
1480 if (m == NULL || !(m->m_flags & M_PKTHDR))
1481 return (EINVAL);
1482
1483 return (m_set_service_class(m, sc));
1484}
1485
1486errno_t
1487mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp)
1488{
1489 u_int32_t flags;
1490
1491 if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL)
1492 return (EINVAL);
1493
1494 *flagsp = 0;
1495 flags = m->m_pkthdr.pkt_flags;
1496 if ((flags & (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) ==
1497 (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR))
1498 *flagsp |= MBUF_PKTAUXF_INET_RESOLVE_RTR;
1499 if ((flags & (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) ==
1500 (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR))
1501 *flagsp |= MBUF_PKTAUXF_INET6_RESOLVE_RTR;
1502
1503 /* These 2 flags are mutually exclusive */
1504 VERIFY((*flagsp &
1505 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) !=
1506 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR));
1507
1508 return (0);
1509}
1510
1511errno_t
1512mbuf_get_driver_scratch(mbuf_t m, u_int8_t **area, size_t *area_len)
1513{
1514 if (m == NULL || area == NULL || area_len == NULL ||
1515 !(m->m_flags & M_PKTHDR))
1516 return (EINVAL);
1517
1518 *area_len = m_scratch_get(m, area);
1519 return (0);
1520}
1521
1522errno_t
1523mbuf_get_unsent_data_bytes(const mbuf_t m, u_int32_t *unsent_data)
1524{
1525 if (m == NULL || unsent_data == NULL || !(m->m_flags & M_PKTHDR))
1526 return (EINVAL);
1527
1528 if (!(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA))
1529 return (EINVAL);
1530
1531 *unsent_data = m->m_pkthdr.bufstatus_if +
1532 m->m_pkthdr.bufstatus_sndbuf;
1533 return (0);
1534}
1535
1536errno_t
1537mbuf_get_buffer_status(const mbuf_t m, mbuf_buffer_status_t *buf_status)
1538{
1539 if (m == NULL || buf_status == NULL || !(m->m_flags & M_PKTHDR) ||
1540 !(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA))
1541 return (EINVAL);
1542
1543 buf_status->buf_interface = m->m_pkthdr.bufstatus_if;
1544 buf_status->buf_sndbuf = m->m_pkthdr.bufstatus_sndbuf;
1545 return (0);
1546}
1547
1548errno_t
1549mbuf_pkt_new_flow(const mbuf_t m, u_int32_t *retval)
1550{
1551 if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR))
1552 return (EINVAL);
1553 if (m->m_pkthdr.pkt_flags & PKTF_NEW_FLOW)
1554 *retval = 1;
1555 else
1556 *retval = 0;
1557 return (0);
1558}
1559
1560errno_t
1561mbuf_last_pkt(const mbuf_t m, u_int32_t *retval)
1562{
1563 if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR))
1564 return (EINVAL);
1565 if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT)
1566 *retval = 1;
1567 else
1568 *retval = 0;
1569 return (0);
1570}
1571
1572errno_t
1573mbuf_get_timestamp(mbuf_t m, u_int64_t *ts, boolean_t *valid)
1574{
1575 if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL)
1576 return (EINVAL);
1577
1578 if ((m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) {
1579 if (valid != NULL)
1580 *valid = FALSE;
1581 *ts = 0;
1582 } else {
1583 if (valid != NULL)
1584 *valid = TRUE;
1585 *ts = m->m_pkthdr.pkt_timestamp;
1586 }
1587 return (0);
1588}
1589
1590errno_t
1591mbuf_set_timestamp(mbuf_t m, u_int64_t ts, boolean_t valid)
1592{
1593 if (m == NULL || !(m->m_flags & M_PKTHDR))
1594 return (EINVAL);
1595
1596 if (valid == FALSE) {
1597 m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID;
1598 m->m_pkthdr.pkt_timestamp = 0;
1599 } else {
1600 m->m_pkthdr.pkt_flags |= PKTF_TS_VALID;
1601 m->m_pkthdr.pkt_timestamp = ts;
1602 }
1603 return (0);
1604}
1605
1606errno_t
1607mbuf_get_status(mbuf_t m, kern_return_t *status)
1608{
1609 if (m == NULL || !(m->m_flags & M_PKTHDR) || status == NULL)
1610 return (EINVAL);
1611
1612 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1613 *status = 0;
1614 } else {
1615 *status = m->m_pkthdr.drv_tx_status;
1616 }
1617 return (0);
1618}
1619
1620static void
1621driver_mtag_init(mbuf_t m)
1622{
1623 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1624 m->m_pkthdr.pkt_flags |= PKTF_DRIVER_MTAG;
1625 bzero(&m->m_pkthdr.driver_mtag,
1626 sizeof(m->m_pkthdr.driver_mtag));
1627 }
1628}
1629
1630errno_t
1631mbuf_set_status(mbuf_t m, kern_return_t status)
1632{
1633 if (m == NULL || !(m->m_flags & M_PKTHDR))
1634 return (EINVAL);
1635
1636 driver_mtag_init(m);
1637
1638 m->m_pkthdr.drv_tx_status = status;
1639
1640 return (0);
1641}
1642
1643errno_t
1644mbuf_get_flowid(mbuf_t m, u_int16_t *flowid)
1645{
1646 if (m == NULL || !(m->m_flags & M_PKTHDR) || flowid == NULL)
1647 return (EINVAL);
1648
1649 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1650 *flowid = 0;
1651 } else {
1652 *flowid = m->m_pkthdr.drv_flowid;
1653 }
1654 return (0);
1655}
1656
1657errno_t
1658mbuf_set_flowid(mbuf_t m, u_int16_t flowid)
1659{
1660 if (m == NULL || !(m->m_flags & M_PKTHDR))
1661 return (EINVAL);
1662
1663 driver_mtag_init(m);
1664
1665 m->m_pkthdr.drv_flowid = flowid;
1666
1667 return (0);
1668}
1669
1670errno_t
1671mbuf_get_tx_compl_data(mbuf_t m, uintptr_t *arg, uintptr_t *data)
1672{
1673 if (m == NULL || !(m->m_flags & M_PKTHDR) || arg == NULL ||
1674 data == NULL)
1675 return (EINVAL);
1676
1677 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1678 *arg = 0;
1679 *data = 0;
1680 } else {
1681 *arg = m->m_pkthdr.drv_tx_compl_arg;
1682 *data = m->m_pkthdr.drv_tx_compl_data;
1683 }
1684 return (0);
1685}
1686
1687errno_t
1688mbuf_set_tx_compl_data(mbuf_t m, uintptr_t arg, uintptr_t data)
1689{
1690 if (m == NULL || !(m->m_flags & M_PKTHDR))
1691 return (EINVAL);
1692
1693 driver_mtag_init(m);
1694
1695 m->m_pkthdr.drv_tx_compl_arg = arg;
1696 m->m_pkthdr.drv_tx_compl_data = data;
1697
1698 return (0);
1699}
1700
1701static u_int32_t
1702get_tx_compl_callback_index_locked(mbuf_tx_compl_func callback)
1703{
1704 u_int32_t i;
1705
1706 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1707 if (mbuf_tx_compl_table[i] == callback) {
1708 return (i);
1709 }
1710 }
1711 return (UINT32_MAX);
1712}
1713
1714static u_int32_t
1715get_tx_compl_callback_index(mbuf_tx_compl_func callback)
1716{
1717 u_int32_t i;
1718
1719 lck_rw_lock_shared(mbuf_tx_compl_tbl_lock);
1720
1721 i = get_tx_compl_callback_index_locked(callback);
1722
1723 lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock);
1724
1725 return (i);
1726}
1727
1728mbuf_tx_compl_func
1729m_get_tx_compl_callback(u_int32_t idx)
1730{
1731 mbuf_tx_compl_func cb;
1732
1733 if (idx >= MAX_MBUF_TX_COMPL_FUNC) {
1734 ASSERT(0);
1735 return (NULL);
1736 }
1737 lck_rw_lock_shared(mbuf_tx_compl_tbl_lock);
1738 cb = mbuf_tx_compl_table[idx];
1739 lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock);
1740 return (cb);
1741}
1742
1743errno_t
1744mbuf_register_tx_compl_callback(mbuf_tx_compl_func callback)
1745{
1746 int i;
1747 errno_t error;
1748
1749 if (callback == NULL)
1750 return (EINVAL);
1751
1752 lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock);
1753
1754 i = get_tx_compl_callback_index_locked(callback);
1755 if (i != -1) {
1756 error = EEXIST;
1757 goto unlock;
1758 }
1759
1760 /* assume the worst */
1761 error = ENOSPC;
1762 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1763 if (mbuf_tx_compl_table[i] == NULL) {
1764 mbuf_tx_compl_table[i] = callback;
1765 error = 0;
1766 goto unlock;
1767 }
1768 }
1769unlock:
1770 lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock);
1771
1772 return (error);
1773}
1774
1775errno_t
1776mbuf_unregister_tx_compl_callback(mbuf_tx_compl_func callback)
1777{
1778 int i;
1779 errno_t error;
1780
1781 if (callback == NULL)
1782 return (EINVAL);
1783
1784 lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock);
1785
1786 /* assume the worst */
1787 error = ENOENT;
1788 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1789 if (mbuf_tx_compl_table[i] == callback) {
1790 mbuf_tx_compl_table[i] = NULL;
1791 error = 0;
1792 goto unlock;
1793 }
1794 }
1795unlock:
1796 lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock);
1797
1798 return (error);
1799}
1800
1801errno_t
1802mbuf_get_timestamp_requested(mbuf_t m, boolean_t *requested)
1803{
1804 if (m == NULL || !(m->m_flags & M_PKTHDR))
1805 return (EINVAL);
1806
1807 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) {
1808 *requested = FALSE;
1809 } else {
1810 *requested = TRUE;
1811 }
1812 return (0);
1813}
1814
1815errno_t
1816mbuf_set_timestamp_requested(mbuf_t m, uintptr_t *pktid,
1817 mbuf_tx_compl_func callback)
1818{
1819 size_t i;
1820
1821 if (m == NULL || !(m->m_flags & M_PKTHDR) || callback == NULL ||
1822 pktid == NULL)
1823 return (EINVAL);
1824
1825 i = get_tx_compl_callback_index(callback);
1826 if (i == UINT32_MAX)
1827 return (ENOENT);
1828
1829#if (DEBUG || DEVELOPMENT)
1830 VERIFY(i < sizeof(m->m_pkthdr.pkt_compl_callbacks));
1831#endif /* (DEBUG || DEVELOPMENT) */
1832
1833 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) {
1834 m->m_pkthdr.pkt_compl_callbacks = 0;
1835 m->m_pkthdr.pkt_flags |= PKTF_TX_COMPL_TS_REQ;
1836 m->m_pkthdr.pkt_compl_context =
1837 atomic_add_32_ov(&mbuf_tx_compl_index, 1);
1838
1839#if (DEBUG || DEVELOPMENT)
1840 if (mbuf_tx_compl_debug != 0) {
1841 OSIncrementAtomic64(&mbuf_tx_compl_outstanding);
1842 }
1843#endif /* (DEBUG || DEVELOPMENT) */
1844 }
1845 m->m_pkthdr.pkt_compl_callbacks |= (1 << i);
1846 *pktid = m->m_pkthdr.pkt_compl_context;
1847
1848 return (0);
1849}
1850
1851void
1852m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp)
1853{
1854 int i;
1855
1856 if (m == NULL)
1857 return;
1858
1859 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0)
1860 return;
1861
1862#if (DEBUG || DEVELOPMENT)
1863 if (mbuf_tx_compl_debug != 0 && ifp != NULL &&
1864 (ifp->if_xflags & IFXF_TIMESTAMP_ENABLED) != 0 &&
1865 (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) {
1866 struct timespec now;
1867
1868 nanouptime(&now);
1869 net_timernsec(&now, &m->m_pkthdr.pkt_timestamp);
1870 }
1871#endif /* (DEBUG || DEVELOPMENT) */
1872
1873 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1874 mbuf_tx_compl_func callback;
1875
1876 if ((m->m_pkthdr.pkt_compl_callbacks & (1 << i)) == 0)
1877 continue;
1878
1879 lck_rw_lock_shared(mbuf_tx_compl_tbl_lock);
1880 callback = mbuf_tx_compl_table[i];
1881 lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock);
1882
1883 if (callback != NULL) {
1884 callback(m->m_pkthdr.pkt_compl_context,
1885 ifp,
1886 (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ?
1887 m->m_pkthdr.pkt_timestamp: 0,
1888 m->m_pkthdr.drv_tx_compl_arg,
1889 m->m_pkthdr.drv_tx_compl_data,
1890 m->m_pkthdr.drv_tx_status);
1891 }
1892 }
1893 m->m_pkthdr.pkt_compl_callbacks = 0;
1894
1895#if (DEBUG || DEVELOPMENT)
1896 if (mbuf_tx_compl_debug != 0) {
1897 OSDecrementAtomic64(&mbuf_tx_compl_outstanding);
1898 if (ifp == NULL)
1899 atomic_add_64(&mbuf_tx_compl_aborted, 1);
1900 }
1901#endif /* (DEBUG || DEVELOPMENT) */
1902}
1903