1/*
2 * Copyright (c) 2000-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */
29
30/*
31 * Copyright (C) 1999 WIDE Project.
32 * All rights reserved.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. Neither the name of the project nor the names of its contributors
43 * may be used to endorse or promote products derived from this software
44 * without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 */
58
59/*
60 * Copyright (c) 1982, 1986, 1988, 1991, 1993
61 * The Regents of the University of California. All rights reserved.
62 *
63 * Redistribution and use in source and binary forms, with or without
64 * modification, are permitted provided that the following conditions
65 * are met:
66 * 1. Redistributions of source code must retain the above copyright
67 * notice, this list of conditions and the following disclaimer.
68 * 2. Redistributions in binary form must reproduce the above copyright
69 * notice, this list of conditions and the following disclaimer in the
70 * documentation and/or other materials provided with the distribution.
71 * 3. All advertising materials mentioning features or use of this software
72 * must display the following acknowledgement:
73 * This product includes software developed by the University of
74 * California, Berkeley and its contributors.
75 * 4. Neither the name of the University nor the names of its contributors
76 * may be used to endorse or promote products derived from this software
77 * without specific prior written permission.
78 *
79 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
80 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
81 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
82 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
83 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
84 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
85 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
86 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
87 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
88 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
89 * SUCH DAMAGE.
90 *
91 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
92 */
93/*
94 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
95 * support for mandatory and extensible security protections. This notice
96 * is included in support of clause 2.2 (b) of the Apple Public License,
97 * Version 2.0.
98 */
99
100
101/*#define PULLDOWN_DEBUG*/
102
103#include <sys/param.h>
104#include <sys/systm.h>
105#include <sys/proc_internal.h>
106#include <sys/malloc.h>
107#include <sys/mbuf.h>
108#include <sys/mcache.h>
109#include <netinet/in.h>
110#include <netinet/ip_var.h>
111#if INET6
112#include <netinet/ip6.h>
113#include <netinet6/ip6_var.h>
114#endif /* INET6 */
115
116#if CONFIG_MACF_NET
117#include <security/mac_framework.h>
118#endif
119
120/*
121 * ensure that [off, off + len) is contiguous on the mbuf chain "m".
122 * packet chain before "off" is kept untouched.
123 * if offp == NULL, the target will start at <retval, 0> on resulting chain.
124 * if offp != NULL, the target will start at <retval, *offp> on resulting chain.
125 *
126 * on error return (NULL return value), original "m" will be freed.
127 *
128 * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster)
129 */
130struct mbuf *
131m_pulldown(struct mbuf *m, int off, int len, int *offp)
132{
133 struct mbuf *n = NULL, *o = NULL;
134 int hlen = 0, tlen = 0, olen = 0;
135 int sharedcluster = 0;
136#if defined(PULLDOWN_STAT) && INET6
137 static struct mbuf *prev = NULL;
138 int prevlen = 0, prevmlen = 0;
139#endif
140
141 /* check invalid arguments. */
142 if (m == NULL)
143 panic("m == NULL in m_pulldown()");
144 if (len > MCLBYTES) {
145 m_freem(m);
146 return NULL; /* impossible */
147 }
148
149#if defined(PULLDOWN_STAT) && INET6
150 ip6stat.ip6s_pulldown++;
151#endif
152
153#if defined(PULLDOWN_STAT) && INET6
154 /* statistics for m_pullup */
155 ip6stat.ip6s_pullup++;
156 if (off + len > MHLEN)
157 ip6stat.ip6s_pullup_fail++;
158 else {
159 int dlen, mlen;
160
161 dlen = (prev == m) ? prevlen : m->m_len;
162 mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m);
163
164 if (dlen >= off + len)
165 ip6stat.ip6s_pullup--; /* call will not be made! */
166 else if ((m->m_flags & M_EXT) != 0) {
167 ip6stat.ip6s_pullup_alloc++;
168 ip6stat.ip6s_pullup_copy++;
169 } else {
170 if (mlen >= off + len)
171 ip6stat.ip6s_pullup_copy++;
172 else {
173 ip6stat.ip6s_pullup_alloc++;
174 ip6stat.ip6s_pullup_copy++;
175 }
176 }
177
178 prevlen = off + len;
179 prevmlen = MHLEN;
180 }
181
182 /* statistics for m_pullup2 */
183 ip6stat.ip6s_pullup2++;
184 if (off + len > MCLBYTES)
185 ip6stat.ip6s_pullup2_fail++;
186 else {
187 int dlen, mlen;
188
189 dlen = (prev == m) ? prevlen : m->m_len;
190 mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m);
191 prevlen = off + len;
192 prevmlen = mlen;
193
194 if (dlen >= off + len)
195 ip6stat.ip6s_pullup2--; /* call will not be made! */
196 else if ((m->m_flags & M_EXT) != 0) {
197 ip6stat.ip6s_pullup2_alloc++;
198 ip6stat.ip6s_pullup2_copy++;
199 prevmlen = (off + len > MHLEN) ? MCLBYTES : MHLEN;
200 } else {
201 if (mlen >= off + len)
202 ip6stat.ip6s_pullup2_copy++;
203 else {
204 ip6stat.ip6s_pullup2_alloc++;
205 ip6stat.ip6s_pullup2_copy++;
206 prevmlen = (off + len > MHLEN) ? MCLBYTES
207 : MHLEN;
208 }
209 }
210 }
211
212 prev = m;
213#endif
214
215#ifdef PULLDOWN_DEBUG
216 {
217 struct mbuf *t;
218 printf("before:");
219 for (t = m; t; t = t->m_next)
220 printf(" %d", t->m_len);
221 printf("\n");
222 }
223#endif
224 n = m;
225
226 /*
227 * Iterate and make n point to the mbuf
228 * within which the first byte at length
229 * offset is contained from the start of
230 * mbuf chain.
231 */
232 while (n != NULL && off > 0) {
233 if (n->m_len > off)
234 break;
235 off -= n->m_len;
236 n = n->m_next;
237 }
238
239 /* be sure to point non-empty mbuf */
240 while (n != NULL && n->m_len == 0)
241 n = n->m_next;
242
243 if (!n) {
244 m_freem(m);
245 return NULL; /* mbuf chain too short */
246 }
247
248 /*
249 * the target data is on <n, off>.
250 * if we got enough data on the mbuf "n", we're done.
251 *
252 * It should be noted, that we should only do this either
253 * when offset is 0, i.e. data is pointing to the start
254 * or when the caller specifies an out argument to get
255 * the offset value in the mbuf to work with data pointer
256 * correctly.
257 *
258 * If offset is not 0 and caller did not provide out-argument
259 * to get offset, we should split the mbuf even when the length
260 * is contained in current mbuf.
261 */
262 if ((off == 0 || offp) && len <= n->m_len - off)
263 goto ok;
264
265#if defined(PULLDOWN_STAT) && INET6
266 ip6stat.ip6s_pulldown_copy++;
267#endif
268
269 /*
270 * when len <= n->m_len - off and off != 0, it is a special case.
271 * len bytes from <n, off> sits in single mbuf, but the caller does
272 * not like the starting position (off).
273 * chop the current mbuf into two pieces, set off to 0.
274 */
275 if (len <= n->m_len - off) {
276 o = m_copym(n, off, n->m_len - off, M_DONTWAIT);
277 if (o == NULL) {
278 m_freem(m);
279 return NULL; /* ENOBUFS */
280 }
281 n->m_len = off;
282 o->m_next = n->m_next;
283 n->m_next = o;
284 n = n->m_next;
285 off = 0;
286 goto ok;
287 }
288
289 /*
290 * we need to take hlen from <n, off> and tlen from <n->m_next, 0>,
291 * and construct contiguous mbuf with m_len == len.
292 * note that hlen + tlen == len, and tlen > 0.
293 *
294 * Read these variables as head length and tail length
295 */
296 hlen = n->m_len - off;
297 tlen = len - hlen;
298
299 /*
300 * ensure that we have enough trailing data on mbuf chain.
301 * if not, we can do nothing about the chain.
302 */
303 olen = 0;
304 for (o = n->m_next; o != NULL; o = o->m_next)
305 olen += o->m_len;
306 if (hlen + olen < len) {
307 m_freem(m);
308 return NULL; /* mbuf chain too short */
309 }
310
311 /*
312 * easy cases first.
313 * we need to use m_copydata() to get data from <n->m_next, 0>.
314 */
315 if ((n->m_flags & M_EXT) == 0)
316 sharedcluster = 0;
317 else {
318 if (m_get_ext_free(n) != NULL)
319 sharedcluster = 1;
320 else if (m_mclhasreference(n))
321 sharedcluster = 1;
322 else
323 sharedcluster = 0;
324 }
325
326 /*
327 * If we have enough space left in current mbuf to accomodate
328 * tail length, copy tail length worth of data starting with next mbuf
329 * and adjust the length of next one accordingly.
330 */
331 if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen
332 && !sharedcluster) {
333 m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len);
334 n->m_len += tlen;
335 m_adj(n->m_next, tlen);
336 goto ok;
337 }
338
339 /*
340 * If have enough leading space in next mbuf to accomodate head length
341 * of current mbuf, and total resulting length of next mbuf is greater
342 * than or equal to requested len bytes, then just copy hlen from
343 * current to the next one and adjust sizes accordingly.
344 */
345 if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen &&
346 (n->m_next->m_len + hlen) >= len && !sharedcluster) {
347 n->m_next->m_data -= hlen;
348 n->m_next->m_len += hlen;
349 bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), hlen);
350 n->m_len -= hlen;
351 n = n->m_next;
352 off = 0;
353 goto ok;
354 }
355
356 /*
357 * now, we need to do the hard way. don't m_copy as there's no room
358 * on both end.
359 */
360#if defined(PULLDOWN_STAT) && INET6
361 ip6stat.ip6s_pulldown_alloc++;
362#endif
363 MGET(o, M_DONTWAIT, m->m_type);
364 if (o == NULL) {
365 m_freem(m);
366 return NULL; /* ENOBUFS */
367 }
368 if (len > MHLEN) { /* use MHLEN just for safety */
369 MCLGET(o, M_DONTWAIT);
370 if ((o->m_flags & M_EXT) == 0) {
371 m_freem(m);
372 m_free(o);
373 return NULL; /* ENOBUFS */
374 }
375 }
376 /* get hlen from <n, off> into <o, 0> */
377 o->m_len = hlen;
378 bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), hlen);
379 n->m_len -= hlen;
380 /* get tlen from <n->m_next, 0> into <o, hlen> */
381 m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len);
382 o->m_len += tlen;
383 m_adj(n->m_next, tlen);
384 o->m_next = n->m_next;
385 n->m_next = o;
386 n = o;
387 off = 0;
388
389ok:
390#ifdef PULLDOWN_DEBUG
391 {
392 struct mbuf *t;
393 printf("after:");
394 for (t = m; t; t = t->m_next)
395 printf("%c%d", t == n ? '*' : ' ', t->m_len);
396 printf(" (off=%d)\n", off);
397 }
398#endif
399 if (offp)
400 *offp = off;
401 return n;
402}
403
404/*
405 * Create and return an m_tag, either by re-using space in a previous tag
406 * or by allocating a new mbuf/cluster
407 */
408struct m_tag *
409m_tag_create(u_int32_t id, u_int16_t type, int len, int wait, struct mbuf *buf)
410{
411 struct m_tag *t = NULL;
412 struct m_tag *p;
413
414 if (len < 0)
415 return (NULL);
416
417 if (len + sizeof (struct m_tag) + sizeof (struct m_taghdr) > MLEN)
418 return (m_tag_alloc(id, type, len, wait));
419
420 /*
421 * We've exhausted all external cases. Now, go through the m_tag
422 * chain and see if we can fit it in any of them.
423 * If not (t == NULL), call m_tag_alloc to store it in a new mbuf.
424 */
425 p = SLIST_FIRST(&buf->m_pkthdr.tags);
426 while(p != NULL) {
427 /* 2KCL m_tag */
428 if (M_TAG_ALIGN(p->m_tag_len) +
429 sizeof (struct m_taghdr) > MLEN) {
430 p = SLIST_NEXT(p, m_tag_link);
431 continue;
432 }
433
434 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
435
436 struct mbuf *m = m_dtom(p);
437 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
438
439 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t)));
440 VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT));
441
442 /* The mbuf can store this m_tag */
443 if (M_TAG_ALIGN(len) <= MLEN - m->m_len) {
444 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
445 VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t)));
446 hdr->refcnt++;
447 m->m_len += M_TAG_ALIGN(len);
448 VERIFY(m->m_len <= MLEN);
449 break;
450 }
451
452 p = SLIST_NEXT(p, m_tag_link);
453 }
454
455 if (t == NULL)
456 return (m_tag_alloc(id, type, len, wait));
457
458 t->m_tag_cookie = M_TAG_VALID_PATTERN;
459 t->m_tag_type = type;
460 t->m_tag_len = len;
461 t->m_tag_id = id;
462 if (len > 0)
463 bzero(t + 1, len);
464 return (t);
465}
466
467/* Get a packet tag structure along with specified data following. */
468struct m_tag *
469m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
470{
471 struct m_tag *t;
472
473 if (len < 0)
474 return (NULL);
475
476 if (M_TAG_ALIGN(len) + sizeof (struct m_taghdr) <= MLEN) {
477 struct mbuf *m = m_get(wait, MT_TAG);
478 struct m_taghdr *hdr;
479
480 if (m == NULL)
481 return (NULL);
482
483 m->m_flags |= M_TAGHDR;
484
485 hdr = (struct m_taghdr *)(void *)m->m_data;
486 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t)));
487 hdr->refcnt = 1;
488 m->m_len += sizeof (struct m_taghdr);
489 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
490 VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t)));
491 m->m_len += M_TAG_ALIGN(len);
492 VERIFY(m->m_len <= MLEN);
493 } else if (len + sizeof (struct m_tag) <= MCLBYTES) {
494 t = (struct m_tag *)(void *)m_mclalloc(wait);
495 } else {
496 t = NULL;
497 }
498
499 if (t == NULL)
500 return (NULL);
501
502 VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t)));
503 t->m_tag_cookie = M_TAG_VALID_PATTERN;
504 t->m_tag_type = type;
505 t->m_tag_len = len;
506 t->m_tag_id = id;
507 if (len > 0)
508 bzero(t + 1, len);
509 return (t);
510}
511
512
513/* Free a packet tag. */
514void
515m_tag_free(struct m_tag *t)
516{
517#if CONFIG_MACF_NET
518 if (t != NULL &&
519 t->m_tag_id == KERNEL_MODULE_TAG_ID &&
520 t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL)
521 mac_mbuf_tag_destroy(t);
522#endif
523 if (t == NULL)
524 return;
525
526 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
527
528 if (M_TAG_ALIGN(t->m_tag_len) + sizeof (struct m_taghdr) <= MLEN) {
529 struct mbuf * m = m_dtom(t);
530 VERIFY(m->m_flags & M_TAGHDR);
531 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
532
533 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t)));
534
535 /* No other tags in this mbuf */
536 if(--hdr->refcnt == 0) {
537 m_free(m);
538 return;
539 }
540
541 /* Pattern-fill the header */
542 u_int64_t *fill_ptr = (u_int64_t *)t;
543 u_int64_t *end_ptr = (u_int64_t *)(t + 1);
544 while (fill_ptr < end_ptr) {
545 *fill_ptr = M_TAG_FREE_PATTERN;
546 fill_ptr++;
547 }
548 } else {
549 m_mclfree((caddr_t)t);
550 }
551}
552
553/* Prepend a packet tag. */
554void
555m_tag_prepend(struct mbuf *m, struct m_tag *t)
556{
557 VERIFY(m != NULL && t != NULL);
558
559 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
560}
561
562/* Unlink a packet tag. */
563void
564m_tag_unlink(struct mbuf *m, struct m_tag *t)
565{
566 VERIFY(m->m_flags & M_PKTHDR);
567 VERIFY(t != NULL && t->m_tag_cookie == M_TAG_VALID_PATTERN);
568
569 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
570}
571
572/* Unlink and free a packet tag. */
573void
574m_tag_delete(struct mbuf *m, struct m_tag *t)
575{
576 m_tag_unlink(m, t);
577 m_tag_free(t);
578}
579
580/* Unlink and free a packet tag chain, starting from given tag. */
581void
582m_tag_delete_chain(struct mbuf *m, struct m_tag *t)
583{
584 struct m_tag *p, *q;
585
586 VERIFY(m->m_flags & M_PKTHDR);
587
588 if (t != NULL) {
589 p = t;
590 } else {
591 p = SLIST_FIRST(&m->m_pkthdr.tags);
592 }
593 if (p == NULL)
594 return;
595
596 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
597 while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) {
598 VERIFY(q->m_tag_cookie == M_TAG_VALID_PATTERN);
599 m_tag_delete(m, q);
600 }
601 m_tag_delete(m, p);
602}
603
604/* Find a tag, starting from a given position. */
605struct m_tag *
606m_tag_locate(struct mbuf *m, u_int32_t id, u_int16_t type, struct m_tag *t)
607{
608 struct m_tag *p;
609
610 VERIFY(m->m_flags & M_PKTHDR);
611
612 if (t == NULL) {
613 p = SLIST_FIRST(&m->m_pkthdr.tags);
614 } else {
615 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
616 p = SLIST_NEXT(t, m_tag_link);
617 }
618 while (p != NULL) {
619 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
620 if (p->m_tag_id == id && p->m_tag_type == type)
621 return (p);
622 p = SLIST_NEXT(p, m_tag_link);
623 }
624 return (NULL);
625}
626
627/* Copy a single tag. */
628struct m_tag *
629m_tag_copy(struct m_tag *t, int how)
630{
631 struct m_tag *p;
632
633 VERIFY(t != NULL);
634
635 p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how);
636 if (p == NULL)
637 return (NULL);
638#if CONFIG_MACF_NET
639 /*
640 * XXXMAC: we should probably pass off the initialization, and
641 * copying here? can we hid that KERNEL_TAG_TYPE_MACLABEL is
642 * special from the mbuf code?
643 */
644 if (t != NULL &&
645 t->m_tag_id == KERNEL_MODULE_TAG_ID &&
646 t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) {
647 if (mac_mbuf_tag_init(p, how) != 0) {
648 m_tag_free(p);
649 return (NULL);
650 }
651 mac_mbuf_tag_copy(t, p);
652 } else
653#endif
654 bcopy(t + 1, p + 1, t->m_tag_len); /* Copy the data */
655 return (p);
656}
657
658/*
659 * Copy two tag chains. The destination mbuf (to) loses any attached
660 * tags even if the operation fails. This should not be a problem, as
661 * m_tag_copy_chain() is typically called with a newly-allocated
662 * destination mbuf.
663 */
664int
665m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how)
666{
667 struct m_tag *p, *t, *tprev = NULL;
668
669 VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR));
670
671 m_tag_delete_chain(to, NULL);
672 SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) {
673 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
674 t = m_tag_copy(p, how);
675 if (t == NULL) {
676 m_tag_delete_chain(to, NULL);
677 return (0);
678 }
679 if (tprev == NULL) {
680 SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link);
681 } else {
682 SLIST_INSERT_AFTER(tprev, t, m_tag_link);
683 tprev = t;
684 }
685 }
686 return (1);
687}
688
689/* Initialize dynamic and static tags on an mbuf. */
690void
691m_tag_init(struct mbuf *m, int all)
692{
693 VERIFY(m->m_flags & M_PKTHDR);
694
695 SLIST_INIT(&m->m_pkthdr.tags);
696 /*
697 * If the caller wants to preserve static mbuf tags
698 * (e.g. m_dup_pkthdr), don't zero them out.
699 */
700 if (all) {
701 bzero(&m->m_pkthdr.builtin_mtag._net_mtag,
702 sizeof (m->m_pkthdr.builtin_mtag._net_mtag));
703 }
704}
705
706/* Get first tag in chain. */
707struct m_tag *
708m_tag_first(struct mbuf *m)
709{
710 VERIFY(m->m_flags & M_PKTHDR);
711
712 return (SLIST_FIRST(&m->m_pkthdr.tags));
713}
714
715/* Get next tag in chain. */
716struct m_tag *
717m_tag_next(struct mbuf *m, struct m_tag *t)
718{
719#pragma unused(m)
720 VERIFY(t != NULL);
721 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
722
723 return (SLIST_NEXT(t, m_tag_link));
724}
725
726int
727m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc)
728{
729 u_int32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */
730
731 return (m_set_service_class(m, m_service_class_from_val(val)));
732}
733
734mbuf_traffic_class_t
735m_get_traffic_class(struct mbuf *m)
736{
737 return (MBUF_SC2TC(m_get_service_class(m)));
738}
739
740int
741m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc)
742{
743 int error = 0;
744
745 VERIFY(m->m_flags & M_PKTHDR);
746
747 if (MBUF_VALID_SC(sc))
748 m->m_pkthdr.pkt_svc = sc;
749 else
750 error = EINVAL;
751
752 return (error);
753}
754
755mbuf_svc_class_t
756m_get_service_class(struct mbuf *m)
757{
758 mbuf_svc_class_t sc;
759
760 VERIFY(m->m_flags & M_PKTHDR);
761
762 if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc))
763 sc = m->m_pkthdr.pkt_svc;
764 else
765 sc = MBUF_SC_BE;
766
767 return (sc);
768}
769
770mbuf_svc_class_t
771m_service_class_from_idx(u_int32_t i)
772{
773 mbuf_svc_class_t sc = MBUF_SC_BE;
774
775 switch (i) {
776 case SCIDX_BK_SYS:
777 return (MBUF_SC_BK_SYS);
778
779 case SCIDX_BK:
780 return (MBUF_SC_BK);
781
782 case SCIDX_BE:
783 return (MBUF_SC_BE);
784
785 case SCIDX_RD:
786 return (MBUF_SC_RD);
787
788 case SCIDX_OAM:
789 return (MBUF_SC_OAM);
790
791 case SCIDX_AV:
792 return (MBUF_SC_AV);
793
794 case SCIDX_RV:
795 return (MBUF_SC_RV);
796
797 case SCIDX_VI:
798 return (MBUF_SC_VI);
799
800 case SCIDX_VO:
801 return (MBUF_SC_VO);
802
803 case SCIDX_CTL:
804 return (MBUF_SC_CTL);
805
806 default:
807 break;
808 }
809
810 VERIFY(0);
811 /* NOTREACHED */
812 return (sc);
813}
814
815mbuf_svc_class_t
816m_service_class_from_val(u_int32_t v)
817{
818 mbuf_svc_class_t sc = MBUF_SC_BE;
819
820 switch (v) {
821 case SCVAL_BK_SYS:
822 return (MBUF_SC_BK_SYS);
823
824 case SCVAL_BK:
825 return (MBUF_SC_BK);
826
827 case SCVAL_BE:
828 return (MBUF_SC_BE);
829
830 case SCVAL_RD:
831 return (MBUF_SC_RD);
832
833 case SCVAL_OAM:
834 return (MBUF_SC_OAM);
835
836 case SCVAL_AV:
837 return (MBUF_SC_AV);
838
839 case SCVAL_RV:
840 return (MBUF_SC_RV);
841
842 case SCVAL_VI:
843 return (MBUF_SC_VI);
844
845 case SCVAL_VO:
846 return (MBUF_SC_VO);
847
848 case SCVAL_CTL:
849 return (MBUF_SC_CTL);
850
851 default:
852 break;
853 }
854
855 VERIFY(0);
856 /* NOTREACHED */
857 return (sc);
858}
859
860uint16_t
861m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff,
862 uint32_t datalen, uint32_t sum)
863{
864 uint32_t total_sub = 0; /* total to subtract */
865 uint32_t mlen = m_pktlen(m); /* frame length */
866 uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */
867 int len;
868
869 ASSERT(bytes <= mlen);
870
871 /*
872 * Take care of excluding (len > 0) or including (len < 0)
873 * extraneous octets at the beginning of the packet, taking
874 * into account the start offset.
875 */
876 len = (dataoff - start);
877 if (len > 0)
878 total_sub = m_sum16(m, start, len);
879 else if (len < 0)
880 sum += m_sum16(m, dataoff, -len);
881
882 /*
883 * Take care of excluding any postpended extraneous octets.
884 */
885 len = (mlen - bytes);
886 if (len > 0) {
887 struct mbuf *m0 = m;
888 uint32_t extra = m_sum16(m, bytes, len);
889 uint32_t off = bytes, off0 = off;
890
891 while (off > 0) {
892 if (__improbable(m == NULL)) {
893 panic("%s: invalid mbuf chain %p [off %u, "
894 "len %u]", __func__, m0, off0, len);
895 /* NOTREACHED */
896 }
897 if (off < m->m_len)
898 break;
899 off -= m->m_len;
900 m = m->m_next;
901 }
902
903 /* if we started on odd-alignment, swap the value */
904 if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1)
905 total_sub += ((extra << 8) & 0xffff) | (extra >> 8);
906 else
907 total_sub += extra;
908
909 total_sub = (total_sub >> 16) + (total_sub & 0xffff);
910 }
911
912 /*
913 * 1's complement subtract any extraneous octets.
914 */
915 if (total_sub != 0) {
916 if (total_sub >= sum)
917 sum = ~(total_sub - sum) & 0xffff;
918 else
919 sum -= total_sub;
920 }
921
922 /* fold 32-bit to 16-bit */
923 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
924 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
925 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
926
927 return (sum & 0xffff);
928}
929
930uint16_t
931m_sum16(struct mbuf *m, uint32_t off, uint32_t len)
932{
933 int mlen;
934
935 /*
936 * Sanity check
937 *
938 * Use m_length2() instead of m_length(), as we cannot rely on
939 * the caller setting m_pkthdr.len correctly, if the mbuf is
940 * a M_PKTHDR one.
941 */
942 if ((mlen = m_length2(m, NULL)) < (off + len)) {
943 panic("%s: mbuf %p len (%d) < off+len (%d+%d)\n", __func__,
944 m, mlen, off, len);
945 /* NOTREACHED */
946 }
947
948 return (os_cpu_in_cksum_mbuf(m, len, off, 0));
949}
950