1/*
2 * Copyright (c) 2016-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24#include <sys/cdefs.h>
25#include <sys/systm.h>
26#include <sys/param.h>
27#include <sys/types.h>
28#include <sys/mcache.h>
29#include <kern/kern_types.h>
30#include <net/pf_pbuf.h>
31#include <netinet/in.h>
32
33void
34pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
35{
36
37 VERIFY((m->m_flags & M_PKTHDR) != 0);
38
39 pbuf->pb_type = PBUF_TYPE_MBUF;
40 pbuf->pb_mbuf = m;
41 pbuf->pb_ifp = ifp;
42 pbuf->pb_next = NULL;
43 pbuf_sync(pbuf);
44}
45
46void
47pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
48{
49
50 pbuf->pb_type = PBUF_TYPE_MEMORY;
51 pbuf->pb_memory = *mp;
52 pbuf->pb_ifp = ifp;
53 pbuf->pb_next = NULL;
54 pbuf_sync(pbuf);
55}
56
57void
58pbuf_destroy(pbuf_t *pbuf)
59{
60
61 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
62 if (pbuf->pb_mbuf) {
63 m_freem(pbuf->pb_mbuf);
64 pbuf->pb_mbuf = NULL;
65 }
66 } else
67 if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
68 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
69 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
70 PBUF_ACTION_DESTROY);
71 } else {
72 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
73 }
74
75 memset(pbuf, 0, sizeof(*pbuf));
76}
77
78void
79pbuf_sync(pbuf_t *pbuf)
80{
81
82 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
83 struct mbuf *m = pbuf->pb_mbuf;
84
85 VERIFY(m != NULL);
86 VERIFY(m->m_flags & M_PKTHDR);
87
88 pbuf->pb_data = mtod(m, void *);
89 pbuf->pb_packet_len = m->m_pkthdr.len;
90 pbuf->pb_contig_len = m->m_len;
91 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
92 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
93 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
94 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
95 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
96 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
97 pbuf->pb_pftag = m_pftag(m);
98 } else
99 if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
100 struct pbuf_memory *nm = &pbuf->pb_memory;
101
102 VERIFY(nm->pm_buffer != NULL);
103 VERIFY(nm->pm_buffer_len != 0);
104 VERIFY(nm->pm_len != 0);
105 VERIFY(nm->pm_len <= nm->pm_buffer_len);
106 VERIFY(nm->pm_offset < nm->pm_len);
107
108 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
109 pbuf->pb_packet_len = nm->pm_len;
110 pbuf->pb_contig_len = nm->pm_len;
111 pbuf->pb_csum_flags = &nm->pm_csum_flags;
112 pbuf->pb_csum_data = &nm->pm_csum_data;
113 pbuf->pb_proto = &nm->pm_proto;
114 pbuf->pb_flowsrc = &nm->pm_flowsrc;
115 pbuf->pb_flowid = &nm->pm_flowid;
116 pbuf->pb_flags = &nm->pm_flags;
117 pbuf->pb_pftag = &nm->pm_pftag;
118 } else
119 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
120}
121
122struct mbuf *
123pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
124{
125 struct mbuf *m = NULL;
126
127 pbuf_sync(pbuf);
128
129 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
130 m = pbuf->pb_mbuf;
131 if (release_ptr) {
132 pbuf->pb_mbuf = NULL;
133 pbuf_destroy(pbuf);
134 }
135 } else
136 if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
137 if (pbuf->pb_packet_len > (u_int)MHLEN) {
138 if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
139 printf("%s: packet too big for cluster (%u)\n",
140 __func__, pbuf->pb_packet_len);
141 return (NULL);
142 }
143 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
144 } else {
145 m = m_gethdr(M_DONTWAIT, MT_DATA);
146 }
147 if (m == NULL)
148 return (NULL);
149
150 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
151 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
152 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
153 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
154 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
155 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
156 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
157
158 if (pbuf->pb_pftag != NULL) {
159 struct pf_mtag *pftag = m_pftag(m);
160
161 if (pftag != NULL)
162 *pftag = *pbuf->pb_pftag;
163 }
164
165 if (release_ptr)
166 pbuf_destroy(pbuf);
167 }
168
169 return (m);
170}
171
172struct mbuf *
173pbuf_clone_to_mbuf(pbuf_t *pbuf)
174{
175 struct mbuf *m = NULL;
176
177 pbuf_sync(pbuf);
178
179 if (pbuf->pb_type == PBUF_TYPE_MBUF)
180 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
181 else
182 if (pbuf->pb_type == PBUF_TYPE_MEMORY)
183 m = pbuf_to_mbuf(pbuf, FALSE);
184 else
185 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
186
187 return (m);
188}
189
190void *
191pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
192{
193
194 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
195 struct mbuf *m = pbuf->pb_mbuf;
196
197 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT))
198 return (NULL);
199
200 if (pbuf->pb_mbuf == NULL) {
201 pbuf_destroy(pbuf);
202 return (NULL);
203 }
204
205 if (m != pbuf->pb_mbuf)
206 pbuf_sync(pbuf);
207
208 } else
209 if (pbuf->pb_type != PBUF_TYPE_MEMORY)
210 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
211
212 return (pbuf->pb_data);
213}
214
215void *
216pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
217{
218 void *rv = NULL;
219
220 VERIFY(off >= 0);
221 VERIFY((u_int)off <= pbuf->pb_packet_len);
222
223 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
224 struct mbuf *m, *n;
225
226 VERIFY(pbuf->pb_mbuf != NULL);
227
228 m = pbuf->pb_mbuf;
229
230 if (off > 0) {
231 /* Split the mbuf chain at the specified boundary */
232 if ((n = m_split(m, off, M_DONTWAIT)) == NULL)
233 return (NULL);
234 } else {
235 n = m;
236 }
237
238 /* Trim old length */
239 m_adj(n, olen);
240
241 /* Prepend new length */
242 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL)
243 return (NULL);
244
245 rv = mtod(n, void *);
246
247 if (off > 0) {
248 /* Merge the two chains */
249 int mlen;
250
251 mlen = n->m_pkthdr.len;
252 m_cat(m, n);
253 m->m_pkthdr.len += mlen;
254 } else {
255 /* The new mbuf becomes the packet header */
256 pbuf->pb_mbuf = n;
257 }
258
259 pbuf_sync(pbuf);
260 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
261 struct pbuf_memory *nm = &pbuf->pb_memory;
262 u_int true_offset, move_len;
263 int delta_len;
264 uint8_t *psrc, *pdst;
265
266 delta_len = nlen - olen;
267 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
268 nm->pm_buffer_len);
269
270 true_offset = (u_int)off + nm->pm_offset;
271 rv = &nm->pm_buffer[true_offset];
272 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
273 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
274 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
275 memmove(pdst, psrc, move_len);
276
277 nm->pm_len += delta_len;
278
279 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
280
281 pbuf_sync(pbuf);
282 } else {
283 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
284 }
285 return (rv);
286}
287
288void *
289pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
290{
291 void *rv = NULL;
292
293 VERIFY(off >= 0);
294 VERIFY(len >= 0);
295 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
296
297 /*
298 * Note: If this fails, then the pbuf is destroyed. This is a
299 * side-effect of m_pulldown().
300 *
301 * PF expects this behaviour so it's not a real problem.
302 */
303 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
304 struct mbuf *n;
305 int moff;
306
307 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
308 if (n == NULL) {
309 /* mbuf is freed by m_pulldown() in this case */
310 pbuf->pb_mbuf = NULL;
311 pbuf_destroy(pbuf);
312 return (NULL);
313 }
314
315 pbuf_sync(pbuf);
316
317 rv = (void *)(mtod(n, uint8_t *) + moff);
318 } else
319 if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
320 /*
321 * This always succeeds since memory pbufs are fully contig.
322 */
323 rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
324 } else
325 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
326
327 return (rv);
328}
329
330void
331pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
332{
333
334 VERIFY(off >= 0);
335 VERIFY(len >= 0);
336 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
337
338 if (pbuf->pb_type == PBUF_TYPE_MBUF)
339 m_copyback(pbuf->pb_mbuf, off, len, src);
340 else
341 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
342 if (len)
343 memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
344 } else
345 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
346}
347
348void
349pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
350{
351
352 VERIFY(off >= 0);
353 VERIFY(len >= 0);
354 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
355
356 if (pbuf->pb_type == PBUF_TYPE_MBUF)
357 m_copydata(pbuf->pb_mbuf, off, len, dst);
358 else
359 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
360 if (len)
361 memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
362 } else
363 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
364}
365
366uint16_t
367pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
368{
369 uint16_t sum = 0;
370
371 if (pbuf->pb_type == PBUF_TYPE_MBUF)
372 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
373 else
374 if (pbuf->pb_type == PBUF_TYPE_MEMORY)
375 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
376 else
377 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
378
379 return (sum);
380}
381
382uint16_t
383pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
384{
385 uint16_t sum = 0;
386
387 if (pbuf->pb_type == PBUF_TYPE_MBUF)
388 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
389 else
390 if (pbuf->pb_type == PBUF_TYPE_MEMORY)
391 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
392 else
393 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
394
395 return (sum);
396}
397
398mbuf_svc_class_t
399pbuf_get_service_class(const pbuf_t *pbuf)
400{
401
402 if (pbuf->pb_type == PBUF_TYPE_MBUF)
403 return m_get_service_class(pbuf->pb_mbuf);
404
405 VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
406
407 return (MBUF_SC_BE);
408}
409