1/*
2 * Copyright (c) 2011-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/cdefs.h>
30
31#include <sys/param.h>
32#include <sys/malloc.h>
33#include <sys/mbuf.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/errno.h>
37#include <sys/mcache.h>
38#include <sys/sysctl.h>
39
40#include <net/if.h>
41#include <net/if_var.h>
42#include <net/if_dl.h>
43#include <net/if_types.h>
44#include <net/net_osdep.h>
45#include <net/pktsched/pktsched.h>
46#include <net/pktsched/pktsched_tcq.h>
47#include <net/pktsched/pktsched_qfq.h>
48#include <net/pktsched/pktsched_fq_codel.h>
49
50#include <pexpert/pexpert.h>
51
52
53u_int32_t machclk_freq = 0;
54u_int64_t machclk_per_sec = 0;
55u_int32_t pktsched_verbose; /* more noise if greater than 1 */
56
57static void init_machclk(void);
58
59SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "pktsched");
60
61SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED,
62 &pktsched_verbose, 0, "Packet scheduler verbosity level");
63
64void
65pktsched_init(void)
66{
67 init_machclk();
68 if (machclk_freq == 0) {
69 panic("%s: no CPU clock available!\n", __func__);
70 /* NOTREACHED */
71 }
72
73 tcq_init();
74 qfq_init();
75}
76
77static void
78init_machclk(void)
79{
80 /*
81 * Initialize machclk_freq using the timerbase frequency
82 * value from device specific info.
83 */
84 machclk_freq = gPEClockFrequencyInfo.timebase_frequency_hz;
85
86 clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
87 &machclk_per_sec);
88}
89
90u_int64_t
91pktsched_abs_to_nsecs(u_int64_t abstime)
92{
93 u_int64_t nsecs;
94
95 absolutetime_to_nanoseconds(abstime, &nsecs);
96 return (nsecs);
97}
98
99u_int64_t
100pktsched_nsecs_to_abstime(u_int64_t nsecs)
101{
102 u_int64_t abstime;
103
104 nanoseconds_to_absolutetime(nsecs, &abstime);
105 return (abstime);
106}
107
108int
109pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
110 classq_pkt_type_t ptype)
111{
112 int error = 0;
113 u_int32_t rflags;
114
115 IFCQ_LOCK_ASSERT_HELD(ifq);
116
117 VERIFY(machclk_freq != 0);
118
119 /* Nothing to do unless the scheduler type changes */
120 if (ifq->ifcq_type == scheduler)
121 return (0);
122
123 /*
124 * Remember the flags that need to be restored upon success, as
125 * they may be cleared when we tear down existing scheduler.
126 */
127 rflags = (ifq->ifcq_flags & IFCQF_ENABLED);
128
129 if (ifq->ifcq_type != PKTSCHEDT_NONE) {
130 (void) pktsched_teardown(ifq);
131
132 /* Teardown should have succeeded */
133 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
134 VERIFY(ifq->ifcq_disc == NULL);
135 VERIFY(ifq->ifcq_enqueue == NULL);
136 VERIFY(ifq->ifcq_dequeue == NULL);
137 VERIFY(ifq->ifcq_dequeue_sc == NULL);
138 VERIFY(ifq->ifcq_request == NULL);
139 }
140
141 switch (scheduler) {
142 case PKTSCHEDT_TCQ:
143 error = tcq_setup_ifclassq(ifq, sflags, ptype);
144 break;
145
146 case PKTSCHEDT_QFQ:
147 error = qfq_setup_ifclassq(ifq, sflags, ptype);
148 break;
149 case PKTSCHEDT_FQ_CODEL:
150 error = fq_if_setup_ifclassq(ifq, sflags, ptype);
151 break;
152 default:
153 error = ENXIO;
154 break;
155 }
156
157 if (error == 0)
158 ifq->ifcq_flags |= rflags;
159
160 return (error);
161}
162
163int
164pktsched_teardown(struct ifclassq *ifq)
165{
166 int error = 0;
167
168 IFCQ_LOCK_ASSERT_HELD(ifq);
169
170 if_qflush(ifq->ifcq_ifp, 1);
171 VERIFY(IFCQ_IS_EMPTY(ifq));
172
173 ifq->ifcq_flags &= ~IFCQF_ENABLED;
174
175 switch (ifq->ifcq_type) {
176 case PKTSCHEDT_NONE:
177 break;
178
179 case PKTSCHEDT_TCQ:
180 error = tcq_teardown_ifclassq(ifq);
181 break;
182
183 case PKTSCHEDT_QFQ:
184 error = qfq_teardown_ifclassq(ifq);
185 break;
186
187 case PKTSCHEDT_FQ_CODEL:
188 error = fq_if_teardown_ifclassq(ifq);
189 break;
190 default:
191 error = ENXIO;
192 break;
193 }
194 return (error);
195}
196
197int
198pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
199 struct if_ifclassq_stats *ifqs)
200{
201 int error;
202
203 IFCQ_LOCK_ASSERT_HELD(ifq);
204
205 switch (ifq->ifcq_type) {
206 case PKTSCHEDT_TCQ:
207 error = tcq_getqstats_ifclassq(ifq, qid, ifqs);
208 break;
209
210 case PKTSCHEDT_QFQ:
211 error = qfq_getqstats_ifclassq(ifq, qid, ifqs);
212 break;
213
214 case PKTSCHEDT_FQ_CODEL:
215 error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
216 break;
217 default:
218 error = ENXIO;
219 break;
220 }
221
222 return (error);
223}
224
225void
226pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp)
227{
228 pkt->pktsched_ptype = ptype;
229 pkt->pktsched_pkt = pp;
230
231 switch (ptype) {
232 case QP_MBUF:
233 pkt->pktsched_plen =
234 (uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt);
235 break;
236
237
238 default:
239 VERIFY(0);
240 /* NOTREACHED */
241 }
242}
243
244void
245pktsched_free_pkt(pktsched_pkt_t *pkt)
246{
247 switch (pkt->pktsched_ptype) {
248 case QP_MBUF:
249 m_freem(pkt->pktsched_pkt);
250 break;
251
252
253 default:
254 VERIFY(0);
255 /* NOTREACHED */
256 }
257
258 pkt->pktsched_pkt = NULL;
259 pkt->pktsched_plen = 0;
260 pkt->pktsched_ptype = 0;
261}
262
263uint32_t
264pktsched_get_pkt_len(pktsched_pkt_t *pkt)
265{
266 return (pkt->pktsched_plen);
267}
268
269mbuf_svc_class_t
270pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
271{
272 mbuf_svc_class_t svc = MBUF_SC_UNSPEC;
273
274 switch (pkt->pktsched_ptype) {
275 case QP_MBUF:
276 svc = m_get_service_class((mbuf_t)pkt->pktsched_pkt);
277 break;
278
279
280 default:
281 VERIFY(0);
282 /* NOTREACHED */
283 }
284
285 return (svc);
286}
287
288void
289pktsched_get_pkt_vars(pktsched_pkt_t *pkt, uint32_t **flags,
290 uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto,
291 uint32_t *tcp_start_seq)
292{
293 switch (pkt->pktsched_ptype) {
294 case QP_MBUF: {
295 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
296 struct pkthdr *pkth = &m->m_pkthdr;
297
298 if (flags != NULL)
299 *flags = &pkth->pkt_flags;
300 if (timestamp != NULL)
301 *timestamp = &pkth->pkt_timestamp;
302 if (flowid != NULL)
303 *flowid = pkth->pkt_flowid;
304 if (flowsrc != NULL)
305 *flowsrc = pkth->pkt_flowsrc;
306 if (proto != NULL)
307 *proto = pkth->pkt_proto;
308 /*
309 * caller should use this value only if PKTF_START_SEQ
310 * is set in the mbuf packet flags
311 */
312 if (tcp_start_seq != NULL)
313 *tcp_start_seq = pkth->tx_start_seq;
314
315 break;
316 }
317
318
319 default:
320 VERIFY(0);
321 /* NOTREACHED */
322 }
323}
324
325struct flowadv_fcentry *
326pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how)
327{
328#pragma unused(ifp)
329 struct flowadv_fcentry *fce = NULL;
330
331 switch (pkt->pktsched_ptype) {
332 case QP_MBUF: {
333 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
334
335 fce = flowadv_alloc_entry(how);
336 if (fce == NULL)
337 break;
338
339 _CASSERT(sizeof (m->m_pkthdr.pkt_flowid) ==
340 sizeof (fce->fce_flowid));
341
342 fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
343 fce->fce_flowid = m->m_pkthdr.pkt_flowid;
344 break;
345 }
346
347
348 default:
349 VERIFY(0);
350 /* NOTREACHED */
351 }
352
353 return (fce);
354}
355
356uint32_t *
357pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags)
358{
359 uint32_t *hashp = NULL;
360
361 switch (pkt->pktsched_ptype) {
362 case QP_MBUF: {
363 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
364 struct pkthdr *pkth = &m->m_pkthdr;
365
366 _CASSERT(sizeof (pkth->pkt_mpriv_hash) == sizeof (uint32_t));
367 _CASSERT(sizeof (pkth->pkt_mpriv_flags) == sizeof (uint32_t));
368
369 *sfb_flags = &pkth->pkt_mpriv_flags;
370 hashp = &pkth->pkt_mpriv_hash;
371 break;
372 }
373
374
375 default:
376 VERIFY(0);
377 /* NOTREACHED */
378 }
379
380 return (hashp);
381}
382