1/*
2 * Copyright (c) 2011-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * traffic class queue
31 */
32
33#include <sys/cdefs.h>
34#include <sys/param.h>
35#include <sys/malloc.h>
36#include <sys/mbuf.h>
37#include <sys/systm.h>
38#include <sys/errno.h>
39#include <sys/kernel.h>
40#include <sys/syslog.h>
41
42#include <kern/zalloc.h>
43
44#include <net/if.h>
45#include <net/net_osdep.h>
46
47#include <net/pktsched/pktsched_tcq.h>
48#include <netinet/in.h>
49
50
51/*
52 * function prototypes
53 */
54static int tcq_enqueue_ifclassq(struct ifclassq *, void *, classq_pkt_type_t,
55 boolean_t *);
56static void *tcq_dequeue_tc_ifclassq(struct ifclassq *, mbuf_svc_class_t,
57 classq_pkt_type_t *);
58static int tcq_request_ifclassq(struct ifclassq *, cqrq_t, void *);
59static int tcq_clear_interface(struct tcq_if *);
60static struct tcq_class *tcq_class_create(struct tcq_if *, int, u_int32_t,
61 int, u_int32_t, classq_pkt_type_t);
62static int tcq_class_destroy(struct tcq_if *, struct tcq_class *);
63static int tcq_destroy_locked(struct tcq_if *);
64static inline int tcq_addq(struct tcq_class *, pktsched_pkt_t *,
65 struct pf_mtag *);
66static inline void tcq_getq(struct tcq_class *, pktsched_pkt_t *);
67static void tcq_purgeq(struct tcq_if *, struct tcq_class *, u_int32_t,
68 u_int32_t *, u_int32_t *);
69static void tcq_purge_sc(struct tcq_if *, cqrq_purge_sc_t *);
70static void tcq_updateq(struct tcq_if *, struct tcq_class *, cqev_t);
71static int tcq_throttle(struct tcq_if *, cqrq_throttle_t *);
72static int tcq_resumeq(struct tcq_if *, struct tcq_class *);
73static int tcq_suspendq(struct tcq_if *, struct tcq_class *);
74static int tcq_stat_sc(struct tcq_if *, cqrq_stat_sc_t *);
75static void tcq_dequeue_cl(struct tcq_if *, struct tcq_class *,
76 mbuf_svc_class_t, pktsched_pkt_t *);
77static inline struct tcq_class *tcq_clh_to_clp(struct tcq_if *, u_int32_t);
78static const char *tcq_style(struct tcq_if *);
79
80#define TCQ_ZONE_MAX 32 /* maximum elements in zone */
81#define TCQ_ZONE_NAME "pktsched_tcq" /* zone name */
82
83static unsigned int tcq_size; /* size of zone element */
84static struct zone *tcq_zone; /* zone for tcq */
85
86#define TCQ_CL_ZONE_MAX 32 /* maximum elements in zone */
87#define TCQ_CL_ZONE_NAME "pktsched_tcq_cl" /* zone name */
88
89static unsigned int tcq_cl_size; /* size of zone element */
90static struct zone *tcq_cl_zone; /* zone for tcq_class */
91
92void
93tcq_init(void)
94{
95 tcq_size = sizeof (struct tcq_if);
96 tcq_zone = zinit(tcq_size, TCQ_ZONE_MAX * tcq_size,
97 0, TCQ_ZONE_NAME);
98 if (tcq_zone == NULL) {
99 panic("%s: failed allocating %s", __func__, TCQ_ZONE_NAME);
100 /* NOTREACHED */
101 }
102 zone_change(tcq_zone, Z_EXPAND, TRUE);
103 zone_change(tcq_zone, Z_CALLERACCT, TRUE);
104
105 tcq_cl_size = sizeof (struct tcq_class);
106 tcq_cl_zone = zinit(tcq_cl_size, TCQ_CL_ZONE_MAX * tcq_cl_size,
107 0, TCQ_CL_ZONE_NAME);
108 if (tcq_cl_zone == NULL) {
109 panic("%s: failed allocating %s", __func__, TCQ_CL_ZONE_NAME);
110 /* NOTREACHED */
111 }
112 zone_change(tcq_cl_zone, Z_EXPAND, TRUE);
113 zone_change(tcq_cl_zone, Z_CALLERACCT, TRUE);
114}
115
116struct tcq_if *
117tcq_alloc(struct ifnet *ifp, int how)
118{
119 struct tcq_if *tif;
120
121 tif = (how == M_WAITOK) ? zalloc(tcq_zone) : zalloc_noblock(tcq_zone);
122 if (tif == NULL)
123 return (NULL);
124
125 bzero(tif, tcq_size);
126 tif->tif_maxpri = -1;
127 tif->tif_ifq = &ifp->if_snd;
128
129 if (pktsched_verbose) {
130 log(LOG_DEBUG, "%s: %s scheduler allocated\n",
131 if_name(ifp), tcq_style(tif));
132 }
133
134 return (tif);
135}
136
137int
138tcq_destroy(struct tcq_if *tif)
139{
140 struct ifclassq *ifq = tif->tif_ifq;
141 int err;
142
143 IFCQ_LOCK(ifq);
144 err = tcq_destroy_locked(tif);
145 IFCQ_UNLOCK(ifq);
146
147 return (err);
148}
149
150static int
151tcq_destroy_locked(struct tcq_if *tif)
152{
153 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
154
155 (void) tcq_clear_interface(tif);
156
157 if (pktsched_verbose) {
158 log(LOG_DEBUG, "%s: %s scheduler destroyed\n",
159 if_name(TCQIF_IFP(tif)), tcq_style(tif));
160 }
161
162 zfree(tcq_zone, tif);
163
164 return (0);
165}
166
167/*
168 * bring the interface back to the initial state by discarding
169 * all the filters and classes.
170 */
171static int
172tcq_clear_interface(struct tcq_if *tif)
173{
174 struct tcq_class *cl;
175 int pri;
176
177 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
178
179 /* clear out the classes */
180 for (pri = 0; pri <= tif->tif_maxpri; pri++)
181 if ((cl = tif->tif_classes[pri]) != NULL)
182 tcq_class_destroy(tif, cl);
183
184 return (0);
185}
186
187/* discard all the queued packets on the interface */
188void
189tcq_purge(struct tcq_if *tif)
190{
191 struct tcq_class *cl;
192 int pri;
193
194 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
195
196 for (pri = 0; pri <= tif->tif_maxpri; pri++) {
197 if ((cl = tif->tif_classes[pri]) != NULL && !qempty(&cl->cl_q))
198 tcq_purgeq(tif, cl, 0, NULL, NULL);
199 }
200 VERIFY(IFCQ_LEN(tif->tif_ifq) == 0);
201}
202
203static void
204tcq_purge_sc(struct tcq_if *tif, cqrq_purge_sc_t *pr)
205{
206 struct ifclassq *ifq = tif->tif_ifq;
207 u_int32_t i;
208
209 IFCQ_LOCK_ASSERT_HELD(ifq);
210
211 VERIFY(pr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(pr->sc));
212 VERIFY(pr->flow != 0);
213
214 if (pr->sc != MBUF_SC_UNSPEC) {
215 i = MBUF_SCIDX(pr->sc);
216 VERIFY(i < IFCQ_SC_MAX);
217
218 tcq_purgeq(tif, ifq->ifcq_disc_slots[i].cl,
219 pr->flow, &pr->packets, &pr->bytes);
220 } else {
221 u_int32_t cnt, len;
222
223 pr->packets = 0;
224 pr->bytes = 0;
225
226 for (i = 0; i < IFCQ_SC_MAX; i++) {
227 tcq_purgeq(tif, ifq->ifcq_disc_slots[i].cl,
228 pr->flow, &cnt, &len);
229 pr->packets += cnt;
230 pr->bytes += len;
231 }
232 }
233}
234
235void
236tcq_event(struct tcq_if *tif, cqev_t ev)
237{
238 struct tcq_class *cl;
239 int pri;
240
241 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
242
243 for (pri = 0; pri <= tif->tif_maxpri; pri++)
244 if ((cl = tif->tif_classes[pri]) != NULL)
245 tcq_updateq(tif, cl, ev);
246}
247
248int
249tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit,
250 int flags, u_int32_t qid, struct tcq_class **clp, classq_pkt_type_t ptype)
251{
252 struct tcq_class *cl;
253
254 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
255
256 /* check parameters */
257 if (priority >= TCQ_MAXPRI)
258 return (EINVAL);
259 if (tif->tif_classes[priority] != NULL)
260 return (EBUSY);
261 if (tcq_clh_to_clp(tif, qid) != NULL)
262 return (EBUSY);
263
264 cl = tcq_class_create(tif, priority, qlimit, flags, qid, ptype);
265 if (cl == NULL)
266 return (ENOMEM);
267
268 if (clp != NULL)
269 *clp = cl;
270
271 return (0);
272}
273
274static struct tcq_class *
275tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit,
276 int flags, u_int32_t qid, classq_pkt_type_t ptype)
277{
278 struct ifnet *ifp;
279 struct ifclassq *ifq;
280 struct tcq_class *cl;
281
282 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
283
284 ifq = tif->tif_ifq;
285 ifp = TCQIF_IFP(tif);
286
287 if ((cl = tif->tif_classes[pri]) != NULL) {
288 /* modify the class instead of creating a new one */
289 if (!qempty(&cl->cl_q))
290 tcq_purgeq(tif, cl, 0, NULL, NULL);
291
292 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
293 sfb_destroy(cl->cl_sfb);
294 cl->cl_qalg.ptr = NULL;
295 qtype(&cl->cl_q) = Q_DROPTAIL;
296 qstate(&cl->cl_q) = QS_RUNNING;
297 VERIFY(qptype(&cl->cl_q) == ptype);
298 } else {
299 cl = zalloc(tcq_cl_zone);
300 if (cl == NULL)
301 return (NULL);
302
303 bzero(cl, tcq_cl_size);
304 }
305
306 tif->tif_classes[pri] = cl;
307 if (flags & TQCF_DEFAULTCLASS)
308 tif->tif_default = cl;
309 if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) {
310 qlimit = IFCQ_MAXLEN(ifq);
311 if (qlimit == 0)
312 qlimit = DEFAULT_QLIMIT; /* use default */
313 }
314 _qinit(&cl->cl_q, Q_DROPTAIL, qlimit, ptype);
315 cl->cl_flags = flags;
316 cl->cl_pri = pri;
317 if (pri > tif->tif_maxpri)
318 tif->tif_maxpri = pri;
319 cl->cl_tif = tif;
320 cl->cl_handle = qid;
321
322 if (flags & TQCF_SFB) {
323 cl->cl_qflags = 0;
324 if (flags & TQCF_ECN) {
325 cl->cl_qflags |= SFBF_ECN;
326 }
327 if (flags & TQCF_FLOWCTL) {
328 cl->cl_qflags |= SFBF_FLOWCTL;
329 }
330 if (flags & TQCF_DELAYBASED) {
331 cl->cl_qflags |= SFBF_DELAYBASED;
332 }
333 if (!(cl->cl_flags & TQCF_LAZY))
334 cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
335 qlimit(&cl->cl_q), cl->cl_qflags);
336 if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY))
337 qtype(&cl->cl_q) = Q_SFB;
338 }
339
340 if (pktsched_verbose) {
341 log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d "
342 "flags=%b\n", if_name(ifp), tcq_style(tif),
343 cl->cl_handle, cl->cl_pri, qlimit, flags, TQCF_BITS);
344 }
345
346 return (cl);
347}
348
349int
350tcq_remove_queue(struct tcq_if *tif, u_int32_t qid)
351{
352 struct tcq_class *cl;
353
354 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
355
356 if ((cl = tcq_clh_to_clp(tif, qid)) == NULL)
357 return (EINVAL);
358
359 return (tcq_class_destroy(tif, cl));
360}
361
362static int
363tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl)
364{
365 struct ifclassq *ifq = tif->tif_ifq;
366 int pri;
367#if !MACH_ASSERT
368#pragma unused(ifq)
369#endif
370 IFCQ_LOCK_ASSERT_HELD(ifq);
371
372 if (!qempty(&cl->cl_q))
373 tcq_purgeq(tif, cl, 0, NULL, NULL);
374
375 tif->tif_classes[cl->cl_pri] = NULL;
376 if (tif->tif_maxpri == cl->cl_pri) {
377 for (pri = cl->cl_pri; pri >= 0; pri--)
378 if (tif->tif_classes[pri] != NULL) {
379 tif->tif_maxpri = pri;
380 break;
381 }
382 if (pri < 0)
383 tif->tif_maxpri = -1;
384 }
385
386 if (tif->tif_default == cl)
387 tif->tif_default = NULL;
388
389 if (cl->cl_qalg.ptr != NULL) {
390 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
391 sfb_destroy(cl->cl_sfb);
392 cl->cl_qalg.ptr = NULL;
393 qtype(&cl->cl_q) = Q_DROPTAIL;
394 qstate(&cl->cl_q) = QS_RUNNING;
395 }
396
397 if (pktsched_verbose) {
398 log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n",
399 if_name(TCQIF_IFP(tif)), tcq_style(tif),
400 cl->cl_handle, cl->cl_pri);
401 }
402
403 zfree(tcq_cl_zone, cl);
404 return (0);
405}
406
407int
408tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, pktsched_pkt_t *pkt,
409 struct pf_mtag *t)
410{
411 struct ifclassq *ifq = tif->tif_ifq;
412 int len, ret;
413
414 IFCQ_LOCK_ASSERT_HELD(ifq);
415 VERIFY(cl == NULL || cl->cl_tif == tif);
416
417 if (cl == NULL) {
418 cl = tcq_clh_to_clp(tif, 0);
419 if (cl == NULL) {
420 cl = tif->tif_default;
421 if (cl == NULL) {
422 IFCQ_CONVERT_LOCK(ifq);
423 return (CLASSQEQ_DROP);
424 }
425 }
426 }
427
428 VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q));
429 len = pktsched_get_pkt_len(pkt);
430
431 ret = tcq_addq(cl, pkt, t);
432 if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) {
433 VERIFY(ret == CLASSQEQ_DROP ||
434 ret == CLASSQEQ_DROP_FC ||
435 ret == CLASSQEQ_DROP_SP);
436 PKTCNTR_ADD(&cl->cl_dropcnt, 1, len);
437 IFCQ_DROP_ADD(ifq, 1, len);
438 return (ret);
439 }
440 IFCQ_INC_LEN(ifq);
441 IFCQ_INC_BYTES(ifq, len);
442
443 /* successfully queued. */
444 return (ret);
445}
446
447/*
448 * note: CLASSQDQ_POLL returns the next packet without removing the packet
449 * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
450 * CLASSQDQ_REMOVE must return the same packet if called immediately
451 * after CLASSQDQ_POLL.
452 */
453void
454tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, pktsched_pkt_t *pkt)
455{
456 tcq_dequeue_cl(tif, NULL, sc, pkt);
457}
458
459static void
460tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl, mbuf_svc_class_t sc,
461 pktsched_pkt_t *pkt)
462{
463 struct ifclassq *ifq = tif->tif_ifq;
464 uint32_t len;
465
466 IFCQ_LOCK_ASSERT_HELD(ifq);
467
468 if (cl == NULL) {
469 cl = tcq_clh_to_clp(tif, MBUF_SCIDX(sc));
470 if (cl == NULL) {
471 pkt->pktsched_pkt = NULL;
472 return;
473 }
474 }
475
476 if (qempty(&cl->cl_q)) {
477 pkt->pktsched_pkt = NULL;
478 return;
479 }
480
481 VERIFY(!IFCQ_IS_EMPTY(ifq));
482
483 tcq_getq(cl, pkt);
484 if (pkt->pktsched_pkt != NULL) {
485 len = pktsched_get_pkt_len(pkt);
486 IFCQ_DEC_LEN(ifq);
487 IFCQ_DEC_BYTES(ifq, len);
488 if (qempty(&cl->cl_q))
489 cl->cl_period++;
490 PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len);
491 IFCQ_XMIT_ADD(ifq, 1, len);
492 }
493}
494
495static inline int
496tcq_addq(struct tcq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t)
497{
498 struct tcq_if *tif = cl->cl_tif;
499 struct ifclassq *ifq = tif->tif_ifq;
500
501 IFCQ_LOCK_ASSERT_HELD(ifq);
502
503 if (q_is_sfb(&cl->cl_q)) {
504 if (cl->cl_sfb == NULL) {
505 struct ifnet *ifp = TCQIF_IFP(tif);
506
507 VERIFY(cl->cl_flags & TQCF_LAZY);
508 cl->cl_flags &= ~TQCF_LAZY;
509 IFCQ_CONVERT_LOCK(ifq);
510
511 cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
512 qlimit(&cl->cl_q), cl->cl_qflags);
513 if (cl->cl_sfb == NULL) {
514 /* fall back to droptail */
515 qtype(&cl->cl_q) = Q_DROPTAIL;
516 cl->cl_flags &= ~TQCF_SFB;
517 cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL);
518
519 log(LOG_ERR, "%s: %s SFB lazy allocation "
520 "failed for qid=%d pri=%d, falling back "
521 "to DROPTAIL\n", if_name(ifp),
522 tcq_style(tif), cl->cl_handle,
523 cl->cl_pri);
524 } else if (tif->tif_throttle != IFNET_THROTTLE_OFF) {
525 /* if there's pending throttling, set it */
526 cqrq_throttle_t tr = { 1, tif->tif_throttle };
527 int err = tcq_throttle(tif, &tr);
528
529 if (err == EALREADY)
530 err = 0;
531 if (err != 0) {
532 tr.level = IFNET_THROTTLE_OFF;
533 (void) tcq_throttle(tif, &tr);
534 }
535 }
536 }
537 if (cl->cl_sfb != NULL)
538 return (sfb_addq(cl->cl_sfb, &cl->cl_q, pkt, t));
539 } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) {
540 IFCQ_CONVERT_LOCK(ifq);
541 return (CLASSQEQ_DROP);
542 }
543
544#if PF_ECN
545 if (cl->cl_flags & TQCF_CLEARDSCP)
546 /* not supported for non-BSD stack packets */
547 VERIFY(pkt->pktsched_ptype == QP_MBUF);
548 write_dsfield(m, t, 0);
549#endif /* PF_ECN */
550
551 VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q));
552 _addq(&cl->cl_q, pkt->pktsched_pkt);
553
554 return (0);
555}
556
557static inline void
558tcq_getq(struct tcq_class *cl, pktsched_pkt_t *pkt)
559{
560 IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq);
561
562 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) {
563 return (sfb_getq(cl->cl_sfb, &cl->cl_q, pkt));
564 }
565
566 return (pktsched_pkt_encap(pkt, qptype(&cl->cl_q), _getq(&cl->cl_q)));
567}
568
569static void
570tcq_purgeq(struct tcq_if *tif, struct tcq_class *cl, u_int32_t flow,
571 u_int32_t *packets, u_int32_t *bytes)
572{
573 struct ifclassq *ifq = tif->tif_ifq;
574 u_int32_t cnt = 0, len = 0, qlen;
575
576 IFCQ_LOCK_ASSERT_HELD(ifq);
577
578 if ((qlen = qlen(&cl->cl_q)) == 0)
579 goto done;
580
581 IFCQ_CONVERT_LOCK(ifq);
582 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
583 sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len);
584 else
585 _flushq_flow(&cl->cl_q, flow, &cnt, &len);
586
587 if (cnt > 0) {
588 VERIFY(qlen(&cl->cl_q) == (qlen - cnt));
589
590 PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len);
591 IFCQ_DROP_ADD(ifq, cnt, len);
592
593 VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0);
594 IFCQ_LEN(ifq) -= cnt;
595
596 if (pktsched_verbose) {
597 log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d "
598 "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n",
599 if_name(TCQIF_IFP(tif)), tcq_style(tif),
600 cl->cl_handle, cl->cl_pri, qlen, qlen(&cl->cl_q),
601 cnt, len, flow);
602 }
603 }
604done:
605 if (packets != NULL)
606 *packets = cnt;
607 if (bytes != NULL)
608 *bytes = len;
609}
610
611static void
612tcq_updateq(struct tcq_if *tif, struct tcq_class *cl, cqev_t ev)
613{
614 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
615
616 if (pktsched_verbose) {
617 log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n",
618 if_name(TCQIF_IFP(tif)), tcq_style(tif),
619 cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev));
620 }
621
622 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
623 return (sfb_updateq(cl->cl_sfb, ev));
624}
625
626int
627tcq_get_class_stats(struct tcq_if *tif, u_int32_t qid,
628 struct tcq_classstats *sp)
629{
630 struct tcq_class *cl;
631
632 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
633
634 if ((cl = tcq_clh_to_clp(tif, qid)) == NULL)
635 return (EINVAL);
636
637 sp->class_handle = cl->cl_handle;
638 sp->priority = cl->cl_pri;
639 sp->qlength = qlen(&cl->cl_q);
640 sp->qlimit = qlimit(&cl->cl_q);
641 sp->period = cl->cl_period;
642 sp->xmitcnt = cl->cl_xmitcnt;
643 sp->dropcnt = cl->cl_dropcnt;
644
645 sp->qtype = qtype(&cl->cl_q);
646 sp->qstate = qstate(&cl->cl_q);
647
648 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
649 sfb_getstats(cl->cl_sfb, &sp->sfb);
650
651 return (0);
652}
653
654static int
655tcq_stat_sc(struct tcq_if *tif, cqrq_stat_sc_t *sr)
656{
657 struct ifclassq *ifq = tif->tif_ifq;
658 struct tcq_class *cl;
659 u_int32_t i;
660
661 IFCQ_LOCK_ASSERT_HELD(ifq);
662
663 VERIFY(sr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(sr->sc));
664
665 i = MBUF_SCIDX(sr->sc);
666 VERIFY(i < IFCQ_SC_MAX);
667
668 cl = ifq->ifcq_disc_slots[i].cl;
669 sr->packets = qlen(&cl->cl_q);
670 sr->bytes = qsize(&cl->cl_q);
671
672 return (0);
673}
674
675/* convert a class handle to the corresponding class pointer */
676static inline struct tcq_class *
677tcq_clh_to_clp(struct tcq_if *tif, u_int32_t chandle)
678{
679 struct tcq_class *cl;
680 int idx;
681
682 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
683
684 for (idx = tif->tif_maxpri; idx >= 0; idx--)
685 if ((cl = tif->tif_classes[idx]) != NULL &&
686 cl->cl_handle == chandle)
687 return (cl);
688
689 return (NULL);
690}
691
692static const char *
693tcq_style(struct tcq_if *tif)
694{
695#pragma unused(tif)
696 return ("TCQ");
697}
698
699/*
700 * tcq_enqueue_ifclassq is an enqueue function to be registered to
701 * (*ifcq_enqueue) in struct ifclassq.
702 */
703static int
704tcq_enqueue_ifclassq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype,
705 boolean_t *pdrop)
706{
707 u_int32_t i = 0;
708 int ret;
709 pktsched_pkt_t pkt;
710 struct pf_mtag *t = NULL;
711
712 IFCQ_LOCK_ASSERT_HELD(ifq);
713
714 if (ptype == QP_MBUF) {
715 struct mbuf *m = p;
716 if (!(m->m_flags & M_PKTHDR)) {
717 /* should not happen */
718 log(LOG_ERR, "%s: packet does not have pkthdr\n",
719 if_name(ifq->ifcq_ifp));
720 IFCQ_CONVERT_LOCK(ifq);
721 m_freem(m);
722 *pdrop = TRUE;
723 return (ENOBUFS);
724 }
725 t = m_pftag(m);
726 i = MBUF_SCIDX(mbuf_get_service_class(m));
727 }
728 VERIFY((u_int32_t)i < IFCQ_SC_MAX);
729
730 pktsched_pkt_encap(&pkt, ptype, p);
731
732 ret = tcq_enqueue(ifq->ifcq_disc,
733 ifq->ifcq_disc_slots[i].cl, &pkt, t);
734
735 if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) {
736 pktsched_free_pkt(&pkt);
737 *pdrop = TRUE;
738 } else {
739 *pdrop = FALSE;
740 }
741
742 switch (ret) {
743 case CLASSQEQ_DROP:
744 ret = ENOBUFS;
745 break;
746 case CLASSQEQ_DROP_FC:
747 ret = EQFULL;
748 break;
749 case CLASSQEQ_DROP_SP:
750 ret = EQSUSPENDED;
751 break;
752 case CLASSQEQ_SUCCESS_FC:
753 ret = EQFULL;
754 break;
755 case CLASSQEQ_SUCCESS:
756 ret = 0;
757 break;
758 default:
759 VERIFY(0);
760 }
761 return (ret);
762}
763
764/*
765 * tcq_dequeue_tc_ifclassq is a dequeue function to be registered to
766 * (*ifcq_dequeue) in struct ifclass.
767 *
768 * note: CLASSQDQ_POLL returns the next packet without removing the packet
769 * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
770 * CLASSQDQ_REMOVE must return the same packet if called immediately
771 * after CLASSQDQ_POLL.
772 */
773static void *
774tcq_dequeue_tc_ifclassq(struct ifclassq *ifq, mbuf_svc_class_t sc,
775 classq_pkt_type_t *ptype)
776{
777 pktsched_pkt_t pkt;
778 u_int32_t i = MBUF_SCIDX(sc);
779
780 VERIFY((u_int32_t)i < IFCQ_SC_MAX);
781
782 bzero(&pkt, sizeof (pkt));
783 (tcq_dequeue_cl(ifq->ifcq_disc, ifq->ifcq_disc_slots[i].cl, sc, &pkt));
784 *ptype = pkt.pktsched_ptype;
785 return (pkt.pktsched_pkt);
786}
787
788static int
789tcq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg)
790{
791 struct tcq_if *tif = (struct tcq_if *)ifq->ifcq_disc;
792 int err = 0;
793
794 IFCQ_LOCK_ASSERT_HELD(ifq);
795
796 switch (req) {
797 case CLASSQRQ_PURGE:
798 tcq_purge(tif);
799 break;
800
801 case CLASSQRQ_PURGE_SC:
802 tcq_purge_sc(tif, (cqrq_purge_sc_t *)arg);
803 break;
804
805 case CLASSQRQ_EVENT:
806 tcq_event(tif, (cqev_t)arg);
807 break;
808
809 case CLASSQRQ_THROTTLE:
810 err = tcq_throttle(tif, (cqrq_throttle_t *)arg);
811 break;
812
813 case CLASSQRQ_STAT_SC:
814 err = tcq_stat_sc(tif, (cqrq_stat_sc_t *)arg);
815 break;
816 }
817 return (err);
818}
819
820int
821tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags,
822 classq_pkt_type_t ptype)
823{
824 struct ifnet *ifp = ifq->ifcq_ifp;
825 struct tcq_class *cl0, *cl1, *cl2, *cl3;
826 struct tcq_if *tif;
827 u_int32_t maxlen = 0, qflags = 0;
828 int err = 0;
829
830 IFCQ_LOCK_ASSERT_HELD(ifq);
831 VERIFY(ifq->ifcq_disc == NULL);
832 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
833
834 if (flags & PKTSCHEDF_QALG_SFB)
835 qflags |= TQCF_SFB;
836 if (flags & PKTSCHEDF_QALG_ECN)
837 qflags |= TQCF_ECN;
838 if (flags & PKTSCHEDF_QALG_FLOWCTL)
839 qflags |= TQCF_FLOWCTL;
840 if (flags & PKTSCHEDF_QALG_DELAYBASED)
841 qflags |= TQCF_DELAYBASED;
842
843 tif = tcq_alloc(ifp, M_WAITOK);
844 if (tif == NULL)
845 return (ENOMEM);
846
847 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0)
848 maxlen = if_sndq_maxlen;
849
850 if ((err = tcq_add_queue(tif, 0, maxlen,
851 qflags | TQCF_LAZY, SCIDX_BK, &cl0, ptype)) != 0)
852 goto cleanup;
853
854 if ((err = tcq_add_queue(tif, 1, maxlen,
855 qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1, ptype)) != 0)
856 goto cleanup;
857
858 if ((err = tcq_add_queue(tif, 2, maxlen,
859 qflags | TQCF_LAZY, SCIDX_VI, &cl2, ptype)) != 0)
860 goto cleanup;
861
862 if ((err = tcq_add_queue(tif, 3, maxlen,
863 qflags, SCIDX_VO, &cl3, ptype)) != 0)
864 goto cleanup;
865
866 err = ifclassq_attach(ifq, PKTSCHEDT_TCQ, tif,
867 tcq_enqueue_ifclassq, NULL, tcq_dequeue_tc_ifclassq,
868 NULL, NULL, tcq_request_ifclassq);
869
870 /* cache these for faster lookup */
871 if (err == 0) {
872 /* Map {BK_SYS,BK} to TC_BK */
873 ifq->ifcq_disc_slots[SCIDX_BK_SYS].qid = SCIDX_BK;
874 ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl = cl0;
875
876 ifq->ifcq_disc_slots[SCIDX_BK].qid = SCIDX_BK;
877 ifq->ifcq_disc_slots[SCIDX_BK].cl = cl0;
878
879 /* Map {BE,RD,OAM} to TC_BE */
880 ifq->ifcq_disc_slots[SCIDX_BE].qid = SCIDX_BE;
881 ifq->ifcq_disc_slots[SCIDX_BE].cl = cl1;
882
883 ifq->ifcq_disc_slots[SCIDX_RD].qid = SCIDX_BE;
884 ifq->ifcq_disc_slots[SCIDX_RD].cl = cl1;
885
886 ifq->ifcq_disc_slots[SCIDX_OAM].qid = SCIDX_BE;
887 ifq->ifcq_disc_slots[SCIDX_OAM].cl = cl1;
888
889 /* Map {AV,RV,VI} to TC_VI */
890 ifq->ifcq_disc_slots[SCIDX_AV].qid = SCIDX_VI;
891 ifq->ifcq_disc_slots[SCIDX_AV].cl = cl2;
892
893 ifq->ifcq_disc_slots[SCIDX_RV].qid = SCIDX_VI;
894 ifq->ifcq_disc_slots[SCIDX_RV].cl = cl2;
895
896 ifq->ifcq_disc_slots[SCIDX_VI].qid = SCIDX_VI;
897 ifq->ifcq_disc_slots[SCIDX_VI].cl = cl2;
898
899 /* Map {VO,CTL} to TC_VO */
900 ifq->ifcq_disc_slots[SCIDX_VO].qid = SCIDX_VO;
901 ifq->ifcq_disc_slots[SCIDX_VO].cl = cl3;
902
903 ifq->ifcq_disc_slots[SCIDX_CTL].qid = SCIDX_VO;
904 ifq->ifcq_disc_slots[SCIDX_CTL].cl = cl3;
905 }
906
907cleanup:
908 if (err != 0)
909 (void) tcq_destroy_locked(tif);
910
911 return (err);
912}
913
914int
915tcq_teardown_ifclassq(struct ifclassq *ifq)
916{
917 struct tcq_if *tif = ifq->ifcq_disc;
918 int i;
919
920 IFCQ_LOCK_ASSERT_HELD(ifq);
921 VERIFY(tif != NULL && ifq->ifcq_type == PKTSCHEDT_TCQ);
922
923 (void) tcq_destroy_locked(tif);
924
925 ifq->ifcq_disc = NULL;
926 for (i = 0; i < IFCQ_SC_MAX; i++) {
927 ifq->ifcq_disc_slots[i].qid = 0;
928 ifq->ifcq_disc_slots[i].cl = NULL;
929 }
930
931 return (ifclassq_detach(ifq));
932}
933
934int
935tcq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot,
936 struct if_ifclassq_stats *ifqs)
937{
938 struct tcq_if *tif = ifq->ifcq_disc;
939
940 IFCQ_LOCK_ASSERT_HELD(ifq);
941 VERIFY(ifq->ifcq_type == PKTSCHEDT_TCQ);
942
943 if (slot >= IFCQ_SC_MAX)
944 return (EINVAL);
945
946 return (tcq_get_class_stats(tif, ifq->ifcq_disc_slots[slot].qid,
947 &ifqs->ifqs_tcq_stats));
948}
949
950static int
951tcq_throttle(struct tcq_if *tif, cqrq_throttle_t *tr)
952{
953 struct ifclassq *ifq = tif->tif_ifq;
954 struct tcq_class *cl;
955 int err = 0;
956
957 IFCQ_LOCK_ASSERT_HELD(ifq);
958
959 if (!tr->set) {
960 tr->level = tif->tif_throttle;
961 return (0);
962 }
963
964 if (tr->level == tif->tif_throttle)
965 return (EALREADY);
966
967 /* Current throttling levels only involve BK_SYS class */
968 cl = ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl;
969
970 switch (tr->level) {
971 case IFNET_THROTTLE_OFF:
972 err = tcq_resumeq(tif, cl);
973 break;
974
975 case IFNET_THROTTLE_OPPORTUNISTIC:
976 err = tcq_suspendq(tif, cl);
977 break;
978
979 default:
980 VERIFY(0);
981 /* NOTREACHED */
982 }
983
984 if (err == 0 || err == ENXIO) {
985 if (pktsched_verbose) {
986 log(LOG_DEBUG, "%s: %s throttling %slevel set %d->%d\n",
987 if_name(TCQIF_IFP(tif)), tcq_style(tif),
988 (err == 0) ? "" : "lazy ", tif->tif_throttle,
989 tr->level);
990 }
991 tif->tif_throttle = tr->level;
992 if (err != 0)
993 err = 0;
994 else
995 tcq_purgeq(tif, cl, 0, NULL, NULL);
996 } else {
997 log(LOG_ERR, "%s: %s unable to set throttling level "
998 "%d->%d [error=%d]\n", if_name(TCQIF_IFP(tif)),
999 tcq_style(tif), tif->tif_throttle, tr->level, err);
1000 }
1001
1002 return (err);
1003}
1004
1005static int
1006tcq_resumeq(struct tcq_if *tif, struct tcq_class *cl)
1007{
1008 struct ifclassq *ifq = tif->tif_ifq;
1009 int err = 0;
1010#if !MACH_ASSERT
1011#pragma unused(ifq)
1012#endif
1013 IFCQ_LOCK_ASSERT_HELD(ifq);
1014
1015 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1016 err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE);
1017
1018 if (err == 0)
1019 qstate(&cl->cl_q) = QS_RUNNING;
1020
1021 return (err);
1022}
1023
1024static int
1025tcq_suspendq(struct tcq_if *tif, struct tcq_class *cl)
1026{
1027 struct ifclassq *ifq = tif->tif_ifq;
1028 int err = 0;
1029#if !MACH_ASSERT
1030#pragma unused(ifq)
1031#endif
1032 IFCQ_LOCK_ASSERT_HELD(ifq);
1033
1034 if (q_is_sfb(&cl->cl_q)) {
1035 if (cl->cl_sfb != NULL) {
1036 err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE);
1037 } else {
1038 VERIFY(cl->cl_flags & TQCF_LAZY);
1039 err = ENXIO; /* delayed throttling */
1040 }
1041 }
1042
1043 if (err == 0 || err == ENXIO)
1044 qstate(&cl->cl_q) = QS_SUSPENDED;
1045
1046 return (err);
1047}
1048