1/*
2 * Copyright (c) 2011-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _NET_CLASSQ_IF_CLASSQ_H_
30#define _NET_CLASSQ_IF_CLASSQ_H_
31
32#ifdef PRIVATE
33#define IFCQ_SC_MAX 10 /* max number of queues */
34
35#ifdef BSD_KERNEL_PRIVATE
36#include <net/classq/classq.h>
37
38/* maximum number of packets stored across all queues */
39#define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048
40
41/* classq request types */
42typedef enum cqrq {
43 CLASSQRQ_PURGE = 1, /* purge all packets */
44 CLASSQRQ_PURGE_SC = 2, /* purge service class (and flow) */
45 CLASSQRQ_EVENT = 3, /* interface events */
46 CLASSQRQ_THROTTLE = 4, /* throttle packets */
47 CLASSQRQ_STAT_SC = 5, /* get service class queue stats */
48} cqrq_t;
49
50/* classq purge_sc request argument */
51typedef struct cqrq_purge_sc {
52 mbuf_svc_class_t sc; /* (in) service class */
53 u_int32_t flow; /* (in) 0 means all flows */
54 u_int32_t packets; /* (out) purged packets */
55 u_int32_t bytes; /* (out) purged bytes */
56} cqrq_purge_sc_t;
57
58/* classq throttle request argument */
59typedef struct cqrq_throttle {
60 u_int32_t set; /* set or get */
61 u_int32_t level; /* (in/out) throttling level */
62} cqrq_throttle_t;
63
64/* classq service class stats request argument */
65typedef struct cqrq_stat_sc {
66 mbuf_svc_class_t sc; /* (in) service class */
67 u_int32_t packets; /* (out) packets enqueued */
68 u_int32_t bytes; /* (out) bytes enqueued */
69} cqrq_stat_sc_t;
70
71/*
72 * A token-bucket regulator limits the rate that a network driver can
73 * dequeue packets from the output queue. Modern cards are able to buffer
74 * a large amount of packets and dequeue too many packets at a time. This
75 * bursty dequeue behavior makes it impossible to schedule packets by
76 * queueing disciplines. A token-bucket is used to control the burst size
77 * in a device independent manner.
78 */
79struct tb_regulator {
80 u_int64_t tbr_rate_raw; /* (unscaled) token bucket rate */
81 u_int32_t tbr_percent; /* token bucket rate in percentage */
82 int64_t tbr_rate; /* (scaled) token bucket rate */
83 int64_t tbr_depth; /* (scaled) token bucket depth */
84
85 int64_t tbr_token; /* (scaled) current token */
86 int64_t tbr_filluptime; /* (scaled) time to fill up bucket */
87 u_int64_t tbr_last; /* last time token was updated */
88
89 /* needed for poll-and-dequeue */
90};
91
92/* simple token bucket meter profile */
93struct tb_profile {
94 u_int64_t rate; /* rate in bit-per-sec */
95 u_int32_t percent; /* rate in percentage */
96 u_int32_t depth; /* depth in bytes */
97};
98
99struct ifclassq;
100enum cqdq_op;
101enum cqrq;
102
103typedef int (*ifclassq_enq_func)(struct ifclassq *, void *, classq_pkt_type_t,
104 boolean_t *);
105typedef void *(*ifclassq_deq_func)(struct ifclassq *, classq_pkt_type_t *);
106typedef void *(*ifclassq_deq_sc_func)(struct ifclassq *,
107 mbuf_svc_class_t, classq_pkt_type_t *);
108typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t,
109 u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *);
110typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *,
111 mbuf_svc_class_t, u_int32_t, u_int32_t, void **, void **,
112 u_int32_t *, u_int32_t *, classq_pkt_type_t *);
113typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *);
114
115/*
116 * Structure defining a queue for a network interface.
117 */
118struct ifclassq {
119 decl_lck_mtx_data(, ifcq_lock);
120
121 struct ifnet *ifcq_ifp; /* back pointer to interface */
122 u_int32_t ifcq_len; /* packet count */
123 u_int32_t ifcq_maxlen;
124 struct pktcntr ifcq_xmitcnt;
125 struct pktcntr ifcq_dropcnt;
126
127 u_int32_t ifcq_type; /* scheduler type */
128 u_int32_t ifcq_flags; /* flags */
129 u_int32_t ifcq_sflags; /* scheduler flags */
130 u_int32_t ifcq_target_qdelay; /* target queue delay */
131 u_int32_t ifcq_bytes; /* bytes count */
132 u_int32_t ifcq_pkt_drop_limit;
133 void *ifcq_disc; /* for scheduler-specific use */
134 /*
135 * ifcq_disc_slots[] represents the leaf classes configured for the
136 * corresponding discpline/scheduler, ordered by their corresponding
137 * service class index. Each slot holds the queue ID used to identify
138 * the class instance, as well as the class instance pointer itself.
139 * The latter is used during enqueue and dequeue in order to avoid the
140 * costs associated with looking up the class pointer based on the
141 * queue ID. The queue ID is used when querying the statistics from
142 * user space.
143 *
144 * Avoiding the use of queue ID during enqueue and dequeue is made
145 * possible by virtue of knowing the particular mbuf service class
146 * associated with the packets. The service class index of the
147 * packet is used as the index to ifcq_disc_slots[].
148 *
149 * ifcq_disc_slots[] therefore also acts as a lookup table which
150 * provides for the mapping between MBUF_SC values and the actual
151 * scheduler classes.
152 */
153 struct ifclassq_disc_slot {
154 u_int32_t qid;
155 void *cl;
156 } ifcq_disc_slots[IFCQ_SC_MAX]; /* for discipline use */
157
158 ifclassq_enq_func ifcq_enqueue;
159 ifclassq_deq_func ifcq_dequeue;
160 ifclassq_deq_sc_func ifcq_dequeue_sc;
161 ifclassq_deq_multi_func ifcq_dequeue_multi;
162 ifclassq_deq_sc_multi_func ifcq_dequeue_sc_multi;
163 ifclassq_req_func ifcq_request;
164
165 /* token bucket regulator */
166 struct tb_regulator ifcq_tbr; /* TBR */
167};
168
169/* ifcq_flags */
170#define IFCQF_READY 0x01 /* ifclassq supports discipline */
171#define IFCQF_ENABLED 0x02 /* ifclassq is in use */
172#define IFCQF_TBR 0x04 /* Token Bucket Regulator is in use */
173
174#define IFCQ_IS_READY(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_READY)
175#define IFCQ_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_ENABLED)
176#define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR)
177
178/* classq enqueue return value */
179/* packet has to be dropped */
180#define CLASSQEQ_DROP (-1)
181/* packet successfully enqueued */
182#define CLASSQEQ_SUCCESS 0
183/* packet enqueued; give flow control feedback */
184#define CLASSQEQ_SUCCESS_FC 1
185/* packet needs to be dropped due to flowcontrol; give flow control feedback */
186#define CLASSQEQ_DROP_FC 2
187/* packet needs to be dropped due to suspension; give flow control feedback */
188#define CLASSQEQ_DROP_SP 3
189
190/* interface event argument for CLASSQRQ_EVENT */
191typedef enum cqev {
192 CLASSQ_EV_INIT = 0,
193 CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */
194 CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */
195 CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */
196 CLASSQ_EV_LINK_UP = 4, /* link is now up */
197 CLASSQ_EV_LINK_DOWN = 5, /* link is now down */
198} cqev_t;
199#endif /* BSD_KERNEL_PRIVATE */
200
201#include <net/pktsched/pktsched_tcq.h>
202#include <net/pktsched/pktsched_qfq.h>
203#include <net/pktsched/pktsched_fq_codel.h>
204
205#ifdef __cplusplus
206extern "C" {
207#endif
208struct if_ifclassq_stats {
209 u_int32_t ifqs_len;
210 u_int32_t ifqs_maxlen;
211 struct pktcntr ifqs_xmitcnt;
212 struct pktcntr ifqs_dropcnt;
213 u_int32_t ifqs_scheduler;
214 union {
215 struct tcq_classstats ifqs_tcq_stats;
216 struct qfq_classstats ifqs_qfq_stats;
217 struct fq_codel_classstats ifqs_fq_codel_stats;
218 };
219} __attribute__((aligned(8)));
220
221#ifdef __cplusplus
222}
223#endif
224
225#ifdef BSD_KERNEL_PRIVATE
226/*
227 * For ifclassq lock
228 */
229#define IFCQ_LOCK_ASSERT_HELD(_ifcq) \
230 LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED)
231
232#define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \
233 LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED)
234
235#define IFCQ_LOCK(_ifcq) \
236 lck_mtx_lock(&(_ifcq)->ifcq_lock)
237
238#define IFCQ_LOCK_SPIN(_ifcq) \
239 lck_mtx_lock_spin(&(_ifcq)->ifcq_lock)
240
241#define IFCQ_CONVERT_LOCK(_ifcq) do { \
242 IFCQ_LOCK_ASSERT_HELD(_ifcq); \
243 lck_mtx_convert_spin(&(_ifcq)->ifcq_lock); \
244} while (0)
245
246#define IFCQ_UNLOCK(_ifcq) \
247 lck_mtx_unlock(&(_ifcq)->ifcq_lock)
248
249/*
250 * For ifclassq operations
251 */
252#define IFCQ_ENQUEUE(_ifq, _p, _t, _err, _drop) do { \
253 (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _p, _t, _drop); \
254} while (0)
255
256#define IFCQ_DEQUEUE(_ifq, _p, _t) do { \
257 (_p) = (*(_ifq)->ifcq_dequeue)(_ifq, _t); \
258} while (0)
259
260#define IFCQ_DEQUEUE_SC(_ifq, _sc, _p, _t) do { \
261 (_p) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, _t); \
262} while (0)
263
264#define IFCQ_TBR_DEQUEUE(_ifcq, _p, _t) do { \
265 (_p) = ifclassq_tbr_dequeue(_ifcq, _t); \
266} while (0)
267
268#define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p, _t) do { \
269 (_p) = ifclassq_tbr_dequeue_sc(_ifcq, _sc, _t); \
270} while (0)
271
272#define IFCQ_PURGE(_ifq) do { \
273 (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_PURGE, NULL); \
274} while (0)
275
276#define IFCQ_PURGE_SC(_ifq, _sc, _flow, _packets, _bytes) do { \
277 cqrq_purge_sc_t _req = { _sc, _flow, 0, 0 }; \
278 (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_PURGE_SC, &_req); \
279 (_packets) = _req.packets; \
280 (_bytes) = _req.bytes; \
281} while (0)
282
283#define IFCQ_UPDATE(_ifq, _ev) do { \
284 (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_EVENT, \
285 (void *)(_ev)); \
286} while (0)
287
288#define IFCQ_SET_THROTTLE(_ifq, _level, _err) do { \
289 cqrq_throttle_t _req = { 1, _level }; \
290 (_err) = (*(_ifq)->ifcq_request) \
291 (_ifq, CLASSQRQ_THROTTLE, &_req); \
292} while (0)
293
294#define IFCQ_GET_THROTTLE(_ifq, _level, _err) do { \
295 cqrq_throttle_t _req = { 0, IFNET_THROTTLE_OFF }; \
296 (_err) = (*(_ifq)->ifcq_request) \
297 (_ifq, CLASSQRQ_THROTTLE, &_req); \
298 (_level) = _req.level; \
299} while (0)
300
301#define IFCQ_LEN_SC(_ifq, _sc, _packets, _bytes, _err) do { \
302 cqrq_stat_sc_t _req = { _sc, 0, 0 }; \
303 (_err) = (*(ifq)->ifcq_request)(_ifq, CLASSQRQ_STAT_SC, &_req); \
304 if ((_packets) != NULL) \
305 (*(_packets)) = _req.packets; \
306 if ((_bytes) != NULL) \
307 (*(_bytes)) = _req.bytes; \
308} while (0)
309
310#define IFCQ_LEN(_ifcq) ((_ifcq)->ifcq_len)
311#define IFCQ_QFULL(_ifcq) (IFCQ_LEN(_ifcq) >= (_ifcq)->ifcq_maxlen)
312#define IFCQ_IS_EMPTY(_ifcq) (IFCQ_LEN(_ifcq) == 0)
313#define IFCQ_INC_LEN(_ifcq) (IFCQ_LEN(_ifcq)++)
314#define IFCQ_DEC_LEN(_ifcq) (IFCQ_LEN(_ifcq)--)
315#define IFCQ_MAXLEN(_ifcq) ((_ifcq)->ifcq_maxlen)
316#define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len))
317#define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay)
318#define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes)
319#define IFCQ_INC_BYTES(_ifcq, _len) \
320 ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len))
321#define IFCQ_DEC_BYTES(_ifcq, _len) \
322 ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len))
323
324#define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \
325 PKTCNTR_ADD(&(_ifcq)->ifcq_xmitcnt, _pkt, _len); \
326} while (0)
327
328#define IFCQ_DROP_ADD(_ifcq, _pkt, _len) do { \
329 PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \
330} while (0)
331
332#define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit)
333
334extern int ifclassq_setup(struct ifnet *, u_int32_t, boolean_t);
335extern void ifclassq_teardown(struct ifnet *);
336extern int ifclassq_pktsched_setup(struct ifclassq *);
337extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t);
338extern u_int32_t ifclassq_get_maxlen(struct ifclassq *);
339extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t,
340 u_int32_t *, u_int32_t *);
341extern errno_t ifclassq_enqueue(struct ifclassq *, void *, classq_pkt_type_t,
342 boolean_t *);
343extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t,
344 void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *);
345extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t,
346 u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *,
347 classq_pkt_type_t *);
348extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *);
349extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t,
350 classq_pkt_type_t *);
351extern void ifclassq_update(struct ifclassq *, cqev_t);
352extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *,
353 ifclassq_enq_func, ifclassq_deq_func, ifclassq_deq_sc_func,
354 ifclassq_deq_multi_func, ifclassq_deq_sc_multi_func, ifclassq_req_func);
355extern int ifclassq_detach(struct ifclassq *);
356extern int ifclassq_getqstats(struct ifclassq *, u_int32_t,
357 void *, u_int32_t *);
358extern const char *ifclassq_ev2str(cqev_t);
359extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t);
360extern void *ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_type_t *);
361extern void *ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t,
362 classq_pkt_type_t *);
363extern void ifclassq_calc_target_qdelay(struct ifnet *ifp,
364 u_int64_t *if_target_qdelay);
365extern void ifclassq_calc_update_interval(u_int64_t *update_interval);
366extern void ifclassq_set_packet_metadata(struct ifclassq *ifq,
367 struct ifnet *ifp, void *p, classq_pkt_type_t ptype);
368extern void ifclassq_reap_caches(boolean_t);
369
370#endif /* BSD_KERNEL_PRIVATE */
371#endif /* PRIVATE */
372#endif /* _NET_CLASSQ_IF_CLASSQ_H_ */
373