1/*
2 * Copyright (c) 2012-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * A note on the MPTCP/NECP-interactions:
31 *
32 * MPTCP uses NECP-callbacks to get notified of interface/policy events.
33 * MPTCP registers to these events at the MPTCP-layer for interface-events
34 * through a call to necp_client_register_multipath_cb.
35 * To get per-flow events (aka per TCP-subflow), we register to it with
36 * necp_client_register_socket_flow. Both registrations happen by using the
37 * necp-client-uuid that comes from the app.
38 *
39 * The locking is rather tricky. In general, we expect the lock-ordering to
40 * happen from necp-fd -> necp->client -> mpp_lock.
41 *
42 * There are however some subtleties.
43 *
44 * 1. When registering the multipath_cb, we are holding the mpp_lock. This is
45 * safe, because it is the very first time this MPTCP-connection goes into NECP.
46 * As we go into NECP we take the NECP-locks and thus are guaranteed that no
47 * NECP-locks will deadlock us. Because these NECP-events will also first take
48 * the NECP-locks. Either they win the race and thus won't find our
49 * MPTCP-connection. Or, MPTCP wins the race and thus it will safely install
50 * the callbacks while holding the NECP lock.
51 *
52 * 2. When registering the subflow-callbacks we must unlock the mpp_lock. This,
53 * because we have already registered callbacks and we might race against an
54 * NECP-event that will match on our socket. So, we have to unlock to be safe.
55 *
56 * 3. When removing the multipath_cb, we do it in mp_pcbdispose(). The
57 * so_usecount has reached 0. We must be careful to not remove the mpp_socket
58 * pointers before we unregistered the callback. Because, again we might be
59 * racing against an NECP-event. Unregistering must happen with an unlocked
60 * mpp_lock, because of the lock-ordering constraint. It could be that
61 * before we had a chance to unregister an NECP-event triggers. That's why
62 * we need to check for the so_usecount in mptcp_session_necp_cb. If we get
63 * there while the socket is being garbage-collected, the use-count will go
64 * down to 0 and we exit. Removal of the multipath_cb again happens by taking
65 * the NECP-locks so any running NECP-events will finish first and exit cleanly.
66 *
67 * 4. When removing the subflow-callback, we do it in in_pcbdispose(). Again,
68 * the socket-lock must be unlocked for lock-ordering constraints. This gets a
69 * bit tricky here, as in tcp_garbage_collect we hold the mp_so and so lock.
70 * So, we drop the mp_so-lock as soon as the subflow is unlinked with
71 * mptcp_subflow_del. Then, in in_pcbdispose we drop the subflow-lock.
72 * If an NECP-event was waiting on the lock in mptcp_subflow_necp_cb, when it
73 * gets it, it will realize that the subflow became non-MPTCP and retry (see
74 * tcp_lock). Then it waits again on the subflow-lock. When we drop this lock
75 * in in_pcbdispose, and enter necp_inpcb_dispose, this one will have to wait
76 * for the NECP-lock (held by the other thread that is taking care of the NECP-
77 * event). So, the event now finally gets the subflow-lock and then hits an
78 * so_usecount that is 0 and exits. Eventually, we can remove the subflow from
79 * the NECP callback.
80 */
81
82#include <sys/param.h>
83#include <sys/systm.h>
84#include <sys/kernel.h>
85#include <sys/mbuf.h>
86#include <sys/mcache.h>
87#include <sys/socket.h>
88#include <sys/socketvar.h>
89#include <sys/syslog.h>
90#include <sys/protosw.h>
91
92#include <kern/zalloc.h>
93#include <kern/locks.h>
94
95#include <mach/sdt.h>
96
97#include <net/if.h>
98#include <netinet/in.h>
99#include <netinet/in_var.h>
100#include <netinet/tcp.h>
101#include <netinet/tcp_fsm.h>
102#include <netinet/tcp_seq.h>
103#include <netinet/tcp_var.h>
104#include <netinet/mptcp_var.h>
105#include <netinet/mptcp.h>
106#include <netinet/mptcp_seq.h>
107#include <netinet/mptcp_opt.h>
108#include <netinet/mptcp_timer.h>
109
110int mptcp_enable = 1;
111SYSCTL_INT(_net_inet_mptcp, OID_AUTO, enable, CTLFLAG_RW | CTLFLAG_LOCKED,
112 &mptcp_enable, 0, "Enable Multipath TCP Support");
113
114/* Number of times to try negotiating MPTCP on SYN retransmissions */
115int mptcp_mpcap_retries = MPTCP_CAPABLE_RETRIES;
116SYSCTL_INT(_net_inet_mptcp, OID_AUTO, mptcp_cap_retr,
117 CTLFLAG_RW | CTLFLAG_LOCKED,
118 &mptcp_mpcap_retries, 0, "Number of MP Capable SYN Retries");
119
120/*
121 * By default, DSS checksum is turned off, revisit if we ever do
122 * MPTCP for non SSL Traffic.
123 */
124int mptcp_dss_csum = 0;
125SYSCTL_INT(_net_inet_mptcp, OID_AUTO, dss_csum, CTLFLAG_RW | CTLFLAG_LOCKED,
126 &mptcp_dss_csum, 0, "Enable DSS checksum");
127
128/*
129 * When mptcp_fail_thresh number of retransmissions are sent, subflow failover
130 * is attempted on a different path.
131 */
132int mptcp_fail_thresh = 1;
133SYSCTL_INT(_net_inet_mptcp, OID_AUTO, fail, CTLFLAG_RW | CTLFLAG_LOCKED,
134 &mptcp_fail_thresh, 0, "Failover threshold");
135
136
137/*
138 * MPTCP subflows have TCP keepalives set to ON. Set a conservative keeptime
139 * as carrier networks mostly have a 30 minute to 60 minute NAT Timeout.
140 * Some carrier networks have a timeout of 10 or 15 minutes.
141 */
142int mptcp_subflow_keeptime = 60*14;
143SYSCTL_INT(_net_inet_mptcp, OID_AUTO, keepalive, CTLFLAG_RW | CTLFLAG_LOCKED,
144 &mptcp_subflow_keeptime, 0, "Keepalive in seconds");
145
146int mptcp_rtthist_rtthresh = 600;
147SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist_thresh, CTLFLAG_RW | CTLFLAG_LOCKED,
148 &mptcp_rtthist_rtthresh, 0, "Rtt threshold");
149
150/*
151 * Use RTO history for sending new data
152 */
153int mptcp_use_rto = 1;
154SYSCTL_INT(_net_inet_mptcp, OID_AUTO, userto, CTLFLAG_RW | CTLFLAG_LOCKED,
155 &mptcp_use_rto, 0, "Disable RTO for subflow selection");
156
157int mptcp_rtothresh = 1500;
158SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rto_thresh, CTLFLAG_RW | CTLFLAG_LOCKED,
159 &mptcp_rtothresh, 0, "RTO threshold");
160
161/*
162 * Probe the preferred path, when it is not in use
163 */
164uint32_t mptcp_probeto = 1000;
165SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probeto, CTLFLAG_RW | CTLFLAG_LOCKED,
166 &mptcp_probeto, 0, "Disable probing by setting to 0");
167
168uint32_t mptcp_probecnt = 5;
169SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED,
170 &mptcp_probecnt, 0, "Number of probe writes");
171
172/*
173 * Static declarations
174 */
175static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t,
176 uint32_t, uint16_t, uint16_t, uint16_t);
177
178static int
179mptcp_reass_present(struct socket *mp_so)
180{
181 struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb;
182 struct tseg_qent *q;
183 int dowakeup = 0;
184 int flags = 0;
185
186 /*
187 * Present data to user, advancing rcv_nxt through
188 * completed sequence space.
189 */
190 if (mp_tp->mpt_state < MPTCPS_ESTABLISHED)
191 return (flags);
192 q = LIST_FIRST(&mp_tp->mpt_segq);
193 if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt)
194 return (flags);
195
196 /*
197 * If there is already another thread doing reassembly for this
198 * connection, it is better to let it finish the job --
199 * (radar 16316196)
200 */
201 if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG)
202 return (flags);
203
204 mp_tp->mpt_flags |= MPTCPF_REASS_INPROG;
205
206 do {
207 mp_tp->mpt_rcvnxt += q->tqe_len;
208 LIST_REMOVE(q, tqe_q);
209 if (mp_so->so_state & SS_CANTRCVMORE) {
210 m_freem(q->tqe_m);
211 } else {
212 flags = !!(q->tqe_m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);
213 if (sbappendstream_rcvdemux(mp_so, q->tqe_m, 0, 0))
214 dowakeup = 1;
215 }
216 zfree(tcp_reass_zone, q);
217 mp_tp->mpt_reassqlen--;
218 q = LIST_FIRST(&mp_tp->mpt_segq);
219 } while (q && q->tqe_m->m_pkthdr.mp_dsn == mp_tp->mpt_rcvnxt);
220 mp_tp->mpt_flags &= ~MPTCPF_REASS_INPROG;
221
222 if (dowakeup)
223 sorwakeup(mp_so); /* done with socket lock held */
224 return (flags);
225
226}
227
228static int
229mptcp_reass(struct socket *mp_so, struct pkthdr *phdr, int *tlenp, struct mbuf *m)
230{
231 struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb;
232 u_int64_t mb_dsn = phdr->mp_dsn;
233 struct tseg_qent *q;
234 struct tseg_qent *p = NULL;
235 struct tseg_qent *nq;
236 struct tseg_qent *te = NULL;
237 u_int16_t qlimit;
238
239 /*
240 * Limit the number of segments in the reassembly queue to prevent
241 * holding on to too many segments (and thus running out of mbufs).
242 * Make sure to let the missing segment through which caused this
243 * queue. Always keep one global queue entry spare to be able to
244 * process the missing segment.
245 */
246 qlimit = min(max(100, mp_so->so_rcv.sb_hiwat >> 10),
247 (tcp_autorcvbuf_max >> 10));
248 if (mb_dsn != mp_tp->mpt_rcvnxt &&
249 (mp_tp->mpt_reassqlen + 1) >= qlimit) {
250 tcpstat.tcps_mptcp_rcvmemdrop++;
251 m_freem(m);
252 *tlenp = 0;
253 return (0);
254 }
255
256 /* Allocate a new queue entry. If we can't, just drop the pkt. XXX */
257 te = (struct tseg_qent *) zalloc(tcp_reass_zone);
258 if (te == NULL) {
259 tcpstat.tcps_mptcp_rcvmemdrop++;
260 m_freem(m);
261 return (0);
262 }
263
264 mp_tp->mpt_reassqlen++;
265
266 /*
267 * Find a segment which begins after this one does.
268 */
269 LIST_FOREACH(q, &mp_tp->mpt_segq, tqe_q) {
270 if (MPTCP_SEQ_GT(q->tqe_m->m_pkthdr.mp_dsn, mb_dsn))
271 break;
272 p = q;
273 }
274
275 /*
276 * If there is a preceding segment, it may provide some of
277 * our data already. If so, drop the data from the incoming
278 * segment. If it provides all of our data, drop us.
279 */
280 if (p != NULL) {
281 int64_t i;
282 /* conversion to int (in i) handles seq wraparound */
283 i = p->tqe_m->m_pkthdr.mp_dsn + p->tqe_len - mb_dsn;
284 if (i > 0) {
285 if (i >= *tlenp) {
286 tcpstat.tcps_mptcp_rcvduppack++;
287 m_freem(m);
288 zfree(tcp_reass_zone, te);
289 te = NULL;
290 mp_tp->mpt_reassqlen--;
291 /*
292 * Try to present any queued data
293 * at the left window edge to the user.
294 * This is needed after the 3-WHS
295 * completes.
296 */
297 goto out;
298 }
299 m_adj(m, i);
300 *tlenp -= i;
301 phdr->mp_dsn += i;
302 }
303 }
304
305 tcpstat.tcps_mp_oodata++;
306
307 /*
308 * While we overlap succeeding segments trim them or,
309 * if they are completely covered, dequeue them.
310 */
311 while (q) {
312 int64_t i = (mb_dsn + *tlenp) - q->tqe_m->m_pkthdr.mp_dsn;
313 if (i <= 0)
314 break;
315
316 if (i < q->tqe_len) {
317 q->tqe_m->m_pkthdr.mp_dsn += i;
318 q->tqe_len -= i;
319 m_adj(q->tqe_m, i);
320 break;
321 }
322
323 nq = LIST_NEXT(q, tqe_q);
324 LIST_REMOVE(q, tqe_q);
325 m_freem(q->tqe_m);
326 zfree(tcp_reass_zone, q);
327 mp_tp->mpt_reassqlen--;
328 q = nq;
329 }
330
331 /* Insert the new segment queue entry into place. */
332 te->tqe_m = m;
333 te->tqe_th = NULL;
334 te->tqe_len = *tlenp;
335
336 if (p == NULL) {
337 LIST_INSERT_HEAD(&mp_tp->mpt_segq, te, tqe_q);
338 } else {
339 LIST_INSERT_AFTER(p, te, tqe_q);
340 }
341
342out:
343 return (mptcp_reass_present(mp_so));
344}
345
346/*
347 * MPTCP input, called when data has been read from a subflow socket.
348 */
349void
350mptcp_input(struct mptses *mpte, struct mbuf *m)
351{
352 struct socket *mp_so;
353 struct mptcb *mp_tp = NULL;
354 int count = 0, wakeup = 0;
355 struct mbuf *save = NULL, *prev = NULL;
356 struct mbuf *freelist = NULL, *tail = NULL;
357
358 VERIFY(m->m_flags & M_PKTHDR);
359
360 mpte_lock_assert_held(mpte); /* same as MP socket lock */
361
362 mp_so = mptetoso(mpte);
363 mp_tp = mpte->mpte_mptcb;
364
365 DTRACE_MPTCP(input);
366
367 mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp);
368
369 /*
370 * Each mbuf contains MPTCP Data Sequence Map
371 * Process the data for reassembly, delivery to MPTCP socket
372 * client, etc.
373 *
374 */
375 count = mp_so->so_rcv.sb_cc;
376
377 /*
378 * In the degraded fallback case, data is accepted without DSS map
379 */
380 if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {
381 struct mbuf *iter;
382 int mb_dfin = 0;
383fallback:
384 mptcp_sbrcv_grow(mp_tp);
385
386 iter = m;
387 while (iter) {
388 if ((iter->m_flags & M_PKTHDR) &&
389 (iter->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
390 mb_dfin = 1;
391 }
392
393 if ((iter->m_flags & M_PKTHDR) && m_pktlen(iter) == 0) {
394 /* Don't add zero-length packets, so jump it! */
395 if (prev == NULL) {
396 m = iter->m_next;
397 m_free(iter);
398 iter = m;
399 } else {
400 prev->m_next = iter->m_next;
401 m_free(iter);
402 iter = prev->m_next;
403 }
404
405 /* It was a zero-length packet so next one must be a pkthdr */
406 VERIFY(iter == NULL || iter->m_flags & M_PKTHDR);
407 } else {
408 prev = iter;
409 iter = iter->m_next;
410 }
411 }
412
413 /*
414 * assume degraded flow as this may be the first packet
415 * without DSS, and the subflow state is not updated yet.
416 */
417 if (sbappendstream_rcvdemux(mp_so, m, 0, 0))
418 sorwakeup(mp_so);
419
420 DTRACE_MPTCP5(receive__degraded, struct mbuf *, m,
421 struct socket *, mp_so,
422 struct sockbuf *, &mp_so->so_rcv,
423 struct sockbuf *, &mp_so->so_snd,
424 struct mptses *, mpte);
425 count = mp_so->so_rcv.sb_cc - count;
426
427 mp_tp->mpt_rcvnxt += count;
428
429 if (mb_dfin) {
430 mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
431 socantrcvmore(mp_so);
432 }
433
434 mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__,
435 count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
436 return;
437 }
438
439 do {
440 u_int64_t mb_dsn;
441 int32_t mb_datalen;
442 int64_t todrop;
443 int mb_dfin = 0;
444
445 /* If fallback occurs, mbufs will not have PKTF_MPTCP set */
446 if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP))
447 goto fallback;
448
449 save = m->m_next;
450 /*
451 * A single TCP packet formed of multiple mbufs
452 * holds DSS mapping in the first mbuf of the chain.
453 * Other mbufs in the chain may have M_PKTHDR set
454 * even though they belong to the same TCP packet
455 * and therefore use the DSS mapping stored in the
456 * first mbuf of the mbuf chain. mptcp_input() can
457 * get an mbuf chain with multiple TCP packets.
458 */
459 while (save && (!(save->m_flags & M_PKTHDR) ||
460 !(save->m_pkthdr.pkt_flags & PKTF_MPTCP))) {
461 prev = save;
462 save = save->m_next;
463 }
464 if (prev)
465 prev->m_next = NULL;
466 else
467 m->m_next = NULL;
468
469 mb_dsn = m->m_pkthdr.mp_dsn;
470 mb_datalen = m->m_pkthdr.mp_rlen;
471
472 todrop = (mb_dsn + mb_datalen) - (mp_tp->mpt_rcvnxt + mp_tp->mpt_rcvwnd);
473 if (todrop > 0) {
474 tcpstat.tcps_mptcp_rcvpackafterwin++;
475
476 if (todrop >= mb_datalen) {
477 if (freelist == NULL)
478 freelist = m;
479 else
480 tail->m_next = m;
481
482 if (prev != NULL)
483 tail = prev;
484 else
485 tail = m;
486
487 m = save;
488 prev = save = NULL;
489 continue;
490 } else {
491 m_adj(m, -todrop);
492 mb_datalen -= todrop;
493 }
494
495 /*
496 * We drop from the right edge of the mbuf, thus the
497 * DATA_FIN is dropped as well
498 */
499 m->m_pkthdr.pkt_flags &= ~PKTF_MPTCP_DFIN;
500 }
501
502
503 if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) {
504 if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen),
505 mp_tp->mpt_rcvnxt)) {
506 if (freelist == NULL)
507 freelist = m;
508 else
509 tail->m_next = m;
510
511 if (prev != NULL)
512 tail = prev;
513 else
514 tail = m;
515
516 m = save;
517 prev = save = NULL;
518 continue;
519 } else {
520 m_adj(m, (mp_tp->mpt_rcvnxt - mb_dsn));
521 }
522 mptcplog((LOG_INFO, "%s: Left Edge %llu\n", __func__,
523 mp_tp->mpt_rcvnxt),
524 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
525 }
526
527 if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) ||
528 !LIST_EMPTY(&mp_tp->mpt_segq)) {
529 mb_dfin = mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m);
530
531 goto next;
532 }
533 mb_dfin = !!(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);
534
535 mptcp_sbrcv_grow(mp_tp);
536
537 if (sbappendstream_rcvdemux(mp_so, m, 0, 0))
538 wakeup = 1;
539
540 DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so,
541 struct sockbuf *, &mp_so->so_rcv,
542 struct sockbuf *, &mp_so->so_snd,
543 struct mptses *, mpte,
544 struct mptcb *, mp_tp);
545 count = mp_so->so_rcv.sb_cc - count;
546 tcpstat.tcps_mp_rcvtotal++;
547 tcpstat.tcps_mp_rcvbytes += count;
548 mptcplog((LOG_DEBUG, "%s: Read %d bytes\n", __func__, count),
549 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
550
551 mp_tp->mpt_rcvnxt += count;
552
553next:
554 if (mb_dfin) {
555 mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
556 socantrcvmore(mp_so);
557 }
558 m = save;
559 prev = save = NULL;
560 count = mp_so->so_rcv.sb_cc;
561 } while (m);
562
563 if (freelist)
564 m_freem(freelist);
565
566 if (wakeup)
567 sorwakeup(mp_so);
568}
569
570boolean_t
571mptcp_can_send_more(struct mptcb *mp_tp, boolean_t ignore_reinject)
572{
573 struct socket *mp_so = mptetoso(mp_tp->mpt_mpte);
574
575 /*
576 * Always send if there is data in the reinject-queue.
577 */
578 if (!ignore_reinject && mp_tp->mpt_mpte->mpte_reinjectq)
579 return (TRUE);
580
581 /*
582 * Don't send, if:
583 *
584 * 1. snd_nxt >= snd_max : Means, basically everything has been sent.
585 * Except when using TFO, we might be doing a 0-byte write.
586 * 2. snd_una + snd_wnd <= snd_nxt: No space in the receiver's window
587 * 3. snd_nxt + 1 == snd_max and we are closing: A DATA_FIN is scheduled.
588 */
589
590 if (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA) && MPTCP_SEQ_GEQ(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax))
591 return (FALSE);
592
593 if (MPTCP_SEQ_LEQ(mp_tp->mpt_snduna + mp_tp->mpt_sndwnd, mp_tp->mpt_sndnxt))
594 return (FALSE);
595
596 if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax && mp_tp->mpt_state > MPTCPS_CLOSE_WAIT)
597 return (FALSE);
598
599 if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2)
600 return (FALSE);
601
602 return (TRUE);
603}
604
605/*
606 * MPTCP output.
607 */
608int
609mptcp_output(struct mptses *mpte)
610{
611 struct mptcb *mp_tp;
612 struct mptsub *mpts;
613 struct mptsub *mpts_tried = NULL;
614 struct socket *mp_so;
615 struct mptsub *preferred_mpts = NULL;
616 uint64_t old_snd_nxt;
617 int error = 0;
618
619 mpte_lock_assert_held(mpte);
620 mp_so = mptetoso(mpte);
621 mp_tp = mpte->mpte_mptcb;
622
623 VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL));
624 mpte->mpte_mppcb->mpp_flags |= MPP_WUPCALL;
625
626 mptcplog((LOG_DEBUG, "%s: snxt %u sndmax %u suna %u swnd %u reinjectq %u state %u\n",
627 __func__, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax,
628 (uint32_t)mp_tp->mpt_snduna, mp_tp->mpt_sndwnd,
629 mpte->mpte_reinjectq ? 1 : 0,
630 mp_tp->mpt_state),
631 MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
632
633 old_snd_nxt = mp_tp->mpt_sndnxt;
634 while (mptcp_can_send_more(mp_tp, FALSE)) {
635 /* get the "best" subflow to be used for transmission */
636 mpts = mptcp_get_subflow(mpte, NULL, &preferred_mpts);
637 if (mpts == NULL) {
638 mptcplog((LOG_INFO, "%s: no subflow\n", __func__),
639 MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG);
640 break;
641 }
642
643 mptcplog((LOG_DEBUG, "%s: using id %u\n", __func__, mpts->mpts_connid),
644 MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
645
646 /* In case there's just one flow, we reattempt later */
647 if (mpts_tried != NULL &&
648 (mpts == mpts_tried || (mpts->mpts_flags & MPTSF_FAILINGOVER))) {
649 mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER;
650 mpts_tried->mpts_flags |= MPTSF_ACTIVE;
651 mptcp_start_timer(mpte, MPTT_REXMT);
652 mptcplog((LOG_DEBUG, "%s: retry later\n", __func__),
653 MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
654 break;
655 }
656
657 /*
658 * Automatic sizing of send socket buffer. Increase the send
659 * socket buffer size if all of the following criteria are met
660 * 1. the receiver has enough buffer space for this data
661 * 2. send buffer is filled to 7/8th with data (so we actually
662 * have data to make use of it);
663 */
664 if (tcp_do_autosendbuf == 1 &&
665 (mp_so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
666 tcp_cansbgrow(&mp_so->so_snd)) {
667 if ((mp_tp->mpt_sndwnd / 4 * 5) >= mp_so->so_snd.sb_hiwat &&
668 mp_so->so_snd.sb_cc >= (mp_so->so_snd.sb_hiwat / 8 * 7)) {
669 if (sbreserve(&mp_so->so_snd,
670 min(mp_so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
671 tcp_autosndbuf_max)) == 1) {
672 mp_so->so_snd.sb_idealsize = mp_so->so_snd.sb_hiwat;
673
674 mptcplog((LOG_DEBUG, "%s: increased snd hiwat to %u lowat %u\n",
675 __func__, mp_so->so_snd.sb_hiwat,
676 mp_so->so_snd.sb_lowat),
677 MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
678 }
679 }
680 }
681
682 DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts,
683 struct socket *, mp_so);
684 error = mptcp_subflow_output(mpte, mpts, 0);
685 if (error) {
686 /* can be a temporary loss of source address or other error */
687 mpts->mpts_flags |= MPTSF_FAILINGOVER;
688 mpts->mpts_flags &= ~MPTSF_ACTIVE;
689 mpts_tried = mpts;
690 if (error != ECANCELED)
691 mptcplog((LOG_ERR, "%s: Error = %d mpts_flags %#x\n", __func__,
692 error, mpts->mpts_flags),
693 MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
694 break;
695 }
696 /* The model is to have only one active flow at a time */
697 mpts->mpts_flags |= MPTSF_ACTIVE;
698 mpts->mpts_probesoon = mpts->mpts_probecnt = 0;
699
700 /* Allows us to update the smoothed rtt */
701 if (mptcp_probeto && mpts != preferred_mpts && preferred_mpts != NULL) {
702 if (preferred_mpts->mpts_probesoon) {
703 if ((tcp_now - preferred_mpts->mpts_probesoon) > mptcp_probeto) {
704 mptcp_subflow_output(mpte, preferred_mpts, MPTCP_SUBOUT_PROBING);
705 if (preferred_mpts->mpts_probecnt >= mptcp_probecnt) {
706 preferred_mpts->mpts_probesoon = 0;
707 preferred_mpts->mpts_probecnt = 0;
708 }
709 }
710 } else {
711 preferred_mpts->mpts_probesoon = tcp_now;
712 preferred_mpts->mpts_probecnt = 0;
713 }
714 }
715
716 if (mpte->mpte_active_sub == NULL) {
717 mpte->mpte_active_sub = mpts;
718 } else if (mpte->mpte_active_sub != mpts) {
719 struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
720 struct tcpcb *acttp = sototcpcb(mpte->mpte_active_sub->mpts_socket);
721
722 mptcplog((LOG_DEBUG, "%s: switch [%u, srtt %d] to [%u, srtt %d]\n", __func__,
723 mpte->mpte_active_sub->mpts_connid, acttp->t_srtt >> TCP_RTT_SHIFT,
724 mpts->mpts_connid, tp->t_srtt >> TCP_RTT_SHIFT),
725 (MPTCP_SENDER_DBG | MPTCP_SOCKET_DBG), MPTCP_LOGLVL_LOG);
726
727 mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE;
728 mpte->mpte_active_sub = mpts;
729
730 mptcpstats_inc_switch(mpte, mpts);
731 }
732 }
733
734 if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) {
735 if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax &&
736 mp_tp->mpt_snduna == mp_tp->mpt_sndnxt)
737 mptcp_finish_usrclosed(mpte);
738 }
739
740 mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_WUPCALL);
741
742 /* subflow errors should not be percolated back up */
743 return (0);
744}
745
746
747static struct mptsub *
748mptcp_choose_subflow(struct mptsub *mpts, struct mptsub *curbest, int *currtt)
749{
750 struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
751
752 /*
753 * Lower RTT? Take it, if it's our first one, or
754 * it doesn't has any loss, or the current one has
755 * loss as well.
756 */
757 if (tp->t_srtt && *currtt > tp->t_srtt &&
758 (curbest == NULL || tp->t_rxtshift == 0 ||
759 sototcpcb(curbest->mpts_socket)->t_rxtshift)) {
760 *currtt = tp->t_srtt;
761 return (mpts);
762 }
763
764 /*
765 * If we find a subflow without loss, take it always!
766 */
767 if (curbest &&
768 sototcpcb(curbest->mpts_socket)->t_rxtshift &&
769 tp->t_rxtshift == 0) {
770 *currtt = tp->t_srtt;
771 return (mpts);
772 }
773
774 return (curbest != NULL ? curbest : mpts);
775}
776
777static struct mptsub *
778mptcp_return_subflow(struct mptsub *mpts)
779{
780 if (mpts && mptcp_subflow_cwnd_space(mpts->mpts_socket) <= 0)
781 return (NULL);
782
783 return (mpts);
784}
785
786/*
787 * Return the most eligible subflow to be used for sending data.
788 */
789struct mptsub *
790mptcp_get_subflow(struct mptses *mpte, struct mptsub *ignore, struct mptsub **preferred)
791{
792 struct tcpcb *besttp, *secondtp;
793 struct inpcb *bestinp, *secondinp;
794 struct mptsub *mpts;
795 struct mptsub *best = NULL;
796 struct mptsub *second_best = NULL;
797 int exp_rtt = INT_MAX, cheap_rtt = INT_MAX;
798
799 /*
800 * First Step:
801 * Choose the best subflow for cellular and non-cellular interfaces.
802 */
803
804 TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
805 struct socket *so = mpts->mpts_socket;
806 struct tcpcb *tp = sototcpcb(so);
807 struct inpcb *inp = sotoinpcb(so);
808
809 mptcplog((LOG_DEBUG, "%s mpts %u ignore %d, mpts_flags %#x, suspended %u sostate %#x tpstate %u cellular %d rtt %u rxtshift %u cheap %u exp %u cwnd %d\n",
810 __func__, mpts->mpts_connid, ignore ? ignore->mpts_connid : -1, mpts->mpts_flags,
811 INP_WAIT_FOR_IF_FEEDBACK(inp), so->so_state, tp->t_state,
812 inp->inp_last_outifp ? IFNET_IS_CELLULAR(inp->inp_last_outifp) : -1,
813 tp->t_srtt, tp->t_rxtshift, cheap_rtt, exp_rtt,
814 mptcp_subflow_cwnd_space(so)),
815 MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
816
817 /*
818 * First, the hard conditions to reject subflows
819 * (e.g., not connected,...)
820 */
821 if (mpts == ignore || inp->inp_last_outifp == NULL)
822 continue;
823
824 if (INP_WAIT_FOR_IF_FEEDBACK(inp))
825 continue;
826
827 /* There can only be one subflow in degraded state */
828 if (mpts->mpts_flags & MPTSF_MP_DEGRADED) {
829 best = mpts;
830 break;
831 }
832
833 /*
834 * If this subflow is waiting to finally send, do it!
835 */
836 if (so->so_flags1 & SOF1_PRECONNECT_DATA)
837 return (mptcp_return_subflow(mpts));
838
839 /*
840 * Only send if the subflow is MP_CAPABLE. The exceptions to
841 * this rule (degraded or TFO) have been taken care of above.
842 */
843 if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE))
844 continue;
845
846 if ((so->so_state & SS_ISDISCONNECTED) ||
847 !(so->so_state & SS_ISCONNECTED) ||
848 !TCPS_HAVEESTABLISHED(tp->t_state) ||
849 tp->t_state > TCPS_CLOSE_WAIT)
850 continue;
851
852 /*
853 * Second, the soft conditions to find the subflow with best
854 * conditions for each set (aka cellular vs non-cellular)
855 */
856 if (IFNET_IS_CELLULAR(inp->inp_last_outifp))
857 second_best = mptcp_choose_subflow(mpts, second_best,
858 &exp_rtt);
859 else
860 best = mptcp_choose_subflow(mpts, best, &cheap_rtt);
861 }
862
863 /*
864 * If there is no preferred or backup subflow, and there is no active
865 * subflow use the last usable subflow.
866 */
867 if (best == NULL)
868 return (mptcp_return_subflow(second_best));
869
870 if (second_best == NULL)
871 return (mptcp_return_subflow(best));
872
873 besttp = sototcpcb(best->mpts_socket);
874 bestinp = sotoinpcb(best->mpts_socket);
875 secondtp = sototcpcb(second_best->mpts_socket);
876 secondinp = sotoinpcb(second_best->mpts_socket);
877
878 if (preferred != NULL)
879 *preferred = mptcp_return_subflow(best);
880
881 /*
882 * Second Step: Among best and second_best. Choose the one that is
883 * most appropriate for this particular service-type.
884 */
885 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) {
886 /*
887 * Only handover if Symptoms tells us to do so.
888 */
889 if (!IFNET_IS_CELLULAR(bestinp->inp_last_outifp) &&
890 mptcp_is_wifi_unusable(mpte) != 0 && mptcp_subflow_is_bad(mpte, best))
891 return (mptcp_return_subflow(second_best));
892
893 return (mptcp_return_subflow(best));
894 } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_INTERACTIVE) {
895 int rtt_thresh = mptcp_rtthist_rtthresh << TCP_RTT_SHIFT;
896 int rto_thresh = mptcp_rtothresh;
897
898 /* Adjust with symptoms information */
899 if (!IFNET_IS_CELLULAR(bestinp->inp_last_outifp) &&
900 mptcp_is_wifi_unusable(mpte) != 0) {
901 rtt_thresh /= 2;
902 rto_thresh /= 2;
903 }
904
905 if (besttp->t_srtt && secondtp->t_srtt &&
906 besttp->t_srtt >= rtt_thresh &&
907 secondtp->t_srtt < rtt_thresh) {
908 tcpstat.tcps_mp_sel_rtt++;
909 mptcplog((LOG_DEBUG, "%s: best cid %d at rtt %d, second cid %d at rtt %d\n", __func__,
910 best->mpts_connid, besttp->t_srtt >> TCP_RTT_SHIFT,
911 second_best->mpts_connid,
912 secondtp->t_srtt >> TCP_RTT_SHIFT),
913 MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG);
914 return (mptcp_return_subflow(second_best));
915 }
916
917 if (mptcp_subflow_is_bad(mpte, best) &&
918 secondtp->t_rxtshift == 0) {
919 return (mptcp_return_subflow(second_best));
920 }
921
922 /* Compare RTOs, select second_best if best's rto exceeds rtothresh */
923 if (besttp->t_rxtcur && secondtp->t_rxtcur &&
924 besttp->t_rxtcur >= rto_thresh &&
925 secondtp->t_rxtcur < rto_thresh) {
926 tcpstat.tcps_mp_sel_rto++;
927 mptcplog((LOG_DEBUG, "%s: best cid %d at rto %d, second cid %d at rto %d\n", __func__,
928 best->mpts_connid, besttp->t_rxtcur,
929 second_best->mpts_connid, secondtp->t_rxtcur),
930 MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG);
931
932 return (mptcp_return_subflow(second_best));
933 }
934
935 /*
936 * None of the above conditions for sending on the secondary
937 * were true. So, let's schedule on the best one, if he still
938 * has some space in the congestion-window.
939 */
940 return (mptcp_return_subflow(best));
941 } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_AGGREGATE) {
942 struct mptsub *tmp;
943
944 /*
945 * We only care about RTT when aggregating
946 */
947 if (besttp->t_srtt > secondtp->t_srtt) {
948 tmp = best;
949 best = second_best;
950 besttp = secondtp;
951 bestinp = secondinp;
952
953 second_best = tmp;
954 secondtp = sototcpcb(second_best->mpts_socket);
955 secondinp = sotoinpcb(second_best->mpts_socket);
956 }
957
958 /* Is there still space in the congestion window? */
959 if (mptcp_subflow_cwnd_space(bestinp->inp_socket) <= 0)
960 return (mptcp_return_subflow(second_best));
961
962 return (mptcp_return_subflow(best));
963 } else {
964 panic("Unknown service-type configured for MPTCP");
965 }
966
967 return (NULL);
968}
969
970static const char *
971mptcp_event_to_str(uint32_t event)
972{
973 const char *c = "UNDEFINED";
974 switch (event) {
975 case MPCE_CLOSE:
976 c = "MPCE_CLOSE";
977 break;
978 case MPCE_RECV_DATA_ACK:
979 c = "MPCE_RECV_DATA_ACK";
980 break;
981 case MPCE_RECV_DATA_FIN:
982 c = "MPCE_RECV_DATA_FIN";
983 break;
984 }
985 return (c);
986}
987
988static const char *
989mptcp_state_to_str(mptcp_state_t state)
990{
991 const char *c = "UNDEFINED";
992 switch (state) {
993 case MPTCPS_CLOSED:
994 c = "MPTCPS_CLOSED";
995 break;
996 case MPTCPS_LISTEN:
997 c = "MPTCPS_LISTEN";
998 break;
999 case MPTCPS_ESTABLISHED:
1000 c = "MPTCPS_ESTABLISHED";
1001 break;
1002 case MPTCPS_CLOSE_WAIT:
1003 c = "MPTCPS_CLOSE_WAIT";
1004 break;
1005 case MPTCPS_FIN_WAIT_1:
1006 c = "MPTCPS_FIN_WAIT_1";
1007 break;
1008 case MPTCPS_CLOSING:
1009 c = "MPTCPS_CLOSING";
1010 break;
1011 case MPTCPS_LAST_ACK:
1012 c = "MPTCPS_LAST_ACK";
1013 break;
1014 case MPTCPS_FIN_WAIT_2:
1015 c = "MPTCPS_FIN_WAIT_2";
1016 break;
1017 case MPTCPS_TIME_WAIT:
1018 c = "MPTCPS_TIME_WAIT";
1019 break;
1020 case MPTCPS_TERMINATE:
1021 c = "MPTCPS_TERMINATE";
1022 break;
1023 }
1024 return (c);
1025}
1026
1027void
1028mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event)
1029{
1030 mpte_lock_assert_held(mp_tp->mpt_mpte);
1031 mptcp_state_t old_state = mp_tp->mpt_state;
1032
1033 DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp,
1034 uint32_t, event);
1035
1036 switch (mp_tp->mpt_state) {
1037 case MPTCPS_CLOSED:
1038 case MPTCPS_LISTEN:
1039 mp_tp->mpt_state = MPTCPS_TERMINATE;
1040 break;
1041
1042 case MPTCPS_ESTABLISHED:
1043 if (event == MPCE_CLOSE) {
1044 mp_tp->mpt_state = MPTCPS_FIN_WAIT_1;
1045 mp_tp->mpt_sndmax += 1; /* adjust for Data FIN */
1046 } else if (event == MPCE_RECV_DATA_FIN) {
1047 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1048 mp_tp->mpt_state = MPTCPS_CLOSE_WAIT;
1049 }
1050 break;
1051
1052 case MPTCPS_CLOSE_WAIT:
1053 if (event == MPCE_CLOSE) {
1054 mp_tp->mpt_state = MPTCPS_LAST_ACK;
1055 mp_tp->mpt_sndmax += 1; /* adjust for Data FIN */
1056 }
1057 break;
1058
1059 case MPTCPS_FIN_WAIT_1:
1060 if (event == MPCE_RECV_DATA_ACK) {
1061 mp_tp->mpt_state = MPTCPS_FIN_WAIT_2;
1062 } else if (event == MPCE_RECV_DATA_FIN) {
1063 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1064 mp_tp->mpt_state = MPTCPS_CLOSING;
1065 }
1066 break;
1067
1068 case MPTCPS_CLOSING:
1069 if (event == MPCE_RECV_DATA_ACK)
1070 mp_tp->mpt_state = MPTCPS_TIME_WAIT;
1071 break;
1072
1073 case MPTCPS_LAST_ACK:
1074 if (event == MPCE_RECV_DATA_ACK)
1075 mptcp_close(mp_tp->mpt_mpte, mp_tp);
1076 break;
1077
1078 case MPTCPS_FIN_WAIT_2:
1079 if (event == MPCE_RECV_DATA_FIN) {
1080 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1081 mp_tp->mpt_state = MPTCPS_TIME_WAIT;
1082 }
1083 break;
1084
1085 case MPTCPS_TIME_WAIT:
1086 case MPTCPS_TERMINATE:
1087 break;
1088
1089 default:
1090 VERIFY(0);
1091 /* NOTREACHED */
1092 }
1093 DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp,
1094 uint32_t, event);
1095 mptcplog((LOG_INFO, "%s: %s to %s on event %s\n", __func__,
1096 mptcp_state_to_str(old_state),
1097 mptcp_state_to_str(mp_tp->mpt_state),
1098 mptcp_event_to_str(event)),
1099 MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG);
1100}
1101
1102/* If you change this function, match up mptcp_update_rcv_state_f */
1103void
1104mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp,
1105 uint16_t csum)
1106{
1107 struct mptcb *mp_tp = tptomptp(tp);
1108 u_int64_t full_dsn = 0;
1109
1110 NTOHL(dss_info->mdss_dsn);
1111 NTOHL(dss_info->mdss_subflow_seqn);
1112 NTOHS(dss_info->mdss_data_len);
1113
1114 /* XXX for autosndbuf grow sb here */
1115 MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn);
1116 mptcp_update_rcv_state_meat(mp_tp, tp,
1117 full_dsn, dss_info->mdss_subflow_seqn, dss_info->mdss_data_len,
1118 csum);
1119
1120}
1121
1122void
1123mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp,
1124 u_int64_t full_dsn, u_int32_t seqn, u_int16_t mdss_data_len,
1125 uint16_t csum)
1126{
1127 if (mdss_data_len == 0) {
1128 mptcplog((LOG_INFO, "%s: Infinite Mapping.\n", __func__),
1129 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);
1130
1131 if ((mp_tp->mpt_flags & MPTCPF_CHECKSUM) && (csum != 0)) {
1132 mptcplog((LOG_ERR, "%s: Bad checksum %x \n", __func__,
1133 csum), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR);
1134 }
1135 mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
1136 return;
1137 }
1138 mptcplog((LOG_DEBUG,
1139 "%s: seqn = %u len = %u full = %u rcvnxt = %u \n", __func__,
1140 seqn, mdss_data_len, (uint32_t)full_dsn, (uint32_t)mp_tp->mpt_rcvnxt),
1141 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
1142
1143 mptcp_notify_mpready(tp->t_inpcb->inp_socket);
1144
1145 tp->t_rcv_map.mpt_dsn = full_dsn;
1146 tp->t_rcv_map.mpt_sseq = seqn;
1147 tp->t_rcv_map.mpt_len = mdss_data_len;
1148 tp->t_rcv_map.mpt_csum = csum;
1149 tp->t_mpflags |= TMPF_EMBED_DSN;
1150}
1151
1152
1153static int
1154mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m,
1155 int hdrlen)
1156{
1157 u_int32_t datalen;
1158
1159 if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP))
1160 return 0;
1161
1162 datalen = m->m_pkthdr.mp_rlen;
1163
1164 /* unacceptable DSS option, fallback to TCP */
1165 if (m->m_pkthdr.len > ((int) datalen + hdrlen)) {
1166 mptcplog((LOG_ERR, "%s: mbuf len %d, MPTCP expected %d",
1167 __func__, m->m_pkthdr.len, datalen),
1168 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);
1169 } else {
1170 return 0;
1171 }
1172 tp->t_mpflags |= TMPF_SND_MPFAIL;
1173 mptcp_notify_mpfail(so);
1174 m_freem(m);
1175 return -1;
1176}
1177
1178int
1179mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1180 int drop_hdrlen)
1181{
1182 mptcp_insert_rmap(tp, m, th);
1183 if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m,
1184 drop_hdrlen) != 0)
1185 return -1;
1186 return 0;
1187}
1188
1189/*
1190 * MPTCP Checksum support
1191 * The checksum is calculated whenever the MPTCP DSS option is included
1192 * in the TCP packet. The checksum includes the sum of the MPTCP psuedo
1193 * header and the actual data indicated by the length specified in the
1194 * DSS option.
1195 */
1196
1197int
1198mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,
1199 uint32_t sseq, uint16_t dlen, uint16_t csum, uint16_t dfin)
1200{
1201 uint16_t mptcp_csum;
1202
1203 mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum, dfin);
1204 if (mptcp_csum) {
1205 tp->t_mpflags |= TMPF_SND_MPFAIL;
1206 mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
1207 m_freem(m);
1208 tcpstat.tcps_mp_badcsum++;
1209 return (-1);
1210 }
1211 return (0);
1212}
1213
1214static uint16_t
1215mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq,
1216 uint16_t dlen, uint16_t csum, uint16_t dfin)
1217{
1218 struct mptcb *mp_tp = tptomptp(tp);
1219 uint16_t real_len = dlen - dfin;
1220 uint32_t sum = 0;
1221
1222 if (mp_tp == NULL)
1223 return (0);
1224
1225 if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM))
1226 return (0);
1227
1228 if (tp->t_mpflags & TMPF_TCP_FALLBACK)
1229 return (0);
1230
1231 /*
1232 * The remote side may send a packet with fewer bytes than the
1233 * claimed DSS checksum length.
1234 */
1235 if ((int)m_length2(m, NULL) < real_len) {
1236 return (0xffff);
1237 }
1238
1239 if (real_len != 0)
1240 sum = m_sum16(m, 0, real_len);
1241
1242 sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum);
1243 ADDCARRY(sum);
1244 DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m,
1245 uint32_t, sum);
1246
1247 mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum),
1248 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
1249 return (~sum & 0xffff);
1250}
1251
1252uint32_t
1253mptcp_output_csum(struct mbuf *m, uint64_t dss_val, uint32_t sseq, uint16_t dlen)
1254{
1255 uint32_t sum = 0;
1256
1257 if (dlen)
1258 sum = m_sum16(m, 0, dlen);
1259
1260 dss_val = mptcp_hton64(dss_val);
1261 sseq = htonl(sseq);
1262 dlen = htons(dlen);
1263 sum += in_pseudo64(dss_val, sseq, dlen);
1264
1265 ADDCARRY(sum);
1266 sum = ~sum & 0xffff;
1267 DTRACE_MPTCP2(checksum__result, struct mbuf *, m, uint32_t, sum);
1268 mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum),
1269 MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
1270
1271 return sum;
1272}
1273
1274/*
1275 * When WiFi signal starts fading, there's more loss and RTT spikes.
1276 * Check if there has been a large spike by comparing against
1277 * a tolerable RTT spike threshold.
1278 */
1279boolean_t
1280mptcp_no_rto_spike(struct socket *so)
1281{
1282 struct tcpcb *tp = intotcpcb(sotoinpcb(so));
1283 int32_t spike = 0;
1284
1285 if (tp->t_rxtcur > mptcp_rtothresh) {
1286 spike = tp->t_rxtcur - mptcp_rtothresh;
1287
1288 mptcplog((LOG_DEBUG, "%s: spike = %d rto = %d best = %d cur = %d\n",
1289 __func__, spike,
1290 tp->t_rxtcur, tp->t_rttbest >> TCP_RTT_SHIFT,
1291 tp->t_rttcur),
1292 (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG);
1293
1294 }
1295
1296 if (spike > 0 ) {
1297 return (FALSE);
1298 } else {
1299 return (TRUE);
1300 }
1301}
1302
1303void
1304mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag)
1305{
1306 VERIFY(mpp->mpp_flags & flag);
1307 mpp->mpp_flags &= ~flag;
1308
1309 if (mptcp_should_defer_upcall(mpp))
1310 return;
1311
1312 if (mpp->mpp_flags & MPP_SHOULD_WORKLOOP) {
1313 mpp->mpp_flags &= ~MPP_SHOULD_WORKLOOP;
1314
1315 mptcp_subflow_workloop(mpp->mpp_pcbe);
1316 }
1317
1318 if (mpp->mpp_flags & MPP_SHOULD_RWAKEUP) {
1319 mpp->mpp_flags &= ~MPP_SHOULD_RWAKEUP;
1320
1321 sorwakeup(mpp->mpp_socket);
1322 }
1323
1324 if (mpp->mpp_flags & MPP_SHOULD_WWAKEUP) {
1325 mpp->mpp_flags &= ~MPP_SHOULD_WWAKEUP;
1326
1327 sowwakeup(mpp->mpp_socket);
1328 }
1329
1330 if (mpp->mpp_flags & MPP_SET_CELLICON) {
1331 mpp->mpp_flags &= ~MPP_SET_CELLICON;
1332
1333 mptcp_set_cellicon(mpp->mpp_pcbe);
1334 }
1335
1336 if (mpp->mpp_flags & MPP_UNSET_CELLICON) {
1337 mpp->mpp_flags &= ~MPP_UNSET_CELLICON;
1338
1339 mptcp_unset_cellicon();
1340 }
1341}
1342
1343void
1344mptcp_ask_for_nat64(struct ifnet *ifp)
1345{
1346 in6_post_msg(ifp, KEV_INET6_REQUEST_NAT64_PREFIX, NULL, NULL);
1347
1348 os_log_info(mptcp_log_handle,
1349 "%s: asked for NAT64-prefix on %s\n", __func__,
1350 ifp->if_name);
1351}
1352
1353static void
1354mptcp_reset_itfinfo(struct mpt_itf_info *info)
1355{
1356 info->ifindex = 0;
1357 info->has_v4_conn = 0;
1358 info->has_v6_conn = 0;
1359 info->has_nat64_conn = 0;
1360}
1361
1362void
1363mptcp_session_necp_cb(void *handle, int action, uint32_t interface_index,
1364 uint32_t necp_flags, __unused bool *viable)
1365{
1366 boolean_t has_v4 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4);
1367 boolean_t has_v6 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV6);
1368 boolean_t has_nat64 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_NAT64);
1369 boolean_t low_power = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_INTERFACE_LOW_POWER);
1370 struct mppcb *mp = (struct mppcb *)handle;
1371 struct mptses *mpte = mptompte(mp);
1372 struct socket *mp_so;
1373 struct mptcb *mp_tp;
1374 int locked = 0;
1375 uint32_t i, ifindex;
1376
1377 ifindex = interface_index;
1378 VERIFY(ifindex != IFSCOPE_NONE);
1379
1380 /* About to be garbage-collected (see note about MPTCP/NECP interactions) */
1381 if (mp->mpp_socket->so_usecount == 0)
1382 return;
1383
1384 if (action != NECP_CLIENT_CBACTION_INITIAL) {
1385 mpte_lock(mpte);
1386 locked = 1;
1387
1388 /* Check again, because it might have changed while waiting */
1389 if (mp->mpp_socket->so_usecount == 0)
1390 goto out;
1391 }
1392
1393 mpte_lock_assert_held(mpte);
1394
1395 mp_tp = mpte->mpte_mptcb;
1396 mp_so = mptetoso(mpte);
1397
1398 os_log_info(mptcp_log_handle, "%s, action: %u ifindex %u usecount %u mpt_flags %#x state %u v4 %u v6 %u nat64 %u power %u\n",
1399 __func__, action, ifindex, mp->mpp_socket->so_usecount, mp_tp->mpt_flags, mp_tp->mpt_state,
1400 has_v4, has_v6, has_nat64, low_power);
1401
1402 /* No need on fallen back sockets */
1403 if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)
1404 goto out;
1405
1406 /*
1407 * When the interface goes in low-power mode we don't want to establish
1408 * new subflows on it. Thus, mark it internally as non-viable.
1409 */
1410 if (low_power)
1411 action = NECP_CLIENT_CBACTION_NONVIABLE;
1412
1413 if (action == NECP_CLIENT_CBACTION_NONVIABLE) {
1414 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1415 if (mpte->mpte_itfinfo[i].ifindex == IFSCOPE_NONE)
1416 continue;
1417
1418 if (mpte->mpte_itfinfo[i].ifindex == ifindex)
1419 mptcp_reset_itfinfo(&mpte->mpte_itfinfo[i]);
1420 }
1421
1422 mptcp_sched_create_subflows(mpte);
1423 } else if (action == NECP_CLIENT_CBACTION_VIABLE ||
1424 action == NECP_CLIENT_CBACTION_INITIAL) {
1425 int found_slot = 0, slot_index = -1;
1426 struct ifnet *ifp;
1427
1428 ifnet_head_lock_shared();
1429 ifp = ifindex2ifnet[ifindex];
1430 ifnet_head_done();
1431
1432 if (ifp == NULL)
1433 goto out;
1434
1435 if (IFNET_IS_EXPENSIVE(ifp) &&
1436 (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE))
1437 goto out;
1438
1439 if (IFNET_IS_CELLULAR(ifp) &&
1440 (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR))
1441 goto out;
1442
1443 if (IS_INTF_CLAT46(ifp))
1444 has_v4 = FALSE;
1445
1446 /* Look for the slot on where to store/update the interface-info. */
1447 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1448 /* Found a potential empty slot where we can put it */
1449 if (mpte->mpte_itfinfo[i].ifindex == 0) {
1450 found_slot = 1;
1451 slot_index = i;
1452 }
1453
1454 /*
1455 * The interface is already in our array. Check if we
1456 * need to update it.
1457 */
1458 if (mpte->mpte_itfinfo[i].ifindex == ifindex &&
1459 (mpte->mpte_itfinfo[i].has_v4_conn != has_v4 ||
1460 mpte->mpte_itfinfo[i].has_v6_conn != has_v6 ||
1461 mpte->mpte_itfinfo[i].has_nat64_conn != has_nat64)) {
1462 found_slot = 1;
1463 slot_index = i;
1464 break;
1465 }
1466
1467 if (mpte->mpte_itfinfo[i].ifindex == ifindex) {
1468 /*
1469 * Ok, it's already there and we don't need
1470 * to update it
1471 */
1472 goto out;
1473 }
1474 }
1475
1476 if ((mpte->mpte_dst.sa_family == AF_INET || mpte->mpte_dst.sa_family == 0) &&
1477 !has_nat64 && !has_v4) {
1478 if (found_slot) {
1479 mpte->mpte_itfinfo[slot_index].has_v4_conn = has_v4;
1480 mpte->mpte_itfinfo[slot_index].has_v6_conn = has_v6;
1481 mpte->mpte_itfinfo[slot_index].has_nat64_conn = has_nat64;
1482 }
1483 mptcp_ask_for_nat64(ifp);
1484 goto out;
1485 }
1486
1487 if (found_slot == 0) {
1488 int new_size = mpte->mpte_itfinfo_size * 2;
1489 struct mpt_itf_info *info = _MALLOC(sizeof(*info) * new_size, M_TEMP, M_ZERO);
1490
1491 if (info == NULL) {
1492 os_log_error(mptcp_log_handle, "%s malloc failed for %u\n",
1493 __func__, new_size);
1494 goto out;
1495 }
1496
1497 memcpy(info, mpte->mpte_itfinfo, mpte->mpte_itfinfo_size * sizeof(*info));
1498
1499 if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE)
1500 _FREE(mpte->mpte_itfinfo, M_TEMP);
1501
1502 /* We allocated a new one, thus the first must be empty */
1503 slot_index = mpte->mpte_itfinfo_size;
1504
1505 mpte->mpte_itfinfo = info;
1506 mpte->mpte_itfinfo_size = new_size;
1507 }
1508
1509 VERIFY(slot_index >= 0 && slot_index < (int)mpte->mpte_itfinfo_size);
1510 mpte->mpte_itfinfo[slot_index].ifindex = ifindex;
1511 mpte->mpte_itfinfo[slot_index].has_v4_conn = has_v4;
1512 mpte->mpte_itfinfo[slot_index].has_v6_conn = has_v6;
1513 mpte->mpte_itfinfo[slot_index].has_nat64_conn = has_nat64;
1514
1515 mptcp_sched_create_subflows(mpte);
1516 }
1517
1518out:
1519 if (locked)
1520 mpte_unlock(mpte);
1521}
1522
1523void
1524mptcp_set_restrictions(struct socket *mp_so)
1525{
1526 struct mptses *mpte = mpsotompte(mp_so);
1527 uint32_t i;
1528
1529 mpte_lock_assert_held(mpte);
1530
1531 ifnet_head_lock_shared();
1532
1533 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1534 struct mpt_itf_info *info = &mpte->mpte_itfinfo[i];
1535 uint32_t ifindex = info->ifindex;
1536 struct ifnet *ifp;
1537
1538 if (ifindex == IFSCOPE_NONE)
1539 continue;
1540
1541 ifp = ifindex2ifnet[ifindex];
1542 if (ifp == NULL)
1543 continue;
1544
1545 if (IFNET_IS_EXPENSIVE(ifp) &&
1546 (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE))
1547 info->ifindex = IFSCOPE_NONE;
1548
1549 if (IFNET_IS_CELLULAR(ifp) &&
1550 (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR))
1551 info->ifindex = IFSCOPE_NONE;
1552 }
1553
1554 ifnet_head_done();
1555}
1556
1557