mptcp_subr.c source code [codebrowser/bsd/netinet/mptcp_subr.c]

1	/*
2	* Copyright (c) 2012-2017 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28
29	#include <kern/locks.h>
30	#include <kern/policy_internal.h>
31	#include <kern/zalloc.h>
32
33	#include <mach/sdt.h>
34
35	#include <sys/domain.h>
36	#include <sys/kdebug.h>
37	#include <sys/kern_control.h>
38	#include <sys/kernel.h>
39	#include <sys/mbuf.h>
40	#include <sys/mcache.h>
41	#include <sys/param.h>
42	#include <sys/proc.h>
43	#include <sys/protosw.h>
44	#include <sys/resourcevar.h>
45	#include <sys/socket.h>
46	#include <sys/socketvar.h>
47	#include <sys/sysctl.h>
48	#include <sys/syslog.h>
49	#include <sys/systm.h>
50
51	#include <net/content_filter.h>
52	#include <net/if.h>
53	#include <net/if_var.h>
54	#include <netinet/in.h>
55	#include <netinet/in_pcb.h>
56	#include <netinet/in_var.h>
57	#include <netinet/tcp.h>
58	#include <netinet/tcp_fsm.h>
59	#include <netinet/tcp_seq.h>
60	#include <netinet/tcp_var.h>
61	#include <netinet/mptcp_var.h>
62	#include <netinet/mptcp.h>
63	#include <netinet/mptcp_opt.h>
64	#include <netinet/mptcp_seq.h>
65	#include <netinet/mptcp_timer.h>
66	#include <libkern/crypto/sha1.h>
67	#if INET6
68	#include <netinet6/in6_pcb.h>
69	#include <netinet6/ip6protosw.h>
70	#endif /* INET6 */
71	#include <dev/random/randomdev.h>
72
73	/*
74	* Notes on MPTCP implementation.
75	*
76	* MPTCP is implemented as <SOCK_STREAM,IPPROTO_TCP> protocol in PF_MULTIPATH
77	* communication domain. The structure mtcbinfo describes the MPTCP instance
78	* of a Multipath protocol in that domain. It is used to keep track of all
79	* MPTCP PCB instances in the system, and is protected by the global lock
80	* mppi_lock.
81	*
82	* An MPTCP socket is opened by calling socket(PF_MULTIPATH, SOCK_STREAM,
83	* IPPROTO_TCP). Upon success, a Multipath PCB gets allocated and along with
84	* it comes an MPTCP Session and an MPTCP PCB. All three structures are
85	* allocated from the same memory block, and each structure has a pointer
86	* to the adjacent ones. The layout is defined by the mpp_mtp structure.
87	* The socket lock (mpp_lock) is used to protect accesses to the Multipath
88	* PCB (mppcb) as well as the MPTCP Session (mptses).
89	*
90	* The MPTCP Session is an MPTCP-specific extension to the Multipath PCB;
91	*
92	* A functioning MPTCP Session consists of one or more subflow sockets. Each
93	* subflow socket is essentially a regular PF_INET/PF_INET6 TCP socket, and is
94	* represented by the mptsub structure. Because each subflow requires access
95	* to the MPTCP Session, the MPTCP socket's so_usecount is bumped up for each
96	* subflow. This gets decremented prior to the subflow's destruction.
97	*
98	* To handle events (read, write, control) from the subflows, we do direct
99	* upcalls into the specific function.
100	*
101	* The whole MPTCP connection is protected by a single lock, the MPTCP socket's
102	* lock. Incoming data on a subflow also ends up taking this single lock. To
103	* achieve the latter, tcp_lock/unlock has been changed to rather use the lock
104	* of the MPTCP-socket.
105	*
106	* An MPTCP socket will be destroyed when its so_usecount drops to zero; this
107	* work is done by the MPTCP garbage collector which is invoked on demand by
108	* the PF_MULTIPATH garbage collector. This process will take place once all
109	* of the subflows have been destroyed.
110	*/
111
112	static void mptcp_attach_to_subf(struct socket , struct* mptcb *, uint8_t);
113	static void mptcp_detach_mptcb_from_subf(struct mptcb , struct* socket *);
114
115	static uint32_t mptcp_gc(struct mppcbinfo *);
116	static int mptcp_subflow_soreceive(struct socket , struct* sockaddr **,
117	struct uio , struct* mbuf , struct mbuf , int *);
118	static int mptcp_subflow_sosend(struct socket , struct* sockaddr *,
119	struct uio , struct* mbuf , struct* mbuf , int*);
120	static void mptcp_subflow_rupcall(struct socket , void* , int*);
121	static void mptcp_subflow_input(struct mptses , struct* mptsub *);
122	static void mptcp_subflow_wupcall(struct socket , void* , int*);
123	static void mptcp_subflow_eupcall1(struct socket , void* *, uint32_t);
124	static void mptcp_update_last_owner(struct socket so, struct* socket *mp_so);
125	static void mptcp_drop_tfo_data(struct mptses , struct* mptsub *);
126
127	static void mptcp_subflow_abort(struct mptsub , int*);
128
129	static void mptcp_send_dfin(struct socket *so);
130
131	/*
132	* Possible return values for subflow event handlers. Note that success
133	* values must be greater or equal than MPTS_EVRET_OK. Values less than that
134	* indicate errors or actions which require immediate attention; they will
135	* prevent the rest of the handlers from processing their respective events
136	* until the next round of events processing.
137	*/
138	typedef enum {
139	MPTS_EVRET_DELETE = `1`, / delete this subflow /
140	MPTS_EVRET_OK = `2`, / OK /
141	MPTS_EVRET_CONNECT_PENDING = `3`, / resume pended connects /
142	MPTS_EVRET_DISCONNECT_FALLBACK = `4`, / abort all but preferred /
143	} ev_ret_t;
144
145	static ev_ret_t mptcp_subflow_events(struct mptses , struct* mptsub , uint64_t );
146	static ev_ret_t mptcp_subflow_propagate_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
147	static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
148	static ev_ret_t mptcp_subflow_failover_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
149	static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
150	static ev_ret_t mptcp_subflow_connected_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
151	static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
152	static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
153	static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
154	static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
155	static ev_ret_t mptcp_subflow_adaptive_rtimo_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
156	static ev_ret_t mptcp_subflow_adaptive_wtimo_ev(struct mptses , struct* mptsub , uint64_t , uint64_t);
157
158	static const char *mptcp_evret2str(ev_ret_t);
159
160	static void mptcp_do_sha1(mptcp_key_t , char* *);
161	static void mptcp_init_local_parms(struct mptses *);
162
163	static unsigned int mptsub_zone_size; / size of mptsub /
164	static struct zone mptsub_zone; /* zone for mptsub /
165
166	static unsigned int mptopt_zone_size; / size of mptopt /
167	static struct zone mptopt_zone; /* zone for mptopt /
168
169	static unsigned int mpt_subauth_entry_size; / size of subf auth entry /
170	static struct zone mpt_subauth_zone; /* zone of subf auth entry /
171
172	struct mppcbinfo mtcbinfo;
173
174	#define MPTCP_SUBFLOW_WRITELEN (8 * 1024) /* bytes to write each time */
175	#define MPTCP_SUBFLOW_READLEN (8 * 1024) /* bytes to read each time */
176
177	SYSCTL_DECL(_net_inet);
178
179	SYSCTL_NODE(_net_inet, OID_AUTO, mptcp, CTLFLAG_RW\|CTLFLAG_LOCKED, `0`, "MPTCP");
180
181	uint32_t mptcp_dbg_area = `31`; / more noise if greater than 1 /
182	SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, dbg_area, CTLFLAG_RW\|CTLFLAG_LOCKED,
183	&mptcp_dbg_area, `0`, "MPTCP debug area");
184
185	uint32_t mptcp_dbg_level = `1`;
186	SYSCTL_INT(_net_inet_mptcp, OID_AUTO, dbg_level, CTLFLAG_RW \| CTLFLAG_LOCKED,
187	&mptcp_dbg_level, `0`, "MPTCP debug level");
188
189	SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, pcbcount, CTLFLAG_RD\|CTLFLAG_LOCKED,
190	&mtcbinfo.mppi_count, `0`, "Number of active PCBs");
191
192
193	static int mptcp_alternate_port = `0`;
194	SYSCTL_INT(_net_inet_mptcp, OID_AUTO, alternate_port, CTLFLAG_RW \| CTLFLAG_LOCKED,
195	&mptcp_alternate_port, `0`, "Set alternate port for MPTCP connections");
196
197	static struct protosw mptcp_subflow_protosw;
198	static struct pr_usrreqs mptcp_subflow_usrreqs;
199	#if INET6
200	static struct ip6protosw mptcp_subflow_protosw6;
201	static struct pr_usrreqs mptcp_subflow_usrreqs6;
202	#endif /* INET6 */
203
204	static uint8_t mptcp_create_subflows_scheduled;
205
206	typedef struct mptcp_subflow_event_entry {
207	uint64_t sofilt_hint_mask;
208	ev_ret_t (*sofilt_hint_ev_hdlr)(
209	struct mptses *mpte,
210	struct mptsub *mpts,
211	uint64_t *p_mpsofilt_hint,
212	uint64_t event);
213	} mptsub_ev_entry_t;
214
215	static uint8_t mptcp_cellicon_is_set;
216	static uint32_t mptcp_last_cellicon_set;
217	#define MPTCP_CELLICON_TOGGLE_RATE (5 * TCP_RETRANSHZ) /* Only toggle every 5 seconds */
218
219	/*
220	* XXX The order of the event handlers below is really
221	* really important. Think twice before changing it.
222	*/
223	static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = {
224	{
225	.sofilt_hint_mask = SO_FILT_HINT_MPCANTRCVMORE,
226	.sofilt_hint_ev_hdlr = mptcp_subflow_mpcantrcvmore_ev,
227	},
228	{
229	.sofilt_hint_mask = SO_FILT_HINT_MPFAILOVER,
230	.sofilt_hint_ev_hdlr = mptcp_subflow_failover_ev,
231	},
232	{
233	.sofilt_hint_mask = SO_FILT_HINT_CONNRESET,
234	.sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev,
235	},
236	{
237	.sofilt_hint_mask = SO_FILT_HINT_MUSTRST,
238	.sofilt_hint_ev_hdlr = mptcp_subflow_mustrst_ev,
239	},
240	{
241	.sofilt_hint_mask = SO_FILT_HINT_CANTRCVMORE,
242	.sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev,
243	},
244	{
245	.sofilt_hint_mask = SO_FILT_HINT_TIMEOUT,
246	.sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev,
247	},
248	{
249	.sofilt_hint_mask = SO_FILT_HINT_NOSRCADDR,
250	.sofilt_hint_ev_hdlr = mptcp_subflow_nosrcaddr_ev,
251	},
252	{
253	.sofilt_hint_mask = SO_FILT_HINT_IFDENIED,
254	.sofilt_hint_ev_hdlr = mptcp_subflow_ifdenied_ev,
255	},
256	{
257	.sofilt_hint_mask = SO_FILT_HINT_CONNECTED,
258	.sofilt_hint_ev_hdlr = mptcp_subflow_connected_ev,
259	},
260	{
261	.sofilt_hint_mask = SO_FILT_HINT_MPSTATUS,
262	.sofilt_hint_ev_hdlr = mptcp_subflow_mpstatus_ev,
263	},
264	{
265	.sofilt_hint_mask = SO_FILT_HINT_DISCONNECTED,
266	.sofilt_hint_ev_hdlr = mptcp_subflow_disconnected_ev,
267	},
268	{
269	.sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_RTIMO,
270	.sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_rtimo_ev,
271	},
272	{
273	.sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_WTIMO,
274	.sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_wtimo_ev,
275	},
276	};
277
278	os_log_t mptcp_log_handle;
279
280	/*
281	* Protocol pr_init callback.
282	*/
283	void
284	mptcp_init(struct protosw pp, struct* domain *dp)
285	{
286	#pragma unused(dp)
287	static int mptcp_initialized = `0`;
288	struct protosw *prp;
289	#if INET6
290	struct ip6protosw *prp6;
291	#endif /* INET6 */
292
293	VERIFY((pp->pr_flags & (PR_INITIALIZED\|PR_ATTACHED)) == PR_ATTACHED);
294
295	/ do this only once /
296	if (mptcp_initialized)
297	return;
298	mptcp_initialized = `1`;
299
300	/*
301	* Since PF_MULTIPATH gets initialized after PF_INET/INET6,
302	* we must be able to find IPPROTO_TCP entries for both.
303	*/
304	prp = pffindproto_locked(PF_INET, IPPROTO_TCP, SOCK_STREAM);
305	VERIFY(prp != NULL);
306	bcopy(prp, &mptcp_subflow_protosw, sizeof (*prp));
307	bcopy(prp->pr_usrreqs, &mptcp_subflow_usrreqs,
308	sizeof (mptcp_subflow_usrreqs));
309	mptcp_subflow_protosw.pr_entry.tqe_next = NULL;
310	mptcp_subflow_protosw.pr_entry.tqe_prev = NULL;
311	mptcp_subflow_protosw.pr_usrreqs = &mptcp_subflow_usrreqs;
312	mptcp_subflow_usrreqs.pru_soreceive = mptcp_subflow_soreceive;
313	mptcp_subflow_usrreqs.pru_sosend = mptcp_subflow_sosend;
314	mptcp_subflow_usrreqs.pru_rcvoob = pru_rcvoob_notsupp;
315	/*
316	* Socket filters shouldn't attach/detach to/from this protosw
317	* since pr_protosw is to be used instead, which points to the
318	* real protocol; if they do, it is a bug and we should panic.
319	*/
320	mptcp_subflow_protosw.pr_filter_head.tqh_first =
321	(struct socket_filter *)(uintptr_t)`0xdeadbeefdeadbeef`;
322	mptcp_subflow_protosw.pr_filter_head.tqh_last =
323	(struct socket_filter **)(uintptr_t)`0xdeadbeefdeadbeef`;
324
325	#if INET6
326	prp6 = (struct ip6protosw *)pffindproto_locked(PF_INET6,
327	IPPROTO_TCP, SOCK_STREAM);
328	VERIFY(prp6 != NULL);
329	bcopy(prp6, &mptcp_subflow_protosw6, sizeof (*prp6));
330	bcopy(prp6->pr_usrreqs, &mptcp_subflow_usrreqs6,
331	sizeof (mptcp_subflow_usrreqs6));
332	mptcp_subflow_protosw6.pr_entry.tqe_next = NULL;
333	mptcp_subflow_protosw6.pr_entry.tqe_prev = NULL;
334	mptcp_subflow_protosw6.pr_usrreqs = &mptcp_subflow_usrreqs6;
335	mptcp_subflow_usrreqs6.pru_soreceive = mptcp_subflow_soreceive;
336	mptcp_subflow_usrreqs6.pru_sosend = mptcp_subflow_sosend;
337	mptcp_subflow_usrreqs6.pru_rcvoob = pru_rcvoob_notsupp;
338	/*
339	* Socket filters shouldn't attach/detach to/from this protosw
340	* since pr_protosw is to be used instead, which points to the
341	* real protocol; if they do, it is a bug and we should panic.
342	*/
343	mptcp_subflow_protosw6.pr_filter_head.tqh_first =
344	(struct socket_filter *)(uintptr_t)`0xdeadbeefdeadbeef`;
345	mptcp_subflow_protosw6.pr_filter_head.tqh_last =
346	(struct socket_filter **)(uintptr_t)`0xdeadbeefdeadbeef`;
347	#endif /* INET6 */
348
349	bzero(&mtcbinfo, sizeof (mtcbinfo));
350	TAILQ_INIT(&mtcbinfo.mppi_pcbs);
351	mtcbinfo.mppi_size = sizeof (struct mpp_mtp);
352	if ((mtcbinfo.mppi_zone = zinit(mtcbinfo.mppi_size,
353	`1024` * mtcbinfo.mppi_size, `8192`, "mptcb")) == NULL) {
354	panic("%s: unable to allocate MPTCP PCB zone\n", __func__);
355	/ NOTREACHED /
356	}
357	zone_change(mtcbinfo.mppi_zone, Z_CALLERACCT, FALSE);
358	zone_change(mtcbinfo.mppi_zone, Z_EXPAND, TRUE);
359
360	mtcbinfo.mppi_lock_grp_attr = lck_grp_attr_alloc_init();
361	mtcbinfo.mppi_lock_grp = lck_grp_alloc_init("mppcb",
362	mtcbinfo.mppi_lock_grp_attr);
363	mtcbinfo.mppi_lock_attr = lck_attr_alloc_init();
364	lck_mtx_init(&mtcbinfo.mppi_lock, mtcbinfo.mppi_lock_grp,
365	mtcbinfo.mppi_lock_attr);
366
367	mtcbinfo.mppi_gc = mptcp_gc;
368	mtcbinfo.mppi_timer = mptcp_timer;
369
370	/ attach to MP domain for garbage collection to take place /
371	mp_pcbinfo_attach(&mtcbinfo);
372
373	mptsub_zone_size = sizeof (struct mptsub);
374	if ((mptsub_zone = zinit(mptsub_zone_size, `1024` * mptsub_zone_size,
375	`8192`, "mptsub")) == NULL) {
376	panic("%s: unable to allocate MPTCP subflow zone\n", __func__);
377	/ NOTREACHED /
378	}
379	zone_change(mptsub_zone, Z_CALLERACCT, FALSE);
380	zone_change(mptsub_zone, Z_EXPAND, TRUE);
381
382	mptopt_zone_size = sizeof (struct mptopt);
383	if ((mptopt_zone = zinit(mptopt_zone_size, `128` * mptopt_zone_size,
384	`1024`, "mptopt")) == NULL) {
385	panic("%s: unable to allocate MPTCP option zone\n", __func__);
386	/ NOTREACHED /
387	}
388	zone_change(mptopt_zone, Z_CALLERACCT, FALSE);
389	zone_change(mptopt_zone, Z_EXPAND, TRUE);
390
391	mpt_subauth_entry_size = sizeof (struct mptcp_subf_auth_entry);
392	if ((mpt_subauth_zone = zinit(mpt_subauth_entry_size,
393	`1024` * mpt_subauth_entry_size, `8192`, "mptauth")) == NULL) {
394	panic("%s: unable to allocate MPTCP address auth zone \n",
395	__func__);
396	/ NOTREACHED /
397	}
398	zone_change(mpt_subauth_zone, Z_CALLERACCT, FALSE);
399	zone_change(mpt_subauth_zone, Z_EXPAND, TRUE);
400
401	mptcp_last_cellicon_set = tcp_now;
402
403	mptcp_log_handle = os_log_create("com.apple.xnu.net.mptcp", "mptcp");
404	}
405
406	int
407	mptcp_get_statsindex(struct mptcp_itf_stats stats, const* struct mptsub *mpts)
408	{
409	const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp;
410
411	int i, index = -`1`;
412
413	if (ifp == NULL) {
414	mptcplog((LOG_ERR, "%s: no ifp on subflow\n", __func__),
415	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
416	return (-`1`);
417	}
418
419	for (i = `0`; i < MPTCP_ITFSTATS_SIZE; i++) {
420	if (stats[i].ifindex == IFSCOPE_NONE) {
421	if (index < `0`)
422	index = i;
423	continue;
424	}
425
426	if (stats[i].ifindex == ifp->if_index) {
427	index = i;
428	return (index);
429	}
430	}
431
432	if (index != -`1`) {
433	stats[index].ifindex = ifp->if_index;
434	if (stats[index].is_expensive == `0`)
435	stats[index].is_expensive = IFNET_IS_CELLULAR(ifp);
436	}
437
438	return (index);
439	}
440
441	void
442	mptcpstats_inc_switch(struct mptses mpte, const* struct mptsub *mpts)
443	{
444	int index;
445
446	tcpstat.tcps_mp_switches++;
447	mpte->mpte_subflow_switches++;
448
449	index = mptcp_get_statsindex(mpte->mpte_itfstats, mpts);
450
451	if (index != -`1`)
452	mpte->mpte_itfstats[index].switches++;
453	}
454
455	/*
456	* Flushes all recorded socket options from an MP socket.
457	*/
458	static void
459	mptcp_flush_sopts(struct mptses *mpte)
460	{
461	struct mptopt mpo, tmpo;
462
463	TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) {
464	mptcp_sopt_remove(mpte, mpo);
465	mptcp_sopt_free(mpo);
466	}
467	VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts));
468	}
469
470	/*
471	* Create an MPTCP session, called as a result of opening a MPTCP socket.
472	*/
473	int
474	mptcp_sescreate(struct mppcb *mpp)
475	{
476	struct mppcbinfo *mppi;
477	struct mptses *mpte;
478	struct mptcb *mp_tp;
479
480	VERIFY(mpp != NULL);
481	mppi = mpp->mpp_pcbinfo;
482	VERIFY(mppi != NULL);
483
484	__IGNORE_WCASTALIGN(mpte = &((struct mpp_mtp *)mpp)->mpp_ses);
485	__IGNORE_WCASTALIGN(mp_tp = &((struct mpp_mtp *)mpp)->mtcb);
486
487	/ MPTCP Multipath PCB Extension /
488	bzero(mpte, sizeof (*mpte));
489	VERIFY(mpp->mpp_pcbe == NULL);
490	mpp->mpp_pcbe = mpte;
491	mpte->mpte_mppcb = mpp;
492	mpte->mpte_mptcb = mp_tp;
493
494	TAILQ_INIT(&mpte->mpte_sopts);
495	TAILQ_INIT(&mpte->mpte_subflows);
496	mpte->mpte_associd = SAE_ASSOCID_ANY;
497	mpte->mpte_connid_last = SAE_CONNID_ANY;
498
499	mpte->mpte_itfinfo = &mpte->_mpte_itfinfo[`0`];
500	mpte->mpte_itfinfo_size = MPTE_ITFINFO_SIZE;
501
502	if (mptcp_alternate_port)
503	mpte->mpte_alternate_port = htons(mptcp_alternate_port);
504
505	/ MPTCP Protocol Control Block /
506	bzero(mp_tp, sizeof (*mp_tp));
507	mp_tp->mpt_mpte = mpte;
508	mp_tp->mpt_state = MPTCPS_CLOSED;
509
510	DTRACE_MPTCP1(session__create, struct mppcb *, mpp);
511
512	return (`0`);
513	}
514
515	static void
516	mptcpstats_get_bytes(struct mptses *mpte, boolean_t initial_cell,
517	uint64_t cellbytes, uint64_t allbytes)
518	{
519	int64_t mycellbytes = `0`;
520	uint64_t myallbytes = `0`;
521	int i;
522
523	for (i = `0`; i < MPTCP_ITFSTATS_SIZE; i++) {
524	if (mpte->mpte_itfstats[i].is_expensive) {
525	mycellbytes += mpte->mpte_itfstats[i].mpis_txbytes;
526	mycellbytes += mpte->mpte_itfstats[i].mpis_rxbytes;
527	}
528
529	myallbytes += mpte->mpte_itfstats[i].mpis_txbytes;
530	myallbytes += mpte->mpte_itfstats[i].mpis_rxbytes;
531	}
532
533	if (initial_cell) {
534	mycellbytes -= mpte->mpte_init_txbytes;
535	mycellbytes -= mpte->mpte_init_txbytes;
536	}
537
538	if (mycellbytes < `0`) {
539	mptcplog((LOG_ERR, "%s cellbytes is %d\n", __func__, mycellbytes),
540	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
541	*cellbytes = `0`;
542	*allbytes = `0`;
543	} else {
544	*cellbytes = mycellbytes;
545	*allbytes = myallbytes;
546	}
547	}
548
549	static void
550	mptcpstats_session_wrapup(struct mptses *mpte)
551	{
552	boolean_t cell = mpte->mpte_initial_cell;
553
554	switch (mpte->mpte_svctype) {
555	case MPTCP_SVCTYPE_HANDOVER:
556	if (mpte->mpte_flags & MPTE_FIRSTPARTY) {
557	tcpstat.tcps_mptcp_fp_handover_attempt++;
558
559	if (cell && mpte->mpte_handshake_success) {
560	tcpstat.tcps_mptcp_fp_handover_success_cell++;
561
562	if (mpte->mpte_used_wifi)
563	tcpstat.tcps_mptcp_handover_wifi_from_cell++;
564	} else if (mpte->mpte_handshake_success) {
565	tcpstat.tcps_mptcp_fp_handover_success_wifi++;
566
567	if (mpte->mpte_used_cell)
568	tcpstat.tcps_mptcp_handover_cell_from_wifi++;
569	}
570	} else {
571	tcpstat.tcps_mptcp_handover_attempt++;
572
573	if (cell && mpte->mpte_handshake_success) {
574	tcpstat.tcps_mptcp_handover_success_cell++;
575
576	if (mpte->mpte_used_wifi)
577	tcpstat.tcps_mptcp_handover_wifi_from_cell++;
578	} else if (mpte->mpte_handshake_success) {
579	tcpstat.tcps_mptcp_handover_success_wifi++;
580
581	if (mpte->mpte_used_cell)
582	tcpstat.tcps_mptcp_handover_cell_from_wifi++;
583	}
584	}
585
586	if (mpte->mpte_handshake_success) {
587	uint64_t cellbytes;
588	uint64_t allbytes;
589
590	mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes);
591
592	tcpstat.tcps_mptcp_handover_cell_bytes += cellbytes;
593	tcpstat.tcps_mptcp_handover_all_bytes += allbytes;
594	}
595	break;
596	case MPTCP_SVCTYPE_INTERACTIVE:
597	if (mpte->mpte_flags & MPTE_FIRSTPARTY) {
598	tcpstat.tcps_mptcp_fp_interactive_attempt++;
599
600	if (mpte->mpte_handshake_success) {
601	tcpstat.tcps_mptcp_fp_interactive_success++;
602
603	if (!cell && mpte->mpte_used_cell)
604	tcpstat.tcps_mptcp_interactive_cell_from_wifi++;
605	}
606	} else {
607	tcpstat.tcps_mptcp_interactive_attempt++;
608
609	if (mpte->mpte_handshake_success) {
610	tcpstat.tcps_mptcp_interactive_success++;
611
612	if (!cell && mpte->mpte_used_cell)
613	tcpstat.tcps_mptcp_interactive_cell_from_wifi++;
614	}
615	}
616
617	if (mpte->mpte_handshake_success) {
618	uint64_t cellbytes;
619	uint64_t allbytes;
620
621	mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes);
622
623	tcpstat.tcps_mptcp_interactive_cell_bytes += cellbytes;
624	tcpstat.tcps_mptcp_interactive_all_bytes += allbytes;
625	}
626	break;
627	case MPTCP_SVCTYPE_AGGREGATE:
628	if (mpte->mpte_flags & MPTE_FIRSTPARTY) {
629	tcpstat.tcps_mptcp_fp_aggregate_attempt++;
630
631	if (mpte->mpte_handshake_success)
632	tcpstat.tcps_mptcp_fp_aggregate_success++;
633	} else {
634	tcpstat.tcps_mptcp_aggregate_attempt++;
635
636	if (mpte->mpte_handshake_success) {
637	tcpstat.tcps_mptcp_aggregate_success++;
638	}
639	}
640
641	if (mpte->mpte_handshake_success) {
642	uint64_t cellbytes;
643	uint64_t allbytes;
644
645	mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes);
646
647	tcpstat.tcps_mptcp_aggregate_cell_bytes += cellbytes;
648	tcpstat.tcps_mptcp_aggregate_all_bytes += allbytes;
649	}
650	break;
651	}
652
653	if (cell && mpte->mpte_handshake_success && mpte->mpte_used_wifi)
654	tcpstat.tcps_mptcp_back_to_wifi++;
655
656	if (mpte->mpte_triggered_cell)
657	tcpstat.tcps_mptcp_triggered_cell++;
658	}
659
660	/*
661	* Destroy an MPTCP session.
662	*/
663	static void
664	mptcp_session_destroy(struct mptses *mpte)
665	{
666	struct mptcb *mp_tp;
667
668	mpte_lock_assert_held(mpte); / same as MP socket lock /
669
670	mp_tp = mpte->mpte_mptcb;
671	VERIFY(mp_tp != NULL);
672
673	mptcpstats_session_wrapup(mpte);
674
675	mptcp_unset_cellicon();
676
677	/*
678	* MPTCP Multipath PCB Extension section
679	*/
680	mptcp_flush_sopts(mpte);
681	VERIFY(TAILQ_EMPTY(&mpte->mpte_subflows) && mpte->mpte_numflows == `0`);
682
683	if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE)
684	_FREE(mpte->mpte_itfinfo, M_TEMP);
685
686	mpte->mpte_itfinfo = NULL;
687
688	m_freem_list(mpte->mpte_reinjectq);
689
690	/*
691	* MPTCP Protocol Control Block section
692	*/
693	DTRACE_MPTCP2(session__destroy, struct mptses *, mpte,
694	struct mptcb *, mp_tp);
695	}
696
697	static boolean_t
698	mptcp_ok_to_create_subflows(struct mptcb *mp_tp)
699	{
700	return (mp_tp->mpt_state >= MPTCPS_ESTABLISHED &&
701	mp_tp->mpt_state < MPTCPS_FIN_WAIT_1 &&
702	!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP));
703	}
704
705	static int
706	mptcp_synthesize_nat64(struct in6_addr addr, uint32_t len, struct* in_addr *addrv4)
707	{
708	static const struct in6_addr well_known_prefix = {
709	.__u6_addr.__u6_addr8 = {`0x00`, `0x64`, `0xff`, `0x9b`, `0x00`, `0x00`,
710	`0x00`, `0x00`, `0x00`, `0x00`, `0x00`, `0x00`,
711	`0x00`, `0x00`, `0x00`, `0x00`},
712	};
713	char buf[MAX_IPv6_STR_LEN];
714	char ptrv4 = (char* *)addrv4;
715	char ptr = (char* *)addr;
716
717	if (IN_ZERONET(ntohl(addrv4->s_addr)) \|\| // 0.0.0.0/8 Source hosts on local network
718	IN_LOOPBACK(ntohl(addrv4->s_addr)) \|\| // 127.0.0.0/8 Loopback
719	IN_LINKLOCAL(ntohl(addrv4->s_addr)) \|\| // 169.254.0.0/16 Link Local
720	IN_DS_LITE(ntohl(addrv4->s_addr)) \|\| // 192.0.0.0/29 DS-Lite
721	IN_6TO4_RELAY_ANYCAST(ntohl(addrv4->s_addr)) \|\| // 192.88.99.0/24 6to4 Relay Anycast
722	IN_MULTICAST(ntohl(addrv4->s_addr)) \|\| // 224.0.0.0/4 Multicast
723	INADDR_BROADCAST == addrv4->s_addr) { // 255.255.255.255/32 Limited Broadcast
724	return (-`1`);
725	}
726
727	/ Check for the well-known prefix /
728	if (len == NAT64_PREFIX_LEN_96 &&
729	IN6_ARE_ADDR_EQUAL(addr, &well_known_prefix)) {
730	if (IN_PRIVATE(ntohl(addrv4->s_addr)) \|\| // 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 Private-Use
731	IN_SHARED_ADDRESS_SPACE(ntohl(addrv4->s_addr))) // 100.64.0.0/10 Shared Address Space
732	return (-`1`);
733	}
734
735	switch (len) {
736	case NAT64_PREFIX_LEN_96:
737	memcpy(ptr + `12`, ptrv4, `4`);
738	break;
739	case NAT64_PREFIX_LEN_64:
740	memcpy(ptr + `9`, ptrv4, `4`);
741	break;
742	case NAT64_PREFIX_LEN_56:
743	memcpy(ptr + `7`, ptrv4, `1`);
744	memcpy(ptr + `9`, ptrv4 + `1`, `3`);
745	break;
746	case NAT64_PREFIX_LEN_48:
747	memcpy(ptr + `6`, ptrv4, `2`);
748	memcpy(ptr + `9`, ptrv4 + `2`, `2`);
749	break;
750	case NAT64_PREFIX_LEN_40:
751	memcpy(ptr + `5`, ptrv4, `3`);
752	memcpy(ptr + `9`, ptrv4 + `3`, `1`);
753	break;
754	case NAT64_PREFIX_LEN_32:
755	memcpy(ptr + `4`, ptrv4, `4`);
756	break;
757	default:
758	panic("NAT64-prefix len is wrong: %u\n", len);
759	}
760
761	os_log_info(mptcp_log_handle, "%s: nat64prefix-len %u synthesized %s\n",
762	__func__, len,
763	inet_ntop(AF_INET6, (void )addr, buf, sizeof*(buf)));
764
765	return (`0`);
766	}
767
768	static void
769	mptcp_trigger_cell_bringup(struct mptses *mpte)
770	{
771	struct socket *mp_so = mptetoso(mpte);
772
773	if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) {
774	uuid_string_t uuidstr;
775	int err;
776
777	err = necp_client_assert_bb_radio_manager(mpsotomppcb(mp_so)->necp_client_uuid,
778	TRUE);
779
780	if (err == `0`)
781	mpte->mpte_triggered_cell = `1`;
782
783	uuid_unparse_upper(mpsotomppcb(mp_so)->necp_client_uuid, uuidstr);
784	os_log_info(mptcp_log_handle, "%s asked irat to bringup cell for uuid %s, err %d\n",
785	__func__, uuidstr, err);
786	} else {
787	os_log_info(mptcp_log_handle, "%s UUID is already null\n", __func__);
788	}
789	}
790
791
792	void
793	mptcp_check_subflows_and_add(struct mptses *mpte)
794	{
795	struct mptcb *mp_tp = mpte->mpte_mptcb;
796	boolean_t cellular_viable = FALSE;
797	boolean_t want_cellular = TRUE;
798	uint32_t i;
799
800	if (!mptcp_ok_to_create_subflows(mp_tp))
801	return;
802
803	for (i = `0`; i < mpte->mpte_itfinfo_size; i++) {
804	struct mpt_itf_info *info;
805	struct mptsub *mpts;
806	struct ifnet *ifp;
807	uint32_t ifindex;
808	int found = `0`;
809
810	info = &mpte->mpte_itfinfo[i];
811
812	if (info->no_mptcp_support)
813	continue;
814
815	ifindex = info->ifindex;
816	if (ifindex == IFSCOPE_NONE)
817	continue;
818
819	ifnet_head_lock_shared();
820	ifp = ifindex2ifnet[ifindex];
821	ifnet_head_done();
822
823	if (ifp == NULL)
824	continue;
825
826	if (IFNET_IS_CELLULAR(ifp))
827	cellular_viable = TRUE;
828
829	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
830	const struct ifnet *subifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp;
831
832	if (subifp == NULL)
833	continue;
834
835	/*
836	* In Handover mode, only create cell subflow if
837	* 1. Wi-Fi Assist is active
838	* 2. Symptoms marked WiFi as weak
839	* 3. We are experiencing RTOs or we are not sending data.
840	*
841	* This covers the scenario, where:
842	* 1. We send and get retransmission timeouts (thus,
843	* we confirmed that WiFi is indeed bad).
844	* 2. We are not sending and the server tries to send.
845	* Establshing a cell-subflow gives the server a
846	* chance to send us some data over cell if WiFi
847	* is dead. We establish the subflow with the
848	* backup-bit set, so the server is not allowed to
849	* send on this subflow as long as WiFi is providing
850	* good performance.
851	*/
852	if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER &&
853	!IFNET_IS_CELLULAR(subifp) &&
854	!(mpts->mpts_flags & (MPTSF_DISCONNECTING \| MPTSF_DISCONNECTED \| MPTSF_CLOSE_REQD)) &&
855	(mptcp_is_wifi_unusable(mpte) == `0` \|\|
856	(sototcpcb(mpts->mpts_socket)->t_rxtshift < mptcp_fail_thresh * `2` &&
857	((mpte->mpte_flags & MPTE_FIRSTPARTY) \|\| mptetoso(mpte)->so_snd.sb_cc)))) {
858	os_log_debug(mptcp_log_handle, "%s handover, wifi state %d rxt %u first-party %u sb_cc %u ifindex %u this %u\n",
859	__func__, mptcp_is_wifi_unusable(mpte),
860	sototcpcb(mpts->mpts_socket)->t_rxtshift,
861	!!(mpte->mpte_flags & MPTE_FIRSTPARTY),
862	mptetoso(mpte)->so_snd.sb_cc,
863	ifindex, subifp->if_index);
864	found = `1`;
865
866	/ We found a proper subflow on WiFi - no need for cell /
867	want_cellular = FALSE;
868	break;
869	} else {
870	os_log_debug(mptcp_log_handle, "%s svc %u cell %u flags %#x unusable %d rtx %u first %u sbcc %u\n",
871	__func__, mpte->mpte_svctype, IFNET_IS_CELLULAR(subifp), mpts->mpts_flags,
872	mptcp_is_wifi_unusable(mpte), sototcpcb(mpts->mpts_socket)->t_rxtshift,
873	!!(mpte->mpte_flags & MPTE_FIRSTPARTY), mptetoso(mpte)->so_snd.sb_cc);
874
875	}
876
877	if (subifp->if_index == ifindex &&
878	!(mpts->mpts_socket->so_state & SS_ISDISCONNECTED) &&
879	sototcpcb(mpts->mpts_socket)->t_state != TCPS_CLOSED) {
880	/*
881	* We found a subflow on this interface.
882	* No need to create a new one.
883	*/
884	found = `1`;
885	break;
886	}
887	}
888
889	if (!found && !(mpte->mpte_flags & MPTE_FIRSTPARTY) &&
890	!(mpte->mpte_flags & MPTE_ACCESS_GRANTED) &&
891	mptcp_developer_mode == `0`) {
892	mptcp_ask_symptoms(mpte);
893	return;
894	}
895
896	if (!found) {
897	struct sockaddr *dst = &mpte->mpte_dst;
898	struct sockaddr_in6 nat64pre;
899
900	if (mpte->mpte_dst.sa_family == AF_INET &&
901	!info->has_v4_conn && info->has_nat64_conn) {
902	struct ipv6_prefix nat64prefixes[NAT64_MAX_NUM_PREFIXES];
903	int error, j;
904
905	bzero(&nat64pre, sizeof(struct sockaddr_in6));
906
907	error = ifnet_get_nat64prefix(ifp, nat64prefixes);
908	if (error) {
909	os_log_error(mptcp_log_handle, "%s: no NAT64-prefix on itf %s, error %d\n",
910	__func__, ifp->if_name, error);
911	continue;
912	}
913
914	for (j = `0`; j < NAT64_MAX_NUM_PREFIXES; j++) {
915	if (nat64prefixes[j].prefix_len != `0`)
916	break;
917	}
918
919	VERIFY(j < NAT64_MAX_NUM_PREFIXES);
920
921	error = mptcp_synthesize_nat64(&nat64prefixes[j].ipv6_prefix,
922	nat64prefixes[j].prefix_len,
923	&mpte->__mpte_dst_v4.sin_addr);
924	if (error != `0`) {
925	os_log_info(mptcp_log_handle, "%s: cannot synthesize this addr\n",
926	__func__);
927	continue;
928	}
929
930	memcpy(&nat64pre.sin6_addr,
931	&nat64prefixes[j].ipv6_prefix,
932	sizeof(nat64pre.sin6_addr));
933	nat64pre.sin6_len = sizeof(struct sockaddr_in6);
934	nat64pre.sin6_family = AF_INET6;
935	nat64pre.sin6_port = mpte->__mpte_dst_v6.sin6_port;
936	nat64pre.sin6_flowinfo = `0`;
937	nat64pre.sin6_scope_id = `0`;
938
939	dst = (struct sockaddr *)&nat64pre;
940	}
941
942	/ Initial subflow started on a NAT64'd address? /
943	if (mpte->mpte_dst.sa_family == AF_INET6 &&
944	mpte->mpte_dst_v4_nat64.sin_family == AF_INET) {
945	dst = (struct sockaddr *)&mpte->mpte_dst_v4_nat64;
946	}
947
948	if (dst->sa_family == AF_INET && !info->has_v4_conn)
949	continue;
950	if (dst->sa_family == AF_INET6 && !info->has_v6_conn)
951	continue;
952
953	mptcp_subflow_add(mpte, NULL, dst, ifindex, NULL);
954	}
955	}
956
957	if (!cellular_viable && want_cellular) {
958	/ Trigger Cell Bringup /
959	mptcp_trigger_cell_bringup(mpte);
960	}
961	}
962
963	/*
964	* Based on the MPTCP Service-type and the state of the subflows, we
965	* will destroy subflows here.
966	*/
967	static void
968	mptcp_check_subflows_and_remove(struct mptses *mpte)
969	{
970	struct mptsub mpts, tmpts;
971	int found_working_subflow = `0`, removed_some = `0`;
972	int wifi_unusable = mptcp_is_wifi_unusable(mpte);
973
974	if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER)
975	return;
976
977	/*
978	* Look for a subflow that is on a non-cellular interface
979	* and actually works (aka, no retransmission timeout).
980	*/
981	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
982	const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp;
983	struct socket *so;
984	struct tcpcb *tp;
985
986	if (ifp == NULL \|\| IFNET_IS_CELLULAR(ifp))
987	continue;
988
989	so = mpts->mpts_socket;
990	tp = sototcpcb(so);
991
992	if (!(mpts->mpts_flags & MPTSF_CONNECTED) \|\|
993	tp->t_state != TCPS_ESTABLISHED)
994	continue;
995
996	/ Is this subflow in good condition? /
997	if (tp->t_rxtshift == `0`)
998	found_working_subflow = `1`;
999
1000	/ Or WiFi is fine /
1001	if (!wifi_unusable)
1002	found_working_subflow = `1`;
1003	}
1004
1005	/*
1006	* Couldn't find a working subflow, let's not remove those on a cellular
1007	* interface.
1008	*/
1009	if (!found_working_subflow)
1010	return;
1011
1012	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
1013	const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp;
1014
1015	/ Only remove cellular subflows /
1016	if (ifp == NULL \|\| !IFNET_IS_CELLULAR(ifp))
1017	continue;
1018
1019	soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MUSTRST);
1020	removed_some = `1`;
1021	}
1022
1023	if (removed_some)
1024	mptcp_unset_cellicon();
1025	}
1026
1027	static void
1028	mptcp_remove_subflows(struct mptses *mpte)
1029	{
1030	struct mptsub mpts, tmpts;
1031
1032	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
1033	if (mpts->mpts_flags & MPTSF_CLOSE_REQD) {
1034	mpts->mpts_flags &= ~MPTSF_CLOSE_REQD;
1035
1036	soevent(mpts->mpts_socket,
1037	SO_FILT_HINT_LOCKED \| SO_FILT_HINT_NOSRCADDR);
1038	}
1039	}
1040	}
1041
1042	static void
1043	mptcp_create_subflows(__unused void *arg)
1044	{
1045	struct mppcb *mpp;
1046
1047	/*
1048	* Start with clearing, because we might be processing connections
1049	* while a new event comes in.
1050	*/
1051	if (OSTestAndClear(`0x01`, &mptcp_create_subflows_scheduled))
1052	mptcplog((LOG_ERR, "%s: bit was already cleared!\n", __func__),
1053	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1054
1055	/ Iterate over all MPTCP connections /
1056
1057	lck_mtx_lock(&mtcbinfo.mppi_lock);
1058
1059	TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) {
1060	struct mptses *mpte;
1061	struct socket *mp_so;
1062
1063	if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS))
1064	continue;
1065
1066	mpp_lock(mpp);
1067
1068	mpp->mpp_flags &= ~MPP_CREATE_SUBFLOWS;
1069
1070	mpte = mpp->mpp_pcbe;
1071	mp_so = mpp->mpp_socket;
1072
1073	VERIFY(mp_so->so_usecount > `0`);
1074
1075	mptcp_check_subflows_and_add(mpte);
1076	mptcp_remove_subflows(mpte);
1077
1078	mp_so->so_usecount--; / See mptcp_sched_create_subflows /
1079	mpp_unlock(mpp);
1080	}
1081
1082	lck_mtx_unlock(&mtcbinfo.mppi_lock);
1083	}
1084
1085	/*
1086	* We need this because we are coming from an NECP-event. This event gets posted
1087	* while holding NECP-locks. The creation of the subflow however leads us back
1088	* into NECP (e.g., to add the necp_cb and also from tcp_connect).
1089	* So, we would deadlock there as we already hold the NECP-lock.
1090	*
1091	* So, let's schedule this separately. It also gives NECP the chance to make
1092	* progress, without having to wait for MPTCP to finish its subflow creation.
1093	*/
1094	void
1095	mptcp_sched_create_subflows(struct mptses *mpte)
1096	{
1097	struct mppcb *mpp = mpte->mpte_mppcb;
1098	struct mptcb *mp_tp = mpte->mpte_mptcb;
1099	struct socket *mp_so = mpp->mpp_socket;
1100
1101	if (!mptcp_ok_to_create_subflows(mp_tp)) {
1102	mptcplog((LOG_DEBUG, "%s: not a good time for subflows, state %u flags %#x",
1103	__func__, mp_tp->mpt_state, mp_tp->mpt_flags),
1104	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
1105	return;
1106	}
1107
1108	if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS)) {
1109	mp_so->so_usecount++; / To prevent it from being free'd in-between /
1110	mpp->mpp_flags \|= MPP_CREATE_SUBFLOWS;
1111	}
1112
1113	if (OSTestAndSet(`0x01`, &mptcp_create_subflows_scheduled))
1114	return;
1115
1116	/ Do the call in 100ms to allow NECP to schedule it on all sockets /
1117	timeout(mptcp_create_subflows, NULL, hz/`10`);
1118	}
1119
1120	/*
1121	* Allocate an MPTCP socket option structure.
1122	*/
1123	struct mptopt *
1124	mptcp_sopt_alloc(int how)
1125	{
1126	struct mptopt *mpo;
1127
1128	mpo = (how == M_WAITOK) ? zalloc(mptopt_zone) :
1129	zalloc_noblock(mptopt_zone);
1130	if (mpo != NULL) {
1131	bzero(mpo, mptopt_zone_size);
1132	}
1133
1134	return (mpo);
1135	}
1136
1137	/*
1138	* Free an MPTCP socket option structure.
1139	*/
1140	void
1141	mptcp_sopt_free(struct mptopt *mpo)
1142	{
1143	VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED));
1144
1145	zfree(mptopt_zone, mpo);
1146	}
1147
1148	/*
1149	* Add a socket option to the MPTCP socket option list.
1150	*/
1151	void
1152	mptcp_sopt_insert(struct mptses mpte, struct* mptopt *mpo)
1153	{
1154	mpte_lock_assert_held(mpte); / same as MP socket lock /
1155	mpo->mpo_flags \|= MPOF_ATTACHED;
1156	TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry);
1157	}
1158
1159	/*
1160	* Remove a socket option from the MPTCP socket option list.
1161	*/
1162	void
1163	mptcp_sopt_remove(struct mptses mpte, struct* mptopt *mpo)
1164	{
1165	mpte_lock_assert_held(mpte); / same as MP socket lock /
1166	VERIFY(mpo->mpo_flags & MPOF_ATTACHED);
1167	mpo->mpo_flags &= ~MPOF_ATTACHED;
1168	TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry);
1169	}
1170
1171	/*
1172	* Search for an existing <sopt_level,sopt_name> socket option.
1173	*/
1174	struct mptopt *
1175	mptcp_sopt_find(struct mptses mpte, struct* sockopt *sopt)
1176	{
1177	struct mptopt *mpo;
1178
1179	mpte_lock_assert_held(mpte); / same as MP socket lock /
1180
1181	TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) {
1182	if (mpo->mpo_level == sopt->sopt_level &&
1183	mpo->mpo_name == sopt->sopt_name)
1184	break;
1185	}
1186	return (mpo);
1187	}
1188
1189	/*
1190	* Allocate a MPTCP subflow structure.
1191	*/
1192	static struct mptsub *
1193	mptcp_subflow_alloc(void)
1194	{
1195	struct mptsub *mpts = zalloc(mptsub_zone);
1196
1197	if (mpts == NULL)
1198	return (NULL);
1199
1200	bzero(mpts, mptsub_zone_size);
1201	return (mpts);
1202	}
1203
1204	/*
1205	* Deallocate a subflow structure, called when all of the references held
1206	* on it have been released. This implies that the subflow has been deleted.
1207	*/
1208	static void
1209	mptcp_subflow_free(struct mptsub *mpts)
1210	{
1211	VERIFY(mpts->mpts_refcnt == `0`);
1212	VERIFY(!(mpts->mpts_flags & MPTSF_ATTACHED));
1213	VERIFY(mpts->mpts_mpte == NULL);
1214	VERIFY(mpts->mpts_socket == NULL);
1215
1216	if (mpts->mpts_src != NULL) {
1217	FREE(mpts->mpts_src, M_SONAME);
1218	mpts->mpts_src = NULL;
1219	}
1220
1221	zfree(mptsub_zone, mpts);
1222	}
1223
1224	static void
1225	mptcp_subflow_addref(struct mptsub *mpts)
1226	{
1227	if (++mpts->mpts_refcnt == `0`)
1228	panic("%s: mpts %p wraparound refcnt\n", __func__, mpts);
1229	/ NOTREACHED /
1230	}
1231
1232	static void
1233	mptcp_subflow_remref(struct mptsub *mpts)
1234	{
1235	if (mpts->mpts_refcnt == `0`) {
1236	panic("%s: mpts %p negative refcnt\n", __func__, mpts);
1237	/ NOTREACHED /
1238	}
1239	if (--mpts->mpts_refcnt > `0`)
1240	return;
1241
1242	/ callee will unlock and destroy lock /
1243	mptcp_subflow_free(mpts);
1244	}
1245
1246	static void
1247	mptcp_subflow_attach(struct mptses mpte, struct* mptsub mpts, struct* socket *so)
1248	{
1249	struct socket *mp_so = mpte->mpte_mppcb->mpp_socket;
1250	struct tcpcb *tp = sototcpcb(so);
1251
1252	/*
1253	* From this moment on, the subflow is linked to the MPTCP-connection.
1254	* Locking,... happens now at the MPTCP-layer
1255	*/
1256	tp->t_mptcb = mpte->mpte_mptcb;
1257	so->so_flags \|= SOF_MP_SUBFLOW;
1258	mp_so->so_usecount++;
1259
1260	/*
1261	* Insert the subflow into the list, and associate the MPTCP PCB
1262	* as well as the the subflow socket. From this point on, removing
1263	* the subflow needs to be done via mptcp_subflow_del().
1264	*/
1265	TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry);
1266	mpte->mpte_numflows++;
1267
1268	atomic_bitset_32(&mpts->mpts_flags, MPTSF_ATTACHED);
1269	mpts->mpts_mpte = mpte;
1270	mpts->mpts_socket = so;
1271	tp->t_mpsub = mpts;
1272	mptcp_subflow_addref(mpts); / for being in MPTCP subflow list /
1273	mptcp_subflow_addref(mpts); / for subflow socket /
1274	}
1275
1276	static void
1277	mptcp_subflow_necp_cb(void handle, __unused int* action,
1278	__unused uint32_t interface_index,
1279	uint32_t necp_flags, bool *viable)
1280	{
1281	boolean_t low_power = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_INTERFACE_LOW_POWER);
1282	struct inpcb inp = (struct* inpcb *)handle;
1283	struct socket *so = inp->inp_socket;
1284	struct mptsub *mpts;
1285	struct mptses *mpte;
1286
1287	if (low_power)
1288	action = NECP_CLIENT_CBACTION_NONVIABLE;
1289
1290	if (action != NECP_CLIENT_CBACTION_NONVIABLE)
1291	return;
1292
1293	/*
1294	* The socket is being garbage-collected. There is nothing to be done
1295	* here.
1296	*/
1297	if (so->so_usecount == `0`)
1298	return;
1299
1300	socket_lock(so, `1`);
1301
1302	/ Check again after we acquired the lock. /
1303	if (so->so_usecount == `0`)
1304	goto out;
1305
1306	mpte = tptomptp(sototcpcb(so))->mpt_mpte;
1307	mpts = sototcpcb(so)->t_mpsub;
1308
1309	os_log_debug(mptcp_log_handle, "%s Subflow on itf %u became non-viable, power %u",
1310	__func__, mpts->mpts_ifscope, low_power);
1311
1312	mpts->mpts_flags \|= MPTSF_CLOSE_REQD;
1313
1314	mptcp_sched_create_subflows(mpte);
1315
1316	if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER && viable != NULL)
1317	*viable = `1`;
1318
1319	out:
1320	socket_unlock(so, `1`);
1321	}
1322
1323	/*
1324	* Create an MPTCP subflow socket.
1325	*/
1326	static int
1327	mptcp_subflow_socreate(struct mptses mpte, struct* mptsub mpts, int* dom,
1328	struct socket **so)
1329	{
1330	lck_mtx_t *subflow_mtx;
1331	struct mptopt smpo, mpo, tmpo;
1332	struct proc *p;
1333	struct socket *mp_so;
1334	int error;
1335
1336	*so = NULL;
1337	mpte_lock_assert_held(mpte); / same as MP socket lock /
1338	mp_so = mptetoso(mpte);
1339
1340	p = proc_find(mp_so->last_pid);
1341	if (p == PROC_NULL) {
1342	mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid),
1343	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1344
1345	return (ESRCH);
1346	}
1347
1348	/*
1349	* Create the subflow socket (multipath subflow, non-blocking.)
1350	*
1351	* This will cause SOF_MP_SUBFLOW socket flag to be set on the subflow
1352	* socket; it will be cleared when the socket is peeled off or closed.
1353	* It also indicates to the underlying TCP to handle MPTCP options.
1354	* A multipath subflow socket implies SS_NOFDREF state.
1355	*/
1356
1357	/*
1358	* Unlock, because tcp_usr_attach ends up in in_pcballoc, which takes
1359	* the ipi-lock. We cannot hold the socket-lock at that point.
1360	*/
1361	mpte_unlock(mpte);
1362	error = socreate_internal(dom, so, SOCK_STREAM, IPPROTO_TCP, p,
1363	SOCF_ASYNC, PROC_NULL);
1364	mpte_lock(mpte);
1365	if (error) {
1366	mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx unable to create subflow socket error %d\n",
1367	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), error),
1368	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1369
1370	proc_rele(p);
1371
1372	mptcp_subflow_free(mpts);
1373	return (error);
1374	}
1375
1376	/*
1377	* We need to protect the setting of SOF_MP_SUBFLOW with a lock, because
1378	* this marks the moment of lock-switch from the TCP-lock to the MPTCP-lock.
1379	* Which is why we also need to get the lock with pr_getlock, as after
1380	* setting the flag, socket_unlock will work on the MPTCP-level lock.
1381	*/
1382	subflow_mtx = ((so)->so_proto->pr_getlock)(so, `0`);
1383	lck_mtx_lock(subflow_mtx);
1384
1385	/*
1386	* Must be the first thing we do, to make sure all pointers for this
1387	* subflow are set.
1388	*/
1389	mptcp_subflow_attach(mpte, mpts, *so);
1390
1391	/*
1392	* A multipath subflow socket is used internally in the kernel,
1393	* therefore it does not have a file desciptor associated by
1394	* default.
1395	*/
1396	(*so)->so_state \|= SS_NOFDREF;
1397
1398	lck_mtx_unlock(subflow_mtx);
1399
1400	/ prevent the socket buffers from being compressed /
1401	(*so)->so_rcv.sb_flags \|= SB_NOCOMPRESS;
1402	(*so)->so_snd.sb_flags \|= SB_NOCOMPRESS;
1403
1404	/ Inherit preconnect and TFO data flags /
1405	if (mp_so->so_flags1 & SOF1_PRECONNECT_DATA)
1406	(*so)->so_flags1 \|= SOF1_PRECONNECT_DATA;
1407	if (mp_so->so_flags1 & SOF1_DATA_IDEMPOTENT)
1408	(*so)->so_flags1 \|= SOF1_DATA_IDEMPOTENT;
1409
1410	/ Inherit uuid and create the related flow. /
1411	if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) {
1412	struct mptcb *mp_tp = mpte->mpte_mptcb;
1413
1414	sotoinpcb(*so)->necp_cb = mptcp_subflow_necp_cb;
1415
1416	/*
1417	* A note on the unlock: With MPTCP, we do multiple times a
1418	* necp_client_register_socket_flow. This is problematic,
1419	* because now the lock-ordering guarantee (first necp-locks,
1420	* then socket-locks) is no more respected. So, we need to
1421	* unlock here.
1422	*/
1423	mpte_unlock(mpte);
1424	error = necp_client_register_socket_flow(mp_so->last_pid,
1425	mpsotomppcb(mp_so)->necp_client_uuid, sotoinpcb(*so));
1426	mpte_lock(mpte);
1427
1428	if (error)
1429	goto out_err;
1430
1431	/ Possible state-change during the unlock above /
1432	if (mp_tp->mpt_state >= MPTCPS_TIME_WAIT \|\|
1433	(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP))
1434	goto out_err;
1435
1436	uuid_copy(sotoinpcb(*so)->necp_client_uuid, mpsotomppcb(mp_so)->necp_client_uuid);
1437	} else {
1438	mptcplog((LOG_NOTICE, "%s: uuid is not set!\n"),
1439	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG);
1440	}
1441
1442	/ inherit the other socket options /
1443	bzero(&smpo, sizeof (smpo));
1444	smpo.mpo_flags \|= MPOF_SUBFLOW_OK;
1445	smpo.mpo_level = SOL_SOCKET;
1446	smpo.mpo_intval = `1`;
1447
1448	/ disable SIGPIPE /
1449	smpo.mpo_name = SO_NOSIGPIPE;
1450	if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != `0`)
1451	goto out_err;
1452
1453	/ find out if the subflow's source address goes away /
1454	smpo.mpo_name = SO_NOADDRERR;
1455	if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != `0`)
1456	goto out_err;
1457
1458	/ enable keepalive /
1459	smpo.mpo_name = SO_KEEPALIVE;
1460	if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != `0`)
1461	goto out_err;
1462
1463	smpo.mpo_level = IPPROTO_TCP;
1464	smpo.mpo_intval = mptcp_subflow_keeptime;
1465	smpo.mpo_name = TCP_KEEPALIVE;
1466	if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != `0`)
1467	goto out_err;
1468
1469	if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED) {
1470	/*
1471	* On secondary subflows we might need to set the cell-fallback
1472	* flag (see conditions in mptcp_subflow_sosetopt).
1473	*/
1474	smpo.mpo_level = SOL_SOCKET;
1475	smpo.mpo_name = SO_MARK_CELLFALLBACK;
1476	smpo.mpo_intval = `1`;
1477	if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != `0`)
1478	goto out_err;
1479	}
1480
1481	/ replay setsockopt(2) on the subflow sockets for eligible options /
1482	TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) {
1483	int interim;
1484
1485	if (!(mpo->mpo_flags & MPOF_SUBFLOW_OK))
1486	continue;
1487
1488	/*
1489	* Skip those that are handled internally; these options
1490	* should not have been recorded and marked with the
1491	* MPOF_SUBFLOW_OK by mptcp_setopt(), but just in case.
1492	*/
1493	if (mpo->mpo_level == SOL_SOCKET &&
1494	(mpo->mpo_name == SO_NOSIGPIPE \|\|
1495	mpo->mpo_name == SO_NOADDRERR \|\|
1496	mpo->mpo_name == SO_KEEPALIVE))
1497	continue;
1498
1499	interim = (mpo->mpo_flags & MPOF_INTERIM);
1500	if (mptcp_subflow_sosetopt(mpte, mpts, mpo) != `0` && interim) {
1501	mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx"
1502	" sopt %s val %d interim record removed\n", __func__,
1503	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
1504	mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name),
1505	mpo->mpo_intval),
1506	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1507	mptcp_sopt_remove(mpte, mpo);
1508	mptcp_sopt_free(mpo);
1509	continue;
1510	}
1511	}
1512
1513	/*
1514	* We need to receive everything that the subflow socket has,
1515	* so use a customized socket receive function. We will undo
1516	* this when the socket is peeled off or closed.
1517	*/
1518	switch (dom) {
1519	case PF_INET:
1520	(*so)->so_proto = &mptcp_subflow_protosw;
1521	break;
1522	#if INET6
1523	case PF_INET6:
1524	(so)->so_proto = (struct* protosw *)&mptcp_subflow_protosw6;
1525	break;
1526	#endif /* INET6 */
1527	default:
1528	VERIFY(`0`);
1529	/ NOTREACHED /
1530	}
1531
1532	proc_rele(p);
1533
1534	DTRACE_MPTCP3(subflow__create, struct mptses *, mpte,
1535	int, dom, int, error);
1536
1537	return (`0`);
1538
1539	out_err:
1540	mptcp_subflow_abort(mpts, error);
1541
1542	proc_rele(p);
1543
1544	mptcplog((LOG_ERR, "%s: subflow socreate failed with error %d\n",
1545	__func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1546
1547	return (error);
1548	}
1549
1550	/*
1551	* Close an MPTCP subflow socket.
1552	*
1553	* Note that this may be called on an embryonic subflow, and the only
1554	* thing that is guaranteed valid is the protocol-user request.
1555	*/
1556	static void
1557	mptcp_subflow_soclose(struct mptsub *mpts)
1558	{
1559	struct socket *so = mpts->mpts_socket;
1560
1561	if (mpts->mpts_flags & MPTSF_CLOSED)
1562	return;
1563
1564	VERIFY(so != NULL);
1565	VERIFY(so->so_flags & SOF_MP_SUBFLOW);
1566	VERIFY((so->so_state & (SS_NBIO\|SS_NOFDREF)) == (SS_NBIO\|SS_NOFDREF));
1567
1568	DTRACE_MPTCP5(subflow__close, struct mptsub *, mpts,
1569	struct socket *, so,
1570	struct sockbuf *, &so->so_rcv,
1571	struct sockbuf *, &so->so_snd,
1572	struct mptses *, mpts->mpts_mpte);
1573
1574	mpts->mpts_flags \|= MPTSF_CLOSED;
1575
1576	if (so->so_retaincnt == `0`) {
1577	soclose_locked(so);
1578
1579	return;
1580	} else {
1581	VERIFY(so->so_usecount > `0`);
1582	so->so_usecount--;
1583	}
1584
1585	return;
1586	}
1587
1588	/*
1589	* Connect an MPTCP subflow socket.
1590	*
1591	* Note that in the pending connect case, the subflow socket may have been
1592	* bound to an interface and/or a source IP address which may no longer be
1593	* around by the time this routine is called; in that case the connect attempt
1594	* will most likely fail.
1595	*/
1596	static int
1597	mptcp_subflow_soconnectx(struct mptses mpte, struct* mptsub *mpts)
1598	{
1599	char dbuf[MAX_IPv6_STR_LEN];
1600	struct socket mp_so, so;
1601	struct mptcb *mp_tp;
1602	struct sockaddr *dst;
1603	struct proc *p;
1604	int af, error, dport;
1605
1606	mp_so = mptetoso(mpte);
1607	mp_tp = mpte->mpte_mptcb;
1608	so = mpts->mpts_socket;
1609	af = mpts->mpts_dst.sa_family;
1610	dst = &mpts->mpts_dst;
1611
1612	VERIFY((mpts->mpts_flags & (MPTSF_CONNECTING\|MPTSF_CONNECTED)) == MPTSF_CONNECTING);
1613	VERIFY(mpts->mpts_socket != NULL);
1614	VERIFY(af == AF_INET \|\| af == AF_INET6);
1615
1616	if (af == AF_INET) {
1617	inet_ntop(af, &SIN(dst)->sin_addr.s_addr, dbuf, sizeof (dbuf));
1618	dport = ntohs(SIN(dst)->sin_port);
1619	} else {
1620	inet_ntop(af, &SIN6(dst)->sin6_addr, dbuf, sizeof (dbuf));
1621	dport = ntohs(SIN6(dst)->sin6_port);
1622	}
1623
1624	os_log_info(mptcp_log_handle,
1625	"%s: ifindex %u dst %s:%d pended %u\n", __func__, mpts->mpts_ifscope,
1626	dbuf, dport, !!(mpts->mpts_flags & MPTSF_CONNECT_PENDING));
1627
1628	p = proc_find(mp_so->last_pid);
1629	if (p == PROC_NULL) {
1630	mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid),
1631	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1632
1633	return (ESRCH);
1634	}
1635
1636	mpts->mpts_flags &= ~MPTSF_CONNECT_PENDING;
1637
1638	mptcp_attach_to_subf(so, mpte->mpte_mptcb, mpte->mpte_addrid_last);
1639
1640	/ connect the subflow socket /
1641	error = soconnectxlocked(so, mpts->mpts_src, &mpts->mpts_dst,
1642	p, mpts->mpts_ifscope,
1643	mpte->mpte_associd, NULL, `0`, NULL, `0`, NULL, NULL);
1644
1645	mpts->mpts_iss = sototcpcb(so)->iss;
1646
1647	/ See tcp_connect_complete /
1648	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED &&
1649	(mp_so->so_flags1 & SOF1_PRECONNECT_DATA)) {
1650	mp_tp->mpt_sndwnd = sototcpcb(so)->snd_wnd;
1651	}
1652
1653	/ Allocate a unique address id per subflow /
1654	mpte->mpte_addrid_last++;
1655	if (mpte->mpte_addrid_last == `0`)
1656	mpte->mpte_addrid_last++;
1657
1658	proc_rele(p);
1659
1660	DTRACE_MPTCP3(subflow__connect, struct mptses *, mpte,
1661	struct mptsub , mpts, int*, error);
1662	if (error)
1663	mptcplog((LOG_ERR, "%s: connectx failed with error %d ifscope %u\n",
1664	__func__, error, mpts->mpts_ifscope),
1665	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
1666
1667	return (error);
1668	}
1669
1670	/*
1671	* MPTCP subflow socket receive routine, derived from soreceive().
1672	*/
1673	static int
1674	mptcp_subflow_soreceive(struct socket so, struct* sockaddr **psa,
1675	struct uio uio, struct* mbuf mp0, struct mbuf controlp, int *flagsp)
1676	{
1677	#pragma unused(uio)
1678	struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte);
1679	int flags, error = `0`;
1680	struct proc *p = current_proc();
1681	struct mbuf m, *mp = mp0;
1682	boolean_t proc_held = FALSE;
1683
1684	mpte_lock_assert_held(tptomptp(sototcpcb(so))->mpt_mpte);
1685	VERIFY(so->so_proto->pr_flags & PR_CONNREQUIRED);
1686
1687	#ifdef MORE_LOCKING_DEBUG
1688	if (so->so_usecount == `1`) {
1689	panic("%s: so=%x no other reference on socket\n", __func__, so);
1690	/ NOTREACHED /
1691	}
1692	#endif
1693	/*
1694	* We return all that is there in the subflow's socket receive buffer
1695	* to the MPTCP layer, so we require that the caller passes in the
1696	* expected parameters.
1697	*/
1698	if (mp == NULL \|\| controlp != NULL)
1699	return (EINVAL);
1700
1701	*mp = NULL;
1702	if (psa != NULL)
1703	*psa = NULL;
1704	if (flagsp != NULL)
1705	flags = *flagsp &~ MSG_EOR;
1706	else
1707	flags = `0`;
1708
1709	if (flags & (MSG_PEEK\|MSG_OOB\|MSG_NEEDSA\|MSG_WAITALL\|MSG_WAITSTREAM))
1710	return (EOPNOTSUPP);
1711
1712	flags \|= (MSG_DONTWAIT\|MSG_NBIO);
1713
1714	/*
1715	* If a recv attempt is made on a previously-accepted socket
1716	* that has been marked as inactive (disconnected), reject
1717	* the request.
1718	*/
1719	if (so->so_flags & SOF_DEFUNCT) {
1720	struct sockbuf *sb = &so->so_rcv;
1721
1722	error = ENOTCONN;
1723	/*
1724	* This socket should have been disconnected and flushed
1725	* prior to being returned from sodefunct(); there should
1726	* be no data on its receive list, so panic otherwise.
1727	*/
1728	if (so->so_state & SS_DEFUNCT)
1729	sb_empty_assert(sb, __func__);
1730	return (error);
1731	}
1732
1733	/*
1734	* See if the socket has been closed (SS_NOFDREF\|SS_CANTRCVMORE)
1735	* and if so just return to the caller. This could happen when
1736	* soreceive() is called by a socket upcall function during the
1737	* time the socket is freed. The socket buffer would have been
1738	* locked across the upcall, therefore we cannot put this thread
1739	* to sleep (else we will deadlock) or return EWOULDBLOCK (else
1740	* we may livelock), because the lock on the socket buffer will
1741	* only be released when the upcall routine returns to its caller.
1742	* Because the socket has been officially closed, there can be
1743	* no further read on it.
1744	*
1745	* A multipath subflow socket would have its SS_NOFDREF set by
1746	* default, so check for SOF_MP_SUBFLOW socket flag; when the
1747	* socket is closed for real, SOF_MP_SUBFLOW would be cleared.
1748	*/
1749	if ((so->so_state & (SS_NOFDREF \| SS_CANTRCVMORE)) ==
1750	(SS_NOFDREF \| SS_CANTRCVMORE) && !(so->so_flags & SOF_MP_SUBFLOW))
1751	return (`0`);
1752
1753	/*
1754	* For consistency with soreceive() semantics, we need to obey
1755	* SB_LOCK in case some other code path has locked the buffer.
1756	*/
1757	error = sblock(&so->so_rcv, `0`);
1758	if (error != `0`)
1759	return (error);
1760
1761	m = so->so_rcv.sb_mb;
1762	if (m == NULL) {
1763	/*
1764	* Panic if we notice inconsistencies in the socket's
1765	* receive list; both sb_mb and sb_cc should correctly
1766	* reflect the contents of the list, otherwise we may
1767	* end up with false positives during select() or poll()
1768	* which could put the application in a bad state.
1769	*/
1770	SB_MB_CHECK(&so->so_rcv);
1771
1772	if (so->so_error != `0`) {
1773	error = so->so_error;
1774	so->so_error = `0`;
1775	goto release;
1776	}
1777
1778	if (so->so_state & SS_CANTRCVMORE) {
1779	goto release;
1780	}
1781
1782	if (!(so->so_state & (SS_ISCONNECTED\|SS_ISCONNECTING))) {
1783	error = ENOTCONN;
1784	goto release;
1785	}
1786
1787	/*
1788	* MSG_DONTWAIT is implicitly defined and this routine will
1789	* never block, so return EWOULDBLOCK when there is nothing.
1790	*/
1791	error = EWOULDBLOCK;
1792	goto release;
1793	}
1794
1795	mptcp_update_last_owner(so, mp_so);
1796
1797	if (mp_so->last_pid != proc_pid(p)) {
1798	p = proc_find(mp_so->last_pid);
1799	if (p == PROC_NULL) {
1800	p = current_proc();
1801	} else {
1802	proc_held = TRUE;
1803	}
1804	}
1805
1806	OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgrcv);
1807	SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 1");
1808	SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1");
1809
1810	while (m != NULL) {
1811	int dlen = `0`, dfin = `0`, error_out = `0`;
1812	struct mbuf *start = m;
1813	uint64_t dsn;
1814	uint32_t sseq;
1815	uint16_t orig_dlen;
1816	uint16_t csum;
1817
1818	VERIFY(m->m_nextpkt == NULL);
1819
1820	if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) {
1821	orig_dlen = dlen = m->m_pkthdr.mp_rlen;
1822	dsn = m->m_pkthdr.mp_dsn;
1823	sseq = m->m_pkthdr.mp_rseq;
1824	csum = m->m_pkthdr.mp_csum;
1825	} else {
1826	/ We did fallback /
1827	mptcp_adj_rmap(so, m, `0`, `0`, `0`, `0`);
1828
1829	sbfree(&so->so_rcv, m);
1830
1831	if (mp != NULL) {
1832	*mp = m;
1833	mp = &m->m_next;
1834	so->so_rcv.sb_mb = m = m->m_next;
1835	*mp = NULL;
1836
1837	}
1838
1839	if (m != NULL) {
1840	so->so_rcv.sb_lastrecord = m;
1841	} else {
1842	SB_EMPTY_FIXUP(&so->so_rcv);
1843	}
1844
1845	continue;
1846	}
1847
1848	if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)
1849	dfin = `1`;
1850
1851	/*
1852	* Check if the full mapping is now present
1853	*/
1854	if ((int)so->so_rcv.sb_cc < dlen - dfin) {
1855	mptcplog((LOG_INFO, "%s not enough data (%u) need %u for dsn %u\n",
1856	__func__, so->so_rcv.sb_cc, dlen, (uint32_t)dsn),
1857	MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);
1858
1859	if (*mp0 == NULL)
1860	error = EWOULDBLOCK;
1861	goto release;
1862	}
1863
1864	/ Now, get the full mapping /
1865	while (dlen > `0`) {
1866	if (mptcp_adj_rmap(so, m, orig_dlen - dlen, dsn, sseq, orig_dlen)) {
1867	error_out = `1`;
1868	error = EIO;
1869	dlen = `0`;
1870	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MUSTRST);
1871	break;
1872	}
1873
1874	dlen -= m->m_len;
1875	sbfree(&so->so_rcv, m);
1876
1877	if (mp != NULL) {
1878	*mp = m;
1879	mp = &m->m_next;
1880	so->so_rcv.sb_mb = m = m->m_next;
1881	*mp = NULL;
1882	}
1883
1884	if (dlen - dfin == `0`)
1885	dlen = `0`;
1886
1887	VERIFY(dlen <= `0` \|\| m);
1888	}
1889
1890	VERIFY(dlen == `0`);
1891
1892	if (m != NULL) {
1893	so->so_rcv.sb_lastrecord = m;
1894	} else {
1895	SB_EMPTY_FIXUP(&so->so_rcv);
1896	}
1897
1898	if (error_out)
1899	goto release;
1900
1901
1902	if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum, dfin)) {
1903	error = EIO;
1904	*mp0 = NULL;
1905	goto release;
1906	}
1907
1908	SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 2");
1909	SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 2");
1910	}
1911
1912	DTRACE_MPTCP3(subflow__receive, struct socket *, so,
1913	struct sockbuf , &so->so_rcv, struct* sockbuf *, &so->so_snd);
1914
1915	if (flagsp != NULL)
1916	*flagsp \|= flags;
1917
1918	release:
1919	sbunlock(&so->so_rcv, TRUE);
1920
1921	if (proc_held)
1922	proc_rele(p);
1923
1924	return (error);
1925
1926	}
1927
1928	/*
1929	* MPTCP subflow socket send routine, derived from sosend().
1930	*/
1931	static int
1932	mptcp_subflow_sosend(struct socket so, struct* sockaddr addr, struct* uio *uio,
1933	struct mbuf top, struct* mbuf control, int* flags)
1934	{
1935	struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte);
1936	struct proc *p = current_proc();
1937	boolean_t en_tracing = FALSE, proc_held = FALSE;
1938	int en_tracing_val;
1939	int sblocked = `1`; / Pretend as if it is already locked, so we won't relock it /
1940	int error;
1941
1942	VERIFY(control == NULL);
1943	VERIFY(addr == NULL);
1944	VERIFY(uio == NULL);
1945	VERIFY(flags == `0`);
1946	VERIFY((so->so_flags & SOF_CONTENT_FILTER) == `0`);
1947
1948	VERIFY(top->m_pkthdr.len > `0` && top->m_pkthdr.len <= UINT16_MAX);
1949	VERIFY(top->m_pkthdr.pkt_flags & PKTF_MPTCP);
1950
1951	/*
1952	* trace if tracing & network (vs. unix) sockets & and
1953	* non-loopback
1954	*/
1955	if (ENTR_SHOULDTRACE &&
1956	(SOCK_CHECK_DOM(so, AF_INET) \|\| SOCK_CHECK_DOM(so, AF_INET6))) {
1957	struct inpcb *inp = sotoinpcb(so);
1958	if (inp->inp_last_outifp != NULL &&
1959	!(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) {
1960	en_tracing = TRUE;
1961	en_tracing_val = top->m_pkthdr.len;
1962	KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_START,
1963	VM_KERNEL_ADDRPERM(so),
1964	((so->so_state & SS_NBIO) ? kEnTrFlagNonBlocking : `0`),
1965	(int64_t)en_tracing_val);
1966	}
1967	}
1968
1969	mptcp_update_last_owner(so, mp_so);
1970
1971	if (mp_so->last_pid != proc_pid(p)) {
1972	p = proc_find(mp_so->last_pid);
1973	if (p == PROC_NULL) {
1974	p = current_proc();
1975	} else {
1976	proc_held = TRUE;
1977	}
1978	}
1979
1980	#if NECP
1981	inp_update_necp_policy(sotoinpcb(so), NULL, NULL, `0`);
1982	#endif /* NECP */
1983
1984	OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgsnd);
1985
1986	error = sosendcheck(so, NULL, top->m_pkthdr.len, `0`, `1`, `0`, &sblocked, NULL);
1987	if (error)
1988	goto out;
1989
1990	error = (*so->so_proto->pr_usrreqs->pru_send)(so, `0`, top, NULL, NULL, p);
1991	top = NULL;
1992
1993	out:
1994	if (top != NULL)
1995	m_freem(top);
1996
1997	if (proc_held)
1998	proc_rele(p);
1999
2000	soclearfastopen(so);
2001
2002	if (en_tracing) {
2003	KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_END,
2004	VM_KERNEL_ADDRPERM(so),
2005	((error == EWOULDBLOCK) ? kEnTrFlagNoWork : `0`),
2006	(int64_t)en_tracing_val);
2007	}
2008
2009	return (error);
2010
2011	}
2012
2013	/*
2014	* Establish an initial MPTCP connection (if first subflow and not yet
2015	* connected), or add a subflow to an existing MPTCP connection.
2016	*/
2017	int
2018	mptcp_subflow_add(struct mptses mpte, struct* sockaddr *src,
2019	struct sockaddr dst, uint32_t ifscope, sae_connid_t pcid)
2020	{
2021	struct socket mp_so, so = NULL;
2022	struct mptcb *mp_tp;
2023	struct mptsub *mpts = NULL;
2024	int af, error = `0`;
2025
2026	mpte_lock_assert_held(mpte); / same as MP socket lock /
2027	mp_so = mptetoso(mpte);
2028	mp_tp = mpte->mpte_mptcb;
2029
2030	if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) {
2031	/ If the remote end sends Data FIN, refuse subflow adds /
2032	mptcplog((LOG_ERR, "%s state %u\n", __func__, mp_tp->mpt_state),
2033	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
2034	error = ENOTCONN;
2035	goto out_err;
2036	}
2037
2038	mpts = mptcp_subflow_alloc();
2039	if (mpts == NULL) {
2040	mptcplog((LOG_ERR, "%s malloc subflow failed\n", __func__),
2041	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
2042	error = ENOMEM;
2043	goto out_err;
2044	}
2045
2046	if (src != NULL) {
2047	int len = src->sa_len;
2048
2049	MALLOC(mpts->mpts_src, struct sockaddr *, len, M_SONAME,
2050	M_WAITOK \| M_ZERO);
2051	if (mpts->mpts_src == NULL) {
2052	mptcplog((LOG_ERR, "%s malloc mpts_src failed", __func__),
2053	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
2054	error = ENOMEM;
2055	goto out_err;
2056	}
2057	bcopy(src, mpts->mpts_src, len);
2058	}
2059
2060	memcpy(&mpts->mpts_dst, dst, dst->sa_len);
2061
2062	af = mpts->mpts_dst.sa_family;
2063
2064	mpts->mpts_ifscope = ifscope;
2065
2066	/ create the subflow socket /
2067	if ((error = mptcp_subflow_socreate(mpte, mpts, af, &so)) != `0`)
2068	/*
2069	* Returning (error) and not cleaning up, because up to here
2070	* all we did is creating mpts.
2071	*
2072	* And the contract is that the call to mptcp_subflow_socreate,
2073	* moves ownership of mpts to mptcp_subflow_socreate.
2074	*/
2075	return (error);
2076
2077	/*
2078	* We may be called from within the kernel. Still need to account this
2079	* one to the real app.
2080	*/
2081	mptcp_update_last_owner(mpts->mpts_socket, mp_so);
2082
2083	/*
2084	* Increment the counter, while avoiding 0 (SAE_CONNID_ANY) and
2085	* -1 (SAE_CONNID_ALL).
2086	*/
2087	mpte->mpte_connid_last++;
2088	if (mpte->mpte_connid_last == SAE_CONNID_ALL \|\|
2089	mpte->mpte_connid_last == SAE_CONNID_ANY)
2090	mpte->mpte_connid_last++;
2091
2092	mpts->mpts_connid = mpte->mpte_connid_last;
2093
2094	mpts->mpts_rel_seq = `1`;
2095
2096	/ Allocate a unique address id per subflow /
2097	mpte->mpte_addrid_last++;
2098	if (mpte->mpte_addrid_last == `0`)
2099	mpte->mpte_addrid_last++;
2100
2101	/ register for subflow socket read/write events /
2102	sock_setupcalls_locked(so, mptcp_subflow_rupcall, mpts, mptcp_subflow_wupcall, mpts, `1`);
2103
2104	/ Register for subflow socket control events /
2105	sock_catchevents_locked(so, mptcp_subflow_eupcall1, mpts,
2106	SO_FILT_HINT_CONNRESET \| SO_FILT_HINT_CANTRCVMORE \|
2107	SO_FILT_HINT_TIMEOUT \| SO_FILT_HINT_NOSRCADDR \|
2108	SO_FILT_HINT_IFDENIED \| SO_FILT_HINT_CONNECTED \|
2109	SO_FILT_HINT_DISCONNECTED \| SO_FILT_HINT_MPFAILOVER \|
2110	SO_FILT_HINT_MPSTATUS \| SO_FILT_HINT_MUSTRST \|
2111	SO_FILT_HINT_MPCANTRCVMORE \| SO_FILT_HINT_ADAPTIVE_RTIMO \|
2112	SO_FILT_HINT_ADAPTIVE_WTIMO);
2113
2114	/ sanity check /
2115	VERIFY(!(mpts->mpts_flags &
2116	(MPTSF_CONNECTING\|MPTSF_CONNECTED\|MPTSF_CONNECT_PENDING)));
2117
2118	/*
2119	* Indicate to the TCP subflow whether or not it should establish
2120	* the initial MPTCP connection, or join an existing one. Fill
2121	* in the connection request structure with additional info needed
2122	* by the underlying TCP (to be used in the TCP options, etc.)
2123	*/
2124	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && mpte->mpte_numflows == `1`) {
2125	mpts->mpts_flags \|= MPTSF_INITIAL_SUB;
2126
2127	if (mp_tp->mpt_state == MPTCPS_CLOSED) {
2128	mptcp_init_local_parms(mpte);
2129	}
2130	soisconnecting(mp_so);
2131
2132	/ If fastopen is requested, set state in mpts /
2133	if (so->so_flags1 & SOF1_PRECONNECT_DATA)
2134	mpts->mpts_flags \|= MPTSF_TFO_REQD;
2135	} else {
2136	if (!(mp_tp->mpt_flags & MPTCPF_JOIN_READY))
2137	mpts->mpts_flags \|= MPTSF_CONNECT_PENDING;
2138	}
2139
2140	mpts->mpts_flags \|= MPTSF_CONNECTING;
2141
2142	if (af == AF_INET \|\| af == AF_INET6) {
2143	char dbuf[MAX_IPv6_STR_LEN];
2144
2145	mptcplog((LOG_DEBUG, "MPTCP Socket: %s "
2146	"mp_so 0x%llx dst %s[%d] cid %d "
2147	"[pending %s]\n", __func__,
2148	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
2149	inet_ntop(af, ((af == AF_INET) ?
2150	(void *)&SIN(&mpts->mpts_dst)->sin_addr.s_addr :
2151	(void *)&SIN6(&mpts->mpts_dst)->sin6_addr),
2152	dbuf, sizeof (dbuf)), ((af == AF_INET) ?
2153	ntohs(SIN(&mpts->mpts_dst)->sin_port) :
2154	ntohs(SIN6(&mpts->mpts_dst)->sin6_port)),
2155	mpts->mpts_connid,
2156	((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ?
2157	"YES" : "NO")),
2158	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2159	}
2160
2161	/ connect right away if first attempt, or if join can be done now /
2162	if (!(mpts->mpts_flags & MPTSF_CONNECT_PENDING))
2163	error = mptcp_subflow_soconnectx(mpte, mpts);
2164
2165	if (error)
2166	goto out_err_close;
2167
2168	if (pcid)
2169	*pcid = mpts->mpts_connid;
2170
2171	return (`0`);
2172
2173	out_err_close:
2174	mptcp_subflow_abort(mpts, error);
2175
2176	return (error);
2177
2178	out_err:
2179	if (mpts)
2180	mptcp_subflow_free(mpts);
2181
2182	return (error);
2183	}
2184
2185	void
2186	mptcpstats_update(struct mptcp_itf_stats stats, struct* mptsub *mpts)
2187	{
2188	int index = mptcp_get_statsindex(stats, mpts);
2189
2190	if (index != -`1`) {
2191	struct inpcb *inp = sotoinpcb(mpts->mpts_socket);
2192
2193	stats[index].mpis_txbytes += inp->inp_stat->txbytes;
2194	stats[index].mpis_rxbytes += inp->inp_stat->rxbytes;
2195	}
2196	}
2197
2198	/*
2199	* Delete/remove a subflow from an MPTCP. The underlying subflow socket
2200	* will no longer be accessible after a subflow is deleted, thus this
2201	* should occur only after the subflow socket has been disconnected.
2202	*/
2203	void
2204	mptcp_subflow_del(struct mptses mpte, struct* mptsub *mpts)
2205	{
2206	struct socket *mp_so = mptetoso(mpte);
2207	struct socket *so = mpts->mpts_socket;
2208	struct tcpcb *tp = sototcpcb(so);
2209
2210	mpte_lock_assert_held(mpte); / same as MP socket lock /
2211	VERIFY(mpts->mpts_mpte == mpte);
2212	VERIFY(mpts->mpts_flags & MPTSF_ATTACHED);
2213	VERIFY(mpte->mpte_numflows != `0`);
2214	VERIFY(mp_so->so_usecount > `0`);
2215
2216	mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx [u=%d,r=%d] cid %d %x error %d\n",
2217	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
2218	mp_so->so_usecount, mp_so->so_retaincnt, mpts->mpts_connid,
2219	mpts->mpts_flags, mp_so->so_error),
2220	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2221
2222	mptcpstats_update(mpte->mpte_itfstats, mpts);
2223	mpte->mpte_init_rxbytes = sotoinpcb(so)->inp_stat->rxbytes;
2224	mpte->mpte_init_txbytes = sotoinpcb(so)->inp_stat->txbytes;
2225
2226	atomic_bitclear_32(&mpts->mpts_flags, MPTSF_ATTACHED);
2227	TAILQ_REMOVE(&mpte->mpte_subflows, mpts, mpts_entry);
2228	mpte->mpte_numflows--;
2229	if (mpte->mpte_active_sub == mpts)
2230	mpte->mpte_active_sub = NULL;
2231
2232	/*
2233	* Drop references held by this subflow socket; there
2234	* will be no further upcalls made from this point.
2235	*/
2236	sock_setupcalls_locked(so, NULL, NULL, NULL, NULL, `0`);
2237	sock_catchevents_locked(so, NULL, NULL, `0`);
2238
2239	mptcp_detach_mptcb_from_subf(mpte->mpte_mptcb, so);
2240
2241	mp_so->so_usecount--; / for subflow socket /
2242	mpts->mpts_mpte = NULL;
2243	mpts->mpts_socket = NULL;
2244
2245	mptcp_subflow_remref(mpts); / for MPTCP subflow list /
2246	mptcp_subflow_remref(mpts); / for subflow socket /
2247
2248	so->so_flags &= ~SOF_MP_SUBFLOW;
2249	tp->t_mptcb = NULL;
2250	tp->t_mpsub = NULL;
2251	}
2252
2253	void
2254	mptcp_subflow_shutdown(struct mptses mpte, struct* mptsub *mpts)
2255	{
2256	struct socket *so = mpts->mpts_socket;
2257	struct mptcb *mp_tp = mpte->mpte_mptcb;
2258	int send_dfin = `0`;
2259
2260	if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT)
2261	send_dfin = `1`;
2262
2263	if (!(so->so_state & (SS_ISDISCONNECTING \| SS_ISDISCONNECTED)) &&
2264	(so->so_state & SS_ISCONNECTED)) {
2265	mptcplog((LOG_DEBUG, "MPTCP subflow shutdown %s: cid %d fin %d\n",
2266	__func__, mpts->mpts_connid, send_dfin),
2267	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2268
2269	if (send_dfin)
2270	mptcp_send_dfin(so);
2271	soshutdownlock(so, SHUT_WR);
2272	}
2273
2274	}
2275
2276	static void
2277	mptcp_subflow_abort(struct mptsub mpts, int* error)
2278	{
2279	struct socket *so = mpts->mpts_socket;
2280	struct tcpcb *tp = sototcpcb(so);
2281
2282	if (mpts->mpts_flags & MPTSF_DISCONNECTED)
2283	return;
2284
2285	mptcplog((LOG_DEBUG, "%s aborting connection state %u\n", __func__, tp->t_state),
2286	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2287
2288	if (tp->t_state != TCPS_CLOSED)
2289	tcp_drop(tp, error);
2290
2291	mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED);
2292	}
2293
2294	/*
2295	* Disconnect a subflow socket.
2296	*/
2297	void
2298	mptcp_subflow_disconnect(struct mptses mpte, struct* mptsub *mpts)
2299	{
2300	struct socket *so;
2301	struct mptcb *mp_tp;
2302	int send_dfin = `0`;
2303
2304	mpte_lock_assert_held(mpte); / same as MP socket lock /
2305
2306	VERIFY(mpts->mpts_mpte == mpte);
2307	VERIFY(mpts->mpts_socket != NULL);
2308
2309	if (mpts->mpts_flags & (MPTSF_DISCONNECTING\|MPTSF_DISCONNECTED))
2310	return;
2311
2312	mpts->mpts_flags \|= MPTSF_DISCONNECTING;
2313
2314	so = mpts->mpts_socket;
2315	mp_tp = mpte->mpte_mptcb;
2316	if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT)
2317	send_dfin = `1`;
2318
2319	if (!(so->so_state & (SS_ISDISCONNECTING \| SS_ISDISCONNECTED)) &&
2320	(so->so_state & SS_ISCONNECTED)) {
2321	mptcplog((LOG_DEBUG, "%s: cid %d fin %d\n",
2322	__func__, mpts->mpts_connid, send_dfin),
2323	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2324
2325	if (send_dfin)
2326	mptcp_send_dfin(so);
2327	(void) soshutdownlock(so, SHUT_RD);
2328	(void) soshutdownlock(so, SHUT_WR);
2329	(void) sodisconnectlocked(so);
2330	}
2331	/*
2332	* Generate a disconnect event for this subflow socket, in case
2333	* the lower layer doesn't do it; this is needed because the
2334	* subflow socket deletion relies on it.
2335	*/
2336	mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED);
2337	}
2338
2339	/*
2340	* Called when the associated subflow socket posted a read event.
2341	*/
2342	static void
2343	mptcp_subflow_rupcall(struct socket so, void* arg, int* waitf)
2344	{
2345	#pragma unused(so, waitf)
2346	struct mptsub mpts = arg, tmpts;
2347	struct mptses *mpte = mpts->mpts_mpte;
2348
2349	VERIFY(mpte != NULL);
2350
2351	if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) {
2352	if (!(mpte->mpte_mppcb->mpp_flags & MPP_RUPCALL))
2353	mpte->mpte_mppcb->mpp_flags \|= MPP_SHOULD_RWAKEUP;
2354	return;
2355	}
2356
2357	mpte->mpte_mppcb->mpp_flags \|= MPP_RUPCALL;
2358	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
2359	if (mpts->mpts_socket->so_usecount == `0`) {
2360	/ Will be removed soon by tcp_garbage_collect /
2361	continue;
2362	}
2363
2364	mptcp_subflow_addref(mpts);
2365	mpts->mpts_socket->so_usecount++;
2366
2367	mptcp_subflow_input(mpte, mpts);
2368
2369	mptcp_subflow_remref(mpts); / ours /
2370
2371	VERIFY(mpts->mpts_socket->so_usecount != `0`);
2372	mpts->mpts_socket->so_usecount--;
2373	}
2374
2375	mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_RUPCALL);
2376	}
2377
2378	/*
2379	* Subflow socket input.
2380	*/
2381	static void
2382	mptcp_subflow_input(struct mptses mpte, struct* mptsub *mpts)
2383	{
2384	struct socket *mp_so = mptetoso(mpte);
2385	struct mbuf *m = NULL;
2386	struct socket *so;
2387	int error, wakeup = `0`;
2388
2389	VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_INPUT));
2390	mpte->mpte_mppcb->mpp_flags \|= MPP_INSIDE_INPUT;
2391
2392	DTRACE_MPTCP2(subflow__input, struct mptses *, mpte,
2393	struct mptsub *, mpts);
2394
2395	if (!(mpts->mpts_flags & MPTSF_CONNECTED))
2396	goto out;
2397
2398	so = mpts->mpts_socket;
2399
2400	error = sock_receive_internal(so, NULL, &m, `0`, NULL);
2401	if (error != `0` && error != EWOULDBLOCK) {
2402	mptcplog((LOG_ERR, "%s: cid %d error %d\n",
2403	__func__, mpts->mpts_connid, error),
2404	MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR);
2405	if (error == ENODATA) {
2406	/*
2407	* Don't ignore ENODATA so as to discover
2408	* nasty middleboxes.
2409	*/
2410	mp_so->so_error = ENODATA;
2411
2412	wakeup = `1`;
2413	goto out;
2414	}
2415	} else if (error == `0`) {
2416	mptcplog((LOG_DEBUG, "%s: cid %d \n", __func__, mpts->mpts_connid),
2417	MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
2418	}
2419
2420	/ In fallback, make sure to accept data on all but one subflow /
2421	if (m && (mpts->mpts_flags & MPTSF_MP_DEGRADED) &&
2422	!(mpts->mpts_flags & MPTSF_ACTIVE)) {
2423	mptcplog((LOG_DEBUG, "%s: degraded and got data on non-active flow\n",
2424	__func__), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
2425	m_freem(m);
2426	goto out;
2427	}
2428
2429	if (m != NULL) {
2430	if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) {
2431	mpte->mpte_mppcb->mpp_flags \|= MPP_SET_CELLICON;
2432
2433	mpte->mpte_used_cell = `1`;
2434	} else {
2435	mpte->mpte_mppcb->mpp_flags \|= MPP_UNSET_CELLICON;
2436
2437	mpte->mpte_used_wifi = `1`;
2438	}
2439
2440	mptcp_input(mpte, m);
2441	}
2442
2443	/ notify protocol that we drained all the data /
2444	if (error == `0` && m != NULL &&
2445	(so->so_proto->pr_flags & PR_WANTRCVD) && so->so_pcb != NULL)
2446	(*so->so_proto->pr_usrreqs->pru_rcvd)(so, `0`);
2447
2448	out:
2449	if (wakeup)
2450	mpte->mpte_mppcb->mpp_flags \|= MPP_SHOULD_RWAKEUP;
2451
2452	mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_INPUT);
2453	}
2454
2455	/*
2456	* Subflow socket write upcall.
2457	*
2458	* Called when the associated subflow socket posted a read event.
2459	*/
2460	static void
2461	mptcp_subflow_wupcall(struct socket so, void* arg, int* waitf)
2462	{
2463	#pragma unused(so, waitf)
2464	struct mptsub *mpts = arg;
2465	struct mptses *mpte = mpts->mpts_mpte;
2466
2467	VERIFY(mpte != NULL);
2468
2469	if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) {
2470	if (!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL))
2471	mpte->mpte_mppcb->mpp_flags \|= MPP_SHOULD_WWAKEUP;
2472	return;
2473	}
2474
2475	mptcp_output(mpte);
2476	}
2477
2478	static boolean_t
2479	mptcp_search_seq_in_sub(struct mbuf m, struct* socket *so)
2480	{
2481	struct mbuf *so_m = so->so_snd.sb_mb;
2482	uint64_t dsn = m->m_pkthdr.mp_dsn;
2483
2484	while (so_m) {
2485	VERIFY(so_m->m_flags & M_PKTHDR);
2486	VERIFY(so_m->m_pkthdr.pkt_flags & PKTF_MPTCP);
2487
2488	/ Part of the segment is covered, don't reinject here /
2489	if (so_m->m_pkthdr.mp_dsn <= dsn &&
2490	so_m->m_pkthdr.mp_dsn + so_m->m_pkthdr.mp_rlen > dsn)
2491	return TRUE;
2492
2493	so_m = so_m->m_next;
2494	}
2495
2496	return FALSE;
2497	}
2498
2499	/*
2500	* Subflow socket output.
2501	*
2502	* Called for sending data from MPTCP to the underlying subflow socket.
2503	*/
2504	int
2505	mptcp_subflow_output(struct mptses mpte, struct* mptsub mpts, int* flags)
2506	{
2507	struct mptcb *mp_tp = mpte->mpte_mptcb;
2508	struct mbuf sb_mb, m, mpt_mbuf = NULL, head, *tail;
2509	struct socket mp_so, so;
2510	struct tcpcb *tp;
2511	uint64_t mpt_dsn = `0`, off = `0`;
2512	int sb_cc = `0`, error = `0`, wakeup = `0`;
2513	uint32_t dss_csum;
2514	uint16_t tot_sent = `0`;
2515	boolean_t reinjected = FALSE;
2516
2517	mpte_lock_assert_held(mpte);
2518
2519	mp_so = mptetoso(mpte);
2520	so = mpts->mpts_socket;
2521	tp = sototcpcb(so);
2522
2523	VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_OUTPUT));
2524	mpte->mpte_mppcb->mpp_flags \|= MPP_INSIDE_OUTPUT;
2525
2526	VERIFY(!INP_WAIT_FOR_IF_FEEDBACK(sotoinpcb(so)));
2527	VERIFY((mpts->mpts_flags & MPTSF_MP_CAPABLE) \|\|
2528	(mpts->mpts_flags & MPTSF_MP_DEGRADED) \|\|
2529	(mpts->mpts_flags & MPTSF_TFO_REQD));
2530	VERIFY(mptcp_subflow_cwnd_space(mpts->mpts_socket) > `0`);
2531
2532	mptcplog((LOG_DEBUG, "%s mpts_flags %#x, mpte_flags %#x cwnd_space %u\n",
2533	__func__, mpts->mpts_flags, mpte->mpte_flags,
2534	mptcp_subflow_cwnd_space(so)),
2535	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
2536	DTRACE_MPTCP2(subflow__output, struct mptses *, mpte,
2537	struct mptsub *, mpts);
2538
2539	/ Remove Addr Option is not sent reliably as per I-D /
2540	if (mpte->mpte_flags & MPTE_SND_REM_ADDR) {
2541	tp->t_rem_aid = mpte->mpte_lost_aid;
2542	tp->t_mpflags \|= TMPF_SND_REM_ADDR;
2543	mpte->mpte_flags &= ~MPTE_SND_REM_ADDR;
2544	}
2545
2546	/*
2547	* The mbuf chains containing the metadata (as well as pointing to
2548	* the user data sitting at the MPTCP output queue) would then be
2549	* sent down to the subflow socket.
2550	*
2551	* Some notes on data sequencing:
2552	*
2553	* a. Each mbuf must be a M_PKTHDR.
2554	* b. MPTCP metadata is stored in the mptcp_pktinfo structure
2555	* in the mbuf pkthdr structure.
2556	* c. Each mbuf containing the MPTCP metadata must have its
2557	* pkt_flags marked with the PKTF_MPTCP flag.
2558	*/
2559
2560	if (mpte->mpte_reinjectq)
2561	sb_mb = mpte->mpte_reinjectq;
2562	else
2563	sb_mb = mp_so->so_snd.sb_mb;
2564
2565	if (sb_mb == NULL) {
2566	mptcplog((LOG_ERR, "%s: No data in MPTCP-sendbuffer! smax %u snxt %u suna %u state %u flags %#x\n",
2567	__func__, (uint32_t)mp_tp->mpt_sndmax, (uint32_t)mp_tp->mpt_sndnxt,
2568	(uint32_t)mp_tp->mpt_snduna, mp_tp->mpt_state, mp_so->so_flags1),
2569	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2570
2571	/ Fix it to prevent looping /
2572	if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna))
2573	mp_tp->mpt_sndnxt = mp_tp->mpt_snduna;
2574	goto out;
2575	}
2576
2577	VERIFY(sb_mb->m_pkthdr.pkt_flags & PKTF_MPTCP);
2578
2579	if (sb_mb->m_pkthdr.mp_rlen == `0` &&
2580	!(so->so_state & SS_ISCONNECTED) &&
2581	(so->so_flags1 & SOF1_PRECONNECT_DATA)) {
2582	tp->t_mpflags \|= TMPF_TFO_REQUEST;
2583	goto zero_len_write;
2584	}
2585
2586	mpt_dsn = sb_mb->m_pkthdr.mp_dsn;
2587
2588	/ First, drop acknowledged data /
2589	if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) {
2590	mptcplog((LOG_ERR, "%s: dropping data, should have been done earlier "
2591	"dsn %u suna %u reinject? %u\n",
2592	__func__, (uint32_t)mpt_dsn,
2593	(uint32_t)mp_tp->mpt_snduna, !!mpte->mpte_reinjectq),
2594	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2595	if (mpte->mpte_reinjectq) {
2596	mptcp_clean_reinjectq(mpte);
2597	} else {
2598	uint64_t len = `0`;
2599	len = mp_tp->mpt_snduna - mpt_dsn;
2600	sbdrop(&mp_so->so_snd, (int)len);
2601	wakeup = `1`;
2602	}
2603	}
2604
2605	/ Check again because of above sbdrop /
2606	if (mp_so->so_snd.sb_mb == NULL && mpte->mpte_reinjectq == NULL) {
2607	mptcplog((LOG_ERR, "%s send-buffer is empty\n", __func__),
2608	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2609	goto out;
2610	}
2611
2612	/*
2613	* In degraded mode, we don't receive data acks, so force free
2614	* mbufs less than snd_nxt
2615	*/
2616	if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) &&
2617	(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) &&
2618	mp_so->so_snd.sb_mb) {
2619	mpt_dsn = mp_so->so_snd.sb_mb->m_pkthdr.mp_dsn;
2620	if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) {
2621	uint64_t len = `0`;
2622	len = mp_tp->mpt_snduna - mpt_dsn;
2623	sbdrop(&mp_so->so_snd, (int)len);
2624	wakeup = `1`;
2625
2626	mptcplog((LOG_ERR, "%s: dropping data in degraded mode, should have been done earlier dsn %u sndnxt %u suna %u\n",
2627	__func__, (uint32_t)mpt_dsn, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_snduna),
2628	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2629	}
2630	}
2631
2632	if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) &&
2633	!(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC)) {
2634	mp_tp->mpt_flags \|= MPTCPF_POST_FALLBACK_SYNC;
2635	so->so_flags1 \|= SOF1_POST_FALLBACK_SYNC;
2636	}
2637
2638	/*
2639	* Adjust the top level notion of next byte used for retransmissions
2640	* and sending FINs.
2641	*/
2642	if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna))
2643	mp_tp->mpt_sndnxt = mp_tp->mpt_snduna;
2644
2645	/ Now determine the offset from which to start transmitting data /
2646	if (mpte->mpte_reinjectq)
2647	sb_mb = mpte->mpte_reinjectq;
2648	else
2649	dont_reinject:
2650	sb_mb = mp_so->so_snd.sb_mb;
2651	if (sb_mb == NULL) {
2652	mptcplog((LOG_ERR, "%s send-buffer is still empty\n", __func__),
2653	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2654	goto out;
2655	}
2656
2657	if (sb_mb == mpte->mpte_reinjectq) {
2658	sb_cc = sb_mb->m_pkthdr.mp_rlen;
2659	off = `0`;
2660
2661	if (mptcp_search_seq_in_sub(sb_mb, so)) {
2662	if (mptcp_can_send_more(mp_tp, TRUE)) {
2663	goto dont_reinject;
2664	}
2665
2666	error = ECANCELED;
2667	goto out;
2668	}
2669
2670	reinjected = TRUE;
2671	} else if (flags & MPTCP_SUBOUT_PROBING) {
2672	sb_cc = sb_mb->m_pkthdr.mp_rlen;
2673	off = `0`;
2674	} else {
2675	sb_cc = min(mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd);
2676
2677	/*
2678	* With TFO, there might be no data at all, thus still go into this
2679	* code-path here.
2680	*/
2681	if ((mp_so->so_flags1 & SOF1_PRECONNECT_DATA) \|\|
2682	MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) {
2683	off = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna;
2684	sb_cc -= off;
2685	} else {
2686	mptcplog((LOG_ERR, "%s this should not happen: sndnxt %u sndmax %u\n",
2687	__func__, (uint32_t)mp_tp->mpt_sndnxt,
2688	(uint32_t)mp_tp->mpt_sndmax),
2689	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2690
2691	goto out;
2692	}
2693	}
2694
2695	sb_cc = min(sb_cc, mptcp_subflow_cwnd_space(so));
2696	if (sb_cc <= `0`) {
2697	mptcplog((LOG_ERR, "%s sb_cc is %d, mp_so->sb_cc %u, sndwnd %u,sndnxt %u sndmax %u cwnd %u\n",
2698	__func__, sb_cc, mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd,
2699	(uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax,
2700	mptcp_subflow_cwnd_space(so)),
2701	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2702	}
2703
2704	sb_cc = min(sb_cc, UINT16_MAX);
2705
2706	/*
2707	* Create a DSN mapping for the data we are about to send. It all
2708	* has the same mapping.
2709	*/
2710	if (reinjected)
2711	mpt_dsn = sb_mb->m_pkthdr.mp_dsn;
2712	else
2713	mpt_dsn = mp_tp->mpt_snduna + off;
2714
2715	mpt_mbuf = sb_mb;
2716	while (mpt_mbuf && reinjected == FALSE &&
2717	(mpt_mbuf->m_pkthdr.mp_rlen == `0` \|\|
2718	mpt_mbuf->m_pkthdr.mp_rlen <= (uint32_t)off)) {
2719	off -= mpt_mbuf->m_pkthdr.mp_rlen;
2720	mpt_mbuf = mpt_mbuf->m_next;
2721	}
2722	if (mpts->mpts_flags & MPTSF_MP_DEGRADED)
2723	mptcplog((LOG_DEBUG, "%s: %u snduna = %u sndnxt = %u probe %d\n",
2724	__func__, mpts->mpts_connid, (uint32_t)mp_tp->mpt_snduna, (uint32_t)mp_tp->mpt_sndnxt,
2725	mpts->mpts_probecnt),
2726	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
2727
2728	VERIFY((mpt_mbuf == NULL) \|\| (mpt_mbuf->m_pkthdr.pkt_flags & PKTF_MPTCP));
2729
2730	head = tail = NULL;
2731
2732	while (tot_sent < sb_cc) {
2733	ssize_t mlen;
2734
2735	mlen = mpt_mbuf->m_len;
2736	mlen -= off;
2737	mlen = min(mlen, sb_cc - tot_sent);
2738
2739	if (mlen < `0`) {
2740	mptcplog((LOG_ERR, "%s mlen %d mp_rlen %u off %u sb_cc %u tot_sent %u\n",
2741	__func__, (int)mlen, mpt_mbuf->m_pkthdr.mp_rlen,
2742	(uint32_t)off, sb_cc, tot_sent),
2743	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2744	goto out;
2745	}
2746
2747	if (mlen == `0`)
2748	goto next;
2749
2750	m = m_copym_mode(mpt_mbuf, (int)off, mlen, M_DONTWAIT,
2751	M_COPYM_MUST_COPY_HDR);
2752	if (m == NULL) {
2753	mptcplog((LOG_ERR, "%s m_copym_mode failed\n", __func__),
2754	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2755	error = ENOBUFS;
2756	break;
2757	}
2758
2759	/ Create a DSN mapping for the data (m_copym does it) /
2760	VERIFY(m->m_flags & M_PKTHDR);
2761	VERIFY(m->m_next == NULL);
2762
2763	m->m_pkthdr.pkt_flags \|= PKTF_MPTCP;
2764	m->m_pkthdr.pkt_flags &= ~PKTF_MPSO;
2765	m->m_pkthdr.mp_dsn = mpt_dsn;
2766	m->m_pkthdr.mp_rseq = mpts->mpts_rel_seq;
2767	m->m_pkthdr.len = mlen;
2768
2769	if (head == NULL) {
2770	head = tail = m;
2771	} else {
2772	tail->m_next = m;
2773	tail = m;
2774	}
2775
2776	tot_sent += mlen;
2777	off = `0`;
2778	next:
2779	mpt_mbuf = mpt_mbuf->m_next;
2780	}
2781
2782	if (reinjected) {
2783	if (sb_cc < sb_mb->m_pkthdr.mp_rlen) {
2784	struct mbuf *n = sb_mb;
2785
2786	while (n) {
2787	n->m_pkthdr.mp_dsn += sb_cc;
2788	n->m_pkthdr.mp_rlen -= sb_cc;
2789	n = n->m_next;
2790	}
2791	m_adj(sb_mb, sb_cc);
2792	} else {
2793	mpte->mpte_reinjectq = sb_mb->m_nextpkt;
2794	m_freem(sb_mb);
2795	}
2796	}
2797
2798	mptcplog((LOG_DEBUG, "%s: Queued dsn %u ssn %u len %u on sub %u\n",
2799	__func__, (uint32_t)mpt_dsn, mpts->mpts_rel_seq,
2800	tot_sent, mpts->mpts_connid), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
2801
2802	if (head && (mp_tp->mpt_flags & MPTCPF_CHECKSUM)) {
2803	dss_csum = mptcp_output_csum(head, mpt_dsn, mpts->mpts_rel_seq,
2804	tot_sent);
2805	}
2806
2807	/ Now, let's update rel-seq and the data-level length /
2808	mpts->mpts_rel_seq += tot_sent;
2809	m = head;
2810	while (m) {
2811	if (mp_tp->mpt_flags & MPTCPF_CHECKSUM)
2812	m->m_pkthdr.mp_csum = dss_csum;
2813	m->m_pkthdr.mp_rlen = tot_sent;
2814	m = m->m_next;
2815	}
2816
2817	if (head != NULL) {
2818	if ((mpts->mpts_flags & MPTSF_TFO_REQD) &&
2819	(tp->t_tfo_stats == `0`))
2820	tp->t_mpflags \|= TMPF_TFO_REQUEST;
2821
2822	error = sock_sendmbuf(so, NULL, head, `0`, NULL);
2823
2824	DTRACE_MPTCP7(send, struct mbuf , m, struct* socket *, so,
2825	struct sockbuf *, &so->so_rcv,
2826	struct sockbuf *, &so->so_snd,
2827	struct mptses , mpte, struct* mptsub *, mpts,
2828	size_t, tot_sent);
2829	}
2830
2831	done_sending:
2832	if (error == `0` \|\|
2833	(error == EWOULDBLOCK && (tp->t_mpflags & TMPF_TFO_REQUEST))) {
2834	uint64_t new_sndnxt = mp_tp->mpt_sndnxt + tot_sent;
2835
2836	if (mpts->mpts_probesoon && mpts->mpts_maxseg && tot_sent) {
2837	tcpstat.tcps_mp_num_probes++;
2838	if ((uint32_t)tot_sent < mpts->mpts_maxseg)
2839	mpts->mpts_probecnt += `1`;
2840	else
2841	mpts->mpts_probecnt +=
2842	tot_sent/mpts->mpts_maxseg;
2843	}
2844
2845	if (!reinjected && !(flags & MPTCP_SUBOUT_PROBING)) {
2846	if (MPTCP_DATASEQ_HIGH32(new_sndnxt) >
2847	MPTCP_DATASEQ_HIGH32(mp_tp->mpt_sndnxt))
2848	mp_tp->mpt_flags \|= MPTCPF_SND_64BITDSN;
2849	mp_tp->mpt_sndnxt = new_sndnxt;
2850	}
2851
2852	mptcp_cancel_timer(mp_tp, MPTT_REXMT);
2853
2854	/ Must be here as mptcp_can_send_more() checks for this /
2855	soclearfastopen(mp_so);
2856
2857	if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) \|\|
2858	(mpts->mpts_probesoon != `0`))
2859	mptcplog((LOG_DEBUG, "%s %u degraded %u wrote %d %d probe %d probedelta %d\n",
2860	__func__, mpts->mpts_connid,
2861	!!(mpts->mpts_flags & MPTSF_MP_DEGRADED),
2862	tot_sent, (int) sb_cc, mpts->mpts_probecnt,
2863	(tcp_now - mpts->mpts_probesoon)),
2864	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
2865
2866	if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) {
2867	mpte->mpte_mppcb->mpp_flags \|= MPP_SET_CELLICON;
2868
2869	mpte->mpte_used_cell = `1`;
2870	} else {
2871	mpte->mpte_mppcb->mpp_flags \|= MPP_UNSET_CELLICON;
2872
2873	mpte->mpte_used_wifi = `1`;
2874	}
2875
2876	/*
2877	* Don't propagate EWOULDBLOCK - it's already taken care of
2878	* in mptcp_usr_send for TFO.
2879	*/
2880	error = `0`;
2881	} else {
2882	mptcplog((LOG_ERR, "%s: %u error %d len %d subflags %#x sostate %#x soerror %u hiwat %u lowat %u\n",
2883	__func__, mpts->mpts_connid, error, tot_sent, so->so_flags, so->so_state, so->so_error, so->so_snd.sb_hiwat, so->so_snd.sb_lowat),
2884	MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR);
2885	}
2886	out:
2887
2888	if (wakeup)
2889	mpte->mpte_mppcb->mpp_flags \|= MPP_SHOULD_WWAKEUP;
2890
2891	mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_OUTPUT);
2892	return (error);
2893
2894	zero_len_write:
2895	/ Opting to call pru_send as no mbuf at subflow level /
2896	error = (*so->so_proto->pr_usrreqs->pru_send)(so, `0`, NULL, NULL,
2897	NULL, current_proc());
2898
2899	goto done_sending;
2900	}
2901
2902	static void
2903	mptcp_add_reinjectq(struct mptses mpte, struct* mbuf *m)
2904	{
2905	struct mbuf n, prev = NULL;
2906
2907	mptcplog((LOG_DEBUG, "%s reinjecting dsn %u dlen %u rseq %u\n",
2908	__func__, (uint32_t)m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen,
2909	m->m_pkthdr.mp_rseq),
2910	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2911
2912	n = mpte->mpte_reinjectq;
2913
2914	/ First, look for an mbuf n, whose data-sequence-number is bigger or*
2915	* equal than m's sequence number.
2916	*/
2917	while (n) {
2918	if (MPTCP_SEQ_GEQ(n->m_pkthdr.mp_dsn, m->m_pkthdr.mp_dsn))
2919	break;
2920
2921	prev = n;
2922
2923	n = n->m_nextpkt;
2924	}
2925
2926	if (n) {
2927	/ m is already fully covered by the next mbuf in the queue /
2928	if (n->m_pkthdr.mp_dsn == m->m_pkthdr.mp_dsn &&
2929	n->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_rlen) {
2930	mptcplog((LOG_DEBUG, "%s fully covered with len %u\n",
2931	__func__, n->m_pkthdr.mp_rlen),
2932	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2933	goto dont_queue;
2934	}
2935
2936	/ m is covering the next mbuf entirely, thus we remove this guy /
2937	if (m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen >= n->m_pkthdr.mp_dsn + n->m_pkthdr.mp_rlen) {
2938	struct mbuf *tmp = n->m_nextpkt;
2939
2940	mptcplog((LOG_DEBUG, "%s m is covering that guy dsn %u len %u dsn %u len %u\n",
2941	__func__, m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen,
2942	n->m_pkthdr.mp_dsn, n->m_pkthdr.mp_rlen),
2943	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2944
2945	m->m_nextpkt = NULL;
2946	if (prev == NULL)
2947	mpte->mpte_reinjectq = tmp;
2948	else
2949	prev->m_nextpkt = tmp;
2950
2951	m_freem(n);
2952	n = tmp;
2953	}
2954
2955	}
2956
2957	if (prev) {
2958	/ m is already fully covered by the previous mbuf in the queue /
2959	if (prev->m_pkthdr.mp_dsn + prev->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_dsn + m->m_pkthdr.len) {
2960	mptcplog((LOG_DEBUG, "%s prev covers us from %u with len %u\n",
2961	__func__, prev->m_pkthdr.mp_dsn, prev->m_pkthdr.mp_rlen),
2962	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
2963	goto dont_queue;
2964	}
2965	}
2966
2967	if (prev == NULL)
2968	mpte->mpte_reinjectq = m;
2969	else
2970	prev->m_nextpkt = m;
2971
2972	m->m_nextpkt = n;
2973
2974	return;
2975
2976	dont_queue:
2977	m_freem(m);
2978	return;
2979	}
2980
2981	static struct mbuf *
2982	mptcp_lookup_dsn(struct mptses *mpte, uint64_t dsn)
2983	{
2984	struct socket *mp_so = mptetoso(mpte);
2985	struct mbuf *m;
2986
2987	m = mp_so->so_snd.sb_mb;
2988
2989	while (m) {
2990	/ If this segment covers what we are looking for, return it. /
2991	if (MPTCP_SEQ_LEQ(m->m_pkthdr.mp_dsn, dsn) &&
2992	MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, dsn))
2993	break;
2994
2995
2996	/ Segment is no more in the queue /
2997	if (MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn, dsn))
2998	return NULL;
2999
3000	m = m->m_next;
3001	}
3002
3003	return m;
3004	}
3005
3006	static struct mbuf *
3007	mptcp_copy_mbuf_list(struct mbuf m, int* len)
3008	{
3009	struct mbuf top = NULL, tail = NULL;
3010	uint64_t dsn;
3011	uint32_t dlen, rseq;
3012
3013	dsn = m->m_pkthdr.mp_dsn;
3014	dlen = m->m_pkthdr.mp_rlen;
3015	rseq = m->m_pkthdr.mp_rseq;
3016
3017	while (len > `0`) {
3018	struct mbuf *n;
3019
3020	VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP));
3021
3022	n = m_copym_mode(m, `0`, m->m_len, M_DONTWAIT, M_COPYM_MUST_COPY_HDR);
3023	if (n == NULL) {
3024	mptcplog((LOG_ERR, "%s m_copym_mode returned NULL\n", __func__),
3025	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
3026	goto err;
3027	}
3028
3029	VERIFY(n->m_flags & M_PKTHDR);
3030	VERIFY(n->m_next == NULL);
3031	VERIFY(n->m_pkthdr.mp_dsn == dsn);
3032	VERIFY(n->m_pkthdr.mp_rlen == dlen);
3033	VERIFY(n->m_pkthdr.mp_rseq == rseq);
3034	VERIFY(n->m_len == m->m_len);
3035
3036	n->m_pkthdr.pkt_flags \|= (PKTF_MPSO \| PKTF_MPTCP);
3037
3038	if (top == NULL)
3039	top = n;
3040
3041	if (tail != NULL)
3042	tail->m_next = n;
3043
3044	tail = n;
3045
3046	len -= m->m_len;
3047	m = m->m_next;
3048	}
3049
3050	return top;
3051
3052	err:
3053	if (top)
3054	m_freem(top);
3055
3056	return NULL;
3057	}
3058
3059	static void
3060	mptcp_reinject_mbufs(struct socket *so)
3061	{
3062	struct tcpcb *tp = sototcpcb(so);
3063	struct mptsub *mpts = tp->t_mpsub;
3064	struct mptcb *mp_tp = tptomptp(tp);
3065	struct mptses *mpte = mp_tp->mpt_mpte;;
3066	struct sockbuf *sb = &so->so_snd;
3067	struct mbuf *m;
3068
3069	m = sb->sb_mb;
3070	while (m) {
3071	struct mbuf n = m->m_next, orig = m;
3072
3073	mptcplog((LOG_DEBUG, "%s working on suna %u relseq %u iss %u len %u pktflags %#x\n",
3074	__func__, tp->snd_una, m->m_pkthdr.mp_rseq, mpts->mpts_iss,
3075	m->m_pkthdr.mp_rlen, m->m_pkthdr.pkt_flags),
3076	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
3077
3078	VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP));
3079
3080	if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_REINJ)
3081	goto next;
3082
3083	/ Has it all already been acknowledged at the data-level? /
3084	if (MPTCP_SEQ_GEQ(mp_tp->mpt_snduna, m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen))
3085	goto next;
3086
3087	/ Part of this has already been acknowledged - lookup in the*
3088	* MPTCP-socket for the segment.
3089	*/
3090	if (SEQ_GT(tp->snd_una - mpts->mpts_iss, m->m_pkthdr.mp_rseq)) {
3091	m = mptcp_lookup_dsn(mpte, m->m_pkthdr.mp_dsn);
3092	if (m == NULL)
3093	goto next;
3094	}
3095
3096	/ Copy the mbuf with headers (aka, DSN-numbers) /
3097	m = mptcp_copy_mbuf_list(m, m->m_pkthdr.mp_rlen);
3098	if (m == NULL)
3099	break;
3100
3101	VERIFY(m->m_nextpkt == NULL);
3102
3103	/ Now, add to the reinject-queue, eliminating overlapping*
3104	* segments
3105	*/
3106	mptcp_add_reinjectq(mpte, m);
3107
3108	orig->m_pkthdr.pkt_flags \|= PKTF_MPTCP_REINJ;
3109
3110	next:
3111	/ mp_rlen can cover multiple mbufs, so advance to the end of it. /
3112	while (n) {
3113	VERIFY((n->m_flags & M_PKTHDR) && (n->m_pkthdr.pkt_flags & PKTF_MPTCP));
3114
3115	if (n->m_pkthdr.mp_dsn != orig->m_pkthdr.mp_dsn)
3116	break;
3117
3118	n->m_pkthdr.pkt_flags \|= PKTF_MPTCP_REINJ;
3119	n = n->m_next;
3120	}
3121
3122	m = n;
3123	}
3124	}
3125
3126	void
3127	mptcp_clean_reinjectq(struct mptses *mpte)
3128	{
3129	struct mptcb *mp_tp = mpte->mpte_mptcb;
3130
3131	mpte_lock_assert_held(mpte);
3132
3133	while (mpte->mpte_reinjectq) {
3134	struct mbuf *m = mpte->mpte_reinjectq;
3135
3136	if (MPTCP_SEQ_GEQ(m->m_pkthdr.mp_dsn, mp_tp->mpt_snduna) \|\|
3137	MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, mp_tp->mpt_snduna))
3138	break;
3139
3140	mpte->mpte_reinjectq = m->m_nextpkt;
3141	m->m_nextpkt = NULL;
3142	m_freem(m);
3143	}
3144	}
3145
3146	/*
3147	* Subflow socket control event upcall.
3148	*/
3149	static void
3150	mptcp_subflow_eupcall1(struct socket so, void* *arg, uint32_t events)
3151	{
3152	#pragma unused(so)
3153	struct mptsub *mpts = arg;
3154	struct mptses *mpte = mpts->mpts_mpte;
3155
3156	VERIFY(mpte != NULL);
3157	mpte_lock_assert_held(mpte);
3158
3159	if ((mpts->mpts_evctl & events) == events)
3160	return;
3161
3162	mpts->mpts_evctl \|= events;
3163
3164	if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) {
3165	mpte->mpte_mppcb->mpp_flags \|= MPP_SHOULD_WORKLOOP;
3166	return;
3167	}
3168
3169	mptcp_subflow_workloop(mpte);
3170	}
3171
3172	/*
3173	* Subflow socket control events.
3174	*
3175	* Called for handling events related to the underlying subflow socket.
3176	*/
3177	static ev_ret_t
3178	mptcp_subflow_events(struct mptses mpte, struct* mptsub *mpts,
3179	uint64_t *p_mpsofilt_hint)
3180	{
3181	ev_ret_t ret = MPTS_EVRET_OK;
3182	int i, mpsub_ev_entry_count = sizeof(mpsub_ev_entry_tbl) /
3183	sizeof(mpsub_ev_entry_tbl[`0`]);
3184
3185	mpte_lock_assert_held(mpte); / same as MP socket lock /
3186
3187	/ bail if there's nothing to process /
3188	if (!mpts->mpts_evctl)
3189	return (ret);
3190
3191	if (mpts->mpts_evctl & (SO_FILT_HINT_CONNRESET\|SO_FILT_HINT_MUSTRST\|
3192	SO_FILT_HINT_CANTSENDMORE\|SO_FILT_HINT_TIMEOUT\|
3193	SO_FILT_HINT_NOSRCADDR\|SO_FILT_HINT_IFDENIED\|
3194	SO_FILT_HINT_DISCONNECTED)) {
3195	mpts->mpts_evctl \|= SO_FILT_HINT_MPFAILOVER;
3196	}
3197
3198	DTRACE_MPTCP3(subflow__events, struct mptses *, mpte,
3199	struct mptsub *, mpts, uint32_t, mpts->mpts_evctl);
3200
3201	mptcplog((LOG_DEBUG, "%s cid %d events=%b\n", __func__,
3202	mpts->mpts_connid, mpts->mpts_evctl, SO_FILT_HINT_BITS),
3203	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE);
3204
3205	/*
3206	* Process all the socket filter hints and reset the hint
3207	* once it is handled
3208	*/
3209	for (i = `0`; i < mpsub_ev_entry_count && mpts->mpts_evctl; i++) {
3210	/*
3211	* Always execute the DISCONNECTED event, because it will wakeup
3212	* the app.
3213	*/
3214	if ((mpts->mpts_evctl & mpsub_ev_entry_tbl[i].sofilt_hint_mask) &&
3215	(ret >= MPTS_EVRET_OK \|\|
3216	mpsub_ev_entry_tbl[i].sofilt_hint_mask == SO_FILT_HINT_DISCONNECTED)) {
3217	mpts->mpts_evctl &= ~mpsub_ev_entry_tbl[i].sofilt_hint_mask;
3218	ev_ret_t error =
3219	mpsub_ev_entry_tbl[i].sofilt_hint_ev_hdlr(mpte, mpts, p_mpsofilt_hint, mpsub_ev_entry_tbl[i].sofilt_hint_mask);
3220	ret = ((error >= MPTS_EVRET_OK) ? MAX(error, ret) : error);
3221	}
3222	}
3223
3224	/*
3225	* We should be getting only events specified via sock_catchevents(),
3226	* so loudly complain if we have any unprocessed one(s).
3227	*/
3228	if (mpts->mpts_evctl \|\| ret < MPTS_EVRET_OK)
3229	mptcplog((LOG_WARNING, "%s%s: cid %d evret %s (%d) unhandled events=%b\n", __func__,
3230	(mpts->mpts_evctl && ret == MPTS_EVRET_OK) ? "MPTCP_ERROR " : "",
3231	mpts->mpts_connid,
3232	mptcp_evret2str(ret), ret, mpts->mpts_evctl, SO_FILT_HINT_BITS),
3233	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3234	else
3235	mptcplog((LOG_DEBUG, "%s: Done, events %b\n", __func__,
3236	mpts->mpts_evctl, SO_FILT_HINT_BITS),
3237	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE);
3238
3239	return (ret);
3240	}
3241
3242	static ev_ret_t
3243	mptcp_subflow_propagate_ev(struct mptses mpte, struct* mptsub *mpts,
3244	uint64_t *p_mpsofilt_hint, uint64_t event)
3245	{
3246	struct socket mp_so, so;
3247	struct mptcb *mp_tp;
3248
3249	mpte_lock_assert_held(mpte); / same as MP socket lock /
3250	VERIFY(mpte->mpte_mppcb != NULL);
3251	mp_so = mptetoso(mpte);
3252	mp_tp = mpte->mpte_mptcb;
3253	so = mpts->mpts_socket;
3254
3255	mptcplog((LOG_DEBUG, "%s: cid %d event %d\n", __func__,
3256	mpts->mpts_connid, event),
3257	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3258
3259	/*
3260	* We got an event for this subflow that might need to be propagated,
3261	* based on the state of the MPTCP connection.
3262	*/
3263	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED \|\|
3264	((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE))) {
3265	mp_so->so_error = so->so_error;
3266	*p_mpsofilt_hint \|= event;
3267	}
3268
3269	return (MPTS_EVRET_OK);
3270	}
3271
3272	/*
3273	* Handle SO_FILT_HINT_NOSRCADDR subflow socket event.
3274	*/
3275	static ev_ret_t
3276	mptcp_subflow_nosrcaddr_ev(struct mptses mpte, struct* mptsub *mpts,
3277	uint64_t *p_mpsofilt_hint, uint64_t event)
3278	{
3279	#pragma unused(p_mpsofilt_hint, event)
3280	struct socket *mp_so;
3281	struct tcpcb *tp;
3282
3283	mpte_lock_assert_held(mpte); / same as MP socket lock /
3284
3285	VERIFY(mpte->mpte_mppcb != NULL);
3286	mp_so = mptetoso(mpte);
3287	tp = intotcpcb(sotoinpcb(mpts->mpts_socket));
3288
3289	/*
3290	* This overwrites any previous mpte_lost_aid to avoid storing
3291	* too much state when the typical case has only two subflows.
3292	*/
3293	mpte->mpte_flags \|= MPTE_SND_REM_ADDR;
3294	mpte->mpte_lost_aid = tp->t_local_aid;
3295
3296	mptcplog((LOG_DEBUG, "%s cid %d\n", __func__, mpts->mpts_connid),
3297	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3298
3299	/*
3300	* The subflow connection has lost its source address.
3301	*/
3302	mptcp_subflow_abort(mpts, EADDRNOTAVAIL);
3303
3304	if (mp_so->so_flags & SOF_NOADDRAVAIL)
3305	mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event);
3306
3307	return (MPTS_EVRET_DELETE);
3308	}
3309
3310	/*
3311	* Handle SO_FILT_HINT_MPCANTRCVMORE subflow socket event that
3312	* indicates that the remote side sent a Data FIN
3313	*/
3314	static ev_ret_t
3315	mptcp_subflow_mpcantrcvmore_ev(struct mptses mpte, struct* mptsub *mpts,
3316	uint64_t *p_mpsofilt_hint, uint64_t event)
3317	{
3318	#pragma unused(event)
3319	struct mptcb *mp_tp;
3320
3321	mpte_lock_assert_held(mpte); / same as MP socket lock /
3322	mp_tp = mpte->mpte_mptcb;
3323
3324	mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__, mpts->mpts_connid),
3325	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3326
3327	/*
3328	* We got a Data FIN for the MPTCP connection.
3329	* The FIN may arrive with data. The data is handed up to the
3330	* mptcp socket and the user is notified so that it may close
3331	* the socket if needed.
3332	*/
3333	if (mp_tp->mpt_state == MPTCPS_CLOSE_WAIT)
3334	*p_mpsofilt_hint \|= SO_FILT_HINT_CANTRCVMORE;
3335
3336	return (MPTS_EVRET_OK); / keep the subflow socket around /
3337	}
3338
3339	/*
3340	* Handle SO_FILT_HINT_MPFAILOVER subflow socket event
3341	*/
3342	static ev_ret_t
3343	mptcp_subflow_failover_ev(struct mptses mpte, struct* mptsub *mpts,
3344	uint64_t *p_mpsofilt_hint, uint64_t event)
3345	{
3346	#pragma unused(event, p_mpsofilt_hint)
3347	struct mptsub *mpts_alt = NULL;
3348	struct socket *alt_so = NULL;
3349	struct socket *mp_so;
3350	int altpath_exists = `0`;
3351
3352	mpte_lock_assert_held(mpte);
3353	mp_so = mptetoso(mpte);
3354	mptcplog((LOG_NOTICE, "%s: mp_so 0x%llx\n", __func__,
3355	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so)),
3356	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3357
3358	mptcp_reinject_mbufs(mpts->mpts_socket);
3359
3360	mpts_alt = mptcp_get_subflow(mpte, mpts, NULL);
3361	/*
3362	* If there is no alternate eligible subflow, ignore the
3363	* failover hint.
3364	*/
3365	if (mpts_alt == NULL) {
3366	mptcplog((LOG_WARNING, "%s: no alternate path\n", __func__),
3367	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3368
3369	goto done;
3370	}
3371
3372	altpath_exists = `1`;
3373	alt_so = mpts_alt->mpts_socket;
3374	if (mpts_alt->mpts_flags & MPTSF_FAILINGOVER) {
3375	/ All data acknowledged and no RTT spike /
3376	if (alt_so->so_snd.sb_cc == `0` && mptcp_no_rto_spike(alt_so)) {
3377	mpts_alt->mpts_flags &= ~MPTSF_FAILINGOVER;
3378	} else {
3379	/ no alternate path available /
3380	altpath_exists = `0`;
3381	}
3382	}
3383
3384	if (altpath_exists) {
3385	mpts_alt->mpts_flags \|= MPTSF_ACTIVE;
3386
3387	mpte->mpte_active_sub = mpts_alt;
3388	mpts->mpts_flags \|= MPTSF_FAILINGOVER;
3389	mpts->mpts_flags &= ~MPTSF_ACTIVE;
3390
3391	mptcplog((LOG_NOTICE, "%s: switched from %d to %d\n",
3392	__func__, mpts->mpts_connid, mpts_alt->mpts_connid),
3393	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3394
3395	mptcpstats_inc_switch(mpte, mpts);
3396
3397	sowwakeup(alt_so);
3398	} else {
3399	mptcplog((LOG_DEBUG, "%s: no alt cid = %d\n", __func__,
3400	mpts->mpts_connid),
3401	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3402	done:
3403	mpts->mpts_socket->so_flags &= ~SOF_MP_TRYFAILOVER;
3404	}
3405
3406	return (MPTS_EVRET_OK);
3407	}
3408
3409	/*
3410	* Handle SO_FILT_HINT_IFDENIED subflow socket event.
3411	*/
3412	static ev_ret_t
3413	mptcp_subflow_ifdenied_ev(struct mptses mpte, struct* mptsub *mpts,
3414	uint64_t *p_mpsofilt_hint, uint64_t event)
3415	{
3416	mpte_lock_assert_held(mpte); / same as MP socket lock /
3417	VERIFY(mpte->mpte_mppcb != NULL);
3418
3419	mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__,
3420	mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3421
3422	/*
3423	* The subflow connection cannot use the outgoing interface, let's
3424	* close this subflow.
3425	*/
3426	mptcp_subflow_abort(mpts, EPERM);
3427
3428	mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event);
3429
3430	return (MPTS_EVRET_DELETE);
3431	}
3432
3433	/*
3434	* https://tools.ietf.org/html/rfc6052#section-2
3435	* https://tools.ietf.org/html/rfc6147#section-5.2
3436	*/
3437	static boolean_t
3438	mptcp_desynthesize_ipv6_addr(const struct in6_addr *addr,
3439	const struct ipv6_prefix *prefix,
3440	struct in_addr *addrv4)
3441	{
3442	char buf[MAX_IPv4_STR_LEN];
3443	char ptrv4 = (char* *)addrv4;
3444	const char ptr = (const* char *)addr;
3445
3446	if (memcmp(addr, &prefix->ipv6_prefix, prefix->prefix_len) != `0`)
3447	return false;
3448
3449	switch (prefix->prefix_len) {
3450	case NAT64_PREFIX_LEN_96:
3451	memcpy(ptrv4, ptr + `12`, `4`);
3452	break;
3453	case NAT64_PREFIX_LEN_64:
3454	memcpy(ptrv4, ptr + `9`, `4`);
3455	break;
3456	case NAT64_PREFIX_LEN_56:
3457	memcpy(ptrv4, ptr + `7`, `1`);
3458	memcpy(ptrv4 + `1`, ptr + `9`, `3`);
3459	break;
3460	case NAT64_PREFIX_LEN_48:
3461	memcpy(ptrv4, ptr + `6`, `2`);
3462	memcpy(ptrv4 + `2`, ptr + `9`, `2`);
3463	break;
3464	case NAT64_PREFIX_LEN_40:
3465	memcpy(ptrv4, ptr + `5`, `3`);
3466	memcpy(ptrv4 + `3`, ptr + `9`, `1`);
3467	break;
3468	case NAT64_PREFIX_LEN_32:
3469	memcpy(ptrv4, ptr + `4`, `4`);
3470	break;
3471	default:
3472	panic("NAT64-prefix len is wrong: %u\n",
3473	prefix->prefix_len);
3474	}
3475
3476	os_log_info(mptcp_log_handle, "%s desynthesized to %s\n", __func__,
3477	inet_ntop(AF_INET, (void )addrv4, buf, sizeof*(buf)));
3478
3479	return true;
3480	}
3481
3482	static void
3483	mptcp_handle_ipv6_connection(struct mptses mpte, const* struct mptsub *mpts)
3484	{
3485	struct ipv6_prefix nat64prefixes[NAT64_MAX_NUM_PREFIXES];
3486	struct socket *so = mpts->mpts_socket;
3487	struct ifnet *ifp;
3488	int j;
3489
3490	ifp = sotoinpcb(so)->inp_last_outifp;
3491
3492	if (ifnet_get_nat64prefix(ifp, nat64prefixes) == ENOENT) {
3493	mptcp_ask_for_nat64(ifp);
3494	return;
3495	}
3496
3497
3498	for (j = `0`; j < NAT64_MAX_NUM_PREFIXES; j++) {
3499	int success;
3500
3501	if (nat64prefixes[j].prefix_len == `0`)
3502	continue;
3503
3504	success = mptcp_desynthesize_ipv6_addr(&mpte->__mpte_dst_v6.sin6_addr,
3505	&nat64prefixes[j],
3506	&mpte->mpte_dst_v4_nat64.sin_addr);
3507	if (success) {
3508	mpte->mpte_dst_v4_nat64.sin_len = sizeof(mpte->mpte_dst_v4_nat64);
3509	mpte->mpte_dst_v4_nat64.sin_family = AF_INET;
3510	mpte->mpte_dst_v4_nat64.sin_port = mpte->__mpte_dst_v6.sin6_port;
3511	break;
3512	}
3513	}
3514	}
3515
3516	/*
3517	* Handle SO_FILT_HINT_CONNECTED subflow socket event.
3518	*/
3519	static ev_ret_t
3520	mptcp_subflow_connected_ev(struct mptses mpte, struct* mptsub *mpts,
3521	uint64_t *p_mpsofilt_hint, uint64_t event)
3522	{
3523	#pragma unused(event, p_mpsofilt_hint)
3524	struct socket mp_so, so;
3525	struct inpcb *inp;
3526	struct tcpcb *tp;
3527	struct mptcb *mp_tp;
3528	int af;
3529	boolean_t mpok = FALSE;
3530
3531	mpte_lock_assert_held(mpte); / same as MP socket lock /
3532	VERIFY(mpte->mpte_mppcb != NULL);
3533
3534	mp_so = mptetoso(mpte);
3535	mp_tp = mpte->mpte_mptcb;
3536	so = mpts->mpts_socket;
3537	tp = sototcpcb(so);
3538	af = mpts->mpts_dst.sa_family;
3539
3540	if (mpts->mpts_flags & MPTSF_CONNECTED)
3541	return (MPTS_EVRET_OK);
3542
3543	if ((mpts->mpts_flags & MPTSF_DISCONNECTED) \|\|
3544	(mpts->mpts_flags & MPTSF_DISCONNECTING)) {
3545	if (!(so->so_state & (SS_ISDISCONNECTING \| SS_ISDISCONNECTED)) &&
3546	(so->so_state & SS_ISCONNECTED)) {
3547	mptcplog((LOG_DEBUG, "%s: cid %d disconnect before tcp connect\n",
3548	__func__, mpts->mpts_connid),
3549	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3550	(void) soshutdownlock(so, SHUT_RD);
3551	(void) soshutdownlock(so, SHUT_WR);
3552	(void) sodisconnectlocked(so);
3553	}
3554	return (MPTS_EVRET_OK);
3555	}
3556
3557	/*
3558	* The subflow connection has been connected. Find out whether it
3559	* is connected as a regular TCP or as a MPTCP subflow. The idea is:
3560	*
3561	* a. If MPTCP connection is not yet established, then this must be
3562	* the first subflow connection. If MPTCP failed to negotiate,
3563	* fallback to regular TCP by degrading this subflow.
3564	*
3565	* b. If MPTCP connection has been established, then this must be
3566	* one of the subsequent subflow connections. If MPTCP failed
3567	* to negotiate, disconnect the connection.
3568	*
3569	* Right now, we simply unblock any waiters at the MPTCP socket layer
3570	* if the MPTCP connection has not been established.
3571	*/
3572
3573	if (so->so_state & SS_ISDISCONNECTED) {
3574	/*
3575	* With MPTCP joins, a connection is connected at the subflow
3576	* level, but the 4th ACK from the server elevates the MPTCP
3577	* subflow to connected state. So there is a small window
3578	* where the subflow could get disconnected before the
3579	* connected event is processed.
3580	*/
3581	return (MPTS_EVRET_OK);
3582	}
3583
3584	if (mpts->mpts_flags & MPTSF_TFO_REQD)
3585	mptcp_drop_tfo_data(mpte, mpts);
3586
3587	mpts->mpts_flags &= ~(MPTSF_CONNECTING \| MPTSF_TFO_REQD);
3588	mpts->mpts_flags \|= MPTSF_CONNECTED;
3589
3590	if (tp->t_mpflags & TMPF_MPTCP_TRUE)
3591	mpts->mpts_flags \|= MPTSF_MP_CAPABLE;
3592
3593	tp->t_mpflags &= ~TMPF_TFO_REQUEST;
3594
3595	/ get/verify the outbound interface /
3596	inp = sotoinpcb(so);
3597
3598	mpts->mpts_maxseg = tp->t_maxseg;
3599
3600	mptcplog((LOG_DEBUG, "%s: cid %d outif %s is %s\n", __func__, mpts->mpts_connid,
3601	((inp->inp_last_outifp != NULL) ? inp->inp_last_outifp->if_xname : "NULL"),
3602	((mpts->mpts_flags & MPTSF_MP_CAPABLE) ? "MPTCP capable" : "a regular TCP")),
3603	(MPTCP_SOCKET_DBG \| MPTCP_EVENTS_DBG), MPTCP_LOGLVL_LOG);
3604
3605	mpok = (mpts->mpts_flags & MPTSF_MP_CAPABLE);
3606
3607	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) {
3608	mp_tp->mpt_state = MPTCPS_ESTABLISHED;
3609	mpte->mpte_associd = mpts->mpts_connid;
3610	DTRACE_MPTCP2(state__change,
3611	struct mptcb *, mp_tp,
3612	uint32_t, `0` / event /);
3613
3614	if (SOCK_DOM(so) == AF_INET) {
3615	in_getsockaddr_s(so, &mpte->__mpte_src_v4);
3616	} else {
3617	in6_getsockaddr_s(so, &mpte->__mpte_src_v6);
3618	}
3619
3620	mpts->mpts_flags \|= MPTSF_ACTIVE;
3621
3622	/ case (a) above /
3623	if (!mpok) {
3624	tcpstat.tcps_mpcap_fallback++;
3625
3626	tp->t_mpflags \|= TMPF_INFIN_SENT;
3627	mptcp_notify_mpfail(so);
3628	} else {
3629	if (IFNET_IS_CELLULAR(inp->inp_last_outifp) &&
3630	mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) {
3631	tp->t_mpflags \|= (TMPF_BACKUP_PATH \| TMPF_SND_MPPRIO);
3632	} else {
3633	mpts->mpts_flags \|= MPTSF_PREFERRED;
3634	}
3635	mpts->mpts_flags \|= MPTSF_MPCAP_CTRSET;
3636	mpte->mpte_nummpcapflows++;
3637
3638	if (SOCK_DOM(so) == AF_INET6)
3639	mptcp_handle_ipv6_connection(mpte, mpts);
3640
3641	mptcp_check_subflows_and_add(mpte);
3642
3643	if (IFNET_IS_CELLULAR(inp->inp_last_outifp))
3644	mpte->mpte_initial_cell = `1`;
3645
3646	mpte->mpte_handshake_success = `1`;
3647	}
3648
3649	mp_tp->mpt_sndwnd = tp->snd_wnd;
3650	mp_tp->mpt_sndwl1 = mp_tp->mpt_rcvnxt;
3651	mp_tp->mpt_sndwl2 = mp_tp->mpt_snduna;
3652	soisconnected(mp_so);
3653
3654	mptcplog((LOG_DEBUG, "%s: MPTCPS_ESTABLISHED for mp_so 0x%llx mpok %u\n",
3655	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpok),
3656	MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG);
3657	} else if (mpok) {
3658	/*
3659	* case (b) above
3660	* In case of additional flows, the MPTCP socket is not
3661	* MPTSF_MP_CAPABLE until an ACK is received from server
3662	* for 3-way handshake. TCP would have guaranteed that this
3663	* is an MPTCP subflow.
3664	*/
3665	if (IFNET_IS_CELLULAR(inp->inp_last_outifp) &&
3666	!(tp->t_mpflags & TMPF_BACKUP_PATH) &&
3667	mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) {
3668	tp->t_mpflags \|= (TMPF_BACKUP_PATH \| TMPF_SND_MPPRIO);
3669	mpts->mpts_flags &= ~MPTSF_PREFERRED;
3670	} else {
3671	mpts->mpts_flags \|= MPTSF_PREFERRED;
3672	}
3673
3674	mpts->mpts_flags \|= MPTSF_MPCAP_CTRSET;
3675	mpte->mpte_nummpcapflows++;
3676
3677	mpts->mpts_rel_seq = `1`;
3678
3679	mptcp_check_subflows_and_remove(mpte);
3680	} else {
3681	unsigned int i;
3682
3683	/ Should we try the alternate port? /
3684	if (mpte->mpte_alternate_port &&
3685	inp->inp_fport != mpte->mpte_alternate_port) {
3686	union sockaddr_in_4_6 dst;
3687	struct sockaddr_in dst_in = (struct* sockaddr_in *)&dst;
3688
3689	memcpy(&dst, &mpts->mpts_dst, mpts->mpts_dst.sa_len);
3690
3691	dst_in->sin_port = mpte->mpte_alternate_port;
3692
3693	mptcp_subflow_add(mpte, NULL, (struct sockaddr *)&dst,
3694	mpts->mpts_ifscope , NULL);
3695	} else { / Else, we tried all we could, mark this interface as non-MPTCP /
3696	for (i = `0`; i < mpte->mpte_itfinfo_size; i++) {
3697	struct mpt_itf_info *info = &mpte->mpte_itfinfo[i];
3698
3699	if (inp->inp_last_outifp->if_index == info->ifindex) {
3700	info->no_mptcp_support = `1`;
3701	break;
3702	}
3703	}
3704	}
3705
3706	tcpstat.tcps_join_fallback++;
3707	if (IFNET_IS_CELLULAR(inp->inp_last_outifp))
3708	tcpstat.tcps_mptcp_cell_proxy++;
3709	else
3710	tcpstat.tcps_mptcp_wifi_proxy++;
3711
3712	soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MUSTRST);
3713
3714	return (MPTS_EVRET_OK);
3715	}
3716
3717	/ This call, just to "book" an entry in the stats-table for this ifindex /
3718	mptcp_get_statsindex(mpte->mpte_itfstats, mpts);
3719
3720	mptcp_output(mpte);
3721
3722	return (MPTS_EVRET_OK); / keep the subflow socket around /
3723	}
3724
3725	/*
3726	* Handle SO_FILT_HINT_DISCONNECTED subflow socket event.
3727	*/
3728	static ev_ret_t
3729	mptcp_subflow_disconnected_ev(struct mptses mpte, struct* mptsub *mpts,
3730	uint64_t *p_mpsofilt_hint, uint64_t event)
3731	{
3732	#pragma unused(event, p_mpsofilt_hint)
3733	struct socket mp_so, so;
3734	struct mptcb *mp_tp;
3735
3736	mpte_lock_assert_held(mpte); / same as MP socket lock /
3737	VERIFY(mpte->mpte_mppcb != NULL);
3738	mp_so = mptetoso(mpte);
3739	mp_tp = mpte->mpte_mptcb;
3740	so = mpts->mpts_socket;
3741
3742	mptcplog((LOG_DEBUG, "%s: cid %d, so_err %d, mpt_state %u fallback %u active %u flags %#x\n",
3743	__func__, mpts->mpts_connid, so->so_error, mp_tp->mpt_state,
3744	!!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP),
3745	!!(mpts->mpts_flags & MPTSF_ACTIVE), sototcpcb(so)->t_mpflags),
3746	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3747
3748	if (mpts->mpts_flags & MPTSF_DISCONNECTED)
3749	return (MPTS_EVRET_DELETE);
3750
3751	mpts->mpts_flags \|= MPTSF_DISCONNECTED;
3752
3753	/ The subflow connection has been disconnected. /
3754
3755	if (mpts->mpts_flags & MPTSF_MPCAP_CTRSET) {
3756	mpte->mpte_nummpcapflows--;
3757	if (mpte->mpte_active_sub == mpts) {
3758	mpte->mpte_active_sub = NULL;
3759	mptcplog((LOG_DEBUG, "%s: resetting active subflow \n",
3760	__func__), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3761	}
3762	mpts->mpts_flags &= ~MPTSF_MPCAP_CTRSET;
3763	}
3764
3765	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED \|\|
3766	((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE)) \|\|
3767	(sototcpcb(so)->t_mpflags & TMPF_FASTCLOSERCV)) {
3768	mptcp_drop(mpte, mp_tp, so->so_error);
3769	}
3770
3771	/*
3772	* Clear flags that are used by getconninfo to return state.
3773	* Retain like MPTSF_DELETEOK for internal purposes.
3774	*/
3775	mpts->mpts_flags &= ~(MPTSF_CONNECTING\|MPTSF_CONNECT_PENDING\|
3776	MPTSF_CONNECTED\|MPTSF_DISCONNECTING\|MPTSF_PREFERRED\|
3777	MPTSF_MP_CAPABLE\|MPTSF_MP_READY\|MPTSF_MP_DEGRADED\|MPTSF_ACTIVE);
3778
3779	return (MPTS_EVRET_DELETE);
3780	}
3781
3782	/*
3783	* Handle SO_FILT_HINT_MPSTATUS subflow socket event
3784	*/
3785	static ev_ret_t
3786	mptcp_subflow_mpstatus_ev(struct mptses mpte, struct* mptsub *mpts,
3787	uint64_t *p_mpsofilt_hint, uint64_t event)
3788	{
3789	#pragma unused(event, p_mpsofilt_hint)
3790	struct socket mp_so, so;
3791	struct mptcb *mp_tp;
3792	ev_ret_t ret = MPTS_EVRET_OK;
3793
3794	mpte_lock_assert_held(mpte); / same as MP socket lock /
3795	VERIFY(mpte->mpte_mppcb != NULL);
3796	mp_so = mptetoso(mpte);
3797	mp_tp = mpte->mpte_mptcb;
3798	so = mpts->mpts_socket;
3799
3800	if (sototcpcb(so)->t_mpflags & TMPF_MPTCP_TRUE)
3801	mpts->mpts_flags \|= MPTSF_MP_CAPABLE;
3802	else
3803	mpts->mpts_flags &= ~MPTSF_MP_CAPABLE;
3804
3805	if (sototcpcb(so)->t_mpflags & TMPF_TCP_FALLBACK) {
3806	if (mpts->mpts_flags & MPTSF_MP_DEGRADED)
3807	goto done;
3808	mpts->mpts_flags \|= MPTSF_MP_DEGRADED;
3809	} else {
3810	mpts->mpts_flags &= ~MPTSF_MP_DEGRADED;
3811	}
3812
3813	if (sototcpcb(so)->t_mpflags & TMPF_MPTCP_READY)
3814	mpts->mpts_flags \|= MPTSF_MP_READY;
3815	else
3816	mpts->mpts_flags &= ~MPTSF_MP_READY;
3817
3818	if (mpts->mpts_flags & MPTSF_MP_DEGRADED) {
3819	mp_tp->mpt_flags \|= MPTCPF_FALLBACK_TO_TCP;
3820	mp_tp->mpt_flags &= ~MPTCPF_JOIN_READY;
3821	}
3822
3823	if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {
3824	VERIFY(!(mp_tp->mpt_flags & MPTCPF_JOIN_READY));
3825	ret = MPTS_EVRET_DISCONNECT_FALLBACK;
3826
3827	m_freem_list(mpte->mpte_reinjectq);
3828	mpte->mpte_reinjectq = NULL;
3829	} else if (mpts->mpts_flags & MPTSF_MP_READY) {
3830	mp_tp->mpt_flags \|= MPTCPF_JOIN_READY;
3831	ret = MPTS_EVRET_CONNECT_PENDING;
3832	}
3833
3834	mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx mpt_flags=%b cid %d mptsf=%b\n",
3835	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
3836	mp_tp->mpt_flags, MPTCPF_BITS, mpts->mpts_connid,
3837	mpts->mpts_flags, MPTSF_BITS),
3838	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3839
3840	done:
3841	return (ret);
3842	}
3843
3844	/*
3845	* Handle SO_FILT_HINT_MUSTRST subflow socket event
3846	*/
3847	static ev_ret_t
3848	mptcp_subflow_mustrst_ev(struct mptses mpte, struct* mptsub *mpts,
3849	uint64_t *p_mpsofilt_hint, uint64_t event)
3850	{
3851	#pragma unused(event)
3852	struct socket mp_so, so;
3853	struct mptcb *mp_tp;
3854	boolean_t is_fastclose;
3855
3856	mpte_lock_assert_held(mpte); / same as MP socket lock /
3857	VERIFY(mpte->mpte_mppcb != NULL);
3858	mp_so = mptetoso(mpte);
3859	mp_tp = mpte->mpte_mptcb;
3860	so = mpts->mpts_socket;
3861
3862	/ We got an invalid option or a fast close /
3863	struct tcptemp *t_template;
3864	struct inpcb *inp = sotoinpcb(so);
3865	struct tcpcb *tp = NULL;
3866
3867	tp = intotcpcb(inp);
3868	so->so_error = ECONNABORTED;
3869
3870	is_fastclose = !!(tp->t_mpflags & TMPF_FASTCLOSERCV);
3871
3872	t_template = tcp_maketemplate(tp);
3873	if (t_template) {
3874	struct tcp_respond_args tra;
3875
3876	bzero(&tra, sizeof(tra));
3877	if (inp->inp_flags & INP_BOUND_IF)
3878	tra.ifscope = inp->inp_boundifp->if_index;
3879	else
3880	tra.ifscope = IFSCOPE_NONE;
3881	tra.awdl_unrestricted = `1`;
3882
3883	tcp_respond(tp, t_template->tt_ipgen,
3884	&t_template->tt_t, (struct mbuf *)NULL,
3885	tp->rcv_nxt, tp->snd_una, TH_RST, &tra);
3886	(void) m_free(dtom(t_template));
3887	mptcplog((LOG_DEBUG, "MPTCP Events: "
3888	"%s: mp_so 0x%llx cid %d \n",
3889	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
3890	so, mpts->mpts_connid),
3891	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG);
3892	}
3893	mptcp_subflow_abort(mpts, ECONNABORTED);
3894
3895	if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && is_fastclose) {
3896	*p_mpsofilt_hint \|= SO_FILT_HINT_CONNRESET;
3897
3898	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED)
3899	mp_so->so_error = ECONNABORTED;
3900	else
3901	mp_so->so_error = ECONNRESET;
3902
3903	/*
3904	* mptcp_drop is being called after processing the events, to fully
3905	* close the MPTCP connection
3906	*/
3907	}
3908
3909	if (mp_tp->mpt_gc_ticks == MPT_GC_TICKS)
3910	mp_tp->mpt_gc_ticks = MPT_GC_TICKS_FAST;
3911
3912	return (MPTS_EVRET_DELETE);
3913	}
3914
3915	static ev_ret_t
3916	mptcp_subflow_adaptive_rtimo_ev(struct mptses mpte, struct* mptsub *mpts,
3917	uint64_t *p_mpsofilt_hint, uint64_t event)
3918	{
3919	#pragma unused(event)
3920	bool found_active = false;
3921
3922	mpts->mpts_flags \|= MPTSF_READ_STALL;
3923
3924	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
3925	struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
3926
3927	if (!TCPS_HAVEESTABLISHED(tp->t_state) \|\|
3928	TCPS_HAVERCVDFIN2(tp->t_state))
3929	continue;
3930
3931	if (!(mpts->mpts_flags & MPTSF_READ_STALL)) {
3932	found_active = true;
3933	break;
3934	}
3935	}
3936
3937	if (!found_active)
3938	*p_mpsofilt_hint \|= SO_FILT_HINT_ADAPTIVE_RTIMO;
3939
3940	return (MPTS_EVRET_OK);
3941	}
3942
3943	static ev_ret_t
3944	mptcp_subflow_adaptive_wtimo_ev(struct mptses mpte, struct* mptsub *mpts,
3945	uint64_t *p_mpsofilt_hint, uint64_t event)
3946	{
3947	#pragma unused(event)
3948	bool found_active = false;
3949
3950	mpts->mpts_flags \|= MPTSF_WRITE_STALL;
3951
3952	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
3953	struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
3954
3955	if (!TCPS_HAVEESTABLISHED(tp->t_state) \|\|
3956	tp->t_state > TCPS_CLOSE_WAIT)
3957	continue;
3958
3959	if (!(mpts->mpts_flags & MPTSF_WRITE_STALL)) {
3960	found_active = true;
3961	break;
3962	}
3963	}
3964
3965	if (!found_active)
3966	*p_mpsofilt_hint \|= SO_FILT_HINT_ADAPTIVE_WTIMO;
3967
3968	return (MPTS_EVRET_OK);
3969	}
3970
3971	static const char *
3972	mptcp_evret2str(ev_ret_t ret)
3973	{
3974	const char *c = "UNKNOWN";
3975
3976	switch (ret) {
3977	case MPTS_EVRET_DELETE:
3978	c = "MPTS_EVRET_DELETE";
3979	break;
3980	case MPTS_EVRET_CONNECT_PENDING:
3981	c = "MPTS_EVRET_CONNECT_PENDING";
3982	break;
3983	case MPTS_EVRET_DISCONNECT_FALLBACK:
3984	c = "MPTS_EVRET_DISCONNECT_FALLBACK";
3985	break;
3986	case MPTS_EVRET_OK:
3987	c = "MPTS_EVRET_OK";
3988	break;
3989	default:
3990	break;
3991	}
3992	return (c);
3993	}
3994
3995	/*
3996	* Issues SOPT_SET on an MPTCP subflow socket; socket must already be locked,
3997	* caller must ensure that the option can be issued on subflow sockets, via
3998	* MPOF_SUBFLOW_OK flag.
3999	*/
4000	int
4001	mptcp_subflow_sosetopt(struct mptses mpte, struct* mptsub mpts, struct* mptopt *mpo)
4002	{
4003	struct socket mp_so, so;
4004	struct sockopt sopt;
4005	int error;
4006
4007	VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK);
4008	mpte_lock_assert_held(mpte);
4009
4010	mp_so = mptetoso(mpte);
4011	so = mpts->mpts_socket;
4012
4013	if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED &&
4014	mpo->mpo_level == SOL_SOCKET &&
4015	mpo->mpo_name == SO_MARK_CELLFALLBACK) {
4016	struct ifnet *ifp = ifindex2ifnet[mpts->mpts_ifscope];
4017
4018	mptcplog((LOG_DEBUG, "%s Setting CELL_FALLBACK, mpte_flags %#x, svctype %u wifi unusable %d lastcell? %d boundcell? %d\n",
4019	__func__, mpte->mpte_flags, mpte->mpte_svctype, mptcp_is_wifi_unusable(mpte),
4020	sotoinpcb(so)->inp_last_outifp ? IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp) : -`1`,
4021	mpts->mpts_ifscope != IFSCOPE_NONE && ifp ? IFNET_IS_CELLULAR(ifp) : -`1`),
4022	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4023
4024	/*
4025	* When we open a new subflow, mark it as cell fallback, if
4026	* this subflow goes over cell.
4027	*
4028	* (except for first-party apps)
4029	*/
4030
4031	if (mpte->mpte_flags & MPTE_FIRSTPARTY)
4032	return (`0`);
4033
4034	if (sotoinpcb(so)->inp_last_outifp &&
4035	!IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp))
4036	return (`0`);
4037
4038	/*
4039	* This here is an OR, because if the app is not binding to the
4040	* interface, then it definitely is not a cell-fallback
4041	* connection.
4042	*/
4043	if (mpts->mpts_ifscope == IFSCOPE_NONE \|\| ifp == NULL \|\|
4044	!IFNET_IS_CELLULAR(ifp))
4045	return (`0`);
4046	}
4047
4048	mpo->mpo_flags &= ~MPOF_INTERIM;
4049
4050	bzero(&sopt, sizeof (sopt));
4051	sopt.sopt_dir = SOPT_SET;
4052	sopt.sopt_level = mpo->mpo_level;
4053	sopt.sopt_name = mpo->mpo_name;
4054	sopt.sopt_val = CAST_USER_ADDR_T(&mpo->mpo_intval);
4055	sopt.sopt_valsize = sizeof (int);
4056	sopt.sopt_p = kernproc;
4057
4058	error = sosetoptlock(so, &sopt, `0`);
4059	if (error == `0`) {
4060	mptcplog((LOG_INFO, "%s: mp_so 0x%llx sopt %s "
4061	"val %d set successful\n", __func__,
4062	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4063	mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name),
4064	mpo->mpo_intval),
4065	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG);
4066	} else {
4067	mptcplog((LOG_ERR, "%s:mp_so 0x%llx sopt %s "
4068	"val %d set error %d\n", __func__,
4069	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4070	mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name),
4071	mpo->mpo_intval, error),
4072	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
4073	}
4074	return (error);
4075	}
4076
4077	/*
4078	* Issues SOPT_GET on an MPTCP subflow socket; socket must already be locked,
4079	* caller must ensure that the option can be issued on subflow sockets, via
4080	* MPOF_SUBFLOW_OK flag.
4081	*/
4082	int
4083	mptcp_subflow_sogetopt(struct mptses mpte, struct* socket *so,
4084	struct mptopt *mpo)
4085	{
4086	struct socket *mp_so;
4087	struct sockopt sopt;
4088	int error;
4089
4090	VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK);
4091	mpte_lock_assert_held(mpte); / same as MP socket lock /
4092	mp_so = mptetoso(mpte);
4093
4094	bzero(&sopt, sizeof (sopt));
4095	sopt.sopt_dir = SOPT_GET;
4096	sopt.sopt_level = mpo->mpo_level;
4097	sopt.sopt_name = mpo->mpo_name;
4098	sopt.sopt_val = CAST_USER_ADDR_T(&mpo->mpo_intval);
4099	sopt.sopt_valsize = sizeof (int);
4100	sopt.sopt_p = kernproc;
4101
4102	error = sogetoptlock(so, &sopt, `0`); / already locked /
4103	if (error == `0`) {
4104	mptcplog((LOG_DEBUG, "MPTCP Socket: "
4105	"%s: mp_so 0x%llx sopt %s "
4106	"val %d get successful\n", __func__,
4107	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4108	mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name),
4109	mpo->mpo_intval),
4110	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4111	} else {
4112	mptcplog((LOG_ERR, "MPTCP Socket: "
4113	"%s: mp_so 0x%llx sopt %s get error %d\n",
4114	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4115	mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), error),
4116	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
4117	}
4118	return (error);
4119	}
4120
4121
4122	/*
4123	* MPTCP garbage collector.
4124	*
4125	* This routine is called by the MP domain on-demand, periodic callout,
4126	* which is triggered when a MPTCP socket is closed. The callout will
4127	* repeat as long as this routine returns a non-zero value.
4128	*/
4129	static uint32_t
4130	mptcp_gc(struct mppcbinfo *mppi)
4131	{
4132	struct mppcb mpp, tmpp;
4133	uint32_t active = `0`;
4134
4135	LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED);
4136
4137	TAILQ_FOREACH_SAFE(mpp, &mppi->mppi_pcbs, mpp_entry, tmpp) {
4138	struct socket *mp_so;
4139	struct mptses *mpte;
4140	struct mptcb *mp_tp;
4141
4142	VERIFY(mpp->mpp_flags & MPP_ATTACHED);
4143	mp_so = mpp->mpp_socket;
4144	VERIFY(mp_so != NULL);
4145	mpte = mptompte(mpp);
4146	VERIFY(mpte != NULL);
4147	mp_tp = mpte->mpte_mptcb;
4148	VERIFY(mp_tp != NULL);
4149
4150	mptcplog((LOG_DEBUG, "MPTCP Socket: "
4151	"%s: mp_so 0x%llx found "
4152	"(u=%d,r=%d,s=%d)\n", __func__,
4153	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_usecount,
4154	mp_so->so_retaincnt, mpp->mpp_state),
4155	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4156
4157	if (!mpte_try_lock(mpte)) {
4158	mptcplog((LOG_DEBUG, "MPTCP Socket: "
4159	"%s: mp_so 0x%llx skipped lock "
4160	"(u=%d,r=%d)\n", __func__,
4161	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4162	mp_so->so_usecount, mp_so->so_retaincnt),
4163	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4164	active++;
4165	continue;
4166	}
4167
4168	/ check again under the lock /
4169	if (mp_so->so_usecount > `0`) {
4170	boolean_t wakeup = FALSE;
4171	struct mptsub mpts, tmpts;
4172
4173	mptcplog((LOG_DEBUG, "MPTCP Socket: "
4174	"%s: mp_so 0x%llx skipped usecount "
4175	"[u=%d,r=%d] %d %d\n", __func__,
4176	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4177	mp_so->so_usecount, mp_so->so_retaincnt,
4178	mp_tp->mpt_gc_ticks,
4179	mp_tp->mpt_state),
4180	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4181
4182	if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1) {
4183	if (mp_tp->mpt_gc_ticks > `0`)
4184	mp_tp->mpt_gc_ticks--;
4185	if (mp_tp->mpt_gc_ticks == `0`) {
4186	wakeup = TRUE;
4187	}
4188	}
4189	if (wakeup) {
4190	TAILQ_FOREACH_SAFE(mpts,
4191	&mpte->mpte_subflows, mpts_entry, tmpts) {
4192	mptcp_subflow_eupcall1(mpts->mpts_socket,
4193	mpts, SO_FILT_HINT_DISCONNECTED);
4194	}
4195	}
4196	mpte_unlock(mpte);
4197	active++;
4198	continue;
4199	}
4200
4201	if (mpp->mpp_state != MPPCB_STATE_DEAD) {
4202	panic("MPTCP Socket: %s: mp_so 0x%llx skipped state "
4203	"[u=%d,r=%d,s=%d]\n", __func__,
4204	(u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4205	mp_so->so_usecount, mp_so->so_retaincnt,
4206	mpp->mpp_state);
4207	}
4208
4209	if (mp_tp->mpt_state == MPTCPS_TIME_WAIT)
4210	mptcp_close(mpte, mp_tp);
4211
4212	mptcp_session_destroy(mpte);
4213
4214	mptcplog((LOG_DEBUG, "MPTCP Socket: "
4215	"%s: mp_so 0x%llx destroyed [u=%d,r=%d]\n",
4216	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
4217	mp_so->so_usecount, mp_so->so_retaincnt),
4218	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4219
4220	DTRACE_MPTCP4(dispose, struct socket *, mp_so,
4221	struct sockbuf *, &mp_so->so_rcv,
4222	struct sockbuf *, &mp_so->so_snd,
4223	struct mppcb *, mpp);
4224
4225	mp_pcbdispose(mpp);
4226	sodealloc(mp_so);
4227	}
4228
4229	return (active);
4230	}
4231
4232	/*
4233	* Drop a MPTCP connection, reporting the specified error.
4234	*/
4235	struct mptses *
4236	mptcp_drop(struct mptses mpte, struct* mptcb mp_tp, int* errno)
4237	{
4238	struct socket *mp_so;
4239
4240	mpte_lock_assert_held(mpte); / same as MP socket lock /
4241	VERIFY(mpte->mpte_mptcb == mp_tp);
4242	mp_so = mptetoso(mpte);
4243
4244	DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp,
4245	uint32_t, `0` / event /);
4246
4247	if (errno == ETIMEDOUT && mp_tp->mpt_softerror != `0`)
4248	errno = mp_tp->mpt_softerror;
4249	mp_so->so_error = errno;
4250
4251	return (mptcp_close(mpte, mp_tp));
4252	}
4253
4254	/*
4255	* Close a MPTCP control block.
4256	*/
4257	struct mptses *
4258	mptcp_close(struct mptses mpte, struct* mptcb *mp_tp)
4259	{
4260	struct socket *mp_so = NULL;
4261	struct mptsub mpts = NULL, tmpts = NULL;
4262
4263	mpte_lock_assert_held(mpte); / same as MP socket lock /
4264	VERIFY(mpte->mpte_mptcb == mp_tp);
4265	mp_so = mptetoso(mpte);
4266
4267	mp_tp->mpt_state = MPTCPS_TERMINATE;
4268
4269	mptcp_freeq(mp_tp);
4270
4271	soisdisconnected(mp_so);
4272
4273	/ Clean up all subflows /
4274	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
4275	mptcp_subflow_disconnect(mpte, mpts);
4276	}
4277
4278	return (NULL);
4279	}
4280
4281	void
4282	mptcp_notify_close(struct socket *so)
4283	{
4284	soevent(so, (SO_FILT_HINT_LOCKED \| SO_FILT_HINT_DISCONNECTED));
4285	}
4286
4287	/*
4288	* MPTCP workloop.
4289	*/
4290	void
4291	mptcp_subflow_workloop(struct mptses *mpte)
4292	{
4293	struct socket *mp_so;
4294	struct mptsub mpts, tmpts;
4295	boolean_t connect_pending = FALSE, disconnect_fallback = FALSE;
4296	uint64_t mpsofilt_hint_mask = SO_FILT_HINT_LOCKED;
4297
4298	mpte_lock_assert_held(mpte);
4299	VERIFY(mpte->mpte_mppcb != NULL);
4300	mp_so = mptetoso(mpte);
4301	VERIFY(mp_so != NULL);
4302
4303	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
4304	ev_ret_t ret;
4305
4306	if (mpts->mpts_socket->so_usecount == `0`) {
4307	/ Will be removed soon by tcp_garbage_collect /
4308	continue;
4309	}
4310
4311	mptcp_subflow_addref(mpts);
4312	mpts->mpts_socket->so_usecount++;
4313
4314	ret = mptcp_subflow_events(mpte, mpts, &mpsofilt_hint_mask);
4315
4316	/*
4317	* If MPTCP socket is closed, disconnect all subflows.
4318	* This will generate a disconnect event which will
4319	* be handled during the next iteration, causing a
4320	* non-zero error to be returned above.
4321	*/
4322	if (mp_so->so_flags & SOF_PCBCLEARING)
4323	mptcp_subflow_disconnect(mpte, mpts);
4324
4325	switch (ret) {
4326	case MPTS_EVRET_OK:
4327	/ nothing to do /
4328	break;
4329	case MPTS_EVRET_DELETE:
4330	mptcp_subflow_soclose(mpts);
4331	break;
4332	case MPTS_EVRET_CONNECT_PENDING:
4333	connect_pending = TRUE;
4334	break;
4335	case MPTS_EVRET_DISCONNECT_FALLBACK:
4336	disconnect_fallback = TRUE;
4337	break;
4338	default:
4339	mptcplog((LOG_DEBUG,
4340	"MPTCP Socket: %s: mptcp_subflow_events "
4341	"returned invalid value: %d\n", __func__,
4342	ret),
4343	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4344	break;
4345	}
4346	mptcp_subflow_remref(mpts); / ours /
4347
4348	VERIFY(mpts->mpts_socket->so_usecount != `0`);
4349	mpts->mpts_socket->so_usecount--;
4350	}
4351
4352	if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) {
4353	VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED);
4354
4355	soevent(mp_so, mpsofilt_hint_mask);
4356	}
4357
4358	if (!connect_pending && !disconnect_fallback)
4359	return;
4360
4361	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
4362	if (disconnect_fallback) {
4363	struct socket *so = NULL;
4364	struct inpcb *inp = NULL;
4365	struct tcpcb *tp = NULL;
4366
4367	if (mpts->mpts_flags & MPTSF_MP_DEGRADED)
4368	continue;
4369
4370	mpts->mpts_flags \|= MPTSF_MP_DEGRADED;
4371
4372	if (mpts->mpts_flags & (MPTSF_DISCONNECTING\|
4373	MPTSF_DISCONNECTED\|MPTSF_CONNECT_PENDING))
4374	continue;
4375
4376	so = mpts->mpts_socket;
4377
4378	/*
4379	* The MPTCP connection has degraded to a fallback
4380	* mode, so there is no point in keeping this subflow
4381	* regardless of its MPTCP-readiness state, unless it
4382	* is the primary one which we use for fallback. This
4383	* assumes that the subflow used for fallback is the
4384	* ACTIVE one.
4385	*/
4386
4387	inp = sotoinpcb(so);
4388	tp = intotcpcb(inp);
4389	tp->t_mpflags &=
4390	~(TMPF_MPTCP_READY\|TMPF_MPTCP_TRUE);
4391	tp->t_mpflags \|= TMPF_TCP_FALLBACK;
4392
4393	if (mpts->mpts_flags & MPTSF_ACTIVE) {
4394	continue;
4395	}
4396	tp->t_mpflags \|= TMPF_RESET;
4397	soevent(so, SO_FILT_HINT_MUSTRST);
4398	} else if (connect_pending) {
4399	/*
4400	* The MPTCP connection has progressed to a state
4401	* where it supports full multipath semantics; allow
4402	* additional joins to be attempted for all subflows
4403	* that are in the PENDING state.
4404	*/
4405	if (mpts->mpts_flags & MPTSF_CONNECT_PENDING) {
4406	int error = mptcp_subflow_soconnectx(mpte, mpts);
4407
4408	if (error)
4409	mptcp_subflow_abort(mpts, error);
4410	}
4411	}
4412	}
4413	}
4414
4415	/*
4416	* Protocol pr_lock callback.
4417	*/
4418	int
4419	mptcp_lock(struct socket mp_so, int* refcount, void *lr)
4420	{
4421	struct mppcb *mpp = mpsotomppcb(mp_so);
4422	void *lr_saved;
4423
4424	if (lr == NULL)
4425	lr_saved = __builtin_return_address(`0`);
4426	else
4427	lr_saved = lr;
4428
4429	if (mpp == NULL) {
4430	panic("%s: so=%p NO PCB! lr=%p lrh= %s\n", __func__,
4431	mp_so, lr_saved, solockhistory_nr(mp_so));
4432	/ NOTREACHED /
4433	}
4434	mpp_lock(mpp);
4435
4436	if (mp_so->so_usecount < `0`) {
4437	panic("%s: so=%p so_pcb=%p lr=%p ref=%x lrh= %s\n", __func__,
4438	mp_so, mp_so->so_pcb, lr_saved, mp_so->so_usecount,
4439	solockhistory_nr(mp_so));
4440	/ NOTREACHED /
4441	}
4442	if (refcount != `0`)
4443	mp_so->so_usecount++;
4444	mp_so->lock_lr[mp_so->next_lock_lr] = lr_saved;
4445	mp_so->next_lock_lr = (mp_so->next_lock_lr + `1`) % SO_LCKDBG_MAX;
4446
4447	return (`0`);
4448	}
4449
4450	/*
4451	* Protocol pr_unlock callback.
4452	*/
4453	int
4454	mptcp_unlock(struct socket mp_so, int* refcount, void *lr)
4455	{
4456	struct mppcb *mpp = mpsotomppcb(mp_so);
4457	void *lr_saved;
4458
4459	if (lr == NULL)
4460	lr_saved = __builtin_return_address(`0`);
4461	else
4462	lr_saved = lr;
4463
4464	if (mpp == NULL) {
4465	panic("%s: so=%p NO PCB usecount=%x lr=%p lrh= %s\n", __func__,
4466	mp_so, mp_so->so_usecount, lr_saved,
4467	solockhistory_nr(mp_so));
4468	/ NOTREACHED /
4469	}
4470	mpp_lock_assert_held(mpp);
4471
4472	if (refcount != `0`)
4473	mp_so->so_usecount--;
4474
4475	if (mp_so->so_usecount < `0`) {
4476	panic("%s: so=%p usecount=%x lrh= %s\n", __func__,
4477	mp_so, mp_so->so_usecount, solockhistory_nr(mp_so));
4478	/ NOTREACHED /
4479	}
4480	mp_so->unlock_lr[mp_so->next_unlock_lr] = lr_saved;
4481	mp_so->next_unlock_lr = (mp_so->next_unlock_lr + `1`) % SO_LCKDBG_MAX;
4482	mpp_unlock(mpp);
4483
4484	return (`0`);
4485	}
4486
4487	/*
4488	* Protocol pr_getlock callback.
4489	*/
4490	lck_mtx_t *
4491	mptcp_getlock(struct socket mp_so, int* flags)
4492	{
4493	struct mppcb *mpp = mpsotomppcb(mp_so);
4494
4495	if (mpp == NULL) {
4496	panic("%s: so=%p NULL so_pcb %s\n", __func__, mp_so,
4497	solockhistory_nr(mp_so));
4498	/ NOTREACHED /
4499	}
4500	if (mp_so->so_usecount < `0`) {
4501	panic("%s: so=%p usecount=%x lrh= %s\n", __func__,
4502	mp_so, mp_so->so_usecount, solockhistory_nr(mp_so));
4503	/ NOTREACHED /
4504	}
4505	return (mpp_getlock(mpp, flags));
4506	}
4507
4508	/*
4509	* MPTCP Join support
4510	*/
4511
4512	static void
4513	mptcp_attach_to_subf(struct socket so, struct* mptcb *mp_tp,
4514	uint8_t addr_id)
4515	{
4516	struct tcpcb *tp = sototcpcb(so);
4517	struct mptcp_subf_auth_entry *sauth_entry;
4518	mpte_lock_assert_held(mp_tp->mpt_mpte);
4519
4520	/*
4521	* The address ID of the first flow is implicitly 0.
4522	*/
4523	if (mp_tp->mpt_state == MPTCPS_CLOSED) {
4524	tp->t_local_aid = `0`;
4525	} else {
4526	tp->t_local_aid = addr_id;
4527	tp->t_mpflags \|= (TMPF_PREESTABLISHED \| TMPF_JOINED_FLOW);
4528	so->so_flags \|= SOF_MP_SEC_SUBFLOW;
4529	}
4530	sauth_entry = zalloc(mpt_subauth_zone);
4531	sauth_entry->msae_laddr_id = tp->t_local_aid;
4532	sauth_entry->msae_raddr_id = `0`;
4533	sauth_entry->msae_raddr_rand = `0`;
4534	try_again:
4535	sauth_entry->msae_laddr_rand = RandomULong();
4536	if (sauth_entry->msae_laddr_rand == `0`)
4537	goto try_again;
4538	LIST_INSERT_HEAD(&mp_tp->mpt_subauth_list, sauth_entry, msae_next);
4539	}
4540
4541	static void
4542	mptcp_detach_mptcb_from_subf(struct mptcb mp_tp, struct* socket *so)
4543	{
4544	struct mptcp_subf_auth_entry *sauth_entry;
4545	struct tcpcb *tp = NULL;
4546	int found = `0`;
4547
4548	tp = sototcpcb(so);
4549	if (tp == NULL)
4550	return;
4551
4552	LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) {
4553	if (sauth_entry->msae_laddr_id == tp->t_local_aid) {
4554	found = `1`;
4555	break;
4556	}
4557	}
4558	if (found) {
4559	LIST_REMOVE(sauth_entry, msae_next);
4560	}
4561
4562	if (found)
4563	zfree(mpt_subauth_zone, sauth_entry);
4564	}
4565
4566	void
4567	mptcp_get_rands(mptcp_addr_id addr_id, struct mptcb mp_tp, u_int32_t lrand,
4568	u_int32_t *rrand)
4569	{
4570	struct mptcp_subf_auth_entry *sauth_entry;
4571	mpte_lock_assert_held(mp_tp->mpt_mpte);
4572
4573	LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) {
4574	if (sauth_entry->msae_laddr_id == addr_id) {
4575	if (lrand)
4576	*lrand = sauth_entry->msae_laddr_rand;
4577	if (rrand)
4578	*rrand = sauth_entry->msae_raddr_rand;
4579	break;
4580	}
4581	}
4582	}
4583
4584	void
4585	mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp,
4586	mptcp_addr_id raddr_id, u_int32_t raddr_rand)
4587	{
4588	struct mptcp_subf_auth_entry *sauth_entry;
4589	mpte_lock_assert_held(mp_tp->mpt_mpte);
4590
4591	LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) {
4592	if (sauth_entry->msae_laddr_id == laddr_id) {
4593	if ((sauth_entry->msae_raddr_id != `0`) &&
4594	(sauth_entry->msae_raddr_id != raddr_id)) {
4595	mptcplog((LOG_ERR, "MPTCP Socket: %s mismatched"
4596	" address ids %d %d \n", __func__, raddr_id,
4597	sauth_entry->msae_raddr_id),
4598	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG);
4599	return;
4600	}
4601	sauth_entry->msae_raddr_id = raddr_id;
4602	if ((sauth_entry->msae_raddr_rand != `0`) &&
4603	(sauth_entry->msae_raddr_rand != raddr_rand)) {
4604	mptcplog((LOG_ERR, "MPTCP Socket: "
4605	"%s: dup SYN_ACK %d %d \n",
4606	__func__, raddr_rand,
4607	sauth_entry->msae_raddr_rand),
4608	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG);
4609	return;
4610	}
4611	sauth_entry->msae_raddr_rand = raddr_rand;
4612	return;
4613	}
4614	}
4615	}
4616
4617	/*
4618	* SHA1 support for MPTCP
4619	*/
4620	static void
4621	mptcp_do_sha1(mptcp_key_t key, char* *sha_digest)
4622	{
4623	SHA1_CTX sha1ctxt;
4624	const unsigned char *sha1_base;
4625	int sha1_size;
4626
4627	sha1_base = (const unsigned char *) key;
4628	sha1_size = sizeof (mptcp_key_t);
4629	SHA1Init(&sha1ctxt);
4630	SHA1Update(&sha1ctxt, sha1_base, sha1_size);
4631	SHA1Final(sha_digest, &sha1ctxt);
4632	}
4633
4634	void
4635	mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2,
4636	u_int32_t rand1, u_int32_t rand2, u_char *digest)
4637	{
4638	SHA1_CTX sha1ctxt;
4639	mptcp_key_t key_ipad[`8`] = {`0`}; / key XOR'd with inner pad /
4640	mptcp_key_t key_opad[`8`] = {`0`}; / key XOR'd with outer pad /
4641	u_int32_t data[`2`];
4642	int i;
4643
4644	bzero(digest, SHA1_RESULTLEN);
4645
4646	/ Set up the Key for HMAC /
4647	key_ipad[`0`] = key1;
4648	key_ipad[`1`] = key2;
4649
4650	key_opad[`0`] = key1;
4651	key_opad[`1`] = key2;
4652
4653	/ Set up the message for HMAC /
4654	data[`0`] = rand1;
4655	data[`1`] = rand2;
4656
4657	/ Key is 512 block length, so no need to compute hash /
4658
4659	/ Compute SHA1(Key XOR opad, SHA1(Key XOR ipad, data)) /
4660
4661	for (i = `0`; i < `8`; i++) {
4662	key_ipad[i] ^= `0x3636363636363636`;
4663	key_opad[i] ^= `0x5c5c5c5c5c5c5c5c`;
4664	}
4665
4666	/ Perform inner SHA1 /
4667	SHA1Init(&sha1ctxt);
4668	SHA1Update(&sha1ctxt, (unsigned char )key_ipad, sizeof* (key_ipad));
4669	SHA1Update(&sha1ctxt, (unsigned char )data, sizeof* (data));
4670	SHA1Final(digest, &sha1ctxt);
4671
4672	/ Perform outer SHA1 /
4673	SHA1Init(&sha1ctxt);
4674	SHA1Update(&sha1ctxt, (unsigned char )key_opad, sizeof* (key_opad));
4675	SHA1Update(&sha1ctxt, (unsigned char *)digest, SHA1_RESULTLEN);
4676	SHA1Final(digest, &sha1ctxt);
4677	}
4678
4679	/*
4680	* corresponds to MAC-B = MAC (Key=(Key-B+Key-A), Msg=(R-B+R-A))
4681	* corresponds to MAC-A = MAC (Key=(Key-A+Key-B), Msg=(R-A+R-B))
4682	*/
4683	void
4684	mptcp_get_hmac(mptcp_addr_id aid, struct mptcb mp_tp, u_char digest)
4685	{
4686	uint32_t lrand, rrand;
4687
4688	mpte_lock_assert_held(mp_tp->mpt_mpte);
4689
4690	lrand = rrand = `0`;
4691	mptcp_get_rands(aid, mp_tp, &lrand, &rrand);
4692	mptcp_hmac_sha1(mp_tp->mpt_localkey, mp_tp->mpt_remotekey, lrand, rrand,
4693	digest);
4694	}
4695
4696	/*
4697	* Authentication data generation
4698	*/
4699	static void
4700	mptcp_generate_token(char sha_digest, int* sha_digest_len, caddr_t token,
4701	int token_len)
4702	{
4703	VERIFY(token_len == sizeof (u_int32_t));
4704	VERIFY(sha_digest_len == SHA1_RESULTLEN);
4705
4706	/ Most significant 32 bits of the SHA1 hash /
4707	bcopy(sha_digest, token, sizeof (u_int32_t));
4708	return;
4709	}
4710
4711	static void
4712	mptcp_generate_idsn(char sha_digest, int* sha_digest_len, caddr_t idsn,
4713	int idsn_len)
4714	{
4715	VERIFY(idsn_len == sizeof (u_int64_t));
4716	VERIFY(sha_digest_len == SHA1_RESULTLEN);
4717
4718	/*
4719	* Least significant 64 bits of the SHA1 hash
4720	*/
4721
4722	idsn[`7`] = sha_digest[`12`];
4723	idsn[`6`] = sha_digest[`13`];
4724	idsn[`5`] = sha_digest[`14`];
4725	idsn[`4`] = sha_digest[`15`];
4726	idsn[`3`] = sha_digest[`16`];
4727	idsn[`2`] = sha_digest[`17`];
4728	idsn[`1`] = sha_digest[`18`];
4729	idsn[`0`] = sha_digest[`19`];
4730	return;
4731	}
4732
4733	static void
4734	mptcp_conn_properties(struct mptcb *mp_tp)
4735	{
4736	/ There is only Version 0 at this time /
4737	mp_tp->mpt_version = MPTCP_STD_VERSION_0;
4738
4739	/ Set DSS checksum flag /
4740	if (mptcp_dss_csum)
4741	mp_tp->mpt_flags \|= MPTCPF_CHECKSUM;
4742
4743	/ Set up receive window /
4744	mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp);
4745
4746	/ Set up gc ticks /
4747	mp_tp->mpt_gc_ticks = MPT_GC_TICKS;
4748	}
4749
4750	static void
4751	mptcp_init_local_parms(struct mptses *mpte)
4752	{
4753	struct mptcb *mp_tp = mpte->mpte_mptcb;
4754	char key_digest[SHA1_RESULTLEN];
4755
4756	read_frandom(&mp_tp->mpt_localkey, sizeof(mp_tp->mpt_localkey));
4757	mptcp_do_sha1(&mp_tp->mpt_localkey, key_digest);
4758
4759	mptcp_generate_token(key_digest, SHA1_RESULTLEN,
4760	(caddr_t)&mp_tp->mpt_localtoken, sizeof (mp_tp->mpt_localtoken));
4761	mptcp_generate_idsn(key_digest, SHA1_RESULTLEN,
4762	(caddr_t)&mp_tp->mpt_local_idsn, sizeof (u_int64_t));
4763
4764	/ The subflow SYN is also first MPTCP byte /
4765	mp_tp->mpt_snduna = mp_tp->mpt_sndmax = mp_tp->mpt_local_idsn + `1`;
4766	mp_tp->mpt_sndnxt = mp_tp->mpt_snduna;
4767
4768	mptcp_conn_properties(mp_tp);
4769	}
4770
4771	int
4772	mptcp_init_remote_parms(struct mptcb *mp_tp)
4773	{
4774	char remote_digest[SHA1_RESULTLEN];
4775	mpte_lock_assert_held(mp_tp->mpt_mpte);
4776
4777	/ Only Version 0 is supported for auth purposes /
4778	if (mp_tp->mpt_version != MPTCP_STD_VERSION_0)
4779	return (-`1`);
4780
4781	/ Setup local and remote tokens and Initial DSNs /
4782	mptcp_do_sha1(&mp_tp->mpt_remotekey, remote_digest);
4783	mptcp_generate_token(remote_digest, SHA1_RESULTLEN,
4784	(caddr_t)&mp_tp->mpt_remotetoken, sizeof (mp_tp->mpt_remotetoken));
4785	mptcp_generate_idsn(remote_digest, SHA1_RESULTLEN,
4786	(caddr_t)&mp_tp->mpt_remote_idsn, sizeof (u_int64_t));
4787	mp_tp->mpt_rcvnxt = mp_tp->mpt_remote_idsn + `1`;
4788
4789	return (`0`);
4790	}
4791
4792	static void
4793	mptcp_send_dfin(struct socket *so)
4794	{
4795	struct tcpcb *tp = NULL;
4796	struct inpcb *inp = NULL;
4797
4798	inp = sotoinpcb(so);
4799	if (!inp)
4800	return;
4801
4802	tp = intotcpcb(inp);
4803	if (!tp)
4804	return;
4805
4806	if (!(tp->t_mpflags & TMPF_RESET))
4807	tp->t_mpflags \|= TMPF_SEND_DFIN;
4808	}
4809
4810	/*
4811	* Data Sequence Mapping routines
4812	*/
4813	void
4814	mptcp_insert_dsn(struct mppcb mpp, struct* mbuf *m)
4815	{
4816	struct mptcb *mp_tp;
4817
4818	if (m == NULL)
4819	return;
4820
4821	__IGNORE_WCASTALIGN(mp_tp = &((struct mpp_mtp *)mpp)->mtcb);
4822	mpte_lock_assert_held(mp_tp->mpt_mpte);
4823
4824	while (m) {
4825	VERIFY(m->m_flags & M_PKTHDR);
4826	m->m_pkthdr.pkt_flags \|= (PKTF_MPTCP \| PKTF_MPSO);
4827	m->m_pkthdr.mp_dsn = mp_tp->mpt_sndmax;
4828	m->m_pkthdr.mp_rlen = m_pktlen(m);
4829	mp_tp->mpt_sndmax += m_pktlen(m);
4830	m = m->m_next;
4831	}
4832	}
4833
4834	void
4835	mptcp_fallback_sbdrop(struct socket so, struct* mbuf m, int* len)
4836	{
4837	struct mptcb *mp_tp = tptomptp(sototcpcb(so));
4838	uint64_t data_ack;
4839	uint64_t dsn;
4840
4841	if (!m \|\| len == `0`)
4842	return;
4843
4844	while (m && len > `0`) {
4845	VERIFY(m->m_flags & M_PKTHDR);
4846	VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP);
4847
4848	data_ack = m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen;
4849	dsn = m->m_pkthdr.mp_dsn;
4850
4851	len -= m->m_len;
4852	m = m->m_next;
4853	}
4854
4855	if (m && len == `0`) {
4856	/*
4857	* If there is one more mbuf in the chain, it automatically means
4858	* that up to m->mp_dsn has been ack'ed.
4859	*
4860	* This means, we actually correct data_ack back down (compared
4861	* to what we set inside the loop - dsn + data_len). Because in
4862	* the loop we are "optimistic" and assume that the full mapping
4863	* will be acked. If that's not the case and we get out of the
4864	* loop with m != NULL, it means only up to m->mp_dsn has been
4865	* really acked.
4866	*/
4867	data_ack = m->m_pkthdr.mp_dsn;
4868	}
4869
4870	if (len < `0`) {
4871	/*
4872	* If len is negative, meaning we acked in the middle of an mbuf,
4873	* only up to this mbuf's data-sequence number has been acked
4874	* at the MPTCP-level.
4875	*/
4876	data_ack = dsn;
4877	}
4878
4879	mptcplog((LOG_DEBUG, "%s inferred ack up to %u\n", __func__, (uint32_t)data_ack),
4880	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
4881	mptcp_data_ack_rcvd(mp_tp, sototcpcb(so), data_ack);
4882	}
4883
4884	void
4885	mptcp_preproc_sbdrop(struct socket so, struct* mbuf m, unsigned* int len)
4886	{
4887	int rewinding = `0`;
4888
4889	/ TFO makes things complicated. /
4890	if (so->so_flags1 & SOF1_TFO_REWIND) {
4891	rewinding = `1`;
4892	so->so_flags1 &= ~SOF1_TFO_REWIND;
4893	}
4894
4895	while (m && (!(so->so_flags & SOF_MP_SUBFLOW) \|\| rewinding)) {
4896	u_int32_t sub_len;
4897	VERIFY(m->m_flags & M_PKTHDR);
4898	VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP);
4899
4900	sub_len = m->m_pkthdr.mp_rlen;
4901
4902	if (sub_len < len) {
4903	m->m_pkthdr.mp_dsn += sub_len;
4904	if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) {
4905	m->m_pkthdr.mp_rseq += sub_len;
4906	}
4907	m->m_pkthdr.mp_rlen = `0`;
4908	len -= sub_len;
4909	} else {
4910	/ sub_len >= len /
4911	if (rewinding == `0`)
4912	m->m_pkthdr.mp_dsn += len;
4913	if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) {
4914	if (rewinding == `0`)
4915	m->m_pkthdr.mp_rseq += len;
4916	}
4917	mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u len %d %d\n",
4918	__func__, (u_int32_t)m->m_pkthdr.mp_dsn,
4919	m->m_pkthdr.mp_rseq, m->m_pkthdr.mp_rlen, len),
4920	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
4921	m->m_pkthdr.mp_rlen -= len;
4922	break;
4923	}
4924	m = m->m_next;
4925	}
4926
4927	if (so->so_flags & SOF_MP_SUBFLOW &&
4928	!(sototcpcb(so)->t_mpflags & TMPF_TFO_REQUEST) &&
4929	!(sototcpcb(so)->t_mpflags & TMPF_RCVD_DACK)) {
4930	/*
4931	* Received an ack without receiving a DATA_ACK.
4932	* Need to fallback to regular TCP (or destroy this subflow).
4933	*/
4934	sototcpcb(so)->t_mpflags \|= TMPF_INFIN_SENT;
4935	mptcp_notify_mpfail(so);
4936	}
4937	}
4938
4939	/ Obtain the DSN mapping stored in the mbuf /
4940	void
4941	mptcp_output_getm_dsnmap32(struct socket so, int* off,
4942	uint32_t dsn, uint32_t relseq, uint16_t data_len, uint16_t dss_csum)
4943	{
4944	u_int64_t dsn64;
4945
4946	mptcp_output_getm_dsnmap64(so, off, &dsn64, relseq, data_len, dss_csum);
4947	*dsn = (u_int32_t)MPTCP_DATASEQ_LOW32(dsn64);
4948	}
4949
4950	void
4951	mptcp_output_getm_dsnmap64(struct socket so, int* off, uint64_t *dsn,
4952	uint32_t relseq, uint16_t data_len,
4953	uint16_t *dss_csum)
4954	{
4955	struct mbuf *m = so->so_snd.sb_mb;
4956	int off_orig = off;
4957
4958	VERIFY(off >= `0`);
4959
4960	/*
4961	* In the subflow socket, the DSN sequencing can be discontiguous,
4962	* but the subflow sequence mapping is contiguous. Use the subflow
4963	* sequence property to find the right mbuf and corresponding dsn
4964	* mapping.
4965	*/
4966
4967	while (m) {
4968	VERIFY(m->m_flags & M_PKTHDR);
4969	VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP);
4970
4971	if (off >= m->m_len) {
4972	off -= m->m_len;
4973	m = m->m_next;
4974	} else {
4975	break;
4976	}
4977	}
4978
4979	VERIFY(m);
4980	VERIFY(off >= `0`);
4981	VERIFY(m->m_pkthdr.mp_rlen <= UINT16_MAX);
4982
4983	*dsn = m->m_pkthdr.mp_dsn;
4984	*relseq = m->m_pkthdr.mp_rseq;
4985	*data_len = m->m_pkthdr.mp_rlen;
4986	*dss_csum = m->m_pkthdr.mp_csum;
4987
4988	mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u data_len %d off %d off_orig %d\n",
4989	__func__, (u_int32_t)(dsn), relseq, *data_len, off, off_orig),
4990	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
4991	}
4992
4993	/*
4994	* Note that this is called only from tcp_input() via mptcp_input_preproc()
4995	* tcp_input() may trim data after the dsn mapping is inserted into the mbuf.
4996	* When it trims data tcp_input calls m_adj() which does not remove the
4997	* m_pkthdr even if the m_len becomes 0 as a result of trimming the mbuf.
4998	* The dsn map insertion cannot be delayed after trim, because data can be in
4999	* the reassembly queue for a while and the DSN option info in tp will be
5000	* overwritten for every new packet received.
5001	* The dsn map will be adjusted just prior to appending to subflow sockbuf
5002	* with mptcp_adj_rmap()
5003	*/
5004	void
5005	mptcp_insert_rmap(struct tcpcb tp, struct* mbuf m, struct* tcphdr *th)
5006	{
5007	VERIFY(m->m_flags & M_PKTHDR);
5008	VERIFY(!(m->m_pkthdr.pkt_flags & PKTF_MPTCP));
5009
5010	if (tp->t_mpflags & TMPF_EMBED_DSN) {
5011	m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn;
5012	m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq;
5013	m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len;
5014	m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum;
5015	if (tp->t_rcv_map.mpt_dfin)
5016	m->m_pkthdr.pkt_flags \|= PKTF_MPTCP_DFIN;
5017
5018	m->m_pkthdr.pkt_flags \|= PKTF_MPTCP;
5019
5020	tp->t_mpflags &= ~TMPF_EMBED_DSN;
5021	tp->t_mpflags \|= TMPF_MPTCP_ACKNOW;
5022	} else if (tp->t_mpflags & TMPF_TCP_FALLBACK) {
5023	if (th->th_flags & TH_FIN)
5024	m->m_pkthdr.pkt_flags \|= PKTF_MPTCP_DFIN;
5025	}
5026	}
5027
5028	int
5029	mptcp_adj_rmap(struct socket so, struct* mbuf m, int* off, uint64_t dsn,
5030	uint32_t rseq, uint16_t dlen)
5031	{
5032	struct mptsub *mpts = sototcpcb(so)->t_mpsub;
5033
5034	if (m_pktlen(m) == `0`)
5035	return (`0`);
5036
5037	if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) {
5038	if (off && (dsn != m->m_pkthdr.mp_dsn \|\|
5039	rseq != m->m_pkthdr.mp_rseq \|\|
5040	dlen != m->m_pkthdr.mp_rlen)) {
5041	mptcplog((LOG_ERR, "%s: Received incorrect second mapping: %llu - %llu , %u - %u, %u - %u\n",
5042	__func__, dsn, m->m_pkthdr.mp_dsn,
5043	rseq, m->m_pkthdr.mp_rseq,
5044	dlen, m->m_pkthdr.mp_rlen),
5045	MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR);
5046	return (-`1`);
5047	}
5048	m->m_pkthdr.mp_dsn += off;
5049	m->m_pkthdr.mp_rseq += off;
5050	m->m_pkthdr.mp_rlen = m->m_pkthdr.len;
5051	} else {
5052	if (!(mpts->mpts_flags & MPTSF_CONFIRMED)) {
5053	/ data arrived without an DSS option mapping /
5054
5055	/ initial subflow can fallback right after SYN handshake /
5056	mptcp_notify_mpfail(so);
5057	}
5058	}
5059
5060	mpts->mpts_flags \|= MPTSF_CONFIRMED;
5061
5062	return (`0`);
5063	}
5064
5065	/*
5066	* Following routines help with failure detection and failover of data
5067	* transfer from one subflow to another.
5068	*/
5069	void
5070	mptcp_act_on_txfail(struct socket *so)
5071	{
5072	struct tcpcb *tp = NULL;
5073	struct inpcb *inp = sotoinpcb(so);
5074
5075	if (inp == NULL)
5076	return;
5077
5078	tp = intotcpcb(inp);
5079	if (tp == NULL)
5080	return;
5081
5082	if (so->so_flags & SOF_MP_TRYFAILOVER)
5083	return;
5084
5085	so->so_flags \|= SOF_MP_TRYFAILOVER;
5086	soevent(so, (SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MPFAILOVER));
5087	}
5088
5089	/*
5090	* Support for MP_FAIL option
5091	*/
5092	int
5093	mptcp_get_map_for_dsn(struct socket so, u_int64_t dsn_fail, u_int32_t tcp_seq)
5094	{
5095	struct mbuf *m = so->so_snd.sb_mb;
5096	u_int64_t dsn;
5097	int off = `0`;
5098	u_int32_t datalen;
5099
5100	if (m == NULL)
5101	return (-`1`);
5102
5103	while (m != NULL) {
5104	VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP);
5105	VERIFY(m->m_flags & M_PKTHDR);
5106	dsn = m->m_pkthdr.mp_dsn;
5107	datalen = m->m_pkthdr.mp_rlen;
5108	if (MPTCP_SEQ_LEQ(dsn, dsn_fail) &&
5109	(MPTCP_SEQ_GEQ(dsn + datalen, dsn_fail))) {
5110	off = dsn_fail - dsn;
5111	*tcp_seq = m->m_pkthdr.mp_rseq + off;
5112	mptcplog((LOG_DEBUG, "%s: %llu %llu \n", __func__, dsn,
5113	dsn_fail), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG);
5114	return (`0`);
5115	}
5116
5117	m = m->m_next;
5118	}
5119
5120	/*
5121	* If there was no mbuf data and a fallback to TCP occurred, there's
5122	* not much else to do.
5123	*/
5124
5125	mptcplog((LOG_ERR, "MPTCP Sender: "
5126	"%s: %llu not found \n", __func__, dsn_fail),
5127	MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG);
5128	return (-`1`);
5129	}
5130
5131	/*
5132	* Support for sending contiguous MPTCP bytes in subflow
5133	* Also for preventing sending data with ACK in 3-way handshake
5134	*/
5135	int32_t
5136	mptcp_adj_sendlen(struct socket *so, int32_t off)
5137	{
5138	struct tcpcb *tp = sototcpcb(so);
5139	struct mptsub *mpts = tp->t_mpsub;
5140	uint64_t mdss_dsn;
5141	uint32_t mdss_subflow_seq;
5142	int mdss_subflow_off;
5143	uint16_t mdss_data_len;
5144	uint16_t dss_csum;
5145
5146	mptcp_output_getm_dsnmap64(so, off, &mdss_dsn, &mdss_subflow_seq,
5147	&mdss_data_len, &dss_csum);
5148
5149	/*
5150	* We need to compute how much of the mapping still remains.
5151	* So, we compute the offset in the send-buffer of the dss-sub-seq.
5152	*/
5153	mdss_subflow_off = (mdss_subflow_seq + mpts->mpts_iss) - tp->snd_una;
5154
5155	/*
5156	* When TFO is used, we are sending the mpts->mpts_iss although the relative
5157	* seq has been set to 1 (while it should be 0).
5158	*/
5159	if (tp->t_mpflags & TMPF_TFO_REQUEST)
5160	mdss_subflow_off--;
5161
5162	if (off < mdss_subflow_off)
5163	printf("%s off %d mdss_subflow_off %d mdss_subflow_seq %u iss %u suna %u\n", __func__,
5164	off, mdss_subflow_off, mdss_subflow_seq, mpts->mpts_iss, tp->snd_una);
5165	VERIFY(off >= mdss_subflow_off);
5166
5167	mptcplog((LOG_DEBUG, "%s dlen %u off %d sub_off %d sub_seq %u iss %u suna %u\n",
5168	__func__, mdss_data_len, off, mdss_subflow_off, mdss_subflow_seq,
5169	mpts->mpts_iss, tp->snd_una), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
5170	return (mdss_data_len - (off - mdss_subflow_off));
5171	}
5172
5173	static uint32_t
5174	mptcp_get_maxseg(struct mptses *mpte)
5175	{
5176	struct mptsub *mpts;
5177	uint32_t maxseg = `0`;
5178
5179	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
5180	struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
5181
5182	if (!TCPS_HAVEESTABLISHED(tp->t_state) \|\|
5183	TCPS_HAVERCVDFIN2(tp->t_state))
5184	continue;
5185
5186	if (tp->t_maxseg > maxseg)
5187	maxseg = tp->t_maxseg;
5188	}
5189
5190	return (maxseg);
5191	}
5192
5193	static uint8_t
5194	mptcp_get_rcvscale(struct mptses *mpte)
5195	{
5196	struct mptsub *mpts;
5197	uint8_t rcvscale = UINT8_MAX;
5198
5199	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
5200	struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
5201
5202	if (!TCPS_HAVEESTABLISHED(tp->t_state) \|\|
5203	TCPS_HAVERCVDFIN2(tp->t_state))
5204	continue;
5205
5206	if (tp->rcv_scale < rcvscale)
5207	rcvscale = tp->rcv_scale;
5208	}
5209
5210	return (rcvscale);
5211	}
5212
5213	/ Similar to tcp_sbrcv_reserve /
5214	static void
5215	mptcp_sbrcv_reserve(struct mptcb mp_tp, struct* sockbuf *sbrcv,
5216	u_int32_t newsize, u_int32_t idealsize)
5217	{
5218	uint8_t rcvscale = mptcp_get_rcvscale(mp_tp->mpt_mpte);
5219
5220	/ newsize should not exceed max /
5221	newsize = min(newsize, tcp_autorcvbuf_max);
5222
5223	/ The receive window scale negotiated at the*
5224	* beginning of the connection will also set a
5225	* limit on the socket buffer size
5226	*/
5227	newsize = min(newsize, TCP_MAXWIN << rcvscale);
5228
5229	/ Set new socket buffer size /
5230	if (newsize > sbrcv->sb_hiwat &&
5231	(sbreserve(sbrcv, newsize) == `1`)) {
5232	sbrcv->sb_idealsize = min(max(sbrcv->sb_idealsize,
5233	(idealsize != `0`) ? idealsize : newsize), tcp_autorcvbuf_max);
5234
5235	/ Again check the limit set by the advertised*
5236	* window scale
5237	*/
5238	sbrcv->sb_idealsize = min(sbrcv->sb_idealsize,
5239	TCP_MAXWIN << rcvscale);
5240	}
5241	}
5242
5243	void
5244	mptcp_sbrcv_grow(struct mptcb *mp_tp)
5245	{
5246	struct mptses *mpte = mp_tp->mpt_mpte;
5247	struct socket *mp_so = mpte->mpte_mppcb->mpp_socket;
5248	struct sockbuf *sbrcv = &mp_so->so_rcv;
5249	uint32_t hiwat_sum = `0`;
5250	uint32_t ideal_sum = `0`;
5251	struct mptsub *mpts;
5252
5253	/*
5254	* Do not grow the receive socket buffer if
5255	* - auto resizing is disabled, globally or on this socket
5256	* - the high water mark already reached the maximum
5257	* - the stream is in background and receive side is being
5258	* throttled
5259	* - if there are segments in reassembly queue indicating loss,
5260	* do not need to increase recv window during recovery as more
5261	* data is not going to be sent. A duplicate ack sent during
5262	* recovery should not change the receive window
5263	*/
5264	if (tcp_do_autorcvbuf == `0` \|\|
5265	(sbrcv->sb_flags & SB_AUTOSIZE) == `0` \|\|
5266	tcp_cansbgrow(sbrcv) == `0` \|\|
5267	sbrcv->sb_hiwat >= tcp_autorcvbuf_max \|\|
5268	(mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) \|\|
5269	!LIST_EMPTY(&mp_tp->mpt_segq)) {
5270	/ Can not resize the socket buffer, just return /
5271	return;
5272	}
5273
5274	/*
5275	* Ideally, we want the rbuf to be (sum_i {bw_i} * rtt_max * 2)
5276	*
5277	* But, for this we first need accurate receiver-RTT estimations, which
5278	* we currently don't have.
5279	*
5280	* Let's use a dummy algorithm for now, just taking the sum of all
5281	* subflow's receive-buffers. It's too low, but that's all we can get
5282	* for now.
5283	*/
5284
5285	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
5286	hiwat_sum += mpts->mpts_socket->so_rcv.sb_hiwat;
5287	ideal_sum += mpts->mpts_socket->so_rcv.sb_idealsize;
5288	}
5289
5290	mptcp_sbrcv_reserve(mp_tp, sbrcv, hiwat_sum, ideal_sum);
5291	}
5292
5293	/*
5294	* Determine if we can grow the recieve socket buffer to avoid sending
5295	* a zero window update to the peer. We allow even socket buffers that
5296	* have fixed size (set by the application) to grow if the resource
5297	* constraints are met. They will also be trimmed after the application
5298	* reads data.
5299	*
5300	* Similar to tcp_sbrcv_grow_rwin
5301	*/
5302	static void
5303	mptcp_sbrcv_grow_rwin(struct mptcb mp_tp, struct* sockbuf *sb)
5304	{
5305	struct socket *mp_so = mp_tp->mpt_mpte->mpte_mppcb->mpp_socket;
5306	u_int32_t rcvbufinc = mptcp_get_maxseg(mp_tp->mpt_mpte) << `4`;
5307	u_int32_t rcvbuf = sb->sb_hiwat;
5308
5309	if (tcp_recv_bg == `1` \|\| IS_TCP_RECV_BG(mp_so))
5310	return;
5311
5312	if (tcp_do_autorcvbuf == `1` &&
5313	tcp_cansbgrow(sb) &&
5314	/ Diff to tcp_sbrcv_grow_rwin /
5315	(mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) == `0` &&
5316	(rcvbuf - sb->sb_cc) < rcvbufinc &&
5317	rcvbuf < tcp_autorcvbuf_max &&
5318	(sb->sb_idealsize > `0` &&
5319	sb->sb_hiwat <= (sb->sb_idealsize + rcvbufinc))) {
5320	sbreserve(sb, min((sb->sb_hiwat + rcvbufinc), tcp_autorcvbuf_max));
5321	}
5322	}
5323
5324	/ Similar to tcp_sbspace /
5325	int32_t
5326	mptcp_sbspace(struct mptcb *mp_tp)
5327	{
5328	struct sockbuf *sb = &mp_tp->mpt_mpte->mpte_mppcb->mpp_socket->so_rcv;
5329	uint32_t rcvbuf;
5330	int32_t space;
5331	int32_t pending = `0`;
5332
5333	mpte_lock_assert_held(mp_tp->mpt_mpte);
5334
5335	mptcp_sbrcv_grow_rwin(mp_tp, sb);
5336
5337	/ hiwat might have changed /
5338	rcvbuf = sb->sb_hiwat;
5339
5340	space = ((int32_t) imin((rcvbuf - sb->sb_cc),
5341	(sb->sb_mbmax - sb->sb_mbcnt)));
5342	if (space < `0`)
5343	space = `0`;
5344
5345	#if CONTENT_FILTER
5346	/ Compensate for data being processed by content filters /
5347	pending = cfil_sock_data_space(sb);
5348	#endif /* CONTENT_FILTER */
5349	if (pending > space)
5350	space = `0`;
5351	else
5352	space -= pending;
5353
5354	return (space);
5355	}
5356
5357	/*
5358	* Support Fallback to Regular TCP
5359	*/
5360	void
5361	mptcp_notify_mpready(struct socket *so)
5362	{
5363	struct tcpcb *tp = NULL;
5364
5365	if (so == NULL)
5366	return;
5367
5368	tp = intotcpcb(sotoinpcb(so));
5369
5370	if (tp == NULL)
5371	return;
5372
5373	DTRACE_MPTCP4(multipath__ready, struct socket *, so,
5374	struct sockbuf , &so->so_rcv, struct* sockbuf *, &so->so_snd,
5375	struct tcpcb *, tp);
5376
5377	if (!(tp->t_mpflags & TMPF_MPTCP_TRUE))
5378	return;
5379
5380	if (tp->t_mpflags & TMPF_MPTCP_READY)
5381	return;
5382
5383	tp->t_mpflags &= ~TMPF_TCP_FALLBACK;
5384	tp->t_mpflags \|= TMPF_MPTCP_READY;
5385
5386	soevent(so, (SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MPSTATUS));
5387	}
5388
5389	void
5390	mptcp_notify_mpfail(struct socket *so)
5391	{
5392	struct tcpcb *tp = NULL;
5393
5394	if (so == NULL)
5395	return;
5396
5397	tp = intotcpcb(sotoinpcb(so));
5398
5399	if (tp == NULL)
5400	return;
5401
5402	DTRACE_MPTCP4(multipath__failed, struct socket *, so,
5403	struct sockbuf , &so->so_rcv, struct* sockbuf *, &so->so_snd,
5404	struct tcpcb *, tp);
5405
5406	if (tp->t_mpflags & TMPF_TCP_FALLBACK)
5407	return;
5408
5409	tp->t_mpflags &= ~(TMPF_MPTCP_READY\|TMPF_MPTCP_TRUE);
5410	tp->t_mpflags \|= TMPF_TCP_FALLBACK;
5411
5412	soevent(so, (SO_FILT_HINT_LOCKED \| SO_FILT_HINT_MPSTATUS));
5413	}
5414
5415	/*
5416	* Keepalive helper function
5417	*/
5418	boolean_t
5419	mptcp_ok_to_keepalive(struct mptcb *mp_tp)
5420	{
5421	boolean_t ret = `1`;
5422	mpte_lock_assert_held(mp_tp->mpt_mpte);
5423
5424	if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) {
5425	ret = `0`;
5426	}
5427	return (ret);
5428	}
5429
5430	/*
5431	* MPTCP t_maxseg adjustment function
5432	*/
5433	int
5434	mptcp_adj_mss(struct tcpcb *tp, boolean_t mtudisc)
5435	{
5436	int mss_lower = `0`;
5437	struct mptcb *mp_tp = tptomptp(tp);
5438
5439	#define MPTCP_COMPUTE_LEN { \
5440	mss_lower = sizeof (struct mptcp_dss_ack_opt); \
5441	if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) \
5442	mss_lower += 2; \
5443	else \
5444	/* adjust to 32-bit boundary + EOL */ \
5445	mss_lower += 2; \
5446	}
5447	if (mp_tp == NULL)
5448	return (`0`);
5449
5450	mpte_lock_assert_held(mp_tp->mpt_mpte);
5451
5452	/*
5453	* For the first subflow and subsequent subflows, adjust mss for
5454	* most common MPTCP option size, for case where tcp_mss is called
5455	* during option processing and MTU discovery.
5456	*/
5457	if (!mtudisc) {
5458	if (tp->t_mpflags & TMPF_MPTCP_TRUE &&
5459	!(tp->t_mpflags & TMPF_JOINED_FLOW)) {
5460	MPTCP_COMPUTE_LEN;
5461	}
5462
5463	if (tp->t_mpflags & TMPF_PREESTABLISHED &&
5464	tp->t_mpflags & TMPF_SENT_JOIN) {
5465	MPTCP_COMPUTE_LEN;
5466	}
5467	} else {
5468	if (tp->t_mpflags & TMPF_MPTCP_TRUE) {
5469	MPTCP_COMPUTE_LEN;
5470	}
5471	}
5472
5473	return (mss_lower);
5474	}
5475
5476	/*
5477	* Update the pid, upid, uuid of the subflow so, based on parent so
5478	*/
5479	void
5480	mptcp_update_last_owner(struct socket so, struct* socket *mp_so)
5481	{
5482	if (so->last_pid != mp_so->last_pid \|\|
5483	so->last_upid != mp_so->last_upid) {
5484	so->last_upid = mp_so->last_upid;
5485	so->last_pid = mp_so->last_pid;
5486	uuid_copy(so->last_uuid, mp_so->last_uuid);
5487	}
5488	so_update_policy(so);
5489	}
5490
5491	static void
5492	fill_mptcp_subflow(struct socket so, mptcp_flow_t flow, struct mptsub *mpts)
5493	{
5494	struct inpcb *inp;
5495
5496	tcp_getconninfo(so, &flow->flow_ci);
5497	inp = sotoinpcb(so);
5498	#if INET6
5499	if ((inp->inp_vflag & INP_IPV6) != `0`) {
5500	flow->flow_src.ss_family = AF_INET6;
5501	flow->flow_dst.ss_family = AF_INET6;
5502	flow->flow_src.ss_len = sizeof(struct sockaddr_in6);
5503	flow->flow_dst.ss_len = sizeof(struct sockaddr_in6);
5504	SIN6(&flow->flow_src)->sin6_port = inp->in6p_lport;
5505	SIN6(&flow->flow_dst)->sin6_port = inp->in6p_fport;
5506	SIN6(&flow->flow_src)->sin6_addr = inp->in6p_laddr;
5507	SIN6(&flow->flow_dst)->sin6_addr = inp->in6p_faddr;
5508	} else
5509	#endif
5510	if ((inp->inp_vflag & INP_IPV4) != `0`) {
5511	flow->flow_src.ss_family = AF_INET;
5512	flow->flow_dst.ss_family = AF_INET;
5513	flow->flow_src.ss_len = sizeof(struct sockaddr_in);
5514	flow->flow_dst.ss_len = sizeof(struct sockaddr_in);
5515	SIN(&flow->flow_src)->sin_port = inp->inp_lport;
5516	SIN(&flow->flow_dst)->sin_port = inp->inp_fport;
5517	SIN(&flow->flow_src)->sin_addr = inp->inp_laddr;
5518	SIN(&flow->flow_dst)->sin_addr = inp->inp_faddr;
5519	}
5520	flow->flow_len = sizeof(*flow);
5521	flow->flow_tcpci_offset = offsetof(mptcp_flow_t, flow_ci);
5522	flow->flow_flags = mpts->mpts_flags;
5523	flow->flow_cid = mpts->mpts_connid;
5524	flow->flow_relseq = mpts->mpts_rel_seq;
5525	flow->flow_soerror = mpts->mpts_socket->so_error;
5526	flow->flow_probecnt = mpts->mpts_probecnt;
5527	}
5528
5529	static int
5530	mptcp_pcblist SYSCTL_HANDLER_ARGS
5531	{
5532	#pragma unused(oidp, arg1, arg2)
5533	int error = `0`, f;
5534	size_t len;
5535	struct mppcb *mpp;
5536	struct mptses *mpte;
5537	struct mptcb *mp_tp;
5538	struct mptsub *mpts;
5539	struct socket *so;
5540	conninfo_mptcp_t mptcpci;
5541	mptcp_flow_t *flows = NULL;
5542
5543	if (req->newptr != USER_ADDR_NULL)
5544	return (EPERM);
5545
5546	lck_mtx_lock(&mtcbinfo.mppi_lock);
5547	if (req->oldptr == USER_ADDR_NULL) {
5548	size_t n = mtcbinfo.mppi_count;
5549	lck_mtx_unlock(&mtcbinfo.mppi_lock);
5550	req->oldidx = (n + n/`8`) * sizeof(conninfo_mptcp_t) +
5551	`4` * (n + n/`8`) * sizeof(mptcp_flow_t);
5552	return (`0`);
5553	}
5554	TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) {
5555	flows = NULL;
5556	mpp_lock(mpp);
5557	VERIFY(mpp->mpp_flags & MPP_ATTACHED);
5558	mpte = mptompte(mpp);
5559	VERIFY(mpte != NULL);
5560	mpte_lock_assert_held(mpte);
5561	mp_tp = mpte->mpte_mptcb;
5562	VERIFY(mp_tp != NULL);
5563
5564	bzero(&mptcpci, sizeof(mptcpci));
5565	mptcpci.mptcpci_state = mp_tp->mpt_state;
5566	mptcpci.mptcpci_flags = mp_tp->mpt_flags;
5567	mptcpci.mptcpci_ltoken = mp_tp->mpt_localtoken;
5568	mptcpci.mptcpci_rtoken = mp_tp->mpt_remotetoken;
5569	mptcpci.mptcpci_notsent_lowat = mp_tp->mpt_notsent_lowat;
5570	mptcpci.mptcpci_snduna = mp_tp->mpt_snduna;
5571	mptcpci.mptcpci_sndnxt = mp_tp->mpt_sndnxt;
5572	mptcpci.mptcpci_sndmax = mp_tp->mpt_sndmax;
5573	mptcpci.mptcpci_lidsn = mp_tp->mpt_local_idsn;
5574	mptcpci.mptcpci_sndwnd = mp_tp->mpt_sndwnd;
5575	mptcpci.mptcpci_rcvnxt = mp_tp->mpt_rcvnxt;
5576	mptcpci.mptcpci_rcvatmark = mp_tp->mpt_rcvnxt;
5577	mptcpci.mptcpci_ridsn = mp_tp->mpt_remote_idsn;
5578	mptcpci.mptcpci_rcvwnd = mp_tp->mpt_rcvwnd;
5579
5580	mptcpci.mptcpci_nflows = mpte->mpte_numflows;
5581	mptcpci.mptcpci_mpte_flags = mpte->mpte_flags;
5582	mptcpci.mptcpci_mpte_addrid = mpte->mpte_addrid_last;
5583	mptcpci.mptcpci_flow_offset =
5584	offsetof(conninfo_mptcp_t, mptcpci_flows);
5585
5586	len = sizeof(flows) mpte->mpte_numflows;
5587	if (mpte->mpte_numflows != `0`) {
5588	flows = _MALLOC(len, M_TEMP, M_WAITOK \| M_ZERO);
5589	if (flows == NULL) {
5590	mpp_unlock(mpp);
5591	break;
5592	}
5593	mptcpci.mptcpci_len = sizeof(mptcpci) +
5594	sizeof(flows) (mptcpci.mptcpci_nflows - `1`);
5595	error = SYSCTL_OUT(req, &mptcpci,
5596	sizeof(mptcpci) - sizeof(mptcp_flow_t));
5597	} else {
5598	mptcpci.mptcpci_len = sizeof(mptcpci);
5599	error = SYSCTL_OUT(req, &mptcpci, sizeof(mptcpci));
5600	}
5601	if (error) {
5602	mpp_unlock(mpp);
5603	FREE(flows, M_TEMP);
5604	break;
5605	}
5606	f = `0`;
5607	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
5608	so = mpts->mpts_socket;
5609	fill_mptcp_subflow(so, &flows[f], mpts);
5610	f++;
5611	}
5612	mpp_unlock(mpp);
5613	if (flows) {
5614	error = SYSCTL_OUT(req, flows, len);
5615	FREE(flows, M_TEMP);
5616	if (error)
5617	break;
5618	}
5619	}
5620	lck_mtx_unlock(&mtcbinfo.mppi_lock);
5621
5622	return (error);
5623	}
5624
5625	SYSCTL_PROC(_net_inet_mptcp, OID_AUTO, pcblist, CTLFLAG_RD \| CTLFLAG_LOCKED,
5626	`0`, `0`, mptcp_pcblist, "S,conninfo_mptcp_t",
5627	"List of active MPTCP connections");
5628
5629	/*
5630	* Set notsent lowat mark on the MPTCB
5631	*/
5632	int
5633	mptcp_set_notsent_lowat(struct mptses mpte, int* optval)
5634	{
5635	struct mptcb *mp_tp = NULL;
5636	int error = `0`;
5637
5638	if (mpte->mpte_mppcb->mpp_flags & MPP_ATTACHED)
5639	mp_tp = mpte->mpte_mptcb;
5640
5641	if (mp_tp)
5642	mp_tp->mpt_notsent_lowat = optval;
5643	else
5644	error = EINVAL;
5645
5646	return (error);
5647	}
5648
5649	u_int32_t
5650	mptcp_get_notsent_lowat(struct mptses *mpte)
5651	{
5652	struct mptcb *mp_tp = NULL;
5653
5654	if (mpte->mpte_mppcb->mpp_flags & MPP_ATTACHED)
5655	mp_tp = mpte->mpte_mptcb;
5656
5657	if (mp_tp)
5658	return (mp_tp->mpt_notsent_lowat);
5659	else
5660	return (`0`);
5661	}
5662
5663	int
5664	mptcp_notsent_lowat_check(struct socket *so)
5665	{
5666	struct mptses *mpte;
5667	struct mppcb *mpp;
5668	struct mptcb *mp_tp;
5669	struct mptsub *mpts;
5670
5671	int notsent = `0`;
5672
5673	mpp = mpsotomppcb(so);
5674	if (mpp == NULL \|\| mpp->mpp_state == MPPCB_STATE_DEAD) {
5675	return (`0`);
5676	}
5677
5678	mpte = mptompte(mpp);
5679	mpte_lock_assert_held(mpte);
5680	mp_tp = mpte->mpte_mptcb;
5681
5682	notsent = so->so_snd.sb_cc;
5683
5684	if ((notsent == `0`) \|\|
5685	((notsent - (mp_tp->mpt_sndnxt - mp_tp->mpt_snduna)) <=
5686	mp_tp->mpt_notsent_lowat)) {
5687	mptcplog((LOG_DEBUG, "MPTCP Sender: "
5688	"lowat %d notsent %d actual %d \n",
5689	mp_tp->mpt_notsent_lowat, notsent,
5690	notsent - (mp_tp->mpt_sndnxt - mp_tp->mpt_snduna)),
5691	MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE);
5692	return (`1`);
5693	}
5694
5695	/ When Nagle's algorithm is not disabled, it is better*
5696	* to wakeup the client even before there is atleast one
5697	* maxseg of data to write.
5698	*/
5699	TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
5700	int retval = `0`;
5701	if (mpts->mpts_flags & MPTSF_ACTIVE) {
5702	struct socket *subf_so = mpts->mpts_socket;
5703	struct tcpcb *tp = intotcpcb(sotoinpcb(subf_so));
5704
5705	notsent = so->so_snd.sb_cc -
5706	(tp->snd_nxt - tp->snd_una);
5707
5708	if ((tp->t_flags & TF_NODELAY) == `0` &&
5709	notsent > `0` && (notsent <= (int)tp->t_maxseg)) {
5710	retval = `1`;
5711	}
5712	mptcplog((LOG_DEBUG, "MPTCP Sender: lowat %d notsent %d"
5713	" nodelay false \n",
5714	mp_tp->mpt_notsent_lowat, notsent),
5715	MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE);
5716	return (retval);
5717	}
5718	}
5719	return (`0`);
5720	}
5721
5722	/ Using Symptoms Advisory to detect poor WiFi or poor Cell /
5723	static kern_ctl_ref mptcp_kern_ctrl_ref = NULL;
5724	static uint32_t mptcp_kern_skt_inuse = `0`;
5725	static uint32_t mptcp_kern_skt_unit;
5726	symptoms_advisory_t mptcp_advisory;
5727
5728	static errno_t
5729	mptcp_symptoms_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
5730	void **unitinfo)
5731	{
5732	#pragma unused(kctlref, sac, unitinfo)
5733
5734	if (OSIncrementAtomic(&mptcp_kern_skt_inuse) > `0`)
5735	os_log_error(mptcp_log_handle, "%s MPTCP kernel-control socket for Symptoms already open!", __func__);
5736
5737	mptcp_kern_skt_unit = sac->sc_unit;
5738
5739	return (`0`);
5740	}
5741
5742	static void
5743	mptcp_allow_uuid(uuid_t uuid)
5744	{
5745	struct mppcb *mpp;
5746
5747	/ Iterate over all MPTCP connections /
5748
5749	lck_mtx_lock(&mtcbinfo.mppi_lock);
5750
5751	TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) {
5752	struct mptses *mpte;
5753	struct socket *mp_so;
5754
5755	mpp_lock(mpp);
5756
5757	mpte = mpp->mpp_pcbe;
5758	mp_so = mpp->mpp_socket;
5759
5760	if (mp_so->so_flags & SOF_DELEGATED &&
5761	uuid_compare(uuid, mp_so->e_uuid))
5762	goto next;
5763	else if (!(mp_so->so_flags & SOF_DELEGATED) &&
5764	uuid_compare(uuid, mp_so->last_uuid))
5765	goto next;
5766
5767	mpte->mpte_flags \|= MPTE_ACCESS_GRANTED;
5768
5769	mptcp_check_subflows_and_add(mpte);
5770	mptcp_remove_subflows(mpte);
5771
5772	mpte->mpte_flags &= ~MPTE_ACCESS_GRANTED;
5773
5774	next:
5775	mpp_unlock(mpp);
5776	}
5777
5778	lck_mtx_unlock(&mtcbinfo.mppi_lock);
5779	}
5780
5781	static void
5782	mptcp_wifi_status_changed(void)
5783	{
5784	struct mppcb *mpp;
5785
5786	/ Iterate over all MPTCP connections /
5787
5788	lck_mtx_lock(&mtcbinfo.mppi_lock);
5789
5790	TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) {
5791	struct mptses *mpte;
5792	struct socket *mp_so;
5793
5794	mpp_lock(mpp);
5795
5796	mpte = mpp->mpp_pcbe;
5797	mp_so = mpp->mpp_socket;
5798
5799	/ Only handover-mode is purely driven by Symptom's Wi-Fi status /
5800	if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER)
5801	goto next;
5802
5803	mptcp_check_subflows_and_add(mpte);
5804	mptcp_check_subflows_and_remove(mpte);
5805
5806	next:
5807	mpp_unlock(mpp);
5808	}
5809
5810	lck_mtx_unlock(&mtcbinfo.mppi_lock);
5811	}
5812
5813	void
5814	mptcp_ask_symptoms(struct mptses *mpte)
5815	{
5816	struct mptcp_symptoms_ask_uuid ask;
5817	struct socket *mp_so;
5818	struct proc *p;
5819	int pid, prio, err;
5820
5821	if (mptcp_kern_skt_unit == `0`) {
5822	os_log_error(mptcp_log_handle, "%s skt_unit is still 0\n", __func__);
5823	return;
5824	}
5825
5826	mp_so = mptetoso(mpte);
5827
5828	if (mp_so->so_flags & SOF_DELEGATED)
5829	pid = mp_so->e_pid;
5830	else
5831	pid = mp_so->last_pid;
5832
5833	p = proc_find(pid);
5834	if (p == PROC_NULL) {
5835	os_log_error(mptcp_log_handle, "%s Couldn't find proc for pid %u\n", __func__, pid);
5836	return;
5837	}
5838
5839	ask.cmd = MPTCP_SYMPTOMS_ASK_UUID;
5840
5841	if (mp_so->so_flags & SOF_DELEGATED)
5842	uuid_copy(ask.uuid, mp_so->e_uuid);
5843	else
5844	uuid_copy(ask.uuid, mp_so->last_uuid);
5845
5846	prio = proc_get_effective_task_policy(proc_task(p), TASK_POLICY_ROLE);
5847
5848	if (prio == TASK_BACKGROUND_APPLICATION)
5849	ask.priority = MPTCP_SYMPTOMS_BACKGROUND;
5850	else if (prio == TASK_FOREGROUND_APPLICATION)
5851	ask.priority = MPTCP_SYMPTOMS_FOREGROUND;
5852	else
5853	ask.priority = MPTCP_SYMPTOMS_UNKNOWN;
5854
5855	err = ctl_enqueuedata(mptcp_kern_ctrl_ref, mptcp_kern_skt_unit,
5856	&ask, sizeof(ask), CTL_DATA_EOR);
5857
5858	os_log_debug(mptcp_log_handle, "%s asked symptoms about pid %u, prio %u, err %d\n",
5859	__func__, pid, ask.priority, err);
5860
5861
5862	proc_rele(p);
5863	}
5864
5865	static errno_t
5866	mptcp_symptoms_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t kcunit,
5867	void *unitinfo)
5868	{
5869	#pragma unused(kctlref, kcunit, unitinfo)
5870
5871	OSDecrementAtomic(&mptcp_kern_skt_inuse);
5872
5873	return (`0`);
5874	}
5875
5876	static errno_t
5877	mptcp_symptoms_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo,
5878	mbuf_t m, int flags)
5879	{
5880	#pragma unused(kctlref, unitinfo, flags)
5881	symptoms_advisory_t *sa = NULL;
5882
5883	if (kcunit != mptcp_kern_skt_unit)
5884	os_log_error(mptcp_log_handle, "%s kcunit %u is different from expected one %u\n",
5885	__func__, kcunit, mptcp_kern_skt_unit);
5886
5887	if (mbuf_pkthdr_len(m) < sizeof(*sa)) {
5888	mbuf_freem(m);
5889	return (EINVAL);
5890	}
5891
5892	if (mbuf_len(m) < sizeof(*sa)) {
5893	mbuf_freem(m);
5894	return (EINVAL);
5895	}
5896
5897	sa = mbuf_data(m);
5898
5899	if (sa->sa_nwk_status != SYMPTOMS_ADVISORY_NOCOMMENT &&
5900	sa->sa_nwk_status != SYMPTOMS_ADVISORY_USEAPP) {
5901	uint8_t old_wifi_status = mptcp_advisory.sa_wifi_status;
5902
5903	mptcplog((LOG_DEBUG, "%s: wifi %d,%d\n",
5904	__func__, sa->sa_wifi_status, mptcp_advisory.sa_wifi_status),
5905	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE);
5906
5907	if ((sa->sa_wifi_status &
5908	(SYMPTOMS_ADVISORY_WIFI_BAD \| SYMPTOMS_ADVISORY_WIFI_OK)) !=
5909	(SYMPTOMS_ADVISORY_WIFI_BAD \| SYMPTOMS_ADVISORY_WIFI_OK))
5910	mptcp_advisory.sa_wifi_status = sa->sa_wifi_status;
5911
5912	if (old_wifi_status != mptcp_advisory.sa_wifi_status)
5913	mptcp_wifi_status_changed();
5914	} else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_NOCOMMENT) {
5915	mptcplog((LOG_DEBUG, "%s: NOCOMMENT wifi %d\n", __func__,
5916	mptcp_advisory.sa_wifi_status),
5917	MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE);
5918	} else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_USEAPP) {
5919	uuid_t uuid;
5920
5921	mptcplog((LOG_DEBUG, "%s Got response about useApp\n", __func__),
5922	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
5923
5924	uuid_copy(uuid, (unsigned char *)(sa + `1`));
5925
5926	mptcp_allow_uuid(uuid);
5927	}
5928
5929	mbuf_freem(m);
5930	return (`0`);
5931	}
5932
5933	void
5934	mptcp_control_register(void)
5935	{
5936	/ Set up the advisory control socket /
5937	struct kern_ctl_reg mptcp_kern_ctl;
5938
5939	bzero(&mptcp_kern_ctl, sizeof(mptcp_kern_ctl));
5940	strlcpy(mptcp_kern_ctl.ctl_name, MPTCP_KERN_CTL_NAME,
5941	sizeof(mptcp_kern_ctl.ctl_name));
5942	mptcp_kern_ctl.ctl_connect = mptcp_symptoms_ctl_connect;
5943	mptcp_kern_ctl.ctl_disconnect = mptcp_symptoms_ctl_disconnect;
5944	mptcp_kern_ctl.ctl_send = mptcp_symptoms_ctl_send;
5945	mptcp_kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED;
5946
5947	(void)ctl_register(&mptcp_kern_ctl, &mptcp_kern_ctrl_ref);
5948	}
5949
5950	/*
5951	* Three return-values:
5952	* 1 : WiFi is bad
5953	* 0 : WiFi is good
5954	* -1 : WiFi-state is unknown, use subflow-only heuristics
5955	*/
5956	int
5957	mptcp_is_wifi_unusable(struct mptses *mpte)
5958	{
5959	if (mpte->mpte_flags & MPTE_FIRSTPARTY) {
5960	if (mptcp_advisory.sa_wifi_status)
5961	return ((mptcp_advisory.sa_wifi_status & SYMPTOMS_ADVISORY_WIFI_BAD) ? `1` : `0`);
5962
5963	/*
5964	* If it's a first-party app and we don't have any info
5965	* about the Wi-Fi state, let's be pessimistic.
5966	*/
5967	return (-`1`);
5968	}
5969
5970	return ((mptcp_advisory.sa_wifi_status & SYMPTOMS_ADVISORY_WIFI_BAD) ? `1` : `0`);
5971	}
5972
5973	boolean_t
5974	mptcp_subflow_is_bad(struct mptses mpte, struct* mptsub *mpts)
5975	{
5976	struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
5977	int fail_thresh = mptcp_fail_thresh;
5978
5979	if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER)
5980	fail_thresh *= `2`;
5981
5982	return (tp->t_rxtshift >= fail_thresh &&
5983	(mptetoso(mpte)->so_snd.sb_cc \|\| mpte->mpte_reinjectq));
5984	}
5985
5986	/ If TFO data is succesfully acked, it must be dropped from the mptcp so /
5987	static void
5988	mptcp_drop_tfo_data(struct mptses mpte, struct* mptsub *mpts)
5989	{
5990	struct socket *mp_so = mptetoso(mpte);
5991	struct socket *so = mpts->mpts_socket;
5992	struct tcpcb *tp = intotcpcb(sotoinpcb(so));
5993	struct mptcb *mp_tp = mpte->mpte_mptcb;
5994
5995	/ If data was sent with SYN, rewind state /
5996	if (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) {
5997	u_int64_t mp_droplen = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna;
5998	unsigned int tcp_droplen = tp->snd_una - tp->iss - `1`;
5999
6000	VERIFY(mp_droplen <= (UINT_MAX));
6001	VERIFY(mp_droplen >= tcp_droplen);
6002
6003	mpts->mpts_flags &= ~MPTSF_TFO_REQD;
6004	mpts->mpts_iss += tcp_droplen;
6005	tp->t_mpflags &= ~TMPF_TFO_REQUEST;
6006
6007	if (mp_droplen > tcp_droplen) {
6008	/ handle partial TCP ack /
6009	mp_so->so_flags1 \|= SOF1_TFO_REWIND;
6010	mp_tp->mpt_sndnxt = mp_tp->mpt_snduna + (mp_droplen - tcp_droplen);
6011	mp_droplen = tcp_droplen;
6012	} else {
6013	/ all data on SYN was acked /
6014	mpts->mpts_rel_seq = `1`;
6015	mp_tp->mpt_sndnxt = mp_tp->mpt_snduna;
6016	}
6017	mp_tp->mpt_sndmax -= tcp_droplen;
6018
6019	if (mp_droplen != `0`) {
6020	VERIFY(mp_so->so_snd.sb_mb != NULL);
6021	sbdrop(&mp_so->so_snd, (int)mp_droplen);
6022	}
6023	mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx cid %d TFO tcp len %d mptcp len %d\n",
6024	__func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so),
6025	mpts->mpts_connid, tcp_droplen, mp_droplen),
6026	MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE);
6027	}
6028	}
6029
6030	int
6031	mptcp_freeq(struct mptcb *mp_tp)
6032	{
6033	struct tseg_qent *q;
6034	int rv = `0`;
6035
6036	while ((q = LIST_FIRST(&mp_tp->mpt_segq)) != NULL) {
6037	LIST_REMOVE(q, tqe_q);
6038	m_freem(q->tqe_m);
6039	zfree(tcp_reass_zone, q);
6040	rv = `1`;
6041	}
6042	mp_tp->mpt_reassqlen = `0`;
6043	return (rv);
6044	}
6045
6046	static int
6047	mptcp_post_event(u_int32_t event_code, int value)
6048	{
6049	struct kev_mptcp_data event_data;
6050	struct kev_msg ev_msg;
6051
6052	memset(&ev_msg, `0`, sizeof(ev_msg));
6053
6054	ev_msg.vendor_code = KEV_VENDOR_APPLE;
6055	ev_msg.kev_class = KEV_NETWORK_CLASS;
6056	ev_msg.kev_subclass = KEV_MPTCP_SUBCLASS;
6057	ev_msg.event_code = event_code;
6058
6059	event_data.value = value;
6060
6061	ev_msg.dv[`0`].data_ptr = &event_data;
6062	ev_msg.dv[`0`].data_length = sizeof(event_data);
6063
6064	return kev_post_msg(&ev_msg);
6065	}
6066
6067	void
6068	mptcp_set_cellicon(struct mptses *mpte)
6069	{
6070	int error;
6071
6072	/ First-party apps (Siri) don't flip the cellicon /
6073	if (mpte->mpte_flags & MPTE_FIRSTPARTY)
6074	return;
6075
6076	/ Remember the last time we set the cellicon (see mptcp_unset_cellicon) /
6077	mptcp_last_cellicon_set = tcp_now;
6078
6079	/ If cellicon is already set, get out of here! /
6080	if (OSTestAndSet(`7`, &mptcp_cellicon_is_set))
6081	return;
6082
6083	error = mptcp_post_event(KEV_MPTCP_CELLUSE, `1`);
6084
6085	if (error)
6086	mptcplog((LOG_ERR, "%s: Setting cellicon failed with %d\n",
6087	__func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
6088	else
6089	mptcplog((LOG_DEBUG, "%s successfully set the cellicon\n", __func__),
6090	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
6091	}
6092
6093	void
6094	mptcp_unset_cellicon(void)
6095	{
6096	int error;
6097
6098	/ If cellicon is already unset, get out of here! /
6099	if (OSTestAndClear(`7`, &mptcp_cellicon_is_set))
6100	return;
6101
6102	/*
6103	* If during the past MPTCP_CELLICON_TOGGLE_RATE seconds we didn't
6104	* explicitly set the cellicon (see mptcp_set_cellicon()), then we unset
6105	* it again.
6106	*/
6107	if (TSTMP_GT(mptcp_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE,
6108	tcp_now)) {
6109	OSTestAndSet(`7`, &mptcp_cellicon_is_set);
6110	return;
6111	}
6112
6113	error = mptcp_post_event(KEV_MPTCP_CELLUSE, `0`);
6114
6115	if (error)
6116	mptcplog((LOG_ERR, "%s: Unsetting cellicon failed with %d\n",
6117	__func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR);
6118	else
6119	mptcplog((LOG_DEBUG, "%s successfully unset the cellicon\n", __func__),
6120	MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE);
6121	}
6122
6123	void
6124	mptcp_reset_rexmit_state(struct tcpcb *tp)
6125	{
6126	struct mptsub *mpts;
6127	struct inpcb *inp;
6128	struct socket *so;
6129
6130	inp = tp->t_inpcb;
6131	if (inp == NULL)
6132	return;
6133
6134	so = inp->inp_socket;
6135	if (so == NULL)
6136	return;
6137
6138	if (!(so->so_flags & SOF_MP_SUBFLOW))
6139	return;
6140
6141	mpts = tp->t_mpsub;
6142
6143	mpts->mpts_flags &= ~MPTSF_WRITE_STALL;
6144	so->so_flags &= ~SOF_MP_TRYFAILOVER;
6145	}
6146
6147	void
6148	mptcp_reset_keepalive(struct tcpcb *tp)
6149	{
6150	struct mptsub *mpts = tp->t_mpsub;
6151
6152	mpts->mpts_flags &= ~MPTSF_READ_STALL;
6153	}
6154
6155

Browse the source code of codebrowser/bsd/netinet/mptcp_subr.c