1/*
2 * Copyright (c) 2011-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <stdint.h>
29#include <sys/param.h>
30#include <sys/mount_internal.h>
31#include <sys/malloc.h>
32#include <sys/queue.h>
33
34#include <libkern/libkern.h>
35#include <libkern/OSAtomic.h>
36#include <kern/debug.h>
37#include <kern/thread.h>
38
39#include <nfs/rpcv2.h>
40#include <nfs/nfsproto.h>
41#include <nfs/nfs.h>
42
43#ifdef NFS_UC_DEBUG
44#define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
45#else
46#define DPRINT(fmt, ...)
47#endif
48
49struct nfsrv_uc_arg {
50 TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
51 socket_t nua_so;
52 struct nfsrv_sock *nua_slp;
53 int nua_waitflag; /* Should always be MBUF_DONTWAIT */
54 uint32_t nua_flags;
55 uint32_t nua_qi;
56};
57#define NFS_UC_QUEUED 0x0001
58
59#define NFS_UC_HASH_SZ 7
60#define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
61
62TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);
63
64static struct nfsrv_uc_queue {
65 lck_mtx_t *ucq_lock;
66 struct nfsrv_uc_q ucq_queue[1];
67 thread_t ucq_thd;
68 uint32_t ucq_flags;
69} nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
70#define NFS_UC_QUEUE_SLEEPING 0x0001
71
72static lck_grp_t *nfsrv_uc_group;
73static lck_mtx_t *nfsrv_uc_shutdown_lock;
74static volatile int nfsrv_uc_shutdown = 0;
75static int32_t nfsrv_uc_thread_count;
76
77extern kern_return_t thread_terminate(thread_t);
78
79#ifdef NFS_UC_Q_DEBUG
80int nfsrv_uc_use_proxy = 1;
81uint32_t nfsrv_uc_queue_limit;
82uint32_t nfsrv_uc_queue_max_seen;
83volatile uint32_t nfsrv_uc_queue_count;
84#endif
85
86/*
87 * Thread that dequeues up-calls and runs the nfsrv_rcv routine
88 */
89static void
90nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
91{
92 int qi = (int)(uintptr_t)arg;
93 int error;
94 struct nfsrv_uc_arg *ep = NULL;
95 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
96
97 DPRINT("nfsrv_uc_thread %d started\n", qi);
98 while (!nfsrv_uc_shutdown) {
99 lck_mtx_lock(myqueue->ucq_lock);
100
101 while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
102 myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING;
103 error = msleep(myqueue, myqueue->ucq_lock, PSOCK, "nfsd_upcall_handler", NULL);
104 myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
105 if (error) {
106 printf("nfsrv_uc_thread received error %d\n", error);
107 }
108 }
109 if (nfsrv_uc_shutdown) {
110 lck_mtx_unlock(myqueue->ucq_lock);
111 break;
112 }
113
114
115 ep = TAILQ_FIRST(myqueue->ucq_queue);
116 DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);
117
118 TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);
119
120 ep->nua_flags &= ~NFS_UC_QUEUED;
121
122 lck_mtx_unlock(myqueue->ucq_lock);
123
124#ifdef NFS_UC_Q_DEBUG
125 OSDecrementAtomic(&nfsrv_uc_queue_count);
126#endif
127
128 DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
129 nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
130 }
131
132 lck_mtx_lock(nfsrv_uc_shutdown_lock);
133 nfsrv_uc_thread_count--;
134 wakeup(&nfsrv_uc_thread_count);
135 lck_mtx_unlock(nfsrv_uc_shutdown_lock);
136
137 thread_terminate(current_thread());
138}
139
140/*
141 * Dequeue a closed nfsrv_sock if needed from the up-call queue.
142 * Call from nfsrv_zapsock
143 */
144void
145nfsrv_uc_dequeue(struct nfsrv_sock *slp)
146{
147 struct nfsrv_uc_arg *ap = slp->ns_ua;
148 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];
149
150 /*
151 * We assume that the socket up-calls have been stop and the socket
152 * is shutting down so no need for acquiring the lock to check that
153 * the flag is cleared.
154 */
155 if (ap == NULL || (ap->nua_flags & NFS_UC_QUEUED) == 0)
156 return;
157 /* If we're queued we might race with nfsrv_uc_thread */
158 lck_mtx_lock(myqueue->ucq_lock);
159 if (ap->nua_flags & NFS_UC_QUEUED) {
160 printf("nfsrv_uc_dequeue remove %p\n", ap);
161 TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
162 ap->nua_flags &= ~NFS_UC_QUEUED;
163#ifdef NFS_UC_Q_DEBUG
164 OSDecrementAtomic(&nfsrv_uc_queue_count);
165#endif
166 }
167 FREE(slp->ns_ua, M_TEMP);
168 slp->ns_ua = NULL;
169 lck_mtx_unlock(myqueue->ucq_lock);
170}
171
172/*
173 * Allocate and initialize globals for nfsrv_sock up-call support.
174 */
175void
176nfsrv_uc_init(void)
177{
178 int i;
179
180 nfsrv_uc_group = lck_grp_alloc_init("nfs_upcall_locks", LCK_GRP_ATTR_NULL);
181 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
182 TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
183 nfsrv_uc_queue_tbl[i].ucq_lock = lck_mtx_alloc_init(nfsrv_uc_group, LCK_ATTR_NULL);
184 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
185 nfsrv_uc_queue_tbl[i].ucq_flags = 0;
186 }
187 nfsrv_uc_shutdown_lock = lck_mtx_alloc_init(nfsrv_uc_group, LCK_ATTR_NULL);
188}
189
190/*
191 * Start up-call threads to service nfsrv_sock(s)
192 * Called from the first call of nfsrv_uc_addsock
193 */
194static void
195nfsrv_uc_start(void)
196{
197 int32_t i;
198 int error;
199
200#ifdef NFS_UC_Q_DEBUG
201 if (!nfsrv_uc_use_proxy)
202 return;
203#endif
204 DPRINT("nfsrv_uc_start\n");
205
206 /* Wait until previous shutdown finishes */
207 lck_mtx_lock(nfsrv_uc_shutdown_lock);
208 while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0)
209 msleep(&nfsrv_uc_thread_count, nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_wait", NULL);
210
211 /* Start up-call threads */
212 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
213 error = kernel_thread_start(nfsrv_uc_thread, (void *)(uintptr_t)i, &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
214 if (!error) {
215 nfsrv_uc_thread_count++;
216 } else {
217 printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
218 }
219 }
220 if (nfsrv_uc_thread_count == 0) {
221 printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
222 goto out;
223 }
224
225out:
226#ifdef NFS_UC_Q_DEBUG
227 nfsrv_uc_queue_count = 0ULL;
228 nfsrv_uc_queue_max_seen = 0ULL;
229#endif
230 lck_mtx_unlock(nfsrv_uc_shutdown_lock);
231}
232
233/*
234 * Stop the up-call threads.
235 * Called from nfsrv_uc_cleanup.
236 */
237static void
238nfsrv_uc_stop(void)
239{
240 int32_t i;
241 int32_t thread_count = nfsrv_uc_thread_count;
242
243 DPRINT("Entering nfsrv_uc_stop\n");
244
245 /* Signal up-call threads to stop */
246 nfsrv_uc_shutdown = 1;
247 for (i = 0; i < thread_count; i++) {
248 lck_mtx_lock(nfsrv_uc_queue_tbl[i].ucq_lock);
249 wakeup(&nfsrv_uc_queue_tbl[i]);
250 lck_mtx_unlock(nfsrv_uc_queue_tbl[i].ucq_lock);
251 }
252
253 /* Wait until they are done shutting down */
254 lck_mtx_lock(nfsrv_uc_shutdown_lock);
255 while (nfsrv_uc_thread_count > 0)
256 msleep(&nfsrv_uc_thread_count, nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_stop", NULL);
257
258 /* Deallocate old threads */
259 for (i = 0; i < nfsrv_uc_thread_count; i++) {
260 if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL)
261 thread_deallocate(nfsrv_uc_queue_tbl[i].ucq_thd);
262 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
263 }
264
265 /* Enable restarting */
266 nfsrv_uc_shutdown = 0;
267 lck_mtx_unlock(nfsrv_uc_shutdown_lock);
268}
269
270/*
271 * Shutdown up-calls for nfsrv_socks.
272 * Make sure nothing is queued on the up-call queues
273 * Shutdown the up-call threads
274 * Called from nfssvc_cleanup.
275 */
276void
277nfsrv_uc_cleanup(void)
278{
279 int i;
280
281 DPRINT("Entering nfsrv_uc_cleanup\n");
282
283 /*
284 * Every thing should be dequeued at this point or will be as sockets are closed
285 * but to be safe, we'll make sure.
286 */
287 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
288 struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];
289
290 lck_mtx_lock(queue->ucq_lock);
291 while (!TAILQ_EMPTY(queue->ucq_queue)) {
292 struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
293 TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
294 ep->nua_flags &= ~NFS_UC_QUEUED;
295 }
296 lck_mtx_unlock(queue->ucq_lock);
297 }
298
299 nfsrv_uc_stop();
300}
301
302/*
303 * This is the nfs up-call routine for server sockets.
304 * We used to set nfsrv_rcv as the up-call routine, but
305 * recently that seems like we are doing to much work for
306 * the interface thread, so we just queue the arguments
307 * that we would have gotten for nfsrv_rcv and let a
308 * worker thread dequeue them and pass them on to nfsrv_rcv.
309 */
310static void
311nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
312{
313 struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg;
314 int qi = uap->nua_qi;
315 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
316
317 lck_mtx_lock(myqueue->ucq_lock);
318 DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
319 DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
320 if (uap == NULL || uap->nua_flags & NFS_UC_QUEUED) {
321 lck_mtx_unlock(myqueue->ucq_lock);
322 return; /* Already queued or freed */
323 }
324
325 uap->nua_so = so;
326 uap->nua_waitflag = waitflag;
327
328 TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);
329
330 uap->nua_flags |= NFS_UC_QUEUED;
331 if (myqueue->ucq_flags | NFS_UC_QUEUE_SLEEPING)
332 wakeup(myqueue);
333
334#ifdef NFS_UC_Q_DEBUG
335 {
336 uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);
337
338 /* This is a bit racey but just for debug */
339 if (count > nfsrv_uc_queue_max_seen)
340 nfsrv_uc_queue_max_seen = count;
341
342 if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
343 panic("nfsd up-call queue limit exceeded\n");
344 }
345 }
346#endif
347 lck_mtx_unlock(myqueue->ucq_lock);
348}
349
350
351/*
352 * Set the up-call routine on the socket associated with the passed in
353 * nfsrv_sock.
354 * Assumes nfsd_mutex is held.
355 */
356void
357nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
358{
359 int on = 1;
360 struct nfsrv_uc_arg *arg;
361
362 if (start && nfsrv_uc_thread_count == 0)
363 nfsrv_uc_start();
364
365 /*
366 * We don't take a lock since once we're up nfsrv_uc_thread_count does
367 * not change until shutdown and then we should not be adding sockets to
368 * generate up-calls.
369 */
370 if (nfsrv_uc_thread_count) {
371 MALLOC(arg, struct nfsrv_uc_arg *, sizeof (struct nfsrv_uc_arg), M_TEMP, M_WAITOK | M_ZERO);
372 if (arg == NULL)
373 goto direct;
374
375 slp->ns_ua = arg;
376 arg->nua_slp = slp;
377 arg->nua_qi = NFS_UC_HASH(slp);
378
379 sock_setupcall(slp->ns_so, nfsrv_uc_proxy, arg);
380 } else {
381direct:
382 slp->ns_ua = NULL;
383 DPRINT("setting nfsrv_rcv up-call\n");
384 sock_setupcall(slp->ns_so, nfsrv_rcv, slp);
385 }
386
387 /* just playin' it safe */
388 sock_setsockopt(slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, &on, sizeof(on));
389
390 return;
391}
392
393