1/*
2 * Copyright (c) 2000-2016 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/proc_internal.h>
38#include <sys/user.h>
39#include <sys/file_internal.h>
40#include <sys/vnode.h>
41#include <sys/kernel.h>
42
43#include <kern/queue.h>
44#include <sys/lock.h>
45#include <kern/thread.h>
46#include <kern/sched_prim.h>
47#include <kern/ast.h>
48
49#include <kern/cpu_number.h>
50#include <vm/vm_kern.h>
51
52#include <kern/task.h>
53#include <mach/time_value.h>
54#include <kern/locks.h>
55#include <kern/policy_internal.h>
56
57#include <sys/systm.h> /* for unix_syscall_return() */
58#include <libkern/OSAtomic.h>
59
60extern void compute_averunnable(void *); /* XXX */
61
62__attribute__((noreturn))
63static void
64_sleep_continue( __unused void *parameter, wait_result_t wresult)
65{
66 struct proc *p = current_proc();
67 thread_t self = current_thread();
68 struct uthread * ut;
69 int sig, catch;
70 int error = 0;
71 int dropmutex, spinmutex;
72
73 ut = get_bsdthread_info(self);
74 catch = ut->uu_pri & PCATCH;
75 dropmutex = ut->uu_pri & PDROP;
76 spinmutex = ut->uu_pri & PSPIN;
77
78 switch (wresult) {
79 case THREAD_TIMED_OUT:
80 error = EWOULDBLOCK;
81 break;
82 case THREAD_AWAKENED:
83 /*
84 * Posix implies any signal should be delivered
85 * first, regardless of whether awakened due
86 * to receiving event.
87 */
88 if (!catch)
89 break;
90 /* else fall through */
91 case THREAD_INTERRUPTED:
92 if (catch) {
93 if (thread_should_abort(self)) {
94 error = EINTR;
95 } else if (SHOULDissignal(p,ut)) {
96 if ((sig = CURSIG(p)) != 0) {
97 if (p->p_sigacts->ps_sigintr & sigmask(sig))
98 error = EINTR;
99 else
100 error = ERESTART;
101 }
102 if (thread_should_abort(self)) {
103 error = EINTR;
104 }
105 } else if( (ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
106 /* due to thread cancel */
107 error = EINTR;
108 }
109 } else
110 error = EINTR;
111 break;
112 }
113
114 if (error == EINTR || error == ERESTART)
115 act_set_astbsd(self);
116
117 if (ut->uu_mtx && !dropmutex) {
118 if (spinmutex)
119 lck_mtx_lock_spin(ut->uu_mtx);
120 else
121 lck_mtx_lock(ut->uu_mtx);
122 }
123 ut->uu_wchan = NULL;
124 ut->uu_wmesg = NULL;
125
126 unix_syscall_return((*ut->uu_continuation)(error));
127}
128
129/*
130 * Give up the processor till a wakeup occurs
131 * on chan, at which time the process
132 * enters the scheduling queue at priority pri.
133 * The most important effect of pri is that when
134 * pri<=PZERO a signal cannot disturb the sleep;
135 * if pri>PZERO signals will be processed.
136 * If pri&PCATCH is set, signals will cause sleep
137 * to return 1, rather than longjmp.
138 * Callers of this routine must be prepared for
139 * premature return, and check that the reason for
140 * sleeping has gone away.
141 *
142 * if msleep was the entry point, than we have a mutex to deal with
143 *
144 * The mutex is unlocked before the caller is blocked, and
145 * relocked before msleep returns unless the priority includes the PDROP
146 * flag... if PDROP is specified, _sleep returns with the mutex unlocked
147 * regardless of whether it actually blocked or not.
148 */
149
150static int
151_sleep(
152 caddr_t chan,
153 int pri,
154 const char *wmsg,
155 u_int64_t abstime,
156 int (*continuation)(int),
157 lck_mtx_t *mtx)
158{
159 struct proc *p;
160 thread_t self = current_thread();
161 struct uthread * ut;
162 int sig, catch;
163 int dropmutex = pri & PDROP;
164 int spinmutex = pri & PSPIN;
165 int wait_result;
166 int error = 0;
167
168 ut = get_bsdthread_info(self);
169
170 p = current_proc();
171 p->p_priority = pri & PRIMASK;
172 /* It can still block in proc_exit() after the teardown. */
173 if (p->p_stats != NULL)
174 OSIncrementAtomicLong(&p->p_stats->p_ru.ru_nvcsw);
175
176 if (pri & PCATCH)
177 catch = THREAD_ABORTSAFE;
178 else
179 catch = THREAD_UNINT;
180
181 /* set wait message & channel */
182 ut->uu_wchan = chan;
183 ut->uu_wmesg = wmsg ? wmsg : "unknown";
184
185 if (mtx != NULL && chan != NULL && (thread_continue_t)continuation == THREAD_CONTINUE_NULL) {
186 int flags;
187
188 if (dropmutex)
189 flags = LCK_SLEEP_UNLOCK;
190 else
191 flags = LCK_SLEEP_DEFAULT;
192
193 if (spinmutex)
194 flags |= LCK_SLEEP_SPIN;
195
196 if (abstime)
197 wait_result = lck_mtx_sleep_deadline(mtx, flags, chan, catch, abstime);
198 else
199 wait_result = lck_mtx_sleep(mtx, flags, chan, catch);
200 }
201 else {
202 if (chan != NULL)
203 assert_wait_deadline(chan, catch, abstime);
204 if (mtx)
205 lck_mtx_unlock(mtx);
206
207 if (catch == THREAD_ABORTSAFE) {
208 if (SHOULDissignal(p,ut)) {
209 if ((sig = CURSIG(p)) != 0) {
210 if (clear_wait(self, THREAD_INTERRUPTED) == KERN_FAILURE)
211 goto block;
212 if (p->p_sigacts->ps_sigintr & sigmask(sig))
213 error = EINTR;
214 else
215 error = ERESTART;
216 if (mtx && !dropmutex) {
217 if (spinmutex)
218 lck_mtx_lock_spin(mtx);
219 else
220 lck_mtx_lock(mtx);
221 }
222 goto out;
223 }
224 }
225 if (thread_should_abort(self)) {
226 if (clear_wait(self, THREAD_INTERRUPTED) == KERN_FAILURE)
227 goto block;
228 error = EINTR;
229
230 if (mtx && !dropmutex) {
231 if (spinmutex)
232 lck_mtx_lock_spin(mtx);
233 else
234 lck_mtx_lock(mtx);
235 }
236 goto out;
237 }
238 }
239
240
241block:
242 if ((thread_continue_t)continuation != THREAD_CONTINUE_NULL) {
243 ut->uu_continuation = continuation;
244 ut->uu_pri = pri;
245 ut->uu_timo = abstime? 1: 0;
246 ut->uu_mtx = mtx;
247 (void) thread_block(_sleep_continue);
248 /* NOTREACHED */
249 }
250
251 wait_result = thread_block(THREAD_CONTINUE_NULL);
252
253 if (mtx && !dropmutex) {
254 if (spinmutex)
255 lck_mtx_lock_spin(mtx);
256 else
257 lck_mtx_lock(mtx);
258 }
259 }
260
261 switch (wait_result) {
262 case THREAD_TIMED_OUT:
263 error = EWOULDBLOCK;
264 break;
265 case THREAD_AWAKENED:
266 case THREAD_RESTART:
267 /*
268 * Posix implies any signal should be delivered
269 * first, regardless of whether awakened due
270 * to receiving event.
271 */
272 if (catch != THREAD_ABORTSAFE)
273 break;
274 /* else fall through */
275 case THREAD_INTERRUPTED:
276 if (catch == THREAD_ABORTSAFE) {
277 if (thread_should_abort(self)) {
278 error = EINTR;
279 } else if (SHOULDissignal(p, ut)) {
280 if ((sig = CURSIG(p)) != 0) {
281 if (p->p_sigacts->ps_sigintr & sigmask(sig))
282 error = EINTR;
283 else
284 error = ERESTART;
285 }
286 if (thread_should_abort(self)) {
287 error = EINTR;
288 }
289 } else if( (ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
290 /* due to thread cancel */
291 error = EINTR;
292 }
293 } else
294 error = EINTR;
295 break;
296 }
297out:
298 if (error == EINTR || error == ERESTART)
299 act_set_astbsd(self);
300 ut->uu_wchan = NULL;
301 ut->uu_wmesg = NULL;
302
303 return (error);
304}
305
306int
307sleep(
308 void *chan,
309 int pri)
310{
311 return _sleep((caddr_t)chan, pri, (char *)NULL, 0, (int (*)(int))0, (lck_mtx_t *)0);
312}
313
314int
315msleep0(
316 void *chan,
317 lck_mtx_t *mtx,
318 int pri,
319 const char *wmsg,
320 int timo,
321 int (*continuation)(int))
322{
323 u_int64_t abstime = 0;
324
325 if (timo)
326 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
327
328 return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, mtx);
329}
330
331int
332msleep(
333 void *chan,
334 lck_mtx_t *mtx,
335 int pri,
336 const char *wmsg,
337 struct timespec *ts)
338{
339 u_int64_t abstime = 0;
340
341 if (ts && (ts->tv_sec || ts->tv_nsec)) {
342 nanoseconds_to_absolutetime((uint64_t)ts->tv_sec * NSEC_PER_SEC + ts->tv_nsec, &abstime );
343 clock_absolutetime_interval_to_deadline( abstime, &abstime );
344 }
345
346 return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, mtx);
347}
348
349int
350msleep1(
351 void *chan,
352 lck_mtx_t *mtx,
353 int pri,
354 const char *wmsg,
355 u_int64_t abstime)
356{
357 return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, mtx);
358}
359
360int
361tsleep(
362 void *chan,
363 int pri,
364 const char *wmsg,
365 int timo)
366{
367 u_int64_t abstime = 0;
368
369 if (timo)
370 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
371 return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, (lck_mtx_t *)0);
372}
373
374int
375tsleep0(
376 void *chan,
377 int pri,
378 const char *wmsg,
379 int timo,
380 int (*continuation)(int))
381{
382 u_int64_t abstime = 0;
383
384 if (timo)
385 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
386 return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, (lck_mtx_t *)0);
387}
388
389int
390tsleep1(
391 void *chan,
392 int pri,
393 const char *wmsg,
394 u_int64_t abstime,
395 int (*continuation)(int))
396{
397 return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, (lck_mtx_t *)0);
398}
399
400/*
401 * Wake up all processes sleeping on chan.
402 */
403void
404wakeup(void *chan)
405{
406 thread_wakeup((caddr_t)chan);
407}
408
409/*
410 * Wake up the first process sleeping on chan.
411 *
412 * Be very sure that the first process is really
413 * the right one to wakeup.
414 */
415void
416wakeup_one(caddr_t chan)
417{
418 thread_wakeup_one((caddr_t)chan);
419}
420
421/*
422 * Compute the priority of a process when running in user mode.
423 * Arrange to reschedule if the resulting priority is better
424 * than that of the current process.
425 */
426void
427resetpriority(struct proc *p)
428{
429 (void)task_importance(p->task, -p->p_nice);
430}
431
432struct loadavg averunnable =
433 { {0, 0, 0}, FSCALE }; /* load average, of runnable procs */
434/*
435 * Constants for averages over 1, 5, and 15 minutes
436 * when sampling at 5 second intervals.
437 */
438static fixpt_t cexp[3] = {
439 (fixpt_t)(0.9200444146293232 * FSCALE), /* exp(-1/12) */
440 (fixpt_t)(0.9834714538216174 * FSCALE), /* exp(-1/60) */
441 (fixpt_t)(0.9944598480048967 * FSCALE), /* exp(-1/180) */
442};
443
444void
445compute_averunnable(void *arg)
446{
447 unsigned int nrun = *(unsigned int *)arg;
448 struct loadavg *avg = &averunnable;
449 int i;
450
451 for (i = 0; i < 3; i++)
452 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
453 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
454}
455