1/*
2 * Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _WORKQUEUE_INTERNAL_H_
30#define _WORKQUEUE_INTERNAL_H_
31
32// Sometimes something gets passed a bucket number and we need a way to express
33// that it's actually the event manager. Use the (0)th bucket for that.
34#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE)
35#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST)
36#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY)
37#define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST)
38#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX
39
40#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared
41#define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr
42
43/* These definitions are only available to the kext, to avoid bleeding
44 * constants and types across the boundary to the userspace library.
45 */
46#ifdef KERNEL
47#pragma mark wq structs
48
49/* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
50 * exported to kernel extensions.
51 */
52#define SCHED_CALL_BLOCK 0x1
53#define SCHED_CALL_UNBLOCK 0x2
54
55/* old workq priority scheme */
56
57#define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */
58#define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */
59#define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */
60#define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */
61
62/* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
63 * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
64 * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
65 * not change. So the factor is now always 5.
66 */
67#define WORKQUEUE_CONSTRAINED_FACTOR 5
68
69#if BSD_KERNEL_PRIVATE
70#include <kern/priority_queue.h>
71#include <kern/thread_call.h>
72#include <kern/turnstile.h>
73#include <mach/kern_return.h>
74#include <sys/queue.h>
75#include <sys/kernel_types.h>
76
77/* struct uthread::uu_workq_flags */
78#define UT_WORKQ_NEW 0x01 /* First return to userspace */
79#define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */
80#define UT_WORKQ_DYING 0x04 /* Thread is being killed */
81#define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */
82#define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */
83#define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */
84#define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */
85#define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */
86
87typedef union workq_threadreq_param_s {
88 struct {
89 uint16_t trp_flags;
90 uint8_t trp_pri;
91 uint8_t trp_pol;
92 uint32_t trp_cpupercent: 8,
93 trp_refillms: 24;
94 };
95 uint64_t trp_value;
96} workq_threadreq_param_t;
97
98#define TRP_PRIORITY 0x1
99#define TRP_POLICY 0x2
100#define TRP_CPUPERCENT 0x4
101#define TRP_RELEASED 0x8000
102
103typedef struct workq_threadreq_s {
104 union {
105 struct priority_queue_entry tr_entry;
106 thread_t tr_binding_thread;
107 };
108 uint32_t tr_flags;
109 uint8_t tr_state;
110 thread_qos_t tr_qos;
111 uint16_t tr_count;
112} *workq_threadreq_t;
113
114TAILQ_HEAD(threadreq_head, workq_threadreq_s);
115
116#define TR_STATE_IDLE 0 /* request isn't in flight */
117#define TR_STATE_NEW 1 /* request is being initiated */
118#define TR_STATE_QUEUED 2 /* request is being queued */
119#define TR_STATE_BINDING 4 /* request is preposted for bind */
120
121#define TR_FLAG_KEVENT 0x01
122#define TR_FLAG_WORKLOOP 0x02
123#define TR_FLAG_OVERCOMMIT 0x04
124#define TR_FLAG_WL_PARAMS 0x08
125#define TR_FLAG_WL_OUTSIDE_QOS 0x10
126
127#if defined(__LP64__)
128typedef unsigned __int128 wq_thactive_t;
129#else
130typedef uint64_t wq_thactive_t;
131#endif
132
133typedef enum {
134 WQ_EXITING = 0x0001,
135 WQ_PROC_SUSPENDED = 0x0002,
136 WQ_DEATH_CALL_SCHEDULED = 0x0004,
137
138 WQ_DELAYED_CALL_SCHEDULED = 0x0010,
139 WQ_DELAYED_CALL_PENDED = 0x0020,
140 WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
141 WQ_IMMEDIATE_CALL_PENDED = 0x0080,
142} workq_state_flags_t;
143
144TAILQ_HEAD(workq_uthread_head, uthread);
145
146struct workqueue {
147 thread_call_t wq_delayed_call;
148 thread_call_t wq_immediate_call;
149 thread_call_t wq_death_call;
150 struct turnstile *wq_turnstile;
151
152 lck_spin_t wq_lock;
153
154 uint64_t wq_thread_call_last_run;
155 struct os_refcnt wq_refcnt;
156 workq_state_flags_t _Atomic wq_flags;
157 uint32_t wq_fulfilled;
158 uint32_t wq_creations;
159 uint32_t wq_timer_interval;
160 uint32_t wq_event_manager_priority;
161 uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */
162 uint16_t wq_thdying_count;
163 uint16_t wq_threads_scheduled;
164 uint16_t wq_constrained_threads_scheduled;
165 uint16_t wq_nthreads;
166 uint16_t wq_thidlecount;
167 uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager
168
169 _Atomic wq_thactive_t wq_thactive;
170 _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];
171
172 struct proc *wq_proc;
173 struct uthread *wq_creator;
174 thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor
175 struct workq_uthread_head wq_thrunlist;
176 struct workq_uthread_head wq_thnewlist;
177 struct workq_uthread_head wq_thidlelist;
178
179 struct priority_queue wq_overcommit_queue;
180 struct priority_queue wq_constrained_queue;
181 struct priority_queue wq_special_queue;
182 workq_threadreq_t wq_event_manager_threadreq;
183};
184
185static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry),
186 "Make sure workq_deallocate_enqueue can cast the workqueue");
187
188#define WORKQUEUE_MAXTHREADS 512
189#define WQ_STALLED_WINDOW_USECS 200
190#define WQ_REDUCE_POOL_WINDOW_USECS 5000000
191#define WQ_MAX_TIMER_INTERVAL_USECS 50000
192
193#pragma mark definitions
194
195struct kqrequest;
196uint32_t _get_pwq_state_kdp(proc_t p);
197
198void workq_exit(struct proc *p);
199void workq_mark_exiting(struct proc *p);
200
201bool workq_is_exiting(struct proc *p);
202
203struct turnstile *workq_turnstile(struct proc *p);
204
205void workq_thread_set_max_qos(struct proc *p, struct kqrequest *kqr);
206
207void workq_thread_terminate(struct proc *p, struct uthread *uth);
208
209#define WORKQ_THREADREQ_SET_AST_ON_FAILURE 0x01
210#define WORKQ_THREADREQ_ATTEMPT_REBIND 0x02
211#define WORKQ_THREADREQ_CAN_CREATE_THREADS 0x04
212#define WORKQ_THREADREQ_CREATOR_TRANSFER 0x08
213#define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10
214
215// called with the kq req lock held
216bool workq_kern_threadreq_initiate(struct proc *p, struct kqrequest *kqr,
217 struct turnstile *ts, thread_qos_t qos, int flags);
218
219// called with the kq req lock held
220void workq_kern_threadreq_modify(struct proc *p, struct kqrequest *kqr,
221 thread_qos_t qos, int flags);
222
223// called with the kq req lock held
224void workq_kern_threadreq_update_inheritor(struct proc *p, struct kqrequest *kqr,
225 thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);
226
227void workq_kern_threadreq_lock(struct proc *p);
228void workq_kern_threadreq_unlock(struct proc *p);
229
230void workq_kern_threadreq_redrive(struct proc *p, int flags);
231
232enum workq_set_self_flags {
233 WORKQ_SET_SELF_QOS_FLAG = 0x1,
234 WORKQ_SET_SELF_VOUCHER_FLAG = 0x2,
235 WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4,
236 WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8,
237 WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
238};
239
240void workq_proc_suspended(struct proc *p);
241void workq_proc_resumed(struct proc *p);
242
243#endif // BSD_KERNEL_PRIVATE
244
245void workq_init(void);
246
247#endif // KERNEL
248
249#endif // _WORKQUEUE_INTERNAL_H_
250