1/*
2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <pexpert/pexpert.h>
30#include <IOKit/IOWorkLoop.h>
31#include <IOKit/IOEventSource.h>
32#include <IOKit/IOInterruptEventSource.h>
33#include <IOKit/IOCommandGate.h>
34#include <IOKit/IOCommandPool.h>
35#include <IOKit/IOTimeStamp.h>
36#include <IOKit/IOKitDebug.h>
37#include <libkern/OSDebug.h>
38#include <kern/thread.h>
39
40#define super OSObject
41
42OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
43
44// Block of unused functions intended for future use
45#if __LP64__
46OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
47OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
48OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
49#else
50OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
51OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
52OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
53#endif
54OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
55OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
56OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
57OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
58OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
59
60enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
61static inline void SETP(void *addr, unsigned int flag)
62 { unsigned char *num = (unsigned char *) addr; *num |= flag; }
63static inline void CLRP(void *addr, unsigned int flag)
64 { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
65static inline bool ISSETP(void *addr, unsigned int flag)
66 { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
67
68#define fFlags loopRestart
69
70#define passiveEventChain reserved->passiveEventChain
71
72#if IOKITSTATS
73
74#define IOStatisticsRegisterCounter() \
75do { \
76 reserved->counter = IOStatistics::registerWorkLoop(this); \
77} while(0)
78
79#define IOStatisticsUnregisterCounter() \
80do { \
81 if (reserved) \
82 IOStatistics::unregisterWorkLoop(reserved->counter); \
83} while(0)
84
85#define IOStatisticsOpenGate() \
86do { \
87 IOStatistics::countWorkLoopOpenGate(reserved->counter); \
88 if (reserved->lockInterval) lockTime(); \
89} while(0)
90#define IOStatisticsCloseGate() \
91do { \
92 IOStatistics::countWorkLoopCloseGate(reserved->counter); \
93 if (reserved->lockInterval) reserved->lockTime = mach_absolute_time(); \
94} while(0)
95
96#define IOStatisticsAttachEventSource() \
97do { \
98 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
99} while(0)
100
101#define IOStatisticsDetachEventSource() \
102do { \
103 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
104} while(0)
105
106#else
107
108#define IOStatisticsRegisterCounter()
109#define IOStatisticsUnregisterCounter()
110#define IOStatisticsOpenGate()
111#define IOStatisticsCloseGate()
112#define IOStatisticsAttachEventSource()
113#define IOStatisticsDetachEventSource()
114
115#endif /* IOKITSTATS */
116
117bool IOWorkLoop::init()
118{
119 // The super init and gateLock allocation MUST be done first.
120 if ( !super::init() )
121 return false;
122
123 // Allocate our ExpansionData if it hasn't been allocated already.
124 if ( !reserved )
125 {
126 reserved = IONew(ExpansionData,1);
127 if ( !reserved )
128 return false;
129
130 bzero(reserved,sizeof(ExpansionData));
131 }
132
133 if ( gateLock == NULL ) {
134 if ( !( gateLock = IORecursiveLockAlloc()) )
135 return false;
136 }
137
138 if ( workToDoLock == NULL ) {
139 if ( !(workToDoLock = IOSimpleLockAlloc()) )
140 return false;
141 IOSimpleLockInit(workToDoLock);
142 workToDo = false;
143 }
144
145 IOStatisticsRegisterCounter();
146
147 if ( controlG == NULL ) {
148 controlG = IOCommandGate::commandGate(
149 this,
150 OSMemberFunctionCast(
151 IOCommandGate::Action,
152 this,
153 &IOWorkLoop::_maintRequest));
154
155 if ( !controlG )
156 return false;
157 // Point the controlGate at the workLoop. Usually addEventSource
158 // does this automatically. The problem is in this case addEventSource
159 // uses the control gate and it has to be bootstrapped.
160 controlG->setWorkLoop(this);
161 if (addEventSource(controlG) != kIOReturnSuccess)
162 return false;
163 }
164
165 if ( workThread == NULL ) {
166 thread_continue_t cptr = OSMemberFunctionCast(
167 thread_continue_t,
168 this,
169 &IOWorkLoop::threadMain);
170 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
171 return false;
172 }
173
174 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
175 return true;
176}
177
178IOWorkLoop *
179IOWorkLoop::workLoop()
180{
181 return IOWorkLoop::workLoopWithOptions(0);
182}
183
184IOWorkLoop *
185IOWorkLoop::workLoopWithOptions(IOOptionBits options)
186{
187 IOWorkLoop *me = new IOWorkLoop;
188
189 if (me && options) {
190 me->reserved = IONew(ExpansionData,1);
191 if (!me->reserved) {
192 me->release();
193 return 0;
194 }
195 bzero(me->reserved,sizeof(ExpansionData));
196 me->reserved->options = options;
197 }
198
199 if (me && !me->init()) {
200 me->release();
201 return 0;
202 }
203
204 return me;
205}
206
207// Free is called twice:
208// First when the atomic retainCount transitions from 1 -> 0
209// Secondly when the work loop itself is commiting hari kari
210// Hence the each leg of the free must be single threaded.
211void IOWorkLoop::free()
212{
213 if (workThread) {
214 IOInterruptState is;
215
216 // If we are here then we must be trying to shut down this work loop
217 // in this case disable all of the event source, mark the loop
218 // as terminating and wakeup the work thread itself and return
219 // Note: we hold the gate across the entire operation mainly for the
220 // benefit of our event sources so we can disable them cleanly.
221 closeGate();
222
223 disableAllEventSources();
224
225 is = IOSimpleLockLockDisableInterrupt(workToDoLock);
226 SETP(&fFlags, kLoopTerminate);
227 thread_wakeup_thread((void *) &workToDo, workThread);
228 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
229
230 openGate();
231 }
232 else /* !workThread */ {
233 IOEventSource *event, *next;
234
235 for (event = eventChain; event; event = next) {
236 next = event->getNext();
237 event->setWorkLoop(0);
238 event->setNext(0);
239 event->release();
240 }
241 eventChain = 0;
242
243 for (event = passiveEventChain; event; event = next) {
244 next = event->getNext();
245 event->setWorkLoop(0);
246 event->setNext(0);
247 event->release();
248 }
249 passiveEventChain = 0;
250
251 // Either we have a partial initialization to clean up
252 // or the workThread itself is performing hari-kari.
253 // Either way clean up all of our resources and return.
254
255 if (controlG) {
256 controlG->workLoop = 0;
257 controlG->release();
258 controlG = 0;
259 }
260
261 if (workToDoLock) {
262 IOSimpleLockFree(workToDoLock);
263 workToDoLock = 0;
264 }
265
266 if (gateLock) {
267 IORecursiveLockFree(gateLock);
268 gateLock = 0;
269 }
270
271 IOStatisticsUnregisterCounter();
272
273 if (reserved) {
274 IODelete(reserved, ExpansionData, 1);
275 reserved = 0;
276 }
277
278 super::free();
279 }
280}
281
282IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
283{
284 if ((workThread)
285 && !thread_has_thread_name(workThread)
286 && (newEvent->owner)
287 && !OSDynamicCast(IOCommandPool, newEvent->owner)) {
288 thread_set_thread_name(workThread, newEvent->owner->getMetaClass()->getClassName());
289 }
290
291 return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
292}
293
294IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
295{
296 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
297}
298
299void IOWorkLoop::enableAllEventSources() const
300{
301 IOEventSource *event;
302
303 for (event = eventChain; event; event = event->getNext())
304 event->enable();
305
306 for (event = passiveEventChain; event; event = event->getNext())
307 event->enable();
308}
309
310void IOWorkLoop::disableAllEventSources() const
311{
312 IOEventSource *event;
313
314 for (event = eventChain; event; event = event->getNext())
315 event->disable();
316
317 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
318 for (event = passiveEventChain; event; event = event->getNext())
319 if (event != controlG) // Don't disable the control gate
320 event->disable();
321}
322
323void IOWorkLoop::enableAllInterrupts() const
324{
325 IOEventSource *event;
326
327 for (event = eventChain; event; event = event->getNext())
328 if (OSDynamicCast(IOInterruptEventSource, event))
329 event->enable();
330}
331
332void IOWorkLoop::disableAllInterrupts() const
333{
334 IOEventSource *event;
335
336 for (event = eventChain; event; event = event->getNext())
337 if (OSDynamicCast(IOInterruptEventSource, event))
338 event->disable();
339}
340
341
342/* virtual */ bool IOWorkLoop::runEventSources()
343{
344 bool res = false;
345 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
346 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
347
348 closeGate();
349 if (ISSETP(&fFlags, kLoopTerminate))
350 goto abort;
351
352 if (traceWL)
353 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this));
354
355 bool more;
356 do {
357 CLRP(&fFlags, kLoopRestart);
358 more = false;
359 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
360 workToDo = false;
361 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
362 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
363 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
364
365 if (traceES)
366 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt));
367
368 more |= evnt->checkForWork();
369
370 if (traceES)
371 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt));
372
373 if (ISSETP(&fFlags, kLoopTerminate))
374 goto abort;
375 else if (fFlags & kLoopRestart) {
376 more = true;
377 break;
378 }
379 }
380 } while (more);
381
382 res = true;
383
384 if (traceWL)
385 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this));
386
387abort:
388 openGate();
389 return res;
390}
391
392/* virtual */ void IOWorkLoop::threadMain()
393{
394restartThread:
395 do {
396 if ( !runEventSources() )
397 goto exitThread;
398
399 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
400 if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
401 assert_wait((void *) &workToDo, false);
402 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
403 thread_continue_t cptr = NULL;
404 if (!reserved || !(kPreciousStack & reserved->options))
405 cptr = OSMemberFunctionCast(
406 thread_continue_t, this, &IOWorkLoop::threadMain);
407 thread_block_parameter(cptr, this);
408 goto restartThread;
409 /* NOTREACHED */
410 }
411
412 // At this point we either have work to do or we need
413 // to commit suicide. But no matter
414 // Clear the simple lock and retore the interrupt state
415 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
416 } while(workToDo);
417
418exitThread:
419 closeGate();
420 thread_t thread = workThread;
421 workThread = 0; // Say we don't have a loop and free ourselves
422 openGate();
423
424 free();
425
426 thread_deallocate(thread);
427 (void) thread_terminate(thread);
428}
429
430IOThread IOWorkLoop::getThread() const
431{
432 return workThread;
433}
434
435bool IOWorkLoop::onThread() const
436{
437 return (IOThreadSelf() == workThread);
438}
439
440bool IOWorkLoop::inGate() const
441{
442 return IORecursiveLockHaveLock(gateLock);
443}
444
445// Internal APIs used by event sources to control the thread
446void IOWorkLoop::signalWorkAvailable()
447{
448 if (workToDoLock) {
449 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
450 workToDo = true;
451 thread_wakeup_thread((void *) &workToDo, workThread);
452 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
453 }
454}
455
456void IOWorkLoop::openGate()
457{
458 IOStatisticsOpenGate();
459 IORecursiveLockUnlock(gateLock);
460}
461
462void IOWorkLoop::closeGate()
463{
464 IORecursiveLockLock(gateLock);
465 IOStatisticsCloseGate();
466}
467
468bool IOWorkLoop::tryCloseGate()
469{
470 bool res = (IORecursiveLockTryLock(gateLock) != 0);
471 if (res) {
472 IOStatisticsCloseGate();
473 }
474 return res;
475}
476
477int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
478{
479 int res;
480 IOStatisticsOpenGate();
481 res = IORecursiveLockSleep(gateLock, event, interuptibleType);
482 IOStatisticsCloseGate();
483 return res;
484}
485
486int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
487{
488 int res;
489 IOStatisticsOpenGate();
490 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
491 IOStatisticsCloseGate();
492 return res;
493}
494
495void IOWorkLoop::wakeupGate(void *event, bool oneThread)
496{
497 IORecursiveLockWakeup(gateLock, event, oneThread);
498}
499
500static IOReturn IOWorkLoopActionToBlock(OSObject *owner,
501 void *arg0, void *arg1,
502 void *arg2, void *arg3)
503{
504 return ((IOWorkLoop::ActionBlock) arg0)();
505}
506
507IOReturn IOWorkLoop::runActionBlock(ActionBlock action)
508{
509 return (runAction(&IOWorkLoopActionToBlock, this, action));
510}
511
512IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
513 void *arg0, void *arg1,
514 void *arg2, void *arg3)
515{
516 IOReturn res;
517
518 // closeGate is recursive so don't worry if we already hold the lock.
519 closeGate();
520 res = (*inAction)(target, arg0, arg1, arg2, arg3);
521 openGate();
522
523 return res;
524}
525
526IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
527{
528 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
529 IOEventSource *inEvent = (IOEventSource *) inD;
530 IOReturn res = kIOReturnSuccess;
531
532 switch (command)
533 {
534 case mAddEvent:
535 if (!inEvent->getWorkLoop()) {
536 SETP(&fFlags, kLoopRestart);
537
538 inEvent->retain();
539 inEvent->setWorkLoop(this);
540 inEvent->setNext(0);
541
542 /* Check if this is a passive or active event source being added */
543 if (eventSourcePerformsWork(inEvent)) {
544
545 if (!eventChain)
546 eventChain = inEvent;
547 else {
548 IOEventSource *event, *next;
549
550 for (event = eventChain; (next = event->getNext()); event = next)
551 ;
552 event->setNext(inEvent);
553
554 }
555
556 }
557 else {
558
559 if (!passiveEventChain)
560 passiveEventChain = inEvent;
561 else {
562 IOEventSource *event, *next;
563
564 for (event = passiveEventChain; (next = event->getNext()); event = next)
565 ;
566 event->setNext(inEvent);
567
568 }
569
570 }
571 IOStatisticsAttachEventSource();
572 }
573 break;
574
575 case mRemoveEvent:
576 if (inEvent->getWorkLoop()) {
577 IOStatisticsDetachEventSource();
578
579 if (eventSourcePerformsWork(inEvent)) {
580 if (eventChain == inEvent)
581 eventChain = inEvent->getNext();
582 else {
583 IOEventSource *event, *next = 0;
584
585 event = eventChain;
586 if (event) while ((next = event->getNext()) && (next != inEvent))
587 event = next;
588
589 if (!next) {
590 res = kIOReturnBadArgument;
591 break;
592 }
593 event->setNext(inEvent->getNext());
594 }
595 }
596 else {
597 if (passiveEventChain == inEvent)
598 passiveEventChain = inEvent->getNext();
599 else {
600 IOEventSource *event, *next = 0;
601
602 event = passiveEventChain;
603 if (event) while ((next = event->getNext()) && (next != inEvent))
604 event = next;
605
606 if (!next) {
607 res = kIOReturnBadArgument;
608 break;
609 }
610 event->setNext(inEvent->getNext());
611 }
612 }
613
614 inEvent->setWorkLoop(0);
615 inEvent->setNext(0);
616 inEvent->release();
617 SETP(&fFlags, kLoopRestart);
618 }
619 break;
620
621 default:
622 return kIOReturnUnsupported;
623 }
624
625 return res;
626}
627
628bool
629IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
630{
631 bool result = true;
632
633 /*
634 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
635 * The assumption is that if you override checkForWork(), you need to be
636 * active and not passive.
637 *
638 * We picked a known quantity controlG that does not override
639 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
640 * the workloop to which this event source is getting attached.
641 *
642 * We do a pointer comparison on the offset in the vtable for inNewEvent against
643 * the offset in the vtable for inReferenceEvent. This works because
644 * IOCommandGate's slot for checkForWork() has the address of
645 * IOEventSource::checkForWork() in it.
646 *
647 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
648 * checkForWork() here. We're just testing to see if it's the same or not.
649 *
650 */
651
652 if (IOEventSource::kPassive & inEventSource->flags) result = false;
653 else if (IOEventSource::kActive & inEventSource->flags) result = true;
654 else if (controlG) {
655 void * ptr1;
656 void * ptr2;
657
658 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
659 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
660
661 if (ptr1 == ptr2)
662 result = false;
663 }
664
665 return result;
666}
667
668void
669IOWorkLoop::lockTime(void)
670{
671 uint64_t time;
672 time = mach_absolute_time() - reserved->lockTime;
673 if (time > reserved->lockInterval)
674 {
675 absolutetime_to_nanoseconds(time, &time);
676 if (kTimeLockPanics & reserved->options) panic("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
677 else OSReportWithBacktrace("IOWorkLoop %p lock time %qd us", this, time / 1000ULL);
678 }
679}
680
681void
682IOWorkLoop::setMaximumLockTime(uint64_t interval, uint32_t options)
683{
684 IORecursiveLockLock(gateLock);
685 reserved->lockInterval = interval;
686 reserved->options = (reserved->options & ~kTimeLockPanics) | (options & kTimeLockPanics);
687 IORecursiveLockUnlock(gateLock);
688}
689const OSSymbol *IOSKCopyKextIdentifierWithAddress(vm_address_t a);
690const OSSymbol *IOSKCopyKextIdentifierWithAddress(vm_address_t a) { OSKext *k = OSKext::lookupKextWithAddress(a); if (k) { const OSSymbol *s = k->getIdentifier(); k->release(); if (s) { s->retain(); return s; } } return NULL; }
691