1/*
2 * Copyright (c) 2010-2018 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#include <kern/kern_types.h>
33#include <kern/ledger.h>
34#include <kern/kalloc.h>
35#include <kern/task.h>
36#include <kern/thread.h>
37
38#include <kern/processor.h>
39#include <kern/machine.h>
40#include <kern/queue.h>
41#include <kern/policy_internal.h>
42
43#include <sys/errno.h>
44
45#include <libkern/OSAtomic.h>
46#include <mach/mach_types.h>
47#include <os/overflow.h>
48
49#include <vm/pmap.h>
50
51/*
52 * Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for
53 * ledger actions (LEDGER_ACTION_BLOCK, etc).
54 */
55#define LF_ENTRY_ACTIVE 0x0001 /* entry is active if set */
56#define LF_WAKE_NEEDED 0x0100 /* one or more threads are asleep */
57#define LF_WAKE_INPROGRESS 0x0200 /* the wait queue is being processed */
58#define LF_REFILL_SCHEDULED 0x0400 /* a refill timer has been set */
59#define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */
60#define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */
61#define LF_WARNED 0x2000 /* callback was called for balance warning */
62#define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */
63#define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */
64#define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */
65
66/* Determine whether a ledger entry exists and has been initialized and active */
67#define ENTRY_VALID(l, e) \
68 (((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) && \
69 (((l)->l_entries[e].le_flags & LF_ENTRY_ACTIVE) == LF_ENTRY_ACTIVE))
70
71#define ASSERT(a) assert(a)
72
73#ifdef LEDGER_DEBUG
74int ledger_debug = 0;
75
76#define lprintf(a) if (ledger_debug) { \
77 printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
78 printf a ; \
79}
80#else
81#define lprintf(a)
82#endif
83
84struct ledger_callback {
85 ledger_callback_t lc_func;
86 const void *lc_param0;
87 const void *lc_param1;
88};
89
90struct entry_template {
91 char et_key[LEDGER_NAME_MAX];
92 char et_group[LEDGER_NAME_MAX];
93 char et_units[LEDGER_NAME_MAX];
94 uint32_t et_flags;
95 struct ledger_callback *et_callback;
96};
97
98lck_grp_t ledger_lck_grp;
99
100/*
101 * Modifying the reference count, table size, or table contents requires
102 * holding the lt_lock. Modfying the table address requires both lt_lock
103 * and setting the inuse bit. This means that the lt_entries field can be
104 * safely dereferenced if you hold either the lock or the inuse bit. The
105 * inuse bit exists solely to allow us to swap in a new, larger entries
106 * table without requiring a full lock to be acquired on each lookup.
107 * Accordingly, the inuse bit should never be held for longer than it takes
108 * to extract a value from the table - i.e., 2 or 3 memory references.
109 */
110struct ledger_template {
111 const char *lt_name;
112 int lt_refs;
113 int lt_cnt;
114 int lt_table_size;
115 volatile uint32_t lt_inuse;
116 lck_mtx_t lt_lock;
117 zone_t lt_zone;
118 bool lt_initialized;
119 struct entry_template *lt_entries;
120};
121
122#define template_lock(template) lck_mtx_lock(&(template)->lt_lock)
123#define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock)
124
125#define TEMPLATE_INUSE(s, t) { \
126 s = splsched(); \
127 while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \
128 ; \
129}
130
131#define TEMPLATE_IDLE(s, t) { \
132 (t)->lt_inuse = 0; \
133 splx(s); \
134}
135
136static int ledger_cnt = 0;
137/* ledger ast helper functions */
138static uint32_t ledger_check_needblock(ledger_t l, uint64_t now);
139static kern_return_t ledger_perform_blocking(ledger_t l);
140static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
141static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
142
143static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
144 int entry, struct ledger_entry *le);
145
146#if 0
147static void
148debug_callback(const void *p0, __unused const void *p1)
149{
150 printf("ledger: resource exhausted [%s] for task %p\n",
151 (const char *)p0, p1);
152}
153#endif
154
155/************************************/
156
157static uint64_t
158abstime_to_nsecs(uint64_t abstime)
159{
160 uint64_t nsecs;
161
162 absolutetime_to_nanoseconds(abstime, &nsecs);
163 return (nsecs);
164}
165
166static uint64_t
167nsecs_to_abstime(uint64_t nsecs)
168{
169 uint64_t abstime;
170
171 nanoseconds_to_absolutetime(nsecs, &abstime);
172 return (abstime);
173}
174
175void
176ledger_init(void)
177{
178 lck_grp_init(&ledger_lck_grp, "ledger", LCK_GRP_ATTR_NULL);
179}
180
181ledger_template_t
182ledger_template_create(const char *name)
183{
184 ledger_template_t template;
185
186 template = (ledger_template_t)kalloc(sizeof (*template));
187 if (template == NULL)
188 return (NULL);
189
190 template->lt_name = name;
191 template->lt_refs = 1;
192 template->lt_cnt = 0;
193 template->lt_table_size = 1;
194 template->lt_inuse = 0;
195 template->lt_zone = NULL;
196 lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL);
197
198 template->lt_entries = (struct entry_template *)
199 kalloc(sizeof (struct entry_template) * template->lt_table_size);
200 if (template->lt_entries == NULL) {
201 kfree(template, sizeof (*template));
202 template = NULL;
203 }
204
205 return (template);
206}
207
208void
209ledger_template_dereference(ledger_template_t template)
210{
211 template_lock(template);
212 template->lt_refs--;
213 template_unlock(template);
214
215 if (template->lt_refs == 0)
216 kfree(template, sizeof (*template));
217}
218
219/*
220 * Add a new entry to the list of entries in a ledger template. There is
221 * currently no mechanism to remove an entry. Implementing such a mechanism
222 * would require us to maintain per-entry reference counts, which we would
223 * prefer to avoid if possible.
224 */
225int
226ledger_entry_add(ledger_template_t template, const char *key,
227 const char *group, const char *units)
228{
229 int idx;
230 struct entry_template *et;
231
232 if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL))
233 return (-1);
234
235 template_lock(template);
236
237 /* If the table is full, attempt to double its size */
238 if (template->lt_cnt == template->lt_table_size) {
239 struct entry_template *new_entries, *old_entries;
240 int old_cnt, old_sz, new_sz = 0;
241 spl_t s;
242
243 old_cnt = template->lt_table_size;
244 old_sz = old_cnt * (int)(sizeof(struct entry_template));
245 /* double old_sz allocation, but check for overflow */
246 if (os_mul_overflow(old_sz, 2, &new_sz)) {
247 template_unlock(template);
248 return -1;
249 }
250 new_entries = kalloc(new_sz);
251 if (new_entries == NULL) {
252 template_unlock(template);
253 return -1;
254 }
255 memcpy(new_entries, template->lt_entries, old_sz);
256 memset(((char *)new_entries) + old_sz, 0, old_sz);
257 /* assume: if the sz didn't overflow, neither will the count */
258 template->lt_table_size = old_cnt * 2;
259
260 old_entries = template->lt_entries;
261
262 TEMPLATE_INUSE(s, template);
263 template->lt_entries = new_entries;
264 TEMPLATE_IDLE(s, template);
265
266 kfree(old_entries, old_sz);
267 }
268
269 et = &template->lt_entries[template->lt_cnt];
270 strlcpy(et->et_key, key, LEDGER_NAME_MAX);
271 strlcpy(et->et_group, group, LEDGER_NAME_MAX);
272 strlcpy(et->et_units, units, LEDGER_NAME_MAX);
273 et->et_flags = LF_ENTRY_ACTIVE;
274 et->et_callback = NULL;
275
276 idx = template->lt_cnt++;
277 template_unlock(template);
278
279 return (idx);
280}
281
282
283kern_return_t
284ledger_entry_setactive(ledger_t ledger, int entry)
285{
286 struct ledger_entry *le;
287
288 if ((ledger == NULL) || (entry < 0) || (entry >= ledger->l_size))
289 return (KERN_INVALID_ARGUMENT);
290
291 le = &ledger->l_entries[entry];
292 if ((le->le_flags & LF_ENTRY_ACTIVE) == 0) {
293 flag_set(&le->le_flags, LF_ENTRY_ACTIVE);
294 }
295 return (KERN_SUCCESS);
296}
297
298
299int
300ledger_key_lookup(ledger_template_t template, const char *key)
301{
302 int idx;
303
304 template_lock(template);
305 for (idx = 0; idx < template->lt_cnt; idx++)
306 if (template->lt_entries != NULL &&
307 (strcmp(key, template->lt_entries[idx].et_key) == 0))
308 break;
309
310 if (idx >= template->lt_cnt)
311 idx = -1;
312 template_unlock(template);
313
314 return (idx);
315}
316
317/*
318 * Complete the initialization of ledger template
319 * by initializing ledger zone. After initializing
320 * the ledger zone, adding an entry in the ledger
321 * template would fail.
322 */
323void
324ledger_template_complete(ledger_template_t template)
325{
326 size_t ledger_size;
327 ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
328 template->lt_zone = zinit(ledger_size, CONFIG_TASK_MAX * ledger_size,
329 ledger_size,
330 template->lt_name);
331 template->lt_initialized = true;
332}
333
334/*
335 * Like ledger_template_complete, except we'll ask
336 * the pmap layer to manage allocations for us.
337 * Meant for ledgers that should be owned by the
338 * pmap layer.
339 */
340void
341ledger_template_complete_secure_alloc(ledger_template_t template)
342{
343 size_t ledger_size;
344 ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
345 pmap_ledger_alloc_init(ledger_size);
346 template->lt_initialized = true;
347}
348
349/*
350 * Create a new ledger based on the specified template. As part of the
351 * ledger creation we need to allocate space for a table of ledger entries.
352 * The size of the table is based on the size of the template at the time
353 * the ledger is created. If additional entries are added to the template
354 * after the ledger is created, they will not be tracked in this ledger.
355 */
356ledger_t
357ledger_instantiate(ledger_template_t template, int entry_type)
358{
359 ledger_t ledger;
360 size_t cnt;
361 int i;
362
363 template_lock(template);
364 template->lt_refs++;
365 cnt = template->lt_cnt;
366 template_unlock(template);
367
368 if (template->lt_zone) {
369 ledger = (ledger_t)zalloc(template->lt_zone);
370 } else {
371 ledger = pmap_ledger_alloc();
372 }
373
374 if (ledger == NULL) {
375 ledger_template_dereference(template);
376 return LEDGER_NULL;
377 }
378
379 ledger->l_template = template;
380 ledger->l_id = ledger_cnt++;
381 ledger->l_refs = 1;
382 ledger->l_size = (int32_t)cnt;
383
384 template_lock(template);
385 assert(ledger->l_size <= template->lt_cnt);
386 for (i = 0; i < ledger->l_size; i++) {
387 struct ledger_entry *le = &ledger->l_entries[i];
388 struct entry_template *et = &template->lt_entries[i];
389
390 le->le_flags = et->et_flags;
391 /* make entry inactive by removing active bit */
392 if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES)
393 flag_clear(&le->le_flags, LF_ENTRY_ACTIVE);
394 /*
395 * If template has a callback, this entry is opted-in,
396 * by default.
397 */
398 if (et->et_callback != NULL)
399 flag_set(&le->le_flags, LEDGER_ACTION_CALLBACK);
400 le->le_credit = 0;
401 le->le_debit = 0;
402 le->le_limit = LEDGER_LIMIT_INFINITY;
403 le->le_warn_level = LEDGER_LIMIT_INFINITY;
404 le->_le.le_refill.le_refill_period = 0;
405 le->_le.le_refill.le_last_refill = 0;
406 }
407 template_unlock(template);
408
409 return (ledger);
410}
411
412static uint32_t
413flag_set(volatile uint32_t *flags, uint32_t bit)
414{
415 return (OSBitOrAtomic(bit, flags));
416}
417
418static uint32_t
419flag_clear(volatile uint32_t *flags, uint32_t bit)
420{
421 return (OSBitAndAtomic(~bit, flags));
422}
423
424/*
425 * Take a reference on a ledger
426 */
427kern_return_t
428ledger_reference(ledger_t ledger)
429{
430 if (!LEDGER_VALID(ledger))
431 return (KERN_INVALID_ARGUMENT);
432 OSIncrementAtomic(&ledger->l_refs);
433 return (KERN_SUCCESS);
434}
435
436int
437ledger_reference_count(ledger_t ledger)
438{
439 if (!LEDGER_VALID(ledger))
440 return (-1);
441
442 return (ledger->l_refs);
443}
444
445/*
446 * Remove a reference on a ledger. If this is the last reference,
447 * deallocate the unused ledger.
448 */
449kern_return_t
450ledger_dereference(ledger_t ledger)
451{
452 int v;
453
454 if (!LEDGER_VALID(ledger))
455 return (KERN_INVALID_ARGUMENT);
456
457 v = OSDecrementAtomic(&ledger->l_refs);
458 ASSERT(v >= 1);
459
460 /* Just released the last reference. Free it. */
461 if (v == 1) {
462 if (ledger->l_template->lt_zone) {
463 zfree(ledger->l_template->lt_zone, ledger);
464 } else {
465 pmap_ledger_free(ledger);
466 }
467 }
468
469 return (KERN_SUCCESS);
470}
471
472/*
473 * Determine whether an entry has exceeded its warning level.
474 */
475static inline int
476warn_level_exceeded(struct ledger_entry *le)
477{
478 ledger_amount_t balance;
479
480 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
481 assert(le->le_debit == 0);
482 } else {
483 assert((le->le_credit >= 0) && (le->le_debit >= 0));
484 }
485
486 /*
487 * XXX - Currently, we only support warnings for ledgers which
488 * use positive limits.
489 */
490 balance = le->le_credit - le->le_debit;
491 if ((le->le_warn_level != LEDGER_LIMIT_INFINITY) && (balance > le->le_warn_level))
492 return (1);
493 return (0);
494}
495
496/*
497 * Determine whether an entry has exceeded its limit.
498 */
499static inline int
500limit_exceeded(struct ledger_entry *le)
501{
502 ledger_amount_t balance;
503
504 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
505 assert(le->le_debit == 0);
506 } else {
507 assert((le->le_credit >= 0) && (le->le_debit >= 0));
508 }
509
510 balance = le->le_credit - le->le_debit;
511 if ((le->le_limit <= 0) && (balance < le->le_limit))
512 return (1);
513
514 if ((le->le_limit > 0) && (balance > le->le_limit))
515 return (1);
516 return (0);
517}
518
519static inline struct ledger_callback *
520entry_get_callback(ledger_t ledger, int entry)
521{
522 struct ledger_callback *callback;
523 spl_t s;
524
525 TEMPLATE_INUSE(s, ledger->l_template);
526 callback = ledger->l_template->lt_entries[entry].et_callback;
527 TEMPLATE_IDLE(s, ledger->l_template);
528
529 return (callback);
530}
531
532/*
533 * If the ledger value is positive, wake up anybody waiting on it.
534 */
535static inline void
536ledger_limit_entry_wakeup(struct ledger_entry *le)
537{
538 uint32_t flags;
539
540 if (!limit_exceeded(le)) {
541 flags = flag_clear(&le->le_flags, LF_CALLED_BACK);
542
543 while (le->le_flags & LF_WAKE_NEEDED) {
544 flag_clear(&le->le_flags, LF_WAKE_NEEDED);
545 thread_wakeup((event_t)le);
546 }
547 }
548}
549
550/*
551 * Refill the coffers.
552 */
553static void
554ledger_refill(uint64_t now, ledger_t ledger, int entry)
555{
556 uint64_t elapsed, period, periods;
557 struct ledger_entry *le;
558 ledger_amount_t balance, due;
559
560 assert(entry >= 0 && entry < ledger->l_size);
561
562 le = &ledger->l_entries[entry];
563
564 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
565
566 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
567 assert(le->le_debit == 0);
568 return;
569 }
570
571 /*
572 * If another thread is handling the refill already, we're not
573 * needed.
574 */
575 if (flag_set(&le->le_flags, LF_REFILL_INPROGRESS) & LF_REFILL_INPROGRESS) {
576 return;
577 }
578
579 /*
580 * If the timestamp we're about to use to refill is older than the
581 * last refill, then someone else has already refilled this ledger
582 * and there's nothing for us to do here.
583 */
584 if (now <= le->_le.le_refill.le_last_refill) {
585 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
586 return;
587 }
588
589 /*
590 * See how many refill periods have passed since we last
591 * did a refill.
592 */
593 period = le->_le.le_refill.le_refill_period;
594 elapsed = now - le->_le.le_refill.le_last_refill;
595 if ((period == 0) || (elapsed < period)) {
596 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
597 return;
598 }
599
600 /*
601 * Optimize for the most common case of only one or two
602 * periods elapsing.
603 */
604 periods = 0;
605 while ((periods < 2) && (elapsed > 0)) {
606 periods++;
607 elapsed -= period;
608 }
609
610 /*
611 * OK, it's been a long time. Do a divide to figure out
612 * how long.
613 */
614 if (elapsed > 0)
615 periods = (now - le->_le.le_refill.le_last_refill) / period;
616
617 balance = le->le_credit - le->le_debit;
618 due = periods * le->le_limit;
619
620 if (balance - due < 0)
621 due = balance;
622
623 assertf(due >= 0,"now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
624
625 OSAddAtomic64(due, &le->le_debit);
626
627 assert(le->le_debit >= 0);
628
629 /*
630 * If we've completely refilled the pool, set the refill time to now.
631 * Otherwise set it to the time at which it last should have been
632 * fully refilled.
633 */
634 if (balance == due)
635 le->_le.le_refill.le_last_refill = now;
636 else
637 le->_le.le_refill.le_last_refill += (le->_le.le_refill.le_refill_period * periods);
638
639 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
640
641 lprintf(("Refill %lld %lld->%lld\n", periods, balance, balance - due));
642 if (!limit_exceeded(le))
643 ledger_limit_entry_wakeup(le);
644}
645
646void
647ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
648 int entry, struct ledger_entry *le)
649{
650 if (le->le_flags & LF_TRACKING_MAX) {
651 ledger_amount_t balance = le->le_credit - le->le_debit;
652
653 if (balance > le->_le._le_max.le_lifetime_max){
654 le->_le._le_max.le_lifetime_max = balance;
655 }
656
657#if CONFIG_LEDGER_INTERVAL_MAX
658 if (balance > le->_le._le_max.le_interval_max) {
659 le->_le._le_max.le_interval_max = balance;
660 }
661#endif /* LEDGER_CONFIG_INTERVAL_MAX */
662 }
663
664 /* Check to see whether we're due a refill */
665 if (le->le_flags & LF_REFILL_SCHEDULED) {
666 assert(!(le->le_flags & LF_TRACKING_MAX));
667
668 uint64_t now = mach_absolute_time();
669 if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period)
670 ledger_refill(now, ledger, entry);
671 }
672
673 if (limit_exceeded(le)) {
674 /*
675 * We've exceeded the limit for this entry. There
676 * are several possible ways to handle it. We can block,
677 * we can execute a callback, or we can ignore it. In
678 * either of the first two cases, we want to set the AST
679 * flag so we can take the appropriate action just before
680 * leaving the kernel. The one caveat is that if we have
681 * already called the callback, we don't want to do it
682 * again until it gets rearmed.
683 */
684 if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
685 (!(le->le_flags & LF_CALLED_BACK) &&
686 entry_get_callback(ledger, entry))) {
687 act_set_astledger_async(thread);
688 }
689 } else {
690 /*
691 * The balance on the account is below the limit.
692 *
693 * If there are any threads blocked on this entry, now would
694 * be a good time to wake them up.
695 */
696 if (le->le_flags & LF_WAKE_NEEDED)
697 ledger_limit_entry_wakeup(le);
698
699 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
700 /*
701 * Client has requested that a callback be invoked whenever
702 * the ledger's balance crosses into or out of the warning
703 * level.
704 */
705 if (warn_level_exceeded(le)) {
706 /*
707 * This ledger's balance is above the warning level.
708 */
709 if ((le->le_flags & LF_WARNED) == 0) {
710 /*
711 * If we are above the warning level and
712 * have not yet invoked the callback,
713 * set the AST so it can be done before returning
714 * to userland.
715 */
716 act_set_astledger_async(thread);
717 }
718 } else {
719 /*
720 * This ledger's balance is below the warning level.
721 */
722 if (le->le_flags & LF_WARNED) {
723 /*
724 * If we are below the warning level and
725 * the LF_WARNED flag is still set, we need
726 * to invoke the callback to let the client
727 * know the ledger balance is now back below
728 * the warning level.
729 */
730 act_set_astledger_async(thread);
731 }
732 }
733 }
734 }
735
736 if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
737 (le->le_credit < le->le_debit)) {
738 panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld\n",
739 ledger, entry, le,
740 le->le_credit,
741 le->le_debit,
742 le->le_credit - le->le_debit);
743 }
744}
745
746void
747ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)
748{
749 struct ledger_entry *le;
750 assert(entry > 0 && entry <= ledger->l_size);
751 le = &ledger->l_entries[entry];
752 ledger_entry_check_new_balance(thread, ledger, entry, le);
753}
754
755/*
756 * Add value to an entry in a ledger for a specific thread.
757 */
758kern_return_t
759ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
760{
761 ledger_amount_t old, new;
762 struct ledger_entry *le;
763
764 if (!ENTRY_VALID(ledger, entry) || (amount < 0))
765 return (KERN_INVALID_VALUE);
766
767 if (amount == 0)
768 return (KERN_SUCCESS);
769
770 le = &ledger->l_entries[entry];
771
772 old = OSAddAtomic64(amount, &le->le_credit);
773 new = old + amount;
774 lprintf(("%p Credit %lld->%lld\n", thread, old, new));
775
776 if (thread) {
777 ledger_entry_check_new_balance(thread, ledger, entry, le);
778 }
779
780 return (KERN_SUCCESS);
781}
782
783/*
784 * Add value to an entry in a ledger.
785 */
786kern_return_t
787ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
788{
789 return ledger_credit_thread(current_thread(), ledger, entry, amount);
790}
791
792/*
793 * Add value to an entry in a ledger; do not check balance after update.
794 */
795kern_return_t
796ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
797{
798 return ledger_credit_thread(NULL, ledger, entry, amount);
799}
800
801/* Add all of one ledger's values into another.
802 * They must have been created from the same template.
803 * This is not done atomically. Another thread (if not otherwise synchronized)
804 * may see bogus values when comparing one entry to another.
805 * As each entry's credit & debit are modified one at a time, the warning/limit
806 * may spuriously trip, or spuriously fail to trip, or another thread (if not
807 * otherwise synchronized) may see a bogus balance.
808 */
809kern_return_t
810ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
811{
812 int i;
813
814 assert(to_ledger->l_template == from_ledger->l_template);
815
816 for (i = 0; i < to_ledger->l_size; i++) {
817 ledger_rollup_entry(to_ledger, from_ledger, i);
818 }
819
820 return (KERN_SUCCESS);
821}
822
823/* Add one ledger entry value to another.
824 * They must have been created from the same template.
825 * Since the credit and debit values are added one
826 * at a time, other thread might read the a bogus value.
827 */
828kern_return_t
829ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
830{
831 struct ledger_entry *from_le, *to_le;
832
833 assert(to_ledger->l_template == from_ledger->l_template);
834 if (ENTRY_VALID(from_ledger, entry) && ENTRY_VALID(to_ledger, entry)) {
835 from_le = &from_ledger->l_entries[entry];
836 to_le = &to_ledger->l_entries[entry];
837 OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
838 OSAddAtomic64(from_le->le_debit, &to_le->le_debit);
839 }
840
841 return (KERN_SUCCESS);
842}
843
844/*
845 * Zero the balance of a ledger by adding to its credit or debit, whichever is smaller.
846 * Note that some clients of ledgers (notably, task wakeup statistics) require that
847 * le_credit only ever increase as a function of ledger_credit().
848 */
849kern_return_t
850ledger_zero_balance(ledger_t ledger, int entry)
851{
852 struct ledger_entry *le;
853 ledger_amount_t debit, credit;
854
855 if (!ENTRY_VALID(ledger, entry))
856 return (KERN_INVALID_VALUE);
857
858 le = &ledger->l_entries[entry];
859
860top:
861 debit = le->le_debit;
862 credit = le->le_credit;
863
864 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
865 assert(le->le_debit == 0);
866 if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
867 goto top;
868 }
869 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
870 } else if (credit > debit) {
871 if (!OSCompareAndSwap64(debit, credit, &le->le_debit))
872 goto top;
873 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
874 } else if (credit < debit) {
875 if (!OSCompareAndSwap64(credit, debit, &le->le_credit))
876 goto top;
877 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
878 }
879
880 return (KERN_SUCCESS);
881}
882
883kern_return_t
884ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit)
885{
886 struct ledger_entry *le;
887
888 if (!ENTRY_VALID(ledger, entry))
889 return (KERN_INVALID_VALUE);
890
891 le = &ledger->l_entries[entry];
892 *limit = le->le_limit;
893
894 lprintf(("ledger_get_limit: %lld\n", *limit));
895
896 return (KERN_SUCCESS);
897}
898
899/*
900 * Adjust the limit of a limited resource. This does not affect the
901 * current balance, so the change doesn't affect the thread until the
902 * next refill.
903 *
904 * warn_level: If non-zero, causes the callback to be invoked when
905 * the balance exceeds this level. Specified as a percentage [of the limit].
906 */
907kern_return_t
908ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit,
909 uint8_t warn_level_percentage)
910{
911 struct ledger_entry *le;
912
913 if (!ENTRY_VALID(ledger, entry))
914 return (KERN_INVALID_VALUE);
915
916 lprintf(("ledger_set_limit: %lld\n", limit));
917 le = &ledger->l_entries[entry];
918
919 if (limit == LEDGER_LIMIT_INFINITY) {
920 /*
921 * Caller wishes to disable the limit. This will implicitly
922 * disable automatic refill, as refills implicitly depend
923 * on the limit.
924 */
925 ledger_disable_refill(ledger, entry);
926 }
927
928 le->le_limit = limit;
929 if (le->le_flags & LF_REFILL_SCHEDULED) {
930 assert(!(le->le_flags & LF_TRACKING_MAX));
931 le->_le.le_refill.le_last_refill = 0;
932 }
933 flag_clear(&le->le_flags, LF_CALLED_BACK);
934 flag_clear(&le->le_flags, LF_WARNED);
935 ledger_limit_entry_wakeup(le);
936
937 if (warn_level_percentage != 0) {
938 assert(warn_level_percentage <= 100);
939 assert(limit > 0); /* no negative limit support for warnings */
940 assert(limit != LEDGER_LIMIT_INFINITY); /* warn % without limit makes no sense */
941 le->le_warn_level = (le->le_limit * warn_level_percentage) / 100;
942 } else {
943 le->le_warn_level = LEDGER_LIMIT_INFINITY;
944 }
945
946 return (KERN_SUCCESS);
947}
948
949#if CONFIG_LEDGER_INTERVAL_MAX
950kern_return_t
951ledger_get_interval_max(ledger_t ledger, int entry,
952 ledger_amount_t *max_interval_balance, int reset)
953{
954 struct ledger_entry *le;
955 le = &ledger->l_entries[entry];
956
957 if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
958 return (KERN_INVALID_VALUE);
959 }
960
961 *max_interval_balance = le->_le._le_max.le_interval_max;
962 lprintf(("ledger_get_interval_max: %lld%s\n", *max_interval_balance,
963 (reset) ? " --> 0" : ""));
964
965 if (reset) {
966 le->_le._le_max.le_interval_max = 0;
967 }
968
969 return (KERN_SUCCESS);
970}
971#endif /* CONFIG_LEDGER_INTERVAL_MAX */
972
973kern_return_t
974ledger_get_lifetime_max(ledger_t ledger, int entry,
975 ledger_amount_t *max_lifetime_balance)
976{
977 struct ledger_entry *le;
978 le = &ledger->l_entries[entry];
979
980 if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
981 return (KERN_INVALID_VALUE);
982 }
983
984 *max_lifetime_balance = le->_le._le_max.le_lifetime_max;
985 lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance));
986
987 return (KERN_SUCCESS);
988}
989
990/*
991 * Enable tracking of periodic maximums for this ledger entry.
992 */
993kern_return_t
994ledger_track_maximum(ledger_template_t template, int entry,
995 __unused int period_in_secs)
996{
997 template_lock(template);
998
999 if ((entry < 0) || (entry >= template->lt_cnt)) {
1000 template_unlock(template);
1001 return (KERN_INVALID_VALUE);
1002 }
1003
1004 /* Refill is incompatible with max tracking. */
1005 if (template->lt_entries[entry].et_flags & LF_REFILL_SCHEDULED) {
1006 return (KERN_INVALID_VALUE);
1007 }
1008
1009 template->lt_entries[entry].et_flags |= LF_TRACKING_MAX;
1010 template_unlock(template);
1011
1012 return (KERN_SUCCESS);
1013}
1014
1015kern_return_t
1016ledger_panic_on_negative(ledger_template_t template, int entry)
1017{
1018 template_lock(template);
1019
1020 if ((entry < 0) || (entry >= template->lt_cnt)) {
1021 template_unlock(template);
1022 return (KERN_INVALID_VALUE);
1023 }
1024
1025 template->lt_entries[entry].et_flags |= LF_PANIC_ON_NEGATIVE;
1026
1027 template_unlock(template);
1028
1029 return (KERN_SUCCESS);
1030}
1031
1032kern_return_t
1033ledger_track_credit_only(ledger_template_t template, int entry)
1034{
1035 template_lock(template);
1036
1037 if ((entry < 0) || (entry >= template->lt_cnt)) {
1038 template_unlock(template);
1039 return (KERN_INVALID_VALUE);
1040 }
1041
1042 template->lt_entries[entry].et_flags |= LF_TRACK_CREDIT_ONLY;
1043
1044 template_unlock(template);
1045
1046 return (KERN_SUCCESS);
1047}
1048
1049/*
1050 * Add a callback to be executed when the resource goes into deficit.
1051 */
1052kern_return_t
1053ledger_set_callback(ledger_template_t template, int entry,
1054 ledger_callback_t func, const void *param0, const void *param1)
1055{
1056 struct entry_template *et;
1057 struct ledger_callback *old_cb, *new_cb;
1058
1059 if ((entry < 0) || (entry >= template->lt_cnt))
1060 return (KERN_INVALID_VALUE);
1061
1062 if (func) {
1063 new_cb = (struct ledger_callback *)kalloc(sizeof (*new_cb));
1064 new_cb->lc_func = func;
1065 new_cb->lc_param0 = param0;
1066 new_cb->lc_param1 = param1;
1067 } else {
1068 new_cb = NULL;
1069 }
1070
1071 template_lock(template);
1072 et = &template->lt_entries[entry];
1073 old_cb = et->et_callback;
1074 et->et_callback = new_cb;
1075 template_unlock(template);
1076 if (old_cb)
1077 kfree(old_cb, sizeof (*old_cb));
1078
1079 return (KERN_SUCCESS);
1080}
1081
1082/*
1083 * Disable callback notification for a specific ledger entry.
1084 *
1085 * Otherwise, if using a ledger template which specified a
1086 * callback function (ledger_set_callback()), it will be invoked when
1087 * the resource goes into deficit.
1088 */
1089kern_return_t
1090ledger_disable_callback(ledger_t ledger, int entry)
1091{
1092 if (!ENTRY_VALID(ledger, entry))
1093 return (KERN_INVALID_VALUE);
1094
1095 /*
1096 * le_warn_level is used to indicate *if* this ledger has a warning configured,
1097 * in addition to what that warning level is set to.
1098 * This means a side-effect of ledger_disable_callback() is that the
1099 * warning level is forgotten.
1100 */
1101 ledger->l_entries[entry].le_warn_level = LEDGER_LIMIT_INFINITY;
1102 flag_clear(&ledger->l_entries[entry].le_flags, LEDGER_ACTION_CALLBACK);
1103 return (KERN_SUCCESS);
1104}
1105
1106/*
1107 * Enable callback notification for a specific ledger entry.
1108 *
1109 * This is only needed if ledger_disable_callback() has previously
1110 * been invoked against an entry; there must already be a callback
1111 * configured.
1112 */
1113kern_return_t
1114ledger_enable_callback(ledger_t ledger, int entry)
1115{
1116 if (!ENTRY_VALID(ledger, entry))
1117 return (KERN_INVALID_VALUE);
1118
1119 assert(entry_get_callback(ledger, entry) != NULL);
1120
1121 flag_set(&ledger->l_entries[entry].le_flags, LEDGER_ACTION_CALLBACK);
1122 return (KERN_SUCCESS);
1123}
1124
1125/*
1126 * Query the automatic refill period for this ledger entry.
1127 *
1128 * A period of 0 means this entry has none configured.
1129 */
1130kern_return_t
1131ledger_get_period(ledger_t ledger, int entry, uint64_t *period)
1132{
1133 struct ledger_entry *le;
1134
1135 if (!ENTRY_VALID(ledger, entry))
1136 return (KERN_INVALID_VALUE);
1137
1138 le = &ledger->l_entries[entry];
1139 *period = abstime_to_nsecs(le->_le.le_refill.le_refill_period);
1140 lprintf(("ledger_get_period: %llx\n", *period));
1141 return (KERN_SUCCESS);
1142}
1143
1144/*
1145 * Adjust the automatic refill period.
1146 */
1147kern_return_t
1148ledger_set_period(ledger_t ledger, int entry, uint64_t period)
1149{
1150 struct ledger_entry *le;
1151
1152 lprintf(("ledger_set_period: %llx\n", period));
1153 if (!ENTRY_VALID(ledger, entry))
1154 return (KERN_INVALID_VALUE);
1155
1156 le = &ledger->l_entries[entry];
1157
1158 /*
1159 * A refill period refills the ledger in multiples of the limit,
1160 * so if you haven't set one yet, you need a lesson on ledgers.
1161 */
1162 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
1163
1164 if (le->le_flags & LF_TRACKING_MAX) {
1165 /*
1166 * Refill is incompatible with rolling max tracking.
1167 */
1168 return (KERN_INVALID_VALUE);
1169 }
1170
1171 le->_le.le_refill.le_refill_period = nsecs_to_abstime(period);
1172
1173 /*
1174 * Set the 'starting time' for the next refill to now. Since
1175 * we're resetting the balance to zero here, we consider this
1176 * moment the starting time for accumulating a balance that
1177 * counts towards the limit.
1178 */
1179 le->_le.le_refill.le_last_refill = mach_absolute_time();
1180 ledger_zero_balance(ledger, entry);
1181
1182 flag_set(&le->le_flags, LF_REFILL_SCHEDULED);
1183
1184 return (KERN_SUCCESS);
1185}
1186
1187/*
1188 * Disable automatic refill.
1189 */
1190kern_return_t
1191ledger_disable_refill(ledger_t ledger, int entry)
1192{
1193 struct ledger_entry *le;
1194
1195 if (!ENTRY_VALID(ledger, entry))
1196 return (KERN_INVALID_VALUE);
1197
1198 le = &ledger->l_entries[entry];
1199
1200 flag_clear(&le->le_flags, LF_REFILL_SCHEDULED);
1201
1202 return (KERN_SUCCESS);
1203}
1204
1205kern_return_t
1206ledger_get_actions(ledger_t ledger, int entry, int *actions)
1207{
1208 if (!ENTRY_VALID(ledger, entry))
1209 return (KERN_INVALID_VALUE);
1210
1211 *actions = ledger->l_entries[entry].le_flags & LEDGER_ACTION_MASK;
1212 lprintf(("ledger_get_actions: %#x\n", *actions));
1213 return (KERN_SUCCESS);
1214}
1215
1216kern_return_t
1217ledger_set_action(ledger_t ledger, int entry, int action)
1218{
1219 lprintf(("ledger_set_action: %#x\n", action));
1220 if (!ENTRY_VALID(ledger, entry))
1221 return (KERN_INVALID_VALUE);
1222
1223 flag_set(&ledger->l_entries[entry].le_flags, action);
1224 return (KERN_SUCCESS);
1225}
1226
1227kern_return_t
1228ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
1229{
1230 struct ledger_entry *le;
1231 ledger_amount_t old, new;
1232
1233 if (!ENTRY_VALID(ledger, entry) || (amount < 0))
1234 return (KERN_INVALID_ARGUMENT);
1235
1236 if (amount == 0)
1237 return (KERN_SUCCESS);
1238
1239 le = &ledger->l_entries[entry];
1240
1241 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1242 assert(le->le_debit == 0);
1243 old = OSAddAtomic64(-amount, &le->le_credit);
1244 new = old - amount;
1245 } else {
1246 old = OSAddAtomic64(amount, &le->le_debit);
1247 new = old + amount;
1248 }
1249 lprintf(("%p Debit %lld->%lld\n", thread, old, new));
1250
1251 if (thread) {
1252 ledger_entry_check_new_balance(thread, ledger, entry, le);
1253 }
1254
1255 return (KERN_SUCCESS);
1256}
1257
1258kern_return_t
1259ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
1260{
1261 return ledger_debit_thread(current_thread(), ledger, entry, amount);
1262}
1263
1264kern_return_t
1265ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
1266{
1267 return ledger_debit_thread(NULL, ledger, entry, amount);
1268}
1269
1270void
1271ledger_ast(thread_t thread)
1272{
1273 struct ledger *l = thread->t_ledger;
1274 struct ledger *thl;
1275 uint32_t block;
1276 uint64_t now;
1277 uint8_t task_flags;
1278 uint8_t task_percentage;
1279 uint64_t task_interval;
1280
1281 kern_return_t ret;
1282 task_t task = thread->task;
1283
1284 lprintf(("Ledger AST for %p\n", thread));
1285
1286 ASSERT(task != NULL);
1287 ASSERT(thread == current_thread());
1288
1289top:
1290 /*
1291 * Take a self-consistent snapshot of the CPU usage monitor parameters. The task
1292 * can change them at any point (with the task locked).
1293 */
1294 task_lock(task);
1295 task_flags = task->rusage_cpu_flags;
1296 task_percentage = task->rusage_cpu_perthr_percentage;
1297 task_interval = task->rusage_cpu_perthr_interval;
1298 task_unlock(task);
1299
1300 /*
1301 * Make sure this thread is up to date with regards to any task-wide per-thread
1302 * CPU limit, but only if it doesn't have a thread-private blocking CPU limit.
1303 */
1304 if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) != 0) &&
1305 ((thread->options & TH_OPT_PRVT_CPULIMIT) == 0)) {
1306 uint8_t percentage;
1307 uint64_t interval;
1308 int action;
1309
1310 thread_get_cpulimit(&action, &percentage, &interval);
1311
1312 /*
1313 * If the thread's CPU limits no longer match the task's, or the
1314 * task has a limit but the thread doesn't, update the limit.
1315 */
1316 if (((thread->options & TH_OPT_PROC_CPULIMIT) == 0) ||
1317 (interval != task_interval) || (percentage != task_percentage)) {
1318 thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, task_percentage, task_interval);
1319 assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0);
1320 }
1321 } else if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) &&
1322 (thread->options & TH_OPT_PROC_CPULIMIT)) {
1323 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
1324
1325 /*
1326 * Task no longer has a per-thread CPU limit; remove this thread's
1327 * corresponding CPU limit.
1328 */
1329 thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0);
1330 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
1331 }
1332
1333 /*
1334 * If the task or thread is being terminated, let's just get on with it
1335 */
1336 if ((l == NULL) || !task->active || task->halting || !thread->active)
1337 return;
1338
1339 /*
1340 * Examine all entries in deficit to see which might be eligble for
1341 * an automatic refill, which require callbacks to be issued, and
1342 * which require blocking.
1343 */
1344 block = 0;
1345 now = mach_absolute_time();
1346
1347 /*
1348 * Note that thread->t_threadledger may have been changed by the
1349 * thread_set_cpulimit() call above - so don't examine it until afterwards.
1350 */
1351 thl = thread->t_threadledger;
1352 if (LEDGER_VALID(thl)) {
1353 block |= ledger_check_needblock(thl, now);
1354 }
1355 block |= ledger_check_needblock(l, now);
1356
1357 /*
1358 * If we are supposed to block on the availability of one or more
1359 * resources, find the first entry in deficit for which we should wait.
1360 * Schedule a refill if necessary and then sleep until the resource
1361 * becomes available.
1362 */
1363 if (block) {
1364 if (LEDGER_VALID(thl)) {
1365 ret = ledger_perform_blocking(thl);
1366 if (ret != KERN_SUCCESS)
1367 goto top;
1368 }
1369 ret = ledger_perform_blocking(l);
1370 if (ret != KERN_SUCCESS)
1371 goto top;
1372 } /* block */
1373}
1374
1375static uint32_t
1376ledger_check_needblock(ledger_t l, uint64_t now)
1377{
1378 int i;
1379 uint32_t flags, block = 0;
1380 struct ledger_entry *le;
1381 struct ledger_callback *lc;
1382
1383
1384 for (i = 0; i < l->l_size; i++) {
1385 le = &l->l_entries[i];
1386
1387 lc = entry_get_callback(l, i);
1388
1389 if (limit_exceeded(le) == FALSE) {
1390 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
1391 /*
1392 * If needed, invoke the callback as a warning.
1393 * This needs to happen both when the balance rises above
1394 * the warning level, and also when it dips back below it.
1395 */
1396 assert(lc != NULL);
1397 /*
1398 * See comments for matching logic in ledger_check_new_balance().
1399 */
1400 if (warn_level_exceeded(le)) {
1401 flags = flag_set(&le->le_flags, LF_WARNED);
1402 if ((flags & LF_WARNED) == 0) {
1403 lc->lc_func(LEDGER_WARNING_ROSE_ABOVE, lc->lc_param0, lc->lc_param1);
1404 }
1405 } else {
1406 flags = flag_clear(&le->le_flags, LF_WARNED);
1407 if (flags & LF_WARNED) {
1408 lc->lc_func(LEDGER_WARNING_DIPPED_BELOW, lc->lc_param0, lc->lc_param1);
1409 }
1410 }
1411 }
1412
1413 continue;
1414 }
1415
1416 /* We're over the limit, so refill if we are eligible and past due. */
1417 if (le->le_flags & LF_REFILL_SCHEDULED) {
1418 assert(!(le->le_flags & LF_TRACKING_MAX));
1419
1420 if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) > now) {
1421 ledger_refill(now, l, i);
1422 if (limit_exceeded(le) == FALSE)
1423 continue;
1424 }
1425 }
1426
1427 if (le->le_flags & LEDGER_ACTION_BLOCK)
1428 block = 1;
1429 if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0)
1430 continue;
1431
1432 /*
1433 * If the LEDGER_ACTION_CALLBACK flag is on, we expect there to
1434 * be a registered callback.
1435 */
1436 assert(lc != NULL);
1437 flags = flag_set(&le->le_flags, LF_CALLED_BACK);
1438 /* Callback has already been called */
1439 if (flags & LF_CALLED_BACK)
1440 continue;
1441 lc->lc_func(FALSE, lc->lc_param0, lc->lc_param1);
1442 }
1443 return(block);
1444}
1445
1446
1447/* return KERN_SUCCESS to continue, KERN_FAILURE to restart */
1448static kern_return_t
1449ledger_perform_blocking(ledger_t l)
1450{
1451 int i;
1452 kern_return_t ret;
1453 struct ledger_entry *le;
1454
1455 for (i = 0; i < l->l_size; i++) {
1456 le = &l->l_entries[i];
1457 if ((!limit_exceeded(le)) ||
1458 ((le->le_flags & LEDGER_ACTION_BLOCK) == 0))
1459 continue;
1460
1461 assert(!(le->le_flags & LF_TRACKING_MAX));
1462
1463 /* Prepare to sleep until the resource is refilled */
1464 ret = assert_wait_deadline(le, THREAD_INTERRUPTIBLE,
1465 le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period);
1466 if (ret != THREAD_WAITING)
1467 return(KERN_SUCCESS);
1468
1469 /* Mark that somebody is waiting on this entry */
1470 flag_set(&le->le_flags, LF_WAKE_NEEDED);
1471
1472 ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL,
1473 AST_LEDGER);
1474 if (ret != THREAD_AWAKENED)
1475 return(KERN_SUCCESS);
1476
1477 /*
1478 * The world may have changed while we were asleep.
1479 * Some other resource we need may have gone into
1480 * deficit. Or maybe we're supposed to die now.
1481 * Go back to the top and reevaluate.
1482 */
1483 return(KERN_FAILURE);
1484 }
1485 return(KERN_SUCCESS);
1486}
1487
1488
1489kern_return_t
1490ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit,
1491 ledger_amount_t *debit)
1492{
1493 struct ledger_entry *le;
1494
1495 if (!ENTRY_VALID(ledger, entry))
1496 return (KERN_INVALID_ARGUMENT);
1497
1498 le = &ledger->l_entries[entry];
1499
1500 *credit = le->le_credit;
1501 *debit = le->le_debit;
1502
1503 return (KERN_SUCCESS);
1504}
1505
1506kern_return_t
1507ledger_reset_callback_state(ledger_t ledger, int entry)
1508{
1509 struct ledger_entry *le;
1510
1511 if (!ENTRY_VALID(ledger, entry))
1512 return (KERN_INVALID_ARGUMENT);
1513
1514 le = &ledger->l_entries[entry];
1515
1516 flag_clear(&le->le_flags, LF_CALLED_BACK);
1517
1518 return (KERN_SUCCESS);
1519}
1520
1521kern_return_t
1522ledger_disable_panic_on_negative(ledger_t ledger, int entry)
1523{
1524 struct ledger_entry *le;
1525
1526 if (!ENTRY_VALID(ledger, entry))
1527 return (KERN_INVALID_ARGUMENT);
1528
1529 le = &ledger->l_entries[entry];
1530
1531 flag_clear(&le->le_flags, LF_PANIC_ON_NEGATIVE);
1532
1533 return (KERN_SUCCESS);
1534}
1535
1536kern_return_t
1537ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative)
1538{
1539 struct ledger_entry *le;
1540
1541 if (!ENTRY_VALID(ledger, entry))
1542 return (KERN_INVALID_ARGUMENT);
1543
1544 le = &ledger->l_entries[entry];
1545
1546 if (le->le_flags & LF_PANIC_ON_NEGATIVE) {
1547 *panic_on_negative = TRUE;
1548 } else {
1549 *panic_on_negative = FALSE;
1550 }
1551
1552 return (KERN_SUCCESS);
1553}
1554
1555kern_return_t
1556ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
1557{
1558 struct ledger_entry *le;
1559
1560 if (!ENTRY_VALID(ledger, entry))
1561 return (KERN_INVALID_ARGUMENT);
1562
1563 le = &ledger->l_entries[entry];
1564
1565 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1566 assert(le->le_debit == 0);
1567 } else {
1568 assert((le->le_credit >= 0) && (le->le_debit >= 0));
1569 }
1570
1571 *balance = le->le_credit - le->le_debit;
1572
1573 return (KERN_SUCCESS);
1574}
1575
1576int
1577ledger_template_info(void **buf, int *len)
1578{
1579 struct ledger_template_info *lti;
1580 struct entry_template *et;
1581 int i;
1582 ledger_t l;
1583
1584 /*
1585 * Since all tasks share a ledger template, we'll just use the
1586 * caller's as the source.
1587 */
1588 l = current_task()->ledger;
1589 if ((*len < 0) || (l == NULL))
1590 return (EINVAL);
1591
1592 if (*len > l->l_size)
1593 *len = l->l_size;
1594 lti = kalloc((*len) * sizeof (struct ledger_template_info));
1595 if (lti == NULL)
1596 return (ENOMEM);
1597 *buf = lti;
1598
1599 template_lock(l->l_template);
1600 et = l->l_template->lt_entries;
1601
1602 for (i = 0; i < *len; i++) {
1603 memset(lti, 0, sizeof (*lti));
1604 strlcpy(lti->lti_name, et->et_key, LEDGER_NAME_MAX);
1605 strlcpy(lti->lti_group, et->et_group, LEDGER_NAME_MAX);
1606 strlcpy(lti->lti_units, et->et_units, LEDGER_NAME_MAX);
1607 et++;
1608 lti++;
1609 }
1610 template_unlock(l->l_template);
1611
1612 return (0);
1613}
1614
1615static void
1616ledger_fill_entry_info(struct ledger_entry *le,
1617 struct ledger_entry_info *lei,
1618 uint64_t now)
1619{
1620 assert(le != NULL);
1621 assert(lei != NULL);
1622
1623 memset(lei, 0, sizeof (*lei));
1624
1625 lei->lei_limit = le->le_limit;
1626 lei->lei_credit = le->le_credit;
1627 lei->lei_debit = le->le_debit;
1628 lei->lei_balance = lei->lei_credit - lei->lei_debit;
1629 lei->lei_refill_period = (le->le_flags & LF_REFILL_SCHEDULED) ?
1630 abstime_to_nsecs(le->_le.le_refill.le_refill_period) : 0;
1631 lei->lei_last_refill = abstime_to_nsecs(now - le->_le.le_refill.le_last_refill);
1632}
1633
1634int
1635ledger_get_task_entry_info_multiple(task_t task, void **buf, int *len)
1636{
1637 struct ledger_entry_info *lei;
1638 struct ledger_entry *le;
1639 uint64_t now = mach_absolute_time();
1640 int i;
1641 ledger_t l;
1642
1643 if ((*len < 0) || ((l = task->ledger) == NULL))
1644 return (EINVAL);
1645
1646 if (*len > l->l_size)
1647 *len = l->l_size;
1648 lei = kalloc((*len) * sizeof (struct ledger_entry_info));
1649 if (lei == NULL)
1650 return (ENOMEM);
1651 *buf = lei;
1652
1653 le = l->l_entries;
1654
1655 for (i = 0; i < *len; i++) {
1656 ledger_fill_entry_info(le, lei, now);
1657 le++;
1658 lei++;
1659 }
1660
1661 return (0);
1662}
1663
1664void
1665ledger_get_entry_info(ledger_t ledger,
1666 int entry,
1667 struct ledger_entry_info *lei)
1668{
1669 uint64_t now = mach_absolute_time();
1670
1671 assert(ledger != NULL);
1672 assert(lei != NULL);
1673
1674 if (entry >= 0 && entry < ledger->l_size) {
1675 struct ledger_entry *le = &ledger->l_entries[entry];
1676 ledger_fill_entry_info(le, lei, now);
1677 }
1678}
1679
1680int
1681ledger_info(task_t task, struct ledger_info *info)
1682{
1683 ledger_t l;
1684
1685 if ((l = task->ledger) == NULL)
1686 return (ENOENT);
1687
1688 memset(info, 0, sizeof (*info));
1689
1690 strlcpy(info->li_name, l->l_template->lt_name, LEDGER_NAME_MAX);
1691 info->li_id = l->l_id;
1692 info->li_entries = l->l_size;
1693 return (0);
1694}
1695
1696#ifdef LEDGER_DEBUG
1697int
1698ledger_limit(task_t task, struct ledger_limit_args *args)
1699{
1700 ledger_t l;
1701 int64_t limit;
1702 int idx;
1703
1704 if ((l = task->ledger) == NULL)
1705 return (EINVAL);
1706
1707 idx = ledger_key_lookup(l->l_template, args->lla_name);
1708 if ((idx < 0) || (idx >= l->l_size))
1709 return (EINVAL);
1710
1711 /*
1712 * XXX - this doesn't really seem like the right place to have
1713 * a context-sensitive conversion of userspace units into kernel
1714 * units. For now I'll handwave and say that the ledger() system
1715 * call isn't meant for civilians to use - they should be using
1716 * the process policy interfaces.
1717 */
1718 if (idx == task_ledgers.cpu_time) {
1719 int64_t nsecs;
1720
1721 if (args->lla_refill_period) {
1722 /*
1723 * If a refill is scheduled, then the limit is
1724 * specified as a percentage of one CPU. The
1725 * syscall specifies the refill period in terms of
1726 * milliseconds, so we need to convert to nsecs.
1727 */
1728 args->lla_refill_period *= 1000000;
1729 nsecs = args->lla_limit *
1730 (args->lla_refill_period / 100);
1731 lprintf(("CPU limited to %lld nsecs per second\n",
1732 nsecs));
1733 } else {
1734 /*
1735 * If no refill is scheduled, then this is a
1736 * fixed amount of CPU time (in nsecs) that can
1737 * be consumed.
1738 */
1739 nsecs = args->lla_limit;
1740 lprintf(("CPU limited to %lld nsecs\n", nsecs));
1741 }
1742 limit = nsecs_to_abstime(nsecs);
1743 } else {
1744 limit = args->lla_limit;
1745 lprintf(("%s limited to %lld\n", args->lla_name, limit));
1746 }
1747
1748 if (args->lla_refill_period > 0)
1749 ledger_set_period(l, idx, args->lla_refill_period);
1750
1751 ledger_set_limit(l, idx, limit);
1752 flag_set(&l->l_entries[idx].le_flags, LEDGER_ACTION_BLOCK);
1753 return (0);
1754}
1755#endif
1756