1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Portions Copyright (c) 2012 by Delphix. All rights reserved.
27 * Portions Copyright (c) 2016 by Joyent, Inc.
28 */
29
30#ifndef _SYS_DTRACE_IMPL_H
31#define _SYS_DTRACE_IMPL_H
32
33/* #pragma ident "@(#)dtrace_impl.h 1.23 07/02/16 SMI" */
34
35#ifdef __cplusplus
36extern "C" {
37#endif
38
39/*
40 * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
41 *
42 * Note: The contents of this file are private to the implementation of the
43 * Solaris system and DTrace subsystem and are subject to change at any time
44 * without notice. Applications and drivers using these interfaces will fail
45 * to run on future releases. These interfaces should not be used for any
46 * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
47 * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
48 */
49
50#include <sys/dtrace.h>
51
52/*
53 * DTrace Implementation Locks
54 */
55extern lck_mtx_t dtrace_procwaitfor_lock;
56
57/*
58 * DTrace Implementation Constants and Typedefs
59 */
60#define DTRACE_MAXPROPLEN 128
61#define DTRACE_DYNVAR_CHUNKSIZE 256
62
63struct dtrace_probe;
64struct dtrace_ecb;
65struct dtrace_predicate;
66struct dtrace_action;
67struct dtrace_provider;
68struct dtrace_state;
69
70typedef struct dtrace_probe dtrace_probe_t;
71typedef struct dtrace_ecb dtrace_ecb_t;
72typedef struct dtrace_predicate dtrace_predicate_t;
73typedef struct dtrace_action dtrace_action_t;
74typedef struct dtrace_provider dtrace_provider_t;
75typedef struct dtrace_meta dtrace_meta_t;
76typedef struct dtrace_state dtrace_state_t;
77typedef uint32_t dtrace_optid_t;
78typedef uint32_t dtrace_specid_t;
79typedef uint64_t dtrace_genid_t;
80
81/*
82 * DTrace Probes
83 *
84 * The probe is the fundamental unit of the DTrace architecture. Probes are
85 * created by DTrace providers, and managed by the DTrace framework. A probe
86 * is identified by a unique <provider, module, function, name> tuple, and has
87 * a unique probe identifier assigned to it. (Some probes are not associated
88 * with a specific point in text; these are called _unanchored probes_ and have
89 * no module or function associated with them.) Probes are represented as a
90 * dtrace_probe structure. To allow quick lookups based on each element of the
91 * probe tuple, probes are hashed by each of provider, module, function and
92 * name. (If a lookup is performed based on a regular expression, a
93 * dtrace_probekey is prepared, and a linear search is performed.) Each probe
94 * is additionally pointed to by a linear array indexed by its identifier. The
95 * identifier is the provider's mechanism for indicating to the DTrace
96 * framework that a probe has fired: the identifier is passed as the first
97 * argument to dtrace_probe(), where it is then mapped into the corresponding
98 * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can
99 * iterate over the probe's list of enabling control blocks; see "DTrace
100 * Enabling Control Blocks", below.)
101 */
102struct dtrace_probe {
103 dtrace_id_t dtpr_id; /* probe identifier */
104 dtrace_ecb_t *dtpr_ecb; /* ECB list; see below */
105 dtrace_ecb_t *dtpr_ecb_last; /* last ECB in list */
106 void *dtpr_arg; /* provider argument */
107 dtrace_cacheid_t dtpr_predcache; /* predicate cache ID */
108 int dtpr_aframes; /* artificial frames */
109 dtrace_provider_t *dtpr_provider; /* pointer to provider */
110 char *dtpr_mod; /* probe's module name */
111 char *dtpr_func; /* probe's function name */
112 char *dtpr_name; /* probe's name */
113 dtrace_probe_t *dtpr_nextprov; /* next in provider hash */
114 dtrace_probe_t *dtpr_prevprov; /* previous in provider hash */
115 dtrace_probe_t *dtpr_nextmod; /* next in module hash */
116 dtrace_probe_t *dtpr_prevmod; /* previous in module hash */
117 dtrace_probe_t *dtpr_nextfunc; /* next in function hash */
118 dtrace_probe_t *dtpr_prevfunc; /* previous in function hash */
119 dtrace_probe_t *dtpr_nextname; /* next in name hash */
120 dtrace_probe_t *dtpr_prevname; /* previous in name hash */
121 dtrace_genid_t dtpr_gen; /* probe generation ID */
122};
123
124typedef int dtrace_probekey_f(const char *, const char *, int);
125
126typedef struct dtrace_probekey {
127 const char *dtpk_prov; /* provider name to match */
128 dtrace_probekey_f *dtpk_pmatch; /* provider matching function */
129 const char *dtpk_mod; /* module name to match */
130 dtrace_probekey_f *dtpk_mmatch; /* module matching function */
131 const char *dtpk_func; /* func name to match */
132 dtrace_probekey_f *dtpk_fmatch; /* func matching function */
133 const char *dtpk_name; /* name to match */
134 dtrace_probekey_f *dtpk_nmatch; /* name matching function */
135 dtrace_id_t dtpk_id; /* identifier to match */
136} dtrace_probekey_t;
137
138typedef struct dtrace_hashbucket {
139 struct dtrace_hashbucket *dthb_next; /* next on hash chain */
140 void *dthb_chain; /* chain of elements */
141 int dthb_len; /* number of probes here */
142} dtrace_hashbucket_t;
143
144typedef const char* dtrace_strkey_f(void*, uintptr_t);
145
146typedef struct dtrace_hash {
147 dtrace_hashbucket_t **dth_tab; /* hash table */
148 int dth_size; /* size of hash table */
149 int dth_mask; /* mask to index into table */
150 int dth_nbuckets; /* total number of buckets */
151 uintptr_t dth_nextoffs; /* offset of next in element */
152 uintptr_t dth_prevoffs; /* offset of prev in element */
153 dtrace_strkey_f *dth_getstr; /* func to retrieve str in element */
154 uintptr_t dth_stroffs; /* offset of str in element */
155} dtrace_hash_t;
156
157/*
158 * DTrace Enabling Control Blocks
159 *
160 * When a provider wishes to fire a probe, it calls into dtrace_probe(),
161 * passing the probe identifier as the first argument. As described above,
162 * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
163 * structure. This structure contains information about the probe, and a
164 * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to
165 * DTrace consumer state, and contains an optional predicate, and a list of
166 * actions. (Shown schematically below.) The ECB abstraction allows a single
167 * probe to be multiplexed across disjoint consumers, or across disjoint
168 * enablings of a single probe within one consumer.
169 *
170 * Enabling Control Block
171 * dtrace_ecb_t
172 * +------------------------+
173 * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
174 * | dtrace_state_t * ------+--------------> State associated with this ECB
175 * | dtrace_predicate_t * --+---------+
176 * | dtrace_action_t * -----+----+ |
177 * | dtrace_ecb_t * ---+ | | | Predicate (if any)
178 * +-------------------+----+ | | dtrace_predicate_t
179 * | | +---> +--------------------+
180 * | | | dtrace_difo_t * ---+----> DIFO
181 * | | +--------------------+
182 * | |
183 * Next ECB | | Action
184 * (if any) | | dtrace_action_t
185 * : +--> +-------------------+
186 * : | dtrace_actkind_t -+------> kind
187 * v | dtrace_difo_t * --+------> DIFO (if any)
188 * | dtrace_recdesc_t -+------> record descr.
189 * | dtrace_action_t * +------+
190 * +-------------------+ |
191 * | Next action
192 * +-------------------------------+ (if any)
193 * |
194 * | Action
195 * | dtrace_action_t
196 * +--> +-------------------+
197 * | dtrace_actkind_t -+------> kind
198 * | dtrace_difo_t * --+------> DIFO (if any)
199 * | dtrace_action_t * +------+
200 * +-------------------+ |
201 * | Next action
202 * +-------------------------------+ (if any)
203 * |
204 * :
205 * v
206 *
207 *
208 * dtrace_probe() iterates over the ECB list. If the ECB needs less space
209 * than is available in the principal buffer, the ECB is processed: if the
210 * predicate is non-NULL, the DIF object is executed. If the result is
211 * non-zero, the action list is processed, with each action being executed
212 * accordingly. When the action list has been completely executed, processing
213 * advances to the next ECB. The ECB abstraction allows disjoint consumers
214 * to multiplex on single probes.
215 *
216 * Execution of the ECB results in consuming dte_size bytes in the buffer
217 * to record data. During execution, dte_needed bytes must be available in
218 * the buffer. This space is used for both recorded data and tuple data.
219 */
220struct dtrace_ecb {
221 dtrace_epid_t dte_epid; /* enabled probe ID */
222 uint32_t dte_alignment; /* required alignment */
223 size_t dte_needed; /* space needed for execution */
224 size_t dte_size; /* size of recorded payload */
225 dtrace_predicate_t *dte_predicate; /* predicate, if any */
226 dtrace_action_t *dte_action; /* actions, if any */
227 dtrace_ecb_t *dte_next; /* next ECB on probe */
228 dtrace_state_t *dte_state; /* pointer to state */
229 uint32_t dte_cond; /* security condition */
230 dtrace_probe_t *dte_probe; /* pointer to probe */
231 dtrace_action_t *dte_action_last; /* last action on ECB */
232 uint64_t dte_uarg; /* library argument */
233};
234
235struct dtrace_predicate {
236 dtrace_difo_t *dtp_difo; /* DIF object */
237 dtrace_cacheid_t dtp_cacheid; /* cache identifier */
238 int dtp_refcnt; /* reference count */
239};
240
241struct dtrace_action {
242 dtrace_actkind_t dta_kind; /* kind of action */
243 uint16_t dta_intuple; /* boolean: in aggregation */
244 uint32_t dta_refcnt; /* reference count */
245 dtrace_difo_t *dta_difo; /* pointer to DIFO */
246 dtrace_recdesc_t dta_rec; /* record description */
247 dtrace_action_t *dta_prev; /* previous action */
248 dtrace_action_t *dta_next; /* next action */
249};
250
251typedef struct dtrace_aggregation {
252 dtrace_action_t dtag_action; /* action; must be first */
253 dtrace_aggid_t dtag_id; /* identifier */
254 dtrace_ecb_t *dtag_ecb; /* corresponding ECB */
255 dtrace_action_t *dtag_first; /* first action in tuple */
256 uint32_t dtag_base; /* base of aggregation */
257 uint8_t dtag_hasarg; /* boolean: has argument */
258 uint64_t dtag_initial; /* initial value */
259 void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t);
260} dtrace_aggregation_t;
261
262/*
263 * DTrace Buffers
264 *
265 * Principal buffers, aggregation buffers, and speculative buffers are all
266 * managed with the dtrace_buffer structure. By default, this structure
267 * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
268 * active and passive buffers, respectively. For speculative buffers,
269 * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
270 * to a scratch buffer. For all buffer types, the dtrace_buffer structure is
271 * always allocated on a per-CPU basis; a single dtrace_buffer structure is
272 * never shared among CPUs. (That is, there is never true sharing of the
273 * dtrace_buffer structure; to prevent false sharing of the structure, it must
274 * always be aligned to the coherence granularity -- generally 64 bytes.)
275 *
276 * One of the critical design decisions of DTrace is that a given ECB always
277 * stores the same quantity and type of data. This is done to assure that the
278 * only metadata required for an ECB's traced data is the EPID. That is, from
279 * the EPID, the consumer can determine the data layout. (The data buffer
280 * layout is shown schematically below.) By assuring that one can determine
281 * data layout from the EPID, the metadata stream can be separated from the
282 * data stream -- simplifying the data stream enormously. The ECB always
283 * proceeds the recorded data as part of the dtrace_rechdr_t structure that
284 * includes the EPID and a high-resolution timestamp used for output ordering
285 * consistency.
286 *
287 * base of data buffer ---> +--------+--------------------+--------+
288 * | rechdr | data | rechdr |
289 * +--------+------+--------+----+--------+
290 * | data | rechdr | data |
291 * +---------------+--------+-------------+
292 * | data, cont. |
293 * +--------+--------------------+--------+
294 * | rechdr | data | |
295 * +--------+--------------------+ |
296 * | || |
297 * | || |
298 * | \/ |
299 * : :
300 * . .
301 * . .
302 * . .
303 * : :
304 * | |
305 * limit of data buffer ---> +--------------------------------------+
306 *
307 * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
308 * principal buffer (both scratch and payload) exceed the available space. If
309 * the ECB's needs exceed available space (and if the principal buffer policy
310 * is the default "switch" policy), the ECB is dropped, the buffer's drop count
311 * is incremented, and processing advances to the next ECB. If the ECB's needs
312 * can be met with the available space, the ECB is processed, but the offset in
313 * the principal buffer is only advanced if the ECB completes processing
314 * without error.
315 *
316 * When a buffer is to be switched (either because the buffer is the principal
317 * buffer with a "switch" policy or because it is an aggregation buffer), a
318 * cross call is issued to the CPU associated with the buffer. In the cross
319 * call context, interrupts are disabled, and the active and the inactive
320 * buffers are atomically switched. This involves switching the data pointers,
321 * copying the various state fields (offset, drops, errors, etc.) into their
322 * inactive equivalents, and clearing the state fields. Because interrupts are
323 * disabled during this procedure, the switch is guaranteed to appear atomic to
324 * dtrace_probe().
325 *
326 * DTrace Ring Buffering
327 *
328 * To process a ring buffer correctly, one must know the oldest valid record.
329 * Processing starts at the oldest record in the buffer and continues until
330 * the end of the buffer is reached. Processing then resumes starting with
331 * the record stored at offset 0 in the buffer, and continues until the
332 * youngest record is processed. If trace records are of a fixed-length,
333 * determining the oldest record is trivial:
334 *
335 * - If the ring buffer has not wrapped, the oldest record is the record
336 * stored at offset 0.
337 *
338 * - If the ring buffer has wrapped, the oldest record is the record stored
339 * at the current offset.
340 *
341 * With variable length records, however, just knowing the current offset
342 * doesn't suffice for determining the oldest valid record: assuming that one
343 * allows for arbitrary data, one has no way of searching forward from the
344 * current offset to find the oldest valid record. (That is, one has no way
345 * of separating data from metadata.) It would be possible to simply refuse to
346 * process any data in the ring buffer between the current offset and the
347 * limit, but this leaves (potentially) an enormous amount of otherwise valid
348 * data unprocessed.
349 *
350 * To effect ring buffering, we track two offsets in the buffer: the current
351 * offset and the _wrapped_ offset. If a request is made to reserve some
352 * amount of data, and the buffer has wrapped, the wrapped offset is
353 * incremented until the wrapped offset minus the current offset is greater
354 * than or equal to the reserve request. This is done by repeatedly looking
355 * up the ECB corresponding to the EPID at the current wrapped offset, and
356 * incrementing the wrapped offset by the size of the data payload
357 * corresponding to that ECB. If this offset is greater than or equal to the
358 * limit of the data buffer, the wrapped offset is set to 0. Thus, the
359 * current offset effectively "chases" the wrapped offset around the buffer.
360 * Schematically:
361 *
362 * base of data buffer ---> +------+--------------------+------+
363 * | EPID | data | EPID |
364 * +------+--------+------+----+------+
365 * | data | EPID | data |
366 * +---------------+------+-----------+
367 * | data, cont. |
368 * +------+---------------------------+
369 * | EPID | data |
370 * current offset ---> +------+---------------------------+
371 * | invalid data |
372 * wrapped offset ---> +------+--------------------+------+
373 * | EPID | data | EPID |
374 * +------+--------+------+----+------+
375 * | data | EPID | data |
376 * +---------------+------+-----------+
377 * : :
378 * . .
379 * . ... valid data ... .
380 * . .
381 * : :
382 * +------+-------------+------+------+
383 * | EPID | data | EPID | data |
384 * +------+------------++------+------+
385 * | data, cont. | leftover |
386 * limit of data buffer ---> +-------------------+--------------+
387 *
388 * If the amount of requested buffer space exceeds the amount of space
389 * available between the current offset and the end of the buffer:
390 *
391 * (1) all words in the data buffer between the current offset and the limit
392 * of the data buffer (marked "leftover", above) are set to
393 * DTRACE_EPIDNONE
394 *
395 * (2) the wrapped offset is set to zero
396 *
397 * (3) the iteration process described above occurs until the wrapped offset
398 * is greater than the amount of desired space.
399 *
400 * The wrapped offset is implemented by (re-)using the inactive offset.
401 * In a "switch" buffer policy, the inactive offset stores the offset in
402 * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
403 * offset.
404 *
405 * DTrace Scratch Buffering
406 *
407 * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
408 * To accommodate such requests easily, scratch memory may be allocated in
409 * the buffer beyond the current offset plus the needed memory of the current
410 * ECB. If there isn't sufficient room in the buffer for the requested amount
411 * of scratch space, the allocation fails and an error is generated. Scratch
412 * memory is tracked in the dtrace_mstate_t and is automatically freed when
413 * the ECB ceases processing. Note that ring buffers cannot allocate their
414 * scratch from the principal buffer -- lest they needlessly overwrite older,
415 * valid data. Ring buffers therefore have their own dedicated scratch buffer
416 * from which scratch is allocated.
417 */
418#define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */
419#define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */
420#define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */
421#define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */
422#define DTRACEBUF_DROPPED 0x0010 /* drops occurred */
423#define DTRACEBUF_ERROR 0x0020 /* errors occurred */
424#define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */
425#define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */
426#define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */
427
428typedef struct dtrace_buffer {
429 uint64_t dtb_offset; /* current offset in buffer */
430 uint64_t dtb_cur_limit; /* current limit before signaling/dropping */
431 uint64_t dtb_limit; /* limit before signaling */
432 uint64_t dtb_size; /* size of buffer */
433 uint32_t dtb_flags; /* flags */
434 uint32_t dtb_drops; /* number of drops */
435 caddr_t dtb_tomax; /* active buffer */
436 caddr_t dtb_xamot; /* inactive buffer */
437 uint32_t dtb_xamot_flags; /* inactive flags */
438 uint32_t dtb_xamot_drops; /* drops in inactive buffer */
439 uint64_t dtb_xamot_offset; /* offset in inactive buffer */
440 uint32_t dtb_errors; /* number of errors */
441 uint32_t dtb_xamot_errors; /* errors in inactive buffer */
442#ifndef _LP64
443 uint64_t dtb_pad1;
444#endif
445 uint64_t dtb_switched; /* time of last switch */
446 uint64_t dtb_interval; /* observed switch interval */
447 uint64_t dtb_pad2[4]; /* pad to avoid false sharing */
448} dtrace_buffer_t;
449
450/*
451 * DTrace Aggregation Buffers
452 *
453 * Aggregation buffers use much of the same mechanism as described above
454 * ("DTrace Buffers"). However, because an aggregation is fundamentally a
455 * hash, there exists dynamic metadata associated with an aggregation buffer
456 * that is not associated with other kinds of buffers. This aggregation
457 * metadata is _only_ relevant for the in-kernel implementation of
458 * aggregations; it is not actually relevant to user-level consumers. To do
459 * this, we allocate dynamic aggregation data (hash keys and hash buckets)
460 * starting below the _limit_ of the buffer, and we allocate data from the
461 * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the
462 * data is copied out; the metadata is simply discarded. Schematically,
463 * aggregation buffers look like:
464 *
465 * base of data buffer ---> +-------+------+-----------+-------+
466 * | aggid | key | value | aggid |
467 * +-------+------+-----------+-------+
468 * | key |
469 * +-------+-------+-----+------------+
470 * | value | aggid | key | value |
471 * +-------+------++-----+------+-----+
472 * | aggid | key | value | |
473 * +-------+------+-------------+ |
474 * | || |
475 * | || |
476 * | \/ |
477 * : :
478 * . .
479 * . .
480 * . .
481 * : :
482 * | /\ |
483 * | || +------------+
484 * | || | |
485 * +---------------------+ |
486 * | hash keys |
487 * | (dtrace_aggkey structures) |
488 * | |
489 * +----------------------------------+
490 * | hash buckets |
491 * | (dtrace_aggbuffer structure) |
492 * | |
493 * limit of data buffer ---> +----------------------------------+
494 *
495 *
496 * As implied above, just as we assure that ECBs always store a constant
497 * amount of data, we assure that a given aggregation -- identified by its
498 * aggregation ID -- always stores data of a constant quantity and type.
499 * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
500 * given record.
501 *
502 * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
503 * aligned. (If this the structure changes such that this becomes false, an
504 * assertion will fail in dtrace_aggregate().)
505 */
506typedef struct dtrace_aggkey {
507 uint32_t dtak_hashval; /* hash value */
508 uint32_t dtak_action:4; /* action -- 4 bits */
509 uint32_t dtak_size:28; /* size -- 28 bits */
510 caddr_t dtak_data; /* data pointer */
511 struct dtrace_aggkey *dtak_next; /* next in hash chain */
512} dtrace_aggkey_t;
513
514typedef struct dtrace_aggbuffer {
515 uintptr_t dtagb_hashsize; /* number of buckets */
516 uintptr_t dtagb_free; /* free list of keys */
517 dtrace_aggkey_t **dtagb_hash; /* hash table */
518} dtrace_aggbuffer_t;
519
520/*
521 * DTrace Speculations
522 *
523 * Speculations have a per-CPU buffer and a global state. Once a speculation
524 * buffer has been comitted or discarded, it cannot be reused until all CPUs
525 * have taken the same action (commit or discard) on their respective
526 * speculative buffer. However, because DTrace probes may execute in arbitrary
527 * context, other CPUs cannot simply be cross-called at probe firing time to
528 * perform the necessary commit or discard. The speculation states thus
529 * optimize for the case that a speculative buffer is only active on one CPU at
530 * the time of a commit() or discard() -- for if this is the case, other CPUs
531 * need not take action, and the speculation is immediately available for
532 * reuse. If the speculation is active on multiple CPUs, it must be
533 * asynchronously cleaned -- potentially leading to a higher rate of dirty
534 * speculative drops. The speculation states are as follows:
535 *
536 * DTRACESPEC_INACTIVE <= Initial state; inactive speculation
537 * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to
538 * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU
539 * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU
540 * DTRACESPEC_COMMITTING <= Currently being commited on one CPU
541 * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
542 * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs
543 *
544 * The state transition diagram is as follows:
545 *
546 * +----------------------------------------------------------+
547 * | |
548 * | +------------+ |
549 * | +-------------------| COMMITTING |<-----------------+ |
550 * | | +------------+ | |
551 * | | copied spec. ^ commit() on | | discard() on
552 * | | into principal | active CPU | | active CPU
553 * | | | commit() | |
554 * V V | | |
555 * +----------+ +--------+ +-----------+
556 * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
557 * +----------+ speculation() +--------+ speculate() +-----------+
558 * ^ ^ | | |
559 * | | | discard() | |
560 * | | asynchronously | discard() on | | speculate()
561 * | | cleaned V inactive CPU | | on inactive
562 * | | +------------+ | | CPU
563 * | +-------------------| DISCARDING |<-----------------+ |
564 * | +------------+ |
565 * | asynchronously ^ |
566 * | copied spec. | discard() |
567 * | into principal +------------------------+ |
568 * | | V
569 * +----------------+ commit() +------------+
570 * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
571 * +----------------+ +------------+
572 */
573typedef enum dtrace_speculation_state {
574 DTRACESPEC_INACTIVE = 0,
575 DTRACESPEC_ACTIVE,
576 DTRACESPEC_ACTIVEONE,
577 DTRACESPEC_ACTIVEMANY,
578 DTRACESPEC_COMMITTING,
579 DTRACESPEC_COMMITTINGMANY,
580 DTRACESPEC_DISCARDING
581} dtrace_speculation_state_t;
582
583typedef struct dtrace_speculation {
584 dtrace_speculation_state_t dtsp_state; /* current speculation state */
585 int dtsp_cleaning; /* non-zero if being cleaned */
586 dtrace_buffer_t *dtsp_buffer; /* speculative buffer */
587} dtrace_speculation_t;
588
589/*
590 * DTrace Dynamic Variables
591 *
592 * The dynamic variable problem is obviously decomposed into two subproblems:
593 * allocating new dynamic storage, and freeing old dynamic storage. The
594 * presence of the second problem makes the first much more complicated -- or
595 * rather, the absence of the second renders the first trivial. This is the
596 * case with aggregations, for which there is effectively no deallocation of
597 * dynamic storage. (Or more accurately, all dynamic storage is deallocated
598 * when a snapshot is taken of the aggregation.) As DTrace dynamic variables
599 * allow for both dynamic allocation and dynamic deallocation, the
600 * implementation of dynamic variables is quite a bit more complicated than
601 * that of their aggregation kin.
602 *
603 * We observe that allocating new dynamic storage is tricky only because the
604 * size can vary -- the allocation problem is much easier if allocation sizes
605 * are uniform. We further observe that in D, the size of dynamic variables is
606 * actually _not_ dynamic -- dynamic variable sizes may be determined by static
607 * analysis of DIF text. (This is true even of putatively dynamically-sized
608 * objects like strings and stacks, the sizes of which are dictated by the
609 * "stringsize" and "stackframes" variables, respectively.) We exploit this by
610 * performing this analysis on all DIF before enabling any probes. For each
611 * dynamic load or store, we calculate the dynamically-allocated size plus the
612 * size of the dtrace_dynvar structure plus the storage required to key the
613 * data. For all DIF, we take the largest value and dub it the _chunksize_.
614 * We then divide dynamic memory into two parts: a hash table that is wide
615 * enough to have every chunk in its own bucket, and a larger region of equal
616 * chunksize units. Whenever we wish to dynamically allocate a variable, we
617 * always allocate a single chunk of memory. Depending on the uniformity of
618 * allocation, this will waste some amount of memory -- but it eliminates the
619 * non-determinism inherent in traditional heap fragmentation.
620 *
621 * Dynamic objects are allocated by storing a non-zero value to them; they are
622 * deallocated by storing a zero value to them. Dynamic variables are
623 * complicated enormously by being shared between CPUs. In particular,
624 * consider the following scenario:
625 *
626 * CPU A CPU B
627 * +---------------------------------+ +---------------------------------+
628 * | | | |
629 * | allocates dynamic object a[123] | | |
630 * | by storing the value 345 to it | | |
631 * | ---------> |
632 * | | | wishing to load from object |
633 * | | | a[123], performs lookup in |
634 * | | | dynamic variable space |
635 * | <--------- |
636 * | deallocates object a[123] by | | |
637 * | storing 0 to it | | |
638 * | | | |
639 * | allocates dynamic object b[567] | | performs load from a[123] |
640 * | by storing the value 789 to it | | |
641 * : : : :
642 * . . . .
643 *
644 * This is obviously a race in the D program, but there are nonetheless only
645 * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly,
646 * CPU B may _not_ see the value 789 for a[123].
647 *
648 * There are essentially two ways to deal with this:
649 *
650 * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load
651 * from a[123], it needs to lock a[123] and hold the lock for the
652 * duration that it wishes to manipulate it.
653 *
654 * (2) Avoid reusing freed chunks until it is known that no CPU is referring
655 * to them.
656 *
657 * The implementation of (1) is rife with complexity, because it requires the
658 * user of a dynamic variable to explicitly decree when they are done using it.
659 * Were all variables by value, this perhaps wouldn't be debilitating -- but
660 * dynamic variables of non-scalar types are tracked by reference. That is, if
661 * a dynamic variable is, say, a string, and that variable is to be traced to,
662 * say, the principal buffer, the DIF emulation code returns to the main
663 * dtrace_probe() loop a pointer to the underlying storage, not the contents of
664 * the storage. Further, code calling on DIF emulation would have to be aware
665 * that the DIF emulation has returned a reference to a dynamic variable that
666 * has been potentially locked. The variable would have to be unlocked after
667 * the main dtrace_probe() loop is finished with the variable, and the main
668 * dtrace_probe() loop would have to be careful to not call any further DIF
669 * emulation while the variable is locked to avoid deadlock. More generally,
670 * if one were to implement (1), DIF emulation code dealing with dynamic
671 * variables could only deal with one dynamic variable at a time (lest deadlock
672 * result). To sum, (1) exports too much subtlety to the users of dynamic
673 * variables -- increasing maintenance burden and imposing serious constraints
674 * on future DTrace development.
675 *
676 * The implementation of (2) is also complex, but the complexity is more
677 * manageable. We need to be sure that when a variable is deallocated, it is
678 * not placed on a traditional free list, but rather on a _dirty_ list. Once a
679 * variable is on a dirty list, it cannot be found by CPUs performing a
680 * subsequent lookup of the variable -- but it may still be in use by other
681 * CPUs. To assure that all CPUs that may be seeing the old variable have
682 * cleared out of probe context, a dtrace_sync() can be issued. Once the
683 * dtrace_sync() has completed, it can be known that all CPUs are done
684 * manipulating the dynamic variable -- the dirty list can be atomically
685 * appended to the free list. Unfortunately, there's a slight hiccup in this
686 * mechanism: dtrace_sync() may not be issued from probe context. The
687 * dtrace_sync() must be therefore issued asynchronously from non-probe
688 * context. For this we rely on the DTrace cleaner, a cyclic that runs at the
689 * "cleanrate" frequency. To ease this implementation, we define several chunk
690 * lists:
691 *
692 * - Dirty. Deallocated chunks, not yet cleaned. Not available.
693 *
694 * - Rinsing. Formerly dirty chunks that are currently being asynchronously
695 * cleaned. Not available, but will be shortly. Dynamic variable
696 * allocation may not spin or block for availability, however.
697 *
698 * - Clean. Clean chunks, ready for allocation -- but not on the free list.
699 *
700 * - Free. Available for allocation.
701 *
702 * Moreover, to avoid absurd contention, _each_ of these lists is implemented
703 * on a per-CPU basis. This is only for performance, not correctness; chunks
704 * may be allocated from another CPU's free list. The algorithm for allocation
705 * then is this:
706 *
707 * (1) Attempt to atomically allocate from current CPU's free list. If list
708 * is non-empty and allocation is successful, allocation is complete.
709 *
710 * (2) If the clean list is non-empty, atomically move it to the free list,
711 * and reattempt (1).
712 *
713 * (3) If the dynamic variable space is in the CLEAN state, look for free
714 * and clean lists on other CPUs by setting the current CPU to the next
715 * CPU, and reattempting (1). If the next CPU is the current CPU (that
716 * is, if all CPUs have been checked), atomically switch the state of
717 * the dynamic variable space based on the following:
718 *
719 * - If no free chunks were found and no dirty chunks were found,
720 * atomically set the state to EMPTY.
721 *
722 * - If dirty chunks were found, atomically set the state to DIRTY.
723 *
724 * - If rinsing chunks were found, atomically set the state to RINSING.
725 *
726 * (4) Based on state of dynamic variable space state, increment appropriate
727 * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
728 * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
729 * RINSING state). Fail the allocation.
730 *
731 * The cleaning cyclic operates with the following algorithm: for all CPUs
732 * with a non-empty dirty list, atomically move the dirty list to the rinsing
733 * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list,
734 * atomically move the rinsing list to the clean list. Perform another
735 * dtrace_sync(). By this point, all CPUs have seen the new clean list; the
736 * state of the dynamic variable space can be restored to CLEAN.
737 *
738 * There exist two final races that merit explanation. The first is a simple
739 * allocation race:
740 *
741 * CPU A CPU B
742 * +---------------------------------+ +---------------------------------+
743 * | | | |
744 * | allocates dynamic object a[123] | | allocates dynamic object a[123] |
745 * | by storing the value 345 to it | | by storing the value 567 to it |
746 * | | | |
747 * : : : :
748 * . . . .
749 *
750 * Again, this is a race in the D program. It can be resolved by having a[123]
751 * hold the value 345 or a[123] hold the value 567 -- but it must be true that
752 * a[123] have only _one_ of these values. (That is, the racing CPUs may not
753 * put the same element twice on the same hash chain.) This is resolved
754 * simply: before the allocation is undertaken, the start of the new chunk's
755 * hash chain is noted. Later, after the allocation is complete, the hash
756 * chain is atomically switched to point to the new element. If this fails
757 * (because of either concurrent allocations or an allocation concurrent with a
758 * deletion), the newly allocated chunk is deallocated to the dirty list, and
759 * the whole process of looking up (and potentially allocating) the dynamic
760 * variable is reattempted.
761 *
762 * The final race is a simple deallocation race:
763 *
764 * CPU A CPU B
765 * +---------------------------------+ +---------------------------------+
766 * | | | |
767 * | deallocates dynamic object | | deallocates dynamic object |
768 * | a[123] by storing the value 0 | | a[123] by storing the value 0 |
769 * | to it | | to it |
770 * | | | |
771 * : : : :
772 * . . . .
773 *
774 * Once again, this is a race in the D program, but it is one that we must
775 * handle without corrupting the underlying data structures. Because
776 * deallocations require the deletion of a chunk from the middle of a hash
777 * chain, we cannot use a single-word atomic operation to remove it. For this,
778 * we add a spin lock to the hash buckets that is _only_ used for deallocations
779 * (allocation races are handled as above). Further, this spin lock is _only_
780 * held for the duration of the delete; before control is returned to the DIF
781 * emulation code, the hash bucket is unlocked.
782 */
783typedef struct dtrace_key {
784 uint64_t dttk_value; /* data value or data pointer */
785 uint64_t dttk_size; /* 0 if by-val, >0 if by-ref */
786} dtrace_key_t;
787
788typedef struct dtrace_tuple {
789 uint32_t dtt_nkeys; /* number of keys in tuple */
790 uint32_t dtt_pad; /* padding */
791 dtrace_key_t dtt_key[1]; /* array of tuple keys */
792} dtrace_tuple_t;
793
794typedef struct dtrace_dynvar {
795 uint64_t dtdv_hashval; /* hash value -- 0 if free */
796 struct dtrace_dynvar *dtdv_next; /* next on list or hash chain */
797 void *dtdv_data; /* pointer to data */
798 dtrace_tuple_t dtdv_tuple; /* tuple key */
799} dtrace_dynvar_t;
800
801typedef enum dtrace_dynvar_op {
802 DTRACE_DYNVAR_ALLOC,
803 DTRACE_DYNVAR_NOALLOC,
804 DTRACE_DYNVAR_DEALLOC
805} dtrace_dynvar_op_t;
806
807typedef struct dtrace_dynhash {
808 dtrace_dynvar_t *dtdh_chain; /* hash chain for this bucket */
809 uintptr_t dtdh_lock; /* deallocation lock */
810#ifdef _LP64
811 uintptr_t dtdh_pad[6]; /* pad to avoid false sharing */
812#else
813 uintptr_t dtdh_pad[14]; /* pad to avoid false sharing */
814#endif
815} dtrace_dynhash_t;
816
817typedef struct dtrace_dstate_percpu {
818 dtrace_dynvar_t *dtdsc_free; /* free list for this CPU */
819 dtrace_dynvar_t *dtdsc_dirty; /* dirty list for this CPU */
820 dtrace_dynvar_t *dtdsc_rinsing; /* rinsing list for this CPU */
821 dtrace_dynvar_t *dtdsc_clean; /* clean list for this CPU */
822 uint64_t dtdsc_drops; /* number of capacity drops */
823 uint64_t dtdsc_dirty_drops; /* number of dirty drops */
824 uint64_t dtdsc_rinsing_drops; /* number of rinsing drops */
825#ifdef _LP64
826 uint64_t dtdsc_pad; /* pad to avoid false sharing */
827#else
828 uint64_t dtdsc_pad[2]; /* pad to avoid false sharing */
829#endif
830} dtrace_dstate_percpu_t;
831
832typedef enum dtrace_dstate_state {
833 DTRACE_DSTATE_CLEAN = 0,
834 DTRACE_DSTATE_EMPTY,
835 DTRACE_DSTATE_DIRTY,
836 DTRACE_DSTATE_RINSING
837} dtrace_dstate_state_t;
838
839typedef struct dtrace_dstate {
840 void *dtds_base; /* base of dynamic var. space */
841 size_t dtds_size; /* size of dynamic var. space */
842 size_t dtds_hashsize; /* number of buckets in hash */
843 size_t dtds_chunksize; /* size of each chunk */
844 dtrace_dynhash_t *dtds_hash; /* pointer to hash table */
845 dtrace_dstate_state_t dtds_state; /* current dynamic var. state */
846 dtrace_dstate_percpu_t *dtds_percpu; /* per-CPU dyn. var. state */
847} dtrace_dstate_t;
848
849/*
850 * DTrace Variable State
851 *
852 * The DTrace variable state tracks user-defined variables in its dtrace_vstate
853 * structure. Each DTrace consumer has exactly one dtrace_vstate structure,
854 * but some dtrace_vstate structures may exist without a corresponding DTrace
855 * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>,
856 * user-defined variables can have one of three scopes:
857 *
858 * DIFV_SCOPE_GLOBAL => global scope
859 * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables)
860 * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables)
861 *
862 * The variable state tracks variables by both their scope and their allocation
863 * type:
864 *
865 * - The dtvs_globals and dtvs_locals members each point to an array of
866 * dtrace_statvar structures. These structures contain both the variable
867 * metadata (dtrace_difv structures) and the underlying storage for all
868 * statically allocated variables, including statically allocated
869 * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
870 *
871 * - The dtvs_tlocals member points to an array of dtrace_difv structures for
872 * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the
873 * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
874 * is allocated out of the dynamic variable space.
875 *
876 * - The dtvs_dynvars member is the dynamic variable state associated with the
877 * variable state. The dynamic variable state (described in "DTrace Dynamic
878 * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
879 * dynamically-allocated DIFV_SCOPE_GLOBAL variables.
880 */
881typedef struct dtrace_statvar {
882 uint64_t dtsv_data; /* data or pointer to it */
883 size_t dtsv_size; /* size of pointed-to data */
884 int dtsv_refcnt; /* reference count */
885 dtrace_difv_t dtsv_var; /* variable metadata */
886} dtrace_statvar_t;
887
888typedef struct dtrace_vstate {
889 dtrace_state_t *dtvs_state; /* back pointer to state */
890 dtrace_statvar_t **dtvs_globals; /* statically-allocated glbls */
891 int dtvs_nglobals; /* number of globals */
892 dtrace_difv_t *dtvs_tlocals; /* thread-local metadata */
893 int dtvs_ntlocals; /* number of thread-locals */
894 dtrace_statvar_t **dtvs_locals; /* clause-local data */
895 int dtvs_nlocals; /* number of clause-locals */
896 dtrace_dstate_t dtvs_dynvars; /* dynamic variable state */
897} dtrace_vstate_t;
898
899/*
900 * DTrace Machine State
901 *
902 * In the process of processing a fired probe, DTrace needs to track and/or
903 * cache some per-CPU state associated with that particular firing. This is
904 * state that is always discarded after the probe firing has completed, and
905 * much of it is not specific to any DTrace consumer, remaining valid across
906 * all ECBs. This state is tracked in the dtrace_mstate structure.
907 */
908#define DTRACE_MSTATE_ARGS 0x00000001
909#define DTRACE_MSTATE_PROBE 0x00000002
910#define DTRACE_MSTATE_EPID 0x00000004
911#define DTRACE_MSTATE_TIMESTAMP 0x00000008
912#define DTRACE_MSTATE_STACKDEPTH 0x00000010
913#define DTRACE_MSTATE_CALLER 0x00000020
914#define DTRACE_MSTATE_IPL 0x00000040
915#define DTRACE_MSTATE_FLTOFFS 0x00000080
916#define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100
917#define DTRACE_MSTATE_USTACKDEPTH 0x00000200
918#define DTRACE_MSTATE_UCALLER 0x00000400
919#define DTRACE_MSTATE_MACHTIMESTAMP 0x00000800
920
921typedef struct dtrace_mstate {
922 uintptr_t dtms_scratch_base; /* base of scratch space */
923 uintptr_t dtms_scratch_ptr; /* current scratch pointer */
924 size_t dtms_scratch_size; /* scratch size */
925 uint32_t dtms_present; /* variables that are present */
926 uint64_t dtms_arg[5]; /* cached arguments */
927 dtrace_epid_t dtms_epid; /* current EPID */
928 uint64_t dtms_timestamp; /* cached timestamp */
929 hrtime_t dtms_walltimestamp; /* cached wall timestamp */
930 uint64_t dtms_machtimestamp; /* cached mach absolute timestamp */
931 int dtms_stackdepth; /* cached stackdepth */
932 int dtms_ustackdepth; /* cached ustackdepth */
933 struct dtrace_probe *dtms_probe; /* current probe */
934 uintptr_t dtms_caller; /* cached caller */
935 uint64_t dtms_ucaller; /* cached user-level caller */
936 int dtms_ipl; /* cached interrupt pri lev */
937 int dtms_fltoffs; /* faulting DIFO offset */
938 uintptr_t dtms_strtok; /* saved strtok() pointer */
939 uintptr_t dtms_strtok_limit; /* upper bound of strtok ptr */
940 uint32_t dtms_access; /* memory access rights */
941 dtrace_difo_t *dtms_difo; /* current dif object */
942} dtrace_mstate_t;
943
944#define DTRACE_COND_OWNER 0x1
945#define DTRACE_COND_USERMODE 0x2
946#define DTRACE_COND_ZONEOWNER 0x4
947
948#define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */
949
950/*
951 * Access flag used by dtrace_mstate.dtms_access.
952 */
953#define DTRACE_ACCESS_KERNEL 0x1 /* the priv to read kmem */
954
955
956/*
957 * DTrace Activity
958 *
959 * Each DTrace consumer is in one of several states, which (for purposes of
960 * avoiding yet-another overloading of the noun "state") we call the current
961 * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on
962 * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may
963 * only transition in one direction; the activity transition diagram is a
964 * directed acyclic graph. The activity transition diagram is as follows:
965 *
966 *
967 *
968 * +----------+ +--------+ +--------+
969 * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
970 * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+
971 * before BEGIN | after BEGIN | | |
972 * | | | |
973 * exit() action | | | |
974 * from BEGIN ECB | | | |
975 * | | | |
976 * v | | |
977 * +----------+ exit() action | | |
978 * +-----------------------------| DRAINING |<-------------------+ | |
979 * | +----------+ | |
980 * | | | |
981 * | dtrace_stop(), | | |
982 * | before END | | |
983 * | | | |
984 * | v | |
985 * | +---------+ +----------+ | |
986 * | | STOPPED |<----------------| COOLDOWN |<----------------------+ |
987 * | +---------+ dtrace_stop(), +----------+ dtrace_stop(), |
988 * | after END before END |
989 * | |
990 * | +--------+ |
991 * +----------------------------->| KILLED |<--------------------------+
992 * deadman timeout or +--------+ deadman timeout or
993 * killed consumer killed consumer
994 *
995 * Note that once a DTrace consumer has stopped tracing, there is no way to
996 * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
997 * the DTrace pseudodevice.
998 */
999typedef enum dtrace_activity {
1000 DTRACE_ACTIVITY_INACTIVE = 0, /* not yet running */
1001 DTRACE_ACTIVITY_WARMUP, /* while starting */
1002 DTRACE_ACTIVITY_ACTIVE, /* running */
1003 DTRACE_ACTIVITY_DRAINING, /* before stopping */
1004 DTRACE_ACTIVITY_COOLDOWN, /* while stopping */
1005 DTRACE_ACTIVITY_STOPPED, /* after stopping */
1006 DTRACE_ACTIVITY_KILLED /* killed */
1007} dtrace_activity_t;
1008
1009
1010/*
1011 * APPLE NOTE: DTrace dof modes implementation
1012 *
1013 * DTrace has four "dof modes". They are:
1014 *
1015 * DTRACE_DOF_MODE_NEVER Never load any dof, period.
1016 * DTRACE_DOF_MODE_LAZY_ON Defer loading dof until later
1017 * DTRACE_DOF_MODE_LAZY_OFF Load all deferred dof now, and any new dof
1018 * DTRACE_DOF_MODE_NON_LAZY Load all dof immediately.
1019 *
1020 * It is legal to transition between the two lazy modes. The NEVER and
1021 * NON_LAZY modes are permanent, and must not change once set.
1022 *
1023 * The current dof mode is kept in dtrace_dof_mode, which is protected by the
1024 * dtrace_dof_mode_lock. This is a RW lock, reads require shared access, writes
1025 * require exclusive access. Because NEVER and NON_LAZY are permanent states,
1026 * it is legal to test for those modes without holding the dof mode lock.
1027 *
1028 * Lock ordering is dof mode lock before any dtrace lock, and before the
1029 * process p_dtrace_sprlock. In general, other locks should not be held when
1030 * taking the dof mode lock. Acquiring the dof mode lock in exclusive mode
1031 * will block process fork, exec, and exit, so it should be held exclusive
1032 * for as short a time as possible.
1033 */
1034
1035#define DTRACE_DOF_MODE_NEVER 0
1036#define DTRACE_DOF_MODE_LAZY_ON 1
1037#define DTRACE_DOF_MODE_LAZY_OFF 2
1038#define DTRACE_DOF_MODE_NON_LAZY 3
1039
1040/*
1041 * dtrace kernel symbol modes are used to control when the kernel may dispose of
1042 * symbol information used by the fbt/sdt provider. The kernel itself, as well as
1043 * every kext, has symbol table/nlist info that has historically been preserved
1044 * for dtrace's use. This allowed dtrace to be lazy about allocating fbt/sdt probes,
1045 * at the expense of keeping the symbol info in the kernel permanently.
1046 *
1047 * Starting in 10.7+, fbt probes may be created from userspace, in the same
1048 * fashion as pid probes. The kernel allows dtrace "first right of refusal"
1049 * whenever symbol data becomes available (such as a kext load). If dtrace is
1050 * active, it will immediately read/copy the needed data, and then the kernel
1051 * may free it. If dtrace is not active, it returns immediately, having done
1052 * no work or allocations, and the symbol data is freed. Should dtrace need
1053 * this data later, it is expected that the userspace client will push the
1054 * data into the kernel via ioctl calls.
1055 *
1056 * The kernel symbol modes are used to control what dtrace does with symbol data:
1057 *
1058 * DTRACE_KERNEL_SYMBOLS_NEVER Effectively disables fbt/sdt
1059 * DTRACE_KERNEL_SYMBOLS_FROM_KERNEL Immediately read/copy symbol data
1060 * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE Wait for symbols from userspace
1061 * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL Immediately read/copy symbol data
1062 *
1063 * It is legal to transition between DTRACE_KERNEL_SYMBOLS_FROM_KERNEL and
1064 * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE. The DTRACE_KERNEL_SYMBOLS_NEVER and
1065 * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL are permanent modes, intended to
1066 * disable fbt probes entirely, or prevent any symbols being loaded from
1067 * userspace.
1068*
1069 * The kernel symbol mode is kept in dtrace_kernel_symbol_mode, which is protected
1070 * by the dtrace_lock.
1071 */
1072
1073#define DTRACE_KERNEL_SYMBOLS_NEVER 0
1074#define DTRACE_KERNEL_SYMBOLS_FROM_KERNEL 1
1075#define DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE 2
1076#define DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL 3
1077
1078
1079/*
1080 * DTrace Helper Implementation
1081 *
1082 * A description of the helper architecture may be found in <sys/dtrace.h>.
1083 * Each process contains a pointer to its helpers in its p_dtrace_helpers
1084 * member. This is a pointer to a dtrace_helpers structure, which contains an
1085 * array of pointers to dtrace_helper structures, helper variable state (shared
1086 * among a process's helpers) and a generation count. (The generation count is
1087 * used to provide an identifier when a helper is added so that it may be
1088 * subsequently removed.) The dtrace_helper structure is self-explanatory,
1089 * containing pointers to the objects needed to execute the helper. Note that
1090 * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more
1091 * than dtrace_helpers_max are allowed per-process.
1092 */
1093#define DTRACE_HELPER_ACTION_USTACK 0
1094#define DTRACE_NHELPER_ACTIONS 1
1095
1096typedef struct dtrace_helper_action {
1097 int dtha_generation; /* helper action generation */
1098 int dtha_nactions; /* number of actions */
1099 dtrace_difo_t *dtha_predicate; /* helper action predicate */
1100 dtrace_difo_t **dtha_actions; /* array of actions */
1101 struct dtrace_helper_action *dtha_next; /* next helper action */
1102} dtrace_helper_action_t;
1103
1104typedef struct dtrace_helper_provider {
1105 int dthp_generation; /* helper provider generation */
1106 uint32_t dthp_ref; /* reference count */
1107 dof_helper_t dthp_prov; /* DOF w/ provider and probes */
1108} dtrace_helper_provider_t;
1109
1110typedef struct dtrace_helpers {
1111 dtrace_helper_action_t **dthps_actions; /* array of helper actions */
1112 dtrace_vstate_t dthps_vstate; /* helper action var. state */
1113 dtrace_helper_provider_t **dthps_provs; /* array of providers */
1114 uint_t dthps_nprovs; /* count of providers */
1115 uint_t dthps_maxprovs; /* provider array size */
1116 int dthps_generation; /* current generation */
1117 pid_t dthps_pid; /* pid of associated proc */
1118 int dthps_deferred; /* helper in deferred list */
1119 struct dtrace_helpers *dthps_next; /* next pointer */
1120 struct dtrace_helpers *dthps_prev; /* prev pointer */
1121} dtrace_helpers_t;
1122
1123/*
1124 * DTrace Helper Action Tracing
1125 *
1126 * Debugging helper actions can be arduous. To ease the development and
1127 * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
1128 * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which
1129 * it is by default on DEBUG kernels), all helper activity will be traced to a
1130 * global, in-kernel ring buffer. Each entry includes a pointer to the specific
1131 * helper, the location within the helper, and a trace of all local variables.
1132 * The ring buffer may be displayed in a human-readable format with the
1133 * ::dtrace_helptrace mdb(1) dcmd.
1134 */
1135#define DTRACE_HELPTRACE_NEXT (-1)
1136#define DTRACE_HELPTRACE_DONE (-2)
1137#define DTRACE_HELPTRACE_ERR (-3)
1138
1139
1140typedef struct dtrace_helptrace {
1141 dtrace_helper_action_t *dtht_helper; /* helper action */
1142 int dtht_where; /* where in helper action */
1143 int dtht_nlocals; /* number of locals */
1144 int dtht_fault; /* type of fault (if any) */
1145 int dtht_fltoffs; /* DIF offset */
1146 uint64_t dtht_illval; /* faulting value */
1147 uint64_t dtht_locals[1]; /* local variables */
1148} dtrace_helptrace_t;
1149
1150/*
1151 * DTrace Credentials
1152 *
1153 * In probe context, we have limited flexibility to examine the credentials
1154 * of the DTrace consumer that created a particular enabling. We use
1155 * the Least Privilege interfaces to cache the consumer's cred pointer and
1156 * some facts about that credential in a dtrace_cred_t structure. These
1157 * can limit the consumer's breadth of visibility and what actions the
1158 * consumer may take.
1159 */
1160#define DTRACE_CRV_ALLPROC 0x01
1161#define DTRACE_CRV_KERNEL 0x02
1162#define DTRACE_CRV_ALLZONE 0x04
1163
1164#define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
1165 DTRACE_CRV_ALLZONE)
1166
1167#define DTRACE_CRA_PROC 0x0001
1168#define DTRACE_CRA_PROC_CONTROL 0x0002
1169#define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER 0x0004
1170#define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE 0x0008
1171#define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG 0x0010
1172#define DTRACE_CRA_KERNEL 0x0020
1173#define DTRACE_CRA_KERNEL_DESTRUCTIVE 0x0040
1174
1175#define DTRACE_CRA_ALL (DTRACE_CRA_PROC | \
1176 DTRACE_CRA_PROC_CONTROL | \
1177 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
1178 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
1179 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
1180 DTRACE_CRA_KERNEL | \
1181 DTRACE_CRA_KERNEL_DESTRUCTIVE)
1182
1183typedef struct dtrace_cred {
1184 cred_t *dcr_cred;
1185 uint8_t dcr_destructive;
1186 uint8_t dcr_visible;
1187 uint16_t dcr_action;
1188} dtrace_cred_t;
1189
1190/*
1191 * DTrace Consumer State
1192 *
1193 * Each DTrace consumer has an associated dtrace_state structure that contains
1194 * its in-kernel DTrace state -- including options, credentials, statistics and
1195 * pointers to ECBs, buffers, speculations and formats. A dtrace_state
1196 * structure is also allocated for anonymous enablings. When anonymous state
1197 * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
1198 * dtrace_state structure.
1199 */
1200struct dtrace_state {
1201 dev_t dts_dev; /* device */
1202 int dts_necbs; /* total number of ECBs */
1203 dtrace_ecb_t **dts_ecbs; /* array of ECBs */
1204 dtrace_epid_t dts_epid; /* next EPID to allocate */
1205 size_t dts_needed; /* greatest needed space */
1206 struct dtrace_state *dts_anon; /* anon. state, if grabbed */
1207 dtrace_activity_t dts_activity; /* current activity */
1208 dtrace_vstate_t dts_vstate; /* variable state */
1209 dtrace_buffer_t *dts_buffer; /* principal buffer */
1210 dtrace_buffer_t *dts_aggbuffer; /* aggregation buffer */
1211 dtrace_speculation_t *dts_speculations; /* speculation array */
1212 int dts_nspeculations; /* number of speculations */
1213 int dts_naggregations; /* number of aggregations */
1214 dtrace_aggregation_t **dts_aggregations; /* aggregation array */
1215 vmem_t *dts_aggid_arena; /* arena for aggregation IDs */
1216 uint64_t dts_errors; /* total number of errors */
1217 uint32_t dts_speculations_busy; /* number of spec. busy */
1218 uint32_t dts_speculations_unavail; /* number of spec unavail */
1219 uint32_t dts_stkstroverflows; /* stack string tab overflows */
1220 uint32_t dts_dblerrors; /* errors in ERROR probes */
1221 uint32_t dts_reserve; /* space reserved for END */
1222 hrtime_t dts_laststatus; /* time of last status */
1223 cyclic_id_t dts_cleaner; /* cleaning cyclic */
1224 cyclic_id_t dts_deadman; /* deadman cyclic */
1225 hrtime_t dts_alive; /* time last alive */
1226 char dts_speculates; /* boolean: has speculations */
1227 char dts_destructive; /* boolean: has dest. actions */
1228 int dts_nformats; /* number of formats */
1229 char **dts_formats; /* format string array */
1230 dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
1231 dtrace_cred_t dts_cred; /* credentials */
1232 size_t dts_nretained; /* number of retained enabs */
1233 uint64_t dts_arg_error_illval;
1234 uint32_t dts_buf_over_limit; /* number of bufs over dtb_limit */
1235};
1236
1237struct dtrace_provider {
1238 dtrace_pattr_t dtpv_attr; /* provider attributes */
1239 dtrace_ppriv_t dtpv_priv; /* provider privileges */
1240 dtrace_pops_t dtpv_pops; /* provider operations */
1241 char *dtpv_name; /* provider name */
1242 void *dtpv_arg; /* provider argument */
1243 uint_t dtpv_defunct; /* boolean: defunct provider */
1244 struct dtrace_provider *dtpv_next; /* next provider */
1245 uint64_t dtpv_probe_count; /* number of associated probes */
1246 uint64_t dtpv_ecb_count; /* number of associated enabled ECBs */
1247};
1248
1249struct dtrace_meta {
1250 dtrace_mops_t dtm_mops; /* meta provider operations */
1251 char *dtm_name; /* meta provider name */
1252 void *dtm_arg; /* meta provider user arg */
1253 uint64_t dtm_count; /* number of associated providers */
1254};
1255
1256/*
1257 * DTrace Enablings
1258 *
1259 * A dtrace_enabling structure is used to track a collection of ECB
1260 * descriptions -- before they have been turned into actual ECBs. This is
1261 * created as a result of DOF processing, and is generally used to generate
1262 * ECBs immediately thereafter. However, enablings are also generally
1263 * retained should the probes they describe be created at a later time; as
1264 * each new module or provider registers with the framework, the retained
1265 * enablings are reevaluated, with any new match resulting in new ECBs. To
1266 * prevent probes from being matched more than once, the enabling tracks the
1267 * last probe generation matched, and only matches probes from subsequent
1268 * generations.
1269 */
1270typedef struct dtrace_enabling {
1271 dtrace_ecbdesc_t **dten_desc; /* all ECB descriptions */
1272 int dten_ndesc; /* number of ECB descriptions */
1273 int dten_maxdesc; /* size of ECB array */
1274 dtrace_vstate_t *dten_vstate; /* associated variable state */
1275 dtrace_genid_t dten_probegen; /* matched probe generation */
1276 dtrace_ecbdesc_t *dten_current; /* current ECB description */
1277 int dten_error; /* current error value */
1278 int dten_primed; /* boolean: set if primed */
1279 struct dtrace_enabling *dten_prev; /* previous enabling */
1280 struct dtrace_enabling *dten_next; /* next enabling */
1281} dtrace_enabling_t;
1282
1283/*
1284 * DTrace Anonymous Enablings
1285 *
1286 * Anonymous enablings are DTrace enablings that are not associated with a
1287 * controlling process, but rather derive their enabling from DOF stored as
1288 * properties in the dtrace.conf file. If there is an anonymous enabling, a
1289 * DTrace consumer state and enabling are created on attach. The state may be
1290 * subsequently grabbed by the first consumer specifying the "grabanon"
1291 * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will
1292 * refuse to unload.
1293 */
1294typedef struct dtrace_anon {
1295 dtrace_state_t *dta_state; /* DTrace consumer state */
1296 dtrace_enabling_t *dta_enabling; /* pointer to enabling */
1297 processorid_t dta_beganon; /* which CPU BEGIN ran on */
1298} dtrace_anon_t;
1299
1300/*
1301 * DTrace Error Debugging
1302 */
1303#if DEBUG
1304#define DTRACE_ERRDEBUG
1305#endif
1306
1307#ifdef DTRACE_ERRDEBUG
1308
1309typedef struct dtrace_errhash {
1310 const char *dter_msg; /* error message */
1311 int dter_count; /* number of times seen */
1312} dtrace_errhash_t;
1313
1314#define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */
1315
1316#endif /* DTRACE_ERRDEBUG */
1317
1318
1319typedef struct dtrace_string dtrace_string_t;
1320
1321typedef struct dtrace_string {
1322 dtrace_string_t *dtst_next;
1323 dtrace_string_t *dtst_prev;
1324 uint32_t dtst_refcount;
1325 char dtst_str[];
1326} dtrace_string_t;
1327
1328/**
1329 * DTrace Matching pre-conditions
1330 *
1331 * Used when matching new probes to discard matching of enablings that
1332 * doesn't match the condition tested by dmc_func
1333 */
1334typedef struct dtrace_match_cond {
1335 int (*dmc_func)(dtrace_probedesc_t*, void*);
1336 void *dmc_data;
1337} dtrace_match_cond_t;
1338
1339
1340/*
1341 * DTrace Toxic Ranges
1342 *
1343 * DTrace supports safe loads from probe context; if the address turns out to
1344 * be invalid, a bit will be set by the kernel indicating that DTrace
1345 * encountered a memory error, and DTrace will propagate the error to the user
1346 * accordingly. However, there may exist some regions of memory in which an
1347 * arbitrary load can change system state, and from which it is impossible to
1348 * recover from such a load after it has been attempted. Examples of this may
1349 * include memory in which programmable I/O registers are mapped (for which a
1350 * read may have some implications for the device) or (in the specific case of
1351 * UltraSPARC-I and -II) the virtual address hole. The platform is required
1352 * to make DTrace aware of these toxic ranges; DTrace will then check that
1353 * target addresses are not in a toxic range before attempting to issue a
1354 * safe load.
1355 */
1356typedef struct dtrace_toxrange {
1357 uintptr_t dtt_base; /* base of toxic range */
1358 uintptr_t dtt_limit; /* limit of toxic range */
1359} dtrace_toxrange_t;
1360
1361extern uint64_t dtrace_getarg(int, int, dtrace_mstate_t*, dtrace_vstate_t*);
1362extern int dtrace_getipl(void);
1363extern uintptr_t dtrace_caller(int);
1364extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
1365extern void *dtrace_casptr(void *, void *, void *);
1366extern void dtrace_copyin(user_addr_t, uintptr_t, size_t, volatile uint16_t *);
1367extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t *);
1368extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *);
1369extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *);
1370extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
1371extern uint64_t dtrace_load64(uintptr_t);
1372extern int dtrace_canload(uint64_t, size_t, dtrace_mstate_t*, dtrace_vstate_t*);
1373
1374extern uint64_t dtrace_getreg(struct regs *, uint_t);
1375extern int dtrace_getstackdepth(int);
1376extern void dtrace_getupcstack(uint64_t *, int);
1377extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
1378extern int dtrace_getustackdepth(void);
1379extern uintptr_t dtrace_fulword(void *);
1380extern uint8_t dtrace_fuword8(user_addr_t);
1381extern uint16_t dtrace_fuword16(user_addr_t);
1382extern uint32_t dtrace_fuword32(user_addr_t);
1383extern uint64_t dtrace_fuword64(user_addr_t);
1384extern int dtrace_proc_waitfor(dtrace_procdesc_t*);
1385extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
1386 int, uint64_t);
1387extern int dtrace_assfail(const char *, const char *, int);
1388extern int dtrace_attached(void);
1389extern hrtime_t dtrace_gethrestime(void);
1390extern void dtrace_isa_init(void);
1391
1392extern void dtrace_flush_caches(void);
1393
1394extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
1395extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1396
1397/*
1398 * DTrace state handling
1399 */
1400extern minor_t dtrace_state_reserve(void);
1401extern dtrace_state_t* dtrace_state_allocate(minor_t minor);
1402extern dtrace_state_t* dtrace_state_get(minor_t minor);
1403extern void dtrace_state_free(minor_t minor);
1404
1405/*
1406 * DTrace restriction checks
1407 */
1408extern void dtrace_restriction_policy_load(void);
1409extern boolean_t dtrace_is_restricted(void);
1410extern boolean_t dtrace_are_restrictions_relaxed(void);
1411extern boolean_t dtrace_fbt_probes_restricted(void);
1412extern boolean_t dtrace_sdt_probes_restricted(void);
1413extern boolean_t dtrace_can_attach_to_proc(proc_t);
1414
1415/*
1416 * DTrace Assertions
1417 *
1418 * DTrace calls ASSERT and VERIFY from probe context. To assure that a failed
1419 * ASSERT or VERIFYdoes not induce a markedly more catastrophic failure (e.g.,
1420 * one from which a dump cannot be gleaned), DTrace must define its own ASSERT
1421 * and VERIFY macros to be ones that may safely be called from probe context.
1422 * This header file must thus be included by any DTrace component that calls
1423 * ASSERT and/or VERIFY from probe context, and _only_ by those components.
1424 * (The only exception to this is kernel debugging infrastructure at user-level
1425 * that doesn't depend on calling ASSERT.)
1426 */
1427#undef ASSERT
1428#undef VERIFY
1429
1430#define VERIFY(EX) ((void)((EX) || \
1431 dtrace_assfail(#EX, __FILE__, __LINE__)))
1432
1433#if DEBUG
1434#define ASSERT(EX) ((void)((EX) || \
1435 dtrace_assfail(#EX, __FILE__, __LINE__)))
1436#else
1437#define ASSERT(X) ((void)0)
1438#endif
1439
1440#ifdef __cplusplus
1441}
1442#endif
1443
1444#endif /* _SYS_DTRACE_IMPL_H */
1445
1446