ktstr/monitor/dump/mod.rs
1//! BPF map state dump for scheduler-failure post-mortem.
2//!
3//! [`dump_state`] is invoked by the freeze coordinator after the vCPU
4//! rendezvous succeeds (see `src/vmm/freeze_coord/mod.rs`). It enumerates every
5//! BPF map in the guest via [`BpfMapAccessor::maps`], filters out
6//! ktstr-internal probes (the framework's own probe and fentry skel
7//! maps), and dispatches per map type:
8//!
9//! - `BPF_MAP_TYPE_ARRAY` (and the `.bss` / `.data` / `.rodata`
10//! global-section maps libbpf creates as single-key arrays) — read
11//! the whole value buffer and render it via [`super::btf_render::render_value_with_mem`]
12//! so embedded `__arena` pointers chase into the captured arena pages.
13//! - `BPF_MAP_TYPE_HASH` — iterate (key, value) pairs, capped at
14//! [`MAX_HASH_ENTRIES`].
15//! - `BPF_MAP_TYPE_PERCPU_ARRAY` — read each CPU's slot for keys
16//! `0..min(max_entries, MAX_PERCPU_KEYS)`.
17//! - Other types — recorded as [`FailureDumpMap::error`] so the operator
18//! sees the gap rather than a silent omission.
19//!
20//! # BTF source — per-map program BTF loading
21//!
22//! The renderer loads each map's program BTF from guest memory at
23//! [`BpfMapInfo::btf_kva`], following the kernel `struct btf`'s
24//! `data`/`data_size`/`base_btf` fields. Split BTF (program types
25//! extending vmlinux) is parsed via [`Btf::from_split_bytes`] with
26//! the host's vmlinux BTF as the base (correct when host kernel ==
27//! guest kernel — ktstr's default and the common CI configuration).
28//! A per-`btf_kva` cache dedupes parses across maps sharing a
29//! program's BTF object. When per-map load fails (still-booting
30//! guest, untranslatable page, corrupted blob), the renderer falls
31//! back to the caller-supplied vmlinux BTF.
32//!
33//! # sdt_alloc pre-pass
34//!
35//! Before the per-map walk runs, [`dump_state`] runs a pre-pass
36//! that locates `sdt_alloc`-backed allocator instances inside the
37//! scheduler's `.bss` and surfaces every live per-task / per-cgroup
38//! allocation as structured records under
39//! [`FailureDumpReport::sdt_allocations`]. The walk runs only when
40//! every prerequisite is present:
41//! - the per-map dump deadline has not been exceeded (the
42//! pre-pass runs before every map render, and an earlier
43//! phase exhausting the budget skips the walk to keep the
44//! dump bounded),
45//! - the arena's `user_vm_start` is 4 GiB-aligned (low 32 bits
46//! zero — the bridge's address arithmetic treats slot starts as
47//! low-32 keys against a `[user_vm_start, user_vm_start + 4 GiB)`
48//! window; misalignment would silently mismap chases),
49//! - the scheduler exposes a `.bss` ARRAY map with non-zero
50//! `btf_kva` (so we can read its raw bytes and have a program
51//! BTF to resolve types against),
52//! - at least one `BPF_MAP_TYPE_ARENA` map snapshot succeeded
53//! (so we have `kern_vm_start` for arena pointer translation),
54//! - the program BTF carries `struct scx_allocator` (the scheduler
55//! links `lib/sdt_alloc.bpf.c`).
56//!
57//! When any prerequisite is missing, the pre-pass leaves
58//! `sdt_allocations` empty rather than failing the dump — the
59//! per-map page-granular [`super::arena::ArenaSnapshot`] still
60//! captures raw arena content for callers that don't need
61//! structured rendering. See [`super::sdt_alloc`] for the walker
62//! design.
63
64mod display;
65mod render_map;
66#[cfg(test)]
67mod tests;
68use render_map::*;
69
70use serde::{Deserialize, Serialize};
71
72use btf_rs::Btf;
73
74use super::arena::{ArenaSnapshot, BpfArenaOffsets, snapshot_arena};
75use super::bpf_map::{
76 BPF_MAP_TYPE_ARENA, BPF_MAP_TYPE_ARRAY, BpfMapAccessor, BpfMapInfo, GuestMemMapAccessor,
77};
78use super::btf_render::RenderedValue;
79use super::sdt_alloc::{
80 SdtAllocOffsets, SdtAllocatorSnapshot, discover_payload_btf_id, walk_sdt_allocator,
81};
82
83/// Borrow-only capture context for per-program runtime stats
84/// (cnt/nsecs/misses) populated alongside the BPF map dump.
85///
86/// Carries a borrowed [`super::bpf_prog::BpfProgAccessor`] plus the
87/// per-CPU offset array obtained from
88/// [`super::symbols::read_per_cpu_offsets`]. [`dump_state`] calls
89/// [`super::bpf_prog::BpfProgAccessor::struct_ops_runtime_stats`]
90/// with the supplied offsets and stores the resulting
91/// [`super::bpf_prog::ProgRuntimeStats`] vector in
92/// [`FailureDumpReport::prog_runtime_stats`].
93///
94/// Pass `None` to skip prog-runtime capture (e.g. when the
95/// accessor could not be constructed because `prog_idr` is
96/// missing or the BPF prog offsets did not resolve). The dump still
97/// renders every map the [`super::bpf_map::BpfMapAccessor`] enumerates.
98pub struct ProgRuntimeCapture<'a> {
99 /// Accessor for walking `prog_idr` and reading per-program
100 /// `bpf_prog_stats` slots. Trait dispatch lets the same dump
101 /// site consume either the guest-memory backend or the planned
102 /// live-host backend without committing to a concrete type.
103 pub accessor: &'a dyn super::bpf_prog::BpfProgAccessor,
104 /// Per-CPU offset array (`__per_cpu_offset[cpu]`) used to address
105 /// each CPU's `bpf_prog_stats` slot for summation.
106 pub per_cpu_offsets: &'a [u64],
107}
108
109/// Borrow-only capture context for per-CPU CPU-time / softirq / IRQ
110/// counters populated alongside the BPF map dump.
111///
112/// Carries the BTF-resolved field offsets for `kernel_cpustat`,
113/// `kernel_stat`, and `tick_sched`, the resolved `.data..percpu`
114/// section offsets of the three per-CPU symbols, and the
115/// `__per_cpu_offset[cpu]` array used to address each CPU's slot.
116///
117/// [`dump_state`] reads each CPU's slot via direct guest-memory
118/// reads against the supplied [`super::reader::GuestMem`] and
119/// records the result into [`FailureDumpReport::per_cpu_time`].
120/// Mirrors [`ProgRuntimeCapture`]'s "borrowed-only, optional"
121/// shape — when `None`, the dump skips the per-CPU time capture
122/// and leaves the field empty.
123///
124/// Skipped silently when the resolver could not locate any of the
125/// three per-CPU symbols (stripped vmlinux), the BTF offsets are
126/// not present (CPU-time accounting types missing), or
127/// `__per_cpu_offset` resolution returned an empty array. The
128/// capture is best-effort diagnostic data; its absence does not
129/// fail the dump.
130pub struct CpuTimeCapture<'a> {
131 /// Guest memory handle used to read each per-CPU slot.
132 pub mem: &'a super::reader::GuestMem,
133 /// BTF-resolved offsets for `kernel_cpustat::cpustat[]`,
134 /// `kernel_stat::softirqs[]`, `kernel_stat::irqs_sum`, and
135 /// optionally `tick_sched::iowait_sleeptime`.
136 pub offsets: &'a super::btf_offsets::CpuTimeOffsets,
137 /// Link-time KVA of the `kernel_cpustat` per-CPU symbol (the
138 /// value `st_value` carries in the vmlinux symbol table — the
139 /// template address the linker assigned). The runtime KVA on
140 /// CPU `cpu` is
141 /// [`super::symbols::per_cpu_kva`]`(kernel_cpustat_kva,
142 /// kaslr_offset, per_cpu_offsets[cpu])`.
143 pub kernel_cpustat_kva: u64,
144 /// Link-time KVA of the `kstat` per-CPU symbol. See
145 /// `kernel_cpustat_kva` for the runtime KVA formula.
146 pub kstat_kva: u64,
147 /// Link-time KVA of the `tick_cpu_sched` per-CPU symbol.
148 /// `None` when the kernel was built without
149 /// `CONFIG_NO_HZ_COMMON`; iowait_sleeptime capture is skipped.
150 pub tick_cpu_sched_kva: Option<u64>,
151 /// Per-CPU offset array (`__per_cpu_offset[cpu]`) — same array
152 /// the BPF prog-stats walker uses (see
153 /// [`super::symbols::read_per_cpu_offsets`]). Length determines
154 /// how many CPUs the walker visits.
155 pub per_cpu_offsets: &'a [u64],
156 /// Guest's `PAGE_OFFSET` (resolved via
157 /// [`super::symbols::resolve_page_offset`]). Used to translate
158 /// each CPU's per-CPU KVA to a guest physical address for the
159 /// memory read.
160 pub page_offset: u64,
161 /// Virtual KASLR offset that per-CPU KVA derivation needs to
162 /// bridge the link-time (`__per_cpu_start_LINK`) and runtime
163 /// (`__per_cpu_start_RUNTIME`) bases. Sourced from the shared
164 /// `kern_virt_kaslr` Arc populated by either the BSP MSR_LSTAR
165 /// derive (`crate::vmm::x86_64::msr_kaslr::read_and_derive`,
166 /// x86_64-only) or the guest-channel KERN_ADDRS `_text`
167 /// subtraction (`crate::vmm::freeze_coord::dispatch`, both
168 /// arches). 0 fallback matches KASLR-off / nokaslr-karg
169 /// semantics and collapses [`super::symbols::per_cpu_kva`] to
170 /// the no-slide formula. On aarch64 without `_text` in
171 /// /proc/kallsyms (kptr_restrict masked) the value stays 0
172 /// and per-CPU resolution relies on the `nokaslr` karg
173 /// (`src/vmm/setup.rs`) instead.
174 pub kaslr_offset: u64,
175}
176
177/// Per-cgroup PSI-irq host-walk inputs (Phase A). Borrowed-only/optional,
178/// mirroring [`CpuTimeCapture`]: the freeze coordinator builds it only when the
179/// cgroup-walk offsets, the `cgrp_dfl_root` symbol, and the `psi_group` offsets
180/// all resolve; otherwise [`DumpContext::cgroup_psi_capture`] is `None` and the
181/// per-cgroup axis reads loud-absent. The walk descends `cgrp_dfl_root` → the
182/// host-held workload-root path → leaf cgroups and reads each leaf's
183/// `cgroup->psi` PSI_IRQ_FULL (see [`super::cgroup_walk`]).
184pub struct CgroupPsiCapture<'a> {
185 /// Guest memory handle used to read the cgroup hierarchy + each psi_group.
186 pub mem: &'a super::reader::GuestMem,
187 /// BTF-resolved cgroup-hierarchy field offsets (`cgroup.{self,kn,psi}`,
188 /// `cgroup_subsys_state.{sibling,children}`, `cgroup_root.cgrp`,
189 /// `kernfs_node.name`).
190 pub cgroup_offsets: &'a super::btf_offsets::CgroupWalkOffsets,
191 /// BTF-resolved `psi_group` offsets (shared with the system-wide walk —
192 /// a per-cgroup psi_group is the same `struct psi_group`).
193 pub psi_offsets: &'a super::btf_offsets::PsiGroupOffsets,
194 /// RUNTIME KVA of the hierarchy root cgroup (`cgrp_dfl_root +
195 /// offsetof(cgroup_root, cgrp)`, with any virtual-KASLR slide applied by
196 /// the caller) — used for the children-list anchor compare at the root
197 /// level. Every descendant is a direct-mapped slab object.
198 pub root_cgroup_kva: u64,
199 /// Guest physical address of the hierarchy root cgroup (the kernel-image
200 /// translation of the link-time root KVA, done by the caller via
201 /// `GuestKernel::text_kva_to_pa`) — the walk's entry read.
202 pub root_cgroup_pa: u64,
203 /// The test's workload-root cgroup path (host-held VM config, default
204 /// `/sys/fs/cgroup/ktstr`). The walk descends ONLY this subtree, so the
205 /// scheduler's separate cgroup does not confound the per-cgroup axis.
206 pub workload_root_path: &'a str,
207 /// Guest `PAGE_OFFSET` for the direct-map (`kva_to_pa`) hops to every
208 /// descendant cgroup / kernfs_node / psi_group.
209 pub page_offset: u64,
210}
211
212/// Borrow-only capture context for per-task enrichment.
213///
214/// Carries the [`super::guest::GuestKernel`] (guest memory + symbol
215/// table), the BTF-resolved task/signal/pid/upid offsets, the cached
216/// sched_class symbol KVAs (for class-name decode and the
217/// PI-boost-out-of-SCX flag), the lock-slowpath symbol cache (for
218/// stack-trace pattern matching), AND the task list itself — a
219/// pre-collected `&[TaskWalkerEntry]` produced by a task walker
220/// (rq->scx walk, DSQ walk, init_task→tasks enumeration).
221///
222/// Mirrors the [`ProgRuntimeCapture`] / [`CpuTimeCapture`]
223/// borrowed-only-optional shape. When `dump_state` receives
224/// `Some(TaskEnrichmentCapture)`, it iterates `tasks` and calls
225/// [`super::task_enrichment::walk_task_enrichment`] for each entry,
226/// pushing results into [`FailureDumpReport::task_enrichments`]. When
227/// `None`, the field stays empty and
228/// [`FailureDumpReport::task_enrichments_unavailable`] gets a
229/// "no task walker available" diagnostic.
230///
231/// The walker producer (rq->scx walker etc.) is responsible for
232/// building this struct. Until walker dispatch lands, no walker
233/// exists; the freeze coordinator passes `None` and the field is
234/// plumbed but empty.
235pub struct TaskEnrichmentCapture<'a> {
236 /// Borrowed GuestKernel — provides memory access, page-table
237 /// translation context, and the vmlinux symbol table.
238 pub kernel: &'a super::guest::GuestKernel,
239 /// BTF-resolved offsets for the task/signal/pid/upid walk.
240 pub offsets: &'a super::btf_offsets::TaskEnrichmentOffsets,
241 /// Cached sched_class symbol KVAs for class decode + PI-boost
242 /// flag.
243 pub sched_classes: &'a super::task_enrichment::SchedClassRegistry,
244 /// Cached lock-slowpath symbol KVAs for stack-PC pattern
245 /// matching.
246 pub lock_slowpaths: &'a super::task_enrichment::LockSlowpathRegistry,
247 /// Tasks the walker discovered, plus per-task metadata
248 /// `walk_task_enrichment` needs (see [`TaskWalkerEntry`]).
249 pub tasks: &'a [TaskWalkerEntry],
250}
251
252/// One entry produced by a task walker (rq->scx, DSQ, etc.) for the
253/// enrichment capture pipeline.
254///
255/// Each task walker discovers task KVAs by traversing the kernel's
256/// own scheduling data structures; the walker also knows which task
257/// was reachable via `rq->scx.runnable_list` (used for the
258/// PI-boost-out-of-SCX flag) and which vCPU's instruction-pointer
259/// matches the running task (used for the lock-slowpath stack
260/// matcher). Capturing those signals at the walker site keeps the
261/// enrichment surface side-effect free — `walk_task_enrichment` only
262/// reads guest memory; it does not perform discovery itself.
263#[derive(Debug, Clone, Copy)]
264pub struct TaskWalkerEntry {
265 /// Kernel virtual address of the `task_struct`.
266 pub task_kva: u64,
267 /// True iff the task was reached via `rq->scx.runnable_list`.
268 /// Required for the PI-boost-out-of-SCX flag — see
269 /// [`super::task_enrichment::TaskEnrichment::pi_boosted_out_of_scx`].
270 pub is_runnable_in_scx: bool,
271 /// Optional instruction pointer for the lock-slowpath stack
272 /// matcher. Pass the corresponding vCPU's
273 /// [`VcpuRegSnapshot::instruction_pointer`] when this task was
274 /// running on that vCPU at freeze time; pass `None` for tasks
275 /// not actively running.
276 pub running_pc: Option<u64>,
277}
278
279/// Per-CPU CPU-time / softirq / IRQ snapshot captured at freeze
280/// time. One entry per CPU index visible to the host walker.
281///
282/// All counter fields are monotonic in the kernel — the freeze
283/// captures the instantaneous value at the moment the vCPUs
284/// rendezvous-park. Diffing two snapshots (or comparing against a
285/// pre-test baseline) is the consumer's job; this type does not
286/// derive deltas.
287///
288/// Field semantics match the kernel sources:
289/// - `cpustat_*_ns`: ns counter from
290/// `kernel_cpustat::cpustat[CPUTIME_*]`. Updated by
291/// `account_user_time` / `account_system_index_time` and
292/// siblings (`kernel/sched/cputime.c`). The kernel stores
293/// nanoseconds; `/proc/stat` divides by `cputime_to_clock_t`.
294/// - `softirqs[i]`: `kernel_stat::softirqs[i]` cumulative count
295/// incremented by `kstat_incr_softirqs_this_cpu` on every
296/// softirq raise. Indexed by `super::btf_offsets::SOFTIRQ_NAMES`.
297/// - `irqs_sum`: `kernel_stat::irqs_sum` cumulative count
298/// incremented by `kstat_incr_irq_this_cpu` on every hardirq.
299/// - `iowait_sleeptime_ns`: `tick_sched::iowait_sleeptime`
300/// accumulated only under NO_HZ when the CPU enters idle with
301/// `nr_iowait > 0`. `None` when CONFIG_NO_HZ_COMMON is off or
302/// the resolver couldn't locate `tick_cpu_sched`.
303#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
304#[non_exhaustive]
305pub struct PerCpuTimeStats {
306 /// CPU index (0-based) this entry describes.
307 pub cpu: u32,
308 /// `cpustat[CPUTIME_USER]` (ns).
309 pub cpustat_user_ns: u64,
310 /// `cpustat[CPUTIME_NICE]` (ns).
311 pub cpustat_nice_ns: u64,
312 /// `cpustat[CPUTIME_SYSTEM]` (ns).
313 pub cpustat_system_ns: u64,
314 /// `cpustat[CPUTIME_SOFTIRQ]` (ns).
315 pub cpustat_softirq_ns: u64,
316 /// `cpustat[CPUTIME_IRQ]` (ns).
317 pub cpustat_irq_ns: u64,
318 /// `cpustat[CPUTIME_IDLE]` (ns).
319 pub cpustat_idle_ns: u64,
320 /// `cpustat[CPUTIME_IOWAIT]` (ns).
321 pub cpustat_iowait_ns: u64,
322 /// `cpustat[CPUTIME_STEAL]` (ns).
323 pub cpustat_steal_ns: u64,
324 /// `kernel_stat::softirqs[]` per-vector cumulative counts.
325 /// Indexed by `super::btf_offsets::SOFTIRQ_NAMES`.
326 pub softirqs: [u64; super::btf_offsets::NR_SOFTIRQS],
327 /// `kernel_stat::irqs_sum` cumulative hardirq count.
328 pub irqs_sum: u64,
329 /// `tick_sched::iowait_sleeptime` accumulated NO_HZ idle time
330 /// with outstanding IO (ns). `None` when NO_HZ disabled or
331 /// `tick_cpu_sched` symbol was absent at resolve time.
332 #[serde(default, skip_serializing_if = "Option::is_none")]
333 pub iowait_sleeptime_ns: Option<u64>,
334}
335
336/// Per-node NUMA event counters captured from
337/// `pglist_data->node_zones[]->vm_numa_event[]` at freeze time.
338///
339/// Each row is one row of NUMA event counters summed across all
340/// zones on a single node. The six counters mirror the kernel's
341/// `enum numa_stat_item` (see `super::btf_offsets::NUMA_HIT`
342/// etc. for the enum-stable indices). All counters are
343/// monotonic-since-boot; consumers diff against a baseline (or
344/// against another node's row) to extract the test-window delta.
345///
346/// Diagnostic value for sched_ext stalls is informational only —
347/// the NUMA balancer is not active for ext tasks. The rows
348/// surface here so an operator triaging a NUMA-aware workload
349/// (e.g. a memory-tiering test) can verify the kernel actually
350/// observed the expected node-locality distribution.
351///
352/// **Live walker status:** the wire shape, BTF offsets
353/// (`super::btf_offsets::NumaStatsOffsets`), and report field
354/// are wired through. The actual host-side walker that resolves
355/// `node_data[]` and reads per-zone counters is pending; until it
356/// lands, the report's [`FailureDumpReport::per_node_numa`] vec
357/// stays empty and
358/// [`FailureDumpReport::per_node_numa_unavailable`] carries the
359/// `"no NUMA walker"` reason.
360#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
361#[non_exhaustive]
362pub struct PerNodeNumaStats {
363 /// NUMA node id this row describes.
364 pub node: u32,
365 /// `vm_numa_event[NUMA_HIT]` summed across zones — pages
366 /// allocated on the requested node when local was preferred.
367 pub numa_hit: u64,
368 /// `vm_numa_event[NUMA_MISS]` — local node full, allocation
369 /// landed on a non-local node.
370 pub numa_miss: u64,
371 /// `vm_numa_event[NUMA_FOREIGN]` — process-policy targeted a
372 /// different node, this node honored the policy.
373 pub numa_foreign: u64,
374 /// `vm_numa_event[NUMA_INTERLEAVE_HIT]` — interleave policy
375 /// allocations that landed on this node.
376 pub numa_interleave_hit: u64,
377 /// `vm_numa_event[NUMA_LOCAL]` — allocations on this node by
378 /// processes running on this node.
379 pub numa_local: u64,
380 /// `vm_numa_event[NUMA_OTHER]` — allocations on this node by
381 /// processes running on a different node.
382 pub numa_other: u64,
383}
384
385/// Reason string written into [`FailureDumpReport::per_node_numa_unavailable`]
386/// when the per-node NUMA walker has not landed yet. Distinct from
387/// other unavailable reasons so a downstream consumer can tell
388/// "walker not implemented" apart from "walker ran and produced
389/// no data" once the live producer ships.
390pub const REASON_NO_NUMA_WALKER: &str = "no NUMA walker (host-side walker pending)";
391
392/// Borrow-only capture context for the per-sample SCX event counter
393/// timeline.
394///
395/// The freeze coordinator forwards the monitor sampler's accumulated
396/// `super::MonitorSample` vec via [`Self::samples`]; the dump path
397/// folds each sample's per-CPU `super::ScxEventCounters` into a
398/// single cross-CPU sum and produces one [`EventCounterSample`] per
399/// monitor tick.
400///
401/// `None` skips the timeline capture; the dump still renders the
402/// rest of the report. Mirrors [`ProgRuntimeCapture`] /
403/// [`CpuTimeCapture`]'s "borrowed-only, optional" shape.
404pub struct EventCounterCapture<'a> {
405 /// Periodic monitor samples gathered between VM start and the
406 /// freeze trigger. Each sample carries per-CPU
407 /// `super::ScxEventCounters` when scx event-stat offsets
408 /// resolved; the dump folder skips samples whose CPUs all
409 /// reported `event_counters: None`.
410 pub samples: &'a [super::MonitorSample],
411}
412
413/// Borrow-only capture context for the rq->scx + DSQ walkers.
414/// Mirrors [`TaskEnrichmentCapture`] / [`CpuTimeCapture`] shape —
415/// `dump_state` consumes everything by reference.
416///
417/// Carries:
418/// - `kernel`: GuestKernel handle for guest-memory reads
419/// (PTE walks, symbol resolution).
420/// - `offsets`: BTF-resolved
421/// [`super::btf_offsets::ScxWalkerOffsets`] covering scx_rq,
422/// scx_sched, scx_sched_pcpu, scx_sched_pnode, scx_dispatch_q,
423/// sched_ext_entity, scx_dsq_list_node, rhashtable, bucket_table,
424/// rhash_head.
425/// - `scx_root_kva`: kernel-text-mapped pointer the walker
426/// dereferences to find the active `scx_sched`.
427/// - `rq_kvas` / `rq_pas`: per-CPU rq KVA + PA arrays; same vecs
428/// the runnable_at scanner uses.
429/// - `per_cpu_offsets`: `__per_cpu_offset[]` array — needed for
430/// per-CPU bypass DSQ resolution.
431/// - `nr_nodes`: NUMA node count, for the per-node global-DSQ
432/// walk. Pass `1` on UMA / unknown configurations; the walker
433/// gracefully skips slots whose pnode pointers are NULL.
434///
435/// When `None` is passed in [`DumpContext::scx_walker_capture`],
436/// the dump emits empty `rq_scx_states` / `dsq_states` and
437/// records `scx_walker_unavailable` with a diagnostic reason.
438pub struct ScxWalkerCapture<'a> {
439 /// Borrowed GuestKernel — provides memory access, page-table
440 /// translation context, and the vmlinux symbol table.
441 pub kernel: &'a super::guest::GuestKernel,
442 /// BTF-resolved offsets for the scx walker.
443 pub offsets: &'a super::btf_offsets::ScxWalkerOffsets,
444 /// `scx_root` symbol KVA (resolved via vmlinux ELF symtab).
445 /// The walker reads `*scx_root` to find the active scx_sched.
446 pub scx_root_kva: u64,
447 /// Per-CPU rq kernel virtual addresses (one per CPU).
448 pub rq_kvas: &'a [u64],
449 /// Per-CPU rq guest physical addresses (parallel to rq_kvas).
450 pub rq_pas: &'a [u64],
451 /// `__per_cpu_offset[]` array, used to address each CPU's
452 /// `scx_sched_pcpu.bypass_dsq`.
453 pub per_cpu_offsets: &'a [u64],
454 /// NUMA node count for the per-node global-DSQ walk. Pass `1`
455 /// on UMA / unknown configurations.
456 pub nr_nodes: u32,
457}
458
459/// One per-monitor-tick snapshot of the 13 SCX_EV_* event counters
460/// summed across every CPU at that tick.
461///
462/// The kernel stores per-CPU `s64` counters in `scx_sched_pcpu`
463/// (kernel/sched/ext.c); the monitor sampler reads them at every
464/// tick and stores per-CPU `event_counters` on each
465/// `super::CpuSnapshot`. The dump path sums across CPUs into the
466/// fields here so a downstream consumer can render the run's
467/// counter timeline (sparkline, delta plot, ...) without
468/// re-iterating the per-CPU vec.
469///
470/// Field semantics match
471/// `super::ScxEventCounters` one-to-one — see that struct's
472/// per-field doc for kernel-source provenance. `total_*` naming
473/// here echoes `super::ScxEventDeltas`'s aggregate-across-window
474/// fields but with per-tick (not per-window) granularity.
475#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
476#[non_exhaustive]
477pub struct EventCounterSample {
478 /// Milliseconds since VM start (mirrors
479 /// `super::MonitorSample::elapsed_ms`). Zero on the first
480 /// sample.
481 pub elapsed_ms: u64,
482 /// Sum of `select_cpu_fallback` across all CPUs at this tick.
483 pub select_cpu_fallback: i64,
484 /// Sum of `dispatch_local_dsq_offline` across all CPUs.
485 pub dispatch_local_dsq_offline: i64,
486 /// Sum of `dispatch_keep_last` across all CPUs.
487 pub dispatch_keep_last: i64,
488 /// Sum of `enq_skip_exiting` across all CPUs.
489 pub enq_skip_exiting: i64,
490 /// Sum of `enq_skip_migration_disabled` across all CPUs.
491 pub enq_skip_migration_disabled: i64,
492 /// Sum of `reenq_immed` across all CPUs.
493 pub reenq_immed: i64,
494 /// Sum of `reenq_local_repeat` across all CPUs.
495 pub reenq_local_repeat: i64,
496 /// Sum of `refill_slice_dfl` across all CPUs.
497 pub refill_slice_dfl: i64,
498 /// Sum of `bypass_duration` across all CPUs (ns).
499 pub bypass_duration: i64,
500 /// Sum of `bypass_dispatch` across all CPUs.
501 pub bypass_dispatch: i64,
502 /// Sum of `bypass_activate` across all CPUs.
503 pub bypass_activate: i64,
504 /// Sum of `insert_not_owned` across all CPUs.
505 pub insert_not_owned: i64,
506 /// Sum of `sub_bypass_dispatch` across all CPUs.
507 pub sub_bypass_dispatch: i64,
508}
509
510impl EventCounterSample {
511 /// Construct from a `super::MonitorSample` by summing every
512 /// CPU's `super::ScxEventCounters`. CPUs whose
513 /// `event_counters` is `None` (event-stat offsets unresolved)
514 /// contribute 0 to every field.
515 ///
516 /// Returns `None` when no CPU on the sample reported event
517 /// counters — propagating that to the timeline would emit a
518 /// row of all zeros that's indistinguishable from a real
519 /// "every counter at zero" tick. Callers filter `None` out.
520 pub fn from_monitor_sample(sample: &super::MonitorSample) -> Option<Self> {
521 let mut any = false;
522 let mut out = Self {
523 elapsed_ms: sample.elapsed_ms,
524 ..Self::default()
525 };
526 for cpu in &sample.cpus {
527 if let Some(ev) = &cpu.event_counters {
528 any = true;
529 // Per-CPU SCX event counters are s64 in the kernel
530 // and originate from BPF map reads of guest memory.
531 // A corrupt counter could trip i64 addition overflow
532 // when summed across many CPUs; saturating_add pins
533 // the sum at i64::{MIN,MAX} rather than panicking
534 // (debug) or wrapping (release) into a misleading
535 // value.
536 out.select_cpu_fallback = out
537 .select_cpu_fallback
538 .saturating_add(ev.select_cpu_fallback);
539 out.dispatch_local_dsq_offline = out
540 .dispatch_local_dsq_offline
541 .saturating_add(ev.dispatch_local_dsq_offline);
542 out.dispatch_keep_last =
543 out.dispatch_keep_last.saturating_add(ev.dispatch_keep_last);
544 out.enq_skip_exiting = out.enq_skip_exiting.saturating_add(ev.enq_skip_exiting);
545 out.enq_skip_migration_disabled = out
546 .enq_skip_migration_disabled
547 .saturating_add(ev.enq_skip_migration_disabled);
548 out.reenq_immed = out.reenq_immed.saturating_add(ev.reenq_immed);
549 out.reenq_local_repeat =
550 out.reenq_local_repeat.saturating_add(ev.reenq_local_repeat);
551 out.refill_slice_dfl = out.refill_slice_dfl.saturating_add(ev.refill_slice_dfl);
552 out.bypass_duration = out.bypass_duration.saturating_add(ev.bypass_duration);
553 out.bypass_dispatch = out.bypass_dispatch.saturating_add(ev.bypass_dispatch);
554 out.bypass_activate = out.bypass_activate.saturating_add(ev.bypass_activate);
555 out.insert_not_owned = out.insert_not_owned.saturating_add(ev.insert_not_owned);
556 out.sub_bypass_dispatch = out
557 .sub_bypass_dispatch
558 .saturating_add(ev.sub_bypass_dispatch);
559 }
560 }
561 if any { Some(out) } else { None }
562 }
563}
564
565/// Render a u64 counter series as a 1-line UTF-8 sparkline.
566///
567/// Maps each value into one of 8 unicode block-element glyphs
568/// (`▁▂▃▄▅▆▇█`) by min-max scaling. Empty input renders as the
569/// empty string; a constant non-zero series renders as repeated
570/// mid-tier glyphs (matches the "no variation" reading in the
571/// data, not as misleading monotonic up-bars). A constant zero
572/// series renders as repeated lowest glyphs.
573///
574/// Used by the `Display` impl for the event-counter timeline. Pure
575/// helper — no allocation outside the returned `String`.
576pub fn render_sparkline(values: &[u64]) -> String {
577 const GLYPHS: &[char] = &['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
578 if values.is_empty() {
579 return String::new();
580 }
581 let min = *values.iter().min().expect("non-empty");
582 let max = *values.iter().max().expect("non-empty");
583 let mut s = String::with_capacity(values.len() * 4);
584 if max == min {
585 let glyph = if max == 0 {
586 GLYPHS[0]
587 } else {
588 GLYPHS[GLYPHS.len() / 2]
589 };
590 for _ in values {
591 s.push(glyph);
592 }
593 return s;
594 }
595 let span = max - min;
596 let last_idx = (GLYPHS.len() - 1) as u64;
597 for &v in values {
598 // Linear scale [min, max] → [0, GLYPHS.len()-1]. Integer
599 // math is sufficient (no rounding artifact at the cost
600 // of one extra glyph step at boundaries).
601 let scaled = ((v - min) * last_idx) / span;
602 let idx = scaled.min(last_idx) as usize;
603 s.push(GLYPHS[idx]);
604 }
605 s
606}
607
608/// Saturating-cast wrapper around [`render_sparkline`] for signed
609/// (i64) counter series. Negative values clamp to 0; the kernel
610/// stores SCX_EV_* as `s64` but every counter is non-negative in
611/// practice, so the saturation only fires on a corrupt read.
612pub fn render_sparkline_i64(values: &[i64]) -> String {
613 let widened: Vec<u64> = values.iter().map(|&v| v.max(0) as u64).collect();
614 render_sparkline(&widened)
615}
616
617/// Snapshot of one vCPU's instruction-pointer / stack-pointer / page-
618/// table-root at freeze time. Re-export of the freeze-side type so
619/// dump consumers don't have to depend on `vmm::exit_dispatch`
620/// internals.
621pub use crate::vmm::exit_dispatch::VcpuRegSnapshot;
622
623/// Schema discriminant value emitted in `FailureDumpReport.schema`.
624///
625/// Consumers that read a `.failure-dump.json` file use the `schema`
626/// field's value to choose between [`FailureDumpReport`],
627/// [`DualFailureDumpReport`], and [`DegradedFailureDumpReport`]
628/// before attempting deserialization. The
629/// [`FailureDumpReportAny::from_json`] dispatcher handles this
630/// routing for in-process consumers.
631/// Values are stable wire constants — extending the dump pipeline
632/// with a new shape adds a new constant rather than changing this
633/// one.
634pub const SCHEMA_SINGLE: &str = "single";
635
636/// Schema discriminant value emitted in `DualFailureDumpReport.schema`.
637/// See [`SCHEMA_SINGLE`] for the discriminant contract.
638pub const SCHEMA_DUAL: &str = "dual";
639
640/// Schema discriminant value emitted in `DegradedFailureDumpReport.schema`.
641///
642/// Carried by failure-dumps the freeze coordinator was unable to
643/// capture as a full [`SCHEMA_SINGLE`] / [`SCHEMA_DUAL`] report — the
644/// trigger fired but rendezvous, gate cross-reference, or KVA
645/// translation aborted the dump path. Per the wire-format contract on
646/// [`SCHEMA_SINGLE`], degraded dumps are a stable variant added by
647/// new constant, not a mutation of the existing two.
648pub const SCHEMA_DEGRADED: &str = "degraded";
649
650/// Reason string written into [`DegradedFailureDumpReport::reason`]
651/// when the freeze coordinator's vCPU rendezvous timed out before
652/// every parked acknowledgement arrived. Wire-format-stable: matches
653/// the operator-grep contract used by every other `REASON_*` constant
654/// in this module. The dynamic detail appended at emit time
655/// (`<timeout_ms>` / `<parked>` / `<expected>`) lets an operator see
656/// which vCPUs stalled without a separate field.
657pub const REASON_DEGRADED_RENDEZVOUS_TIMEOUT: &str =
658 "vCPU rendezvous timed out before parked acknowledgement";
659
660/// Reason string written into [`DegradedFailureDumpReport::reason`]
661/// when the host kill signal flipped during the vCPU freeze
662/// rendezvous, short-circuiting the wait before every parked
663/// acknowledgement arrived. Distinct from
664/// [`REASON_DEGRADED_RENDEZVOUS_TIMEOUT`] (which fires only when the
665/// 30s deadline expires): a kill-mid-rendezvous typically lands in
666/// the rendezvous loop within milliseconds, so the elapsed_ms in the
667/// dynamic detail (appended at emit time) reads as a small number
668/// and the "timed out" label is internally contradictory. The kill
669/// sources are SCHED_EXIT propagation from a vCPU thread, watchdog
670/// hard-deadline expiry, and panic-hook flips. Wire-format-stable;
671/// matches the operator-grep contract used by every other `REASON_*`
672/// constant.
673pub const REASON_DEGRADED_KILL_DURING_RENDEZVOUS: &str =
674 "vCPU rendezvous aborted by external kill before parked acknowledgement";
675
676/// Snapshot tag used when the early-snapshot trigger fires but
677/// `freeze_and_dispatch(FreezeMode::Capture { gate_on_exit_kind: false })` returns `Degraded` (early-half
678/// rendezvous timeout). The freeze coordinator writes the degraded
679/// JSON to a sibling path named via
680/// `super::super::vmm::freeze_coord::snapshot_tagged_path` using
681/// this tag — main `{stem}.failure-dump.json` is preserved for the
682/// subsequent late-trigger emission. Operator-readable wire-format
683/// constant: kebab-case, stable across releases.
684///
685/// MAINTENANCE: adding a new `SNAPSHOT_TAG_*` const requires updating
686/// BOTH (a) the `ALL_SNAPSHOT_TAGS` slice below AND (b) the
687/// `expected` hand-list in
688/// `all_snapshot_tags_enumerates_every_pub_const_in_module` in
689/// `src/monitor/dump/tests.rs`. The pinning test catches slice-vs-
690/// expected divergence; it does NOT catch a const added without
691/// updating either (both arrays stay at the same length).
692pub const SNAPSHOT_TAG_EARLY_DEGRADED: &str = "early-degraded";
693
694/// Snapshot tag used when dual-snapshot mode held a Captured early
695/// snapshot AND the late-trigger path returned `Degraded`. The early
696/// snapshot is written to a sibling path with this tag while the
697/// late degraded JSON occupies the main dump path. Distinguishes
698/// "early itself degraded" ([`SNAPSHOT_TAG_EARLY_DEGRADED`]) from
699/// "early captured, late degraded" (this tag) so an operator browsing
700/// the dump directory knows which case produced which file. Every
701/// captured snapshot reaches disk.
702pub const SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED: &str = "early-pre-late-degraded";
703
704/// Snapshot tag used when dual-snapshot mode held a Captured early
705/// snapshot AND the late-trigger path ran AND returned `Suppressed`
706/// (the gate examined `*scx_root->exit_kind`, found it below
707/// SCX_EXIT_ERROR, and decided no failure dump warranted). Distinct
708/// from [`SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED`]: this tag
709/// means the late trigger DID fire and the gate explicitly decided
710/// clean exit. Operator triage: scheduler recovered from the early
711/// stall and reached a clean shutdown via the SCX_EXIT_NONE /
712/// SCX_EXIT_DONE path. The early observation (runnable-age spike)
713/// is independently meaningful and reaches disk at the tagged
714/// sibling. Symmetric with
715/// [`SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED`] — tagged sibling rather
716/// than main path so the main `{stem}.failure-dump.json` keeps the
717/// "scheduler had a failure-class exit" semantic.
718pub const SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED: &str = "early-only-late-suppressed";
719
720/// Snapshot tag used when dual-snapshot mode held a Captured early
721/// snapshot AND the late-trigger path NEVER FIRED for the run (no
722/// `err_exit_detected` BPF latch flip; the scheduler never reached
723/// an error-class late event). Distinct from
724/// [`SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED`]: this tag means the
725/// late trigger never ran at all, NOT that it ran and decided
726/// clean. Operator triage: scheduler crossed the half-watchdog
727/// runnable-age threshold (early-trigger fired) but then either
728/// recovered or terminated before reaching the late-trigger path —
729/// `freeze_state` stayed at `Idle` or `TookEarly` through coord
730/// exit. The end-of-coord drain emits the early observation to the
731/// tagged sibling rather than letting it drop with the closure.
732pub const SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED: &str = "early-only-late-never-fired";
733
734/// Canonical enumeration of every `SNAPSHOT_TAG_*` constant in this
735/// module. Tests that negative-scan tag locations (e.g. asserting no
736/// file landed at any wrong tag) iterate this slice rather than
737/// hardcoding the 4-element list. Adding a new `SNAPSHOT_TAG_*`
738/// constant requires updating this slice — the pinning test
739/// `all_snapshot_tags_enumerates_every_pub_const_in_module` in
740/// `src/monitor/dump/tests.rs` will fail until the new tag is added,
741/// flagging the inconsistency at test-time.
742///
743/// Order: NEVER_FIRED first because it is the default tag in
744/// `EarlySnapshotGuard::drain_to_disk`'s `unwrap_or` arm at
745/// src/vmm/freeze_coord/mod.rs (the guard struct is `pub(super)` so
746/// rustdoc cannot intra-doc-link it from this module; cite by file
747/// path matches the cross-ref convention used elsewhere in this
748/// batch). The other three follow in dispatch-arm order:
749/// SUPPRESSED (late-trigger Suppressed write-failure) → PRE_LATE_DEGRADED
750/// (late-trigger Degraded write-failure) → EARLY_DEGRADED (early-
751/// trigger Degraded direct write) — so readers can map slice index
752/// to dispatch arm by inspection.
753///
754/// `#[cfg(test)]` — production code never iterates this slice;
755/// only the negative-scan tests and the pinning canary reference it.
756/// Gated to keep the lib's dead-code lint clean.
757#[cfg(test)]
758pub const ALL_SNAPSHOT_TAGS: &[&str] = &[
759 SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED,
760 SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED,
761 SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED,
762 SNAPSHOT_TAG_EARLY_DEGRADED,
763];
764
765/// Reason string written into [`FailureDumpReport::prog_runtime_stats_unavailable`]
766/// when [`DumpContext::prog_capture`] was supplied but the per-program
767/// walker found no struct_ops programs in `prog_idr` at freeze time.
768/// Wire-format-stable: an operator parsing the sidecar JSON looks for
769/// this exact string to distinguish from the prog-accessor-missing
770/// case.
771pub const REASON_NO_STRUCT_OPS_LOADED: &str = "no struct_ops programs loaded";
772
773/// Reason string written into [`FailureDumpReport::prog_runtime_stats_unavailable`]
774/// when [`DumpContext::prog_capture`] was `None`. Distinguishes from
775/// [`REASON_NO_STRUCT_OPS_LOADED`] — the walker never ran in this case
776/// because the accessor wasn't constructed (e.g. `prog_idr` symbol
777/// missing).
778pub const REASON_PROG_ACCESSOR_UNAVAILABLE: &str = "prog accessor unavailable";
779
780/// Reason string written into [`FailureDumpReport::task_enrichments_unavailable`]
781/// when [`DumpContext::task_enrichment_capture`] was supplied but
782/// every walker entry produced no enrichment (idle guest with no
783/// runnable scx tasks at the freeze instant).
784pub const REASON_TASK_WALKER_ZERO_TASKS: &str = "task walker yielded zero tasks";
785
786/// Reason string written into [`FailureDumpReport::task_enrichments_unavailable`]
787/// when [`DumpContext::task_enrichment_capture`] was `None`.
788/// Distinguishes from [`REASON_TASK_WALKER_ZERO_TASKS`] — the walker
789/// never ran because the capture wasn't supplied.
790pub const REASON_NO_TASK_WALKER: &str = "no task walker available";
791
792/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
793/// when offsets resolved AND the walker found rq->scx + local DSQ
794/// data BUT `*scx_root == 0` — no scheduler attached at the freeze
795/// instant. The sched-rooted passes (bypass / global / user-hash)
796/// have nothing to walk, but the rq->scx and per-CPU local DSQ
797/// captures still produced data. Surfaces a distinct reason so the
798/// operator knows the scheduler isn't loaded vs. the walker is
799/// broken.
800pub const REASON_SCX_ROOT_NULL: &str = "scx_root is NULL (no scheduler attached)";
801
802/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
803/// when [`DumpContext::scx_walker_capture`] was supplied AND every
804/// offset sub-group resolved BUT the walker reached no rq, no DSQ,
805/// and no scx_sched state at all (every read failed). Distinct from
806/// [`REASON_SCX_ROOT_NULL`]: that case has rq->scx + local DSQ data
807/// but no sched_state; this case has nothing.
808pub const REASON_SCX_WALKER_NO_STATE: &str = "scx walker reached no state";
809
810/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
811/// when [`DumpContext::scx_walker_capture`] was `None`. Distinguishes
812/// from [`REASON_SCX_WALKER_NO_STATE`] — the walker never ran at all
813/// because no capture was supplied.
814pub const REASON_NO_SCX_WALKER: &str = "no scx walker capture";
815
816/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
817/// when the sdt_alloc pre-pass could not run because the scheduler's
818/// arena `user_vm_start` was not 4 GiB-aligned. See the gate in
819/// [`dump_state`]: low-32 keying of the per-pass
820/// [`crate::monitor::dump::render_map::ArenaSlotIndex`] is only
821/// correct when `user_vm_start & 0xFFFF_FFFF == 0`.
822pub const REASON_SDT_ALLOC_UNALIGNED_USER_VM: &str =
823 "user_vm_start is not 4 GiB-aligned; low-32 keying disabled";
824
825/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
826/// when the dump enumerated no `*.bss` ARRAY map with a non-zero
827/// `btf_kva`. Without the scheduler's `.bss` bytes the pre-pass cannot
828/// read any allocator's in-memory state, and without `btf_kva` the
829/// program BTF that names `struct scx_allocator` is not loadable.
830pub const REASON_SDT_ALLOC_NO_BSS: &str = "no scheduler .bss map (or .bss has no program BTF)";
831
832/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
833/// when no `BPF_MAP_TYPE_ARENA` map snapshot was captured: the
834/// pre-pass has no `kern_vm_start` to translate arena pointers
835/// against, so no allocator slot can be walked.
836pub const REASON_SDT_ALLOC_NO_ARENA: &str = "no arena map captured (kern_vm_start unavailable)";
837
838/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
839/// when the scheduler's program BTF does not carry `struct scx_allocator`
840/// or its peer types (`sdt_pool`, `sdt_desc`, `sdt_chunk`) — the scheduler
841/// does not link `lib/sdt_alloc.bpf.c` and there are no allocator slots
842/// to walk. [`crate::monitor::sdt_alloc::SdtAllocOffsets::from_btf`]
843/// returns the underlying `anyhow::Error` describing which struct was
844/// missing; the dump caller folds that into this reason string.
845pub const REASON_SDT_ALLOC_NO_TYPE: &str = "scheduler BTF does not declare struct scx_allocator";
846
847/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
848/// when every prerequisite resolved but no `.bss` variable of type
849/// `struct scx_allocator` was discovered. The scheduler links
850/// `lib/sdt_alloc.bpf.c` for its types but has not declared a typed
851/// allocator instance.
852pub const REASON_SDT_ALLOC_NO_INSTANCE: &str = "no scx_allocator instance in .bss";
853
854/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
855/// when the per-map dump deadline was exhausted before the pre-pass
856/// could run. The `dump_truncated_at_us` field also surfaces the
857/// truncation; this string disambiguates "deadline" from the other
858/// no-data causes when a consumer is scanning sdt_alloc-related
859/// diagnostics specifically.
860pub const REASON_SDT_ALLOC_DEADLINE_EXCEEDED: &str =
861 "dump deadline exceeded before sdt_alloc pre-pass could run";
862
863/// Cross-CPU sum of every per-CPU diagnostic counter slot in the
864/// probe BPF program's `.bss` `ktstr_pcpu_counters` array.
865///
866/// The probe declares one fixed-shape per-CPU array
867/// (`pcpu_counter ktstr_pcpu_counters[MAX_CPUS][KTSTR_PCPU_NR]` —
868/// see `src/bpf/probe.bpf.c`); each tracepoint / kprobe handler
869/// bumps a slot via `ktstr_pcpu_inc(KTSTR_PCPU_<NAME>)`. The host
870/// reader sums across the CPU axis to recover the cumulative count
871/// each handler reports. Field names mirror the slot names from
872/// `enum ktstr_pcpu_idx` so an operator can walk back from the
873/// failure-dump field to the probe source by exact name.
874///
875/// All counters are monotonic-since-probe-attach. Zero values
876/// indicate either "the corresponding tracepoint never fired" (the
877/// common case for `pi_*` and `lock_contend_*` on tests that don't
878/// exercise PI / lock contention) or "the tracepoint never attached"
879/// (e.g. `preempt_*` on a kernel without
880/// `CONFIG_TRACE_PREEMPT_TOGGLE`); the counter alone cannot
881/// distinguish those two cases — pair with the attach-state surface
882/// in `crate::probe::process::ProbeDiagnostics` when the
883/// distinction matters.
884#[derive(Debug, Clone, Default, Serialize, Deserialize)]
885#[non_exhaustive]
886pub struct ProbeBssCounters {
887 /// `KTSTR_PCPU_PROBE_COUNT` summed across CPUs — total kprobe
888 /// fires past the `ktstr_enabled` gate.
889 pub probe_count: u64,
890 /// `KTSTR_PCPU_KPROBE_RETURNS` summed across CPUs — kprobe fires
891 /// that committed an entry to `probe_data` (past `func_meta_map`
892 /// lookup and scratch-slot allocation).
893 pub kprobe_returns: u64,
894 /// `KTSTR_PCPU_META_MISS` summed across CPUs — kprobe fires
895 /// whose IP missed `func_meta_map`. `probe_count -
896 /// kprobe_returns` is the total bail count; `meta_miss` is the
897 /// subset whose bail came from the `func_meta_map` lookup.
898 pub meta_miss: u64,
899 /// `KTSTR_PCPU_RINGBUF_DROPS` summed across CPUs — failed
900 /// `bpf_ringbuf_reserve` calls inside the trigger handler.
901 pub ringbuf_drops: u64,
902 /// `KTSTR_PCPU_TIMELINE_COUNT` summed across CPUs — successful
903 /// timeline-event submissions across the three timeline
904 /// tracepoints (sched_switch + sched_migrate_task + sched_wakeup).
905 pub timeline_count: u64,
906 /// `KTSTR_PCPU_TIMELINE_DROPS` summed across CPUs — timeline
907 /// submissions that failed because the dedicated
908 /// `timeline_events` ringbuf was full at submit time.
909 pub timeline_drops: u64,
910 /// `KTSTR_PCPU_PI_COUNT` summed across CPUs — PI boost / unboost
911 /// records committed via `fexit/rt_mutex_setprio`.
912 pub pi_count: u64,
913 /// `KTSTR_PCPU_PI_ORPHAN_FEXITS` summed across CPUs — fexit
914 /// fires whose entry-side snapshot was never recorded (attach
915 /// race or `pi_scratch` overflow).
916 pub pi_orphan_fexits: u64,
917 /// `KTSTR_PCPU_PI_CLASS_CHANGE_COUNT` summed across CPUs —
918 /// PI events that observed a `sched_class` flip from fentry
919 /// to fexit (e.g. CFS → RT under a boost).
920 pub pi_class_change_count: u64,
921 /// `KTSTR_PCPU_PI_DROPS` summed across CPUs — TL_EVT_PI_BOOST
922 /// submissions that failed because the timeline ringbuf was
923 /// full at the PI fexit handler.
924 pub pi_drops: u64,
925 /// `KTSTR_PCPU_LOCK_CONTEND_COUNT` summed across CPUs —
926 /// `tp_btf/contention_begin` fires that committed a
927 /// TL_EVT_LOCK_CONTEND timeline record.
928 pub lock_contend_count: u64,
929 /// `KTSTR_PCPU_LOCK_CONTEND_DROPS` summed across CPUs —
930 /// TL_EVT_LOCK_CONTEND submissions that failed because the
931 /// timeline ringbuf was full.
932 pub lock_contend_drops: u64,
933 /// `KTSTR_PCPU_PREEMPT_DISABLE_COUNT` summed across CPUs —
934 /// `tp_btf/preempt_disable` outermost-transition fires.
935 pub preempt_disable_count: u64,
936 /// `KTSTR_PCPU_PREEMPT_ENABLE_COUNT` summed across CPUs —
937 /// `tp_btf/preempt_enable` outermost-transition fires.
938 pub preempt_enable_count: u64,
939 /// `KTSTR_PCPU_TRIGGER_COUNT` summed across CPUs — every accepted
940 /// selected trigger fire. The raw `scx_vexit` return program
941 /// also counts non-error kinds like DONE / UNREG; the global-era
942 /// `scx_dump_state` program counts only error-class entries.
943 pub trigger_count: u64,
944}
945
946/// Host-side context for a failure that struck before the freeze
947/// coordinator could capture any BPF state — the guest booted but cgroup
948/// or scheduler setup failed, so the payload never ran and only a
949/// placeholder report exists. Everything here was already in the host's
950/// hands (`vmm::VmResult`) at placeholder-write time; embedding it makes
951/// the artifact diagnosable instead of rendering `<unavailable>` for
952/// every section. All fields optional so the shape is purely additive.
953#[derive(Debug, Clone, Default, Serialize, Deserialize)]
954#[non_exhaustive]
955pub struct GuestSetupFailureContext {
956 /// Stage label from `classify_init_stage` — the deepest lifecycle
957 /// phase the guest reached before failing.
958 #[serde(default, skip_serializing_if = "Option::is_none")]
959 pub init_stage: Option<String>,
960 /// Final guest lifecycle stage per the host progress ledger
961 /// (`VmResult::final_guest_phase`), Debug-rendered.
962 #[serde(default, skip_serializing_if = "Option::is_none")]
963 pub final_guest_phase: Option<String>,
964 /// Guest exit code surfaced through the SHM ring or COM2 sentinel.
965 #[serde(default, skip_serializing_if = "Option::is_none")]
966 pub guest_exit_code: Option<i32>,
967 /// Scheduler attach outcome decoded from the lifecycle frames
968 /// (`verifier::attach_outcome_from_messages`), including the guest's
969 /// not-attached reason suffix when present.
970 #[serde(default, skip_serializing_if = "Option::is_none")]
971 pub scheduler_attach_outcome: Option<String>,
972 /// Bounded tail of the guest console / kernel log (COM1,
973 /// `VmResult::stderr`).
974 #[serde(default, skip_serializing_if = "Option::is_none")]
975 pub console_tail: Option<String>,
976 /// Bounded tail of the guest stdout / init log (COM2,
977 /// `VmResult::output`) — carries the `ktstr-init:` setup errors.
978 #[serde(default, skip_serializing_if = "Option::is_none")]
979 pub init_log_tail: Option<String>,
980 /// Bounded tail of the scheduler's own merged stdout/stderr shipped
981 /// over the bulk port (`SCHED_LOG` chunks) — carries the
982 /// scheduler's argv/config parse errors when it died pre-attach.
983 #[serde(default, skip_serializing_if = "Option::is_none")]
984 pub scheduler_log_tail: Option<String>,
985}
986
987impl GuestSetupFailureContext {
988 /// True when no field carries data — the Display impl and the
989 /// empty-report guard treat such a context as absent.
990 pub fn is_empty(&self) -> bool {
991 self.init_stage.is_none()
992 && self.final_guest_phase.is_none()
993 && self.guest_exit_code.is_none()
994 && self.scheduler_attach_outcome.is_none()
995 && self.console_tail.is_none()
996 && self.init_log_tail.is_none()
997 && self.scheduler_log_tail.is_none()
998 }
999}
1000
1001/// Top-level failure-dump report. One per freeze trigger.
1002#[derive(Debug, Clone, Serialize, Deserialize)]
1003#[non_exhaustive]
1004pub struct FailureDumpReport {
1005 /// Wire-format discriminant. Always `"single"` for this variant,
1006 /// pinning [`SCHEMA_SINGLE`]. Consumers branch on this to
1007 /// choose between [`FailureDumpReport`], [`DualFailureDumpReport`],
1008 /// and [`DegradedFailureDumpReport`] before deserializing. Single
1009 /// and Dual share top-level field names that would collide without
1010 /// an explicit tag; Degraded carries a distinct field set
1011 /// (`reason`, `watchpoint_hit`, `bss_latch_state`, `exit_kind`,
1012 /// `elapsed_ms`) but still gets the tag so
1013 /// [`FailureDumpReportAny::from_json`] can dispatch uniformly.
1014 pub schema: String,
1015 /// One entry per BPF map enumerated. Order matches the IDR walk
1016 /// (i.e. allocation order); the report is otherwise unsorted so
1017 /// callers that want a stable view should sort by name.
1018 pub maps: Vec<FailureDumpMap>,
1019 /// Per-vCPU register snapshots captured on each vCPU thread at
1020 /// freeze time. Index matches vCPU id (BSP at 0, APs at 1..N).
1021 /// `None` when a vCPU never parked (rendezvous timeout) or its
1022 /// `KVM_GET_REGS` failed mid-shutdown. Attached to the report by
1023 /// the freeze coordinator after `dump_state` returns.
1024 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1025 pub vcpu_regs: Vec<Option<VcpuRegSnapshot>>,
1026 /// Obj name of the currently-attached scheduler, identified by
1027 /// matching each `BPF_MAP_TYPE_STRUCT_OPS` map's `value_kva` (the
1028 /// guest-KVA of its `kvalue.data` payload) against the dereferenced
1029 /// `*scx_root` value (the guest-KVA of the active `struct scx_sched`,
1030 /// which is also the KVA of `scx_sched.ops` since `ops` sits at
1031 /// offset 0). When the match succeeds, the struct_ops map's name
1032 /// carries the obj prefix (libbpf convention: `<obj>.<struct_ops_var>`);
1033 /// the prefix is split at the first `.` and stored here.
1034 ///
1035 /// `None` when:
1036 /// - `scx_sched_state` is unavailable (no scheduler attached, BTF
1037 /// missing the `scx_sched` type, or `*scx_root` could not be
1038 /// resolved at capture time).
1039 /// - No `BPF_MAP_TYPE_STRUCT_OPS` map had `value_kva` matching the
1040 /// active sched_kva (capture race during a mid-attach window, or
1041 /// the struct_ops map's value_kva was not yet populated).
1042 /// - The matched map's name lacks a `<obj>.` prefix.
1043 ///
1044 /// [`crate::scenario::snapshot::Snapshot::active`] uses this as
1045 /// the principled tiebreaker when the projection sees multiple
1046 /// obj prefixes in global-section maps. On `None` the consumer
1047 /// falls back to the prefix-grouping heuristic (single obj →
1048 /// that one; multiple obj → `NoActiveScheduler` with a
1049 /// diagnostic naming the observed obj_names + the walker's
1050 /// failure cause).
1051 #[serde(default, skip_serializing_if = "Option::is_none")]
1052 pub active_obj_name: Option<String>,
1053 /// Guest-KVAs of every `struct bpf_map` belonging to the
1054 /// currently-attached scheduler's loaded BPF object, captured
1055 /// alongside [`Self::active_obj_name`] from the same
1056 /// `*scx_root → struct_ops map → owning bpf_prog → used_maps`
1057 /// walk. The walker enumerates the matched struct_ops prog's
1058 /// `used_maps` array and records each entry's KVA so a
1059 /// downstream filter can identify the active scheduler's maps
1060 /// uniquely — even when two scheduler instances loaded from the
1061 /// SAME binary coexist post-[`crate::scenario::ops::Op::ReplaceScheduler`]
1062 /// (where their bss / data / rodata maps share the
1063 /// `<obj_name>.` prefix and cannot be distinguished by name
1064 /// alone).
1065 ///
1066 /// Empty when:
1067 /// - [`Self::active_obj_name`] is `None` (walker did not
1068 /// resolve the active obj — same reasons; see that field's
1069 /// doc).
1070 /// - The matched prog's `used_maps` could not be safely read
1071 /// (torn race per the kernel's `used_map_cnt` / `used_maps`
1072 /// pointer publication TOCTOU described at
1073 /// `monitor/bpf_prog.rs::find_active_struct_ops_obj_no_target`).
1074 /// - The walker ran but the kernel published an empty
1075 /// `used_maps` for the active prog (no maps registered — an
1076 /// unusual but legal sched_ext shape).
1077 ///
1078 /// **KVA aliasing caveat:** kernel BPF map allocations are
1079 /// vmalloc/slab-backed; a freed map's KVA can be reassigned to a
1080 /// new allocation across captures. [`crate::scenario::snapshot::Snapshot::active`]
1081 /// combines this set with [`Self::active_obj_name`] (both must
1082 /// match) to reject the aliasing case — a KVA hit whose owning
1083 /// map name does not share the active obj prefix is treated as
1084 /// stale and falls through to the obj-prefix heuristic.
1085 ///
1086 /// **Within-run identity ONLY.** KVAs reflect kernel address
1087 /// space allocation at capture time (subject to KASLR slide).
1088 /// Stable for the life of the map within a single VM run; NOT
1089 /// comparable across runs. Never persist or compare against
1090 /// checked-in baselines.
1091 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1092 pub active_map_kvas: Vec<u64>,
1093 /// Structured per-allocation views from sdt_alloc-backed
1094 /// allocators. One entry per discovered allocator; each carries
1095 /// every live leaf slot (capped at
1096 /// `super::sdt_alloc::MAX_SDT_ALLOC_ENTRIES`) BTF-rendered to
1097 /// named field views. Empty when no scheduler-side allocator
1098 /// could be located, when arena offsets / sdt_alloc offsets are
1099 /// absent, or when the program BTF lacks the `scx_allocator`
1100 /// type (scheduler doesn't link `lib/sdt_alloc.bpf.c`).
1101 ///
1102 /// Populated alongside the page-granular `ArenaSnapshot` in
1103 /// each map: a consumer can read either representation depending
1104 /// on whether they want raw bytes or named-field allocations.
1105 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1106 pub sdt_allocations: Vec<SdtAllocatorSnapshot>,
1107 /// Live `scx_static` bump-allocator regions discovered in
1108 /// `.bss`. One entry per `struct scx_static` instance with
1109 /// `memory != 0` and an in-range `(off, max_alloc_bytes)` pair.
1110 /// Distinct from [`Self::sdt_allocations`]: scx_static is a
1111 /// program-lifetime bump allocator (`lib/sdt_alloc.bpf.c:577`)
1112 /// with no per-allocation header, so the surfaced view is
1113 /// region-granular ranges rather than per-slot named allocations.
1114 /// Empty when the scheduler doesn't link `lib/sdt_alloc.bpf.c`,
1115 /// when the program BTF lacks `struct scx_static`, or when no
1116 /// `scx_static` instance has been initialised at freeze time.
1117 ///
1118 /// The dump pipeline uses the same ranges to populate the
1119 /// renderer's `ScxStaticRangeIndex` so a deferred-resolve arena
1120 /// chase whose target lives inside scx_static memory can
1121 /// fail-closed cleanly (no per-slot type recovery is possible
1122 /// without a per-call-site type hook from cast analysis).
1123 #[serde(
1124 default,
1125 skip_serializing_if = "super::scx_static_alloc::ScxStaticSnapshot::is_empty"
1126 )]
1127 pub scx_static_ranges: super::scx_static_alloc::ScxStaticSnapshot,
1128 /// Diagnostic reason for `sdt_allocations` being empty.
1129 ///
1130 /// - `None` → either the pre-pass ran and produced records (the vec
1131 /// is non-empty), or the pre-pass ran cleanly but the scheduler
1132 /// simply has no live allocations (the vec is empty for legitimate
1133 /// reasons that aren't worth a diagnostic). Default.
1134 /// - `Some(REASON_SDT_ALLOC_*)` → the pre-pass skipped before it
1135 /// could surface any allocator state. The string identifies which
1136 /// prerequisite was missing: deadline exhaustion, unaligned
1137 /// `user_vm_start`, missing scheduler `.bss`, missing arena
1138 /// snapshot, scheduler BTF without `struct scx_allocator`, or no
1139 /// `.bss` `scx_allocator` instance.
1140 ///
1141 /// Distinct from `dump_truncated_at_us` (which records deadline
1142 /// truncation across the whole dump) and from
1143 /// [`Self::scx_static_ranges`] (which has its own walker independent
1144 /// of the typed-allocator pre-pass). Mirrors the
1145 /// `prog_runtime_stats_unavailable` / `task_enrichments_unavailable`
1146 /// pattern.
1147 #[serde(default, skip_serializing_if = "Option::is_none")]
1148 pub sdt_alloc_unavailable: Option<String>,
1149 /// Per-program BPF runtime stats summed across CPUs at freeze
1150 /// time (cnt, nsecs, misses). One entry per discovered
1151 /// struct_ops BPF program. Empty when no struct_ops programs are
1152 /// loaded OR when the prog accessor was unavailable to
1153 /// `dump_state` — see [`Self::prog_runtime_stats_unavailable`]
1154 /// for the reason.
1155 ///
1156 /// Per-CPU offset resolution failure does NOT empty the vec —
1157 /// each program still contributes one entry, but with
1158 /// `cnt`/`nsecs`/`misses` summed only over CPUs whose per-CPU
1159 /// `bpf_prog_stats` slot translated successfully (out-of-range
1160 /// CPUs return None per `super::bpf_map::read_percpu_array_value`
1161 /// semantics).
1162 ///
1163 /// See `super::bpf_prog::ProgRuntimeStats` for field semantics
1164 /// and the kernel-source-grounded provenance of each counter.
1165 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1166 pub prog_runtime_stats: Vec<super::bpf_prog::ProgRuntimeStats>,
1167 /// Diagnostic reason for `prog_runtime_stats` being empty.
1168 ///
1169 /// Distinguishes the three causes a consumer can't otherwise tell
1170 /// apart from an empty vec:
1171 /// - `None` (field absent on wire) → vec was populated normally
1172 /// (or the dump path didn't run). Default.
1173 /// - `Some("no struct_ops programs loaded")` → walker ran, no
1174 /// struct_ops programs were in `prog_idr` at freeze time.
1175 /// - `Some("prog accessor unavailable")` → caller passed
1176 /// `prog_capture: None`. Typical causes: `prog_idr` symbol
1177 /// missing, `BpfProgOffsets` BTF parse failed, or
1178 /// `__per_cpu_offset` resolution didn't yield non-zero offsets
1179 /// yet (still-booting guest).
1180 ///
1181 /// Set by `dump_state` only when prog_runtime_stats ends up
1182 /// empty AND a definite cause is identifiable; left None
1183 /// otherwise so the field stays absent in the JSON for
1184 /// already-populated dumps.
1185 #[serde(default, skip_serializing_if = "Option::is_none")]
1186 pub prog_runtime_stats_unavailable: Option<String>,
1187 /// Per-CPU CPU-time / softirq / IRQ counters captured from
1188 /// `kernel_cpustat`, `kernel_stat`, and (under NO_HZ)
1189 /// `tick_sched`. One entry per CPU enumerated by the walker.
1190 /// Empty when the dump caller passed no `CpuTimeCapture` or
1191 /// when symbol/BTF resolution failed.
1192 ///
1193 /// See `PerCpuTimeStats` for field semantics. Surfaces the
1194 /// per-CPU interrupt and idle-time data the failure dump
1195 /// otherwise leaves implicit (the existing scx walker reads
1196 /// `rq->nr_iowait` but not the cumulative time accounting).
1197 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1198 pub per_cpu_time: Vec<PerCpuTimeStats>,
1199 /// Per-cgroup PSI-irq samples for the test's workload cgroups, host-walked
1200 /// from the cgroup hierarchy at this freeze (Phase A). One entry per
1201 /// workload-root leaf cgroup with per-cgroup PSI accounting enabled. Empty
1202 /// when the dump caller passed no `CgroupPsiCapture`, the workload root
1203 /// isn't present yet, or `psi_cgroups_enabled` is off — loud-absent. RAW
1204 /// values; decoded + folded at the metric layer. See
1205 /// `super::cgroup_walk::CgroupPsiStat`.
1206 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1207 pub cgroup_psi: Vec<super::cgroup_walk::CgroupPsiStat>,
1208 /// Per-node NUMA event counters captured from
1209 /// `pglist_data->node_zones[]->vm_numa_event[]`. One row per
1210 /// NUMA node enumerated by the walker. Empty when the live
1211 /// walker has not landed yet (the BTF offsets and wire shape
1212 /// are wired; the reader is a follow-up).
1213 ///
1214 /// See `PerNodeNumaStats` for field semantics; see
1215 /// [`Self::per_node_numa_unavailable`] for the "why empty"
1216 /// diagnostic.
1217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1218 pub per_node_numa: Vec<PerNodeNumaStats>,
1219 /// Diagnostic reason for `per_node_numa` being empty.
1220 /// `None` when the vec was populated normally (or the dump
1221 /// path didn't run); `Some(REASON_NO_NUMA_WALKER)` until the
1222 /// host-side walker lands.
1223 #[serde(default, skip_serializing_if = "Option::is_none")]
1224 pub per_node_numa_unavailable: Option<String>,
1225 /// Per-task failure-dump enrichments — identity (pid, tgid,
1226 /// comm), process tree (group_leader, real_parent, pgid, sid,
1227 /// nr_threads), scheduling (prio family, sched_class name,
1228 /// scx.weight, core_cookie), context-switch counters, watchdog
1229 /// disambiguation flag, and lock-slowpath stack matches.
1230 ///
1231 /// One entry per task the dump path's task walker reaches —
1232 /// today's task walkers are the rq->scx walker and the DSQ
1233 /// walker; both produce task KVAs that get enriched here.
1234 /// Empty when no task walker ran (typical until walker
1235 /// dispatch lands) or when the `TaskEnrichmentCapture` was
1236 /// absent.
1237 ///
1238 /// See `super::task_enrichment::TaskEnrichment` for field
1239 /// semantics; see [`Self::task_enrichments_unavailable`] for the
1240 /// "why empty" diagnostic.
1241 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1242 pub task_enrichments: Vec<super::task_enrichment::TaskEnrichment>,
1243 /// Diagnostic reason for `task_enrichments` being empty.
1244 ///
1245 /// - `None` → vec was populated normally (or the dump path
1246 /// didn't run).
1247 /// - `Some("no task walker available")` → the
1248 /// `TaskEnrichmentCapture` was missing from
1249 /// `DumpContext`. Until DSQ + rq->scx walker dispatch
1250 /// lands, this is the expected steady state for the dump
1251 /// pipeline; the offsets + walker library is wired and
1252 /// ready to populate as soon as a task-list producer hooks
1253 /// in.
1254 /// - `Some("task walker yielded zero tasks")` → walker
1255 /// produced no task KVAs (frozen guest with no runnable /
1256 /// queued scx tasks at the dump instant — possible on a
1257 /// completely-idle stall trigger).
1258 #[serde(default, skip_serializing_if = "Option::is_none")]
1259 pub task_enrichments_unavailable: Option<String>,
1260 /// Per-monitor-tick SCX_EV_* event counter timeline. Each entry
1261 /// is the cross-CPU sum of the 13 SCX_EV_* counters at one
1262 /// monitor sample. Empty when the dump caller passed no
1263 /// `EventCounterCapture` or no sample reported event counters
1264 /// (event-stat offsets unresolved, scx_root unset). Renderers
1265 /// build sparklines / per-counter delta plots from this vec.
1266 ///
1267 /// See `EventCounterSample` for field semantics; the kernel-
1268 /// source provenance lives on
1269 /// `super::ScxEventCounters` field doc.
1270 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1271 pub event_counter_timeline: Vec<EventCounterSample>,
1272 /// Per-CPU `rq->scx` snapshots — scalar fields the kernel's
1273 /// own `scx_dump_state` reads plus the runnable_list per-task
1274 /// KVAs that fed into the per-task enrichment capture.
1275 /// One entry per CPU walked. Empty when the
1276 /// `ScxWalkerCapture` was absent or every CPU's translate
1277 /// failed.
1278 ///
1279 /// See `super::scx_walker::RqScxState` for field semantics.
1280 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1281 pub rq_scx_states: Vec<super::scx_walker::RqScxState>,
1282 /// Per-DSQ snapshots — local, bypass, global, and user DSQs
1283 /// reachable from `*scx_root`. Each entry carries `nr` (depth),
1284 /// `seq` (BPF-iter counter), and the queued task KVAs.
1285 /// Surfaces data the kernel's own `scx_dump_state` does not
1286 /// emit (per-DSQ depth enumeration), so this vec adds value
1287 /// even on a kernel that prints its own dump.
1288 ///
1289 /// Empty when the `ScxWalkerCapture` was absent.
1290 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1291 pub dsq_states: Vec<super::scx_walker::DsqState>,
1292 /// Top-level `scx_sched` state captured from `*scx_root`:
1293 /// aborting flag, bypass_depth, exit_kind. `None` when no
1294 /// scheduler is attached or `*scx_root` was unreadable.
1295 #[serde(default, skip_serializing_if = "Option::is_none")]
1296 pub scx_sched_state: Option<super::scx_walker::ScxSchedState>,
1297 /// Diagnostic reason for `rq_scx_states` / `dsq_states` /
1298 /// `scx_sched_state` being absent. Mirrors the
1299 /// `prog_runtime_stats_unavailable` / `task_enrichments_unavailable`
1300 /// pattern.
1301 #[serde(default, skip_serializing_if = "Option::is_none")]
1302 pub scx_walker_unavailable: Option<String>,
1303 /// Per-vCPU hardware perf counter snapshot captured at the
1304 /// instant the failure dump fired. One entry per vCPU; index
1305 /// matches vCPU id (0 = BSP, 1..N = APs). `None` per-entry when
1306 /// the freeze-time `read(2)` failed for that vCPU. Empty vec
1307 /// when `DumpContext::perf_capture` was None (perf
1308 /// unavailable on this host) or the read errored wholesale.
1309 ///
1310 /// `exclude_host=1` means each counter ticks only during guest
1311 /// execution; the values here record the cumulative count from
1312 /// the start of the run. Diff against any
1313 /// `super::CpuSnapshot::vcpu_perf` in the monitor timeline to
1314 /// recover the count over a freeze-aligned window. See
1315 /// `super::perf_counters::VcpuPerfSample` for field semantics
1316 /// and the multiplexing math.
1317 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1318 pub vcpu_perf_at_freeze: Vec<Option<super::perf_counters::VcpuPerfSample>>,
1319 /// Microseconds from dump_state entry to the phase that exceeded
1320 /// the soft deadline supplied via `DumpContext::deadline`. `None`
1321 /// when no deadline was supplied, when every phase finished within
1322 /// the deadline, or when the deadline check happened before the
1323 /// dump started any heavy phase. A `Some(us)` value means the dump
1324 /// truncated remaining work (skipped further maps / tasks /
1325 /// walkers) at that elapsed offset to keep the freeze window
1326 /// bounded — the freeze coordinator's parked vCPUs cannot
1327 /// service guest IRQs or MMIO traps while the dump is running,
1328 /// so unbounded dump latency stretches every guest's KVM_RUN
1329 /// pause and risks the freeze rendezvous timeout firing on the
1330 /// next iteration.
1331 #[serde(default, skip_serializing_if = "Option::is_none")]
1332 pub dump_truncated_at_us: Option<u64>,
1333 /// Count of scheduler-under-test maps the per-map render loop
1334 /// skipped because the soft deadline had already been crossed
1335 /// (`dump_truncated_at_us` records WHEN, this records HOW MANY).
1336 /// `0` on a complete dump. A skipped map is absent from `maps`
1337 /// entirely — without this count a consumer reading `maps`
1338 /// cannot tell "the scheduler has N maps" from "the scheduler
1339 /// has N+k maps but k were dropped by truncation", so a
1340 /// degraded dump would silently under-report map state. Excludes
1341 /// ktstr's own framework maps, which are filtered before the
1342 /// deadline check and never counted here.
1343 #[serde(default, skip_serializing_if = "is_zero_u32")]
1344 pub maps_truncated: u32,
1345 /// Probe BPF program's per-CPU diagnostic counter snapshot
1346 /// (see `ProbeBssCounters`). Populated by the host-side
1347 /// reader in `decode_probe_counters_snapshot` which sums
1348 /// each `KTSTR_PCPU_*` slot across CPUs. `None` when the
1349 /// probe `.bss` map isn't enumerated (probe not loaded), the
1350 /// program BTF can't be parsed, or the array's offset doesn't
1351 /// resolve.
1352 ///
1353 /// A populated `trigger_count > 0` is the structural signal
1354 /// that the selected BPF scheduler-exit handler fired during
1355 /// the run — distinct from the boolean `trigger_fired` flag
1356 /// in `super::probe::process::ProbeDiagnostics` (which
1357 /// also records host-side observations like a watchdog
1358 /// teardown). The cross-product is the failure-dump E2E
1359 /// test's structural assertion: a stall scenario must show
1360 /// both flag=true AND `trigger_count > 0`, otherwise the
1361 /// probe attached without firing or fired without the host
1362 /// observing.
1363 #[serde(default, skip_serializing_if = "Option::is_none")]
1364 pub probe_counters: Option<ProbeBssCounters>,
1365 /// Host-side setup-failure evidence embedded when this report is a
1366 /// placeholder written for a pre-capture guest failure (payload
1367 /// never ran). `None` on real captures and on placeholders written
1368 /// where no `VmResult` exists (build/run errors before a VM ran).
1369 #[serde(default, skip_serializing_if = "Option::is_none")]
1370 pub guest_setup_failure: Option<GuestSetupFailureContext>,
1371 /// `true` when this report was produced by
1372 /// [`Self::placeholder`] — i.e. the capture pipeline could
1373 /// not produce real data (typical cause: freeze rendezvous
1374 /// timed out). Periodic-sample temporal assertions skip
1375 /// placeholder reports rather than treating their empty
1376 /// vectors as "no progress" signals; the `*_unavailable`
1377 /// fields carry the reason string for human consumers, but
1378 /// the boolean flag is the machine-checkable discriminant a
1379 /// pattern can branch on without re-deriving placeholder
1380 /// status from the absence of every field.
1381 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1382 pub is_placeholder: bool,
1383}
1384
1385impl Default for FailureDumpReport {
1386 /// Empty report with `schema = "single"`. Pinning the schema
1387 /// here keeps `FailureDumpReport::default()` and a
1388 /// freshly-constructed `FailureDumpReport { ..., schema:
1389 /// SCHEMA_SINGLE.into(), ... }` indistinguishable to consumers,
1390 /// so the schema discriminant is never quietly missing on a
1391 /// default-built report.
1392 fn default() -> Self {
1393 Self {
1394 schema: SCHEMA_SINGLE.to_string(),
1395 active_map_kvas: Vec::new(),
1396 maps: Vec::new(),
1397 vcpu_regs: Vec::new(),
1398 sdt_allocations: Vec::new(),
1399 scx_static_ranges: super::scx_static_alloc::ScxStaticSnapshot::default(),
1400 sdt_alloc_unavailable: None,
1401 prog_runtime_stats: Vec::new(),
1402 prog_runtime_stats_unavailable: None,
1403 per_cpu_time: Vec::new(),
1404 cgroup_psi: Vec::new(),
1405 per_node_numa: Vec::new(),
1406 per_node_numa_unavailable: None,
1407 task_enrichments: Vec::new(),
1408 task_enrichments_unavailable: None,
1409 event_counter_timeline: Vec::new(),
1410 rq_scx_states: Vec::new(),
1411 dsq_states: Vec::new(),
1412 scx_sched_state: None,
1413 scx_walker_unavailable: None,
1414 vcpu_perf_at_freeze: Vec::new(),
1415 dump_truncated_at_us: None,
1416 maps_truncated: 0,
1417 probe_counters: None,
1418 guest_setup_failure: None,
1419 is_placeholder: false,
1420 active_obj_name: None,
1421 }
1422 }
1423}
1424
1425impl FailureDumpReport {
1426 /// Build a placeholder report for a capture that could not
1427 /// produce real data. Every `*_unavailable` field is set to
1428 /// `Some(reason)` so downstream consumers (`perf-delta`,
1429 /// failure-rendering tooling) can distinguish "capture
1430 /// happened, no data" from "capture path failed for reason X".
1431 /// All vector / option fields stay at their `Default` empty
1432 /// state so the report is structurally a real
1433 /// `FailureDumpReport`, not a sentinel that breaks consumer
1434 /// type contracts.
1435 ///
1436 /// Used by the freeze coordinator's user-watchpoint dispatch,
1437 /// periodic-capture drain, and final-drain teardown — every
1438 /// site that needs to publish a "capture attempted, did not
1439 /// land" entry on the snapshot bridge.
1440 pub fn placeholder(reason: impl Into<String>) -> Self {
1441 let reason = reason.into();
1442 Self {
1443 prog_runtime_stats_unavailable: Some(reason.clone()),
1444 per_node_numa_unavailable: Some(reason.clone()),
1445 task_enrichments_unavailable: Some(reason.clone()),
1446 scx_walker_unavailable: Some(reason.clone()),
1447 sdt_alloc_unavailable: Some(reason),
1448 is_placeholder: true,
1449 ..Self::default()
1450 }
1451 }
1452}
1453
1454/// Identify the obj name (libbpf `<obj>` prefix) of the
1455/// currently-attached scheduler from the captured BPF state, plus
1456/// the live scheduler's `used_maps` KVA set when available.
1457///
1458/// **No `*scx_root` dependency -- works on every supported kernel.**
1459/// `scx_root` (the global pointer to the active `struct scx_sched`)
1460/// only exists on v6.16+ (added by commit 48e126777386); pre-6.16
1461/// kernels track the active scheduler via the global `scx_ops`
1462/// (`struct sched_ext_ops`) plus the atomic enable-state instead, so
1463/// any `*scx_root`-based identification is blind on 6.14/6.15. Both
1464/// paths below read only `prog_idr` / struct_ops map names /
1465/// global-section map names, which are present and BTF-offset-stable
1466/// across the whole range, so this helper resolves the active obj
1467/// uniformly regardless of kernel version.
1468///
1469/// **PRIMARY: target-free `prog_idr` walk.** Delegates to
1470/// `prog_accessor.find_active_struct_ops_obj_no_target` (see
1471/// `monitor::bpf_prog`), which walks `prog_idr` for an alive
1472/// `BPF_PROG_TYPE_STRUCT_OPS` prog whose `aux->used_maps` carries a
1473/// sibling `<obj>.bss/.data/.rodata` global-section map, and returns
1474/// that prog's obj prefix + full `used_map_kvas` snapshot. The
1475/// returned prefix is cross-checked against the captured `maps[]`
1476/// (a `<prefix>.bss/.data/.rodata` must exist) so a torn `used_maps`
1477/// read cannot publish a garbage prefix. The walk takes no target
1478/// and reads live guest memory, so it produces the correct live
1479/// scheduler even mid-swap and on pre-6.16 kernels. Used only when
1480/// the walker returns a NON-EMPTY `used_map_kvas`; an empty
1481/// whitelist cannot disambiguate two same-prefix copies downstream.
1482///
1483/// The walker matches the FIRST such prog and does not gate on the
1484/// kernel enable-state, so uniqueness rests on ktstr's swap
1485/// sequencing (kill the outgoing scheduler and wait for its process
1486/// to exit before loading the next), NOT on a kernel
1487/// old-prog-removed-before-new-prog-added guarantee: the kernel
1488/// does not serialize those (a detached struct_ops prog leaves
1489/// `prog_idr` only when its owning map's last fd closes and an RCU
1490/// grace elapses). See `find_active_struct_ops_obj_no_target` for
1491/// the single-tenant threat model.
1492///
1493/// **FALLBACK: prefix grouping over struct_ops map names.** Runs
1494/// when `prog_walker` is `None` (the prog accessor has not published
1495/// yet) OR the walker found no global-section-bearing prog OR its
1496/// whitelist was empty. Picks the first `BPF_MAP_TYPE_STRUCT_OPS`
1497/// map whose name prefix has an UNAMBIGUOUS global-section sibling
1498/// set (each of `.bss/.data/.rodata` count <= 1) and returns
1499/// `(prefix, vec![])`. Section counts use full-name equality so they
1500/// stay in lockstep with the consumer's classifier at
1501/// [`crate::scenario::snapshot::Snapshot::active`] and the walker's
1502/// `strip_suffix` in
1503/// `monitor::bpf_prog::extract_global_section_obj_prefix` (private
1504/// helper, cited by path rather than intra-doc link). A multi-copy
1505/// prefix (the same-binary `Op::ReplaceScheduler` swap window leaves
1506/// the dying scheduler's globals beside the new scheduler's) is
1507/// skipped here: the empty-whitelist `(prefix, vec![])` cannot
1508/// disambiguate downstream, so the helper instead returns `None` and
1509/// the consumer surfaces an actionable `NoActiveScheduler`.
1510///
1511/// Returns `None` when neither path resolves: the walker was absent
1512/// / found nothing / returned an unbacked or empty result, AND no
1513/// struct_ops map had an unambiguous global-section sibling set. On
1514/// `None`, callers fall back to
1515/// [`crate::scenario::snapshot::Snapshot::active`]'s own
1516/// prefix-grouping, whose per-section-count check enforces the same
1517/// multi-copy detection, so a helper-`None` does not silently
1518/// downgrade into the prefix-only `AmbiguousVar` surface.
1519fn identify_active_obj_from_struct_ops(
1520 maps: &[super::bpf_map::BpfMapInfo],
1521 prog_walker: Option<(
1522 &dyn super::bpf_prog::BpfProgAccessor,
1523 &super::btf_offsets::BpfMapOffsets,
1524 )>,
1525) -> Option<(String, Vec<u64>)> {
1526 // PRIMARY PATH: target-free prog_idr walk. Returns the first
1527 // alive `BPF_PROG_TYPE_STRUCT_OPS` prog whose `aux->used_maps`
1528 // contains a sibling `<obj>.bss/.data/.rodata` global-section map,
1529 // with that prog's obj prefix + full `used_map_kvas` snapshot.
1530 // Reads live guest memory and takes no target, so it resolves the
1531 // live scheduler on every supported kernel -- including pre-6.16,
1532 // where `scx_root` does not exist. In ktstr scenarios only one
1533 // matching prog is alive at a time: ktstr's swap sequence kills
1534 // the outgoing scheduler and waits for its process to exit before
1535 // loading the next, which closes the outgoing struct_ops map's
1536 // last fd and lets the RCU-deferred map+prog free remove the OLD
1537 // prog from `prog_idr`. (The kernel does NOT serialize
1538 // old-prog-removal before new-prog-add; the single-alive-prog
1539 // property is ktstr's sequencing, not a kernel invariant.) The
1540 // returned prefix is cross-checked against the captured `maps[]`
1541 // so a torn `used_maps` read can't publish a garbage prefix.
1542 if let Some((prog_accessor, map_offsets)) = prog_walker
1543 && let Some(walker_match) = prog_accessor.find_active_struct_ops_obj_no_target(map_offsets)
1544 && !walker_match.used_map_kvas.is_empty()
1545 {
1546 // Defense against torn used_maps reads: the walker's returned
1547 // prefix MUST appear as `<prefix>.<section>` in the captured
1548 // `maps[]`. A walker that read garbage from a mid-mutation
1549 // used_maps window would name an obj prefix that no captured
1550 // map matches; the cross-check rejects that case.
1551 let (wb, wd, wr) = count_global_sections_for_prefix(maps, &walker_match.obj_name);
1552 if wb + wd + wr > 0 {
1553 return Some((walker_match.obj_name, walker_match.used_map_kvas));
1554 }
1555 }
1556
1557 // FALLBACK PATH: prefix grouping by struct_ops map name. Runs
1558 // when the prog walker is unavailable (`prog_walker` is `None`,
1559 // e.g. `owned_prog_accessor` hasn't published yet at boot) OR the
1560 // live STRUCT_OPS prog has no global-section maps in its
1561 // used_maps (libbpf-named struct_ops case without a `.bss/.data/
1562 // .rodata` sibling, observed when the scheduler keeps all its
1563 // state in non-libbpf-named maps).
1564 //
1565 // Pick the first STRUCT_OPS map whose prefix has an unambiguous
1566 // global-section sibling set (each section ≤ 1). Multi-copy
1567 // collisions skip -- the consumer surfaces NoActiveScheduler with
1568 // an actionable diagnostic.
1569 for active_struct_ops in maps
1570 .iter()
1571 .filter(|m| m.map_type == super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS)
1572 {
1573 if active_struct_ops.map_kva == 0 {
1574 continue;
1575 }
1576 let so_name = active_struct_ops.name();
1577 let Some(prefix) = so_name.split('.').next().filter(|s| !s.is_empty()) else {
1578 continue;
1579 };
1580 let (bss_count, data_count, rodata_count) = count_global_sections_for_prefix(maps, prefix);
1581 let has_matching_global = bss_count + data_count + rodata_count > 0;
1582 let unambiguous = bss_count <= 1 && data_count <= 1 && rodata_count <= 1;
1583 if has_matching_global && unambiguous {
1584 return Some((prefix.to_string(), Vec::new()));
1585 }
1586 }
1587 None
1588}
1589
1590/// Count captured maps named exactly `<prefix>.bss`, `<prefix>.data`,
1591/// `<prefix>.rodata`. Full-name equality (not prefix matching) so it
1592/// aligns with the consumer's `name.ends_with(".bss"/...)` classifier
1593/// at [`crate::scenario::snapshot::Snapshot::active`] and with the
1594/// walker's `strip_suffix(".bss")` in
1595/// `monitor::bpf_prog::extract_global_section_obj_prefix` (private
1596/// helper, cited by path rather than intra-doc link). A hypothetical
1597/// `<prefix>.bss.shared` map would count as zero here (the walker
1598/// treats it the same way), so the counts stay in lockstep across
1599/// the three sites that classify global-section maps for a scheduler
1600/// obj.
1601///
1602/// Skips `BPF_MAP_TYPE_STRUCT_OPS` maps so the active scheduler's
1603/// own struct_ops map (which `Snapshot::active` filters by type as
1604/// well) never inflates the global-section totals.
1605fn count_global_sections_for_prefix(
1606 maps: &[super::bpf_map::BpfMapInfo],
1607 prefix: &str,
1608) -> (usize, usize, usize) {
1609 let bss_name = format!("{prefix}.bss");
1610 let data_name = format!("{prefix}.data");
1611 let rodata_name = format!("{prefix}.rodata");
1612 maps.iter()
1613 .filter(|m| m.map_type != super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS)
1614 .fold((0usize, 0usize, 0usize), |(b, d, r), m| {
1615 let n = m.name();
1616 (
1617 b + usize::from(n == bss_name),
1618 d + usize::from(n == data_name),
1619 r + usize::from(n == rodata_name),
1620 )
1621 })
1622}
1623
1624/// Pair of failure-dump snapshots captured at two points in a stall.
1625///
1626/// `early` is taken when the host-side runnable_at scanner observes
1627/// any task with `jiffies - p->scx.runnable_at > watchdog_timeout/2`
1628/// (mirrors the kernel's `check_rq_for_timeouts` walk over
1629/// `rq->scx.runnable_list`). `late` is taken at the same trigger as
1630/// the single-snapshot path: the BPF probe's
1631/// `ktstr_err_exit_detected` latch flipping after a sched_ext
1632/// error-class exit.
1633///
1634/// `early == None` when the watchdog half-way threshold never
1635/// triggered before `late` fired (e.g. an immediate scheduler error
1636/// in `init_task` before any task became runnable). Diffing
1637/// `late` against `early` shows what BPF state changed during the
1638/// stall window — the value-add over the single-snapshot dump.
1639///
1640/// **No user toggle — auto-repro engages this automatically.** Only
1641/// the auto-repro VM emits this shape;
1642/// `crate::test_support::probe::attempt_auto_repro` is the
1643/// single call site flipping the builder's `dual_snapshot` flag,
1644/// and there is no public ktstr surface for asking for it from a
1645/// primary VM. Test authors don't need to know about it — when an
1646/// auto-repro fires, the file at
1647/// `<test>-<variant_hash>.repro.failure-dump.json` changes shape from
1648/// [`FailureDumpReport`] to this wrapper.
1649///
1650/// Note: there is no `Default` impl. The `late` field is required
1651/// by the doc invariant ("the freeze coordinator only writes a
1652/// `DualFailureDumpReport` after the late snapshot has been
1653/// captured"); a `Default::default()` would have produced a wrapper
1654/// with an empty late report whose `maps`/`vcpu_regs` vectors
1655/// silently lie about a successful capture. Construct via the
1656/// struct literal with an explicit `late: FailureDumpReport`.
1657#[derive(Debug, Clone, Serialize, Deserialize)]
1658#[non_exhaustive]
1659pub struct DualFailureDumpReport {
1660 /// Wire-format discriminant. Always `"dual"` for this variant,
1661 /// pinning [`SCHEMA_DUAL`]. Mirror of [`FailureDumpReport::schema`]
1662 /// — consumers branch on it before deserializing.
1663 pub schema: String,
1664 /// Snapshot at the watchdog half-way point. `None` when the
1665 /// stall fired before the half-way scanner crossed its threshold.
1666 #[serde(default, skip_serializing_if = "Option::is_none")]
1667 pub early: Option<FailureDumpReport>,
1668 /// Snapshot at the error-exit latch trigger. Always present
1669 /// (the freeze coordinator only writes a `DualFailureDumpReport`
1670 /// after the late snapshot has been captured; if the run ends
1671 /// with only an early snapshot the file is not written at all).
1672 pub late: FailureDumpReport,
1673 /// Maximum `jiffies - p->scx.runnable_at` observed by the
1674 /// runnable_at scanner at the moment the early snapshot fired.
1675 /// Zero when `early` is `None`.
1676 ///
1677 /// To recover the kernel's full `watchdog_timeout`, double
1678 /// [`Self::early_threshold_jiffies`] — the scanner trigger
1679 /// fires at half the watchdog, so the threshold field carries
1680 /// `watchdog_timeout / 2`. Diff `early_max_age_jiffies` against
1681 /// `2 * early_threshold_jiffies` to see how close the system
1682 /// was to the SCX_EXIT_ERROR_STALL emission line at the
1683 /// early-trigger point.
1684 #[serde(default, skip_serializing_if = "is_zero_u64")]
1685 pub early_max_age_jiffies: u64,
1686 /// The half-way trigger threshold the scanner compared against
1687 /// when capturing the early snapshot, expressed in guest
1688 /// jiffies. Equals `(watchdog_timeout_ms * CONFIG_HZ) / 1000 / 2`
1689 /// at the moment the snapshot fired. Zero when `early` is
1690 /// `None`.
1691 ///
1692 /// Surfaced alongside `early_max_age_jiffies` so a downstream
1693 /// consumer reading the JSON does not have to recompute the
1694 /// kernel-internal jiffies arithmetic to reproduce the
1695 /// trigger condition.
1696 #[serde(default, skip_serializing_if = "is_zero_u64")]
1697 pub early_threshold_jiffies: u64,
1698 /// Structured reason the early snapshot is absent. `None` when
1699 /// the early snapshot was captured (the [`Self::early`] field is
1700 /// `Some`). When the early field is `None`, this carries a short
1701 /// machine-friendly string identifying which of the known
1702 /// failure modes occurred:
1703 ///
1704 /// - `"scan prerequisites unavailable: <prereq>"` — the
1705 /// per-CPU `runnable_at` scan never resolved its dependencies
1706 /// (most often `<prereq>` names the missing kernel symbol /
1707 /// BTF entry).
1708 /// - `"max_age never crossed threshold (peak={peak}j,
1709 /// threshold={threshold}j)"` — the scan ran but the maximum
1710 /// observed runnable-age stayed below the half-way mark for
1711 /// the whole VM lifetime. Indicates a non-stall err-class exit
1712 /// (e.g. `scx_bpf_error()`).
1713 /// - `"scx_tick stall — no per-task runnable_at data"` — the
1714 /// stall path that drove the late capture has no per-task
1715 /// `runnable_at` to scan (the kernel's "watchdog failed to
1716 /// check in" path raises `SCX_EXIT_ERROR_STALL` from the
1717 /// scx_tick kernel side without any task on
1718 /// `rq->scx.runnable_list`).
1719 ///
1720 /// Display rendering at `super::display` surfaces this string
1721 /// directly; the previous "stall fired before half-way threshold,
1722 /// or runnable_at scan setup failed" generic text is replaced
1723 /// with the structured reason whenever this field is `Some`.
1724 #[serde(default, skip_serializing_if = "Option::is_none")]
1725 pub early_skipped_reason: Option<String>,
1726}
1727
1728fn is_zero_u64(v: &u64) -> bool {
1729 *v == 0
1730}
1731
1732fn is_zero_u32(v: &u32) -> bool {
1733 *v == 0
1734}
1735
1736/// Top-level degraded failure-dump report. Emitted by the freeze
1737/// coordinator when a real error-class trigger fires but the dump
1738/// path aborts before a full [`FailureDumpReport`] can be captured —
1739/// today only the vCPU rendezvous-timeout path produces this shape.
1740///
1741/// Carries the partial state the coordinator did collect (per-vCPU
1742/// registers from any vCPU that parked before timeout) plus the
1743/// observable trigger state at the moment of degradation
1744/// (watchpoint hit, BPF `.bss` latch status, live `exit_kind` if the
1745/// gate read it). An operator inspecting the JSON learns WHY the
1746/// dump degraded from the `reason` field and WHICH vCPUs stalled
1747/// from the `vcpu_regs` Vec's per-slot `None` / `Some` pattern.
1748///
1749/// Schema discriminant: [`SCHEMA_DEGRADED`]. Parsed via the same
1750/// [`FailureDumpReportAny::from_json`] dispatcher as the other two
1751/// variants.
1752#[derive(Debug, Clone, Serialize, Deserialize)]
1753#[non_exhaustive]
1754pub struct DegradedFailureDumpReport {
1755 /// Wire-format discriminant. Always `"degraded"` for this
1756 /// variant, pinning [`SCHEMA_DEGRADED`]. Mirror of
1757 /// [`FailureDumpReport::schema`] / [`DualFailureDumpReport::schema`]
1758 /// — consumers branch on it before deserializing.
1759 pub schema: String,
1760 /// Operator-readable reason the dump degraded. Carries one of
1761 /// the `REASON_DEGRADED_*` constants as the canonical prefix,
1762 /// followed by dynamic detail filled in at emit time (e.g.
1763 /// timeout milliseconds, parked-vCPU counts). Stable wire format
1764 /// per the [`SCHEMA_DEGRADED`] discriminant contract: new degraded
1765 /// causes add new `REASON_DEGRADED_*` constants rather than
1766 /// mutating the existing ones.
1767 pub reason: String,
1768 /// Per-vCPU register snapshots collected before degradation.
1769 /// Index matches vCPU id (BSP at 0, APs at 1..N). `None` entries
1770 /// identify the vCPUs that never parked (the operator's
1771 /// signal for which vCPUs stalled) — distinct from
1772 /// [`FailureDumpReport::vcpu_regs`]'s `None`, which usually
1773 /// means `KVM_GET_REGS` failed mid-shutdown.
1774 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1775 pub vcpu_regs: Vec<Option<VcpuRegSnapshot>>,
1776 /// Hardware-watchpoint hit state at degradation. `true` when
1777 /// the freeze-coordinator's `*scx_root->exit_kind` watchpoint
1778 /// fired on a vCPU thread; `false` when only the BPF `.bss`
1779 /// latch fired (or the trigger source was a deferred-capture
1780 /// request).
1781 pub watchpoint_hit: bool,
1782 /// BPF probe `.bss` latch state at degradation. One of
1783 /// `"triggered"` (probe latched err exit), `"not_triggered"`
1784 /// (latch readable, value still 0), `"out_of_bounds"` (cached
1785 /// `.bss` PA no longer 4-byte-readable — probe map freed
1786 /// mid-run), or `"not_resolved"` (cached `.bss` PA was never
1787 /// populated). Mirror of `crate::vmm::freeze_coord`'s
1788 /// internal `BssReadState` enum, serialised as the snake-case
1789 /// of each variant. String-typed for wire-format stability with
1790 /// the rest of the `REASON_*` / state-name surface — see
1791 /// [`SCHEMA_DEGRADED`] for the contract.
1792 pub bss_latch_state: String,
1793 /// Live `*scx_root->exit_kind` value at degradation, when the
1794 /// gate read it. `None` when the dump path aborted before
1795 /// reaching the gate (rendezvous timed out earlier) or when the
1796 /// KVA translation failed. `Some(kind)` carries the raw `u32`
1797 /// from `enum scx_exit_kind` — operators read it against
1798 /// the kernel's `scx_exit_kind` enum definition to identify
1799 /// whether the scheduler's intended exit class matched the
1800 /// trigger that fired.
1801 #[serde(default, skip_serializing_if = "Option::is_none")]
1802 pub exit_kind: Option<u32>,
1803 /// Wall-clock milliseconds from the freeze trigger (capture
1804 /// start) to the degraded-emit decision. Lets an operator see
1805 /// how long the coordinator spent trying to capture before
1806 /// giving up. Mirrors the `elapsed_ms` field
1807 /// [`FailureDumpReport`] surfaces via the post-dump log line —
1808 /// here it's structured so consumers can read it without
1809 /// parsing the log.
1810 #[serde(default, skip_serializing_if = "is_zero_u64")]
1811 pub elapsed_ms: u64,
1812}
1813
1814/// Either-or wrapper that owns a parsed [`FailureDumpReport`],
1815/// [`DualFailureDumpReport`], or [`DegradedFailureDumpReport`]. Lets
1816/// a consumer hold and render a failure-dump file without prematurely
1817/// committing to one schema — the discriminant lives in the JSON's
1818/// `schema` field, not in the type the consumer holds.
1819///
1820/// Centralises the schema-tag dispatch logic that previously lived
1821/// inline at every read site (the auto-repro tail renderer, the
1822/// failure-dump-e2e test, any future consumer that wants to inspect
1823/// any shape). Use [`Self::from_json`] to parse an arbitrary
1824/// failure-dump JSON blob; the Display impl forwards to the
1825/// underlying report's existing Display so the rendered output is
1826/// indistinguishable from holding the unwrapped report directly.
1827///
1828/// `non_exhaustive` so a future fourth schema can be added without
1829/// breaking external pattern matches.
1830#[non_exhaustive]
1831pub enum FailureDumpReportAny {
1832 /// Single-snapshot report, schema=`"single"`. Emitted by the
1833 /// primary VM's freeze coordinator when an error-class SCX exit
1834 /// fires.
1835 Single(Box<FailureDumpReport>),
1836 /// Dual-snapshot wrapper, schema=`"dual"`. Emitted by the
1837 /// auto-repro VM when the dual-snapshot path is enabled. Carries
1838 /// optional `early` + required `late` snapshots plus jiffies
1839 /// metadata for the early-trigger condition.
1840 ///
1841 /// Boxed to keep [`FailureDumpReportAny`]'s on-stack size bounded
1842 /// — `DualFailureDumpReport` carries the early+late snapshots
1843 /// inline and is roughly 2x the size of [`FailureDumpReport`].
1844 Dual(Box<DualFailureDumpReport>),
1845 /// Degraded report, schema=`"degraded"`. Emitted when an
1846 /// error-class trigger fires but the dump path aborts before a
1847 /// full single/dual report can be captured — today only the
1848 /// vCPU rendezvous-timeout path produces this shape. Carries
1849 /// partial vCPU register data + trigger-state diagnostics
1850 /// instead of the full map / scx-walker output.
1851 ///
1852 /// Boxed for size parity with the other variants.
1853 Degraded(Box<DegradedFailureDumpReport>),
1854}
1855
1856impl FailureDumpReportAny {
1857 /// Parse a failure-dump JSON blob, choosing the variant by the
1858 /// `schema` field. Returns `None` on any of:
1859 ///
1860 /// - the blob does not parse as JSON
1861 /// - the `schema` field is absent (degraded variant requires an
1862 /// explicit discriminant; the previous "absent ⇒ single"
1863 /// fallback would silently mis-route a richer wrapper as a
1864 /// lossy single shape)
1865 /// - the `schema` field carries an unknown value
1866 /// - the typed deserialisation under the chosen schema fails
1867 pub fn from_json(json: &str) -> Option<Self> {
1868 let value: serde_json::Value = serde_json::from_str(json).ok()?;
1869 let schema = value.get("schema").and_then(|v| v.as_str())?;
1870 match schema {
1871 SCHEMA_DUAL => serde_json::from_str(json)
1872 .ok()
1873 .map(|d| Self::Dual(Box::new(d))),
1874 SCHEMA_SINGLE => serde_json::from_str(json)
1875 .ok()
1876 .map(|r| Self::Single(Box::new(r))),
1877 SCHEMA_DEGRADED => serde_json::from_str(json)
1878 .ok()
1879 .map(|d| Self::Degraded(Box::new(d))),
1880 _ => None,
1881 }
1882 }
1883}
1884
1885/// Rendering of one BPF map's contents.
1886///
1887/// Unifies the map-type rendering paths under a single
1888/// representation: single-entry ARRAY maps (incl. the
1889/// `.bss`/`.data`/`.rodata` global sections) populate `value`;
1890/// multi-entry ARRAY maps populate `array_entries`; keyed HASH maps
1891/// populate `entries`; per-CPU maps populate `percpu_entries`.
1892/// Exactly one of these is non-empty for a successful render; on
1893/// failure `error` is set and the rest empty.
1894#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1895#[non_exhaustive]
1896pub struct FailureDumpMap {
1897 /// Map name as registered with the kernel. Truncated to
1898 /// `BPF_OBJ_NAME_LEN` (16) by the kernel; libbpf composes
1899 /// `"<obj_name>.<section>"` for global-section maps.
1900 pub name: String,
1901 /// Guest-KVA of this map's `struct bpf_map` allocation. Unique
1902 /// per loaded map instance — two map copies sharing the same
1903 /// `name` (e.g. two `<obj>.bss` maps from two scheduler
1904 /// instances loaded from the same binary post-
1905 /// [`crate::scenario::ops::Op::ReplaceScheduler`]) have distinct
1906 /// KVAs and are distinguishable on this field alone.
1907 ///
1908 /// Sourced from [`crate::monitor::bpf_map::BpfMapInfo::map_kva`]
1909 /// at capture time. Within-run stable (the kernel does not
1910 /// relocate `struct bpf_map`); not comparable across runs
1911 /// (KASLR slide differs).
1912 ///
1913 /// `0` when capture did not record a KVA (e.g., synthetic test
1914 /// fixtures constructed via `..Default::default()`); consumers
1915 /// treating `0` as "no kernel identity" gracefully fall back to
1916 /// name-based matching.
1917 #[serde(default, skip_serializing_if = "is_zero_u64")]
1918 pub map_kva: u64,
1919 /// Raw `map_type` from `struct bpf_map` (e.g. `BPF_MAP_TYPE_ARRAY`).
1920 /// Kept as `u32` rather than an enum to avoid bumping a serde
1921 /// schema each time the kernel adds a kind.
1922 pub map_type: u32,
1923 /// Declared per-entry value size. Captured even when rendering
1924 /// fails so the operator can see the map shape.
1925 pub value_size: u32,
1926 /// Declared maximum entry count from `struct bpf_map.max_entries`.
1927 /// Surfaces alongside the rendered slice so a consumer can spot
1928 /// when the dump shows fewer entries than the map declares
1929 /// (e.g. ARRAY / HASH truncated at `MAX_ARRAY_KEYS` /
1930 /// `MAX_HASH_ENTRIES`; PERCPU_ARRAY truncated at
1931 /// `MAX_PERCPU_KEYS`).
1932 pub max_entries: u32,
1933 /// Single-value render for a single-entry ARRAY map
1934 /// (`max_entries <= 1`, incl. the `.bss`/`.data`/`.rodata`
1935 /// global sections). Multi-entry ARRAY maps use `array_entries`.
1936 #[serde(default, skip_serializing_if = "Option::is_none")]
1937 pub value: Option<RenderedValue>,
1938 /// (key, value) entries for HASH maps.
1939 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1940 pub entries: Vec<FailureDumpEntry>,
1941 /// Per-entry values for a multi-entry `BPF_MAP_TYPE_ARRAY` map,
1942 /// indexed by the array key (`u32`). Populated for
1943 /// `max_entries > 1`; the single-entry case uses `value`, so
1944 /// exactly one of `value` / `array_entries` is set for an ARRAY
1945 /// render. Capped at `MAX_ARRAY_KEYS`; truncation and per-key
1946 /// read failures surface in `error`.
1947 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1948 pub array_entries: Vec<FailureDumpArrayEntry>,
1949 /// Per-CPU slots for PERCPU_ARRAY maps. Outer Vec indexed by key,
1950 /// inner Vec indexed by CPU id.
1951 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1952 pub percpu_entries: Vec<FailureDumpPercpuEntry>,
1953 /// Per-key per-CPU slots for `PERCPU_HASH` / `LRU_PERCPU_HASH`
1954 /// maps. Same shape as `percpu_entries` but the outer key is
1955 /// arbitrary bytes (rendered via BTF when a key type id is
1956 /// available, hex otherwise) instead of the implicit u32 key
1957 /// of a per-CPU array.
1958 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1959 pub percpu_hash_entries: Vec<FailureDumpPercpuHashEntry>,
1960 /// Page snapshot for `BPF_MAP_TYPE_ARENA` maps. `None` for all
1961 /// other map types.
1962 #[serde(default, skip_serializing_if = "Option::is_none")]
1963 pub arena: Option<ArenaSnapshot>,
1964 /// Position counters and capacity for `BPF_MAP_TYPE_RINGBUF` /
1965 /// `BPF_MAP_TYPE_USER_RINGBUF` maps. Surfaces stuck-consumer
1966 /// diagnostics — pending bytes far below the watermark plus
1967 /// non-zero `pending_pos` indicates a producer holding a
1968 /// reservation; pending bytes near capacity indicates a stalled
1969 /// consumer. `None` for non-ringbuf maps or when the BTF offsets
1970 /// for `bpf_ringbuf_map` / `bpf_ringbuf` weren't resolvable.
1971 #[serde(default, skip_serializing_if = "Option::is_none")]
1972 pub ringbuf: Option<FailureDumpRingbuf>,
1973 /// Per-bucket trace summary for `BPF_MAP_TYPE_STACK_TRACE` maps.
1974 /// `None` for non-STACK_TRACE maps or when the BTF offsets for
1975 /// `bpf_stack_map` / `stack_map_bucket` weren't resolvable.
1976 #[serde(default, skip_serializing_if = "Option::is_none")]
1977 pub stack_trace: Option<FailureDumpStackTrace>,
1978 /// Populated-slot summary for FD-array families (`PROG_ARRAY`,
1979 /// `PERF_EVENT_ARRAY`, `CGROUP_ARRAY`, `ARRAY_OF_MAPS`,
1980 /// `HASH_OF_MAPS`, `DEVMAP*`, `SOCKMAP*`, `CPUMAP`, `XSKMAP`,
1981 /// `REUSEPORT_SOCKARRAY`). `None` for non-FD-array maps.
1982 #[serde(default, skip_serializing_if = "Option::is_none")]
1983 pub fd_array: Option<FailureDumpFdArray>,
1984 /// Reason this map's contents are missing or partial. Empty on
1985 /// successful render.
1986 #[serde(default, skip_serializing_if = "Option::is_none")]
1987 pub error: Option<String>,
1988}
1989
1990/// Ringbuf occupancy snapshot read from `struct bpf_ringbuf` at the
1991/// freeze instant.
1992///
1993/// Capacity, consumer/producer positions, and the in-flight reservation
1994/// frontier (`pending_pos`) are all that's readable without walking the
1995/// records. Pending bytes (= `producer_pos - consumer_pos`, computed
1996/// with unsigned wraparound) is the operator-visible indicator: low
1997/// values = consumer keeping up; values approaching capacity = consumer
1998/// stuck or kernel producer overrunning. A non-zero gap between
1999/// `producer_pos` and `pending_pos` means a producer is mid-reserve
2000/// and the consumer can't advance past `pending_pos`.
2001///
2002/// Read via `crate::monitor::btf_offsets::BpfRingbufOffsets`; rendered
2003/// in `render_ringbuf_state`.
2004#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2005#[non_exhaustive]
2006pub struct FailureDumpRingbuf {
2007 /// Ring data area capacity in bytes (= `mask + 1`). Always a
2008 /// power of two; matches the map's declared `max_entries`.
2009 pub capacity: u64,
2010 /// Consumer position. Byte index of the next record userspace
2011 /// will read. Monotonically advances; the kernel never writes
2012 /// here.
2013 pub consumer_pos: u64,
2014 /// Producer position. Byte index past the last reserved record.
2015 /// Monotonically advances; updated by the kernel on each
2016 /// `bpf_ringbuf_reserve`.
2017 pub producer_pos: u64,
2018 /// Pending position. Byte index of the oldest in-flight (still
2019 /// being filled) reservation. Records below `pending_pos` are
2020 /// committed and visible to the consumer; records between
2021 /// `pending_pos` and `producer_pos` are reserved but not yet
2022 /// committed.
2023 pub pending_pos: u64,
2024 /// Pending bytes (= `producer_pos.wrapping_sub(consumer_pos)`).
2025 /// 0 = consumer caught up; capacity = ring full / consumer
2026 /// stalled. Computed with unsigned wraparound to match the
2027 /// kernel's dispatch-path arithmetic.
2028 pub pending_bytes: u64,
2029}
2030
2031/// Per-bucket summary of populated stack traces in a STACK_TRACE map.
2032///
2033/// Each `entry` is one populated bucket whose pointer was non-null at
2034/// the freeze instant. `nr` is the number of trace samples (PCs) in
2035/// the bucket; `pcs` carries the actual u64 PC values when readable
2036/// (build-id stacks render the raw bytes hex since the per-entry
2037/// shape is `struct bpf_stack_build_id`, not a u64). The dump caps
2038/// per-bucket entries at `MAX_STACK_TRACE_PCS` to bound memory.
2039#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2040#[non_exhaustive]
2041pub struct FailureDumpStackTrace {
2042 /// `bpf_stack_map.n_buckets` — the rounded-up power-of-two slot
2043 /// count. Iteration upper bound; differs from `max_entries` which
2044 /// the kernel rounds.
2045 pub n_buckets: u32,
2046 /// One entry per non-null bucket pointer. Sorted by bucket id.
2047 pub entries: Vec<FailureDumpStackTraceEntry>,
2048 /// True when any populated bucket was truncated at
2049 /// `MAX_STACK_TRACE_PCS` PCs.
2050 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2051 pub truncated: bool,
2052 /// Count of buckets whose pointer slot or bucket struct could
2053 /// not be translated to a guest physical address at capture
2054 /// time (an unmapped page in the bucket array or a dangling
2055 /// bucket KVA). These buckets are absent from `entries`
2056 /// entirely. `0` when every non-null bucket was readable.
2057 /// Without this count `entries.len()` undercounts the live
2058 /// buckets and the gap reads as "fewer stacks present" rather
2059 /// than "stacks present but unreadable".
2060 #[serde(default, skip_serializing_if = "is_zero_u32")]
2061 pub buckets_unreadable: u32,
2062}
2063
2064/// One populated stack trace from a STACK_TRACE map.
2065#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2066#[non_exhaustive]
2067pub struct FailureDumpStackTraceEntry {
2068 /// Bucket id (= stack ID returned by `bpf_get_stackid`).
2069 pub bucket_id: u32,
2070 /// Number of trace samples (kernel `stack_map_bucket.nr`).
2071 pub nr: u32,
2072 /// PC values (u64) when the map is in non-build-id mode. Empty
2073 /// when `BPF_F_STACK_BUILD_ID` is set on the map (each entry
2074 /// is then a `bpf_stack_build_id` record — its raw bytes land
2075 /// in `data_hex`).
2076 #[serde(default, skip_serializing_if = "Vec::is_empty")]
2077 pub pcs: Vec<u64>,
2078 /// Hex-encoded raw bucket data bytes. Always populated alongside
2079 /// `pcs` so the operator can decode build-id stacks or correlate
2080 /// trace samples with the wire format.
2081 pub data_hex: String,
2082}
2083
2084/// Per-FD-array snapshot of populated indices.
2085///
2086/// FD-array families store `void *` slots in `bpf_array.ptrs`; each
2087/// slot is either NULL (empty) or a kernel pointer (struct bpf_prog *,
2088/// struct file *, etc.). The dump path reads up to
2089/// `MAX_FD_ARRAY_SLOTS` slots, counts non-zero, and lists the
2090/// populated indices.
2091#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2092#[non_exhaustive]
2093pub struct FailureDumpFdArray {
2094 /// Number of populated (non-zero) slots seen.
2095 pub populated: u32,
2096 /// Total slots scanned. Capped at `MAX_FD_ARRAY_SLOTS`.
2097 pub scanned: u32,
2098 /// Indices of populated slots. Truncated to
2099 /// `MAX_FD_ARRAY_INDICES` entries.
2100 #[serde(default, skip_serializing_if = "Vec::is_empty")]
2101 pub indices: Vec<u32>,
2102 /// True when iteration capped at `MAX_FD_ARRAY_SLOTS` and
2103 /// `scanned < max_entries`.
2104 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2105 pub truncated: bool,
2106 /// True when `populated > indices.len()` because
2107 /// `MAX_FD_ARRAY_INDICES` capped the index list.
2108 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2109 pub indices_truncated: bool,
2110 /// Count of scanned slots whose KVA could not be translated to a
2111 /// guest physical address (an unmapped page in the `ptrs` flex
2112 /// array). These slots are neither confirmed empty nor counted
2113 /// as `populated`, so `populated` is a lower bound when this is
2114 /// non-zero. `0` when every scanned slot was readable.
2115 #[serde(default, skip_serializing_if = "is_zero_u32")]
2116 pub unreadable: u32,
2117}
2118
2119/// One (key, value) pair from a hash map. Both sides are rendered via
2120/// BTF when key/value type ids are available; a `None` rendering
2121/// preserves the raw bytes.
2122#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2123#[non_exhaustive]
2124pub struct FailureDumpEntry {
2125 /// Rendered key. `None` when no BTF type is available for the key.
2126 #[serde(default, skip_serializing_if = "Option::is_none")]
2127 pub key: Option<RenderedValue>,
2128 /// Hex-encoded raw key bytes. Kept alongside `key` so the operator
2129 /// can correlate rendered output with the wire format.
2130 pub key_hex: String,
2131 /// Rendered value. `None` when no BTF type is available.
2132 #[serde(default, skip_serializing_if = "Option::is_none")]
2133 pub value: Option<RenderedValue>,
2134 /// Hex-encoded raw value bytes.
2135 pub value_hex: String,
2136 /// Typed render of the per-entry sdt_alloc payload, when the value
2137 /// carries a `struct sdt_data __arena *` field that points into a
2138 /// captured arena page and a payload type id was discovered for
2139 /// the matching allocator. `None` when the entry carries no arena
2140 /// pointer to chase, no allocator metadata was found, the payload
2141 /// type was ambiguous, or the arena read failed.
2142 ///
2143 /// `value` already renders the surface struct (e.g.
2144 /// `scx_task_map_val { tid, tptr, data: 0x100000... -> sdt_data {
2145 /// tid: { idx, genn } } }`), but `sdt_data.payload[]` is a flex
2146 /// array — BTF reports its size as 0, so the per-task struct that
2147 /// actually lives in the payload bytes never decodes through the
2148 /// surface render. This field carries that decoded payload
2149 /// alongside the surface struct so the operator sees both views
2150 /// at once.
2151 #[serde(default, skip_serializing_if = "Option::is_none")]
2152 pub payload: Option<RenderedValue>,
2153}
2154
2155/// One entry of a multi-entry `BPF_MAP_TYPE_ARRAY` map: the array key
2156/// (`u32`) and its rendered value. Mirrors [`FailureDumpPercpuEntry`]'s
2157/// typed `u32` key (ARRAY keys are kernel-imposed indices, not
2158/// user-typed bytes) but carries a single value rather than a per-CPU
2159/// vector.
2160///
2161/// `value` is `None` only when the entry's guest page was unmapped at
2162/// the freeze instant; a BTF-render miss falls back to
2163/// `RenderedValue::Bytes` (hex), so `None` unambiguously means
2164/// "unreadable", not "un-rendered". ARRAY values are not
2165/// `sdt_data`-arena-chased — no in-tree sched_ext ARRAY stores arena
2166/// pointers; add a `payload` field here if one ever does.
2167#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2168#[non_exhaustive]
2169pub struct FailureDumpArrayEntry {
2170 /// Array index (kernel key).
2171 pub key: u32,
2172 /// Rendered value (BTF when `btf_value_type_id` is non-zero, hex
2173 /// fallback otherwise). `None` when the entry was unreadable.
2174 #[serde(default, skip_serializing_if = "Option::is_none")]
2175 pub value: Option<RenderedValue>,
2176}
2177
2178/// One key from a per-CPU array, with one rendered value per CPU
2179/// (None for CPUs whose per-CPU page was unmapped or out-of-range).
2180#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2181#[non_exhaustive]
2182pub struct FailureDumpPercpuEntry {
2183 pub key: u32,
2184 pub per_cpu: Vec<Option<RenderedValue>>,
2185}
2186
2187/// One key from a `PERCPU_HASH` / `LRU_PERCPU_HASH` map, with one
2188/// rendered value per CPU. Mirrors [`FailureDumpEntry`] for the key
2189/// side (rendered + hex) and [`FailureDumpPercpuEntry`] for the
2190/// per-CPU value vector.
2191#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2192#[non_exhaustive]
2193pub struct FailureDumpPercpuHashEntry {
2194 /// Rendered key. `None` when no BTF type is available for the key.
2195 #[serde(default, skip_serializing_if = "Option::is_none")]
2196 pub key: Option<RenderedValue>,
2197 /// Hex-encoded raw key bytes.
2198 pub key_hex: String,
2199 /// One slot per CPU. `None` when the CPU's per-CPU slot was
2200 /// unmapped or out-of-range; `Some` rendered (BTF when value type
2201 /// id is non-zero) or hex bytes otherwise.
2202 pub per_cpu: Vec<Option<RenderedValue>>,
2203}
2204
2205/// Sanity cap on a single BTF blob read.
2206///
2207/// BPF program BTF is normally <100 KB; vmlinux BTF caps around
2208/// ~10 MB. A bogus `data_size` (corrupted `struct btf`) shouldn't
2209/// pull megabytes of unrelated guest memory into the renderer or the
2210/// freeze coordinator. Shared between [`load_program_btf_kva`] and
2211/// `vmm::load_probe_bss_offset`; defining it here keeps the bound
2212/// in one place so a future tightening doesn't drift between sites.
2213pub(crate) const MAX_BTF_BLOB: usize = 32 * 1024 * 1024;
2214
2215/// Hard cap on the per-task enrichment loop inside [`dump_state`].
2216///
2217/// A hostile or pathologically broken guest can produce a runnable_list
2218/// chain whose length is bounded only by the number of
2219/// `task_struct`s in the kernel — tens of thousands on a busy box.
2220/// Each enrichment call walks task/signal/pid/upid offsets, the
2221/// sched_class registry, and the lock-slowpath stack matcher, so an
2222/// uncapped loop turns the freeze window from milliseconds into
2223/// minutes. 4096 is well above any healthy SCX runnable_list depth
2224/// (the kernel's own watchdog fires long before that many tasks
2225/// queue up) and still bounds the worst-case freeze cost. When the
2226/// cap fires, [`dump_state`] truncates without enriching the tail
2227/// and stamps [`FailureDumpReport::dump_truncated_at_us`] so the
2228/// operator knows to attribute missing tasks to truncation rather
2229/// than walker failure.
2230pub const MAX_ENRICHED_TASKS: usize = 4096;
2231
2232/// Bare-named ktstr framework maps to skip during enumeration.
2233///
2234/// These are declared in `src/bpf/probe.bpf.c` without a libbpf
2235/// `<obj>.<section>` prefix (`SEC(".maps")` declarations like
2236/// `func_meta_map`, `probe_data`, `probe_scratch`, `ktstr_events`);
2237/// the kernel registers them under the bare names listed here.
2238/// They're framework-internal — the user looking at a failure dump
2239/// for their scheduler doesn't care about ktstr's own kprobe
2240/// scratch — so the dump path drops them.
2241///
2242/// The framework's ringbuf is named `ktstr_events` (not `events`)
2243/// so a user scheduler that legitimately names its own ringbuf
2244/// `events` is not silently dropped from the dump.
2245///
2246/// Future ktstr probe additions need to be added here AND the
2247/// matching `<obj_name>.` prefix needs to be in the
2248/// `render_map`-internal starts_with list (see `dump_state`).
2249const KTSTR_INTERNAL_MAPS: &[&str] = &[
2250 "func_meta_map",
2251 "probe_data",
2252 "probe_scratch",
2253 "ktstr_events",
2254];
2255
2256/// All inputs the failure-dump renderer needs, bundled so future
2257/// capture sites (DSQ walker, rq->scx walker, NUMA stats, ...) can
2258/// land as new optional fields without churning every call site.
2259///
2260/// `accessor` is currently the concrete guest-memory backend. The
2261/// trait dispatch claim in [`BpfMapAccessor`]'s module-level doc
2262/// is aspirational: `dump_state` reaches through the accessor for
2263/// map enumeration AND for the sdt_alloc post-pass walk, which
2264/// needs the underlying [`super::guest::GuestKernel`] handle —
2265/// only the guest-memory backend exposes that. When the live-host
2266/// backend lands, sdt_alloc walking will move into a
2267/// backend-specific path and `accessor` here can become
2268/// `&'a dyn BpfMapAccessor`.
2269///
2270/// `arena_offsets` and `prog_capture` are both optional borrows
2271/// (uniform shape): `None` for either disables that
2272/// capture leg without affecting the rest. A scheduler running on
2273/// an older kernel without arena support lands here with
2274/// `arena_offsets: None` and the failure dump renders maps + regs
2275/// without arena pages; a setup where the BpfProgAccessor couldn't
2276/// resolve `prog_idr` lands with `prog_capture: None` and
2277/// `prog_runtime_stats` stays empty.
2278pub struct DumpContext<'a> {
2279 /// BPF map accessor. Concrete guest-memory backend today; see
2280 /// the type-level doc for why this is not `&dyn BpfMapAccessor`.
2281 pub accessor: &'a GuestMemMapAccessor<'a>,
2282 /// Host-resolved vmlinux BTF. The renderer uses it as the base
2283 /// for split-BTF parsing on programs that ship their own type
2284 /// info; it's also the fallback when a map's program BTF can't
2285 /// be loaded.
2286 pub btf: &'a Btf,
2287 /// Guest's `nr_cpu_ids`. Forwarded into per-CPU map rendering
2288 /// so PERCPU_ARRAY readers know how many slots to enumerate.
2289 /// Pass `1` for non-percpu-only dumps if the caller doesn't
2290 /// have the value handy.
2291 pub num_cpus: u32,
2292 /// BTF-resolved arena field offsets. Enables
2293 /// `BPF_MAP_TYPE_ARENA` page snapshotting via the accessor
2294 /// trait's `read_arena_pages`. `None` skips arena rendering
2295 /// (older kernel without arena support, or BTF lacking
2296 /// `struct bpf_arena`).
2297 pub arena_offsets: Option<&'a BpfArenaOffsets>,
2298 /// Per-program runtime stats capture. `None` skips
2299 /// prog-runtime capture; the dump still renders every map the
2300 /// accessor enumerates.
2301 pub prog_capture: Option<&'a ProgRuntimeCapture<'a>>,
2302 /// Per-CPU CPU-time / softirq / IRQ capture. `None` skips the
2303 /// per-CPU time walk; the rest of the dump still renders. Same
2304 /// "borrowed-only, optional" shape as
2305 /// [`Self::prog_capture`] / [`Self::arena_offsets`] so a
2306 /// future capture site lands as another optional field without
2307 /// churning the call sites already plumbed through here.
2308 pub cpu_time_capture: Option<&'a CpuTimeCapture<'a>>,
2309 /// Per-cgroup PSI-irq capture (Phase A). `None` skips the cgroup
2310 /// hierarchy walk; the rest of the dump still renders. Same
2311 /// borrowed-only/optional shape as [`Self::cpu_time_capture`].
2312 pub cgroup_psi_capture: Option<&'a CgroupPsiCapture<'a>>,
2313 /// Per-task enrichment capture. `None` skips the per-task walk
2314 /// and `task_enrichments` stays empty; the rest of the dump
2315 /// still renders.
2316 ///
2317 /// Today's freeze coordinator passes `None` because the DSQ
2318 /// and rq->scx task walkers have not yet landed dispatch. The
2319 /// `TaskEnrichmentOffsets` + `SchedClassRegistry` + the
2320 /// `walk_task_enrichment` library are wired and ready —
2321 /// the producer side just needs to populate
2322 /// [`TaskEnrichmentCapture::tasks`] from the rq->scx walker.
2323 pub task_enrichment_capture: Option<&'a TaskEnrichmentCapture<'a>>,
2324 /// SCX_EV_* event counter timeline capture. `None` skips
2325 /// timeline rendering and `event_counter_timeline` stays
2326 /// empty; the rest of the dump still renders. Same
2327 /// "borrowed-only, optional" shape as
2328 /// [`Self::cpu_time_capture`].
2329 pub event_counter_capture: Option<&'a EventCounterCapture<'a>>,
2330 /// SCX rq->scx + DSQ walker capture. `None` skips the walk;
2331 /// `rq_scx_states` / `dsq_states` / `scx_sched_state` stay
2332 /// empty/None and `scx_walker_unavailable` records why.
2333 pub scx_walker_capture: Option<&'a ScxWalkerCapture<'a>>,
2334 /// Host-side per-vCPU hardware perf counters (cycles,
2335 /// instructions, cache-misses, branch-misses) opened with
2336 /// `exclude_host=1`, so each counter only ticks during guest
2337 /// execution. `None` skips the freeze-time read; the
2338 /// [`FailureDumpReport::vcpu_perf_at_freeze`] vec stays empty.
2339 /// See [`super::perf_counters`] for the kernel-source-grounded
2340 /// rationale and capture semantics.
2341 ///
2342 /// The same capture is shared (via `Arc` in the freeze
2343 /// coordinator) with the per-tick monitor sampler; per-tick
2344 /// samples land on each [`super::CpuSnapshot::vcpu_perf`]. The
2345 /// freeze-time read here records the absolute counter values at
2346 /// the instant the failure dump fired, which lets a consumer
2347 /// diff against any earlier sample to compute IPC over a
2348 /// freeze-aligned window.
2349 pub perf_capture: Option<&'a super::perf_counters::PerfCountersCapture>,
2350 /// Soft deadline for the dump's heavy phases (per-map render
2351 /// loop, walk_rq_scx, walk_local_dsqs, walk_dsqs sched-rooted,
2352 /// walk_task_enrichment, sdt_alloc post-pass). When supplied,
2353 /// each phase boundary checks `Instant::now() > deadline`; the
2354 /// first crossing truncates remaining work and stamps
2355 /// [`FailureDumpReport::dump_truncated_at_us`]. `None` disables
2356 /// the bailout — the dump runs every phase to completion.
2357 ///
2358 /// Set by the freeze coordinator to `capture_start +
2359 /// watchdog_timeout/2` so a slow dump can't keep vCPUs parked
2360 /// past the kernel's own SCX_EXIT_ERROR_STALL emission line. The
2361 /// deadline is a soft bound: each phase that has already started
2362 /// runs to completion before checking, so the actual elapsed
2363 /// time at truncation can exceed the deadline by one phase's
2364 /// worth of work.
2365 pub deadline: Option<std::time::Instant>,
2366 /// BPF cast-analysis output for the scheduler's program object,
2367 /// produced once at builder time by parsing the scheduler
2368 /// binary's `.bpf.objs` ELF blob (no libbpf, no kernel
2369 /// interaction). Threaded into every per-map [`RenderMapCtx`]
2370 /// so the renderer's
2371 /// [`super::btf_render::MemReader::cast_lookup`] can promote
2372 /// `u64` fields the analyzer flagged into typed-pointer
2373 /// renders. `None` skips cast-driven promotion entirely (every
2374 /// `u64` renders as a plain unsigned counter, the
2375 /// pre-integration default); same effect as passing an empty
2376 /// map but cheaper to thread.
2377 pub cast_map: Option<&'a super::cast_analysis::CastMap>,
2378 /// Unique alloc_sizes captured from `scx_static_alloc_internal`
2379 /// call sites. Threaded to the renderer as a last-resort fallback.
2380 pub alloc_size_types: &'a [(u64, String)],
2381 /// Cross-BTF Fwd resolution context: every parsed embedded
2382 /// BPF object's program BTF plus a name-keyed index over
2383 /// every complete struct/union across them. Threaded into
2384 /// every per-map [`RenderMapCtx`] so the renderer's
2385 /// [`super::btf_render::MemReader::cross_btf_resolve_fwd`]
2386 /// can chase a `BTF_KIND_FWD` whose body lives in a sibling
2387 /// embedded object's BTF. Borrowed slices in the
2388 /// `(btfs, fwd_index)` pair point into the
2389 /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] the
2390 /// freeze coordinator holds alive via `Arc` for the dump
2391 /// pass; `None` (no scheduler binary, or analyzer found no
2392 /// complete struct/union definitions) keeps the renderer's
2393 /// "forward declaration; body not in this BTF" skip path
2394 /// intact.
2395 pub cross_btf_fwd_index: Option<CrossBtfFwdIndex<'a>>,
2396}
2397
2398/// Per-dump cross-BTF Fwd resolution context: every parsed program
2399/// BTF the cast-analysis pre-pass discovered, plus a name-keyed
2400/// index over the complete (`!is_fwd`) struct/union definitions
2401/// across them.
2402///
2403/// Built once at the freeze-coordinator side from
2404/// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] and
2405/// threaded through [`DumpContext::cross_btf_fwd_index`] into
2406/// every per-map `AccessorMemReader`. The renderer's
2407/// [`super::btf_render::MemReader::cross_btf_resolve_fwd`]
2408/// override range-looks up the hit and returns a
2409/// [`super::btf_render::CrossBtfRef`] whose `btf` borrow points at
2410/// the matching `Arc<Btf>` inside `btfs`.
2411///
2412/// Empty `btfs` / empty `fwd_index` are valid (no scheduler binary,
2413/// or analyzer found no Struct/Union definitions); the bridge stays
2414/// dormant and the chase falls through to the legacy
2415/// "forward declaration" skip path.
2416pub struct CrossBtfFwdIndex<'a> {
2417 /// Every parsed program BTF in the order
2418 /// `crate::vmm::cast_analysis_load::iter_embedded_bpf_objects`
2419 /// yielded the embedded objects. Index 0 is the first object's
2420 /// BTF, etc. Empty when the scheduler binary had no parseable
2421 /// `.bpf.objs`. Borrowed from the
2422 /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] held
2423 /// alive by the freeze coordinator's `Arc` for the dump pass.
2424 pub btfs: &'a [std::sync::Arc<Btf>],
2425 /// `name -> FwdIndexEntry` over every complete struct/union
2426 /// across `btfs`. See
2427 /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput::fwd_index`]
2428 /// for the construction policy (first-write-wins on duplicate
2429 /// names, anonymous types skipped).
2430 pub fwd_index:
2431 &'a std::collections::HashMap<String, crate::vmm::cast_analysis_load::FwdIndexEntry>,
2432}
2433
2434/// Reconstruct an `ScxSchedState` from the probe BPF program's
2435/// `.bss` snapshot (`ktstr_exit_*` vars).
2436///
2437/// Used as a fallback by [`dump_state`] when
2438/// [`super::scx_walker::read_scx_sched_state`] returned `None`
2439/// because `*scx_root == 0` at freeze time. The probe's selected
2440/// scheduler-exit handler captured the same scalars BEFORE kernel teardown
2441/// nulled `scx_root`, so this path produces a coherent view of
2442/// what the scheduler looked like AT THE INSTANT IT ERRORED OUT —
2443/// which is exactly the state an operator wants to debug.
2444///
2445/// Returns `None` when:
2446/// - the probe `.bss` map isn't loaded yet (boot-race window),
2447/// - the probe's program BTF can't be parsed,
2448/// - the snapshot's `ktstr_exit_kind_snap` is still 0 (latch
2449/// never fired this run, so the snapshot is empty defaults),
2450/// - or any individual var lookup / read fails wholesale.
2451///
2452/// Variable names match the probe BPF declarations one-for-one
2453/// (`ktstr_exit_aborting`, `ktstr_exit_bypass_depth`,
2454/// `ktstr_exit_kind_snap`, `ktstr_exit_sched_kva`,
2455/// `ktstr_exit_watchdog_timeout`); each is resolved by name via the
2456/// program-BTF Datasec walk so a future addition / reorder of `.bss`
2457/// vars does not silently misalign offsets.
2458/// Decode the probe BPF program's per-CPU counter array
2459/// (`ktstr_pcpu_counters`) and sum each slot across CPUs.
2460///
2461/// The probe declares `pcpu_counter ktstr_pcpu_counters[MAX_CPUS]
2462/// [KTSTR_PCPU_NR]` in `.bss`; each `pcpu_counter` is a single
2463/// `long` field forced to 128-byte alignment, so each per-CPU slot
2464/// occupies its own cacheline. The host walks each
2465/// `(cpu, slot)` 8-byte slice and sums into a [`ProbeBssCounters`]
2466/// — see the BPF source for the
2467/// `ktstr_pcpu_inc(KTSTR_PCPU_<NAME>)` fire sites.
2468///
2469/// Returns `None` when:
2470/// - the probe `.bss` map isn't loaded yet (boot-race window),
2471/// - the probe's program BTF can't be parsed,
2472/// - the BTF doesn't carry a `ktstr_pcpu_counters` var (probe
2473/// build that pre-dates the per-CPU conversion), or
2474/// - the array's bytes can't be read wholesale.
2475///
2476/// All values use `u64` for wire compatibility; the underlying
2477/// kernel `long` is signed but every fire site only ever
2478/// increments, so a positive cumulative count is the only outcome
2479/// in practice. Negative reads (would indicate guest-memory
2480/// corruption) saturate to 0 via `as u64`.
2481fn decode_probe_counters_snapshot(
2482 accessor: &GuestMemMapAccessor<'_>,
2483 base_btf: &Btf,
2484) -> Option<ProbeBssCounters> {
2485 use super::bpf_map::BpfMapAccessor;
2486
2487 // Slot indices must match `enum ktstr_pcpu_idx` in
2488 // src/bpf/probe.bpf.c. A reorder in the BPF source breaks
2489 // every reader; the explicit constants here keep the slot
2490 // mapping localized and reviewable.
2491 const PCPU_PROBE_COUNT: usize = 0;
2492 const PCPU_KPROBE_RETURNS: usize = 1;
2493 const PCPU_META_MISS: usize = 2;
2494 const PCPU_RINGBUF_DROPS: usize = 3;
2495 const PCPU_TIMELINE_COUNT: usize = 4;
2496 const PCPU_TIMELINE_DROPS: usize = 5;
2497 const PCPU_PI_COUNT: usize = 6;
2498 const PCPU_PI_ORPHAN_FEXITS: usize = 7;
2499 const PCPU_PI_CLASS_CHANGE_COUNT: usize = 8;
2500 const PCPU_PI_DROPS: usize = 9;
2501 const PCPU_LOCK_CONTEND_COUNT: usize = 10;
2502 const PCPU_LOCK_CONTEND_DROPS: usize = 11;
2503 const PCPU_PREEMPT_DISABLE_COUNT: usize = 12;
2504 const PCPU_PREEMPT_ENABLE_COUNT: usize = 13;
2505 const PCPU_TRIGGER_COUNT: usize = 14;
2506 const PCPU_NR: usize = 15;
2507 /// Per-CPU slot stride in bytes — `pcpu_counter` is forced to
2508 /// 128-byte alignment in the BPF source so each slot occupies
2509 /// one cacheline. Mirroring the alignment here keeps the
2510 /// host-side walk in lockstep with the BPF storage layout;
2511 /// any future change to the alignment must update both.
2512 const PCPU_SLOT_STRIDE: usize = 128;
2513 /// Per-CPU dimension. Matches `MAX_CPUS` in `src/bpf/probe.bpf.c`
2514 /// (CPU_MASK + 1 = 256). Walking every CPU slot is cheap (256
2515 /// CPUs × 15 slots × 8 bytes = 30 KB of reads); slots beyond
2516 /// the actual `nr_cpus` are zero-init `.bss` and contribute
2517 /// nothing to the sum.
2518 const MAX_CPUS: usize = 256;
2519
2520 // Locate the probe's `.bss` map. Same suffix the freeze
2521 // coordinator's lazy-discovery path uses (matched by suffix
2522 // to avoid colliding with a scheduler-under-test's own
2523 // `.bss`).
2524 let bss_map = accessor.find_array_map("probe_bp.bss")?;
2525 if bss_map.btf_kva == 0 {
2526 // Probe not yet loaded — accessor enumerated a stub.
2527 return None;
2528 }
2529
2530 // Load the probe's program BTF as split BTF on top of the
2531 // host vmlinux BTF (matches the freeze coordinator's
2532 // load_probe_bss_offset pattern). Failure is silent — the
2533 // dump path stays best-effort and falls through to None so
2534 // the caller leaves `probe_counters` as None rather than
2535 // emitting a misleading partial.
2536 let prog_btf = load_program_btf_kva(accessor, bss_map.btf_kva, base_btf)?;
2537
2538 // Resolve the array's byte offset within the `.bss` Datasec.
2539 // A missing var (e.g. probe build that pre-dates the per-CPU
2540 // conversion) means the snapshot wasn't emitted — bail.
2541 let array_off = super::btf_offsets::resolve_var_offset_in_section(
2542 &prog_btf,
2543 ".bss",
2544 "ktstr_pcpu_counters",
2545 )? as usize;
2546
2547 // Read the entire array as one slab — 256 * 15 * 128 = 480 KiB.
2548 // A single slab read is cheaper than 256 * 17 individual reads
2549 // through the page-walking accessor; the read primitive
2550 // tolerates over-large requests (truncates at the map's
2551 // value_size) so a future MAX_CPUS / PCPU_NR shrink doesn't
2552 // need a coordinated host update.
2553 let total_bytes = MAX_CPUS * PCPU_NR * PCPU_SLOT_STRIDE;
2554 let array_bytes = accessor.read_value(&bss_map, array_off, total_bytes)?;
2555 if array_bytes.len() < total_bytes {
2556 // Short read — the map's value_size bounds were tighter
2557 // than the array's compile-time shape. A future probe
2558 // build that shrinks MAX_CPUS or PCPU_NR is the expected
2559 // case; bail rather than misalign the slot indexing.
2560 return None;
2561 }
2562
2563 // Sum every CPU's slot. Each slot's `long value` lives at
2564 // offset 0 within the cacheline-aligned `pcpu_counter`
2565 // struct, so the per-(cpu, slot) byte offset is
2566 // `(cpu * PCPU_NR + slot) * PCPU_SLOT_STRIDE`.
2567 let sum_slot = |slot: usize| -> u64 {
2568 let mut total: u64 = 0;
2569 for cpu in 0..MAX_CPUS {
2570 let off = (cpu * PCPU_NR + slot) * PCPU_SLOT_STRIDE;
2571 // BPF runs in little-endian byte order on every
2572 // host arch ktstr targets (x86_64, aarch64). A future
2573 // big-endian host would need an arch gate — flagged
2574 // in the probe BPF source's byte-order section.
2575 let mut buf = [0u8; 8];
2576 buf.copy_from_slice(&array_bytes[off..off + 8]);
2577 // The kernel's `long` is signed but counters only
2578 // increment; cast through `i64` then to `u64` to
2579 // saturate any negative value (corruption signal) to 0.
2580 let v = i64::from_le_bytes(buf);
2581 if v > 0 {
2582 total = total.saturating_add(v as u64);
2583 }
2584 }
2585 total
2586 };
2587
2588 Some(ProbeBssCounters {
2589 probe_count: sum_slot(PCPU_PROBE_COUNT),
2590 kprobe_returns: sum_slot(PCPU_KPROBE_RETURNS),
2591 meta_miss: sum_slot(PCPU_META_MISS),
2592 ringbuf_drops: sum_slot(PCPU_RINGBUF_DROPS),
2593 timeline_count: sum_slot(PCPU_TIMELINE_COUNT),
2594 timeline_drops: sum_slot(PCPU_TIMELINE_DROPS),
2595 pi_count: sum_slot(PCPU_PI_COUNT),
2596 pi_orphan_fexits: sum_slot(PCPU_PI_ORPHAN_FEXITS),
2597 pi_class_change_count: sum_slot(PCPU_PI_CLASS_CHANGE_COUNT),
2598 pi_drops: sum_slot(PCPU_PI_DROPS),
2599 lock_contend_count: sum_slot(PCPU_LOCK_CONTEND_COUNT),
2600 lock_contend_drops: sum_slot(PCPU_LOCK_CONTEND_DROPS),
2601 preempt_disable_count: sum_slot(PCPU_PREEMPT_DISABLE_COUNT),
2602 preempt_enable_count: sum_slot(PCPU_PREEMPT_ENABLE_COUNT),
2603 trigger_count: sum_slot(PCPU_TRIGGER_COUNT),
2604 })
2605}
2606
2607/// Return whether the exact probe-counter failure-dump read path is
2608/// fully usable.
2609///
2610/// This intentionally calls [`decode_probe_counters_snapshot`] rather
2611/// than duplicating a shallower map/BTF probe. A true result therefore
2612/// proves accessor adoption, `probe_bp.bss` discovery, split-BTF
2613/// parsing, `ktstr_pcpu_counters` offset resolution, and the same full
2614/// 480-KiB slab read the eventual dump performs. Counter values may
2615/// still be zero before workload execution; this edge describes
2616/// decodability, not activity.
2617pub(crate) fn probe_counters_snapshot_ready(
2618 accessor: &GuestMemMapAccessor<'_>,
2619 base_btf: &Btf,
2620) -> bool {
2621 decode_probe_counters_snapshot(accessor, base_btf).is_some()
2622}
2623
2624fn decode_probe_sched_state_snapshot(
2625 accessor: &GuestMemMapAccessor<'_>,
2626 base_btf: &Btf,
2627) -> Option<super::scx_walker::ScxSchedState> {
2628 use super::bpf_map::BpfMapAccessor;
2629
2630 // Locate the probe's `.bss` map. Same suffix the freeze
2631 // coordinator's lazy-discovery path uses (matched by suffix to
2632 // avoid colliding with a scheduler-under-test's own `.bss`).
2633 let bss_map = accessor.find_array_map("probe_bp.bss")?;
2634 if bss_map.btf_kva == 0 {
2635 // Probe not yet loaded — accessor enumerated a stub. The
2636 // var offsets live in the program BTF the loader hasn't
2637 // attached yet.
2638 return None;
2639 }
2640
2641 // Load the probe's program BTF as a split BTF on top of the
2642 // host vmlinux BTF (matches the freeze coordinator's
2643 // load_probe_bss_offset pattern). Failure is silent — the dump
2644 // path stays best-effort and falls through to None so the
2645 // caller leaves `scx_sched_state` as None rather than emitting
2646 // a misleading partial.
2647 let prog_btf = load_program_btf_kva(accessor, bss_map.btf_kva, base_btf)?;
2648
2649 // Resolve each `ktstr_exit_*` var's byte offset within the
2650 // `.bss` Datasec. A missing var (e.g. probe build that pre-
2651 // dates the snapshot vars) means the snapshot wasn't emitted —
2652 // bail rather than render zero defaults that would alias as
2653 // "scheduler healthy and exited cleanly".
2654 let kind_off = super::btf_offsets::resolve_var_offset_in_section(
2655 &prog_btf,
2656 ".bss",
2657 "ktstr_exit_kind_snap",
2658 )?;
2659 let aborting_off = super::btf_offsets::resolve_var_offset_in_section(
2660 &prog_btf,
2661 ".bss",
2662 "ktstr_exit_aborting",
2663 )?;
2664 let bypass_depth_off = super::btf_offsets::resolve_var_offset_in_section(
2665 &prog_btf,
2666 ".bss",
2667 "ktstr_exit_bypass_depth",
2668 )?;
2669 let sched_kva_off = super::btf_offsets::resolve_var_offset_in_section(
2670 &prog_btf,
2671 ".bss",
2672 "ktstr_exit_sched_kva",
2673 )?;
2674 let watchdog_timeout_off = super::btf_offsets::resolve_var_offset_in_section(
2675 &prog_btf,
2676 ".bss",
2677 "ktstr_exit_watchdog_timeout",
2678 )?;
2679
2680 // Read each var's bytes via the accessor. `.bss` maps have a
2681 // single key (zero) and the value bytes ARE the section bytes,
2682 // so `read_value(map, off, size)` is the read primitive. A
2683 // failed read on any field bails the whole snapshot — partial
2684 // values would mislead the consumer.
2685 let kind_bytes = accessor.read_value(&bss_map, kind_off as usize, 4)?;
2686 let kind = u32::from_le_bytes(kind_bytes.as_slice().try_into().ok()?);
2687
2688 // The snapshot is sticky: `ktstr_exit_kind_snap` stays at 0
2689 // until the selected BPF handler latches an error-class exit. A
2690 // 0 here means the latch never fired — the snapshot vars are
2691 // all at their initial 0/false defaults and the dump should
2692 // honour `*scx_root == 0` as "no scheduler state to surface"
2693 // rather than render a fake healthy-exit ScxSchedState.
2694 if kind == 0 {
2695 return None;
2696 }
2697
2698 let aborting_bytes = accessor.read_value(&bss_map, aborting_off as usize, 1)?;
2699 let aborting = aborting_bytes.first().copied()? != 0;
2700
2701 let bypass_depth_bytes = accessor.read_value(&bss_map, bypass_depth_off as usize, 4)?;
2702 let bypass_depth = i32::from_le_bytes(bypass_depth_bytes.as_slice().try_into().ok()?);
2703
2704 let sched_kva_bytes = accessor.read_value(&bss_map, sched_kva_off as usize, 8)?;
2705 let sched_kva = u64::from_le_bytes(sched_kva_bytes.as_slice().try_into().ok()?);
2706
2707 let watchdog_timeout_bytes = accessor.read_value(&bss_map, watchdog_timeout_off as usize, 8)?;
2708 let watchdog_timeout = u64::from_le_bytes(watchdog_timeout_bytes.as_slice().try_into().ok()?);
2709
2710 Some(super::scx_walker::ScxSchedState {
2711 aborting,
2712 bypass_depth,
2713 exit_kind: kind,
2714 watchdog_timeout: Some(watchdog_timeout),
2715 source: Some(super::scx_walker::SCX_SCHED_STATE_SOURCE_BSS.to_string()),
2716 // `sched_kva == 0` would mean the BPF probe handler ran
2717 // BEFORE a scheduler identity was available. The tracepoint and
2718 // fexit shapes always carry `sch`; the global-era dump shape has no
2719 // per-scheduler object and intentionally leaves this zero. Surface it as
2720 // None so the consumer can distinguish "snapshot data exists
2721 // but no slab address" from "snapshot has the address" via
2722 // a single Option rather than a magic-zero check.
2723 sched_kva: if sched_kva == 0 {
2724 None
2725 } else {
2726 Some(sched_kva)
2727 },
2728 })
2729}
2730
2731/// Snapshot every BPF map visible to the host accessor.
2732///
2733/// The dump is best-effort: a map that fails to render lands in the
2734/// report with `error: Some(...)` rather than aborting the whole walk,
2735/// so a single corrupt map can't blind the operator to the rest of
2736/// the scheduler's state.
2737pub fn dump_state(ctx: DumpContext<'_>) -> FailureDumpReport {
2738 let DumpContext {
2739 accessor,
2740 btf,
2741 num_cpus,
2742 arena_offsets,
2743 prog_capture,
2744 cpu_time_capture,
2745 cgroup_psi_capture,
2746 task_enrichment_capture,
2747 event_counter_capture,
2748 scx_walker_capture,
2749 perf_capture,
2750 deadline,
2751 cast_map,
2752 cross_btf_fwd_index,
2753 alloc_size_types,
2754 } = ctx;
2755 let cross_btf_fwd_index_ref = cross_btf_fwd_index.as_ref();
2756 // Wall-clock origin for per-phase elapsed_us tracing and the
2757 // soft-deadline bailout. Each heavy phase compares
2758 // `Instant::now()` against `deadline` AFTER it finishes, so a
2759 // truncation captures the phase's data before short-circuiting
2760 // the remaining ones (consistent with the doc on
2761 // [`DumpContext::deadline`]).
2762 let dump_start = std::time::Instant::now();
2763 // Tracks the elapsed_us of the first phase to observe a deadline
2764 // crossing. Stamped onto [`FailureDumpReport::dump_truncated_at_us`]
2765 // at the end so the operator can attribute missing maps / tasks /
2766 // walker results to truncation rather than walker failure.
2767 let mut truncated_at_us: Option<u64> = None;
2768 // Helper closure: returns `true` once the deadline (if any) has
2769 // been crossed. Sets `truncated_at_us` on the FIRST crossing so
2770 // the report records WHERE truncation began, not the last phase
2771 // to short-circuit. Idempotent on repeated calls — once stamped,
2772 // every later phase sees the same elapsed_us.
2773 let deadline_exceeded = |truncated_at: &mut Option<u64>| -> bool {
2774 if let Some(deadline) = deadline {
2775 let now = std::time::Instant::now();
2776 if now > deadline {
2777 if truncated_at.is_none() {
2778 let elapsed_us = dump_start.elapsed().as_micros() as u64;
2779 *truncated_at = Some(elapsed_us);
2780 tracing::warn!(
2781 elapsed_us,
2782 "dump_state: deadline exceeded, truncating remaining phases"
2783 );
2784 }
2785 return true;
2786 }
2787 }
2788 false
2789 };
2790 let maps = accessor.maps();
2791 let (prog_runtime_stats, prog_runtime_stats_unavailable) = match prog_capture {
2792 Some(cap) => {
2793 let stats = cap.accessor.struct_ops_runtime_stats(cap.per_cpu_offsets);
2794 let reason = if stats.is_empty() {
2795 Some(REASON_NO_STRUCT_OPS_LOADED.to_string())
2796 } else {
2797 None
2798 };
2799 (stats, reason)
2800 }
2801 None => (
2802 Vec::new(),
2803 Some(REASON_PROG_ACCESSOR_UNAVAILABLE.to_string()),
2804 ),
2805 };
2806 let per_cpu_time = match cpu_time_capture {
2807 Some(cap) => collect_per_cpu_time(cap),
2808 None => Vec::new(),
2809 };
2810 // Per-cgroup PSI-irq for the test's workload leaves (Phase A). Empty
2811 // when no capture was supplied, the workload root isn't present, or
2812 // psi_cgroups is off — loud-absent.
2813 let cgroup_psi = match cgroup_psi_capture {
2814 Some(cap) => collect_cgroup_psi(cap),
2815 None => Vec::new(),
2816 };
2817 let task_enrichment_t0 = std::time::Instant::now();
2818 let (task_enrichments, task_enrichments_unavailable) = match task_enrichment_capture {
2819 Some(cap) => {
2820 // Cap iteration AND Vec capacity at MAX_ENRICHED_TASKS so
2821 // a hostile guest with a corrupt or absurdly long
2822 // runnable_list can't drag the freeze window into the
2823 // tens-of-seconds range.
2824 let total = cap.tasks.len();
2825 let cap_n = total.min(MAX_ENRICHED_TASKS);
2826 let mut enrichments = Vec::with_capacity(cap_n);
2827 for entry in cap.tasks.iter().take(cap_n) {
2828 if let Some(e) = super::task_enrichment::walk_task_enrichment(
2829 cap.kernel,
2830 entry.task_kva,
2831 cap.offsets,
2832 cap.sched_classes,
2833 cap.lock_slowpaths,
2834 entry.is_runnable_in_scx,
2835 entry.running_pc,
2836 ) {
2837 enrichments.push(e);
2838 }
2839 }
2840 if total > cap_n {
2841 tracing::warn!(
2842 cap = MAX_ENRICHED_TASKS,
2843 total,
2844 "dump_state task_enrichment: capped at MAX_ENRICHED_TASKS, dropping tail"
2845 );
2846 }
2847 let reason = if enrichments.is_empty() {
2848 tracing::debug!(
2849 tasks_count = total,
2850 "dump_state task_enrichment: walker yielded zero entries — \
2851 scx_tasks list and rq->scx.runnable_list both empty, or every \
2852 walk_task_enrichment call returned None (translate failures)",
2853 );
2854 Some(REASON_TASK_WALKER_ZERO_TASKS.to_string())
2855 } else {
2856 None
2857 };
2858 (enrichments, reason)
2859 }
2860 None => {
2861 tracing::debug!(
2862 "dump_state task_enrichment: capture is None — \
2863 freeze coordinator passed no TaskEnrichmentCapture \
2864 (scx_owned, scx_walker_offsets, or task_enrichment_offsets unresolved)",
2865 );
2866 (Vec::new(), Some(REASON_NO_TASK_WALKER.to_string()))
2867 }
2868 };
2869 tracing::debug!(
2870 elapsed_us = task_enrichment_t0.elapsed().as_micros() as u64,
2871 enriched = task_enrichments.len(),
2872 "dump_state phase: walk_task_enrichment"
2873 );
2874 deadline_exceeded(&mut truncated_at_us);
2875 let event_counter_timeline = match event_counter_capture {
2876 Some(cap) => cap
2877 .samples
2878 .iter()
2879 .filter_map(EventCounterSample::from_monitor_sample)
2880 .collect(),
2881 None => Vec::new(),
2882 };
2883 let (rq_scx_states, dsq_states, scx_sched_state, scx_walker_unavailable) =
2884 match scx_walker_capture {
2885 Some(cap) => {
2886 // Sub-group offsets resolved per kernel struct;
2887 // surface the absent groups in the diagnostic so a
2888 // partial walk announces which passes were skipped.
2889 let missing = cap.offsets.missing_groups();
2890
2891 // 1. Read scalar scx_sched state and recover the
2892 // sched_pa for the sched-rooted DSQ walker passes.
2893 // `sched_state` is None when the BTF lacked the
2894 // `sched` sub-group OR when *scx_root == 0
2895 // (no scheduler attached) — both surface as a
2896 // None scx_sched_state in the report. The
2897 // distinction is encoded in `scx_walker_unavailable`
2898 // via REASON_SCX_ROOT_NULL.
2899 let (sched_pa_opt, sched_state) = match super::scx_walker::read_scx_sched_state(
2900 cap.kernel,
2901 cap.scx_root_kva,
2902 cap.offsets,
2903 ) {
2904 Some((sched_kva, state)) => {
2905 // Translate sched_kva → PA (slab/vmalloc; use
2906 // translate_any_kva via the GuestKernel handle).
2907 let mem = cap.kernel.mem();
2908 let walk = cap.kernel.walk_context();
2909 let pa = super::idr::translate_any_kva(
2910 mem,
2911 walk.cr3_pa,
2912 walk.page_offset,
2913 sched_kva,
2914 walk.l5,
2915 walk.tcr_el1,
2916 );
2917 (pa, Some(state))
2918 }
2919 None => {
2920 // Live read failed — `*scx_root == 0` because
2921 // the scheduler has already torn down by
2922 // freeze time. Fall back to the BPF .bss
2923 // snapshot the probe's selected handler latched
2924 // at error-exit time. The snapshot is the
2925 // strict subset of scheduler state the host
2926 // renderer needs; the sched_pa stays None
2927 // because the slab page that backed the live
2928 // `scx_sched` was freed during teardown and
2929 // the sched-rooted DSQ passes (per-node
2930 // global, user dsq_hash) cannot reach it any
2931 // longer. The caller's `unavail` selector
2932 // below now sees `Some(state)` and skips
2933 // REASON_SCX_ROOT_NULL — the consumer reads
2934 // `state.source = "bss_snapshot"` to
2935 // distinguish snapshot from live.
2936 let snap = decode_probe_sched_state_snapshot(accessor, btf);
2937 if snap.is_some() {
2938 tracing::debug!(
2939 scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
2940 "dump_state scx walker: live read returned None; \
2941 BPF .bss snapshot fallback populated scx_sched_state \
2942 (scheduler torn down before freeze, snapshot \
2943 captured at err-exit instant)",
2944 );
2945 }
2946 (None, snap)
2947 }
2948 };
2949
2950 // 2. Per-CPU rq->scx walk. Per-CPU runs only when the
2951 // rq + scx_rq + task sub-groups are present;
2952 // walk_rq_scx returns None to skip otherwise.
2953 let walk_rq_scx_t0 = std::time::Instant::now();
2954 let mut rq_states = Vec::with_capacity(cap.rq_kvas.len());
2955 if !deadline_exceeded(&mut truncated_at_us) {
2956 for (cpu, (&rq_kva, &rq_pa)) in
2957 cap.rq_kvas.iter().zip(cap.rq_pas.iter()).enumerate()
2958 {
2959 if let Some((state, _entries)) = super::scx_walker::walk_rq_scx(
2960 cap.kernel,
2961 cpu as u32,
2962 rq_kva,
2963 rq_pa,
2964 cap.offsets,
2965 ) {
2966 rq_states.push(state);
2967 }
2968 }
2969 }
2970 tracing::debug!(
2971 elapsed_us = walk_rq_scx_t0.elapsed().as_micros() as u64,
2972 cpus = cap.rq_kvas.len(),
2973 rq_states = rq_states.len(),
2974 "dump_state phase: walk_rq_scx"
2975 );
2976
2977 // 3. Per-CPU local DSQ walk runs unconditionally —
2978 // `rq->scx.local_dsq` is initialized at boot
2979 // (init_dsq from kernel/sched/ext.c:4581 for every
2980 // possible CPU) and survives scheduler teardown,
2981 // so it produces data even when *scx_root is NULL.
2982 // This is the data source that survives
2983 // scx_bypass's runnable_list drain
2984 // (kernel/sched/ext.c:5448-5548) during teardown.
2985 let walk_local_dsqs_t0 = std::time::Instant::now();
2986 let mut dsqs: Vec<super::scx_walker::DsqState> = Vec::new();
2987 if !deadline_exceeded(&mut truncated_at_us)
2988 && let Some((local_states, _entries)) = super::scx_walker::walk_local_dsqs(
2989 cap.kernel,
2990 cap.rq_kvas,
2991 cap.rq_pas,
2992 cap.per_cpu_offsets,
2993 cap.offsets,
2994 )
2995 {
2996 dsqs.extend(local_states);
2997 }
2998 tracing::debug!(
2999 elapsed_us = walk_local_dsqs_t0.elapsed().as_micros() as u64,
3000 local_dsqs = dsqs.len(),
3001 "dump_state phase: walk_local_dsqs"
3002 );
3003
3004 // 4. Sched-rooted DSQ passes (per-CPU bypass, per-node
3005 // global, user dsq_hash) require the sched_pa we
3006 // resolved in step 1. Without it, no scheduler is
3007 // attached and these DSQs don't exist at all.
3008 let walk_dsqs_t0 = std::time::Instant::now();
3009 if !deadline_exceeded(&mut truncated_at_us)
3010 && let Some(sched_pa) = sched_pa_opt
3011 {
3012 let (sched_states, _entries) = super::scx_walker::walk_dsqs(
3013 cap.kernel,
3014 sched_pa,
3015 cap.per_cpu_offsets,
3016 cap.nr_nodes,
3017 cap.offsets,
3018 );
3019 dsqs.extend(sched_states);
3020 }
3021 tracing::debug!(
3022 elapsed_us = walk_dsqs_t0.elapsed().as_micros() as u64,
3023 total_dsqs = dsqs.len(),
3024 "dump_state phase: walk_dsqs"
3025 );
3026
3027 // Diagnostic priority:
3028 // 1. Partial-degradation (sub-group(s) missing) —
3029 // announces exactly which passes were skipped.
3030 // 2. *scx_root is NULL — sched/bypass/global/user
3031 // passes blinded but rq->scx + local DSQ still
3032 // work; surface this distinct reason so the
3033 // operator knows the scheduler isn't attached.
3034 // 3. Walker reached no state at all — typical when
3035 // every read fails.
3036 // 4. None — every pass had data to surface.
3037 let unavail = if !missing.is_empty() {
3038 tracing::debug!(
3039 missing_groups = ?missing,
3040 rq_states_count = rq_states.len(),
3041 dsq_count = dsqs.len(),
3042 sched_state_some = sched_state.is_some(),
3043 "dump_state scx walker: partial degradation — missing BTF sub-groups",
3044 );
3045 Some(format!(
3046 "scx walker partial: missing offset groups [{}]",
3047 missing.join(", ")
3048 ))
3049 } else if sched_state.is_none() {
3050 tracing::debug!(
3051 scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
3052 rq_states_count = rq_states.len(),
3053 dsq_count = dsqs.len(),
3054 "dump_state scx walker: scx_root is NULL — no scheduler attached; \
3055 rq->scx and local DSQ captures populated, sched/bypass/global/user passes blinded",
3056 );
3057 Some(REASON_SCX_ROOT_NULL.to_string())
3058 } else if rq_states.is_empty() && dsqs.is_empty() {
3059 tracing::debug!(
3060 scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
3061 "dump_state scx walker: every walker read failed — no rq->scx, no DSQ, but sched_state present",
3062 );
3063 Some(REASON_SCX_WALKER_NO_STATE.to_string())
3064 } else {
3065 None
3066 };
3067 (rq_states, dsqs, sched_state, unavail)
3068 }
3069 None => {
3070 tracing::debug!(
3071 "dump_state scx walker: capture is None — \
3072 freeze coordinator passed no ScxWalkerCapture (offsets/symbols/per_cpu_offsets unresolved)",
3073 );
3074 (
3075 Vec::new(),
3076 Vec::new(),
3077 None,
3078 Some(REASON_NO_SCX_WALKER.to_string()),
3079 )
3080 }
3081 };
3082 // Freeze-time per-vCPU perf-counter snapshot. With `exclude_host=1`
3083 // each counter ticks only during guest execution; the freeze
3084 // coordinator has parked every vCPU before reaching this site, so
3085 // the read returns the cumulative count at the last guest exit
3086 // for each vCPU. A single per-vCPU read failure is recorded as
3087 // `None` for that entry; a failure on one vCPU does not blank the
3088 // others. When `perf_capture` is None the vec stays empty (the
3089 // host lacked perf, or `perf_event_open` failed at run start).
3090 let vcpu_perf_at_freeze: Vec<Option<super::perf_counters::VcpuPerfSample>> = match perf_capture
3091 {
3092 Some(cap) => cap.per_vcpu.iter().map(|p| p.read().ok()).collect(),
3093 None => Vec::new(),
3094 };
3095
3096 // Snapshot the probe's per-CPU diagnostic counters before the
3097 // per-map render loop walks `.bss` itself — the read goes
3098 // through the same `read_value` path the renderer uses, but
3099 // captures the array as a structured `ProbeBssCounters` rather
3100 // than the BTF Datasec render. Best-effort: a None result
3101 // (probe not loaded, BTF missing the var) leaves the report's
3102 // `probe_counters` empty and the existing `.bss` map render
3103 // still surfaces the raw bytes.
3104 let probe_counters = decode_probe_counters_snapshot(accessor, btf);
3105
3106 // Resolve the active-scheduler obj name. The prog accessor +
3107 // BpfMapOffsets pair powers the target-free `prog_idr` walker
3108 // (the primary path; needs no `scx_root`, so it works on
3109 // pre-6.16 kernels too) -- when absent, the helper degrades to the
3110 // prefix-grouping fallback over struct_ops map names.
3111 let prog_walker = prog_capture.map(|cap| {
3112 (
3113 cap.accessor as &dyn super::bpf_prog::BpfProgAccessor,
3114 accessor.offsets(),
3115 )
3116 });
3117 let (active_obj_name, active_map_kvas) =
3118 match identify_active_obj_from_struct_ops(&maps, prog_walker) {
3119 Some((name, kvas)) => (Some(name), kvas),
3120 None => (None, Vec::new()),
3121 };
3122 let mut report = FailureDumpReport {
3123 active_map_kvas,
3124 maps: Vec::with_capacity(maps.len()),
3125 prog_runtime_stats,
3126 prog_runtime_stats_unavailable,
3127 per_cpu_time,
3128 cgroup_psi,
3129 // Per-node NUMA wire fields: empty Vec + the well-defined
3130 // diagnostic string until the host-side walker lands.
3131 per_node_numa: Vec::new(),
3132 per_node_numa_unavailable: Some(REASON_NO_NUMA_WALKER.to_string()),
3133 task_enrichments,
3134 task_enrichments_unavailable,
3135 event_counter_timeline,
3136 rq_scx_states,
3137 dsq_states,
3138 scx_sched_state,
3139 scx_walker_unavailable,
3140 vcpu_perf_at_freeze,
3141 probe_counters,
3142 active_obj_name,
3143 ..Default::default()
3144 };
3145
3146 // Per-map program-BTF cache, keyed by `btf_kva`. Each unique
3147 // `struct btf *` lives in the kernel BTF IDR — multiple maps from
3148 // the same BPF program point at the same KVA, so caching dedupes
3149 // the heavy `Btf::from_bytes`/`from_split_bytes` parse across them
3150 // (a scheduler with N maps backed by one BPF object pays one
3151 // parse, not N). Lookups go through this cache before falling
3152 // back to the caller-supplied vmlinux `btf`.
3153 //
3154 // Populated by an explicit pre-pass below so the sdt_alloc walk
3155 // can read it before the per-map render loop runs (the renderer
3156 // needs the resulting allocator metadata via `RenderMapCtx`).
3157 let mut program_btfs: std::collections::HashMap<u64, Btf> = std::collections::HashMap::new();
3158
3159 // Pre-pass: locate the first non-internal `BPF_MAP_TYPE_ARENA`
3160 // map (skipping the same ktstr-internal name set the main loop
3161 // skips) and snapshot it once before any map renders. This lets
3162 // the per-map `MemReader` chase `__arena` pointers no matter
3163 // which slot the arena map occupies in the iteration order —
3164 // the previous design ran `snapshot_arena` lazily inside
3165 // `render_map`'s arena arm, so non-arena maps that rendered
3166 // earlier saw `arena_snapshot: None` and silently failed every
3167 // arena pointer chase. `lib/arena_map.h` declares one `__weak`
3168 // arena per BPF object so a single shared snapshot covers every
3169 // `__arena` pointer the scheduler emits; additional arena maps
3170 // (multi-object schedulers, theoretical) still get their own
3171 // snapshot inside `render_map`'s arena arm — they just don't
3172 // contribute to the cross-map pointer-chase context.
3173 let shared_arena_snapshot: Option<(BpfMapInfo, ArenaSnapshot)> =
3174 arena_offsets.and_then(|off| {
3175 for info in &maps {
3176 let name = info.name();
3177 if name.starts_with("probe_bp.")
3178 || name.starts_with("fentry_p.")
3179 || name == "probe_bp"
3180 || name == "fentry_p"
3181 || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
3182 {
3183 continue;
3184 }
3185 if info.map_type == BPF_MAP_TYPE_ARENA {
3186 let snap = snapshot_arena(accessor.kernel(), info, off);
3187 return Some((info.clone(), snap));
3188 }
3189 }
3190 None
3191 });
3192 let shared_arena_ref: Option<(&ArenaSnapshot, u64)> = shared_arena_snapshot
3193 .as_ref()
3194 .map(|(info, snap)| (snap, info.map_kva));
3195
3196 // Cache `kern_vm_start` from the pre-pass snapshot for the
3197 // sdt_alloc walk. Pulling directly from `shared_arena_snapshot`
3198 // (rather than scraping each rendered map's `arena` field in the
3199 // main loop) keeps the walk gating decoupled from per-map render
3200 // order — the data the walker needs is finalized before the
3201 // loop runs.
3202 let arena_kern_vm_start: u64 = shared_arena_snapshot
3203 .as_ref()
3204 .map(|(_, snap)| snap.kern_vm_start)
3205 .unwrap_or(0);
3206
3207 // Pre-pass: load every non-internal map's program BTF and locate
3208 // the scheduler's `.bss` raw bytes. Both inputs feed the
3209 // sdt_alloc walk below — moving them out of the main render loop
3210 // means the allocator metadata that decoration in the
3211 // TASK_STORAGE arm needs (`elem_size`, `target_type_id`) is
3212 // available BEFORE any map renders, instead of getting derived
3213 // post-loop only to be unusable for per-entry payload chase.
3214 let mut sched_bss_bytes: Option<(Vec<u8>, u64)> = None; // (bytes, btf_kva)
3215 for info in &maps {
3216 let name = info.name();
3217 if name.starts_with("probe_bp.")
3218 || name.starts_with("fentry_p.")
3219 || name == "probe_bp"
3220 || name == "fentry_p"
3221 || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
3222 {
3223 continue;
3224 }
3225 if info.btf_kva != 0
3226 && !program_btfs.contains_key(&info.btf_kva)
3227 && let Some(loaded) = accessor.load_program_btf(info, btf)
3228 {
3229 program_btfs.insert(info.btf_kva, loaded);
3230 }
3231 if sched_bss_bytes.is_none()
3232 && info.map_type == BPF_MAP_TYPE_ARRAY
3233 && info.btf_kva != 0
3234 && name.ends_with(".bss")
3235 && let Some(bytes) = accessor.read_value(info, 0, info.value_size as usize)
3236 {
3237 sched_bss_bytes = Some((bytes, info.btf_kva));
3238 }
3239 }
3240
3241 // Pre-pass: walk sdt_alloc trees if all prerequisites lined up.
3242 // Runs BEFORE the main render loop so the allocator metadata it
3243 // discovers (`elem_size`, `target_type_id`,
3244 // `data_header_size`) is available to per-map decoration —
3245 // specifically, the TASK_STORAGE arm uses it to expand each
3246 // entry's `struct sdt_data __arena *` pointer into a typed
3247 // payload render via [`render_map`]'s
3248 // [`crate::monitor::dump::render_map::SdtAllocMeta`].
3249 //
3250 // The walk is best-effort and silent: any missing prerequisite
3251 // (no scheduler .bss, no arena window, no program BTF, no
3252 // `scx_allocator` type) leaves `sdt_allocations` empty rather
3253 // than failing the dump. `sdt_alloc_metas` stays empty in the
3254 // same cases, so each per-entry payload field also degrades to
3255 // `None` (the surface struct still renders).
3256 //
3257 // Build the dump-pass arena page index here too — once outside
3258 // the per-map loop so each per-map `mem_reader` borrows the
3259 // existing table instead of rebuilding it. The sdt_alloc walk
3260 // below uses the same index for its own MemReader.
3261 let arena_page_index = crate::monitor::dump::render_map::build_arena_page_index(
3262 shared_arena_snapshot.as_ref().map(|(_, snap)| snap),
3263 );
3264 let sdt_alloc_t0 = std::time::Instant::now();
3265 // Every typed allocator the program declares; the per-map
3266 // selector in [`render_map`] picks the matching entry by name
3267 // (e.g. `scx_task_allocator` matches `scx_task_map`). A
3268 // single-allocator scheduler hits the unique-candidate path —
3269 // every map gets that allocator. A multi-allocator scheduler
3270 // (per-task + per-cgroup) lets each local-storage map render
3271 // its own payload type instead of forcing the renderer to give
3272 // up.
3273 let mut sdt_alloc_metas: Vec<crate::monitor::dump::render_map::SdtAllocMeta> = Vec::new();
3274 // `slot_start → ArenaSlotInfo` lookup populated as each
3275 // allocator walk completes.
3276 // [`MemReader::resolve_arena_type`] consults this index via a
3277 // range lookup: given a chased address, find the slot whose
3278 // `[slot_start, slot_start + elem_size)` range contains it,
3279 // then route on `offset_in_slot`:
3280 //
3281 // - `offset_in_slot == 0` (slot-start pointer, e.g. the
3282 // `data` field of `scx_task_map_val` storing the raw
3283 // `sdt_alloc()` return) → render the payload skipping
3284 // `header_size` bytes of header.
3285 // - `offset_in_slot == header_size` (payload-start pointer,
3286 // e.g. the return of `scx_task_data(p)` cached in
3287 // `cached_taskc_raw`) → render the payload directly.
3288 // - Other in-slot offsets → no resolve; the renderer falls
3289 // back to its existing skip behaviour.
3290 //
3291 // Built incrementally inside the walk loop so the
3292 // per-allocator snapshot moves into `report.sdt_allocations`
3293 // after each iteration without a clone.
3294 //
3295 // [`crate::monitor::sdt_alloc::TreeWalker::emit_leaf`]
3296 // populates each [`SdtAllocEntry::user_addr`] as
3297 // `data_ptr & 0xFFFF_FFFF` — the slot-START address windowed
3298 // to the low 32 bits. The index keys directly on this
3299 // windowed slot start, paired with an [`ArenaSlotInfo`] that
3300 // carries `elem_size`, `header_size`, and the payload BTF
3301 // type id so the [`MemReader::resolve_arena_type`] range
3302 // lookup has every value it needs to decide the chase shape.
3303 //
3304 // Slot non-overlap invariant: the kernel allocator places
3305 // slots back-to-back inside one `sdt_chunk` and never re-uses
3306 // a position while the bitmap still has it marked allocated
3307 // (see `lib/sdt_alloc.bpf.c::scx_alloc_internal`'s
3308 // bitmap-then-data ordering). Two distinct slots cannot have
3309 // overlapping `[start, start + elem_size)` ranges, so
3310 // dedup-on-exact-key here is sufficient — we cannot land on a
3311 // case where `slot_a + elem_a > slot_b > slot_a` with
3312 // `slot_b` separately keyed.
3313 //
3314 // Duplicates (two slots reporting the same slot start,
3315 // indicating a stale snapshot from a freed allocation racing
3316 // with the freeze) keep the FIRST entry; this matches the
3317 // [`build_arena_page_index`] policy on duplicate user_addr
3318 // pages and emits a `tracing::warn!` line so an operator
3319 // diagnosing a wrong-render can spot the collision.
3320 let mut arena_slot_index = crate::monitor::dump::render_map::ArenaSlotIndex::new();
3321 // 4 GiB-alignment invariant: the bridge keys on the low 32
3322 // bits of slot start. That is correct iff `user_vm_start` is
3323 // 4 GiB-aligned — `slot_full_addr - slot_low32 == user_vm_start`
3324 // and the renderer reconstructs full addresses by masking
3325 // chased values with `0xFFFF_FFFF`. Every in-tree scx scheduler
3326 // sets `map_extra` to a 4 GiB-aligned value (`1 << 32`,
3327 // `1 << 44`); the kernel auto-pick path in
3328 // `bpf_arena_map_alloc` (kernel/bpf/arena.c) rounds the user
3329 // VM area up to `SZ_4G` before mounting. The kernel does
3330 // accept arbitrary `map_extra` from userspace, so an
3331 // out-of-tree scheduler could in theory pass an unaligned
3332 // value — surface a warning and skip the index build rather
3333 // than silently misroute every chase.
3334 let user_vm_aligned = shared_arena_snapshot
3335 .as_ref()
3336 .map(|(_, snap)| snap.user_vm_start & 0xFFFF_FFFF == 0)
3337 .unwrap_or(false);
3338 if !user_vm_aligned && let Some((_, snap)) = shared_arena_snapshot.as_ref() {
3339 tracing::warn!(
3340 user_vm_start = format_args!("{:#x}", snap.user_vm_start),
3341 "sdt_alloc bridge skipped: user_vm_start is not 4 GiB-aligned; \
3342 low-32 keying would misroute every chase",
3343 );
3344 }
3345 // `user_vm_start == 0` is technically 4 GiB-aligned (and the
3346 // gate above accepts it), but the
3347 // [`super::dump::render_map::is_arena_addr_in_snapshot`] helper
3348 // rejects every address when `user_vm_start == 0` — silently
3349 // disabling the bridge. Surface a warn so an operator
3350 // diagnosing missing typed-pointer renders sees the cause
3351 // (likely a snapshot capture failure that produced an
3352 // uninitialized arena VM start).
3353 if let Some((_, snap)) = shared_arena_snapshot.as_ref()
3354 && snap.user_vm_start == 0
3355 {
3356 tracing::warn!(
3357 "sdt_alloc bridge effectively disabled: user_vm_start == 0 \
3358 (snapshot capture may have failed before resolving the \
3359 arena's user VM window); every chase will skip with \
3360 `is_arena_addr` = false",
3361 );
3362 }
3363 // Resolve the unavailable reason in the order the gate checks
3364 // run. The first failing prerequisite wins — subsequent reasons
3365 // (which would all surface for the same missing prerequisite) are
3366 // suppressed. `None` here means the pre-pass actually runs; the
3367 // sdt_alloc_unavailable field is then populated post-loop based
3368 // on whether any allocator was discovered.
3369 let sdt_alloc_skip_reason: Option<&'static str> = if deadline_exceeded(&mut truncated_at_us) {
3370 Some(REASON_SDT_ALLOC_DEADLINE_EXCEEDED)
3371 } else if !user_vm_aligned {
3372 Some(REASON_SDT_ALLOC_UNALIGNED_USER_VM)
3373 } else if sched_bss_bytes.is_none() {
3374 Some(REASON_SDT_ALLOC_NO_BSS)
3375 } else if arena_kern_vm_start == 0 {
3376 Some(REASON_SDT_ALLOC_NO_ARENA)
3377 } else if let Some((_, btf_kva)) = sched_bss_bytes.as_ref()
3378 && !program_btfs.contains_key(btf_kva)
3379 {
3380 Some(REASON_SDT_ALLOC_NO_BSS)
3381 } else if let Some((_, btf_kva)) = sched_bss_bytes.as_ref()
3382 && let Some(prog_btf) = program_btfs.get(btf_kva)
3383 && SdtAllocOffsets::from_btf(prog_btf).is_err()
3384 {
3385 Some(REASON_SDT_ALLOC_NO_TYPE)
3386 } else {
3387 None
3388 };
3389 if let Some(reason) = sdt_alloc_skip_reason {
3390 report.sdt_alloc_unavailable = Some(reason.to_string());
3391 }
3392 // Track whether the pre-pass body ran (every prerequisite
3393 // satisfied). Distinct from `sdt_alloc_skip_reason`: if the
3394 // body runs but discovers no `.bss` instance of
3395 // `struct scx_allocator`, the unavailable reason flips to
3396 // [`REASON_SDT_ALLOC_NO_INSTANCE`] AFTER the loop.
3397 let mut sdt_alloc_pre_pass_ran = false;
3398 if !deadline_exceeded(&mut truncated_at_us)
3399 && user_vm_aligned
3400 && let Some((bss_bytes, btf_kva)) = sched_bss_bytes
3401 && arena_kern_vm_start != 0
3402 && let Some(prog_btf) = program_btfs.get(&btf_kva)
3403 && let Ok(sdt_offsets) = SdtAllocOffsets::from_btf(prog_btf)
3404 {
3405 sdt_alloc_pre_pass_ran = true;
3406 // One MemReader for every leaf payload render, so an
3407 // arena pointer embedded in a per-task / per-cgroup
3408 // sdt_alloc payload chases into typed contents instead
3409 // of opaque hex.
3410 //
3411 // The arena type index is intentionally `None` on this
3412 // pre-pass reader: the walk produces the entries the
3413 // index is built from, so the index does not yet exist
3414 // when the leaf payload renders run. A nested `__arena
3415 // *` pointer inside a payload that targets a separate
3416 // allocator slot whose payload type is forward-declared
3417 // in the program BTF degrades to the existing chase
3418 // behaviour during the pre-pass; the index is wired
3419 // into the per-map renders below where the typical
3420 // bridge call site lives (TASK_STORAGE / HASH maps
3421 // holding `struct sdt_data __arena *` entry pointers).
3422 let sdt_mem = accessor.mem_reader(
3423 shared_arena_snapshot.as_ref().map(|(_, snap)| snap),
3424 &arena_page_index,
3425 num_cpus,
3426 // Threaded in from [`DumpContext::cast_map`]: same
3427 // cast-analysis output the per-map renderer below
3428 // consumes. Letting the sdt_alloc pre-pass see it
3429 // means typed-allocator payload chases (per-task /
3430 // per-cgroup contents inside arena) get the same
3431 // `u64` → typed-pointer promotion as the rest of
3432 // the dump, instead of degrading to plain counters
3433 // for fields the analyzer recovered.
3434 cast_map,
3435 None,
3436 // The sdt_alloc pre-pass populates `sdt_alloc_metas`
3437 // itself; the metas slice is empty until this loop
3438 // finishes, so pass an empty slice here. The
3439 // metas-driven fallback only fires for the per-map
3440 // renders below (after this loop has produced every
3441 // allocator's metadata).
3442 &[],
3443 // The sdt_alloc pre-pass reads BTF type metadata for
3444 // the typed allocator payload from the scheduler's
3445 // own program BTF; no cross-BTF Fwd resolution is
3446 // needed here. The per-map renders below pass the
3447 // built `cross_btf_fwd_index` where it matters.
3448 None,
3449 // The sdt_alloc pre-pass runs BEFORE the
3450 // [`crate::monitor::scx_static_alloc`] walker (the
3451 // walks are independent, but the per-allocator leaf
3452 // payload renders here happen during the sdt_alloc
3453 // walk's own loop, so the scx_static index is not yet
3454 // built). Pass `None` so the bridge stays a no-op for
3455 // these pre-pass renders; the per-map renders below
3456 // pass the built `scx_static_index` where it matters.
3457 None,
3458 // The sdt_alloc pre-pass IS the surface that produces
3459 // the rendered-slot set; pass `None` so no dedup gate
3460 // fires while the typed-allocator surface itself is
3461 // being assembled. The per-map renders below pass the
3462 // built set where it matters.
3463 None,
3464 alloc_size_types,
3465 // The sdt_alloc pre-pass walks against the scheduler's
3466 // program BTF identified by `btf_kva` (the same BTF the
3467 // allocator metadata was resolved against — see
3468 // `append_arena_slot_index_for_allocator` below). Pass
3469 // this kva as the requesting-BTF identifier so the
3470 // pre-pass's own leaf renders compare against it
3471 // (currently the index is still being built so the gate
3472 // never fires here, but the threading keeps the contract
3473 // consistent across all `mem_reader` call sites).
3474 btf_kva,
3475 );
3476 // Locate every sdt_alloc allocator instance declared in
3477 // `.bss`. The Datasec walk gives us each variable's name and
3478 // offset; we filter to types matching `struct scx_allocator`
3479 // by re-resolving the var's chained type. A scheduler may
3480 // declare more than one allocator (e.g. one per-task, one
3481 // per-cgroup) so we iterate all of them.
3482 for (var_name, var_offset, var_type_id) in iter_bss_vars_with_type(prog_btf, ".bss") {
3483 // Only walk vars whose type is `struct scx_allocator`.
3484 if !is_scx_allocator_type(prog_btf, var_type_id) {
3485 continue;
3486 }
3487 // Slice the in-bss bytes for one full `struct scx_allocator`.
3488 // The size comes from BTF (resolved into `allocator_size`
3489 // by `SdtAllocOffsets::from_btf`); using the BTF-reported
3490 // size means a future field appended to scx_allocator
3491 // doesn't silently slip past the slice end.
3492 let Some(slice_end) = var_offset.checked_add(sdt_offsets.allocator_size) else {
3493 continue;
3494 };
3495 let slice = match bss_bytes.get(var_offset..slice_end) {
3496 Some(s) => s,
3497 None => continue,
3498 };
3499
3500 // Discover the payload BTF type id from the elem_size
3501 // we'd read in the walker. We do a small read here just
3502 // to drive the heuristic; the walker re-reads it.
3503 let pool_off = sdt_offsets.allocator_pool + sdt_offsets.pool_elem_size;
3504 let elem_size = if pool_off + 8 <= slice.len() {
3505 let mut buf = [0u8; 8];
3506 buf.copy_from_slice(&slice[pool_off..pool_off + 8]);
3507 u64::from_le_bytes(buf)
3508 } else {
3509 0
3510 };
3511 let payload_size =
3512 elem_size.saturating_sub(sdt_offsets.data_header_size as u64) as usize;
3513 // `prog_btf` is split BTF: the scheduler's program types
3514 // layered on the vmlinux base. `discover_payload_btf_id`
3515 // probes only the program section's id range (via
3516 // `Btf::split`), so vmlinux base `*_ctx` structs of the
3517 // same size cannot shadow the scheduler's payload struct.
3518 let choice = discover_payload_btf_id(prog_btf, payload_size, &var_name);
3519
3520 let snap = walk_sdt_allocator(
3521 accessor.kernel(),
3522 arena_kern_vm_start,
3523 slice,
3524 &sdt_offsets,
3525 prog_btf,
3526 choice.target_type_id,
3527 choice.reason.clone(),
3528 var_name.clone(),
3529 &sdt_mem,
3530 );
3531 // Accumulate every allocator with a typed payload AND
3532 // append its live slots to the bridge index. The
3533 // per-map selector (`select_sdt_alloc_meta`) picks the
3534 // right one by matching `var_name` (the .bss symbol —
3535 // e.g. `scx_task_allocator`) against each rendered map's
3536 // name (e.g. `scx_task_map`). Schedulers that declare
3537 // multiple typed allocators no longer lose payload
3538 // expansion — each map renders against the matching
3539 // allocator's payload type. Only allocators with a
3540 // resolved payload type id contribute to the bridge
3541 // index — without a typed payload there is no useful
3542 // BTF id to surface to the renderer, and the index
3543 // would just point every chase at 0 (which the bridge
3544 // gate filters as "no payload type").
3545 if choice.target_type_id != 0 {
3546 sdt_alloc_metas.push(crate::monitor::dump::render_map::SdtAllocMeta {
3547 allocator_name: var_name.clone(),
3548 elem_size,
3549 header_size: sdt_offsets.data_header_size,
3550 target_type_id: choice.target_type_id,
3551 kern_vm_start: arena_kern_vm_start,
3552 });
3553 // Append this allocator's slots to the bridge index.
3554 // The helper handles the size-fits-u32 check, the
3555 // dedup-on-duplicate-slot-start, and the
3556 // `tracing::warn!` collision diagnostic — see
3557 // [`append_arena_slot_index_for_allocator`] for the
3558 // full contract. Bridge gate (`target_type_id
3559 // != 0`) is encoded inside the helper as well; the
3560 // outer guard here is a fast-path bail before we
3561 // even allocate metadata for the allocator.
3562 crate::monitor::dump::render_map::append_arena_slot_index_for_allocator(
3563 &mut arena_slot_index,
3564 &var_name,
3565 choice.target_type_id,
3566 sdt_offsets.data_header_size,
3567 elem_size,
3568 &snap.all_slot_addrs,
3569 // Stamp the slot with the program BTF the
3570 // `target_type_id` was resolved against. The
3571 // per-map renderer's
3572 // [`MemReader::resolve_arena_type`] gate
3573 // compares this against each requesting map's
3574 // `btf_kva` and suppresses the hit on mismatch
3575 // so the BTF id cannot leak into a sibling
3576 // BTF's id space (multi-`.bpf.o` schedulers).
3577 btf_kva,
3578 );
3579 }
3580 // Surface only allocators with a non-empty result OR a
3581 // diagnostic elem_size; an all-zero snapshot from a
3582 // never-initialized allocator is just noise.
3583 if !snap.entries.is_empty() || snap.elem_size != 0 {
3584 report.sdt_allocations.push(snap);
3585 }
3586 }
3587 }
3588 // Post-loop: if the pre-pass ran but discovered nothing, the
3589 // scheduler has program BTF + arena + bss but no `struct
3590 // scx_allocator` declared in `.bss`. Surface a distinct
3591 // diagnostic so an operator can tell the schedule-doesn't-link
3592 // case from the link-but-no-instance case.
3593 if sdt_alloc_pre_pass_ran
3594 && report.sdt_allocations.is_empty()
3595 && report.sdt_alloc_unavailable.is_none()
3596 {
3597 report.sdt_alloc_unavailable = Some(REASON_SDT_ALLOC_NO_INSTANCE.to_string());
3598 }
3599 let arena_slot_index_ref = if arena_slot_index.is_empty() {
3600 None
3601 } else {
3602 Some(&arena_slot_index)
3603 };
3604 // Build the rendered-slot set for arena chase dedup. Keys on
3605 // every slot that is actually rendered under
3606 // `report.sdt_allocations` — typed AND untyped allocators both
3607 // contribute via [`SdtAllocEntry::user_addr`] (emit_leaf records
3608 // an entry on every leaf it visits, regardless of whether
3609 // payload BTF resolution succeeded). Keying on
3610 // `arena_slot_index.keys()` alone would have missed slots from
3611 // allocators whose `target_type_id == 0` — those allocators
3612 // never reach `append_arena_slot_index_for_allocator` (the
3613 // helper short-circuits on a zero target id) yet their slots
3614 // ARE rendered in `report.sdt_allocations`.
3615 //
3616 // Two address keys per entry: the slot start (`user_addr`) and
3617 // the payload start (`user_addr + header_size`). Chase targets
3618 // resolved via `scx_task_data(p)` and similar helpers point at
3619 // the payload, not the slot start; without the payload-start
3620 // key the dedup misses every chase that uses the helper-
3621 // computed pointer.
3622 //
3623 // Slots past [`super::sdt_alloc::MAX_SDT_ALLOC_ENTRIES`] are
3624 // intentionally excluded — they are NOT rendered in
3625 // `snap.entries`, so the per-map renderer must surface their
3626 // payload (otherwise the truncated tail would appear nowhere
3627 // in the dump). The walker's `truncated` flag is the operator-
3628 // visible signal that some slots are only available via the
3629 // per-map render path.
3630 //
3631 // The per-map renderer's [`MemReader::is_already_rendered`]
3632 // consults this set to skip re-rendering the same allocation
3633 // when a TASK_STORAGE / HASH map's value pointer chases back
3634 // into it. Only TYPED allocators (target_type_id != 0) enter
3635 // the set — untyped pre-pass renders (hex fallback) must not
3636 // suppress per-map chases because the cast analyzer's shape
3637 // inference may resolve a concrete target type the heuristic
3638 // missed.
3639 let header_size_by_allocator: std::collections::HashMap<&str, usize> = sdt_alloc_metas
3640 .iter()
3641 .map(|meta| (meta.allocator_name.as_str(), meta.header_size))
3642 .collect();
3643 let rendered_slot_addrs: std::collections::HashSet<u32> = report
3644 .sdt_allocations
3645 .iter()
3646 .filter_map(|snap| {
3647 // Only dedup slots from allocators with a resolved
3648 // payload type. Untyped allocators (target_type_id == 0,
3649 // rendered as hex in the pre-pass) must NOT suppress
3650 // per-map chases — the cast analyzer's shape inference
3651 // may resolve a concrete target type that the pre-pass
3652 // heuristic missed.
3653 let &header_size = header_size_by_allocator.get(snap.allocator_name.as_str())?;
3654 Some(snap.entries.iter().flat_map(move |e| {
3655 let slot_start = e.user_addr as u32;
3656 let payload_start = slot_start.wrapping_add(header_size as u32);
3657 [slot_start, payload_start]
3658 }))
3659 })
3660 .flatten()
3661 .collect();
3662 let rendered_slot_addrs_ref = if rendered_slot_addrs.is_empty() {
3663 None
3664 } else {
3665 Some(&rendered_slot_addrs)
3666 };
3667 tracing::debug!(
3668 elapsed_us = sdt_alloc_t0.elapsed().as_micros() as u64,
3669 allocations = report.sdt_allocations.len(),
3670 index_entries = arena_slot_index.len(),
3671 "dump_state phase: sdt_alloc"
3672 );
3673
3674 // Pre-pass: walk every `scx_static` bump-allocator instance in
3675 // `.bss` and surface its live-allocated range. Distinct from the
3676 // sdt_alloc per-instance allocator walk above:
3677 //
3678 // - sdt_alloc (`struct scx_allocator`) hands out fixed-stride
3679 // slots via a 3-level radix tree with per-slot metadata; the
3680 // walker produces one entry per live slot keyed on slot start.
3681 // - scx_static (`struct scx_static`) is a flat bump allocator
3682 // with no per-slot metadata; the walker produces one entry
3683 // per live region keyed on the region's base address.
3684 //
3685 // The walk runs only when:
3686 // - we have a scheduler `.bss` blob to read from (re-located
3687 // here because the sdt_alloc walk above consumed the
3688 // pre-pass `sched_bss_bytes` Option),
3689 // - we have a program BTF to resolve `struct scx_static`
3690 // against,
3691 // - the program BTF carries `struct scx_static`.
3692 //
3693 // When any prerequisite is missing, the walk leaves
3694 // `report.scx_static_ranges` empty (default) rather than failing
3695 // the dump — schedulers that don't link `lib/sdt_alloc.bpf.c` or
3696 // don't use the static allocator simply skip the walk.
3697 //
3698 // Membership-only: the walker produces an UNTYPED range index.
3699 // Per-allocation type recovery requires a per-call-site type
3700 // hook from cast analysis that does not exist today (see the
3701 // module-level doc in [`crate::monitor::scx_static_alloc`] for
3702 // why). When the renderer's deferred-resolve arena chase lands
3703 // on an address inside an scx_static range, the bridge
3704 // recognises the address as "in scx_static memory" and
3705 // fails closed (returns `None` from `resolve_arena_type`)
3706 // rather than returning a wrong type — the "no invalid data
3707 // made" contract.
3708 let scx_static_t0 = std::time::Instant::now();
3709 if !deadline_exceeded(&mut truncated_at_us)
3710 && let Some((bss_bytes, prog_btf)) = relocate_sched_bss(&maps, accessor, &program_btfs)
3711 && let Ok(scx_static_offsets) =
3712 crate::monitor::scx_static_alloc::ScxStaticOffsets::from_btf(prog_btf)
3713 {
3714 let snap = crate::monitor::scx_static_alloc::walk_scx_static(
3715 &bss_bytes,
3716 &scx_static_offsets,
3717 iter_bss_vars_with_type(prog_btf, ".bss"),
3718 |type_id| is_scx_static_type(prog_btf, type_id),
3719 );
3720 report.scx_static_ranges = snap;
3721 }
3722 let scx_static_index =
3723 crate::monitor::scx_static_alloc::build_scx_static_range_index(&report.scx_static_ranges);
3724 let scx_static_index_ref = if scx_static_index.is_empty() {
3725 None
3726 } else {
3727 Some(&scx_static_index)
3728 };
3729 tracing::debug!(
3730 elapsed_us = scx_static_t0.elapsed().as_micros() as u64,
3731 ranges = report.scx_static_ranges.ranges.len(),
3732 skipped = report.scx_static_ranges.skipped,
3733 index_entries = scx_static_index.len(),
3734 "dump_state phase: scx_static"
3735 );
3736
3737 let render_map_t0 = std::time::Instant::now();
3738 let mut maps_rendered: usize = 0;
3739 let mut maps_truncated: usize = 0;
3740 for info in maps {
3741 // Skip ktstr's own framework maps so the report only shows
3742 // the scheduler-under-test's state. Three distinct shapes
3743 // need filtering:
3744 //
3745 // 1. Global-section maps from the probe skeleton: libbpf
3746 // composes `<obj_name>.<section>` so `probe_bp.bss`,
3747 // `probe_bp.data`, `probe_bp.rodata` all match the
3748 // `probe_bp.` prefix. (`probe_bp` matching the bare obj
3749 // name covers any single-name section the kernel might
3750 // surface, though libbpf today always adds the suffix.)
3751 // 2. Global-section maps from the fentry skeleton, named
3752 // with the `fentry_p.` prefix following the same
3753 // libbpf convention.
3754 // 3. Bare-named maps declared via `SEC(".maps")` in
3755 // src/bpf/probe.bpf.c — these don't get an obj prefix
3756 // because they're not from a global section. The
3757 // explicit denylist [`KTSTR_INTERNAL_MAPS`] enumerates
3758 // them.
3759 //
3760 // A future tighter filter would consult bpf_prog ownership
3761 // (the program-attachment ID list pinned to each map), but
3762 // name-based filtering is enough today and avoids loading
3763 // the full prog_idr walk on the freeze hot path.
3764 {
3765 let info_name = info.name();
3766 if info_name.starts_with("probe_bp.")
3767 || info_name.starts_with("fentry_p.")
3768 || info_name == "probe_bp"
3769 || info_name == "fentry_p"
3770 || KTSTR_INTERNAL_MAPS.contains(&info_name.as_ref())
3771 {
3772 continue;
3773 }
3774 }
3775
3776 // Deadline check before each map render — bigger maps
3777 // (large hashes, arenas) can each take a meaningful slice
3778 // of the freeze window, so we re-check between renders to
3779 // bound the worst case rather than letting one
3780 // straggler push us past the watchdog.
3781 if deadline_exceeded(&mut truncated_at_us) {
3782 maps_truncated += 1;
3783 continue;
3784 }
3785
3786 // Per-map BTF resolution.
3787 //
3788 // The map's `btf_value_type_id` / `btf_key_type_id` index
3789 // the *map's own* BTF, NOT the kernel vmlinux BTF — when
3790 // `btf_kva != 0` the type IDs are program-local and using
3791 // vmlinux BTF with them would resolve to unrelated kernel
3792 // types (or out-of-range nonsense). So:
3793 //
3794 // - `BPF_MAP_TYPE_STRUCT_OPS` → use vmlinux
3795 // BTF. The wrapper struct `bpf_struct_ops_<name>` is
3796 // declared in the kernel's vmlinux BTF and the
3797 // wrapper type id stored on the map (in
3798 // `btf_vmlinux_value_type_id`) indexes vmlinux. Using
3799 // the program BTF here would fail to resolve the
3800 // wrapper.
3801 // - `btf_kva != 0` AND program BTF loaded by pre-pass → use it.
3802 // - `btf_kva != 0` AND program BTF load failed in pre-pass
3803 // → render hex-only (None map_btf), no fallback.
3804 // - `btf_kva == 0` (kernel-builtin map) → use the
3805 // caller-supplied vmlinux BTF; the type IDs (if any)
3806 // genuinely index vmlinux BTF in this case.
3807 let map_btf: Option<&Btf> = if info.map_type == super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS {
3808 Some(btf)
3809 } else if info.btf_kva != 0 {
3810 program_btfs.get(&info.btf_kva)
3811 } else {
3812 Some(btf)
3813 };
3814
3815 let rendered = render_map(
3816 &RenderMapCtx {
3817 accessor,
3818 btf: map_btf,
3819 num_cpus,
3820 arena_offsets,
3821 shared_arena: shared_arena_ref,
3822 arena_page_index: &arena_page_index,
3823 sdt_alloc_metas: &sdt_alloc_metas,
3824 // Threaded in from
3825 // [`DumpContext::cast_map`]: the BPF
3826 // cast-analysis output for the scheduler's
3827 // program object. `Some(&map)` lets the
3828 // renderer promote `u64` fields the analyzer
3829 // flagged into typed-pointer renders via
3830 // [`super::btf_render::MemReader::cast_lookup`];
3831 // `None` keeps every `u64` rendered as a plain
3832 // unsigned counter (the trait default).
3833 cast_map,
3834 // Built from the sdt_alloc pre-pass above:
3835 // `slot_start → ArenaSlotInfo` for every live
3836 // allocator slot. Lets the renderer range-lookup
3837 // the slot a chased arena address falls in and
3838 // recover a `BTF_KIND_FWD` pointee's real
3839 // struct id (plus a `header_skip` byte count)
3840 // via [`MemReader::resolve_arena_type`] — a
3841 // `struct sdt_data __arena *` field (or a `data`
3842 // field caching the raw `sdt_alloc()` return)
3843 // whose pointee body lives in the sdt_alloc
3844 // library's BTF still chases as the typed
3845 // per-task / per-cgroup struct, instead of
3846 // skipping with "forward declaration; body not
3847 // in this BTF". `None` when no allocator with a
3848 // typed payload was discovered.
3849 arena_slot_index: arena_slot_index_ref,
3850 // Threaded in from
3851 // [`DumpContext::cross_btf_fwd_index`]: the
3852 // cross-BTF Fwd resolution context populated by
3853 // the cast-analysis pre-pass over every embedded
3854 // BPF object's BTF. `Some(&idx)` lets the
3855 // renderer chase a `BTF_KIND_FWD` whose body
3856 // lives in a sibling embedded object via
3857 // [`MemReader::cross_btf_resolve_fwd`]. `None`
3858 // keeps the renderer's "forward declaration;
3859 // body not in this BTF" skip path intact.
3860 cross_btf_fwd_index: cross_btf_fwd_index_ref,
3861 // Built from the scx_static pre-pass above:
3862 // `start_low32 → size` for every live
3863 // `scx_static` bump-allocator region. Lets the
3864 // renderer's
3865 // [`MemReader::resolve_arena_type`]
3866 // diagnose-and-skip on a chased arena address
3867 // that lands inside scx_static memory — the
3868 // bridge cannot recover a per-allocation type
3869 // (no per-slot header) so the chase falls through
3870 // to the historical Fwd-skip / cross-BTF
3871 // resolution path. `None` when no live
3872 // `scx_static` instance was discovered.
3873 scx_static_index: scx_static_index_ref,
3874 // Built from the sdt_alloc pre-pass above: the
3875 // low-32 windowed `slot_start` of every allocator
3876 // slot already rendered into
3877 // `report.sdt_allocations`. Lets the renderer's
3878 // [`MemReader::is_already_rendered`] short-circuit
3879 // the arena chase when a TASK_STORAGE / HASH
3880 // map's value pointer lands in a slot the
3881 // typed-allocator surface already shows — no
3882 // duplicate payload in the dump. `None` when no
3883 // allocator pre-pass produced any rendered slot.
3884 rendered_slot_addrs: rendered_slot_addrs_ref,
3885 alloc_size_types,
3886 },
3887 &info,
3888 );
3889
3890 report.maps.push(rendered);
3891 maps_rendered += 1;
3892 }
3893 tracing::debug!(
3894 elapsed_us = render_map_t0.elapsed().as_micros() as u64,
3895 rendered = maps_rendered,
3896 truncated = maps_truncated,
3897 "dump_state phase: per-map render"
3898 );
3899
3900 report.dump_truncated_at_us = truncated_at_us;
3901 report.maps_truncated = maps_truncated as u32;
3902 report
3903}
3904
3905/// Walk every CPU's `kernel_cpustat`, `kernel_stat`, and (under
3906/// NO_HZ) `tick_sched` slots and produce a [`PerCpuTimeStats`]
3907/// vector — one entry per CPU index in `cap.per_cpu_offsets`.
3908///
3909/// Reads against the supplied [`super::reader::GuestMem`]. Each CPU's
3910/// per-CPU base for a symbol `S` is `S + per_cpu_offsets[cpu]`,
3911/// converted to a guest physical offset via
3912/// [`super::symbols::kva_to_pa`] using the supplied `page_offset`
3913/// (the standard direct-mapping translation; per-CPU pages always
3914/// live in the direct mapping).
3915///
3916/// `cpustat[]` is read as 8 contiguous u64s starting at the
3917/// resolved offset (length matches the indices captured —
3918/// CPUTIME_USER through CPUTIME_STEAL — leaving CPUTIME_GUEST /
3919/// CPUTIME_GUEST_NICE / CPUTIME_FORCEIDLE unread; the dump
3920/// surfaces them as zero in the unread slots, which is acceptable
3921/// since they're virt-guest specific or kernel-config gated and
3922/// distinct from the failure-dump narrative). `softirqs[]` reads
3923/// as `NR_SOFTIRQS` u32s, widened to u64 for the report. `irqs_sum`
3924/// is `unsigned long` (read as u64 — 64-bit only kernels are the
3925/// supported configuration). `iowait_sleeptime` is `ktime_t` /
3926/// `s64`; the value is cast to u64 (the kernel never produces
3927/// negative iowait time).
3928fn collect_per_cpu_time(cap: &CpuTimeCapture<'_>) -> Vec<PerCpuTimeStats> {
3929 use super::btf_offsets::{
3930 CPUTIME_IDLE, CPUTIME_IOWAIT, CPUTIME_IRQ, CPUTIME_NICE, CPUTIME_SOFTIRQ, CPUTIME_STEAL,
3931 CPUTIME_SYSTEM, CPUTIME_USER, NR_SOFTIRQS,
3932 };
3933 let mut out = Vec::with_capacity(cap.per_cpu_offsets.len());
3934 for (cpu_idx, &per_cpu_off) in cap.per_cpu_offsets.iter().enumerate() {
3935 let cpu = cpu_idx as u32;
3936
3937 // kernel_cpustat::cpustat[N]: each slot is a u64 in nsec.
3938 // Read CPUTIME_USER through CPUTIME_STEAL (indices 0..=7).
3939 let cpustat_kva =
3940 super::symbols::per_cpu_kva(cap.kernel_cpustat_kva, cap.kaslr_offset, per_cpu_off);
3941 let cpustat_pa = super::symbols::kva_to_pa(cpustat_kva, cap.page_offset);
3942 let cpustat_base = cap.offsets.kernel_cpustat_cpustat;
3943 let read_cpustat = |idx: usize| -> u64 {
3944 // sizeof(u64) == 8.
3945 cap.mem.read_u64(cpustat_pa, cpustat_base + idx * 8)
3946 };
3947 let cpustat_user_ns = read_cpustat(CPUTIME_USER);
3948 let cpustat_nice_ns = read_cpustat(CPUTIME_NICE);
3949 let cpustat_system_ns = read_cpustat(CPUTIME_SYSTEM);
3950 let cpustat_softirq_ns = read_cpustat(CPUTIME_SOFTIRQ);
3951 let cpustat_irq_ns = read_cpustat(CPUTIME_IRQ);
3952 let cpustat_idle_ns = read_cpustat(CPUTIME_IDLE);
3953 let cpustat_iowait_ns = read_cpustat(CPUTIME_IOWAIT);
3954 let cpustat_steal_ns = read_cpustat(CPUTIME_STEAL);
3955
3956 // kernel_stat::softirqs[N]: each slot is a u32 (count).
3957 // Widen to u64 for reporting consistency with cpustat.
3958 let kstat_kva = super::symbols::per_cpu_kva(cap.kstat_kva, cap.kaslr_offset, per_cpu_off);
3959 let kstat_pa = super::symbols::kva_to_pa(kstat_kva, cap.page_offset);
3960 let mut softirqs = [0u64; NR_SOFTIRQS];
3961 for (i, slot) in softirqs.iter_mut().enumerate() {
3962 // sizeof(unsigned int) == 4.
3963 *slot = cap
3964 .mem
3965 .read_u32(kstat_pa, cap.offsets.kstat_softirqs + i * 4) as u64;
3966 }
3967
3968 // kernel_stat::irqs_sum: unsigned long. 64-bit only
3969 // kernels are supported, so read as u64.
3970 let irqs_sum = cap.mem.read_u64(kstat_pa, cap.offsets.kstat_irqs_sum);
3971
3972 // tick_sched::iowait_sleeptime: ktime_t (s64) ns,
3973 // accumulated only under NO_HZ when the CPU enters idle
3974 // with nr_iowait > 0. Skip when the symbol or BTF offset
3975 // is absent.
3976 let iowait_sleeptime_ns = cap
3977 .tick_cpu_sched_kva
3978 .zip(cap.offsets.tick_sched_iowait_sleeptime)
3979 .map(|(tick_sym_kva, off)| {
3980 let kva = super::symbols::per_cpu_kva(tick_sym_kva, cap.kaslr_offset, per_cpu_off);
3981 let pa = super::symbols::kva_to_pa(kva, cap.page_offset);
3982 cap.mem.read_u64(pa, off)
3983 });
3984
3985 out.push(PerCpuTimeStats {
3986 cpu,
3987 cpustat_user_ns,
3988 cpustat_nice_ns,
3989 cpustat_system_ns,
3990 cpustat_softirq_ns,
3991 cpustat_irq_ns,
3992 cpustat_idle_ns,
3993 cpustat_iowait_ns,
3994 cpustat_steal_ns,
3995 softirqs,
3996 irqs_sum,
3997 iowait_sleeptime_ns,
3998 });
3999 }
4000 out
4001}
4002
4003/// Walk the test's workload cgroups and read each leaf's PSI_IRQ_FULL (Phase A).
4004/// Thin adapter over
4005/// [`super::cgroup_walk::collect_workload_cgroup_psi`] that unpacks the
4006/// borrowed [`CgroupPsiCapture`]. Empty when no workload leaf has per-cgroup
4007/// PSI accounting (loud-absent).
4008fn collect_cgroup_psi(cap: &CgroupPsiCapture<'_>) -> Vec<super::cgroup_walk::CgroupPsiStat> {
4009 super::cgroup_walk::collect_workload_cgroup_psi(
4010 cap.mem,
4011 cap.cgroup_offsets,
4012 cap.psi_offsets,
4013 cap.root_cgroup_kva,
4014 cap.root_cgroup_pa,
4015 cap.workload_root_path,
4016 cap.page_offset,
4017 )
4018}
4019
4020/// Walk a Datasec section by name, yielding `(var_name, byte_offset,
4021/// type_id)` for every variable declared in it.
4022///
4023/// Used by [`dump_state`] to enumerate `.bss` variables when looking
4024/// for `scx_allocator` instances. Returns an empty iterator when the
4025/// Datasec doesn't exist or any chained Var resolution fails — the
4026/// caller treats that as "no sdt_alloc state to surface" rather than
4027/// a hard error.
4028fn iter_bss_vars_with_type(btf: &Btf, section_name: &str) -> Vec<(String, usize, u32)> {
4029 use btf_rs::BtfType;
4030 let mut out = Vec::new();
4031 let Ok(candidates) = btf.resolve_types_by_name(section_name) else {
4032 return out;
4033 };
4034 for ty in candidates {
4035 let btf_rs::Type::Datasec(ds) = ty else {
4036 continue;
4037 };
4038 for var_info in &ds.variables {
4039 let Ok(chained) = btf.resolve_chained_type(var_info) else {
4040 continue;
4041 };
4042 let btf_rs::Type::Var(var) = chained else {
4043 continue;
4044 };
4045 let Ok(name) = btf.resolve_name(&var) else {
4046 continue;
4047 };
4048 // The Var's type_id points to the variable's actual
4049 // type (e.g. struct scx_allocator). var_info.offset() is
4050 // the byte offset within the Datasec.
4051 let Some(type_id) = var.get_type_id() else {
4052 continue;
4053 };
4054 out.push((name, var_info.offset() as usize, type_id));
4055 }
4056 }
4057 out
4058}
4059
4060/// True iff `type_id` resolves to a struct named `scx_allocator`,
4061/// stripping the BTF modifier chain en route. The five modifier
4062/// kinds the loop unwraps — `Const`, `Volatile`, `Typedef`,
4063/// `Restrict`, `TypeTag` — are the complete set the kernel BPF
4064/// pipeline emits for global variable types in `.bss`. Any other
4065/// kind in the chain (Ptr, Array, etc.) terminates the lookup with
4066/// a non-match.
4067fn is_scx_allocator_type(btf: &Btf, type_id: u32) -> bool {
4068 use btf_rs::Type as T;
4069 // Mirror the modifier-chain pattern in
4070 // `btf_offsets::resolve_member_composite` — resolve the
4071 // chained type via the BtfType trait object so the type
4072 // aliases (Const = Volatile, TypeTag = Typedef) all share the
4073 // same path through the loop.
4074 let Ok(mut t) = btf.resolve_type_by_id(type_id) else {
4075 return false;
4076 };
4077 for _ in 0..20 {
4078 match t {
4079 T::Struct(s) => {
4080 return btf.resolve_name(&s).is_ok_and(|n| n == "scx_allocator");
4081 }
4082 T::Const(_) | T::Volatile(_) | T::Typedef(_) | T::Restrict(_) | T::TypeTag(_) => {
4083 let Some(btf_ty) = t.as_btf_type() else {
4084 return false;
4085 };
4086 let Ok(next) = btf.resolve_chained_type(btf_ty) else {
4087 return false;
4088 };
4089 t = next;
4090 }
4091 _ => return false,
4092 }
4093 }
4094 false
4095}
4096
4097/// True iff `type_id` resolves to a struct named `scx_static`,
4098/// stripping the BTF modifier chain en route. Mirrors the
4099/// modifier-handling shape of [`is_scx_allocator_type`] — the five
4100/// modifier kinds (`Const`, `Volatile`, `Typedef`, `Restrict`,
4101/// `TypeTag`) are the complete set the kernel BPF pipeline emits for
4102/// global variable types in `.bss`; any other kind terminates the
4103/// lookup with a non-match.
4104///
4105/// Used by the [`crate::monitor::scx_static_alloc::walk_scx_static`]
4106/// pre-pass to filter `.bss` Vars to only `struct scx_static`
4107/// instances. A scheduler that doesn't link `lib/sdt_alloc.bpf.c`
4108/// has no such Var; the filter rejects every candidate and the
4109/// walker produces an empty snapshot.
4110fn is_scx_static_type(btf: &Btf, type_id: u32) -> bool {
4111 use btf_rs::Type as T;
4112 let Ok(mut t) = btf.resolve_type_by_id(type_id) else {
4113 return false;
4114 };
4115 for _ in 0..20 {
4116 match t {
4117 T::Struct(s) => {
4118 return btf.resolve_name(&s).is_ok_and(|n| n == "scx_static");
4119 }
4120 T::Const(_) | T::Volatile(_) | T::Typedef(_) | T::Restrict(_) | T::TypeTag(_) => {
4121 let Some(btf_ty) = t.as_btf_type() else {
4122 return false;
4123 };
4124 let Ok(next) = btf.resolve_chained_type(btf_ty) else {
4125 return false;
4126 };
4127 t = next;
4128 }
4129 _ => return false,
4130 }
4131 }
4132 false
4133}
4134
4135/// Locate the scheduler's `.bss` array map and pull out (raw bytes,
4136/// program BTF) for the [`crate::monitor::scx_static_alloc`] pre-pass.
4137///
4138/// The earlier sdt_alloc pre-pass at the top of [`dump_state`]
4139/// already collected `sched_bss_bytes` once but the if-let chain at
4140/// the sdt_alloc walk consumed the `Option`. Re-locating here keeps
4141/// both walkers independent: each owns its own bss-bytes read so a
4142/// future ordering change can't accidentally leave one walker
4143/// without input. The cost is one extra map walk; small compared to
4144/// the per-map render loop that follows.
4145///
4146/// Returns `None` when:
4147/// - no `*.bss` map exists (libbpf only creates this map when the
4148/// program has any global non-const data),
4149/// - the map's `btf_kva == 0` (no program BTF — type resolution
4150/// would fail),
4151/// - the program BTF for that `btf_kva` was not loaded in the
4152/// pre-pass (parse failed earlier; the caller already logged),
4153/// - the map's value bytes can't be read.
4154fn relocate_sched_bss<'btf>(
4155 maps: &[BpfMapInfo],
4156 accessor: &GuestMemMapAccessor<'_>,
4157 program_btfs: &'btf std::collections::HashMap<u64, Btf>,
4158) -> Option<(Vec<u8>, &'btf Btf)> {
4159 for info in maps {
4160 let name = info.name();
4161 if name.starts_with("probe_bp.")
4162 || name.starts_with("fentry_p.")
4163 || name == "probe_bp"
4164 || name == "fentry_p"
4165 || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
4166 {
4167 continue;
4168 }
4169 if info.map_type == BPF_MAP_TYPE_ARRAY
4170 && info.btf_kva != 0
4171 && name.ends_with(".bss")
4172 && let Some(prog_btf) = program_btfs.get(&info.btf_kva)
4173 && let Some(bytes) = accessor.read_value(info, 0, info.value_size as usize)
4174 {
4175 return Some((bytes, prog_btf));
4176 }
4177 }
4178 None
4179}
4180
4181/// Load a BPF program's `struct btf` from guest memory at `btf_kva`.
4182///
4183/// Reads the kernel `struct btf` at `btf_kva`, follows its `data` /
4184/// `data_size` / `base_btf` fields, fetches the raw BTF blob via
4185/// page-walked vmalloc reads, and parses it. When `base_btf` is
4186/// non-NULL the program's BTF is split atop the vmlinux BTF (the
4187/// kernel's own base BTF) — pass the host's already-parsed vmlinux
4188/// `Btf` as the split base so type IDs resolve correctly.
4189///
4190/// Returns `None` when any step fails: missing offsets, untranslatable
4191/// pages, or `Btf::from_bytes` rejection (truncated / corrupted blob).
4192/// Failure is silent and the caller falls back to the host vmlinux
4193/// BTF — the dump is best-effort, a partial render still beats no
4194/// render.
4195///
4196/// Distinct from the [`super::bpf_map::BpfMapAccessor::load_program_btf`]
4197/// trait method (which dispatches across backends): this free function
4198/// is the guest-memory backend's actual KVA-based loader. The trait
4199/// method on `GuestMemMapAccessor` just forwards here.
4200pub(super) fn load_program_btf_kva(
4201 accessor: &GuestMemMapAccessor<'_>,
4202 btf_kva: u64,
4203 base_btf: &Btf,
4204) -> Option<Btf> {
4205 let kernel = accessor.kernel();
4206 let offsets = accessor.offsets();
4207 let mem = kernel.mem();
4208 let walk = kernel.walk_context();
4209
4210 // `struct btf` may be kmalloc'd (direct map) or vmalloc'd; use
4211 // translate_any_kva.
4212 let btf_pa = super::idr::translate_any_kva(
4213 mem,
4214 walk.cr3_pa,
4215 walk.page_offset,
4216 btf_kva,
4217 walk.l5,
4218 walk.tcr_el1,
4219 )?;
4220 let data_kva = mem.read_u64(btf_pa, offsets.btf_data);
4221 let data_size = mem.read_u32(btf_pa, offsets.btf_data_size) as usize;
4222 let base_kva = mem.read_u64(btf_pa, offsets.btf_base_btf);
4223
4224 if data_kva == 0 || data_size == 0 {
4225 return None;
4226 }
4227
4228 if data_size > MAX_BTF_BLOB {
4229 return None;
4230 }
4231
4232 // The BTF blob is vmalloc-backed — `btf->data` is allocated via
4233 // vmalloc / kvmalloc inside `kernel/bpf/btf.c`'s
4234 // `btf_parse_*` paths. Use the chunked vmalloc reader so a
4235 // 100 KB blob doesn't pay 100K syscalls of byte-wise translate.
4236 // The chunked reader honours all-or-nothing semantics, so a
4237 // short read returns None directly; no extra length check needed.
4238 let blob = kernel.read_kva_bytes_chunked(data_kva, data_size)?;
4239
4240 if base_kva != 0 {
4241 // Split BTF: the program's types extend the kernel's
4242 // vmlinux BTF. Pass the host's parsed vmlinux Btf as the
4243 // base so cross-base type IDs (e.g. `task_struct`) resolve.
4244 //
4245 // Uses host vmlinux BTF as split base — correct when host
4246 // kernel == guest kernel (ktstr's default and the common
4247 // CI configuration). A guest running a different kernel
4248 // version would silently mis-render cross-base type
4249 // references; flagged as a known limitation in the module
4250 // doc above.
4251 Btf::from_split_bytes(&blob, base_btf).ok()
4252 } else {
4253 Btf::from_bytes(&blob).ok()
4254 }
4255}
4256
4257/// Render a byte slice as space-separated hex pairs.
4258///
4259/// `pub(crate)` so [`super::sdt_alloc`] can reuse the same wire shape
4260/// for its hex-fallback payload renderings — keeps the dump's hex
4261/// output consistent across both renderers.
4262pub(crate) fn hex_dump(bytes: &[u8]) -> String {
4263 use std::fmt::Write;
4264 let mut s = String::with_capacity(bytes.len() * 3);
4265 for (i, b) in bytes.iter().enumerate() {
4266 if i > 0 {
4267 s.push(' ');
4268 }
4269 // unwrap is safe: write! to String never fails.
4270 let _ = write!(s, "{b:02x}");
4271 }
4272 s
4273}