ktstr/monitor/dump/
mod.rs

1//! BPF map state dump for scheduler-failure post-mortem.
2//!
3//! [`dump_state`] is invoked by the freeze coordinator after the vCPU
4//! rendezvous succeeds (see `src/vmm/freeze_coord/mod.rs`). It enumerates every
5//! BPF map in the guest via [`BpfMapAccessor::maps`], filters out
6//! ktstr-internal probes (the framework's own probe and fentry skel
7//! maps), and dispatches per map type:
8//!
9//! - `BPF_MAP_TYPE_ARRAY` (and the `.bss` / `.data` / `.rodata`
10//!   global-section maps libbpf creates as single-key arrays) — read
11//!   the whole value buffer and render it via [`super::btf_render::render_value_with_mem`]
12//!   so embedded `__arena` pointers chase into the captured arena pages.
13//! - `BPF_MAP_TYPE_HASH` — iterate (key, value) pairs, capped at
14//!   [`MAX_HASH_ENTRIES`].
15//! - `BPF_MAP_TYPE_PERCPU_ARRAY` — read each CPU's slot for keys
16//!   `0..min(max_entries, MAX_PERCPU_KEYS)`.
17//! - Other types — recorded as [`FailureDumpMap::error`] so the operator
18//!   sees the gap rather than a silent omission.
19//!
20//! # BTF source — per-map program BTF loading
21//!
22//! The renderer loads each map's program BTF from guest memory at
23//! [`BpfMapInfo::btf_kva`], following the kernel `struct btf`'s
24//! `data`/`data_size`/`base_btf` fields. Split BTF (program types
25//! extending vmlinux) is parsed via [`Btf::from_split_bytes`] with
26//! the host's vmlinux BTF as the base (correct when host kernel ==
27//! guest kernel — ktstr's default and the common CI configuration).
28//! A per-`btf_kva` cache dedupes parses across maps sharing a
29//! program's BTF object. When per-map load fails (still-booting
30//! guest, untranslatable page, corrupted blob), the renderer falls
31//! back to the caller-supplied vmlinux BTF.
32//!
33//! # sdt_alloc pre-pass
34//!
35//! Before the per-map walk runs, [`dump_state`] runs a pre-pass
36//! that locates `sdt_alloc`-backed allocator instances inside the
37//! scheduler's `.bss` and surfaces every live per-task / per-cgroup
38//! allocation as structured records under
39//! [`FailureDumpReport::sdt_allocations`]. The walk runs only when
40//! every prerequisite is present:
41//!   - the per-map dump deadline has not been exceeded (the
42//!     pre-pass runs before every map render, and an earlier
43//!     phase exhausting the budget skips the walk to keep the
44//!     dump bounded),
45//!   - the arena's `user_vm_start` is 4 GiB-aligned (low 32 bits
46//!     zero — the bridge's address arithmetic treats slot starts as
47//!     low-32 keys against a `[user_vm_start, user_vm_start + 4 GiB)`
48//!     window; misalignment would silently mismap chases),
49//!   - the scheduler exposes a `.bss` ARRAY map with non-zero
50//!     `btf_kva` (so we can read its raw bytes and have a program
51//!     BTF to resolve types against),
52//!   - at least one `BPF_MAP_TYPE_ARENA` map snapshot succeeded
53//!     (so we have `kern_vm_start` for arena pointer translation),
54//!   - the program BTF carries `struct scx_allocator` (the scheduler
55//!     links `lib/sdt_alloc.bpf.c`).
56//!
57//! When any prerequisite is missing, the pre-pass leaves
58//! `sdt_allocations` empty rather than failing the dump — the
59//! per-map page-granular [`super::arena::ArenaSnapshot`] still
60//! captures raw arena content for callers that don't need
61//! structured rendering. See [`super::sdt_alloc`] for the walker
62//! design.
63
64mod display;
65mod render_map;
66#[cfg(test)]
67mod tests;
68use render_map::*;
69
70use serde::{Deserialize, Serialize};
71
72use btf_rs::Btf;
73
74use super::arena::{ArenaSnapshot, BpfArenaOffsets, snapshot_arena};
75use super::bpf_map::{
76    BPF_MAP_TYPE_ARENA, BPF_MAP_TYPE_ARRAY, BpfMapAccessor, BpfMapInfo, GuestMemMapAccessor,
77};
78use super::btf_render::RenderedValue;
79use super::sdt_alloc::{
80    SdtAllocOffsets, SdtAllocatorSnapshot, discover_payload_btf_id, walk_sdt_allocator,
81};
82
83/// Borrow-only capture context for per-program runtime stats
84/// (cnt/nsecs/misses) populated alongside the BPF map dump.
85///
86/// Carries a borrowed [`super::bpf_prog::BpfProgAccessor`] plus the
87/// per-CPU offset array obtained from
88/// [`super::symbols::read_per_cpu_offsets`]. [`dump_state`] calls
89/// [`super::bpf_prog::BpfProgAccessor::struct_ops_runtime_stats`]
90/// with the supplied offsets and stores the resulting
91/// [`super::bpf_prog::ProgRuntimeStats`] vector in
92/// [`FailureDumpReport::prog_runtime_stats`].
93///
94/// Pass `None` to skip prog-runtime capture (e.g. when the
95/// accessor could not be constructed because `prog_idr` is
96/// missing or the BPF prog offsets did not resolve). The dump still
97/// renders every map the [`super::bpf_map::BpfMapAccessor`] enumerates.
98pub struct ProgRuntimeCapture<'a> {
99    /// Accessor for walking `prog_idr` and reading per-program
100    /// `bpf_prog_stats` slots. Trait dispatch lets the same dump
101    /// site consume either the guest-memory backend or the planned
102    /// live-host backend without committing to a concrete type.
103    pub accessor: &'a dyn super::bpf_prog::BpfProgAccessor,
104    /// Per-CPU offset array (`__per_cpu_offset[cpu]`) used to address
105    /// each CPU's `bpf_prog_stats` slot for summation.
106    pub per_cpu_offsets: &'a [u64],
107}
108
109/// Borrow-only capture context for per-CPU CPU-time / softirq / IRQ
110/// counters populated alongside the BPF map dump.
111///
112/// Carries the BTF-resolved field offsets for `kernel_cpustat`,
113/// `kernel_stat`, and `tick_sched`, the resolved `.data..percpu`
114/// section offsets of the three per-CPU symbols, and the
115/// `__per_cpu_offset[cpu]` array used to address each CPU's slot.
116///
117/// [`dump_state`] reads each CPU's slot via direct guest-memory
118/// reads against the supplied [`super::reader::GuestMem`] and
119/// records the result into [`FailureDumpReport::per_cpu_time`].
120/// Mirrors [`ProgRuntimeCapture`]'s "borrowed-only, optional"
121/// shape — when `None`, the dump skips the per-CPU time capture
122/// and leaves the field empty.
123///
124/// Skipped silently when the resolver could not locate any of the
125/// three per-CPU symbols (stripped vmlinux), the BTF offsets are
126/// not present (CPU-time accounting types missing), or
127/// `__per_cpu_offset` resolution returned an empty array. The
128/// capture is best-effort diagnostic data; its absence does not
129/// fail the dump.
130pub struct CpuTimeCapture<'a> {
131    /// Guest memory handle used to read each per-CPU slot.
132    pub mem: &'a super::reader::GuestMem,
133    /// BTF-resolved offsets for `kernel_cpustat::cpustat[]`,
134    /// `kernel_stat::softirqs[]`, `kernel_stat::irqs_sum`, and
135    /// optionally `tick_sched::iowait_sleeptime`.
136    pub offsets: &'a super::btf_offsets::CpuTimeOffsets,
137    /// Link-time KVA of the `kernel_cpustat` per-CPU symbol (the
138    /// value `st_value` carries in the vmlinux symbol table — the
139    /// template address the linker assigned). The runtime KVA on
140    /// CPU `cpu` is
141    /// [`super::symbols::per_cpu_kva`]`(kernel_cpustat_kva,
142    /// kaslr_offset, per_cpu_offsets[cpu])`.
143    pub kernel_cpustat_kva: u64,
144    /// Link-time KVA of the `kstat` per-CPU symbol. See
145    /// `kernel_cpustat_kva` for the runtime KVA formula.
146    pub kstat_kva: u64,
147    /// Link-time KVA of the `tick_cpu_sched` per-CPU symbol.
148    /// `None` when the kernel was built without
149    /// `CONFIG_NO_HZ_COMMON`; iowait_sleeptime capture is skipped.
150    pub tick_cpu_sched_kva: Option<u64>,
151    /// Per-CPU offset array (`__per_cpu_offset[cpu]`) — same array
152    /// the BPF prog-stats walker uses (see
153    /// [`super::symbols::read_per_cpu_offsets`]). Length determines
154    /// how many CPUs the walker visits.
155    pub per_cpu_offsets: &'a [u64],
156    /// Guest's `PAGE_OFFSET` (resolved via
157    /// [`super::symbols::resolve_page_offset`]). Used to translate
158    /// each CPU's per-CPU KVA to a guest physical address for the
159    /// memory read.
160    pub page_offset: u64,
161    /// Virtual KASLR offset that per-CPU KVA derivation needs to
162    /// bridge the link-time (`__per_cpu_start_LINK`) and runtime
163    /// (`__per_cpu_start_RUNTIME`) bases. Sourced from the shared
164    /// `kern_virt_kaslr` Arc populated by either the BSP MSR_LSTAR
165    /// derive (`crate::vmm::x86_64::msr_kaslr::read_and_derive`,
166    /// x86_64-only) or the guest-channel KERN_ADDRS `_text`
167    /// subtraction (`crate::vmm::freeze_coord::dispatch`, both
168    /// arches). 0 fallback matches KASLR-off / nokaslr-karg
169    /// semantics and collapses [`super::symbols::per_cpu_kva`] to
170    /// the no-slide formula. On aarch64 without `_text` in
171    /// /proc/kallsyms (kptr_restrict masked) the value stays 0
172    /// and per-CPU resolution relies on the `nokaslr` karg
173    /// (`src/vmm/setup.rs`) instead.
174    pub kaslr_offset: u64,
175}
176
177/// Per-cgroup PSI-irq host-walk inputs (Phase A). Borrowed-only/optional,
178/// mirroring [`CpuTimeCapture`]: the freeze coordinator builds it only when the
179/// cgroup-walk offsets, the `cgrp_dfl_root` symbol, and the `psi_group` offsets
180/// all resolve; otherwise [`DumpContext::cgroup_psi_capture`] is `None` and the
181/// per-cgroup axis reads loud-absent. The walk descends `cgrp_dfl_root` → the
182/// host-held workload-root path → leaf cgroups and reads each leaf's
183/// `cgroup->psi` PSI_IRQ_FULL (see [`super::cgroup_walk`]).
184pub struct CgroupPsiCapture<'a> {
185    /// Guest memory handle used to read the cgroup hierarchy + each psi_group.
186    pub mem: &'a super::reader::GuestMem,
187    /// BTF-resolved cgroup-hierarchy field offsets (`cgroup.{self,kn,psi}`,
188    /// `cgroup_subsys_state.{sibling,children}`, `cgroup_root.cgrp`,
189    /// `kernfs_node.name`).
190    pub cgroup_offsets: &'a super::btf_offsets::CgroupWalkOffsets,
191    /// BTF-resolved `psi_group` offsets (shared with the system-wide walk —
192    /// a per-cgroup psi_group is the same `struct psi_group`).
193    pub psi_offsets: &'a super::btf_offsets::PsiGroupOffsets,
194    /// RUNTIME KVA of the hierarchy root cgroup (`cgrp_dfl_root +
195    /// offsetof(cgroup_root, cgrp)`, with any virtual-KASLR slide applied by
196    /// the caller) — used for the children-list anchor compare at the root
197    /// level. Every descendant is a direct-mapped slab object.
198    pub root_cgroup_kva: u64,
199    /// Guest physical address of the hierarchy root cgroup (the kernel-image
200    /// translation of the link-time root KVA, done by the caller via
201    /// `GuestKernel::text_kva_to_pa`) — the walk's entry read.
202    pub root_cgroup_pa: u64,
203    /// The test's workload-root cgroup path (host-held VM config, default
204    /// `/sys/fs/cgroup/ktstr`). The walk descends ONLY this subtree, so the
205    /// scheduler's separate cgroup does not confound the per-cgroup axis.
206    pub workload_root_path: &'a str,
207    /// Guest `PAGE_OFFSET` for the direct-map (`kva_to_pa`) hops to every
208    /// descendant cgroup / kernfs_node / psi_group.
209    pub page_offset: u64,
210}
211
212/// Borrow-only capture context for per-task enrichment.
213///
214/// Carries the [`super::guest::GuestKernel`] (guest memory + symbol
215/// table), the BTF-resolved task/signal/pid/upid offsets, the cached
216/// sched_class symbol KVAs (for class-name decode and the
217/// PI-boost-out-of-SCX flag), the lock-slowpath symbol cache (for
218/// stack-trace pattern matching), AND the task list itself — a
219/// pre-collected `&[TaskWalkerEntry]` produced by a task walker
220/// (rq->scx walk, DSQ walk, init_task→tasks enumeration).
221///
222/// Mirrors the [`ProgRuntimeCapture`] / [`CpuTimeCapture`]
223/// borrowed-only-optional shape. When `dump_state` receives
224/// `Some(TaskEnrichmentCapture)`, it iterates `tasks` and calls
225/// [`super::task_enrichment::walk_task_enrichment`] for each entry,
226/// pushing results into [`FailureDumpReport::task_enrichments`]. When
227/// `None`, the field stays empty and
228/// [`FailureDumpReport::task_enrichments_unavailable`] gets a
229/// "no task walker available" diagnostic.
230///
231/// The walker producer (rq->scx walker etc.) is responsible for
232/// building this struct. Until walker dispatch lands, no walker
233/// exists; the freeze coordinator passes `None` and the field is
234/// plumbed but empty.
235pub struct TaskEnrichmentCapture<'a> {
236    /// Borrowed GuestKernel — provides memory access, page-table
237    /// translation context, and the vmlinux symbol table.
238    pub kernel: &'a super::guest::GuestKernel,
239    /// BTF-resolved offsets for the task/signal/pid/upid walk.
240    pub offsets: &'a super::btf_offsets::TaskEnrichmentOffsets,
241    /// Cached sched_class symbol KVAs for class decode + PI-boost
242    /// flag.
243    pub sched_classes: &'a super::task_enrichment::SchedClassRegistry,
244    /// Cached lock-slowpath symbol KVAs for stack-PC pattern
245    /// matching.
246    pub lock_slowpaths: &'a super::task_enrichment::LockSlowpathRegistry,
247    /// Tasks the walker discovered, plus per-task metadata
248    /// `walk_task_enrichment` needs (see [`TaskWalkerEntry`]).
249    pub tasks: &'a [TaskWalkerEntry],
250}
251
252/// One entry produced by a task walker (rq->scx, DSQ, etc.) for the
253/// enrichment capture pipeline.
254///
255/// Each task walker discovers task KVAs by traversing the kernel's
256/// own scheduling data structures; the walker also knows which task
257/// was reachable via `rq->scx.runnable_list` (used for the
258/// PI-boost-out-of-SCX flag) and which vCPU's instruction-pointer
259/// matches the running task (used for the lock-slowpath stack
260/// matcher). Capturing those signals at the walker site keeps the
261/// enrichment surface side-effect free — `walk_task_enrichment` only
262/// reads guest memory; it does not perform discovery itself.
263#[derive(Debug, Clone, Copy)]
264pub struct TaskWalkerEntry {
265    /// Kernel virtual address of the `task_struct`.
266    pub task_kva: u64,
267    /// True iff the task was reached via `rq->scx.runnable_list`.
268    /// Required for the PI-boost-out-of-SCX flag — see
269    /// [`super::task_enrichment::TaskEnrichment::pi_boosted_out_of_scx`].
270    pub is_runnable_in_scx: bool,
271    /// Optional instruction pointer for the lock-slowpath stack
272    /// matcher. Pass the corresponding vCPU's
273    /// [`VcpuRegSnapshot::instruction_pointer`] when this task was
274    /// running on that vCPU at freeze time; pass `None` for tasks
275    /// not actively running.
276    pub running_pc: Option<u64>,
277}
278
279/// Per-CPU CPU-time / softirq / IRQ snapshot captured at freeze
280/// time. One entry per CPU index visible to the host walker.
281///
282/// All counter fields are monotonic in the kernel — the freeze
283/// captures the instantaneous value at the moment the vCPUs
284/// rendezvous-park. Diffing two snapshots (or comparing against a
285/// pre-test baseline) is the consumer's job; this type does not
286/// derive deltas.
287///
288/// Field semantics match the kernel sources:
289///   - `cpustat_*_ns`: ns counter from
290///     `kernel_cpustat::cpustat[CPUTIME_*]`. Updated by
291///     `account_user_time` / `account_system_index_time` and
292///     siblings (`kernel/sched/cputime.c`). The kernel stores
293///     nanoseconds; `/proc/stat` divides by `cputime_to_clock_t`.
294///   - `softirqs[i]`: `kernel_stat::softirqs[i]` cumulative count
295///     incremented by `kstat_incr_softirqs_this_cpu` on every
296///     softirq raise. Indexed by `super::btf_offsets::SOFTIRQ_NAMES`.
297///   - `irqs_sum`: `kernel_stat::irqs_sum` cumulative count
298///     incremented by `kstat_incr_irq_this_cpu` on every hardirq.
299///   - `iowait_sleeptime_ns`: `tick_sched::iowait_sleeptime`
300///     accumulated only under NO_HZ when the CPU enters idle with
301///     `nr_iowait > 0`. `None` when CONFIG_NO_HZ_COMMON is off or
302///     the resolver couldn't locate `tick_cpu_sched`.
303#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
304#[non_exhaustive]
305pub struct PerCpuTimeStats {
306    /// CPU index (0-based) this entry describes.
307    pub cpu: u32,
308    /// `cpustat[CPUTIME_USER]` (ns).
309    pub cpustat_user_ns: u64,
310    /// `cpustat[CPUTIME_NICE]` (ns).
311    pub cpustat_nice_ns: u64,
312    /// `cpustat[CPUTIME_SYSTEM]` (ns).
313    pub cpustat_system_ns: u64,
314    /// `cpustat[CPUTIME_SOFTIRQ]` (ns).
315    pub cpustat_softirq_ns: u64,
316    /// `cpustat[CPUTIME_IRQ]` (ns).
317    pub cpustat_irq_ns: u64,
318    /// `cpustat[CPUTIME_IDLE]` (ns).
319    pub cpustat_idle_ns: u64,
320    /// `cpustat[CPUTIME_IOWAIT]` (ns).
321    pub cpustat_iowait_ns: u64,
322    /// `cpustat[CPUTIME_STEAL]` (ns).
323    pub cpustat_steal_ns: u64,
324    /// `kernel_stat::softirqs[]` per-vector cumulative counts.
325    /// Indexed by `super::btf_offsets::SOFTIRQ_NAMES`.
326    pub softirqs: [u64; super::btf_offsets::NR_SOFTIRQS],
327    /// `kernel_stat::irqs_sum` cumulative hardirq count.
328    pub irqs_sum: u64,
329    /// `tick_sched::iowait_sleeptime` accumulated NO_HZ idle time
330    /// with outstanding IO (ns). `None` when NO_HZ disabled or
331    /// `tick_cpu_sched` symbol was absent at resolve time.
332    #[serde(default, skip_serializing_if = "Option::is_none")]
333    pub iowait_sleeptime_ns: Option<u64>,
334}
335
336/// Per-node NUMA event counters captured from
337/// `pglist_data->node_zones[]->vm_numa_event[]` at freeze time.
338///
339/// Each row is one row of NUMA event counters summed across all
340/// zones on a single node. The six counters mirror the kernel's
341/// `enum numa_stat_item` (see `super::btf_offsets::NUMA_HIT`
342/// etc. for the enum-stable indices). All counters are
343/// monotonic-since-boot; consumers diff against a baseline (or
344/// against another node's row) to extract the test-window delta.
345///
346/// Diagnostic value for sched_ext stalls is informational only —
347/// the NUMA balancer is not active for ext tasks. The rows
348/// surface here so an operator triaging a NUMA-aware workload
349/// (e.g. a memory-tiering test) can verify the kernel actually
350/// observed the expected node-locality distribution.
351///
352/// **Live walker status:** the wire shape, BTF offsets
353/// (`super::btf_offsets::NumaStatsOffsets`), and report field
354/// are wired through. The actual host-side walker that resolves
355/// `node_data[]` and reads per-zone counters is pending; until it
356/// lands, the report's [`FailureDumpReport::per_node_numa`] vec
357/// stays empty and
358/// [`FailureDumpReport::per_node_numa_unavailable`] carries the
359/// `"no NUMA walker"` reason.
360#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
361#[non_exhaustive]
362pub struct PerNodeNumaStats {
363    /// NUMA node id this row describes.
364    pub node: u32,
365    /// `vm_numa_event[NUMA_HIT]` summed across zones — pages
366    /// allocated on the requested node when local was preferred.
367    pub numa_hit: u64,
368    /// `vm_numa_event[NUMA_MISS]` — local node full, allocation
369    /// landed on a non-local node.
370    pub numa_miss: u64,
371    /// `vm_numa_event[NUMA_FOREIGN]` — process-policy targeted a
372    /// different node, this node honored the policy.
373    pub numa_foreign: u64,
374    /// `vm_numa_event[NUMA_INTERLEAVE_HIT]` — interleave policy
375    /// allocations that landed on this node.
376    pub numa_interleave_hit: u64,
377    /// `vm_numa_event[NUMA_LOCAL]` — allocations on this node by
378    /// processes running on this node.
379    pub numa_local: u64,
380    /// `vm_numa_event[NUMA_OTHER]` — allocations on this node by
381    /// processes running on a different node.
382    pub numa_other: u64,
383}
384
385/// Reason string written into [`FailureDumpReport::per_node_numa_unavailable`]
386/// when the per-node NUMA walker has not landed yet. Distinct from
387/// other unavailable reasons so a downstream consumer can tell
388/// "walker not implemented" apart from "walker ran and produced
389/// no data" once the live producer ships.
390pub const REASON_NO_NUMA_WALKER: &str = "no NUMA walker (host-side walker pending)";
391
392/// Borrow-only capture context for the per-sample SCX event counter
393/// timeline.
394///
395/// The freeze coordinator forwards the monitor sampler's accumulated
396/// `super::MonitorSample` vec via [`Self::samples`]; the dump path
397/// folds each sample's per-CPU `super::ScxEventCounters` into a
398/// single cross-CPU sum and produces one [`EventCounterSample`] per
399/// monitor tick.
400///
401/// `None` skips the timeline capture; the dump still renders the
402/// rest of the report. Mirrors [`ProgRuntimeCapture`] /
403/// [`CpuTimeCapture`]'s "borrowed-only, optional" shape.
404pub struct EventCounterCapture<'a> {
405    /// Periodic monitor samples gathered between VM start and the
406    /// freeze trigger. Each sample carries per-CPU
407    /// `super::ScxEventCounters` when scx event-stat offsets
408    /// resolved; the dump folder skips samples whose CPUs all
409    /// reported `event_counters: None`.
410    pub samples: &'a [super::MonitorSample],
411}
412
413/// Borrow-only capture context for the rq->scx + DSQ walkers.
414/// Mirrors [`TaskEnrichmentCapture`] / [`CpuTimeCapture`] shape —
415/// `dump_state` consumes everything by reference.
416///
417/// Carries:
418/// - `kernel`: GuestKernel handle for guest-memory reads
419///   (PTE walks, symbol resolution).
420/// - `offsets`: BTF-resolved
421///   [`super::btf_offsets::ScxWalkerOffsets`] covering scx_rq,
422///   scx_sched, scx_sched_pcpu, scx_sched_pnode, scx_dispatch_q,
423///   sched_ext_entity, scx_dsq_list_node, rhashtable, bucket_table,
424///   rhash_head.
425/// - `scx_root_kva`: kernel-text-mapped pointer the walker
426///   dereferences to find the active `scx_sched`.
427/// - `rq_kvas` / `rq_pas`: per-CPU rq KVA + PA arrays; same vecs
428///   the runnable_at scanner uses.
429/// - `per_cpu_offsets`: `__per_cpu_offset[]` array — needed for
430///   per-CPU bypass DSQ resolution.
431/// - `nr_nodes`: NUMA node count, for the per-node global-DSQ
432///   walk. Pass `1` on UMA / unknown configurations; the walker
433///   gracefully skips slots whose pnode pointers are NULL.
434///
435/// When `None` is passed in [`DumpContext::scx_walker_capture`],
436/// the dump emits empty `rq_scx_states` / `dsq_states` and
437/// records `scx_walker_unavailable` with a diagnostic reason.
438pub struct ScxWalkerCapture<'a> {
439    /// Borrowed GuestKernel — provides memory access, page-table
440    /// translation context, and the vmlinux symbol table.
441    pub kernel: &'a super::guest::GuestKernel,
442    /// BTF-resolved offsets for the scx walker.
443    pub offsets: &'a super::btf_offsets::ScxWalkerOffsets,
444    /// `scx_root` symbol KVA (resolved via vmlinux ELF symtab).
445    /// The walker reads `*scx_root` to find the active scx_sched.
446    pub scx_root_kva: u64,
447    /// Per-CPU rq kernel virtual addresses (one per CPU).
448    pub rq_kvas: &'a [u64],
449    /// Per-CPU rq guest physical addresses (parallel to rq_kvas).
450    pub rq_pas: &'a [u64],
451    /// `__per_cpu_offset[]` array, used to address each CPU's
452    /// `scx_sched_pcpu.bypass_dsq`.
453    pub per_cpu_offsets: &'a [u64],
454    /// NUMA node count for the per-node global-DSQ walk. Pass `1`
455    /// on UMA / unknown configurations.
456    pub nr_nodes: u32,
457}
458
459/// One per-monitor-tick snapshot of the 13 SCX_EV_* event counters
460/// summed across every CPU at that tick.
461///
462/// The kernel stores per-CPU `s64` counters in `scx_sched_pcpu`
463/// (kernel/sched/ext.c); the monitor sampler reads them at every
464/// tick and stores per-CPU `event_counters` on each
465/// `super::CpuSnapshot`. The dump path sums across CPUs into the
466/// fields here so a downstream consumer can render the run's
467/// counter timeline (sparkline, delta plot, ...) without
468/// re-iterating the per-CPU vec.
469///
470/// Field semantics match
471/// `super::ScxEventCounters` one-to-one — see that struct's
472/// per-field doc for kernel-source provenance. `total_*` naming
473/// here echoes `super::ScxEventDeltas`'s aggregate-across-window
474/// fields but with per-tick (not per-window) granularity.
475#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
476#[non_exhaustive]
477pub struct EventCounterSample {
478    /// Milliseconds since VM start (mirrors
479    /// `super::MonitorSample::elapsed_ms`). Zero on the first
480    /// sample.
481    pub elapsed_ms: u64,
482    /// Sum of `select_cpu_fallback` across all CPUs at this tick.
483    pub select_cpu_fallback: i64,
484    /// Sum of `dispatch_local_dsq_offline` across all CPUs.
485    pub dispatch_local_dsq_offline: i64,
486    /// Sum of `dispatch_keep_last` across all CPUs.
487    pub dispatch_keep_last: i64,
488    /// Sum of `enq_skip_exiting` across all CPUs.
489    pub enq_skip_exiting: i64,
490    /// Sum of `enq_skip_migration_disabled` across all CPUs.
491    pub enq_skip_migration_disabled: i64,
492    /// Sum of `reenq_immed` across all CPUs.
493    pub reenq_immed: i64,
494    /// Sum of `reenq_local_repeat` across all CPUs.
495    pub reenq_local_repeat: i64,
496    /// Sum of `refill_slice_dfl` across all CPUs.
497    pub refill_slice_dfl: i64,
498    /// Sum of `bypass_duration` across all CPUs (ns).
499    pub bypass_duration: i64,
500    /// Sum of `bypass_dispatch` across all CPUs.
501    pub bypass_dispatch: i64,
502    /// Sum of `bypass_activate` across all CPUs.
503    pub bypass_activate: i64,
504    /// Sum of `insert_not_owned` across all CPUs.
505    pub insert_not_owned: i64,
506    /// Sum of `sub_bypass_dispatch` across all CPUs.
507    pub sub_bypass_dispatch: i64,
508}
509
510impl EventCounterSample {
511    /// Construct from a `super::MonitorSample` by summing every
512    /// CPU's `super::ScxEventCounters`. CPUs whose
513    /// `event_counters` is `None` (event-stat offsets unresolved)
514    /// contribute 0 to every field.
515    ///
516    /// Returns `None` when no CPU on the sample reported event
517    /// counters — propagating that to the timeline would emit a
518    /// row of all zeros that's indistinguishable from a real
519    /// "every counter at zero" tick. Callers filter `None` out.
520    pub fn from_monitor_sample(sample: &super::MonitorSample) -> Option<Self> {
521        let mut any = false;
522        let mut out = Self {
523            elapsed_ms: sample.elapsed_ms,
524            ..Self::default()
525        };
526        for cpu in &sample.cpus {
527            if let Some(ev) = &cpu.event_counters {
528                any = true;
529                // Per-CPU SCX event counters are s64 in the kernel
530                // and originate from BPF map reads of guest memory.
531                // A corrupt counter could trip i64 addition overflow
532                // when summed across many CPUs; saturating_add pins
533                // the sum at i64::{MIN,MAX} rather than panicking
534                // (debug) or wrapping (release) into a misleading
535                // value.
536                out.select_cpu_fallback = out
537                    .select_cpu_fallback
538                    .saturating_add(ev.select_cpu_fallback);
539                out.dispatch_local_dsq_offline = out
540                    .dispatch_local_dsq_offline
541                    .saturating_add(ev.dispatch_local_dsq_offline);
542                out.dispatch_keep_last =
543                    out.dispatch_keep_last.saturating_add(ev.dispatch_keep_last);
544                out.enq_skip_exiting = out.enq_skip_exiting.saturating_add(ev.enq_skip_exiting);
545                out.enq_skip_migration_disabled = out
546                    .enq_skip_migration_disabled
547                    .saturating_add(ev.enq_skip_migration_disabled);
548                out.reenq_immed = out.reenq_immed.saturating_add(ev.reenq_immed);
549                out.reenq_local_repeat =
550                    out.reenq_local_repeat.saturating_add(ev.reenq_local_repeat);
551                out.refill_slice_dfl = out.refill_slice_dfl.saturating_add(ev.refill_slice_dfl);
552                out.bypass_duration = out.bypass_duration.saturating_add(ev.bypass_duration);
553                out.bypass_dispatch = out.bypass_dispatch.saturating_add(ev.bypass_dispatch);
554                out.bypass_activate = out.bypass_activate.saturating_add(ev.bypass_activate);
555                out.insert_not_owned = out.insert_not_owned.saturating_add(ev.insert_not_owned);
556                out.sub_bypass_dispatch = out
557                    .sub_bypass_dispatch
558                    .saturating_add(ev.sub_bypass_dispatch);
559            }
560        }
561        if any { Some(out) } else { None }
562    }
563}
564
565/// Render a u64 counter series as a 1-line UTF-8 sparkline.
566///
567/// Maps each value into one of 8 unicode block-element glyphs
568/// (`▁▂▃▄▅▆▇█`) by min-max scaling. Empty input renders as the
569/// empty string; a constant non-zero series renders as repeated
570/// mid-tier glyphs (matches the "no variation" reading in the
571/// data, not as misleading monotonic up-bars). A constant zero
572/// series renders as repeated lowest glyphs.
573///
574/// Used by the `Display` impl for the event-counter timeline. Pure
575/// helper — no allocation outside the returned `String`.
576pub fn render_sparkline(values: &[u64]) -> String {
577    const GLYPHS: &[char] = &['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
578    if values.is_empty() {
579        return String::new();
580    }
581    let min = *values.iter().min().expect("non-empty");
582    let max = *values.iter().max().expect("non-empty");
583    let mut s = String::with_capacity(values.len() * 4);
584    if max == min {
585        let glyph = if max == 0 {
586            GLYPHS[0]
587        } else {
588            GLYPHS[GLYPHS.len() / 2]
589        };
590        for _ in values {
591            s.push(glyph);
592        }
593        return s;
594    }
595    let span = max - min;
596    let last_idx = (GLYPHS.len() - 1) as u64;
597    for &v in values {
598        // Linear scale [min, max] → [0, GLYPHS.len()-1]. Integer
599        // math is sufficient (no rounding artifact at the cost
600        // of one extra glyph step at boundaries).
601        let scaled = ((v - min) * last_idx) / span;
602        let idx = scaled.min(last_idx) as usize;
603        s.push(GLYPHS[idx]);
604    }
605    s
606}
607
608/// Saturating-cast wrapper around [`render_sparkline`] for signed
609/// (i64) counter series. Negative values clamp to 0; the kernel
610/// stores SCX_EV_* as `s64` but every counter is non-negative in
611/// practice, so the saturation only fires on a corrupt read.
612pub fn render_sparkline_i64(values: &[i64]) -> String {
613    let widened: Vec<u64> = values.iter().map(|&v| v.max(0) as u64).collect();
614    render_sparkline(&widened)
615}
616
617/// Snapshot of one vCPU's instruction-pointer / stack-pointer / page-
618/// table-root at freeze time. Re-export of the freeze-side type so
619/// dump consumers don't have to depend on `vmm::exit_dispatch`
620/// internals.
621pub use crate::vmm::exit_dispatch::VcpuRegSnapshot;
622
623/// Schema discriminant value emitted in `FailureDumpReport.schema`.
624///
625/// Consumers that read a `.failure-dump.json` file use the `schema`
626/// field's value to choose between [`FailureDumpReport`],
627/// [`DualFailureDumpReport`], and [`DegradedFailureDumpReport`]
628/// before attempting deserialization. The
629/// [`FailureDumpReportAny::from_json`] dispatcher handles this
630/// routing for in-process consumers.
631/// Values are stable wire constants — extending the dump pipeline
632/// with a new shape adds a new constant rather than changing this
633/// one.
634pub const SCHEMA_SINGLE: &str = "single";
635
636/// Schema discriminant value emitted in `DualFailureDumpReport.schema`.
637/// See [`SCHEMA_SINGLE`] for the discriminant contract.
638pub const SCHEMA_DUAL: &str = "dual";
639
640/// Schema discriminant value emitted in `DegradedFailureDumpReport.schema`.
641///
642/// Carried by failure-dumps the freeze coordinator was unable to
643/// capture as a full [`SCHEMA_SINGLE`] / [`SCHEMA_DUAL`] report — the
644/// trigger fired but rendezvous, gate cross-reference, or KVA
645/// translation aborted the dump path. Per the wire-format contract on
646/// [`SCHEMA_SINGLE`], degraded dumps are a stable variant added by
647/// new constant, not a mutation of the existing two.
648pub const SCHEMA_DEGRADED: &str = "degraded";
649
650/// Reason string written into [`DegradedFailureDumpReport::reason`]
651/// when the freeze coordinator's vCPU rendezvous timed out before
652/// every parked acknowledgement arrived. Wire-format-stable: matches
653/// the operator-grep contract used by every other `REASON_*` constant
654/// in this module. The dynamic detail appended at emit time
655/// (`<timeout_ms>` / `<parked>` / `<expected>`) lets an operator see
656/// which vCPUs stalled without a separate field.
657pub const REASON_DEGRADED_RENDEZVOUS_TIMEOUT: &str =
658    "vCPU rendezvous timed out before parked acknowledgement";
659
660/// Reason string written into [`DegradedFailureDumpReport::reason`]
661/// when the host kill signal flipped during the vCPU freeze
662/// rendezvous, short-circuiting the wait before every parked
663/// acknowledgement arrived. Distinct from
664/// [`REASON_DEGRADED_RENDEZVOUS_TIMEOUT`] (which fires only when the
665/// 30s deadline expires): a kill-mid-rendezvous typically lands in
666/// the rendezvous loop within milliseconds, so the elapsed_ms in the
667/// dynamic detail (appended at emit time) reads as a small number
668/// and the "timed out" label is internally contradictory. The kill
669/// sources are SCHED_EXIT propagation from a vCPU thread, watchdog
670/// hard-deadline expiry, and panic-hook flips. Wire-format-stable;
671/// matches the operator-grep contract used by every other `REASON_*`
672/// constant.
673pub const REASON_DEGRADED_KILL_DURING_RENDEZVOUS: &str =
674    "vCPU rendezvous aborted by external kill before parked acknowledgement";
675
676/// Snapshot tag used when the early-snapshot trigger fires but
677/// `freeze_and_dispatch(FreezeMode::Capture { gate_on_exit_kind: false })` returns `Degraded` (early-half
678/// rendezvous timeout). The freeze coordinator writes the degraded
679/// JSON to a sibling path named via
680/// `super::super::vmm::freeze_coord::snapshot_tagged_path` using
681/// this tag — main `{stem}.failure-dump.json` is preserved for the
682/// subsequent late-trigger emission. Operator-readable wire-format
683/// constant: kebab-case, stable across releases.
684///
685/// MAINTENANCE: adding a new `SNAPSHOT_TAG_*` const requires updating
686/// BOTH (a) the `ALL_SNAPSHOT_TAGS` slice below AND (b) the
687/// `expected` hand-list in
688/// `all_snapshot_tags_enumerates_every_pub_const_in_module` in
689/// `src/monitor/dump/tests.rs`. The pinning test catches slice-vs-
690/// expected divergence; it does NOT catch a const added without
691/// updating either (both arrays stay at the same length).
692pub const SNAPSHOT_TAG_EARLY_DEGRADED: &str = "early-degraded";
693
694/// Snapshot tag used when dual-snapshot mode held a Captured early
695/// snapshot AND the late-trigger path returned `Degraded`. The early
696/// snapshot is written to a sibling path with this tag while the
697/// late degraded JSON occupies the main dump path. Distinguishes
698/// "early itself degraded" ([`SNAPSHOT_TAG_EARLY_DEGRADED`]) from
699/// "early captured, late degraded" (this tag) so an operator browsing
700/// the dump directory knows which case produced which file. Every
701/// captured snapshot reaches disk.
702pub const SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED: &str = "early-pre-late-degraded";
703
704/// Snapshot tag used when dual-snapshot mode held a Captured early
705/// snapshot AND the late-trigger path ran AND returned `Suppressed`
706/// (the gate examined `*scx_root->exit_kind`, found it below
707/// SCX_EXIT_ERROR, and decided no failure dump warranted). Distinct
708/// from [`SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED`]: this tag
709/// means the late trigger DID fire and the gate explicitly decided
710/// clean exit. Operator triage: scheduler recovered from the early
711/// stall and reached a clean shutdown via the SCX_EXIT_NONE /
712/// SCX_EXIT_DONE path. The early observation (runnable-age spike)
713/// is independently meaningful and reaches disk at the tagged
714/// sibling. Symmetric with
715/// [`SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED`] — tagged sibling rather
716/// than main path so the main `{stem}.failure-dump.json` keeps the
717/// "scheduler had a failure-class exit" semantic.
718pub const SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED: &str = "early-only-late-suppressed";
719
720/// Snapshot tag used when dual-snapshot mode held a Captured early
721/// snapshot AND the late-trigger path NEVER FIRED for the run (no
722/// `err_exit_detected` BPF latch flip; the scheduler never reached
723/// an error-class late event). Distinct from
724/// [`SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED`]: this tag means the
725/// late trigger never ran at all, NOT that it ran and decided
726/// clean. Operator triage: scheduler crossed the half-watchdog
727/// runnable-age threshold (early-trigger fired) but then either
728/// recovered or terminated before reaching the late-trigger path —
729/// `freeze_state` stayed at `Idle` or `TookEarly` through coord
730/// exit. The end-of-coord drain emits the early observation to the
731/// tagged sibling rather than letting it drop with the closure.
732pub const SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED: &str = "early-only-late-never-fired";
733
734/// Canonical enumeration of every `SNAPSHOT_TAG_*` constant in this
735/// module. Tests that negative-scan tag locations (e.g. asserting no
736/// file landed at any wrong tag) iterate this slice rather than
737/// hardcoding the 4-element list. Adding a new `SNAPSHOT_TAG_*`
738/// constant requires updating this slice — the pinning test
739/// `all_snapshot_tags_enumerates_every_pub_const_in_module` in
740/// `src/monitor/dump/tests.rs` will fail until the new tag is added,
741/// flagging the inconsistency at test-time.
742///
743/// Order: NEVER_FIRED first because it is the default tag in
744/// `EarlySnapshotGuard::drain_to_disk`'s `unwrap_or` arm at
745/// src/vmm/freeze_coord/mod.rs (the guard struct is `pub(super)` so
746/// rustdoc cannot intra-doc-link it from this module; cite by file
747/// path matches the cross-ref convention used elsewhere in this
748/// batch). The other three follow in dispatch-arm order:
749/// SUPPRESSED (late-trigger Suppressed write-failure) → PRE_LATE_DEGRADED
750/// (late-trigger Degraded write-failure) → EARLY_DEGRADED (early-
751/// trigger Degraded direct write) — so readers can map slice index
752/// to dispatch arm by inspection.
753///
754/// `#[cfg(test)]` — production code never iterates this slice;
755/// only the negative-scan tests and the pinning canary reference it.
756/// Gated to keep the lib's dead-code lint clean.
757#[cfg(test)]
758pub const ALL_SNAPSHOT_TAGS: &[&str] = &[
759    SNAPSHOT_TAG_EARLY_ONLY_LATE_NEVER_FIRED,
760    SNAPSHOT_TAG_EARLY_ONLY_LATE_SUPPRESSED,
761    SNAPSHOT_TAG_EARLY_PRE_LATE_DEGRADED,
762    SNAPSHOT_TAG_EARLY_DEGRADED,
763];
764
765/// Reason string written into [`FailureDumpReport::prog_runtime_stats_unavailable`]
766/// when [`DumpContext::prog_capture`] was supplied but the per-program
767/// walker found no struct_ops programs in `prog_idr` at freeze time.
768/// Wire-format-stable: an operator parsing the sidecar JSON looks for
769/// this exact string to distinguish from the prog-accessor-missing
770/// case.
771pub const REASON_NO_STRUCT_OPS_LOADED: &str = "no struct_ops programs loaded";
772
773/// Reason string written into [`FailureDumpReport::prog_runtime_stats_unavailable`]
774/// when [`DumpContext::prog_capture`] was `None`. Distinguishes from
775/// [`REASON_NO_STRUCT_OPS_LOADED`] — the walker never ran in this case
776/// because the accessor wasn't constructed (e.g. `prog_idr` symbol
777/// missing).
778pub const REASON_PROG_ACCESSOR_UNAVAILABLE: &str = "prog accessor unavailable";
779
780/// Reason string written into [`FailureDumpReport::task_enrichments_unavailable`]
781/// when [`DumpContext::task_enrichment_capture`] was supplied but
782/// every walker entry produced no enrichment (idle guest with no
783/// runnable scx tasks at the freeze instant).
784pub const REASON_TASK_WALKER_ZERO_TASKS: &str = "task walker yielded zero tasks";
785
786/// Reason string written into [`FailureDumpReport::task_enrichments_unavailable`]
787/// when [`DumpContext::task_enrichment_capture`] was `None`.
788/// Distinguishes from [`REASON_TASK_WALKER_ZERO_TASKS`] — the walker
789/// never ran because the capture wasn't supplied.
790pub const REASON_NO_TASK_WALKER: &str = "no task walker available";
791
792/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
793/// when offsets resolved AND the walker found rq->scx + local DSQ
794/// data BUT `*scx_root == 0` — no scheduler attached at the freeze
795/// instant. The sched-rooted passes (bypass / global / user-hash)
796/// have nothing to walk, but the rq->scx and per-CPU local DSQ
797/// captures still produced data. Surfaces a distinct reason so the
798/// operator knows the scheduler isn't loaded vs. the walker is
799/// broken.
800pub const REASON_SCX_ROOT_NULL: &str = "scx_root is NULL (no scheduler attached)";
801
802/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
803/// when [`DumpContext::scx_walker_capture`] was supplied AND every
804/// offset sub-group resolved BUT the walker reached no rq, no DSQ,
805/// and no scx_sched state at all (every read failed). Distinct from
806/// [`REASON_SCX_ROOT_NULL`]: that case has rq->scx + local DSQ data
807/// but no sched_state; this case has nothing.
808pub const REASON_SCX_WALKER_NO_STATE: &str = "scx walker reached no state";
809
810/// Reason string written into [`FailureDumpReport::scx_walker_unavailable`]
811/// when [`DumpContext::scx_walker_capture`] was `None`. Distinguishes
812/// from [`REASON_SCX_WALKER_NO_STATE`] — the walker never ran at all
813/// because no capture was supplied.
814pub const REASON_NO_SCX_WALKER: &str = "no scx walker capture";
815
816/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
817/// when the sdt_alloc pre-pass could not run because the scheduler's
818/// arena `user_vm_start` was not 4 GiB-aligned. See the gate in
819/// [`dump_state`]: low-32 keying of the per-pass
820/// [`crate::monitor::dump::render_map::ArenaSlotIndex`] is only
821/// correct when `user_vm_start & 0xFFFF_FFFF == 0`.
822pub const REASON_SDT_ALLOC_UNALIGNED_USER_VM: &str =
823    "user_vm_start is not 4 GiB-aligned; low-32 keying disabled";
824
825/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
826/// when the dump enumerated no `*.bss` ARRAY map with a non-zero
827/// `btf_kva`. Without the scheduler's `.bss` bytes the pre-pass cannot
828/// read any allocator's in-memory state, and without `btf_kva` the
829/// program BTF that names `struct scx_allocator` is not loadable.
830pub const REASON_SDT_ALLOC_NO_BSS: &str = "no scheduler .bss map (or .bss has no program BTF)";
831
832/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
833/// when no `BPF_MAP_TYPE_ARENA` map snapshot was captured: the
834/// pre-pass has no `kern_vm_start` to translate arena pointers
835/// against, so no allocator slot can be walked.
836pub const REASON_SDT_ALLOC_NO_ARENA: &str = "no arena map captured (kern_vm_start unavailable)";
837
838/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
839/// when the scheduler's program BTF does not carry `struct scx_allocator`
840/// or its peer types (`sdt_pool`, `sdt_desc`, `sdt_chunk`) — the scheduler
841/// does not link `lib/sdt_alloc.bpf.c` and there are no allocator slots
842/// to walk. [`crate::monitor::sdt_alloc::SdtAllocOffsets::from_btf`]
843/// returns the underlying `anyhow::Error` describing which struct was
844/// missing; the dump caller folds that into this reason string.
845pub const REASON_SDT_ALLOC_NO_TYPE: &str = "scheduler BTF does not declare struct scx_allocator";
846
847/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
848/// when every prerequisite resolved but no `.bss` variable of type
849/// `struct scx_allocator` was discovered. The scheduler links
850/// `lib/sdt_alloc.bpf.c` for its types but has not declared a typed
851/// allocator instance.
852pub const REASON_SDT_ALLOC_NO_INSTANCE: &str = "no scx_allocator instance in .bss";
853
854/// Reason string written into [`FailureDumpReport::sdt_alloc_unavailable`]
855/// when the per-map dump deadline was exhausted before the pre-pass
856/// could run. The `dump_truncated_at_us` field also surfaces the
857/// truncation; this string disambiguates "deadline" from the other
858/// no-data causes when a consumer is scanning sdt_alloc-related
859/// diagnostics specifically.
860pub const REASON_SDT_ALLOC_DEADLINE_EXCEEDED: &str =
861    "dump deadline exceeded before sdt_alloc pre-pass could run";
862
863/// Cross-CPU sum of every per-CPU diagnostic counter slot in the
864/// probe BPF program's `.bss` `ktstr_pcpu_counters` array.
865///
866/// The probe declares one fixed-shape per-CPU array
867/// (`pcpu_counter ktstr_pcpu_counters[MAX_CPUS][KTSTR_PCPU_NR]` —
868/// see `src/bpf/probe.bpf.c`); each tracepoint / kprobe handler
869/// bumps a slot via `ktstr_pcpu_inc(KTSTR_PCPU_<NAME>)`. The host
870/// reader sums across the CPU axis to recover the cumulative count
871/// each handler reports. Field names mirror the slot names from
872/// `enum ktstr_pcpu_idx` so an operator can walk back from the
873/// failure-dump field to the probe source by exact name.
874///
875/// All counters are monotonic-since-probe-attach. Zero values
876/// indicate either "the corresponding tracepoint never fired" (the
877/// common case for `pi_*` and `lock_contend_*` on tests that don't
878/// exercise PI / lock contention) or "the tracepoint never attached"
879/// (e.g. `preempt_*` on a kernel without
880/// `CONFIG_TRACE_PREEMPT_TOGGLE`); the counter alone cannot
881/// distinguish those two cases — pair with the attach-state surface
882/// in `crate::probe::process::ProbeDiagnostics` when the
883/// distinction matters.
884#[derive(Debug, Clone, Default, Serialize, Deserialize)]
885#[non_exhaustive]
886pub struct ProbeBssCounters {
887    /// `KTSTR_PCPU_PROBE_COUNT` summed across CPUs — total kprobe
888    /// fires past the `ktstr_enabled` gate.
889    pub probe_count: u64,
890    /// `KTSTR_PCPU_KPROBE_RETURNS` summed across CPUs — kprobe fires
891    /// that committed an entry to `probe_data` (past `func_meta_map`
892    /// lookup and scratch-slot allocation).
893    pub kprobe_returns: u64,
894    /// `KTSTR_PCPU_META_MISS` summed across CPUs — kprobe fires
895    /// whose IP missed `func_meta_map`. `probe_count -
896    /// kprobe_returns` is the total bail count; `meta_miss` is the
897    /// subset whose bail came from the `func_meta_map` lookup.
898    pub meta_miss: u64,
899    /// `KTSTR_PCPU_RINGBUF_DROPS` summed across CPUs — failed
900    /// `bpf_ringbuf_reserve` calls inside the trigger handler.
901    pub ringbuf_drops: u64,
902    /// `KTSTR_PCPU_TIMELINE_COUNT` summed across CPUs — successful
903    /// timeline-event submissions across the three timeline
904    /// tracepoints (sched_switch + sched_migrate_task + sched_wakeup).
905    pub timeline_count: u64,
906    /// `KTSTR_PCPU_TIMELINE_DROPS` summed across CPUs — timeline
907    /// submissions that failed because the dedicated
908    /// `timeline_events` ringbuf was full at submit time.
909    pub timeline_drops: u64,
910    /// `KTSTR_PCPU_PI_COUNT` summed across CPUs — PI boost / unboost
911    /// records committed via `fexit/rt_mutex_setprio`.
912    pub pi_count: u64,
913    /// `KTSTR_PCPU_PI_ORPHAN_FEXITS` summed across CPUs — fexit
914    /// fires whose entry-side snapshot was never recorded (attach
915    /// race or `pi_scratch` overflow).
916    pub pi_orphan_fexits: u64,
917    /// `KTSTR_PCPU_PI_CLASS_CHANGE_COUNT` summed across CPUs —
918    /// PI events that observed a `sched_class` flip from fentry
919    /// to fexit (e.g. CFS → RT under a boost).
920    pub pi_class_change_count: u64,
921    /// `KTSTR_PCPU_PI_DROPS` summed across CPUs — TL_EVT_PI_BOOST
922    /// submissions that failed because the timeline ringbuf was
923    /// full at the PI fexit handler.
924    pub pi_drops: u64,
925    /// `KTSTR_PCPU_LOCK_CONTEND_COUNT` summed across CPUs —
926    /// `tp_btf/contention_begin` fires that committed a
927    /// TL_EVT_LOCK_CONTEND timeline record.
928    pub lock_contend_count: u64,
929    /// `KTSTR_PCPU_LOCK_CONTEND_DROPS` summed across CPUs —
930    /// TL_EVT_LOCK_CONTEND submissions that failed because the
931    /// timeline ringbuf was full.
932    pub lock_contend_drops: u64,
933    /// `KTSTR_PCPU_PREEMPT_DISABLE_COUNT` summed across CPUs —
934    /// `tp_btf/preempt_disable` outermost-transition fires.
935    pub preempt_disable_count: u64,
936    /// `KTSTR_PCPU_PREEMPT_ENABLE_COUNT` summed across CPUs —
937    /// `tp_btf/preempt_enable` outermost-transition fires.
938    pub preempt_enable_count: u64,
939    /// `KTSTR_PCPU_TRIGGER_COUNT` summed across CPUs — every accepted
940    /// selected trigger fire. The raw `scx_vexit` return program
941    /// also counts non-error kinds like DONE / UNREG; the global-era
942    /// `scx_dump_state` program counts only error-class entries.
943    pub trigger_count: u64,
944}
945
946/// Host-side context for a failure that struck before the freeze
947/// coordinator could capture any BPF state — the guest booted but cgroup
948/// or scheduler setup failed, so the payload never ran and only a
949/// placeholder report exists. Everything here was already in the host's
950/// hands (`vmm::VmResult`) at placeholder-write time; embedding it makes
951/// the artifact diagnosable instead of rendering `<unavailable>` for
952/// every section. All fields optional so the shape is purely additive.
953#[derive(Debug, Clone, Default, Serialize, Deserialize)]
954#[non_exhaustive]
955pub struct GuestSetupFailureContext {
956    /// Stage label from `classify_init_stage` — the deepest lifecycle
957    /// phase the guest reached before failing.
958    #[serde(default, skip_serializing_if = "Option::is_none")]
959    pub init_stage: Option<String>,
960    /// Final guest lifecycle stage per the host progress ledger
961    /// (`VmResult::final_guest_phase`), Debug-rendered.
962    #[serde(default, skip_serializing_if = "Option::is_none")]
963    pub final_guest_phase: Option<String>,
964    /// Guest exit code surfaced through the SHM ring or COM2 sentinel.
965    #[serde(default, skip_serializing_if = "Option::is_none")]
966    pub guest_exit_code: Option<i32>,
967    /// Scheduler attach outcome decoded from the lifecycle frames
968    /// (`verifier::attach_outcome_from_messages`), including the guest's
969    /// not-attached reason suffix when present.
970    #[serde(default, skip_serializing_if = "Option::is_none")]
971    pub scheduler_attach_outcome: Option<String>,
972    /// Bounded tail of the guest console / kernel log (COM1,
973    /// `VmResult::stderr`).
974    #[serde(default, skip_serializing_if = "Option::is_none")]
975    pub console_tail: Option<String>,
976    /// Bounded tail of the guest stdout / init log (COM2,
977    /// `VmResult::output`) — carries the `ktstr-init:` setup errors.
978    #[serde(default, skip_serializing_if = "Option::is_none")]
979    pub init_log_tail: Option<String>,
980    /// Bounded tail of the scheduler's own merged stdout/stderr shipped
981    /// over the bulk port (`SCHED_LOG` chunks) — carries the
982    /// scheduler's argv/config parse errors when it died pre-attach.
983    #[serde(default, skip_serializing_if = "Option::is_none")]
984    pub scheduler_log_tail: Option<String>,
985}
986
987impl GuestSetupFailureContext {
988    /// True when no field carries data — the Display impl and the
989    /// empty-report guard treat such a context as absent.
990    pub fn is_empty(&self) -> bool {
991        self.init_stage.is_none()
992            && self.final_guest_phase.is_none()
993            && self.guest_exit_code.is_none()
994            && self.scheduler_attach_outcome.is_none()
995            && self.console_tail.is_none()
996            && self.init_log_tail.is_none()
997            && self.scheduler_log_tail.is_none()
998    }
999}
1000
1001/// Top-level failure-dump report. One per freeze trigger.
1002#[derive(Debug, Clone, Serialize, Deserialize)]
1003#[non_exhaustive]
1004pub struct FailureDumpReport {
1005    /// Wire-format discriminant. Always `"single"` for this variant,
1006    /// pinning [`SCHEMA_SINGLE`]. Consumers branch on this to
1007    /// choose between [`FailureDumpReport`], [`DualFailureDumpReport`],
1008    /// and [`DegradedFailureDumpReport`] before deserializing. Single
1009    /// and Dual share top-level field names that would collide without
1010    /// an explicit tag; Degraded carries a distinct field set
1011    /// (`reason`, `watchpoint_hit`, `bss_latch_state`, `exit_kind`,
1012    /// `elapsed_ms`) but still gets the tag so
1013    /// [`FailureDumpReportAny::from_json`] can dispatch uniformly.
1014    pub schema: String,
1015    /// One entry per BPF map enumerated. Order matches the IDR walk
1016    /// (i.e. allocation order); the report is otherwise unsorted so
1017    /// callers that want a stable view should sort by name.
1018    pub maps: Vec<FailureDumpMap>,
1019    /// Per-vCPU register snapshots captured on each vCPU thread at
1020    /// freeze time. Index matches vCPU id (BSP at 0, APs at 1..N).
1021    /// `None` when a vCPU never parked (rendezvous timeout) or its
1022    /// `KVM_GET_REGS` failed mid-shutdown. Attached to the report by
1023    /// the freeze coordinator after `dump_state` returns.
1024    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1025    pub vcpu_regs: Vec<Option<VcpuRegSnapshot>>,
1026    /// Obj name of the currently-attached scheduler, identified by
1027    /// matching each `BPF_MAP_TYPE_STRUCT_OPS` map's `value_kva` (the
1028    /// guest-KVA of its `kvalue.data` payload) against the dereferenced
1029    /// `*scx_root` value (the guest-KVA of the active `struct scx_sched`,
1030    /// which is also the KVA of `scx_sched.ops` since `ops` sits at
1031    /// offset 0). When the match succeeds, the struct_ops map's name
1032    /// carries the obj prefix (libbpf convention: `<obj>.<struct_ops_var>`);
1033    /// the prefix is split at the first `.` and stored here.
1034    ///
1035    /// `None` when:
1036    /// - `scx_sched_state` is unavailable (no scheduler attached, BTF
1037    ///   missing the `scx_sched` type, or `*scx_root` could not be
1038    ///   resolved at capture time).
1039    /// - No `BPF_MAP_TYPE_STRUCT_OPS` map had `value_kva` matching the
1040    ///   active sched_kva (capture race during a mid-attach window, or
1041    ///   the struct_ops map's value_kva was not yet populated).
1042    /// - The matched map's name lacks a `<obj>.` prefix.
1043    ///
1044    /// [`crate::scenario::snapshot::Snapshot::active`] uses this as
1045    /// the principled tiebreaker when the projection sees multiple
1046    /// obj prefixes in global-section maps. On `None` the consumer
1047    /// falls back to the prefix-grouping heuristic (single obj →
1048    /// that one; multiple obj → `NoActiveScheduler` with a
1049    /// diagnostic naming the observed obj_names + the walker's
1050    /// failure cause).
1051    #[serde(default, skip_serializing_if = "Option::is_none")]
1052    pub active_obj_name: Option<String>,
1053    /// Guest-KVAs of every `struct bpf_map` belonging to the
1054    /// currently-attached scheduler's loaded BPF object, captured
1055    /// alongside [`Self::active_obj_name`] from the same
1056    /// `*scx_root → struct_ops map → owning bpf_prog → used_maps`
1057    /// walk. The walker enumerates the matched struct_ops prog's
1058    /// `used_maps` array and records each entry's KVA so a
1059    /// downstream filter can identify the active scheduler's maps
1060    /// uniquely — even when two scheduler instances loaded from the
1061    /// SAME binary coexist post-[`crate::scenario::ops::Op::ReplaceScheduler`]
1062    /// (where their bss / data / rodata maps share the
1063    /// `<obj_name>.` prefix and cannot be distinguished by name
1064    /// alone).
1065    ///
1066    /// Empty when:
1067    /// - [`Self::active_obj_name`] is `None` (walker did not
1068    ///   resolve the active obj — same reasons; see that field's
1069    ///   doc).
1070    /// - The matched prog's `used_maps` could not be safely read
1071    ///   (torn race per the kernel's `used_map_cnt` / `used_maps`
1072    ///   pointer publication TOCTOU described at
1073    ///   `monitor/bpf_prog.rs::find_active_struct_ops_obj_no_target`).
1074    /// - The walker ran but the kernel published an empty
1075    ///   `used_maps` for the active prog (no maps registered — an
1076    ///   unusual but legal sched_ext shape).
1077    ///
1078    /// **KVA aliasing caveat:** kernel BPF map allocations are
1079    /// vmalloc/slab-backed; a freed map's KVA can be reassigned to a
1080    /// new allocation across captures. [`crate::scenario::snapshot::Snapshot::active`]
1081    /// combines this set with [`Self::active_obj_name`] (both must
1082    /// match) to reject the aliasing case — a KVA hit whose owning
1083    /// map name does not share the active obj prefix is treated as
1084    /// stale and falls through to the obj-prefix heuristic.
1085    ///
1086    /// **Within-run identity ONLY.** KVAs reflect kernel address
1087    /// space allocation at capture time (subject to KASLR slide).
1088    /// Stable for the life of the map within a single VM run; NOT
1089    /// comparable across runs. Never persist or compare against
1090    /// checked-in baselines.
1091    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1092    pub active_map_kvas: Vec<u64>,
1093    /// Structured per-allocation views from sdt_alloc-backed
1094    /// allocators. One entry per discovered allocator; each carries
1095    /// every live leaf slot (capped at
1096    /// `super::sdt_alloc::MAX_SDT_ALLOC_ENTRIES`) BTF-rendered to
1097    /// named field views. Empty when no scheduler-side allocator
1098    /// could be located, when arena offsets / sdt_alloc offsets are
1099    /// absent, or when the program BTF lacks the `scx_allocator`
1100    /// type (scheduler doesn't link `lib/sdt_alloc.bpf.c`).
1101    ///
1102    /// Populated alongside the page-granular `ArenaSnapshot` in
1103    /// each map: a consumer can read either representation depending
1104    /// on whether they want raw bytes or named-field allocations.
1105    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1106    pub sdt_allocations: Vec<SdtAllocatorSnapshot>,
1107    /// Live `scx_static` bump-allocator regions discovered in
1108    /// `.bss`. One entry per `struct scx_static` instance with
1109    /// `memory != 0` and an in-range `(off, max_alloc_bytes)` pair.
1110    /// Distinct from [`Self::sdt_allocations`]: scx_static is a
1111    /// program-lifetime bump allocator (`lib/sdt_alloc.bpf.c:577`)
1112    /// with no per-allocation header, so the surfaced view is
1113    /// region-granular ranges rather than per-slot named allocations.
1114    /// Empty when the scheduler doesn't link `lib/sdt_alloc.bpf.c`,
1115    /// when the program BTF lacks `struct scx_static`, or when no
1116    /// `scx_static` instance has been initialised at freeze time.
1117    ///
1118    /// The dump pipeline uses the same ranges to populate the
1119    /// renderer's `ScxStaticRangeIndex` so a deferred-resolve arena
1120    /// chase whose target lives inside scx_static memory can
1121    /// fail-closed cleanly (no per-slot type recovery is possible
1122    /// without a per-call-site type hook from cast analysis).
1123    #[serde(
1124        default,
1125        skip_serializing_if = "super::scx_static_alloc::ScxStaticSnapshot::is_empty"
1126    )]
1127    pub scx_static_ranges: super::scx_static_alloc::ScxStaticSnapshot,
1128    /// Diagnostic reason for `sdt_allocations` being empty.
1129    ///
1130    /// - `None` → either the pre-pass ran and produced records (the vec
1131    ///   is non-empty), or the pre-pass ran cleanly but the scheduler
1132    ///   simply has no live allocations (the vec is empty for legitimate
1133    ///   reasons that aren't worth a diagnostic). Default.
1134    /// - `Some(REASON_SDT_ALLOC_*)` → the pre-pass skipped before it
1135    ///   could surface any allocator state. The string identifies which
1136    ///   prerequisite was missing: deadline exhaustion, unaligned
1137    ///   `user_vm_start`, missing scheduler `.bss`, missing arena
1138    ///   snapshot, scheduler BTF without `struct scx_allocator`, or no
1139    ///   `.bss` `scx_allocator` instance.
1140    ///
1141    /// Distinct from `dump_truncated_at_us` (which records deadline
1142    /// truncation across the whole dump) and from
1143    /// [`Self::scx_static_ranges`] (which has its own walker independent
1144    /// of the typed-allocator pre-pass). Mirrors the
1145    /// `prog_runtime_stats_unavailable` / `task_enrichments_unavailable`
1146    /// pattern.
1147    #[serde(default, skip_serializing_if = "Option::is_none")]
1148    pub sdt_alloc_unavailable: Option<String>,
1149    /// Per-program BPF runtime stats summed across CPUs at freeze
1150    /// time (cnt, nsecs, misses). One entry per discovered
1151    /// struct_ops BPF program. Empty when no struct_ops programs are
1152    /// loaded OR when the prog accessor was unavailable to
1153    /// `dump_state` — see [`Self::prog_runtime_stats_unavailable`]
1154    /// for the reason.
1155    ///
1156    /// Per-CPU offset resolution failure does NOT empty the vec —
1157    /// each program still contributes one entry, but with
1158    /// `cnt`/`nsecs`/`misses` summed only over CPUs whose per-CPU
1159    /// `bpf_prog_stats` slot translated successfully (out-of-range
1160    /// CPUs return None per `super::bpf_map::read_percpu_array_value`
1161    /// semantics).
1162    ///
1163    /// See `super::bpf_prog::ProgRuntimeStats` for field semantics
1164    /// and the kernel-source-grounded provenance of each counter.
1165    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1166    pub prog_runtime_stats: Vec<super::bpf_prog::ProgRuntimeStats>,
1167    /// Diagnostic reason for `prog_runtime_stats` being empty.
1168    ///
1169    /// Distinguishes the three causes a consumer can't otherwise tell
1170    /// apart from an empty vec:
1171    /// - `None` (field absent on wire) → vec was populated normally
1172    ///   (or the dump path didn't run). Default.
1173    /// - `Some("no struct_ops programs loaded")` → walker ran, no
1174    ///   struct_ops programs were in `prog_idr` at freeze time.
1175    /// - `Some("prog accessor unavailable")` → caller passed
1176    ///   `prog_capture: None`. Typical causes: `prog_idr` symbol
1177    ///   missing, `BpfProgOffsets` BTF parse failed, or
1178    ///   `__per_cpu_offset` resolution didn't yield non-zero offsets
1179    ///   yet (still-booting guest).
1180    ///
1181    /// Set by `dump_state` only when prog_runtime_stats ends up
1182    /// empty AND a definite cause is identifiable; left None
1183    /// otherwise so the field stays absent in the JSON for
1184    /// already-populated dumps.
1185    #[serde(default, skip_serializing_if = "Option::is_none")]
1186    pub prog_runtime_stats_unavailable: Option<String>,
1187    /// Per-CPU CPU-time / softirq / IRQ counters captured from
1188    /// `kernel_cpustat`, `kernel_stat`, and (under NO_HZ)
1189    /// `tick_sched`. One entry per CPU enumerated by the walker.
1190    /// Empty when the dump caller passed no `CpuTimeCapture` or
1191    /// when symbol/BTF resolution failed.
1192    ///
1193    /// See `PerCpuTimeStats` for field semantics. Surfaces the
1194    /// per-CPU interrupt and idle-time data the failure dump
1195    /// otherwise leaves implicit (the existing scx walker reads
1196    /// `rq->nr_iowait` but not the cumulative time accounting).
1197    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1198    pub per_cpu_time: Vec<PerCpuTimeStats>,
1199    /// Per-cgroup PSI-irq samples for the test's workload cgroups, host-walked
1200    /// from the cgroup hierarchy at this freeze (Phase A). One entry per
1201    /// workload-root leaf cgroup with per-cgroup PSI accounting enabled. Empty
1202    /// when the dump caller passed no `CgroupPsiCapture`, the workload root
1203    /// isn't present yet, or `psi_cgroups_enabled` is off — loud-absent. RAW
1204    /// values; decoded + folded at the metric layer. See
1205    /// `super::cgroup_walk::CgroupPsiStat`.
1206    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1207    pub cgroup_psi: Vec<super::cgroup_walk::CgroupPsiStat>,
1208    /// Per-node NUMA event counters captured from
1209    /// `pglist_data->node_zones[]->vm_numa_event[]`. One row per
1210    /// NUMA node enumerated by the walker. Empty when the live
1211    /// walker has not landed yet (the BTF offsets and wire shape
1212    /// are wired; the reader is a follow-up).
1213    ///
1214    /// See `PerNodeNumaStats` for field semantics; see
1215    /// [`Self::per_node_numa_unavailable`] for the "why empty"
1216    /// diagnostic.
1217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1218    pub per_node_numa: Vec<PerNodeNumaStats>,
1219    /// Diagnostic reason for `per_node_numa` being empty.
1220    /// `None` when the vec was populated normally (or the dump
1221    /// path didn't run); `Some(REASON_NO_NUMA_WALKER)` until the
1222    /// host-side walker lands.
1223    #[serde(default, skip_serializing_if = "Option::is_none")]
1224    pub per_node_numa_unavailable: Option<String>,
1225    /// Per-task failure-dump enrichments — identity (pid, tgid,
1226    /// comm), process tree (group_leader, real_parent, pgid, sid,
1227    /// nr_threads), scheduling (prio family, sched_class name,
1228    /// scx.weight, core_cookie), context-switch counters, watchdog
1229    /// disambiguation flag, and lock-slowpath stack matches.
1230    ///
1231    /// One entry per task the dump path's task walker reaches —
1232    /// today's task walkers are the rq->scx walker and the DSQ
1233    /// walker; both produce task KVAs that get enriched here.
1234    /// Empty when no task walker ran (typical until walker
1235    /// dispatch lands) or when the `TaskEnrichmentCapture` was
1236    /// absent.
1237    ///
1238    /// See `super::task_enrichment::TaskEnrichment` for field
1239    /// semantics; see [`Self::task_enrichments_unavailable`] for the
1240    /// "why empty" diagnostic.
1241    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1242    pub task_enrichments: Vec<super::task_enrichment::TaskEnrichment>,
1243    /// Diagnostic reason for `task_enrichments` being empty.
1244    ///
1245    /// - `None` → vec was populated normally (or the dump path
1246    ///   didn't run).
1247    /// - `Some("no task walker available")` → the
1248    ///   `TaskEnrichmentCapture` was missing from
1249    ///   `DumpContext`. Until DSQ + rq->scx walker dispatch
1250    ///   lands, this is the expected steady state for the dump
1251    ///   pipeline; the offsets + walker library is wired and
1252    ///   ready to populate as soon as a task-list producer hooks
1253    ///   in.
1254    /// - `Some("task walker yielded zero tasks")` → walker
1255    ///   produced no task KVAs (frozen guest with no runnable /
1256    ///   queued scx tasks at the dump instant — possible on a
1257    ///   completely-idle stall trigger).
1258    #[serde(default, skip_serializing_if = "Option::is_none")]
1259    pub task_enrichments_unavailable: Option<String>,
1260    /// Per-monitor-tick SCX_EV_* event counter timeline. Each entry
1261    /// is the cross-CPU sum of the 13 SCX_EV_* counters at one
1262    /// monitor sample. Empty when the dump caller passed no
1263    /// `EventCounterCapture` or no sample reported event counters
1264    /// (event-stat offsets unresolved, scx_root unset). Renderers
1265    /// build sparklines / per-counter delta plots from this vec.
1266    ///
1267    /// See `EventCounterSample` for field semantics; the kernel-
1268    /// source provenance lives on
1269    /// `super::ScxEventCounters` field doc.
1270    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1271    pub event_counter_timeline: Vec<EventCounterSample>,
1272    /// Per-CPU `rq->scx` snapshots — scalar fields the kernel's
1273    /// own `scx_dump_state` reads plus the runnable_list per-task
1274    /// KVAs that fed into the per-task enrichment capture.
1275    /// One entry per CPU walked. Empty when the
1276    /// `ScxWalkerCapture` was absent or every CPU's translate
1277    /// failed.
1278    ///
1279    /// See `super::scx_walker::RqScxState` for field semantics.
1280    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1281    pub rq_scx_states: Vec<super::scx_walker::RqScxState>,
1282    /// Per-DSQ snapshots — local, bypass, global, and user DSQs
1283    /// reachable from `*scx_root`. Each entry carries `nr` (depth),
1284    /// `seq` (BPF-iter counter), and the queued task KVAs.
1285    /// Surfaces data the kernel's own `scx_dump_state` does not
1286    /// emit (per-DSQ depth enumeration), so this vec adds value
1287    /// even on a kernel that prints its own dump.
1288    ///
1289    /// Empty when the `ScxWalkerCapture` was absent.
1290    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1291    pub dsq_states: Vec<super::scx_walker::DsqState>,
1292    /// Top-level `scx_sched` state captured from `*scx_root`:
1293    /// aborting flag, bypass_depth, exit_kind. `None` when no
1294    /// scheduler is attached or `*scx_root` was unreadable.
1295    #[serde(default, skip_serializing_if = "Option::is_none")]
1296    pub scx_sched_state: Option<super::scx_walker::ScxSchedState>,
1297    /// Diagnostic reason for `rq_scx_states` / `dsq_states` /
1298    /// `scx_sched_state` being absent. Mirrors the
1299    /// `prog_runtime_stats_unavailable` / `task_enrichments_unavailable`
1300    /// pattern.
1301    #[serde(default, skip_serializing_if = "Option::is_none")]
1302    pub scx_walker_unavailable: Option<String>,
1303    /// Per-vCPU hardware perf counter snapshot captured at the
1304    /// instant the failure dump fired. One entry per vCPU; index
1305    /// matches vCPU id (0 = BSP, 1..N = APs). `None` per-entry when
1306    /// the freeze-time `read(2)` failed for that vCPU. Empty vec
1307    /// when `DumpContext::perf_capture` was None (perf
1308    /// unavailable on this host) or the read errored wholesale.
1309    ///
1310    /// `exclude_host=1` means each counter ticks only during guest
1311    /// execution; the values here record the cumulative count from
1312    /// the start of the run. Diff against any
1313    /// `super::CpuSnapshot::vcpu_perf` in the monitor timeline to
1314    /// recover the count over a freeze-aligned window. See
1315    /// `super::perf_counters::VcpuPerfSample` for field semantics
1316    /// and the multiplexing math.
1317    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1318    pub vcpu_perf_at_freeze: Vec<Option<super::perf_counters::VcpuPerfSample>>,
1319    /// Microseconds from dump_state entry to the phase that exceeded
1320    /// the soft deadline supplied via `DumpContext::deadline`. `None`
1321    /// when no deadline was supplied, when every phase finished within
1322    /// the deadline, or when the deadline check happened before the
1323    /// dump started any heavy phase. A `Some(us)` value means the dump
1324    /// truncated remaining work (skipped further maps / tasks /
1325    /// walkers) at that elapsed offset to keep the freeze window
1326    /// bounded — the freeze coordinator's parked vCPUs cannot
1327    /// service guest IRQs or MMIO traps while the dump is running,
1328    /// so unbounded dump latency stretches every guest's KVM_RUN
1329    /// pause and risks the freeze rendezvous timeout firing on the
1330    /// next iteration.
1331    #[serde(default, skip_serializing_if = "Option::is_none")]
1332    pub dump_truncated_at_us: Option<u64>,
1333    /// Count of scheduler-under-test maps the per-map render loop
1334    /// skipped because the soft deadline had already been crossed
1335    /// (`dump_truncated_at_us` records WHEN, this records HOW MANY).
1336    /// `0` on a complete dump. A skipped map is absent from `maps`
1337    /// entirely — without this count a consumer reading `maps`
1338    /// cannot tell "the scheduler has N maps" from "the scheduler
1339    /// has N+k maps but k were dropped by truncation", so a
1340    /// degraded dump would silently under-report map state. Excludes
1341    /// ktstr's own framework maps, which are filtered before the
1342    /// deadline check and never counted here.
1343    #[serde(default, skip_serializing_if = "is_zero_u32")]
1344    pub maps_truncated: u32,
1345    /// Probe BPF program's per-CPU diagnostic counter snapshot
1346    /// (see `ProbeBssCounters`). Populated by the host-side
1347    /// reader in `decode_probe_counters_snapshot` which sums
1348    /// each `KTSTR_PCPU_*` slot across CPUs. `None` when the
1349    /// probe `.bss` map isn't enumerated (probe not loaded), the
1350    /// program BTF can't be parsed, or the array's offset doesn't
1351    /// resolve.
1352    ///
1353    /// A populated `trigger_count > 0` is the structural signal
1354    /// that the selected BPF scheduler-exit handler fired during
1355    /// the run — distinct from the boolean `trigger_fired` flag
1356    /// in `super::probe::process::ProbeDiagnostics` (which
1357    /// also records host-side observations like a watchdog
1358    /// teardown). The cross-product is the failure-dump E2E
1359    /// test's structural assertion: a stall scenario must show
1360    /// both flag=true AND `trigger_count > 0`, otherwise the
1361    /// probe attached without firing or fired without the host
1362    /// observing.
1363    #[serde(default, skip_serializing_if = "Option::is_none")]
1364    pub probe_counters: Option<ProbeBssCounters>,
1365    /// Host-side setup-failure evidence embedded when this report is a
1366    /// placeholder written for a pre-capture guest failure (payload
1367    /// never ran). `None` on real captures and on placeholders written
1368    /// where no `VmResult` exists (build/run errors before a VM ran).
1369    #[serde(default, skip_serializing_if = "Option::is_none")]
1370    pub guest_setup_failure: Option<GuestSetupFailureContext>,
1371    /// `true` when this report was produced by
1372    /// [`Self::placeholder`] — i.e. the capture pipeline could
1373    /// not produce real data (typical cause: freeze rendezvous
1374    /// timed out). Periodic-sample temporal assertions skip
1375    /// placeholder reports rather than treating their empty
1376    /// vectors as "no progress" signals; the `*_unavailable`
1377    /// fields carry the reason string for human consumers, but
1378    /// the boolean flag is the machine-checkable discriminant a
1379    /// pattern can branch on without re-deriving placeholder
1380    /// status from the absence of every field.
1381    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
1382    pub is_placeholder: bool,
1383}
1384
1385impl Default for FailureDumpReport {
1386    /// Empty report with `schema = "single"`. Pinning the schema
1387    /// here keeps `FailureDumpReport::default()` and a
1388    /// freshly-constructed `FailureDumpReport { ..., schema:
1389    /// SCHEMA_SINGLE.into(), ... }` indistinguishable to consumers,
1390    /// so the schema discriminant is never quietly missing on a
1391    /// default-built report.
1392    fn default() -> Self {
1393        Self {
1394            schema: SCHEMA_SINGLE.to_string(),
1395            active_map_kvas: Vec::new(),
1396            maps: Vec::new(),
1397            vcpu_regs: Vec::new(),
1398            sdt_allocations: Vec::new(),
1399            scx_static_ranges: super::scx_static_alloc::ScxStaticSnapshot::default(),
1400            sdt_alloc_unavailable: None,
1401            prog_runtime_stats: Vec::new(),
1402            prog_runtime_stats_unavailable: None,
1403            per_cpu_time: Vec::new(),
1404            cgroup_psi: Vec::new(),
1405            per_node_numa: Vec::new(),
1406            per_node_numa_unavailable: None,
1407            task_enrichments: Vec::new(),
1408            task_enrichments_unavailable: None,
1409            event_counter_timeline: Vec::new(),
1410            rq_scx_states: Vec::new(),
1411            dsq_states: Vec::new(),
1412            scx_sched_state: None,
1413            scx_walker_unavailable: None,
1414            vcpu_perf_at_freeze: Vec::new(),
1415            dump_truncated_at_us: None,
1416            maps_truncated: 0,
1417            probe_counters: None,
1418            guest_setup_failure: None,
1419            is_placeholder: false,
1420            active_obj_name: None,
1421        }
1422    }
1423}
1424
1425impl FailureDumpReport {
1426    /// Build a placeholder report for a capture that could not
1427    /// produce real data. Every `*_unavailable` field is set to
1428    /// `Some(reason)` so downstream consumers (`perf-delta`,
1429    /// failure-rendering tooling) can distinguish "capture
1430    /// happened, no data" from "capture path failed for reason X".
1431    /// All vector / option fields stay at their `Default` empty
1432    /// state so the report is structurally a real
1433    /// `FailureDumpReport`, not a sentinel that breaks consumer
1434    /// type contracts.
1435    ///
1436    /// Used by the freeze coordinator's user-watchpoint dispatch,
1437    /// periodic-capture drain, and final-drain teardown — every
1438    /// site that needs to publish a "capture attempted, did not
1439    /// land" entry on the snapshot bridge.
1440    pub fn placeholder(reason: impl Into<String>) -> Self {
1441        let reason = reason.into();
1442        Self {
1443            prog_runtime_stats_unavailable: Some(reason.clone()),
1444            per_node_numa_unavailable: Some(reason.clone()),
1445            task_enrichments_unavailable: Some(reason.clone()),
1446            scx_walker_unavailable: Some(reason.clone()),
1447            sdt_alloc_unavailable: Some(reason),
1448            is_placeholder: true,
1449            ..Self::default()
1450        }
1451    }
1452}
1453
1454/// Identify the obj name (libbpf `<obj>` prefix) of the
1455/// currently-attached scheduler from the captured BPF state, plus
1456/// the live scheduler's `used_maps` KVA set when available.
1457///
1458/// **No `*scx_root` dependency -- works on every supported kernel.**
1459/// `scx_root` (the global pointer to the active `struct scx_sched`)
1460/// only exists on v6.16+ (added by commit 48e126777386); pre-6.16
1461/// kernels track the active scheduler via the global `scx_ops`
1462/// (`struct sched_ext_ops`) plus the atomic enable-state instead, so
1463/// any `*scx_root`-based identification is blind on 6.14/6.15. Both
1464/// paths below read only `prog_idr` / struct_ops map names /
1465/// global-section map names, which are present and BTF-offset-stable
1466/// across the whole range, so this helper resolves the active obj
1467/// uniformly regardless of kernel version.
1468///
1469/// **PRIMARY: target-free `prog_idr` walk.** Delegates to
1470/// `prog_accessor.find_active_struct_ops_obj_no_target` (see
1471/// `monitor::bpf_prog`), which walks `prog_idr` for an alive
1472/// `BPF_PROG_TYPE_STRUCT_OPS` prog whose `aux->used_maps` carries a
1473/// sibling `<obj>.bss/.data/.rodata` global-section map, and returns
1474/// that prog's obj prefix + full `used_map_kvas` snapshot. The
1475/// returned prefix is cross-checked against the captured `maps[]`
1476/// (a `<prefix>.bss/.data/.rodata` must exist) so a torn `used_maps`
1477/// read cannot publish a garbage prefix. The walk takes no target
1478/// and reads live guest memory, so it produces the correct live
1479/// scheduler even mid-swap and on pre-6.16 kernels. Used only when
1480/// the walker returns a NON-EMPTY `used_map_kvas`; an empty
1481/// whitelist cannot disambiguate two same-prefix copies downstream.
1482///
1483/// The walker matches the FIRST such prog and does not gate on the
1484/// kernel enable-state, so uniqueness rests on ktstr's swap
1485/// sequencing (kill the outgoing scheduler and wait for its process
1486/// to exit before loading the next), NOT on a kernel
1487/// old-prog-removed-before-new-prog-added guarantee: the kernel
1488/// does not serialize those (a detached struct_ops prog leaves
1489/// `prog_idr` only when its owning map's last fd closes and an RCU
1490/// grace elapses). See `find_active_struct_ops_obj_no_target` for
1491/// the single-tenant threat model.
1492///
1493/// **FALLBACK: prefix grouping over struct_ops map names.** Runs
1494/// when `prog_walker` is `None` (the prog accessor has not published
1495/// yet) OR the walker found no global-section-bearing prog OR its
1496/// whitelist was empty. Picks the first `BPF_MAP_TYPE_STRUCT_OPS`
1497/// map whose name prefix has an UNAMBIGUOUS global-section sibling
1498/// set (each of `.bss/.data/.rodata` count <= 1) and returns
1499/// `(prefix, vec![])`. Section counts use full-name equality so they
1500/// stay in lockstep with the consumer's classifier at
1501/// [`crate::scenario::snapshot::Snapshot::active`] and the walker's
1502/// `strip_suffix` in
1503/// `monitor::bpf_prog::extract_global_section_obj_prefix` (private
1504/// helper, cited by path rather than intra-doc link). A multi-copy
1505/// prefix (the same-binary `Op::ReplaceScheduler` swap window leaves
1506/// the dying scheduler's globals beside the new scheduler's) is
1507/// skipped here: the empty-whitelist `(prefix, vec![])` cannot
1508/// disambiguate downstream, so the helper instead returns `None` and
1509/// the consumer surfaces an actionable `NoActiveScheduler`.
1510///
1511/// Returns `None` when neither path resolves: the walker was absent
1512/// / found nothing / returned an unbacked or empty result, AND no
1513/// struct_ops map had an unambiguous global-section sibling set. On
1514/// `None`, callers fall back to
1515/// [`crate::scenario::snapshot::Snapshot::active`]'s own
1516/// prefix-grouping, whose per-section-count check enforces the same
1517/// multi-copy detection, so a helper-`None` does not silently
1518/// downgrade into the prefix-only `AmbiguousVar` surface.
1519fn identify_active_obj_from_struct_ops(
1520    maps: &[super::bpf_map::BpfMapInfo],
1521    prog_walker: Option<(
1522        &dyn super::bpf_prog::BpfProgAccessor,
1523        &super::btf_offsets::BpfMapOffsets,
1524    )>,
1525) -> Option<(String, Vec<u64>)> {
1526    // PRIMARY PATH: target-free prog_idr walk. Returns the first
1527    // alive `BPF_PROG_TYPE_STRUCT_OPS` prog whose `aux->used_maps`
1528    // contains a sibling `<obj>.bss/.data/.rodata` global-section map,
1529    // with that prog's obj prefix + full `used_map_kvas` snapshot.
1530    // Reads live guest memory and takes no target, so it resolves the
1531    // live scheduler on every supported kernel -- including pre-6.16,
1532    // where `scx_root` does not exist. In ktstr scenarios only one
1533    // matching prog is alive at a time: ktstr's swap sequence kills
1534    // the outgoing scheduler and waits for its process to exit before
1535    // loading the next, which closes the outgoing struct_ops map's
1536    // last fd and lets the RCU-deferred map+prog free remove the OLD
1537    // prog from `prog_idr`. (The kernel does NOT serialize
1538    // old-prog-removal before new-prog-add; the single-alive-prog
1539    // property is ktstr's sequencing, not a kernel invariant.) The
1540    // returned prefix is cross-checked against the captured `maps[]`
1541    // so a torn `used_maps` read can't publish a garbage prefix.
1542    if let Some((prog_accessor, map_offsets)) = prog_walker
1543        && let Some(walker_match) = prog_accessor.find_active_struct_ops_obj_no_target(map_offsets)
1544        && !walker_match.used_map_kvas.is_empty()
1545    {
1546        // Defense against torn used_maps reads: the walker's returned
1547        // prefix MUST appear as `<prefix>.<section>` in the captured
1548        // `maps[]`. A walker that read garbage from a mid-mutation
1549        // used_maps window would name an obj prefix that no captured
1550        // map matches; the cross-check rejects that case.
1551        let (wb, wd, wr) = count_global_sections_for_prefix(maps, &walker_match.obj_name);
1552        if wb + wd + wr > 0 {
1553            return Some((walker_match.obj_name, walker_match.used_map_kvas));
1554        }
1555    }
1556
1557    // FALLBACK PATH: prefix grouping by struct_ops map name. Runs
1558    // when the prog walker is unavailable (`prog_walker` is `None`,
1559    // e.g. `owned_prog_accessor` hasn't published yet at boot) OR the
1560    // live STRUCT_OPS prog has no global-section maps in its
1561    // used_maps (libbpf-named struct_ops case without a `.bss/.data/
1562    // .rodata` sibling, observed when the scheduler keeps all its
1563    // state in non-libbpf-named maps).
1564    //
1565    // Pick the first STRUCT_OPS map whose prefix has an unambiguous
1566    // global-section sibling set (each section ≤ 1). Multi-copy
1567    // collisions skip -- the consumer surfaces NoActiveScheduler with
1568    // an actionable diagnostic.
1569    for active_struct_ops in maps
1570        .iter()
1571        .filter(|m| m.map_type == super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS)
1572    {
1573        if active_struct_ops.map_kva == 0 {
1574            continue;
1575        }
1576        let so_name = active_struct_ops.name();
1577        let Some(prefix) = so_name.split('.').next().filter(|s| !s.is_empty()) else {
1578            continue;
1579        };
1580        let (bss_count, data_count, rodata_count) = count_global_sections_for_prefix(maps, prefix);
1581        let has_matching_global = bss_count + data_count + rodata_count > 0;
1582        let unambiguous = bss_count <= 1 && data_count <= 1 && rodata_count <= 1;
1583        if has_matching_global && unambiguous {
1584            return Some((prefix.to_string(), Vec::new()));
1585        }
1586    }
1587    None
1588}
1589
1590/// Count captured maps named exactly `<prefix>.bss`, `<prefix>.data`,
1591/// `<prefix>.rodata`. Full-name equality (not prefix matching) so it
1592/// aligns with the consumer's `name.ends_with(".bss"/...)` classifier
1593/// at [`crate::scenario::snapshot::Snapshot::active`] and with the
1594/// walker's `strip_suffix(".bss")` in
1595/// `monitor::bpf_prog::extract_global_section_obj_prefix` (private
1596/// helper, cited by path rather than intra-doc link). A hypothetical
1597/// `<prefix>.bss.shared` map would count as zero here (the walker
1598/// treats it the same way), so the counts stay in lockstep across
1599/// the three sites that classify global-section maps for a scheduler
1600/// obj.
1601///
1602/// Skips `BPF_MAP_TYPE_STRUCT_OPS` maps so the active scheduler's
1603/// own struct_ops map (which `Snapshot::active` filters by type as
1604/// well) never inflates the global-section totals.
1605fn count_global_sections_for_prefix(
1606    maps: &[super::bpf_map::BpfMapInfo],
1607    prefix: &str,
1608) -> (usize, usize, usize) {
1609    let bss_name = format!("{prefix}.bss");
1610    let data_name = format!("{prefix}.data");
1611    let rodata_name = format!("{prefix}.rodata");
1612    maps.iter()
1613        .filter(|m| m.map_type != super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS)
1614        .fold((0usize, 0usize, 0usize), |(b, d, r), m| {
1615            let n = m.name();
1616            (
1617                b + usize::from(n == bss_name),
1618                d + usize::from(n == data_name),
1619                r + usize::from(n == rodata_name),
1620            )
1621        })
1622}
1623
1624/// Pair of failure-dump snapshots captured at two points in a stall.
1625///
1626/// `early` is taken when the host-side runnable_at scanner observes
1627/// any task with `jiffies - p->scx.runnable_at > watchdog_timeout/2`
1628/// (mirrors the kernel's `check_rq_for_timeouts` walk over
1629/// `rq->scx.runnable_list`). `late` is taken at the same trigger as
1630/// the single-snapshot path: the BPF probe's
1631/// `ktstr_err_exit_detected` latch flipping after a sched_ext
1632/// error-class exit.
1633///
1634/// `early == None` when the watchdog half-way threshold never
1635/// triggered before `late` fired (e.g. an immediate scheduler error
1636/// in `init_task` before any task became runnable). Diffing
1637/// `late` against `early` shows what BPF state changed during the
1638/// stall window — the value-add over the single-snapshot dump.
1639///
1640/// **No user toggle — auto-repro engages this automatically.** Only
1641/// the auto-repro VM emits this shape;
1642/// `crate::test_support::probe::attempt_auto_repro` is the
1643/// single call site flipping the builder's `dual_snapshot` flag,
1644/// and there is no public ktstr surface for asking for it from a
1645/// primary VM. Test authors don't need to know about it — when an
1646/// auto-repro fires, the file at
1647/// `<test>-<variant_hash>.repro.failure-dump.json` changes shape from
1648/// [`FailureDumpReport`] to this wrapper.
1649///
1650/// Note: there is no `Default` impl. The `late` field is required
1651/// by the doc invariant ("the freeze coordinator only writes a
1652/// `DualFailureDumpReport` after the late snapshot has been
1653/// captured"); a `Default::default()` would have produced a wrapper
1654/// with an empty late report whose `maps`/`vcpu_regs` vectors
1655/// silently lie about a successful capture. Construct via the
1656/// struct literal with an explicit `late: FailureDumpReport`.
1657#[derive(Debug, Clone, Serialize, Deserialize)]
1658#[non_exhaustive]
1659pub struct DualFailureDumpReport {
1660    /// Wire-format discriminant. Always `"dual"` for this variant,
1661    /// pinning [`SCHEMA_DUAL`]. Mirror of [`FailureDumpReport::schema`]
1662    /// — consumers branch on it before deserializing.
1663    pub schema: String,
1664    /// Snapshot at the watchdog half-way point. `None` when the
1665    /// stall fired before the half-way scanner crossed its threshold.
1666    #[serde(default, skip_serializing_if = "Option::is_none")]
1667    pub early: Option<FailureDumpReport>,
1668    /// Snapshot at the error-exit latch trigger. Always present
1669    /// (the freeze coordinator only writes a `DualFailureDumpReport`
1670    /// after the late snapshot has been captured; if the run ends
1671    /// with only an early snapshot the file is not written at all).
1672    pub late: FailureDumpReport,
1673    /// Maximum `jiffies - p->scx.runnable_at` observed by the
1674    /// runnable_at scanner at the moment the early snapshot fired.
1675    /// Zero when `early` is `None`.
1676    ///
1677    /// To recover the kernel's full `watchdog_timeout`, double
1678    /// [`Self::early_threshold_jiffies`] — the scanner trigger
1679    /// fires at half the watchdog, so the threshold field carries
1680    /// `watchdog_timeout / 2`. Diff `early_max_age_jiffies` against
1681    /// `2 * early_threshold_jiffies` to see how close the system
1682    /// was to the SCX_EXIT_ERROR_STALL emission line at the
1683    /// early-trigger point.
1684    #[serde(default, skip_serializing_if = "is_zero_u64")]
1685    pub early_max_age_jiffies: u64,
1686    /// The half-way trigger threshold the scanner compared against
1687    /// when capturing the early snapshot, expressed in guest
1688    /// jiffies. Equals `(watchdog_timeout_ms * CONFIG_HZ) / 1000 / 2`
1689    /// at the moment the snapshot fired. Zero when `early` is
1690    /// `None`.
1691    ///
1692    /// Surfaced alongside `early_max_age_jiffies` so a downstream
1693    /// consumer reading the JSON does not have to recompute the
1694    /// kernel-internal jiffies arithmetic to reproduce the
1695    /// trigger condition.
1696    #[serde(default, skip_serializing_if = "is_zero_u64")]
1697    pub early_threshold_jiffies: u64,
1698    /// Structured reason the early snapshot is absent. `None` when
1699    /// the early snapshot was captured (the [`Self::early`] field is
1700    /// `Some`). When the early field is `None`, this carries a short
1701    /// machine-friendly string identifying which of the known
1702    /// failure modes occurred:
1703    ///
1704    /// - `"scan prerequisites unavailable: <prereq>"` — the
1705    ///   per-CPU `runnable_at` scan never resolved its dependencies
1706    ///   (most often `<prereq>` names the missing kernel symbol /
1707    ///   BTF entry).
1708    /// - `"max_age never crossed threshold (peak={peak}j,
1709    ///   threshold={threshold}j)"` — the scan ran but the maximum
1710    ///   observed runnable-age stayed below the half-way mark for
1711    ///   the whole VM lifetime. Indicates a non-stall err-class exit
1712    ///   (e.g. `scx_bpf_error()`).
1713    /// - `"scx_tick stall — no per-task runnable_at data"` — the
1714    ///   stall path that drove the late capture has no per-task
1715    ///   `runnable_at` to scan (the kernel's "watchdog failed to
1716    ///   check in" path raises `SCX_EXIT_ERROR_STALL` from the
1717    ///   scx_tick kernel side without any task on
1718    ///   `rq->scx.runnable_list`).
1719    ///
1720    /// Display rendering at `super::display` surfaces this string
1721    /// directly; the previous "stall fired before half-way threshold,
1722    /// or runnable_at scan setup failed" generic text is replaced
1723    /// with the structured reason whenever this field is `Some`.
1724    #[serde(default, skip_serializing_if = "Option::is_none")]
1725    pub early_skipped_reason: Option<String>,
1726}
1727
1728fn is_zero_u64(v: &u64) -> bool {
1729    *v == 0
1730}
1731
1732fn is_zero_u32(v: &u32) -> bool {
1733    *v == 0
1734}
1735
1736/// Top-level degraded failure-dump report. Emitted by the freeze
1737/// coordinator when a real error-class trigger fires but the dump
1738/// path aborts before a full [`FailureDumpReport`] can be captured —
1739/// today only the vCPU rendezvous-timeout path produces this shape.
1740///
1741/// Carries the partial state the coordinator did collect (per-vCPU
1742/// registers from any vCPU that parked before timeout) plus the
1743/// observable trigger state at the moment of degradation
1744/// (watchpoint hit, BPF `.bss` latch status, live `exit_kind` if the
1745/// gate read it). An operator inspecting the JSON learns WHY the
1746/// dump degraded from the `reason` field and WHICH vCPUs stalled
1747/// from the `vcpu_regs` Vec's per-slot `None` / `Some` pattern.
1748///
1749/// Schema discriminant: [`SCHEMA_DEGRADED`]. Parsed via the same
1750/// [`FailureDumpReportAny::from_json`] dispatcher as the other two
1751/// variants.
1752#[derive(Debug, Clone, Serialize, Deserialize)]
1753#[non_exhaustive]
1754pub struct DegradedFailureDumpReport {
1755    /// Wire-format discriminant. Always `"degraded"` for this
1756    /// variant, pinning [`SCHEMA_DEGRADED`]. Mirror of
1757    /// [`FailureDumpReport::schema`] / [`DualFailureDumpReport::schema`]
1758    /// — consumers branch on it before deserializing.
1759    pub schema: String,
1760    /// Operator-readable reason the dump degraded. Carries one of
1761    /// the `REASON_DEGRADED_*` constants as the canonical prefix,
1762    /// followed by dynamic detail filled in at emit time (e.g.
1763    /// timeout milliseconds, parked-vCPU counts). Stable wire format
1764    /// per the [`SCHEMA_DEGRADED`] discriminant contract: new degraded
1765    /// causes add new `REASON_DEGRADED_*` constants rather than
1766    /// mutating the existing ones.
1767    pub reason: String,
1768    /// Per-vCPU register snapshots collected before degradation.
1769    /// Index matches vCPU id (BSP at 0, APs at 1..N). `None` entries
1770    /// identify the vCPUs that never parked (the operator's
1771    /// signal for which vCPUs stalled) — distinct from
1772    /// [`FailureDumpReport::vcpu_regs`]'s `None`, which usually
1773    /// means `KVM_GET_REGS` failed mid-shutdown.
1774    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1775    pub vcpu_regs: Vec<Option<VcpuRegSnapshot>>,
1776    /// Hardware-watchpoint hit state at degradation. `true` when
1777    /// the freeze-coordinator's `*scx_root->exit_kind` watchpoint
1778    /// fired on a vCPU thread; `false` when only the BPF `.bss`
1779    /// latch fired (or the trigger source was a deferred-capture
1780    /// request).
1781    pub watchpoint_hit: bool,
1782    /// BPF probe `.bss` latch state at degradation. One of
1783    /// `"triggered"` (probe latched err exit), `"not_triggered"`
1784    /// (latch readable, value still 0), `"out_of_bounds"` (cached
1785    /// `.bss` PA no longer 4-byte-readable — probe map freed
1786    /// mid-run), or `"not_resolved"` (cached `.bss` PA was never
1787    /// populated). Mirror of `crate::vmm::freeze_coord`'s
1788    /// internal `BssReadState` enum, serialised as the snake-case
1789    /// of each variant. String-typed for wire-format stability with
1790    /// the rest of the `REASON_*` / state-name surface — see
1791    /// [`SCHEMA_DEGRADED`] for the contract.
1792    pub bss_latch_state: String,
1793    /// Live `*scx_root->exit_kind` value at degradation, when the
1794    /// gate read it. `None` when the dump path aborted before
1795    /// reaching the gate (rendezvous timed out earlier) or when the
1796    /// KVA translation failed. `Some(kind)` carries the raw `u32`
1797    /// from `enum scx_exit_kind` — operators read it against
1798    /// the kernel's `scx_exit_kind` enum definition to identify
1799    /// whether the scheduler's intended exit class matched the
1800    /// trigger that fired.
1801    #[serde(default, skip_serializing_if = "Option::is_none")]
1802    pub exit_kind: Option<u32>,
1803    /// Wall-clock milliseconds from the freeze trigger (capture
1804    /// start) to the degraded-emit decision. Lets an operator see
1805    /// how long the coordinator spent trying to capture before
1806    /// giving up. Mirrors the `elapsed_ms` field
1807    /// [`FailureDumpReport`] surfaces via the post-dump log line —
1808    /// here it's structured so consumers can read it without
1809    /// parsing the log.
1810    #[serde(default, skip_serializing_if = "is_zero_u64")]
1811    pub elapsed_ms: u64,
1812}
1813
1814/// Either-or wrapper that owns a parsed [`FailureDumpReport`],
1815/// [`DualFailureDumpReport`], or [`DegradedFailureDumpReport`]. Lets
1816/// a consumer hold and render a failure-dump file without prematurely
1817/// committing to one schema — the discriminant lives in the JSON's
1818/// `schema` field, not in the type the consumer holds.
1819///
1820/// Centralises the schema-tag dispatch logic that previously lived
1821/// inline at every read site (the auto-repro tail renderer, the
1822/// failure-dump-e2e test, any future consumer that wants to inspect
1823/// any shape). Use [`Self::from_json`] to parse an arbitrary
1824/// failure-dump JSON blob; the Display impl forwards to the
1825/// underlying report's existing Display so the rendered output is
1826/// indistinguishable from holding the unwrapped report directly.
1827///
1828/// `non_exhaustive` so a future fourth schema can be added without
1829/// breaking external pattern matches.
1830#[non_exhaustive]
1831pub enum FailureDumpReportAny {
1832    /// Single-snapshot report, schema=`"single"`. Emitted by the
1833    /// primary VM's freeze coordinator when an error-class SCX exit
1834    /// fires.
1835    Single(Box<FailureDumpReport>),
1836    /// Dual-snapshot wrapper, schema=`"dual"`. Emitted by the
1837    /// auto-repro VM when the dual-snapshot path is enabled. Carries
1838    /// optional `early` + required `late` snapshots plus jiffies
1839    /// metadata for the early-trigger condition.
1840    ///
1841    /// Boxed to keep [`FailureDumpReportAny`]'s on-stack size bounded
1842    /// — `DualFailureDumpReport` carries the early+late snapshots
1843    /// inline and is roughly 2x the size of [`FailureDumpReport`].
1844    Dual(Box<DualFailureDumpReport>),
1845    /// Degraded report, schema=`"degraded"`. Emitted when an
1846    /// error-class trigger fires but the dump path aborts before a
1847    /// full single/dual report can be captured — today only the
1848    /// vCPU rendezvous-timeout path produces this shape. Carries
1849    /// partial vCPU register data + trigger-state diagnostics
1850    /// instead of the full map / scx-walker output.
1851    ///
1852    /// Boxed for size parity with the other variants.
1853    Degraded(Box<DegradedFailureDumpReport>),
1854}
1855
1856impl FailureDumpReportAny {
1857    /// Parse a failure-dump JSON blob, choosing the variant by the
1858    /// `schema` field. Returns `None` on any of:
1859    ///
1860    /// - the blob does not parse as JSON
1861    /// - the `schema` field is absent (degraded variant requires an
1862    ///   explicit discriminant; the previous "absent ⇒ single"
1863    ///   fallback would silently mis-route a richer wrapper as a
1864    ///   lossy single shape)
1865    /// - the `schema` field carries an unknown value
1866    /// - the typed deserialisation under the chosen schema fails
1867    pub fn from_json(json: &str) -> Option<Self> {
1868        let value: serde_json::Value = serde_json::from_str(json).ok()?;
1869        let schema = value.get("schema").and_then(|v| v.as_str())?;
1870        match schema {
1871            SCHEMA_DUAL => serde_json::from_str(json)
1872                .ok()
1873                .map(|d| Self::Dual(Box::new(d))),
1874            SCHEMA_SINGLE => serde_json::from_str(json)
1875                .ok()
1876                .map(|r| Self::Single(Box::new(r))),
1877            SCHEMA_DEGRADED => serde_json::from_str(json)
1878                .ok()
1879                .map(|d| Self::Degraded(Box::new(d))),
1880            _ => None,
1881        }
1882    }
1883}
1884
1885/// Rendering of one BPF map's contents.
1886///
1887/// Unifies the map-type rendering paths under a single
1888/// representation: single-entry ARRAY maps (incl. the
1889/// `.bss`/`.data`/`.rodata` global sections) populate `value`;
1890/// multi-entry ARRAY maps populate `array_entries`; keyed HASH maps
1891/// populate `entries`; per-CPU maps populate `percpu_entries`.
1892/// Exactly one of these is non-empty for a successful render; on
1893/// failure `error` is set and the rest empty.
1894#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1895#[non_exhaustive]
1896pub struct FailureDumpMap {
1897    /// Map name as registered with the kernel. Truncated to
1898    /// `BPF_OBJ_NAME_LEN` (16) by the kernel; libbpf composes
1899    /// `"<obj_name>.<section>"` for global-section maps.
1900    pub name: String,
1901    /// Guest-KVA of this map's `struct bpf_map` allocation. Unique
1902    /// per loaded map instance — two map copies sharing the same
1903    /// `name` (e.g. two `<obj>.bss` maps from two scheduler
1904    /// instances loaded from the same binary post-
1905    /// [`crate::scenario::ops::Op::ReplaceScheduler`]) have distinct
1906    /// KVAs and are distinguishable on this field alone.
1907    ///
1908    /// Sourced from [`crate::monitor::bpf_map::BpfMapInfo::map_kva`]
1909    /// at capture time. Within-run stable (the kernel does not
1910    /// relocate `struct bpf_map`); not comparable across runs
1911    /// (KASLR slide differs).
1912    ///
1913    /// `0` when capture did not record a KVA (e.g., synthetic test
1914    /// fixtures constructed via `..Default::default()`); consumers
1915    /// treating `0` as "no kernel identity" gracefully fall back to
1916    /// name-based matching.
1917    #[serde(default, skip_serializing_if = "is_zero_u64")]
1918    pub map_kva: u64,
1919    /// Raw `map_type` from `struct bpf_map` (e.g. `BPF_MAP_TYPE_ARRAY`).
1920    /// Kept as `u32` rather than an enum to avoid bumping a serde
1921    /// schema each time the kernel adds a kind.
1922    pub map_type: u32,
1923    /// Declared per-entry value size. Captured even when rendering
1924    /// fails so the operator can see the map shape.
1925    pub value_size: u32,
1926    /// Declared maximum entry count from `struct bpf_map.max_entries`.
1927    /// Surfaces alongside the rendered slice so a consumer can spot
1928    /// when the dump shows fewer entries than the map declares
1929    /// (e.g. ARRAY / HASH truncated at `MAX_ARRAY_KEYS` /
1930    /// `MAX_HASH_ENTRIES`; PERCPU_ARRAY truncated at
1931    /// `MAX_PERCPU_KEYS`).
1932    pub max_entries: u32,
1933    /// Single-value render for a single-entry ARRAY map
1934    /// (`max_entries <= 1`, incl. the `.bss`/`.data`/`.rodata`
1935    /// global sections). Multi-entry ARRAY maps use `array_entries`.
1936    #[serde(default, skip_serializing_if = "Option::is_none")]
1937    pub value: Option<RenderedValue>,
1938    /// (key, value) entries for HASH maps.
1939    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1940    pub entries: Vec<FailureDumpEntry>,
1941    /// Per-entry values for a multi-entry `BPF_MAP_TYPE_ARRAY` map,
1942    /// indexed by the array key (`u32`). Populated for
1943    /// `max_entries > 1`; the single-entry case uses `value`, so
1944    /// exactly one of `value` / `array_entries` is set for an ARRAY
1945    /// render. Capped at `MAX_ARRAY_KEYS`; truncation and per-key
1946    /// read failures surface in `error`.
1947    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1948    pub array_entries: Vec<FailureDumpArrayEntry>,
1949    /// Per-CPU slots for PERCPU_ARRAY maps. Outer Vec indexed by key,
1950    /// inner Vec indexed by CPU id.
1951    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1952    pub percpu_entries: Vec<FailureDumpPercpuEntry>,
1953    /// Per-key per-CPU slots for `PERCPU_HASH` / `LRU_PERCPU_HASH`
1954    /// maps. Same shape as `percpu_entries` but the outer key is
1955    /// arbitrary bytes (rendered via BTF when a key type id is
1956    /// available, hex otherwise) instead of the implicit u32 key
1957    /// of a per-CPU array.
1958    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1959    pub percpu_hash_entries: Vec<FailureDumpPercpuHashEntry>,
1960    /// Page snapshot for `BPF_MAP_TYPE_ARENA` maps. `None` for all
1961    /// other map types.
1962    #[serde(default, skip_serializing_if = "Option::is_none")]
1963    pub arena: Option<ArenaSnapshot>,
1964    /// Position counters and capacity for `BPF_MAP_TYPE_RINGBUF` /
1965    /// `BPF_MAP_TYPE_USER_RINGBUF` maps. Surfaces stuck-consumer
1966    /// diagnostics — pending bytes far below the watermark plus
1967    /// non-zero `pending_pos` indicates a producer holding a
1968    /// reservation; pending bytes near capacity indicates a stalled
1969    /// consumer. `None` for non-ringbuf maps or when the BTF offsets
1970    /// for `bpf_ringbuf_map` / `bpf_ringbuf` weren't resolvable.
1971    #[serde(default, skip_serializing_if = "Option::is_none")]
1972    pub ringbuf: Option<FailureDumpRingbuf>,
1973    /// Per-bucket trace summary for `BPF_MAP_TYPE_STACK_TRACE` maps.
1974    /// `None` for non-STACK_TRACE maps or when the BTF offsets for
1975    /// `bpf_stack_map` / `stack_map_bucket` weren't resolvable.
1976    #[serde(default, skip_serializing_if = "Option::is_none")]
1977    pub stack_trace: Option<FailureDumpStackTrace>,
1978    /// Populated-slot summary for FD-array families (`PROG_ARRAY`,
1979    /// `PERF_EVENT_ARRAY`, `CGROUP_ARRAY`, `ARRAY_OF_MAPS`,
1980    /// `HASH_OF_MAPS`, `DEVMAP*`, `SOCKMAP*`, `CPUMAP`, `XSKMAP`,
1981    /// `REUSEPORT_SOCKARRAY`). `None` for non-FD-array maps.
1982    #[serde(default, skip_serializing_if = "Option::is_none")]
1983    pub fd_array: Option<FailureDumpFdArray>,
1984    /// Reason this map's contents are missing or partial. Empty on
1985    /// successful render.
1986    #[serde(default, skip_serializing_if = "Option::is_none")]
1987    pub error: Option<String>,
1988}
1989
1990/// Ringbuf occupancy snapshot read from `struct bpf_ringbuf` at the
1991/// freeze instant.
1992///
1993/// Capacity, consumer/producer positions, and the in-flight reservation
1994/// frontier (`pending_pos`) are all that's readable without walking the
1995/// records. Pending bytes (= `producer_pos - consumer_pos`, computed
1996/// with unsigned wraparound) is the operator-visible indicator: low
1997/// values = consumer keeping up; values approaching capacity = consumer
1998/// stuck or kernel producer overrunning. A non-zero gap between
1999/// `producer_pos` and `pending_pos` means a producer is mid-reserve
2000/// and the consumer can't advance past `pending_pos`.
2001///
2002/// Read via `crate::monitor::btf_offsets::BpfRingbufOffsets`; rendered
2003/// in `render_ringbuf_state`.
2004#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2005#[non_exhaustive]
2006pub struct FailureDumpRingbuf {
2007    /// Ring data area capacity in bytes (= `mask + 1`). Always a
2008    /// power of two; matches the map's declared `max_entries`.
2009    pub capacity: u64,
2010    /// Consumer position. Byte index of the next record userspace
2011    /// will read. Monotonically advances; the kernel never writes
2012    /// here.
2013    pub consumer_pos: u64,
2014    /// Producer position. Byte index past the last reserved record.
2015    /// Monotonically advances; updated by the kernel on each
2016    /// `bpf_ringbuf_reserve`.
2017    pub producer_pos: u64,
2018    /// Pending position. Byte index of the oldest in-flight (still
2019    /// being filled) reservation. Records below `pending_pos` are
2020    /// committed and visible to the consumer; records between
2021    /// `pending_pos` and `producer_pos` are reserved but not yet
2022    /// committed.
2023    pub pending_pos: u64,
2024    /// Pending bytes (= `producer_pos.wrapping_sub(consumer_pos)`).
2025    /// 0 = consumer caught up; capacity = ring full / consumer
2026    /// stalled. Computed with unsigned wraparound to match the
2027    /// kernel's dispatch-path arithmetic.
2028    pub pending_bytes: u64,
2029}
2030
2031/// Per-bucket summary of populated stack traces in a STACK_TRACE map.
2032///
2033/// Each `entry` is one populated bucket whose pointer was non-null at
2034/// the freeze instant. `nr` is the number of trace samples (PCs) in
2035/// the bucket; `pcs` carries the actual u64 PC values when readable
2036/// (build-id stacks render the raw bytes hex since the per-entry
2037/// shape is `struct bpf_stack_build_id`, not a u64). The dump caps
2038/// per-bucket entries at `MAX_STACK_TRACE_PCS` to bound memory.
2039#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2040#[non_exhaustive]
2041pub struct FailureDumpStackTrace {
2042    /// `bpf_stack_map.n_buckets` — the rounded-up power-of-two slot
2043    /// count. Iteration upper bound; differs from `max_entries` which
2044    /// the kernel rounds.
2045    pub n_buckets: u32,
2046    /// One entry per non-null bucket pointer. Sorted by bucket id.
2047    pub entries: Vec<FailureDumpStackTraceEntry>,
2048    /// True when any populated bucket was truncated at
2049    /// `MAX_STACK_TRACE_PCS` PCs.
2050    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2051    pub truncated: bool,
2052    /// Count of buckets whose pointer slot or bucket struct could
2053    /// not be translated to a guest physical address at capture
2054    /// time (an unmapped page in the bucket array or a dangling
2055    /// bucket KVA). These buckets are absent from `entries`
2056    /// entirely. `0` when every non-null bucket was readable.
2057    /// Without this count `entries.len()` undercounts the live
2058    /// buckets and the gap reads as "fewer stacks present" rather
2059    /// than "stacks present but unreadable".
2060    #[serde(default, skip_serializing_if = "is_zero_u32")]
2061    pub buckets_unreadable: u32,
2062}
2063
2064/// One populated stack trace from a STACK_TRACE map.
2065#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2066#[non_exhaustive]
2067pub struct FailureDumpStackTraceEntry {
2068    /// Bucket id (= stack ID returned by `bpf_get_stackid`).
2069    pub bucket_id: u32,
2070    /// Number of trace samples (kernel `stack_map_bucket.nr`).
2071    pub nr: u32,
2072    /// PC values (u64) when the map is in non-build-id mode. Empty
2073    /// when `BPF_F_STACK_BUILD_ID` is set on the map (each entry
2074    /// is then a `bpf_stack_build_id` record — its raw bytes land
2075    /// in `data_hex`).
2076    #[serde(default, skip_serializing_if = "Vec::is_empty")]
2077    pub pcs: Vec<u64>,
2078    /// Hex-encoded raw bucket data bytes. Always populated alongside
2079    /// `pcs` so the operator can decode build-id stacks or correlate
2080    /// trace samples with the wire format.
2081    pub data_hex: String,
2082}
2083
2084/// Per-FD-array snapshot of populated indices.
2085///
2086/// FD-array families store `void *` slots in `bpf_array.ptrs`; each
2087/// slot is either NULL (empty) or a kernel pointer (struct bpf_prog *,
2088/// struct file *, etc.). The dump path reads up to
2089/// `MAX_FD_ARRAY_SLOTS` slots, counts non-zero, and lists the
2090/// populated indices.
2091#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2092#[non_exhaustive]
2093pub struct FailureDumpFdArray {
2094    /// Number of populated (non-zero) slots seen.
2095    pub populated: u32,
2096    /// Total slots scanned. Capped at `MAX_FD_ARRAY_SLOTS`.
2097    pub scanned: u32,
2098    /// Indices of populated slots. Truncated to
2099    /// `MAX_FD_ARRAY_INDICES` entries.
2100    #[serde(default, skip_serializing_if = "Vec::is_empty")]
2101    pub indices: Vec<u32>,
2102    /// True when iteration capped at `MAX_FD_ARRAY_SLOTS` and
2103    /// `scanned < max_entries`.
2104    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2105    pub truncated: bool,
2106    /// True when `populated > indices.len()` because
2107    /// `MAX_FD_ARRAY_INDICES` capped the index list.
2108    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
2109    pub indices_truncated: bool,
2110    /// Count of scanned slots whose KVA could not be translated to a
2111    /// guest physical address (an unmapped page in the `ptrs` flex
2112    /// array). These slots are neither confirmed empty nor counted
2113    /// as `populated`, so `populated` is a lower bound when this is
2114    /// non-zero. `0` when every scanned slot was readable.
2115    #[serde(default, skip_serializing_if = "is_zero_u32")]
2116    pub unreadable: u32,
2117}
2118
2119/// One (key, value) pair from a hash map. Both sides are rendered via
2120/// BTF when key/value type ids are available; a `None` rendering
2121/// preserves the raw bytes.
2122#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2123#[non_exhaustive]
2124pub struct FailureDumpEntry {
2125    /// Rendered key. `None` when no BTF type is available for the key.
2126    #[serde(default, skip_serializing_if = "Option::is_none")]
2127    pub key: Option<RenderedValue>,
2128    /// Hex-encoded raw key bytes. Kept alongside `key` so the operator
2129    /// can correlate rendered output with the wire format.
2130    pub key_hex: String,
2131    /// Rendered value. `None` when no BTF type is available.
2132    #[serde(default, skip_serializing_if = "Option::is_none")]
2133    pub value: Option<RenderedValue>,
2134    /// Hex-encoded raw value bytes.
2135    pub value_hex: String,
2136    /// Typed render of the per-entry sdt_alloc payload, when the value
2137    /// carries a `struct sdt_data __arena *` field that points into a
2138    /// captured arena page and a payload type id was discovered for
2139    /// the matching allocator. `None` when the entry carries no arena
2140    /// pointer to chase, no allocator metadata was found, the payload
2141    /// type was ambiguous, or the arena read failed.
2142    ///
2143    /// `value` already renders the surface struct (e.g.
2144    /// `scx_task_map_val { tid, tptr, data: 0x100000... -> sdt_data {
2145    /// tid: { idx, genn } } }`), but `sdt_data.payload[]` is a flex
2146    /// array — BTF reports its size as 0, so the per-task struct that
2147    /// actually lives in the payload bytes never decodes through the
2148    /// surface render. This field carries that decoded payload
2149    /// alongside the surface struct so the operator sees both views
2150    /// at once.
2151    #[serde(default, skip_serializing_if = "Option::is_none")]
2152    pub payload: Option<RenderedValue>,
2153}
2154
2155/// One entry of a multi-entry `BPF_MAP_TYPE_ARRAY` map: the array key
2156/// (`u32`) and its rendered value. Mirrors [`FailureDumpPercpuEntry`]'s
2157/// typed `u32` key (ARRAY keys are kernel-imposed indices, not
2158/// user-typed bytes) but carries a single value rather than a per-CPU
2159/// vector.
2160///
2161/// `value` is `None` only when the entry's guest page was unmapped at
2162/// the freeze instant; a BTF-render miss falls back to
2163/// `RenderedValue::Bytes` (hex), so `None` unambiguously means
2164/// "unreadable", not "un-rendered". ARRAY values are not
2165/// `sdt_data`-arena-chased — no in-tree sched_ext ARRAY stores arena
2166/// pointers; add a `payload` field here if one ever does.
2167#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2168#[non_exhaustive]
2169pub struct FailureDumpArrayEntry {
2170    /// Array index (kernel key).
2171    pub key: u32,
2172    /// Rendered value (BTF when `btf_value_type_id` is non-zero, hex
2173    /// fallback otherwise). `None` when the entry was unreadable.
2174    #[serde(default, skip_serializing_if = "Option::is_none")]
2175    pub value: Option<RenderedValue>,
2176}
2177
2178/// One key from a per-CPU array, with one rendered value per CPU
2179/// (None for CPUs whose per-CPU page was unmapped or out-of-range).
2180#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2181#[non_exhaustive]
2182pub struct FailureDumpPercpuEntry {
2183    pub key: u32,
2184    pub per_cpu: Vec<Option<RenderedValue>>,
2185}
2186
2187/// One key from a `PERCPU_HASH` / `LRU_PERCPU_HASH` map, with one
2188/// rendered value per CPU. Mirrors [`FailureDumpEntry`] for the key
2189/// side (rendered + hex) and [`FailureDumpPercpuEntry`] for the
2190/// per-CPU value vector.
2191#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2192#[non_exhaustive]
2193pub struct FailureDumpPercpuHashEntry {
2194    /// Rendered key. `None` when no BTF type is available for the key.
2195    #[serde(default, skip_serializing_if = "Option::is_none")]
2196    pub key: Option<RenderedValue>,
2197    /// Hex-encoded raw key bytes.
2198    pub key_hex: String,
2199    /// One slot per CPU. `None` when the CPU's per-CPU slot was
2200    /// unmapped or out-of-range; `Some` rendered (BTF when value type
2201    /// id is non-zero) or hex bytes otherwise.
2202    pub per_cpu: Vec<Option<RenderedValue>>,
2203}
2204
2205/// Sanity cap on a single BTF blob read.
2206///
2207/// BPF program BTF is normally <100 KB; vmlinux BTF caps around
2208/// ~10 MB. A bogus `data_size` (corrupted `struct btf`) shouldn't
2209/// pull megabytes of unrelated guest memory into the renderer or the
2210/// freeze coordinator. Shared between [`load_program_btf_kva`] and
2211/// `vmm::load_probe_bss_offset`; defining it here keeps the bound
2212/// in one place so a future tightening doesn't drift between sites.
2213pub(crate) const MAX_BTF_BLOB: usize = 32 * 1024 * 1024;
2214
2215/// Hard cap on the per-task enrichment loop inside [`dump_state`].
2216///
2217/// A hostile or pathologically broken guest can produce a runnable_list
2218/// chain whose length is bounded only by the number of
2219/// `task_struct`s in the kernel — tens of thousands on a busy box.
2220/// Each enrichment call walks task/signal/pid/upid offsets, the
2221/// sched_class registry, and the lock-slowpath stack matcher, so an
2222/// uncapped loop turns the freeze window from milliseconds into
2223/// minutes. 4096 is well above any healthy SCX runnable_list depth
2224/// (the kernel's own watchdog fires long before that many tasks
2225/// queue up) and still bounds the worst-case freeze cost. When the
2226/// cap fires, [`dump_state`] truncates without enriching the tail
2227/// and stamps [`FailureDumpReport::dump_truncated_at_us`] so the
2228/// operator knows to attribute missing tasks to truncation rather
2229/// than walker failure.
2230pub const MAX_ENRICHED_TASKS: usize = 4096;
2231
2232/// Bare-named ktstr framework maps to skip during enumeration.
2233///
2234/// These are declared in `src/bpf/probe.bpf.c` without a libbpf
2235/// `<obj>.<section>` prefix (`SEC(".maps")` declarations like
2236/// `func_meta_map`, `probe_data`, `probe_scratch`, `ktstr_events`);
2237/// the kernel registers them under the bare names listed here.
2238/// They're framework-internal — the user looking at a failure dump
2239/// for their scheduler doesn't care about ktstr's own kprobe
2240/// scratch — so the dump path drops them.
2241///
2242/// The framework's ringbuf is named `ktstr_events` (not `events`)
2243/// so a user scheduler that legitimately names its own ringbuf
2244/// `events` is not silently dropped from the dump.
2245///
2246/// Future ktstr probe additions need to be added here AND the
2247/// matching `<obj_name>.` prefix needs to be in the
2248/// `render_map`-internal starts_with list (see `dump_state`).
2249const KTSTR_INTERNAL_MAPS: &[&str] = &[
2250    "func_meta_map",
2251    "probe_data",
2252    "probe_scratch",
2253    "ktstr_events",
2254];
2255
2256/// All inputs the failure-dump renderer needs, bundled so future
2257/// capture sites (DSQ walker, rq->scx walker, NUMA stats, ...) can
2258/// land as new optional fields without churning every call site.
2259///
2260/// `accessor` is currently the concrete guest-memory backend. The
2261/// trait dispatch claim in [`BpfMapAccessor`]'s module-level doc
2262/// is aspirational: `dump_state` reaches through the accessor for
2263/// map enumeration AND for the sdt_alloc post-pass walk, which
2264/// needs the underlying [`super::guest::GuestKernel`] handle —
2265/// only the guest-memory backend exposes that. When the live-host
2266/// backend lands, sdt_alloc walking will move into a
2267/// backend-specific path and `accessor` here can become
2268/// `&'a dyn BpfMapAccessor`.
2269///
2270/// `arena_offsets` and `prog_capture` are both optional borrows
2271/// (uniform shape): `None` for either disables that
2272/// capture leg without affecting the rest. A scheduler running on
2273/// an older kernel without arena support lands here with
2274/// `arena_offsets: None` and the failure dump renders maps + regs
2275/// without arena pages; a setup where the BpfProgAccessor couldn't
2276/// resolve `prog_idr` lands with `prog_capture: None` and
2277/// `prog_runtime_stats` stays empty.
2278pub struct DumpContext<'a> {
2279    /// BPF map accessor. Concrete guest-memory backend today; see
2280    /// the type-level doc for why this is not `&dyn BpfMapAccessor`.
2281    pub accessor: &'a GuestMemMapAccessor<'a>,
2282    /// Host-resolved vmlinux BTF. The renderer uses it as the base
2283    /// for split-BTF parsing on programs that ship their own type
2284    /// info; it's also the fallback when a map's program BTF can't
2285    /// be loaded.
2286    pub btf: &'a Btf,
2287    /// Guest's `nr_cpu_ids`. Forwarded into per-CPU map rendering
2288    /// so PERCPU_ARRAY readers know how many slots to enumerate.
2289    /// Pass `1` for non-percpu-only dumps if the caller doesn't
2290    /// have the value handy.
2291    pub num_cpus: u32,
2292    /// BTF-resolved arena field offsets. Enables
2293    /// `BPF_MAP_TYPE_ARENA` page snapshotting via the accessor
2294    /// trait's `read_arena_pages`. `None` skips arena rendering
2295    /// (older kernel without arena support, or BTF lacking
2296    /// `struct bpf_arena`).
2297    pub arena_offsets: Option<&'a BpfArenaOffsets>,
2298    /// Per-program runtime stats capture. `None` skips
2299    /// prog-runtime capture; the dump still renders every map the
2300    /// accessor enumerates.
2301    pub prog_capture: Option<&'a ProgRuntimeCapture<'a>>,
2302    /// Per-CPU CPU-time / softirq / IRQ capture. `None` skips the
2303    /// per-CPU time walk; the rest of the dump still renders. Same
2304    /// "borrowed-only, optional" shape as
2305    /// [`Self::prog_capture`] / [`Self::arena_offsets`] so a
2306    /// future capture site lands as another optional field without
2307    /// churning the call sites already plumbed through here.
2308    pub cpu_time_capture: Option<&'a CpuTimeCapture<'a>>,
2309    /// Per-cgroup PSI-irq capture (Phase A). `None` skips the cgroup
2310    /// hierarchy walk; the rest of the dump still renders. Same
2311    /// borrowed-only/optional shape as [`Self::cpu_time_capture`].
2312    pub cgroup_psi_capture: Option<&'a CgroupPsiCapture<'a>>,
2313    /// Per-task enrichment capture. `None` skips the per-task walk
2314    /// and `task_enrichments` stays empty; the rest of the dump
2315    /// still renders.
2316    ///
2317    /// Today's freeze coordinator passes `None` because the DSQ
2318    /// and rq->scx task walkers have not yet landed dispatch. The
2319    /// `TaskEnrichmentOffsets` + `SchedClassRegistry` + the
2320    /// `walk_task_enrichment` library are wired and ready —
2321    /// the producer side just needs to populate
2322    /// [`TaskEnrichmentCapture::tasks`] from the rq->scx walker.
2323    pub task_enrichment_capture: Option<&'a TaskEnrichmentCapture<'a>>,
2324    /// SCX_EV_* event counter timeline capture. `None` skips
2325    /// timeline rendering and `event_counter_timeline` stays
2326    /// empty; the rest of the dump still renders. Same
2327    /// "borrowed-only, optional" shape as
2328    /// [`Self::cpu_time_capture`].
2329    pub event_counter_capture: Option<&'a EventCounterCapture<'a>>,
2330    /// SCX rq->scx + DSQ walker capture. `None` skips the walk;
2331    /// `rq_scx_states` / `dsq_states` / `scx_sched_state` stay
2332    /// empty/None and `scx_walker_unavailable` records why.
2333    pub scx_walker_capture: Option<&'a ScxWalkerCapture<'a>>,
2334    /// Host-side per-vCPU hardware perf counters (cycles,
2335    /// instructions, cache-misses, branch-misses) opened with
2336    /// `exclude_host=1`, so each counter only ticks during guest
2337    /// execution. `None` skips the freeze-time read; the
2338    /// [`FailureDumpReport::vcpu_perf_at_freeze`] vec stays empty.
2339    /// See [`super::perf_counters`] for the kernel-source-grounded
2340    /// rationale and capture semantics.
2341    ///
2342    /// The same capture is shared (via `Arc` in the freeze
2343    /// coordinator) with the per-tick monitor sampler; per-tick
2344    /// samples land on each [`super::CpuSnapshot::vcpu_perf`]. The
2345    /// freeze-time read here records the absolute counter values at
2346    /// the instant the failure dump fired, which lets a consumer
2347    /// diff against any earlier sample to compute IPC over a
2348    /// freeze-aligned window.
2349    pub perf_capture: Option<&'a super::perf_counters::PerfCountersCapture>,
2350    /// Soft deadline for the dump's heavy phases (per-map render
2351    /// loop, walk_rq_scx, walk_local_dsqs, walk_dsqs sched-rooted,
2352    /// walk_task_enrichment, sdt_alloc post-pass). When supplied,
2353    /// each phase boundary checks `Instant::now() > deadline`; the
2354    /// first crossing truncates remaining work and stamps
2355    /// [`FailureDumpReport::dump_truncated_at_us`]. `None` disables
2356    /// the bailout — the dump runs every phase to completion.
2357    ///
2358    /// Set by the freeze coordinator to `capture_start +
2359    /// watchdog_timeout/2` so a slow dump can't keep vCPUs parked
2360    /// past the kernel's own SCX_EXIT_ERROR_STALL emission line. The
2361    /// deadline is a soft bound: each phase that has already started
2362    /// runs to completion before checking, so the actual elapsed
2363    /// time at truncation can exceed the deadline by one phase's
2364    /// worth of work.
2365    pub deadline: Option<std::time::Instant>,
2366    /// BPF cast-analysis output for the scheduler's program object,
2367    /// produced once at builder time by parsing the scheduler
2368    /// binary's `.bpf.objs` ELF blob (no libbpf, no kernel
2369    /// interaction). Threaded into every per-map [`RenderMapCtx`]
2370    /// so the renderer's
2371    /// [`super::btf_render::MemReader::cast_lookup`] can promote
2372    /// `u64` fields the analyzer flagged into typed-pointer
2373    /// renders. `None` skips cast-driven promotion entirely (every
2374    /// `u64` renders as a plain unsigned counter, the
2375    /// pre-integration default); same effect as passing an empty
2376    /// map but cheaper to thread.
2377    pub cast_map: Option<&'a super::cast_analysis::CastMap>,
2378    /// Unique alloc_sizes captured from `scx_static_alloc_internal`
2379    /// call sites. Threaded to the renderer as a last-resort fallback.
2380    pub alloc_size_types: &'a [(u64, String)],
2381    /// Cross-BTF Fwd resolution context: every parsed embedded
2382    /// BPF object's program BTF plus a name-keyed index over
2383    /// every complete struct/union across them. Threaded into
2384    /// every per-map [`RenderMapCtx`] so the renderer's
2385    /// [`super::btf_render::MemReader::cross_btf_resolve_fwd`]
2386    /// can chase a `BTF_KIND_FWD` whose body lives in a sibling
2387    /// embedded object's BTF. Borrowed slices in the
2388    /// `(btfs, fwd_index)` pair point into the
2389    /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] the
2390    /// freeze coordinator holds alive via `Arc` for the dump
2391    /// pass; `None` (no scheduler binary, or analyzer found no
2392    /// complete struct/union definitions) keeps the renderer's
2393    /// "forward declaration; body not in this BTF" skip path
2394    /// intact.
2395    pub cross_btf_fwd_index: Option<CrossBtfFwdIndex<'a>>,
2396}
2397
2398/// Per-dump cross-BTF Fwd resolution context: every parsed program
2399/// BTF the cast-analysis pre-pass discovered, plus a name-keyed
2400/// index over the complete (`!is_fwd`) struct/union definitions
2401/// across them.
2402///
2403/// Built once at the freeze-coordinator side from
2404/// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] and
2405/// threaded through [`DumpContext::cross_btf_fwd_index`] into
2406/// every per-map `AccessorMemReader`. The renderer's
2407/// [`super::btf_render::MemReader::cross_btf_resolve_fwd`]
2408/// override range-looks up the hit and returns a
2409/// [`super::btf_render::CrossBtfRef`] whose `btf` borrow points at
2410/// the matching `Arc<Btf>` inside `btfs`.
2411///
2412/// Empty `btfs` / empty `fwd_index` are valid (no scheduler binary,
2413/// or analyzer found no Struct/Union definitions); the bridge stays
2414/// dormant and the chase falls through to the legacy
2415/// "forward declaration" skip path.
2416pub struct CrossBtfFwdIndex<'a> {
2417    /// Every parsed program BTF in the order
2418    /// `crate::vmm::cast_analysis_load::iter_embedded_bpf_objects`
2419    /// yielded the embedded objects. Index 0 is the first object's
2420    /// BTF, etc. Empty when the scheduler binary had no parseable
2421    /// `.bpf.objs`. Borrowed from the
2422    /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput`] held
2423    /// alive by the freeze coordinator's `Arc` for the dump pass.
2424    pub btfs: &'a [std::sync::Arc<Btf>],
2425    /// `name -> FwdIndexEntry` over every complete struct/union
2426    /// across `btfs`. See
2427    /// [`crate::vmm::cast_analysis_load::CastAnalysisOutput::fwd_index`]
2428    /// for the construction policy (first-write-wins on duplicate
2429    /// names, anonymous types skipped).
2430    pub fwd_index:
2431        &'a std::collections::HashMap<String, crate::vmm::cast_analysis_load::FwdIndexEntry>,
2432}
2433
2434/// Reconstruct an `ScxSchedState` from the probe BPF program's
2435/// `.bss` snapshot (`ktstr_exit_*` vars).
2436///
2437/// Used as a fallback by [`dump_state`] when
2438/// [`super::scx_walker::read_scx_sched_state`] returned `None`
2439/// because `*scx_root == 0` at freeze time. The probe's selected
2440/// scheduler-exit handler captured the same scalars BEFORE kernel teardown
2441/// nulled `scx_root`, so this path produces a coherent view of
2442/// what the scheduler looked like AT THE INSTANT IT ERRORED OUT —
2443/// which is exactly the state an operator wants to debug.
2444///
2445/// Returns `None` when:
2446///   - the probe `.bss` map isn't loaded yet (boot-race window),
2447///   - the probe's program BTF can't be parsed,
2448///   - the snapshot's `ktstr_exit_kind_snap` is still 0 (latch
2449///     never fired this run, so the snapshot is empty defaults),
2450///   - or any individual var lookup / read fails wholesale.
2451///
2452/// Variable names match the probe BPF declarations one-for-one
2453/// (`ktstr_exit_aborting`, `ktstr_exit_bypass_depth`,
2454/// `ktstr_exit_kind_snap`, `ktstr_exit_sched_kva`,
2455/// `ktstr_exit_watchdog_timeout`); each is resolved by name via the
2456/// program-BTF Datasec walk so a future addition / reorder of `.bss`
2457/// vars does not silently misalign offsets.
2458/// Decode the probe BPF program's per-CPU counter array
2459/// (`ktstr_pcpu_counters`) and sum each slot across CPUs.
2460///
2461/// The probe declares `pcpu_counter ktstr_pcpu_counters[MAX_CPUS]
2462/// [KTSTR_PCPU_NR]` in `.bss`; each `pcpu_counter` is a single
2463/// `long` field forced to 128-byte alignment, so each per-CPU slot
2464/// occupies its own cacheline. The host walks each
2465/// `(cpu, slot)` 8-byte slice and sums into a [`ProbeBssCounters`]
2466/// — see the BPF source for the
2467/// `ktstr_pcpu_inc(KTSTR_PCPU_<NAME>)` fire sites.
2468///
2469/// Returns `None` when:
2470///   - the probe `.bss` map isn't loaded yet (boot-race window),
2471///   - the probe's program BTF can't be parsed,
2472///   - the BTF doesn't carry a `ktstr_pcpu_counters` var (probe
2473///     build that pre-dates the per-CPU conversion), or
2474///   - the array's bytes can't be read wholesale.
2475///
2476/// All values use `u64` for wire compatibility; the underlying
2477/// kernel `long` is signed but every fire site only ever
2478/// increments, so a positive cumulative count is the only outcome
2479/// in practice. Negative reads (would indicate guest-memory
2480/// corruption) saturate to 0 via `as u64`.
2481fn decode_probe_counters_snapshot(
2482    accessor: &GuestMemMapAccessor<'_>,
2483    base_btf: &Btf,
2484) -> Option<ProbeBssCounters> {
2485    use super::bpf_map::BpfMapAccessor;
2486
2487    // Slot indices must match `enum ktstr_pcpu_idx` in
2488    // src/bpf/probe.bpf.c. A reorder in the BPF source breaks
2489    // every reader; the explicit constants here keep the slot
2490    // mapping localized and reviewable.
2491    const PCPU_PROBE_COUNT: usize = 0;
2492    const PCPU_KPROBE_RETURNS: usize = 1;
2493    const PCPU_META_MISS: usize = 2;
2494    const PCPU_RINGBUF_DROPS: usize = 3;
2495    const PCPU_TIMELINE_COUNT: usize = 4;
2496    const PCPU_TIMELINE_DROPS: usize = 5;
2497    const PCPU_PI_COUNT: usize = 6;
2498    const PCPU_PI_ORPHAN_FEXITS: usize = 7;
2499    const PCPU_PI_CLASS_CHANGE_COUNT: usize = 8;
2500    const PCPU_PI_DROPS: usize = 9;
2501    const PCPU_LOCK_CONTEND_COUNT: usize = 10;
2502    const PCPU_LOCK_CONTEND_DROPS: usize = 11;
2503    const PCPU_PREEMPT_DISABLE_COUNT: usize = 12;
2504    const PCPU_PREEMPT_ENABLE_COUNT: usize = 13;
2505    const PCPU_TRIGGER_COUNT: usize = 14;
2506    const PCPU_NR: usize = 15;
2507    /// Per-CPU slot stride in bytes — `pcpu_counter` is forced to
2508    /// 128-byte alignment in the BPF source so each slot occupies
2509    /// one cacheline. Mirroring the alignment here keeps the
2510    /// host-side walk in lockstep with the BPF storage layout;
2511    /// any future change to the alignment must update both.
2512    const PCPU_SLOT_STRIDE: usize = 128;
2513    /// Per-CPU dimension. Matches `MAX_CPUS` in `src/bpf/probe.bpf.c`
2514    /// (CPU_MASK + 1 = 256). Walking every CPU slot is cheap (256
2515    /// CPUs × 15 slots × 8 bytes = 30 KB of reads); slots beyond
2516    /// the actual `nr_cpus` are zero-init `.bss` and contribute
2517    /// nothing to the sum.
2518    const MAX_CPUS: usize = 256;
2519
2520    // Locate the probe's `.bss` map. Same suffix the freeze
2521    // coordinator's lazy-discovery path uses (matched by suffix
2522    // to avoid colliding with a scheduler-under-test's own
2523    // `.bss`).
2524    let bss_map = accessor.find_array_map("probe_bp.bss")?;
2525    if bss_map.btf_kva == 0 {
2526        // Probe not yet loaded — accessor enumerated a stub.
2527        return None;
2528    }
2529
2530    // Load the probe's program BTF as split BTF on top of the
2531    // host vmlinux BTF (matches the freeze coordinator's
2532    // load_probe_bss_offset pattern). Failure is silent — the
2533    // dump path stays best-effort and falls through to None so
2534    // the caller leaves `probe_counters` as None rather than
2535    // emitting a misleading partial.
2536    let prog_btf = load_program_btf_kva(accessor, bss_map.btf_kva, base_btf)?;
2537
2538    // Resolve the array's byte offset within the `.bss` Datasec.
2539    // A missing var (e.g. probe build that pre-dates the per-CPU
2540    // conversion) means the snapshot wasn't emitted — bail.
2541    let array_off = super::btf_offsets::resolve_var_offset_in_section(
2542        &prog_btf,
2543        ".bss",
2544        "ktstr_pcpu_counters",
2545    )? as usize;
2546
2547    // Read the entire array as one slab — 256 * 15 * 128 = 480 KiB.
2548    // A single slab read is cheaper than 256 * 17 individual reads
2549    // through the page-walking accessor; the read primitive
2550    // tolerates over-large requests (truncates at the map's
2551    // value_size) so a future MAX_CPUS / PCPU_NR shrink doesn't
2552    // need a coordinated host update.
2553    let total_bytes = MAX_CPUS * PCPU_NR * PCPU_SLOT_STRIDE;
2554    let array_bytes = accessor.read_value(&bss_map, array_off, total_bytes)?;
2555    if array_bytes.len() < total_bytes {
2556        // Short read — the map's value_size bounds were tighter
2557        // than the array's compile-time shape. A future probe
2558        // build that shrinks MAX_CPUS or PCPU_NR is the expected
2559        // case; bail rather than misalign the slot indexing.
2560        return None;
2561    }
2562
2563    // Sum every CPU's slot. Each slot's `long value` lives at
2564    // offset 0 within the cacheline-aligned `pcpu_counter`
2565    // struct, so the per-(cpu, slot) byte offset is
2566    // `(cpu * PCPU_NR + slot) * PCPU_SLOT_STRIDE`.
2567    let sum_slot = |slot: usize| -> u64 {
2568        let mut total: u64 = 0;
2569        for cpu in 0..MAX_CPUS {
2570            let off = (cpu * PCPU_NR + slot) * PCPU_SLOT_STRIDE;
2571            // BPF runs in little-endian byte order on every
2572            // host arch ktstr targets (x86_64, aarch64). A future
2573            // big-endian host would need an arch gate — flagged
2574            // in the probe BPF source's byte-order section.
2575            let mut buf = [0u8; 8];
2576            buf.copy_from_slice(&array_bytes[off..off + 8]);
2577            // The kernel's `long` is signed but counters only
2578            // increment; cast through `i64` then to `u64` to
2579            // saturate any negative value (corruption signal) to 0.
2580            let v = i64::from_le_bytes(buf);
2581            if v > 0 {
2582                total = total.saturating_add(v as u64);
2583            }
2584        }
2585        total
2586    };
2587
2588    Some(ProbeBssCounters {
2589        probe_count: sum_slot(PCPU_PROBE_COUNT),
2590        kprobe_returns: sum_slot(PCPU_KPROBE_RETURNS),
2591        meta_miss: sum_slot(PCPU_META_MISS),
2592        ringbuf_drops: sum_slot(PCPU_RINGBUF_DROPS),
2593        timeline_count: sum_slot(PCPU_TIMELINE_COUNT),
2594        timeline_drops: sum_slot(PCPU_TIMELINE_DROPS),
2595        pi_count: sum_slot(PCPU_PI_COUNT),
2596        pi_orphan_fexits: sum_slot(PCPU_PI_ORPHAN_FEXITS),
2597        pi_class_change_count: sum_slot(PCPU_PI_CLASS_CHANGE_COUNT),
2598        pi_drops: sum_slot(PCPU_PI_DROPS),
2599        lock_contend_count: sum_slot(PCPU_LOCK_CONTEND_COUNT),
2600        lock_contend_drops: sum_slot(PCPU_LOCK_CONTEND_DROPS),
2601        preempt_disable_count: sum_slot(PCPU_PREEMPT_DISABLE_COUNT),
2602        preempt_enable_count: sum_slot(PCPU_PREEMPT_ENABLE_COUNT),
2603        trigger_count: sum_slot(PCPU_TRIGGER_COUNT),
2604    })
2605}
2606
2607/// Return whether the exact probe-counter failure-dump read path is
2608/// fully usable.
2609///
2610/// This intentionally calls [`decode_probe_counters_snapshot`] rather
2611/// than duplicating a shallower map/BTF probe. A true result therefore
2612/// proves accessor adoption, `probe_bp.bss` discovery, split-BTF
2613/// parsing, `ktstr_pcpu_counters` offset resolution, and the same full
2614/// 480-KiB slab read the eventual dump performs. Counter values may
2615/// still be zero before workload execution; this edge describes
2616/// decodability, not activity.
2617pub(crate) fn probe_counters_snapshot_ready(
2618    accessor: &GuestMemMapAccessor<'_>,
2619    base_btf: &Btf,
2620) -> bool {
2621    decode_probe_counters_snapshot(accessor, base_btf).is_some()
2622}
2623
2624fn decode_probe_sched_state_snapshot(
2625    accessor: &GuestMemMapAccessor<'_>,
2626    base_btf: &Btf,
2627) -> Option<super::scx_walker::ScxSchedState> {
2628    use super::bpf_map::BpfMapAccessor;
2629
2630    // Locate the probe's `.bss` map. Same suffix the freeze
2631    // coordinator's lazy-discovery path uses (matched by suffix to
2632    // avoid colliding with a scheduler-under-test's own `.bss`).
2633    let bss_map = accessor.find_array_map("probe_bp.bss")?;
2634    if bss_map.btf_kva == 0 {
2635        // Probe not yet loaded — accessor enumerated a stub. The
2636        // var offsets live in the program BTF the loader hasn't
2637        // attached yet.
2638        return None;
2639    }
2640
2641    // Load the probe's program BTF as a split BTF on top of the
2642    // host vmlinux BTF (matches the freeze coordinator's
2643    // load_probe_bss_offset pattern). Failure is silent — the dump
2644    // path stays best-effort and falls through to None so the
2645    // caller leaves `scx_sched_state` as None rather than emitting
2646    // a misleading partial.
2647    let prog_btf = load_program_btf_kva(accessor, bss_map.btf_kva, base_btf)?;
2648
2649    // Resolve each `ktstr_exit_*` var's byte offset within the
2650    // `.bss` Datasec. A missing var (e.g. probe build that pre-
2651    // dates the snapshot vars) means the snapshot wasn't emitted —
2652    // bail rather than render zero defaults that would alias as
2653    // "scheduler healthy and exited cleanly".
2654    let kind_off = super::btf_offsets::resolve_var_offset_in_section(
2655        &prog_btf,
2656        ".bss",
2657        "ktstr_exit_kind_snap",
2658    )?;
2659    let aborting_off = super::btf_offsets::resolve_var_offset_in_section(
2660        &prog_btf,
2661        ".bss",
2662        "ktstr_exit_aborting",
2663    )?;
2664    let bypass_depth_off = super::btf_offsets::resolve_var_offset_in_section(
2665        &prog_btf,
2666        ".bss",
2667        "ktstr_exit_bypass_depth",
2668    )?;
2669    let sched_kva_off = super::btf_offsets::resolve_var_offset_in_section(
2670        &prog_btf,
2671        ".bss",
2672        "ktstr_exit_sched_kva",
2673    )?;
2674    let watchdog_timeout_off = super::btf_offsets::resolve_var_offset_in_section(
2675        &prog_btf,
2676        ".bss",
2677        "ktstr_exit_watchdog_timeout",
2678    )?;
2679
2680    // Read each var's bytes via the accessor. `.bss` maps have a
2681    // single key (zero) and the value bytes ARE the section bytes,
2682    // so `read_value(map, off, size)` is the read primitive. A
2683    // failed read on any field bails the whole snapshot — partial
2684    // values would mislead the consumer.
2685    let kind_bytes = accessor.read_value(&bss_map, kind_off as usize, 4)?;
2686    let kind = u32::from_le_bytes(kind_bytes.as_slice().try_into().ok()?);
2687
2688    // The snapshot is sticky: `ktstr_exit_kind_snap` stays at 0
2689    // until the selected BPF handler latches an error-class exit. A
2690    // 0 here means the latch never fired — the snapshot vars are
2691    // all at their initial 0/false defaults and the dump should
2692    // honour `*scx_root == 0` as "no scheduler state to surface"
2693    // rather than render a fake healthy-exit ScxSchedState.
2694    if kind == 0 {
2695        return None;
2696    }
2697
2698    let aborting_bytes = accessor.read_value(&bss_map, aborting_off as usize, 1)?;
2699    let aborting = aborting_bytes.first().copied()? != 0;
2700
2701    let bypass_depth_bytes = accessor.read_value(&bss_map, bypass_depth_off as usize, 4)?;
2702    let bypass_depth = i32::from_le_bytes(bypass_depth_bytes.as_slice().try_into().ok()?);
2703
2704    let sched_kva_bytes = accessor.read_value(&bss_map, sched_kva_off as usize, 8)?;
2705    let sched_kva = u64::from_le_bytes(sched_kva_bytes.as_slice().try_into().ok()?);
2706
2707    let watchdog_timeout_bytes = accessor.read_value(&bss_map, watchdog_timeout_off as usize, 8)?;
2708    let watchdog_timeout = u64::from_le_bytes(watchdog_timeout_bytes.as_slice().try_into().ok()?);
2709
2710    Some(super::scx_walker::ScxSchedState {
2711        aborting,
2712        bypass_depth,
2713        exit_kind: kind,
2714        watchdog_timeout: Some(watchdog_timeout),
2715        source: Some(super::scx_walker::SCX_SCHED_STATE_SOURCE_BSS.to_string()),
2716        // `sched_kva == 0` would mean the BPF probe handler ran
2717        // BEFORE a scheduler identity was available. The tracepoint and
2718        // fexit shapes always carry `sch`; the global-era dump shape has no
2719        // per-scheduler object and intentionally leaves this zero. Surface it as
2720        // None so the consumer can distinguish "snapshot data exists
2721        // but no slab address" from "snapshot has the address" via
2722        // a single Option rather than a magic-zero check.
2723        sched_kva: if sched_kva == 0 {
2724            None
2725        } else {
2726            Some(sched_kva)
2727        },
2728    })
2729}
2730
2731/// Snapshot every BPF map visible to the host accessor.
2732///
2733/// The dump is best-effort: a map that fails to render lands in the
2734/// report with `error: Some(...)` rather than aborting the whole walk,
2735/// so a single corrupt map can't blind the operator to the rest of
2736/// the scheduler's state.
2737pub fn dump_state(ctx: DumpContext<'_>) -> FailureDumpReport {
2738    let DumpContext {
2739        accessor,
2740        btf,
2741        num_cpus,
2742        arena_offsets,
2743        prog_capture,
2744        cpu_time_capture,
2745        cgroup_psi_capture,
2746        task_enrichment_capture,
2747        event_counter_capture,
2748        scx_walker_capture,
2749        perf_capture,
2750        deadline,
2751        cast_map,
2752        cross_btf_fwd_index,
2753        alloc_size_types,
2754    } = ctx;
2755    let cross_btf_fwd_index_ref = cross_btf_fwd_index.as_ref();
2756    // Wall-clock origin for per-phase elapsed_us tracing and the
2757    // soft-deadline bailout. Each heavy phase compares
2758    // `Instant::now()` against `deadline` AFTER it finishes, so a
2759    // truncation captures the phase's data before short-circuiting
2760    // the remaining ones (consistent with the doc on
2761    // [`DumpContext::deadline`]).
2762    let dump_start = std::time::Instant::now();
2763    // Tracks the elapsed_us of the first phase to observe a deadline
2764    // crossing. Stamped onto [`FailureDumpReport::dump_truncated_at_us`]
2765    // at the end so the operator can attribute missing maps / tasks /
2766    // walker results to truncation rather than walker failure.
2767    let mut truncated_at_us: Option<u64> = None;
2768    // Helper closure: returns `true` once the deadline (if any) has
2769    // been crossed. Sets `truncated_at_us` on the FIRST crossing so
2770    // the report records WHERE truncation began, not the last phase
2771    // to short-circuit. Idempotent on repeated calls — once stamped,
2772    // every later phase sees the same elapsed_us.
2773    let deadline_exceeded = |truncated_at: &mut Option<u64>| -> bool {
2774        if let Some(deadline) = deadline {
2775            let now = std::time::Instant::now();
2776            if now > deadline {
2777                if truncated_at.is_none() {
2778                    let elapsed_us = dump_start.elapsed().as_micros() as u64;
2779                    *truncated_at = Some(elapsed_us);
2780                    tracing::warn!(
2781                        elapsed_us,
2782                        "dump_state: deadline exceeded, truncating remaining phases"
2783                    );
2784                }
2785                return true;
2786            }
2787        }
2788        false
2789    };
2790    let maps = accessor.maps();
2791    let (prog_runtime_stats, prog_runtime_stats_unavailable) = match prog_capture {
2792        Some(cap) => {
2793            let stats = cap.accessor.struct_ops_runtime_stats(cap.per_cpu_offsets);
2794            let reason = if stats.is_empty() {
2795                Some(REASON_NO_STRUCT_OPS_LOADED.to_string())
2796            } else {
2797                None
2798            };
2799            (stats, reason)
2800        }
2801        None => (
2802            Vec::new(),
2803            Some(REASON_PROG_ACCESSOR_UNAVAILABLE.to_string()),
2804        ),
2805    };
2806    let per_cpu_time = match cpu_time_capture {
2807        Some(cap) => collect_per_cpu_time(cap),
2808        None => Vec::new(),
2809    };
2810    // Per-cgroup PSI-irq for the test's workload leaves (Phase A). Empty
2811    // when no capture was supplied, the workload root isn't present, or
2812    // psi_cgroups is off — loud-absent.
2813    let cgroup_psi = match cgroup_psi_capture {
2814        Some(cap) => collect_cgroup_psi(cap),
2815        None => Vec::new(),
2816    };
2817    let task_enrichment_t0 = std::time::Instant::now();
2818    let (task_enrichments, task_enrichments_unavailable) = match task_enrichment_capture {
2819        Some(cap) => {
2820            // Cap iteration AND Vec capacity at MAX_ENRICHED_TASKS so
2821            // a hostile guest with a corrupt or absurdly long
2822            // runnable_list can't drag the freeze window into the
2823            // tens-of-seconds range.
2824            let total = cap.tasks.len();
2825            let cap_n = total.min(MAX_ENRICHED_TASKS);
2826            let mut enrichments = Vec::with_capacity(cap_n);
2827            for entry in cap.tasks.iter().take(cap_n) {
2828                if let Some(e) = super::task_enrichment::walk_task_enrichment(
2829                    cap.kernel,
2830                    entry.task_kva,
2831                    cap.offsets,
2832                    cap.sched_classes,
2833                    cap.lock_slowpaths,
2834                    entry.is_runnable_in_scx,
2835                    entry.running_pc,
2836                ) {
2837                    enrichments.push(e);
2838                }
2839            }
2840            if total > cap_n {
2841                tracing::warn!(
2842                    cap = MAX_ENRICHED_TASKS,
2843                    total,
2844                    "dump_state task_enrichment: capped at MAX_ENRICHED_TASKS, dropping tail"
2845                );
2846            }
2847            let reason = if enrichments.is_empty() {
2848                tracing::debug!(
2849                    tasks_count = total,
2850                    "dump_state task_enrichment: walker yielded zero entries — \
2851                     scx_tasks list and rq->scx.runnable_list both empty, or every \
2852                     walk_task_enrichment call returned None (translate failures)",
2853                );
2854                Some(REASON_TASK_WALKER_ZERO_TASKS.to_string())
2855            } else {
2856                None
2857            };
2858            (enrichments, reason)
2859        }
2860        None => {
2861            tracing::debug!(
2862                "dump_state task_enrichment: capture is None — \
2863                 freeze coordinator passed no TaskEnrichmentCapture \
2864                 (scx_owned, scx_walker_offsets, or task_enrichment_offsets unresolved)",
2865            );
2866            (Vec::new(), Some(REASON_NO_TASK_WALKER.to_string()))
2867        }
2868    };
2869    tracing::debug!(
2870        elapsed_us = task_enrichment_t0.elapsed().as_micros() as u64,
2871        enriched = task_enrichments.len(),
2872        "dump_state phase: walk_task_enrichment"
2873    );
2874    deadline_exceeded(&mut truncated_at_us);
2875    let event_counter_timeline = match event_counter_capture {
2876        Some(cap) => cap
2877            .samples
2878            .iter()
2879            .filter_map(EventCounterSample::from_monitor_sample)
2880            .collect(),
2881        None => Vec::new(),
2882    };
2883    let (rq_scx_states, dsq_states, scx_sched_state, scx_walker_unavailable) =
2884        match scx_walker_capture {
2885            Some(cap) => {
2886                // Sub-group offsets resolved per kernel struct;
2887                // surface the absent groups in the diagnostic so a
2888                // partial walk announces which passes were skipped.
2889                let missing = cap.offsets.missing_groups();
2890
2891                // 1. Read scalar scx_sched state and recover the
2892                //    sched_pa for the sched-rooted DSQ walker passes.
2893                //    `sched_state` is None when the BTF lacked the
2894                //    `sched` sub-group OR when *scx_root == 0
2895                //    (no scheduler attached) — both surface as a
2896                //    None scx_sched_state in the report. The
2897                //    distinction is encoded in `scx_walker_unavailable`
2898                //    via REASON_SCX_ROOT_NULL.
2899                let (sched_pa_opt, sched_state) = match super::scx_walker::read_scx_sched_state(
2900                    cap.kernel,
2901                    cap.scx_root_kva,
2902                    cap.offsets,
2903                ) {
2904                    Some((sched_kva, state)) => {
2905                        // Translate sched_kva → PA (slab/vmalloc; use
2906                        // translate_any_kva via the GuestKernel handle).
2907                        let mem = cap.kernel.mem();
2908                        let walk = cap.kernel.walk_context();
2909                        let pa = super::idr::translate_any_kva(
2910                            mem,
2911                            walk.cr3_pa,
2912                            walk.page_offset,
2913                            sched_kva,
2914                            walk.l5,
2915                            walk.tcr_el1,
2916                        );
2917                        (pa, Some(state))
2918                    }
2919                    None => {
2920                        // Live read failed — `*scx_root == 0` because
2921                        // the scheduler has already torn down by
2922                        // freeze time. Fall back to the BPF .bss
2923                        // snapshot the probe's selected handler latched
2924                        // at error-exit time. The snapshot is the
2925                        // strict subset of scheduler state the host
2926                        // renderer needs; the sched_pa stays None
2927                        // because the slab page that backed the live
2928                        // `scx_sched` was freed during teardown and
2929                        // the sched-rooted DSQ passes (per-node
2930                        // global, user dsq_hash) cannot reach it any
2931                        // longer. The caller's `unavail` selector
2932                        // below now sees `Some(state)` and skips
2933                        // REASON_SCX_ROOT_NULL — the consumer reads
2934                        // `state.source = "bss_snapshot"` to
2935                        // distinguish snapshot from live.
2936                        let snap = decode_probe_sched_state_snapshot(accessor, btf);
2937                        if snap.is_some() {
2938                            tracing::debug!(
2939                                scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
2940                                "dump_state scx walker: live read returned None; \
2941                                 BPF .bss snapshot fallback populated scx_sched_state \
2942                                 (scheduler torn down before freeze, snapshot \
2943                                 captured at err-exit instant)",
2944                            );
2945                        }
2946                        (None, snap)
2947                    }
2948                };
2949
2950                // 2. Per-CPU rq->scx walk. Per-CPU runs only when the
2951                //    rq + scx_rq + task sub-groups are present;
2952                //    walk_rq_scx returns None to skip otherwise.
2953                let walk_rq_scx_t0 = std::time::Instant::now();
2954                let mut rq_states = Vec::with_capacity(cap.rq_kvas.len());
2955                if !deadline_exceeded(&mut truncated_at_us) {
2956                    for (cpu, (&rq_kva, &rq_pa)) in
2957                        cap.rq_kvas.iter().zip(cap.rq_pas.iter()).enumerate()
2958                    {
2959                        if let Some((state, _entries)) = super::scx_walker::walk_rq_scx(
2960                            cap.kernel,
2961                            cpu as u32,
2962                            rq_kva,
2963                            rq_pa,
2964                            cap.offsets,
2965                        ) {
2966                            rq_states.push(state);
2967                        }
2968                    }
2969                }
2970                tracing::debug!(
2971                    elapsed_us = walk_rq_scx_t0.elapsed().as_micros() as u64,
2972                    cpus = cap.rq_kvas.len(),
2973                    rq_states = rq_states.len(),
2974                    "dump_state phase: walk_rq_scx"
2975                );
2976
2977                // 3. Per-CPU local DSQ walk runs unconditionally —
2978                //    `rq->scx.local_dsq` is initialized at boot
2979                //    (init_dsq from kernel/sched/ext.c:4581 for every
2980                //    possible CPU) and survives scheduler teardown,
2981                //    so it produces data even when *scx_root is NULL.
2982                //    This is the data source that survives
2983                //    scx_bypass's runnable_list drain
2984                //    (kernel/sched/ext.c:5448-5548) during teardown.
2985                let walk_local_dsqs_t0 = std::time::Instant::now();
2986                let mut dsqs: Vec<super::scx_walker::DsqState> = Vec::new();
2987                if !deadline_exceeded(&mut truncated_at_us)
2988                    && let Some((local_states, _entries)) = super::scx_walker::walk_local_dsqs(
2989                        cap.kernel,
2990                        cap.rq_kvas,
2991                        cap.rq_pas,
2992                        cap.per_cpu_offsets,
2993                        cap.offsets,
2994                    )
2995                {
2996                    dsqs.extend(local_states);
2997                }
2998                tracing::debug!(
2999                    elapsed_us = walk_local_dsqs_t0.elapsed().as_micros() as u64,
3000                    local_dsqs = dsqs.len(),
3001                    "dump_state phase: walk_local_dsqs"
3002                );
3003
3004                // 4. Sched-rooted DSQ passes (per-CPU bypass, per-node
3005                //    global, user dsq_hash) require the sched_pa we
3006                //    resolved in step 1. Without it, no scheduler is
3007                //    attached and these DSQs don't exist at all.
3008                let walk_dsqs_t0 = std::time::Instant::now();
3009                if !deadline_exceeded(&mut truncated_at_us)
3010                    && let Some(sched_pa) = sched_pa_opt
3011                {
3012                    let (sched_states, _entries) = super::scx_walker::walk_dsqs(
3013                        cap.kernel,
3014                        sched_pa,
3015                        cap.per_cpu_offsets,
3016                        cap.nr_nodes,
3017                        cap.offsets,
3018                    );
3019                    dsqs.extend(sched_states);
3020                }
3021                tracing::debug!(
3022                    elapsed_us = walk_dsqs_t0.elapsed().as_micros() as u64,
3023                    total_dsqs = dsqs.len(),
3024                    "dump_state phase: walk_dsqs"
3025                );
3026
3027                // Diagnostic priority:
3028                //   1. Partial-degradation (sub-group(s) missing) —
3029                //      announces exactly which passes were skipped.
3030                //   2. *scx_root is NULL — sched/bypass/global/user
3031                //      passes blinded but rq->scx + local DSQ still
3032                //      work; surface this distinct reason so the
3033                //      operator knows the scheduler isn't attached.
3034                //   3. Walker reached no state at all — typical when
3035                //      every read fails.
3036                //   4. None — every pass had data to surface.
3037                let unavail = if !missing.is_empty() {
3038                    tracing::debug!(
3039                        missing_groups = ?missing,
3040                        rq_states_count = rq_states.len(),
3041                        dsq_count = dsqs.len(),
3042                        sched_state_some = sched_state.is_some(),
3043                        "dump_state scx walker: partial degradation — missing BTF sub-groups",
3044                    );
3045                    Some(format!(
3046                        "scx walker partial: missing offset groups [{}]",
3047                        missing.join(", ")
3048                    ))
3049                } else if sched_state.is_none() {
3050                    tracing::debug!(
3051                        scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
3052                        rq_states_count = rq_states.len(),
3053                        dsq_count = dsqs.len(),
3054                        "dump_state scx walker: scx_root is NULL — no scheduler attached; \
3055                         rq->scx and local DSQ captures populated, sched/bypass/global/user passes blinded",
3056                    );
3057                    Some(REASON_SCX_ROOT_NULL.to_string())
3058                } else if rq_states.is_empty() && dsqs.is_empty() {
3059                    tracing::debug!(
3060                        scx_root_kva = format_args!("{:#x}", cap.scx_root_kva),
3061                        "dump_state scx walker: every walker read failed — no rq->scx, no DSQ, but sched_state present",
3062                    );
3063                    Some(REASON_SCX_WALKER_NO_STATE.to_string())
3064                } else {
3065                    None
3066                };
3067                (rq_states, dsqs, sched_state, unavail)
3068            }
3069            None => {
3070                tracing::debug!(
3071                    "dump_state scx walker: capture is None — \
3072                     freeze coordinator passed no ScxWalkerCapture (offsets/symbols/per_cpu_offsets unresolved)",
3073                );
3074                (
3075                    Vec::new(),
3076                    Vec::new(),
3077                    None,
3078                    Some(REASON_NO_SCX_WALKER.to_string()),
3079                )
3080            }
3081        };
3082    // Freeze-time per-vCPU perf-counter snapshot. With `exclude_host=1`
3083    // each counter ticks only during guest execution; the freeze
3084    // coordinator has parked every vCPU before reaching this site, so
3085    // the read returns the cumulative count at the last guest exit
3086    // for each vCPU. A single per-vCPU read failure is recorded as
3087    // `None` for that entry; a failure on one vCPU does not blank the
3088    // others. When `perf_capture` is None the vec stays empty (the
3089    // host lacked perf, or `perf_event_open` failed at run start).
3090    let vcpu_perf_at_freeze: Vec<Option<super::perf_counters::VcpuPerfSample>> = match perf_capture
3091    {
3092        Some(cap) => cap.per_vcpu.iter().map(|p| p.read().ok()).collect(),
3093        None => Vec::new(),
3094    };
3095
3096    // Snapshot the probe's per-CPU diagnostic counters before the
3097    // per-map render loop walks `.bss` itself — the read goes
3098    // through the same `read_value` path the renderer uses, but
3099    // captures the array as a structured `ProbeBssCounters` rather
3100    // than the BTF Datasec render. Best-effort: a None result
3101    // (probe not loaded, BTF missing the var) leaves the report's
3102    // `probe_counters` empty and the existing `.bss` map render
3103    // still surfaces the raw bytes.
3104    let probe_counters = decode_probe_counters_snapshot(accessor, btf);
3105
3106    // Resolve the active-scheduler obj name. The prog accessor +
3107    // BpfMapOffsets pair powers the target-free `prog_idr` walker
3108    // (the primary path; needs no `scx_root`, so it works on
3109    // pre-6.16 kernels too) -- when absent, the helper degrades to the
3110    // prefix-grouping fallback over struct_ops map names.
3111    let prog_walker = prog_capture.map(|cap| {
3112        (
3113            cap.accessor as &dyn super::bpf_prog::BpfProgAccessor,
3114            accessor.offsets(),
3115        )
3116    });
3117    let (active_obj_name, active_map_kvas) =
3118        match identify_active_obj_from_struct_ops(&maps, prog_walker) {
3119            Some((name, kvas)) => (Some(name), kvas),
3120            None => (None, Vec::new()),
3121        };
3122    let mut report = FailureDumpReport {
3123        active_map_kvas,
3124        maps: Vec::with_capacity(maps.len()),
3125        prog_runtime_stats,
3126        prog_runtime_stats_unavailable,
3127        per_cpu_time,
3128        cgroup_psi,
3129        // Per-node NUMA wire fields: empty Vec + the well-defined
3130        // diagnostic string until the host-side walker lands.
3131        per_node_numa: Vec::new(),
3132        per_node_numa_unavailable: Some(REASON_NO_NUMA_WALKER.to_string()),
3133        task_enrichments,
3134        task_enrichments_unavailable,
3135        event_counter_timeline,
3136        rq_scx_states,
3137        dsq_states,
3138        scx_sched_state,
3139        scx_walker_unavailable,
3140        vcpu_perf_at_freeze,
3141        probe_counters,
3142        active_obj_name,
3143        ..Default::default()
3144    };
3145
3146    // Per-map program-BTF cache, keyed by `btf_kva`. Each unique
3147    // `struct btf *` lives in the kernel BTF IDR — multiple maps from
3148    // the same BPF program point at the same KVA, so caching dedupes
3149    // the heavy `Btf::from_bytes`/`from_split_bytes` parse across them
3150    // (a scheduler with N maps backed by one BPF object pays one
3151    // parse, not N). Lookups go through this cache before falling
3152    // back to the caller-supplied vmlinux `btf`.
3153    //
3154    // Populated by an explicit pre-pass below so the sdt_alloc walk
3155    // can read it before the per-map render loop runs (the renderer
3156    // needs the resulting allocator metadata via `RenderMapCtx`).
3157    let mut program_btfs: std::collections::HashMap<u64, Btf> = std::collections::HashMap::new();
3158
3159    // Pre-pass: locate the first non-internal `BPF_MAP_TYPE_ARENA`
3160    // map (skipping the same ktstr-internal name set the main loop
3161    // skips) and snapshot it once before any map renders. This lets
3162    // the per-map `MemReader` chase `__arena` pointers no matter
3163    // which slot the arena map occupies in the iteration order —
3164    // the previous design ran `snapshot_arena` lazily inside
3165    // `render_map`'s arena arm, so non-arena maps that rendered
3166    // earlier saw `arena_snapshot: None` and silently failed every
3167    // arena pointer chase. `lib/arena_map.h` declares one `__weak`
3168    // arena per BPF object so a single shared snapshot covers every
3169    // `__arena` pointer the scheduler emits; additional arena maps
3170    // (multi-object schedulers, theoretical) still get their own
3171    // snapshot inside `render_map`'s arena arm — they just don't
3172    // contribute to the cross-map pointer-chase context.
3173    let shared_arena_snapshot: Option<(BpfMapInfo, ArenaSnapshot)> =
3174        arena_offsets.and_then(|off| {
3175            for info in &maps {
3176                let name = info.name();
3177                if name.starts_with("probe_bp.")
3178                    || name.starts_with("fentry_p.")
3179                    || name == "probe_bp"
3180                    || name == "fentry_p"
3181                    || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
3182                {
3183                    continue;
3184                }
3185                if info.map_type == BPF_MAP_TYPE_ARENA {
3186                    let snap = snapshot_arena(accessor.kernel(), info, off);
3187                    return Some((info.clone(), snap));
3188                }
3189            }
3190            None
3191        });
3192    let shared_arena_ref: Option<(&ArenaSnapshot, u64)> = shared_arena_snapshot
3193        .as_ref()
3194        .map(|(info, snap)| (snap, info.map_kva));
3195
3196    // Cache `kern_vm_start` from the pre-pass snapshot for the
3197    // sdt_alloc walk. Pulling directly from `shared_arena_snapshot`
3198    // (rather than scraping each rendered map's `arena` field in the
3199    // main loop) keeps the walk gating decoupled from per-map render
3200    // order — the data the walker needs is finalized before the
3201    // loop runs.
3202    let arena_kern_vm_start: u64 = shared_arena_snapshot
3203        .as_ref()
3204        .map(|(_, snap)| snap.kern_vm_start)
3205        .unwrap_or(0);
3206
3207    // Pre-pass: load every non-internal map's program BTF and locate
3208    // the scheduler's `.bss` raw bytes. Both inputs feed the
3209    // sdt_alloc walk below — moving them out of the main render loop
3210    // means the allocator metadata that decoration in the
3211    // TASK_STORAGE arm needs (`elem_size`, `target_type_id`) is
3212    // available BEFORE any map renders, instead of getting derived
3213    // post-loop only to be unusable for per-entry payload chase.
3214    let mut sched_bss_bytes: Option<(Vec<u8>, u64)> = None; // (bytes, btf_kva)
3215    for info in &maps {
3216        let name = info.name();
3217        if name.starts_with("probe_bp.")
3218            || name.starts_with("fentry_p.")
3219            || name == "probe_bp"
3220            || name == "fentry_p"
3221            || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
3222        {
3223            continue;
3224        }
3225        if info.btf_kva != 0
3226            && !program_btfs.contains_key(&info.btf_kva)
3227            && let Some(loaded) = accessor.load_program_btf(info, btf)
3228        {
3229            program_btfs.insert(info.btf_kva, loaded);
3230        }
3231        if sched_bss_bytes.is_none()
3232            && info.map_type == BPF_MAP_TYPE_ARRAY
3233            && info.btf_kva != 0
3234            && name.ends_with(".bss")
3235            && let Some(bytes) = accessor.read_value(info, 0, info.value_size as usize)
3236        {
3237            sched_bss_bytes = Some((bytes, info.btf_kva));
3238        }
3239    }
3240
3241    // Pre-pass: walk sdt_alloc trees if all prerequisites lined up.
3242    // Runs BEFORE the main render loop so the allocator metadata it
3243    // discovers (`elem_size`, `target_type_id`,
3244    // `data_header_size`) is available to per-map decoration —
3245    // specifically, the TASK_STORAGE arm uses it to expand each
3246    // entry's `struct sdt_data __arena *` pointer into a typed
3247    // payload render via [`render_map`]'s
3248    // [`crate::monitor::dump::render_map::SdtAllocMeta`].
3249    //
3250    // The walk is best-effort and silent: any missing prerequisite
3251    // (no scheduler .bss, no arena window, no program BTF, no
3252    // `scx_allocator` type) leaves `sdt_allocations` empty rather
3253    // than failing the dump. `sdt_alloc_metas` stays empty in the
3254    // same cases, so each per-entry payload field also degrades to
3255    // `None` (the surface struct still renders).
3256    //
3257    // Build the dump-pass arena page index here too — once outside
3258    // the per-map loop so each per-map `mem_reader` borrows the
3259    // existing table instead of rebuilding it. The sdt_alloc walk
3260    // below uses the same index for its own MemReader.
3261    let arena_page_index = crate::monitor::dump::render_map::build_arena_page_index(
3262        shared_arena_snapshot.as_ref().map(|(_, snap)| snap),
3263    );
3264    let sdt_alloc_t0 = std::time::Instant::now();
3265    // Every typed allocator the program declares; the per-map
3266    // selector in [`render_map`] picks the matching entry by name
3267    // (e.g. `scx_task_allocator` matches `scx_task_map`). A
3268    // single-allocator scheduler hits the unique-candidate path —
3269    // every map gets that allocator. A multi-allocator scheduler
3270    // (per-task + per-cgroup) lets each local-storage map render
3271    // its own payload type instead of forcing the renderer to give
3272    // up.
3273    let mut sdt_alloc_metas: Vec<crate::monitor::dump::render_map::SdtAllocMeta> = Vec::new();
3274    // `slot_start → ArenaSlotInfo` lookup populated as each
3275    // allocator walk completes.
3276    // [`MemReader::resolve_arena_type`] consults this index via a
3277    // range lookup: given a chased address, find the slot whose
3278    // `[slot_start, slot_start + elem_size)` range contains it,
3279    // then route on `offset_in_slot`:
3280    //
3281    //   - `offset_in_slot == 0` (slot-start pointer, e.g. the
3282    //     `data` field of `scx_task_map_val` storing the raw
3283    //     `sdt_alloc()` return) → render the payload skipping
3284    //     `header_size` bytes of header.
3285    //   - `offset_in_slot == header_size` (payload-start pointer,
3286    //     e.g. the return of `scx_task_data(p)` cached in
3287    //     `cached_taskc_raw`) → render the payload directly.
3288    //   - Other in-slot offsets → no resolve; the renderer falls
3289    //     back to its existing skip behaviour.
3290    //
3291    // Built incrementally inside the walk loop so the
3292    // per-allocator snapshot moves into `report.sdt_allocations`
3293    // after each iteration without a clone.
3294    //
3295    // [`crate::monitor::sdt_alloc::TreeWalker::emit_leaf`]
3296    // populates each [`SdtAllocEntry::user_addr`] as
3297    // `data_ptr & 0xFFFF_FFFF` — the slot-START address windowed
3298    // to the low 32 bits. The index keys directly on this
3299    // windowed slot start, paired with an [`ArenaSlotInfo`] that
3300    // carries `elem_size`, `header_size`, and the payload BTF
3301    // type id so the [`MemReader::resolve_arena_type`] range
3302    // lookup has every value it needs to decide the chase shape.
3303    //
3304    // Slot non-overlap invariant: the kernel allocator places
3305    // slots back-to-back inside one `sdt_chunk` and never re-uses
3306    // a position while the bitmap still has it marked allocated
3307    // (see `lib/sdt_alloc.bpf.c::scx_alloc_internal`'s
3308    // bitmap-then-data ordering). Two distinct slots cannot have
3309    // overlapping `[start, start + elem_size)` ranges, so
3310    // dedup-on-exact-key here is sufficient — we cannot land on a
3311    // case where `slot_a + elem_a > slot_b > slot_a` with
3312    // `slot_b` separately keyed.
3313    //
3314    // Duplicates (two slots reporting the same slot start,
3315    // indicating a stale snapshot from a freed allocation racing
3316    // with the freeze) keep the FIRST entry; this matches the
3317    // [`build_arena_page_index`] policy on duplicate user_addr
3318    // pages and emits a `tracing::warn!` line so an operator
3319    // diagnosing a wrong-render can spot the collision.
3320    let mut arena_slot_index = crate::monitor::dump::render_map::ArenaSlotIndex::new();
3321    // 4 GiB-alignment invariant: the bridge keys on the low 32
3322    // bits of slot start. That is correct iff `user_vm_start` is
3323    // 4 GiB-aligned — `slot_full_addr - slot_low32 == user_vm_start`
3324    // and the renderer reconstructs full addresses by masking
3325    // chased values with `0xFFFF_FFFF`. Every in-tree scx scheduler
3326    // sets `map_extra` to a 4 GiB-aligned value (`1 << 32`,
3327    // `1 << 44`); the kernel auto-pick path in
3328    // `bpf_arena_map_alloc` (kernel/bpf/arena.c) rounds the user
3329    // VM area up to `SZ_4G` before mounting. The kernel does
3330    // accept arbitrary `map_extra` from userspace, so an
3331    // out-of-tree scheduler could in theory pass an unaligned
3332    // value — surface a warning and skip the index build rather
3333    // than silently misroute every chase.
3334    let user_vm_aligned = shared_arena_snapshot
3335        .as_ref()
3336        .map(|(_, snap)| snap.user_vm_start & 0xFFFF_FFFF == 0)
3337        .unwrap_or(false);
3338    if !user_vm_aligned && let Some((_, snap)) = shared_arena_snapshot.as_ref() {
3339        tracing::warn!(
3340            user_vm_start = format_args!("{:#x}", snap.user_vm_start),
3341            "sdt_alloc bridge skipped: user_vm_start is not 4 GiB-aligned; \
3342             low-32 keying would misroute every chase",
3343        );
3344    }
3345    // `user_vm_start == 0` is technically 4 GiB-aligned (and the
3346    // gate above accepts it), but the
3347    // [`super::dump::render_map::is_arena_addr_in_snapshot`] helper
3348    // rejects every address when `user_vm_start == 0` — silently
3349    // disabling the bridge. Surface a warn so an operator
3350    // diagnosing missing typed-pointer renders sees the cause
3351    // (likely a snapshot capture failure that produced an
3352    // uninitialized arena VM start).
3353    if let Some((_, snap)) = shared_arena_snapshot.as_ref()
3354        && snap.user_vm_start == 0
3355    {
3356        tracing::warn!(
3357            "sdt_alloc bridge effectively disabled: user_vm_start == 0 \
3358             (snapshot capture may have failed before resolving the \
3359             arena's user VM window); every chase will skip with \
3360             `is_arena_addr` = false",
3361        );
3362    }
3363    // Resolve the unavailable reason in the order the gate checks
3364    // run. The first failing prerequisite wins — subsequent reasons
3365    // (which would all surface for the same missing prerequisite) are
3366    // suppressed. `None` here means the pre-pass actually runs; the
3367    // sdt_alloc_unavailable field is then populated post-loop based
3368    // on whether any allocator was discovered.
3369    let sdt_alloc_skip_reason: Option<&'static str> = if deadline_exceeded(&mut truncated_at_us) {
3370        Some(REASON_SDT_ALLOC_DEADLINE_EXCEEDED)
3371    } else if !user_vm_aligned {
3372        Some(REASON_SDT_ALLOC_UNALIGNED_USER_VM)
3373    } else if sched_bss_bytes.is_none() {
3374        Some(REASON_SDT_ALLOC_NO_BSS)
3375    } else if arena_kern_vm_start == 0 {
3376        Some(REASON_SDT_ALLOC_NO_ARENA)
3377    } else if let Some((_, btf_kva)) = sched_bss_bytes.as_ref()
3378        && !program_btfs.contains_key(btf_kva)
3379    {
3380        Some(REASON_SDT_ALLOC_NO_BSS)
3381    } else if let Some((_, btf_kva)) = sched_bss_bytes.as_ref()
3382        && let Some(prog_btf) = program_btfs.get(btf_kva)
3383        && SdtAllocOffsets::from_btf(prog_btf).is_err()
3384    {
3385        Some(REASON_SDT_ALLOC_NO_TYPE)
3386    } else {
3387        None
3388    };
3389    if let Some(reason) = sdt_alloc_skip_reason {
3390        report.sdt_alloc_unavailable = Some(reason.to_string());
3391    }
3392    // Track whether the pre-pass body ran (every prerequisite
3393    // satisfied). Distinct from `sdt_alloc_skip_reason`: if the
3394    // body runs but discovers no `.bss` instance of
3395    // `struct scx_allocator`, the unavailable reason flips to
3396    // [`REASON_SDT_ALLOC_NO_INSTANCE`] AFTER the loop.
3397    let mut sdt_alloc_pre_pass_ran = false;
3398    if !deadline_exceeded(&mut truncated_at_us)
3399        && user_vm_aligned
3400        && let Some((bss_bytes, btf_kva)) = sched_bss_bytes
3401        && arena_kern_vm_start != 0
3402        && let Some(prog_btf) = program_btfs.get(&btf_kva)
3403        && let Ok(sdt_offsets) = SdtAllocOffsets::from_btf(prog_btf)
3404    {
3405        sdt_alloc_pre_pass_ran = true;
3406        // One MemReader for every leaf payload render, so an
3407        // arena pointer embedded in a per-task / per-cgroup
3408        // sdt_alloc payload chases into typed contents instead
3409        // of opaque hex.
3410        //
3411        // The arena type index is intentionally `None` on this
3412        // pre-pass reader: the walk produces the entries the
3413        // index is built from, so the index does not yet exist
3414        // when the leaf payload renders run. A nested `__arena
3415        // *` pointer inside a payload that targets a separate
3416        // allocator slot whose payload type is forward-declared
3417        // in the program BTF degrades to the existing chase
3418        // behaviour during the pre-pass; the index is wired
3419        // into the per-map renders below where the typical
3420        // bridge call site lives (TASK_STORAGE / HASH maps
3421        // holding `struct sdt_data __arena *` entry pointers).
3422        let sdt_mem = accessor.mem_reader(
3423            shared_arena_snapshot.as_ref().map(|(_, snap)| snap),
3424            &arena_page_index,
3425            num_cpus,
3426            // Threaded in from [`DumpContext::cast_map`]: same
3427            // cast-analysis output the per-map renderer below
3428            // consumes. Letting the sdt_alloc pre-pass see it
3429            // means typed-allocator payload chases (per-task /
3430            // per-cgroup contents inside arena) get the same
3431            // `u64` → typed-pointer promotion as the rest of
3432            // the dump, instead of degrading to plain counters
3433            // for fields the analyzer recovered.
3434            cast_map,
3435            None,
3436            // The sdt_alloc pre-pass populates `sdt_alloc_metas`
3437            // itself; the metas slice is empty until this loop
3438            // finishes, so pass an empty slice here. The
3439            // metas-driven fallback only fires for the per-map
3440            // renders below (after this loop has produced every
3441            // allocator's metadata).
3442            &[],
3443            // The sdt_alloc pre-pass reads BTF type metadata for
3444            // the typed allocator payload from the scheduler's
3445            // own program BTF; no cross-BTF Fwd resolution is
3446            // needed here. The per-map renders below pass the
3447            // built `cross_btf_fwd_index` where it matters.
3448            None,
3449            // The sdt_alloc pre-pass runs BEFORE the
3450            // [`crate::monitor::scx_static_alloc`] walker (the
3451            // walks are independent, but the per-allocator leaf
3452            // payload renders here happen during the sdt_alloc
3453            // walk's own loop, so the scx_static index is not yet
3454            // built). Pass `None` so the bridge stays a no-op for
3455            // these pre-pass renders; the per-map renders below
3456            // pass the built `scx_static_index` where it matters.
3457            None,
3458            // The sdt_alloc pre-pass IS the surface that produces
3459            // the rendered-slot set; pass `None` so no dedup gate
3460            // fires while the typed-allocator surface itself is
3461            // being assembled. The per-map renders below pass the
3462            // built set where it matters.
3463            None,
3464            alloc_size_types,
3465            // The sdt_alloc pre-pass walks against the scheduler's
3466            // program BTF identified by `btf_kva` (the same BTF the
3467            // allocator metadata was resolved against — see
3468            // `append_arena_slot_index_for_allocator` below). Pass
3469            // this kva as the requesting-BTF identifier so the
3470            // pre-pass's own leaf renders compare against it
3471            // (currently the index is still being built so the gate
3472            // never fires here, but the threading keeps the contract
3473            // consistent across all `mem_reader` call sites).
3474            btf_kva,
3475        );
3476        // Locate every sdt_alloc allocator instance declared in
3477        // `.bss`. The Datasec walk gives us each variable's name and
3478        // offset; we filter to types matching `struct scx_allocator`
3479        // by re-resolving the var's chained type. A scheduler may
3480        // declare more than one allocator (e.g. one per-task, one
3481        // per-cgroup) so we iterate all of them.
3482        for (var_name, var_offset, var_type_id) in iter_bss_vars_with_type(prog_btf, ".bss") {
3483            // Only walk vars whose type is `struct scx_allocator`.
3484            if !is_scx_allocator_type(prog_btf, var_type_id) {
3485                continue;
3486            }
3487            // Slice the in-bss bytes for one full `struct scx_allocator`.
3488            // The size comes from BTF (resolved into `allocator_size`
3489            // by `SdtAllocOffsets::from_btf`); using the BTF-reported
3490            // size means a future field appended to scx_allocator
3491            // doesn't silently slip past the slice end.
3492            let Some(slice_end) = var_offset.checked_add(sdt_offsets.allocator_size) else {
3493                continue;
3494            };
3495            let slice = match bss_bytes.get(var_offset..slice_end) {
3496                Some(s) => s,
3497                None => continue,
3498            };
3499
3500            // Discover the payload BTF type id from the elem_size
3501            // we'd read in the walker. We do a small read here just
3502            // to drive the heuristic; the walker re-reads it.
3503            let pool_off = sdt_offsets.allocator_pool + sdt_offsets.pool_elem_size;
3504            let elem_size = if pool_off + 8 <= slice.len() {
3505                let mut buf = [0u8; 8];
3506                buf.copy_from_slice(&slice[pool_off..pool_off + 8]);
3507                u64::from_le_bytes(buf)
3508            } else {
3509                0
3510            };
3511            let payload_size =
3512                elem_size.saturating_sub(sdt_offsets.data_header_size as u64) as usize;
3513            // `prog_btf` is split BTF: the scheduler's program types
3514            // layered on the vmlinux base. `discover_payload_btf_id`
3515            // probes only the program section's id range (via
3516            // `Btf::split`), so vmlinux base `*_ctx` structs of the
3517            // same size cannot shadow the scheduler's payload struct.
3518            let choice = discover_payload_btf_id(prog_btf, payload_size, &var_name);
3519
3520            let snap = walk_sdt_allocator(
3521                accessor.kernel(),
3522                arena_kern_vm_start,
3523                slice,
3524                &sdt_offsets,
3525                prog_btf,
3526                choice.target_type_id,
3527                choice.reason.clone(),
3528                var_name.clone(),
3529                &sdt_mem,
3530            );
3531            // Accumulate every allocator with a typed payload AND
3532            // append its live slots to the bridge index. The
3533            // per-map selector (`select_sdt_alloc_meta`) picks the
3534            // right one by matching `var_name` (the .bss symbol —
3535            // e.g. `scx_task_allocator`) against each rendered map's
3536            // name (e.g. `scx_task_map`). Schedulers that declare
3537            // multiple typed allocators no longer lose payload
3538            // expansion — each map renders against the matching
3539            // allocator's payload type. Only allocators with a
3540            // resolved payload type id contribute to the bridge
3541            // index — without a typed payload there is no useful
3542            // BTF id to surface to the renderer, and the index
3543            // would just point every chase at 0 (which the bridge
3544            // gate filters as "no payload type").
3545            if choice.target_type_id != 0 {
3546                sdt_alloc_metas.push(crate::monitor::dump::render_map::SdtAllocMeta {
3547                    allocator_name: var_name.clone(),
3548                    elem_size,
3549                    header_size: sdt_offsets.data_header_size,
3550                    target_type_id: choice.target_type_id,
3551                    kern_vm_start: arena_kern_vm_start,
3552                });
3553                // Append this allocator's slots to the bridge index.
3554                // The helper handles the size-fits-u32 check, the
3555                // dedup-on-duplicate-slot-start, and the
3556                // `tracing::warn!` collision diagnostic — see
3557                // [`append_arena_slot_index_for_allocator`] for the
3558                // full contract. Bridge gate (`target_type_id
3559                // != 0`) is encoded inside the helper as well; the
3560                // outer guard here is a fast-path bail before we
3561                // even allocate metadata for the allocator.
3562                crate::monitor::dump::render_map::append_arena_slot_index_for_allocator(
3563                    &mut arena_slot_index,
3564                    &var_name,
3565                    choice.target_type_id,
3566                    sdt_offsets.data_header_size,
3567                    elem_size,
3568                    &snap.all_slot_addrs,
3569                    // Stamp the slot with the program BTF the
3570                    // `target_type_id` was resolved against. The
3571                    // per-map renderer's
3572                    // [`MemReader::resolve_arena_type`] gate
3573                    // compares this against each requesting map's
3574                    // `btf_kva` and suppresses the hit on mismatch
3575                    // so the BTF id cannot leak into a sibling
3576                    // BTF's id space (multi-`.bpf.o` schedulers).
3577                    btf_kva,
3578                );
3579            }
3580            // Surface only allocators with a non-empty result OR a
3581            // diagnostic elem_size; an all-zero snapshot from a
3582            // never-initialized allocator is just noise.
3583            if !snap.entries.is_empty() || snap.elem_size != 0 {
3584                report.sdt_allocations.push(snap);
3585            }
3586        }
3587    }
3588    // Post-loop: if the pre-pass ran but discovered nothing, the
3589    // scheduler has program BTF + arena + bss but no `struct
3590    // scx_allocator` declared in `.bss`. Surface a distinct
3591    // diagnostic so an operator can tell the schedule-doesn't-link
3592    // case from the link-but-no-instance case.
3593    if sdt_alloc_pre_pass_ran
3594        && report.sdt_allocations.is_empty()
3595        && report.sdt_alloc_unavailable.is_none()
3596    {
3597        report.sdt_alloc_unavailable = Some(REASON_SDT_ALLOC_NO_INSTANCE.to_string());
3598    }
3599    let arena_slot_index_ref = if arena_slot_index.is_empty() {
3600        None
3601    } else {
3602        Some(&arena_slot_index)
3603    };
3604    // Build the rendered-slot set for arena chase dedup. Keys on
3605    // every slot that is actually rendered under
3606    // `report.sdt_allocations` — typed AND untyped allocators both
3607    // contribute via [`SdtAllocEntry::user_addr`] (emit_leaf records
3608    // an entry on every leaf it visits, regardless of whether
3609    // payload BTF resolution succeeded). Keying on
3610    // `arena_slot_index.keys()` alone would have missed slots from
3611    // allocators whose `target_type_id == 0` — those allocators
3612    // never reach `append_arena_slot_index_for_allocator` (the
3613    // helper short-circuits on a zero target id) yet their slots
3614    // ARE rendered in `report.sdt_allocations`.
3615    //
3616    // Two address keys per entry: the slot start (`user_addr`) and
3617    // the payload start (`user_addr + header_size`). Chase targets
3618    // resolved via `scx_task_data(p)` and similar helpers point at
3619    // the payload, not the slot start; without the payload-start
3620    // key the dedup misses every chase that uses the helper-
3621    // computed pointer.
3622    //
3623    // Slots past [`super::sdt_alloc::MAX_SDT_ALLOC_ENTRIES`] are
3624    // intentionally excluded — they are NOT rendered in
3625    // `snap.entries`, so the per-map renderer must surface their
3626    // payload (otherwise the truncated tail would appear nowhere
3627    // in the dump). The walker's `truncated` flag is the operator-
3628    // visible signal that some slots are only available via the
3629    // per-map render path.
3630    //
3631    // The per-map renderer's [`MemReader::is_already_rendered`]
3632    // consults this set to skip re-rendering the same allocation
3633    // when a TASK_STORAGE / HASH map's value pointer chases back
3634    // into it. Only TYPED allocators (target_type_id != 0) enter
3635    // the set — untyped pre-pass renders (hex fallback) must not
3636    // suppress per-map chases because the cast analyzer's shape
3637    // inference may resolve a concrete target type the heuristic
3638    // missed.
3639    let header_size_by_allocator: std::collections::HashMap<&str, usize> = sdt_alloc_metas
3640        .iter()
3641        .map(|meta| (meta.allocator_name.as_str(), meta.header_size))
3642        .collect();
3643    let rendered_slot_addrs: std::collections::HashSet<u32> = report
3644        .sdt_allocations
3645        .iter()
3646        .filter_map(|snap| {
3647            // Only dedup slots from allocators with a resolved
3648            // payload type. Untyped allocators (target_type_id == 0,
3649            // rendered as hex in the pre-pass) must NOT suppress
3650            // per-map chases — the cast analyzer's shape inference
3651            // may resolve a concrete target type that the pre-pass
3652            // heuristic missed.
3653            let &header_size = header_size_by_allocator.get(snap.allocator_name.as_str())?;
3654            Some(snap.entries.iter().flat_map(move |e| {
3655                let slot_start = e.user_addr as u32;
3656                let payload_start = slot_start.wrapping_add(header_size as u32);
3657                [slot_start, payload_start]
3658            }))
3659        })
3660        .flatten()
3661        .collect();
3662    let rendered_slot_addrs_ref = if rendered_slot_addrs.is_empty() {
3663        None
3664    } else {
3665        Some(&rendered_slot_addrs)
3666    };
3667    tracing::debug!(
3668        elapsed_us = sdt_alloc_t0.elapsed().as_micros() as u64,
3669        allocations = report.sdt_allocations.len(),
3670        index_entries = arena_slot_index.len(),
3671        "dump_state phase: sdt_alloc"
3672    );
3673
3674    // Pre-pass: walk every `scx_static` bump-allocator instance in
3675    // `.bss` and surface its live-allocated range. Distinct from the
3676    // sdt_alloc per-instance allocator walk above:
3677    //
3678    //   - sdt_alloc (`struct scx_allocator`) hands out fixed-stride
3679    //     slots via a 3-level radix tree with per-slot metadata; the
3680    //     walker produces one entry per live slot keyed on slot start.
3681    //   - scx_static (`struct scx_static`) is a flat bump allocator
3682    //     with no per-slot metadata; the walker produces one entry
3683    //     per live region keyed on the region's base address.
3684    //
3685    // The walk runs only when:
3686    //   - we have a scheduler `.bss` blob to read from (re-located
3687    //     here because the sdt_alloc walk above consumed the
3688    //     pre-pass `sched_bss_bytes` Option),
3689    //   - we have a program BTF to resolve `struct scx_static`
3690    //     against,
3691    //   - the program BTF carries `struct scx_static`.
3692    //
3693    // When any prerequisite is missing, the walk leaves
3694    // `report.scx_static_ranges` empty (default) rather than failing
3695    // the dump — schedulers that don't link `lib/sdt_alloc.bpf.c` or
3696    // don't use the static allocator simply skip the walk.
3697    //
3698    // Membership-only: the walker produces an UNTYPED range index.
3699    // Per-allocation type recovery requires a per-call-site type
3700    // hook from cast analysis that does not exist today (see the
3701    // module-level doc in [`crate::monitor::scx_static_alloc`] for
3702    // why). When the renderer's deferred-resolve arena chase lands
3703    // on an address inside an scx_static range, the bridge
3704    // recognises the address as "in scx_static memory" and
3705    // fails closed (returns `None` from `resolve_arena_type`)
3706    // rather than returning a wrong type — the "no invalid data
3707    // made" contract.
3708    let scx_static_t0 = std::time::Instant::now();
3709    if !deadline_exceeded(&mut truncated_at_us)
3710        && let Some((bss_bytes, prog_btf)) = relocate_sched_bss(&maps, accessor, &program_btfs)
3711        && let Ok(scx_static_offsets) =
3712            crate::monitor::scx_static_alloc::ScxStaticOffsets::from_btf(prog_btf)
3713    {
3714        let snap = crate::monitor::scx_static_alloc::walk_scx_static(
3715            &bss_bytes,
3716            &scx_static_offsets,
3717            iter_bss_vars_with_type(prog_btf, ".bss"),
3718            |type_id| is_scx_static_type(prog_btf, type_id),
3719        );
3720        report.scx_static_ranges = snap;
3721    }
3722    let scx_static_index =
3723        crate::monitor::scx_static_alloc::build_scx_static_range_index(&report.scx_static_ranges);
3724    let scx_static_index_ref = if scx_static_index.is_empty() {
3725        None
3726    } else {
3727        Some(&scx_static_index)
3728    };
3729    tracing::debug!(
3730        elapsed_us = scx_static_t0.elapsed().as_micros() as u64,
3731        ranges = report.scx_static_ranges.ranges.len(),
3732        skipped = report.scx_static_ranges.skipped,
3733        index_entries = scx_static_index.len(),
3734        "dump_state phase: scx_static"
3735    );
3736
3737    let render_map_t0 = std::time::Instant::now();
3738    let mut maps_rendered: usize = 0;
3739    let mut maps_truncated: usize = 0;
3740    for info in maps {
3741        // Skip ktstr's own framework maps so the report only shows
3742        // the scheduler-under-test's state. Three distinct shapes
3743        // need filtering:
3744        //
3745        // 1. Global-section maps from the probe skeleton: libbpf
3746        //    composes `<obj_name>.<section>` so `probe_bp.bss`,
3747        //    `probe_bp.data`, `probe_bp.rodata` all match the
3748        //    `probe_bp.` prefix. (`probe_bp` matching the bare obj
3749        //    name covers any single-name section the kernel might
3750        //    surface, though libbpf today always adds the suffix.)
3751        // 2. Global-section maps from the fentry skeleton, named
3752        //    with the `fentry_p.` prefix following the same
3753        //    libbpf convention.
3754        // 3. Bare-named maps declared via `SEC(".maps")` in
3755        //    src/bpf/probe.bpf.c — these don't get an obj prefix
3756        //    because they're not from a global section. The
3757        //    explicit denylist [`KTSTR_INTERNAL_MAPS`] enumerates
3758        //    them.
3759        //
3760        // A future tighter filter would consult bpf_prog ownership
3761        // (the program-attachment ID list pinned to each map), but
3762        // name-based filtering is enough today and avoids loading
3763        // the full prog_idr walk on the freeze hot path.
3764        {
3765            let info_name = info.name();
3766            if info_name.starts_with("probe_bp.")
3767                || info_name.starts_with("fentry_p.")
3768                || info_name == "probe_bp"
3769                || info_name == "fentry_p"
3770                || KTSTR_INTERNAL_MAPS.contains(&info_name.as_ref())
3771            {
3772                continue;
3773            }
3774        }
3775
3776        // Deadline check before each map render — bigger maps
3777        // (large hashes, arenas) can each take a meaningful slice
3778        // of the freeze window, so we re-check between renders to
3779        // bound the worst case rather than letting one
3780        // straggler push us past the watchdog.
3781        if deadline_exceeded(&mut truncated_at_us) {
3782            maps_truncated += 1;
3783            continue;
3784        }
3785
3786        // Per-map BTF resolution.
3787        //
3788        // The map's `btf_value_type_id` / `btf_key_type_id` index
3789        // the *map's own* BTF, NOT the kernel vmlinux BTF — when
3790        // `btf_kva != 0` the type IDs are program-local and using
3791        // vmlinux BTF with them would resolve to unrelated kernel
3792        // types (or out-of-range nonsense). So:
3793        //
3794        //   - `BPF_MAP_TYPE_STRUCT_OPS`              → use vmlinux
3795        //     BTF. The wrapper struct `bpf_struct_ops_<name>` is
3796        //     declared in the kernel's vmlinux BTF and the
3797        //     wrapper type id stored on the map (in
3798        //     `btf_vmlinux_value_type_id`) indexes vmlinux. Using
3799        //     the program BTF here would fail to resolve the
3800        //     wrapper.
3801        //   - `btf_kva != 0` AND program BTF loaded by pre-pass → use it.
3802        //   - `btf_kva != 0` AND program BTF load failed in pre-pass
3803        //     → render hex-only (None map_btf), no fallback.
3804        //   - `btf_kva == 0` (kernel-builtin map)      → use the
3805        //     caller-supplied vmlinux BTF; the type IDs (if any)
3806        //     genuinely index vmlinux BTF in this case.
3807        let map_btf: Option<&Btf> = if info.map_type == super::bpf_map::BPF_MAP_TYPE_STRUCT_OPS {
3808            Some(btf)
3809        } else if info.btf_kva != 0 {
3810            program_btfs.get(&info.btf_kva)
3811        } else {
3812            Some(btf)
3813        };
3814
3815        let rendered = render_map(
3816            &RenderMapCtx {
3817                accessor,
3818                btf: map_btf,
3819                num_cpus,
3820                arena_offsets,
3821                shared_arena: shared_arena_ref,
3822                arena_page_index: &arena_page_index,
3823                sdt_alloc_metas: &sdt_alloc_metas,
3824                // Threaded in from
3825                // [`DumpContext::cast_map`]: the BPF
3826                // cast-analysis output for the scheduler's
3827                // program object. `Some(&map)` lets the
3828                // renderer promote `u64` fields the analyzer
3829                // flagged into typed-pointer renders via
3830                // [`super::btf_render::MemReader::cast_lookup`];
3831                // `None` keeps every `u64` rendered as a plain
3832                // unsigned counter (the trait default).
3833                cast_map,
3834                // Built from the sdt_alloc pre-pass above:
3835                // `slot_start → ArenaSlotInfo` for every live
3836                // allocator slot. Lets the renderer range-lookup
3837                // the slot a chased arena address falls in and
3838                // recover a `BTF_KIND_FWD` pointee's real
3839                // struct id (plus a `header_skip` byte count)
3840                // via [`MemReader::resolve_arena_type`] — a
3841                // `struct sdt_data __arena *` field (or a `data`
3842                // field caching the raw `sdt_alloc()` return)
3843                // whose pointee body lives in the sdt_alloc
3844                // library's BTF still chases as the typed
3845                // per-task / per-cgroup struct, instead of
3846                // skipping with "forward declaration; body not
3847                // in this BTF". `None` when no allocator with a
3848                // typed payload was discovered.
3849                arena_slot_index: arena_slot_index_ref,
3850                // Threaded in from
3851                // [`DumpContext::cross_btf_fwd_index`]: the
3852                // cross-BTF Fwd resolution context populated by
3853                // the cast-analysis pre-pass over every embedded
3854                // BPF object's BTF. `Some(&idx)` lets the
3855                // renderer chase a `BTF_KIND_FWD` whose body
3856                // lives in a sibling embedded object via
3857                // [`MemReader::cross_btf_resolve_fwd`]. `None`
3858                // keeps the renderer's "forward declaration;
3859                // body not in this BTF" skip path intact.
3860                cross_btf_fwd_index: cross_btf_fwd_index_ref,
3861                // Built from the scx_static pre-pass above:
3862                // `start_low32 → size` for every live
3863                // `scx_static` bump-allocator region. Lets the
3864                // renderer's
3865                // [`MemReader::resolve_arena_type`]
3866                // diagnose-and-skip on a chased arena address
3867                // that lands inside scx_static memory — the
3868                // bridge cannot recover a per-allocation type
3869                // (no per-slot header) so the chase falls through
3870                // to the historical Fwd-skip / cross-BTF
3871                // resolution path. `None` when no live
3872                // `scx_static` instance was discovered.
3873                scx_static_index: scx_static_index_ref,
3874                // Built from the sdt_alloc pre-pass above: the
3875                // low-32 windowed `slot_start` of every allocator
3876                // slot already rendered into
3877                // `report.sdt_allocations`. Lets the renderer's
3878                // [`MemReader::is_already_rendered`] short-circuit
3879                // the arena chase when a TASK_STORAGE / HASH
3880                // map's value pointer lands in a slot the
3881                // typed-allocator surface already shows — no
3882                // duplicate payload in the dump. `None` when no
3883                // allocator pre-pass produced any rendered slot.
3884                rendered_slot_addrs: rendered_slot_addrs_ref,
3885                alloc_size_types,
3886            },
3887            &info,
3888        );
3889
3890        report.maps.push(rendered);
3891        maps_rendered += 1;
3892    }
3893    tracing::debug!(
3894        elapsed_us = render_map_t0.elapsed().as_micros() as u64,
3895        rendered = maps_rendered,
3896        truncated = maps_truncated,
3897        "dump_state phase: per-map render"
3898    );
3899
3900    report.dump_truncated_at_us = truncated_at_us;
3901    report.maps_truncated = maps_truncated as u32;
3902    report
3903}
3904
3905/// Walk every CPU's `kernel_cpustat`, `kernel_stat`, and (under
3906/// NO_HZ) `tick_sched` slots and produce a [`PerCpuTimeStats`]
3907/// vector — one entry per CPU index in `cap.per_cpu_offsets`.
3908///
3909/// Reads against the supplied [`super::reader::GuestMem`]. Each CPU's
3910/// per-CPU base for a symbol `S` is `S + per_cpu_offsets[cpu]`,
3911/// converted to a guest physical offset via
3912/// [`super::symbols::kva_to_pa`] using the supplied `page_offset`
3913/// (the standard direct-mapping translation; per-CPU pages always
3914/// live in the direct mapping).
3915///
3916/// `cpustat[]` is read as 8 contiguous u64s starting at the
3917/// resolved offset (length matches the indices captured —
3918/// CPUTIME_USER through CPUTIME_STEAL — leaving CPUTIME_GUEST /
3919/// CPUTIME_GUEST_NICE / CPUTIME_FORCEIDLE unread; the dump
3920/// surfaces them as zero in the unread slots, which is acceptable
3921/// since they're virt-guest specific or kernel-config gated and
3922/// distinct from the failure-dump narrative). `softirqs[]` reads
3923/// as `NR_SOFTIRQS` u32s, widened to u64 for the report. `irqs_sum`
3924/// is `unsigned long` (read as u64 — 64-bit only kernels are the
3925/// supported configuration). `iowait_sleeptime` is `ktime_t` /
3926/// `s64`; the value is cast to u64 (the kernel never produces
3927/// negative iowait time).
3928fn collect_per_cpu_time(cap: &CpuTimeCapture<'_>) -> Vec<PerCpuTimeStats> {
3929    use super::btf_offsets::{
3930        CPUTIME_IDLE, CPUTIME_IOWAIT, CPUTIME_IRQ, CPUTIME_NICE, CPUTIME_SOFTIRQ, CPUTIME_STEAL,
3931        CPUTIME_SYSTEM, CPUTIME_USER, NR_SOFTIRQS,
3932    };
3933    let mut out = Vec::with_capacity(cap.per_cpu_offsets.len());
3934    for (cpu_idx, &per_cpu_off) in cap.per_cpu_offsets.iter().enumerate() {
3935        let cpu = cpu_idx as u32;
3936
3937        // kernel_cpustat::cpustat[N]: each slot is a u64 in nsec.
3938        // Read CPUTIME_USER through CPUTIME_STEAL (indices 0..=7).
3939        let cpustat_kva =
3940            super::symbols::per_cpu_kva(cap.kernel_cpustat_kva, cap.kaslr_offset, per_cpu_off);
3941        let cpustat_pa = super::symbols::kva_to_pa(cpustat_kva, cap.page_offset);
3942        let cpustat_base = cap.offsets.kernel_cpustat_cpustat;
3943        let read_cpustat = |idx: usize| -> u64 {
3944            // sizeof(u64) == 8.
3945            cap.mem.read_u64(cpustat_pa, cpustat_base + idx * 8)
3946        };
3947        let cpustat_user_ns = read_cpustat(CPUTIME_USER);
3948        let cpustat_nice_ns = read_cpustat(CPUTIME_NICE);
3949        let cpustat_system_ns = read_cpustat(CPUTIME_SYSTEM);
3950        let cpustat_softirq_ns = read_cpustat(CPUTIME_SOFTIRQ);
3951        let cpustat_irq_ns = read_cpustat(CPUTIME_IRQ);
3952        let cpustat_idle_ns = read_cpustat(CPUTIME_IDLE);
3953        let cpustat_iowait_ns = read_cpustat(CPUTIME_IOWAIT);
3954        let cpustat_steal_ns = read_cpustat(CPUTIME_STEAL);
3955
3956        // kernel_stat::softirqs[N]: each slot is a u32 (count).
3957        // Widen to u64 for reporting consistency with cpustat.
3958        let kstat_kva = super::symbols::per_cpu_kva(cap.kstat_kva, cap.kaslr_offset, per_cpu_off);
3959        let kstat_pa = super::symbols::kva_to_pa(kstat_kva, cap.page_offset);
3960        let mut softirqs = [0u64; NR_SOFTIRQS];
3961        for (i, slot) in softirqs.iter_mut().enumerate() {
3962            // sizeof(unsigned int) == 4.
3963            *slot = cap
3964                .mem
3965                .read_u32(kstat_pa, cap.offsets.kstat_softirqs + i * 4) as u64;
3966        }
3967
3968        // kernel_stat::irqs_sum: unsigned long. 64-bit only
3969        // kernels are supported, so read as u64.
3970        let irqs_sum = cap.mem.read_u64(kstat_pa, cap.offsets.kstat_irqs_sum);
3971
3972        // tick_sched::iowait_sleeptime: ktime_t (s64) ns,
3973        // accumulated only under NO_HZ when the CPU enters idle
3974        // with nr_iowait > 0. Skip when the symbol or BTF offset
3975        // is absent.
3976        let iowait_sleeptime_ns = cap
3977            .tick_cpu_sched_kva
3978            .zip(cap.offsets.tick_sched_iowait_sleeptime)
3979            .map(|(tick_sym_kva, off)| {
3980                let kva = super::symbols::per_cpu_kva(tick_sym_kva, cap.kaslr_offset, per_cpu_off);
3981                let pa = super::symbols::kva_to_pa(kva, cap.page_offset);
3982                cap.mem.read_u64(pa, off)
3983            });
3984
3985        out.push(PerCpuTimeStats {
3986            cpu,
3987            cpustat_user_ns,
3988            cpustat_nice_ns,
3989            cpustat_system_ns,
3990            cpustat_softirq_ns,
3991            cpustat_irq_ns,
3992            cpustat_idle_ns,
3993            cpustat_iowait_ns,
3994            cpustat_steal_ns,
3995            softirqs,
3996            irqs_sum,
3997            iowait_sleeptime_ns,
3998        });
3999    }
4000    out
4001}
4002
4003/// Walk the test's workload cgroups and read each leaf's PSI_IRQ_FULL (Phase A).
4004/// Thin adapter over
4005/// [`super::cgroup_walk::collect_workload_cgroup_psi`] that unpacks the
4006/// borrowed [`CgroupPsiCapture`]. Empty when no workload leaf has per-cgroup
4007/// PSI accounting (loud-absent).
4008fn collect_cgroup_psi(cap: &CgroupPsiCapture<'_>) -> Vec<super::cgroup_walk::CgroupPsiStat> {
4009    super::cgroup_walk::collect_workload_cgroup_psi(
4010        cap.mem,
4011        cap.cgroup_offsets,
4012        cap.psi_offsets,
4013        cap.root_cgroup_kva,
4014        cap.root_cgroup_pa,
4015        cap.workload_root_path,
4016        cap.page_offset,
4017    )
4018}
4019
4020/// Walk a Datasec section by name, yielding `(var_name, byte_offset,
4021/// type_id)` for every variable declared in it.
4022///
4023/// Used by [`dump_state`] to enumerate `.bss` variables when looking
4024/// for `scx_allocator` instances. Returns an empty iterator when the
4025/// Datasec doesn't exist or any chained Var resolution fails — the
4026/// caller treats that as "no sdt_alloc state to surface" rather than
4027/// a hard error.
4028fn iter_bss_vars_with_type(btf: &Btf, section_name: &str) -> Vec<(String, usize, u32)> {
4029    use btf_rs::BtfType;
4030    let mut out = Vec::new();
4031    let Ok(candidates) = btf.resolve_types_by_name(section_name) else {
4032        return out;
4033    };
4034    for ty in candidates {
4035        let btf_rs::Type::Datasec(ds) = ty else {
4036            continue;
4037        };
4038        for var_info in &ds.variables {
4039            let Ok(chained) = btf.resolve_chained_type(var_info) else {
4040                continue;
4041            };
4042            let btf_rs::Type::Var(var) = chained else {
4043                continue;
4044            };
4045            let Ok(name) = btf.resolve_name(&var) else {
4046                continue;
4047            };
4048            // The Var's type_id points to the variable's actual
4049            // type (e.g. struct scx_allocator). var_info.offset() is
4050            // the byte offset within the Datasec.
4051            let Some(type_id) = var.get_type_id() else {
4052                continue;
4053            };
4054            out.push((name, var_info.offset() as usize, type_id));
4055        }
4056    }
4057    out
4058}
4059
4060/// True iff `type_id` resolves to a struct named `scx_allocator`,
4061/// stripping the BTF modifier chain en route. The five modifier
4062/// kinds the loop unwraps — `Const`, `Volatile`, `Typedef`,
4063/// `Restrict`, `TypeTag` — are the complete set the kernel BPF
4064/// pipeline emits for global variable types in `.bss`. Any other
4065/// kind in the chain (Ptr, Array, etc.) terminates the lookup with
4066/// a non-match.
4067fn is_scx_allocator_type(btf: &Btf, type_id: u32) -> bool {
4068    use btf_rs::Type as T;
4069    // Mirror the modifier-chain pattern in
4070    // `btf_offsets::resolve_member_composite` — resolve the
4071    // chained type via the BtfType trait object so the type
4072    // aliases (Const = Volatile, TypeTag = Typedef) all share the
4073    // same path through the loop.
4074    let Ok(mut t) = btf.resolve_type_by_id(type_id) else {
4075        return false;
4076    };
4077    for _ in 0..20 {
4078        match t {
4079            T::Struct(s) => {
4080                return btf.resolve_name(&s).is_ok_and(|n| n == "scx_allocator");
4081            }
4082            T::Const(_) | T::Volatile(_) | T::Typedef(_) | T::Restrict(_) | T::TypeTag(_) => {
4083                let Some(btf_ty) = t.as_btf_type() else {
4084                    return false;
4085                };
4086                let Ok(next) = btf.resolve_chained_type(btf_ty) else {
4087                    return false;
4088                };
4089                t = next;
4090            }
4091            _ => return false,
4092        }
4093    }
4094    false
4095}
4096
4097/// True iff `type_id` resolves to a struct named `scx_static`,
4098/// stripping the BTF modifier chain en route. Mirrors the
4099/// modifier-handling shape of [`is_scx_allocator_type`] — the five
4100/// modifier kinds (`Const`, `Volatile`, `Typedef`, `Restrict`,
4101/// `TypeTag`) are the complete set the kernel BPF pipeline emits for
4102/// global variable types in `.bss`; any other kind terminates the
4103/// lookup with a non-match.
4104///
4105/// Used by the [`crate::monitor::scx_static_alloc::walk_scx_static`]
4106/// pre-pass to filter `.bss` Vars to only `struct scx_static`
4107/// instances. A scheduler that doesn't link `lib/sdt_alloc.bpf.c`
4108/// has no such Var; the filter rejects every candidate and the
4109/// walker produces an empty snapshot.
4110fn is_scx_static_type(btf: &Btf, type_id: u32) -> bool {
4111    use btf_rs::Type as T;
4112    let Ok(mut t) = btf.resolve_type_by_id(type_id) else {
4113        return false;
4114    };
4115    for _ in 0..20 {
4116        match t {
4117            T::Struct(s) => {
4118                return btf.resolve_name(&s).is_ok_and(|n| n == "scx_static");
4119            }
4120            T::Const(_) | T::Volatile(_) | T::Typedef(_) | T::Restrict(_) | T::TypeTag(_) => {
4121                let Some(btf_ty) = t.as_btf_type() else {
4122                    return false;
4123                };
4124                let Ok(next) = btf.resolve_chained_type(btf_ty) else {
4125                    return false;
4126                };
4127                t = next;
4128            }
4129            _ => return false,
4130        }
4131    }
4132    false
4133}
4134
4135/// Locate the scheduler's `.bss` array map and pull out (raw bytes,
4136/// program BTF) for the [`crate::monitor::scx_static_alloc`] pre-pass.
4137///
4138/// The earlier sdt_alloc pre-pass at the top of [`dump_state`]
4139/// already collected `sched_bss_bytes` once but the if-let chain at
4140/// the sdt_alloc walk consumed the `Option`. Re-locating here keeps
4141/// both walkers independent: each owns its own bss-bytes read so a
4142/// future ordering change can't accidentally leave one walker
4143/// without input. The cost is one extra map walk; small compared to
4144/// the per-map render loop that follows.
4145///
4146/// Returns `None` when:
4147///   - no `*.bss` map exists (libbpf only creates this map when the
4148///     program has any global non-const data),
4149///   - the map's `btf_kva == 0` (no program BTF — type resolution
4150///     would fail),
4151///   - the program BTF for that `btf_kva` was not loaded in the
4152///     pre-pass (parse failed earlier; the caller already logged),
4153///   - the map's value bytes can't be read.
4154fn relocate_sched_bss<'btf>(
4155    maps: &[BpfMapInfo],
4156    accessor: &GuestMemMapAccessor<'_>,
4157    program_btfs: &'btf std::collections::HashMap<u64, Btf>,
4158) -> Option<(Vec<u8>, &'btf Btf)> {
4159    for info in maps {
4160        let name = info.name();
4161        if name.starts_with("probe_bp.")
4162            || name.starts_with("fentry_p.")
4163            || name == "probe_bp"
4164            || name == "fentry_p"
4165            || KTSTR_INTERNAL_MAPS.contains(&name.as_ref())
4166        {
4167            continue;
4168        }
4169        if info.map_type == BPF_MAP_TYPE_ARRAY
4170            && info.btf_kva != 0
4171            && name.ends_with(".bss")
4172            && let Some(prog_btf) = program_btfs.get(&info.btf_kva)
4173            && let Some(bytes) = accessor.read_value(info, 0, info.value_size as usize)
4174        {
4175            return Some((bytes, prog_btf));
4176        }
4177    }
4178    None
4179}
4180
4181/// Load a BPF program's `struct btf` from guest memory at `btf_kva`.
4182///
4183/// Reads the kernel `struct btf` at `btf_kva`, follows its `data` /
4184/// `data_size` / `base_btf` fields, fetches the raw BTF blob via
4185/// page-walked vmalloc reads, and parses it. When `base_btf` is
4186/// non-NULL the program's BTF is split atop the vmlinux BTF (the
4187/// kernel's own base BTF) — pass the host's already-parsed vmlinux
4188/// `Btf` as the split base so type IDs resolve correctly.
4189///
4190/// Returns `None` when any step fails: missing offsets, untranslatable
4191/// pages, or `Btf::from_bytes` rejection (truncated / corrupted blob).
4192/// Failure is silent and the caller falls back to the host vmlinux
4193/// BTF — the dump is best-effort, a partial render still beats no
4194/// render.
4195///
4196/// Distinct from the [`super::bpf_map::BpfMapAccessor::load_program_btf`]
4197/// trait method (which dispatches across backends): this free function
4198/// is the guest-memory backend's actual KVA-based loader. The trait
4199/// method on `GuestMemMapAccessor` just forwards here.
4200pub(super) fn load_program_btf_kva(
4201    accessor: &GuestMemMapAccessor<'_>,
4202    btf_kva: u64,
4203    base_btf: &Btf,
4204) -> Option<Btf> {
4205    let kernel = accessor.kernel();
4206    let offsets = accessor.offsets();
4207    let mem = kernel.mem();
4208    let walk = kernel.walk_context();
4209
4210    // `struct btf` may be kmalloc'd (direct map) or vmalloc'd; use
4211    // translate_any_kva.
4212    let btf_pa = super::idr::translate_any_kva(
4213        mem,
4214        walk.cr3_pa,
4215        walk.page_offset,
4216        btf_kva,
4217        walk.l5,
4218        walk.tcr_el1,
4219    )?;
4220    let data_kva = mem.read_u64(btf_pa, offsets.btf_data);
4221    let data_size = mem.read_u32(btf_pa, offsets.btf_data_size) as usize;
4222    let base_kva = mem.read_u64(btf_pa, offsets.btf_base_btf);
4223
4224    if data_kva == 0 || data_size == 0 {
4225        return None;
4226    }
4227
4228    if data_size > MAX_BTF_BLOB {
4229        return None;
4230    }
4231
4232    // The BTF blob is vmalloc-backed — `btf->data` is allocated via
4233    // vmalloc / kvmalloc inside `kernel/bpf/btf.c`'s
4234    // `btf_parse_*` paths. Use the chunked vmalloc reader so a
4235    // 100 KB blob doesn't pay 100K syscalls of byte-wise translate.
4236    // The chunked reader honours all-or-nothing semantics, so a
4237    // short read returns None directly; no extra length check needed.
4238    let blob = kernel.read_kva_bytes_chunked(data_kva, data_size)?;
4239
4240    if base_kva != 0 {
4241        // Split BTF: the program's types extend the kernel's
4242        // vmlinux BTF. Pass the host's parsed vmlinux Btf as the
4243        // base so cross-base type IDs (e.g. `task_struct`) resolve.
4244        //
4245        // Uses host vmlinux BTF as split base — correct when host
4246        // kernel == guest kernel (ktstr's default and the common
4247        // CI configuration). A guest running a different kernel
4248        // version would silently mis-render cross-base type
4249        // references; flagged as a known limitation in the module
4250        // doc above.
4251        Btf::from_split_bytes(&blob, base_btf).ok()
4252    } else {
4253        Btf::from_bytes(&blob).ok()
4254    }
4255}
4256
4257/// Render a byte slice as space-separated hex pairs.
4258///
4259/// `pub(crate)` so [`super::sdt_alloc`] can reuse the same wire shape
4260/// for its hex-fallback payload renderings — keeps the dump's hex
4261/// output consistent across both renderers.
4262pub(crate) fn hex_dump(bytes: &[u8]) -> String {
4263    use std::fmt::Write;
4264    let mut s = String::with_capacity(bytes.len() * 3);
4265    for (i, b) in bytes.iter().enumerate() {
4266        if i > 0 {
4267            s.push(' ');
4268        }
4269        // unwrap is safe: write! to String never fails.
4270        let _ = write!(s, "{b:02x}");
4271    }
4272    s
4273}