ktstr/test_support/sidecar/
mod.rs

1//! Per-run sidecar JSON — the durable record of a ktstr test outcome.
2//!
3//! Every test (pass, fail, or skip) writes a [`SidecarResult`] to a
4//! JSON file under the run's sidecar directory; downstream analysis
5//! (`cargo ktstr stats`, CI dashboards) aggregates those files to
6//! compute pass/fail rates, verifier stats, callback profiles, and
7//! KVM stats across gauntlet variants.
8//!
9//! Responsibilities owned by this module:
10//! - [`SidecarResult`]: the on-disk schema. Writer-side: every field
11//!   is always emitted — `null` for `None`, `[]` for empty `Vec` —
12//!   with no `skip_serializing_if` and no `serde(default)`. Reader-
13//!   side: serde's native `Option<T>` deserialize tolerates absence
14//!   (a missing key parses as `None`); non-`Option` fields (e.g.
15//!   `test_name`, `passed`, `stats`) are hard-required and a missing
16//!   key fails deserialize. The contract is intentionally asymmetric
17//!   so a future producer that drops an `Option` field still parses
18//!   on older readers, while the current writer guarantees full
19//!   round-trip symmetry. Pre-1.0: old sidecar JSON is disposable;
20//!   regenerate by re-running the test rather than relying on the
21//!   reader-side tolerance for migration.
22//! - [`collect_sidecars`]: load every `*.ktstr.json` under a directory
23//!   (one level of subdirectories for per-job gauntlet layouts).
24//! - [`write_sidecar`] / [`write_skip_sidecar`]: serialize one run to
25//!   disk; variant-hash the discriminating fields so gauntlet variants
26//!   don't clobber each other.
27//! - [`sidecar_dir`], [`runs_root`], [`newest_run_dir`]: resolve where
28//!   sidecars live (env override, or
29//!   `{target}/ktstr/{kernel}-{project_commit}` where
30//!   `{project_commit}` is the project tree's HEAD short hex from
31//!   [`detect_project_commit`], suffixed `-dirty` when the
32//!   worktree differs).
33//! - [`format_run_dirname`]: render the
34//!   `{kernel}-{project_commit}` leaf name from the resolved
35//!   kernel + commit slots, substituting the literal `unknown`
36//!   when either probe returned `None` so the dirname stays
37//!   filesystem-safe (see the unknown-commit collision
38//!   semantics in the runs guide).
39//! - [`is_run_directory`]: predicate consumed by run-listing
40//!   walkers ([`newest_run_dir`] here, `sorted_run_entries` in
41//!   `crate::stats`). Filters non-directories and dotfile
42//!   subdirectories (notably the `.locks/` flock-sentinel
43//!   subdirectory) so the lock infrastructure cannot pollute
44//!   `cargo ktstr stats list` output or claim the "most recent
45//!   run" bucket.
46//! - [`reset_run_dir_for_session`]: shallow-wipe `*.ktstr.json`
47//!   files when the run epoch changes so a re-run at the same
48//!   `{kernel}-{project_commit}` key produces a last-writer-wins
49//!   snapshot rather than an append-only archive. Raw no-token
50//!   calls retain a process-local once gate.
51//! - [`acquire_run_dir_publication_lock`]: cross-process publication
52//!   rail on `{runs_root}/.locks/{key}.lock`. Writers whose run-epoch
53//!   sentinel already matches take `LOCK_SH` and publish concurrently;
54//!   the first writer or an epoch transition takes `LOCK_EX` for
55//!   wipe -> sentinel -> first publication. Holding SH through atomic
56//!   rename prevents an EX reset from crossing either the primary write
57//!   or its final-verdict rewrite.
58//!   The override branch (operator-chosen
59//!   `KTSTR_SIDECAR_DIR`) skips the flock for the same reason
60//!   it skips pre-clear: the operator owns the directory's
61//!   contents.
62//! - [`warn_unknown_project_commit_once`]: one-shot stderr warning
63//!   on first sidecar write when `detect_project_commit` returns
64//!   `None` (test process not in a git repo) so concurrent or
65//!   successive non-git runs colliding on `{kernel}-unknown`
66//!   surface the disambiguation hint
67//!   (`KTSTR_SIDECAR_DIR=…` or place the tree under git) at
68//!   first invocation rather than as a silent collision.
69//! - [`format_verifier_stats`], [`format_callback_profile`],
70//!   [`format_kvm_stats`]: human-readable summaries from a
71//!   `Vec<SidecarResult>` for CLI output.
72//! - [`detect_kernel_version`]: read the kernel version from
73//!   `KTSTR_KERNEL` cache metadata for sidecar-dir naming and the
74//!   `kernel_version` field, with fallback to
75//!   `include/config/kernel.release` in the kernel source tree
76//!   when the cache metadata is absent or does not carry a
77//!   version (e.g. a raw source-tree path set in `KTSTR_KERNEL`
78//!   rather than a cache key).
79//! - [`detect_kernel_commit`]: read the kernel SOURCE TREE's git
80//!   HEAD short hex (with `-dirty` suffix when worktree differs
81//!   from the index or HEAD differs from the index) for the
82//!   `kernel_commit` field. Distinct from `kernel_version`
83//!   (release string from `kernel.release`) and `project_commit`
84//!   (ktstr framework HEAD): this records "what kernel commit
85//!   produced this run" so two runs of the same `kernel_version`
86//!   but different WIP source trees compare distinctly.
87
88use std::path::PathBuf;
89
90use anyhow::Context;
91
92use crate::assert::{AssertResult, ScenarioStats};
93use crate::monitor::MonitorSummary;
94use crate::sync::MutexExt;
95use crate::test_support::PayloadMetrics;
96use crate::timeline::StimulusEvent;
97use crate::vmm;
98
99use super::entry::KtstrTestEntry;
100use super::timefmt::{generate_run_id, now_iso8601};
101
102/// Which time base denominates this sidecar's workload-THROUGHPUT rate
103/// metrics (`taobench_*_ops_per_cpu_sec` /
104/// `schbench_*loops_per_cpu_sec` / `iteration_rate`, and any future
105/// throughput rate). Some rate key names predate the denomination change, so
106/// a number from a wall-era sidecar is NOT comparable to a CPU-era sidecar —
107/// the compare
108/// pipeline keys its row pairing and its group-averaging on this marker so
109/// cross-denomination values are never silently folded or diffed (see
110/// `PairingKey::from_row` and the `denomination_mismatches` counter in
111/// `CompareReport`).
112///
113/// Event-frequency metrics (IRQ/sec, per-second RPS samples) and latency
114/// metrics are wall-based in BOTH eras — a frequency's meaning is real time —
115/// so they are outside this marker's scope.
116///
117/// `#[serde(default)]` on the carrying field + `Default = Wall`: a sidecar
118/// written before the marker existed is by definition wall-era.
119#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
120#[serde(rename_all = "snake_case")]
121pub enum ThroughputDenomination {
122    /// Throughput rates are work / wall-second (pre-marker sidecars).
123    #[default]
124    Wall,
125    /// Throughput rates are work / CPU-second received
126    /// (`CLOCK_THREAD_CPUTIME_ID`) — the current denomination.
127    CpuSec,
128}
129
130impl ThroughputDenomination {
131    /// Stable string form used in pairing keys and diagnostics.
132    pub fn as_str(self) -> &'static str {
133        match self {
134            ThroughputDenomination::Wall => "wall",
135            ThroughputDenomination::CpuSec => "cpu_sec",
136        }
137    }
138}
139
140/// Test result sidecar written to KTSTR_SIDECAR_DIR for post-run analysis.
141#[derive(Debug, serde::Serialize, serde::Deserialize)]
142pub struct SidecarResult {
143    /// Fully qualified test name (matches `KtstrTestEntry::name`,
144    /// the bare function name without the `ktstr/` nextest prefix).
145    pub test_name: String,
146    /// Rendered topology label (e.g. `1n2l4c1t`) for the variant this
147    /// sidecar describes.
148    pub topology: String,
149    /// Scheduler name (matches `Scheduler::name`); `"eevdf"` for
150    /// tests run without an scx scheduler.
151    pub scheduler: String,
152    /// Best-effort git commit of the scheduler binary used for this
153    /// run. Currently ALWAYS `None` for every `SchedulerSpec`
154    /// variant — no variant today has a reliable commit source.
155    /// The field is reserved on the schema so stats tooling can
156    /// enrich it once a reliable source exists (e.g. a
157    /// `--version` probe or ELF-note read on the resolved
158    /// scheduler binary). See
159    /// [`crate::test_support::SchedulerSpec::scheduler_commit`]
160    /// for the full per-variant rationale.
161    ///
162    /// Writer always emits (`"scheduler_commit": null` on absence).
163    /// Reader-side: serde's native `Option<T>` deserialize tolerates
164    /// absence (a missing key parses as `None`); see the module-level
165    /// doc for the full asymmetric contract that governs every
166    /// nullable on this struct.
167    pub scheduler_commit: Option<String>,
168    /// How the userspace scheduler binary was resolved for this run —
169    /// the snake_case [`crate::test_support::ResolveSource::as_str`] tag
170    /// (`"path"`, `"env_var"`, `"path_lookup"`, `"auto_built"`,
171    /// `"not_found"`). Historical sidecars may carry retired tags
172    /// (`"sibling_dir"`, `"target_debug"`, `"target_release"`) from the
173    /// pre-built-fallback cascade the resolver no longer runs; the stats
174    /// CLI reads `resolve_source` as an opaque string, so those still
175    /// filter and display. Provenance, not identity: distinct from
176    /// [`SidecarResult::scheduler_commit`] (the binary's git commit) —
177    /// this records the resolution PATH, so the stats CLI can answer "was
178    /// this run's scheduler auto-built from the workspace HEAD, or
179    /// resolved from a possibly-stale `$PATH` binary?".
180    /// `"auto_built"` is the only tag whose source commit is known to
181    /// match the workspace tree; every other tag carries the stale-binary
182    /// hazard documented on the [`crate::test_support::ResolveSource`]
183    /// variant.
184    ///
185    /// Writer always emits (`"resolve_source": null` on absence — the
186    /// skip-sidecar path resolves no binary). Reader-side: serde's native
187    /// `Option<T>` deserialize tolerates absence (a missing key parses as
188    /// `None`); see the module-level doc for the full asymmetric
189    /// contract. Excluded from `sidecar_variant_hash` for the same
190    /// cross-host grouping reason as `scheduler_commit` / `run_source`:
191    /// two runs of the same semantic variant resolved via different
192    /// discovery paths must still bucket together.
193    pub resolve_source: Option<String>,
194    /// Best-effort git HEAD of the ktstr project tree at sidecar-
195    /// write time. Captured by `detect_project_commit` via
196    /// `gix::discover` from the test process's current working
197    /// directory; walks up to find the enclosing repo and reads
198    /// HEAD short-hex, suffixing `-dirty` when index-vs-HEAD or
199    /// worktree-vs-index changes are observed (submodules ignored,
200    /// matching the [`crate::fetch::local_source`] dirty-detection
201    /// pattern). `None` when cwd is not inside any git repo, or
202    /// when the gix probe fails for any reason — this is metadata,
203    /// not a gate, so probe failure must not abort the run.
204    ///
205    /// Distinct from [`SidecarResult::scheduler_commit`]: that
206    /// field tracks the userspace scheduler binary's commit
207    /// (currently always `None` per its own doc); this field
208    /// tracks the ktstr framework / test-runner commit, so the
209    /// stats CLI can answer "which version of the harness produced
210    /// this sidecar?" without inspecting the scheduler.
211    ///
212    /// Writer always emits (`"project_commit": null` on absence).
213    /// Reader-side: serde's native `Option<T>` deserialize tolerates
214    /// absence (a missing key parses as `None`) — see the module-
215    /// level doc for the full asymmetric contract. Excluded from
216    /// `sidecar_variant_hash` for the same cross-host grouping
217    /// reason `scheduler_commit` is excluded: two runs of the same
218    /// semantic variant on different ktstr commits must still bucket
219    /// together so `perf-delta` can diff them; the commit-drift
220    /// detection inspects this field directly via `--project-commit`
221    /// / `--a-project-commit` / `--b-project-commit`.
222    pub project_commit: Option<String>,
223    /// Binary payload name (matches `Payload::name` when
224    /// `entry.payload` is set). `None` when the test declared no
225    /// binary payload. Writer always emits (`"payload": null` on
226    /// absence); reader-side, serde's native `Option<T>` deserialize
227    /// tolerates absence — see the module-level doc for the full
228    /// asymmetric contract.
229    pub payload: Option<String>,
230    /// Per-payload extracted metrics collected from `ctx.payload(X).run()`
231    /// / `.spawn().wait()` call sites during the test body.
232    ///
233    /// One [`PayloadMetrics`] per invocation, in the order the calls
234    /// ran. Empty when no payload calls were made (scheduler-only
235    /// tests, or a binary-only test where the body bailed before
236    /// running the payload). Writer always emits (`"metrics": []` in
237    /// that case); reader-side, this `Vec` field is hard-required —
238    /// non-`Option` fields fail deserialize on absence. See the
239    /// module-level doc for the full contract.
240    pub metrics: Vec<PayloadMetrics>,
241    /// True when the run is a real pass — every assertion that
242    /// ran produced a positive verdict. Mirrors
243    /// [`crate::assert::AssertResult::is_pass`]. Mutually
244    /// exclusive with [`Self::skipped`] and [`Self::inconclusive`]:
245    /// the three bits `(passed, skipped, inconclusive)` form a
246    /// strict 4-state encoding where at most one is set per
247    /// record. The fourth state — Fail — is the all-false case
248    /// (no dedicated bit; [`Self::is_fail`] derives it). A real
249    /// pass requires `!skipped && !inconclusive` AND at least one
250    /// observed assertion (the empty / all-skip case routes
251    /// through [`Self::skipped`] instead).
252    pub passed: bool,
253    /// True when the run was skipped (e.g. topology mismatch,
254    /// missing resource, in-VM `AssertResult::skip` return).
255    /// Mutually exclusive with [`Self::passed`] (Pass requires a
256    /// real assertion; an all-skip stream is Skip, not Pass) and
257    /// with [`Self::inconclusive`]. Stats tooling subtracts
258    /// `skipped` runs from "pass count" so non-executions are not
259    /// reported as passes.
260    pub skipped: bool,
261    /// True when at least one assertion was [`Outcome::Inconclusive`](crate::assert::Outcome::Inconclusive) —
262    /// the run ran but a zero-denominator ratio gate could not be
263    /// evaluated (e.g. zero iterations across all workers under a
264    /// `max_migration_ratio` check). Mutually exclusive with
265    /// [`Self::passed`] and [`Self::skipped`]; in the
266    /// `Fail > Inconclusive > Pass > Skip` lattice, Inconclusive
267    /// dominates Pass/Skip but loses to Fail, so a run with both
268    /// Inconclusive and Fail outcomes records `inconclusive = false,
269    /// passed = false` (Fail wins) — `inconclusive = true` requires
270    /// `!is_fail() && !is_pass() && !is_skip()`.
271    ///
272    /// Distinct from `passed = false` (Fail) and `skipped = true`
273    /// (precondition unmet) so CI gates and stats tooling can
274    /// triage zero-denominator runs as "workload didn't produce
275    /// the signal the assertion needed" rather than misclassifying
276    /// them as silent passes (prior to the [`Outcome::Inconclusive`](crate::assert::Outcome::Inconclusive)
277    /// variant the zero-denominator case fell out as Pass) or as
278    /// hard failures.
279    pub inconclusive: bool,
280    /// True when the persisted verdict (`passed`/`skipped`/
281    /// `inconclusive`) is the POST-inversion FINAL outcome of a run
282    /// whose underlying scenario actually failed — i.e. an
283    /// `expect_err` / `expect_auto_repro` test whose induced failure was
284    /// inverted to a pass. Set by the sidecar finalize
285    /// (`finalize_sidecar_verdict`) after dispatch resolves the
286    /// verdict; `false` for an ordinary pass/skip/fail.
287    ///
288    /// The verdict bits carry the FINAL outcome so the footer, `stats`
289    /// analysis, and `replay` match nextest's exit code. This flag
290    /// preserves the one fact that overwrite loses: that the run's
291    /// telemetry is failure-mode-dominated (a deliberately short /
292    /// stalled run). `perf-delta` ORs it into its exclusion guard so
293    /// an inverted-to-pass row is still kept OUT of the regression math
294    /// (its induced-crash telemetry is not real scheduler behavior).
295    pub expected_failure: bool,
296    /// Aggregate per-cgroup statistics merged across every worker.
297    pub stats: ScenarioStats,
298    /// Monitor summary. `None` means the monitor loop did not run
299    /// (host-only tests, early VM failure) or sample collection
300    /// produced no valid data. Writer always emits (`"monitor": null`
301    /// on absence); reader-side, serde's native `Option<T>`
302    /// deserialize tolerates absence — see the module-level doc.
303    pub monitor: Option<MonitorSummary>,
304    /// Periodic-capture coverage for this run: how many periodic snapshot
305    /// boundaries actually fired (`periodic_fired`) out of the configured
306    /// `num_snapshots` target (`periodic_target`). Carried verbatim from
307    /// [`crate::prelude::VmResult`] so cross-run tooling can read the
308    /// coverage off the persisted sidecar (previously only the in-memory
309    /// result exposed it). `0`/`0` for runs with no periodic captures
310    /// configured. Hard-required `u32` fields — old sidecars predating
311    /// them re-generate on the next run (sidecar data is disposable).
312    pub periodic_fired: u32,
313    /// See [`Self::periodic_fired`].
314    pub periodic_target: u32,
315    /// Guest vCPU count and the effective host-CPU budget the vCPU threads
316    /// ran on, carried verbatim from [`crate::prelude::VmResult`]. Drive
317    /// the `cpu-budget` comparison Dimension (cross-budget runs are not
318    /// paired — confining 32 vCPUs to 4 host CPUs measures something else)
319    /// and the overcommit marker: `cpu_budget < vcpus` means the host
320    /// time-sliced the guest's vCPUs, confounding the timing metrics
321    /// (wake-latency / off-CPU / run-delay — schedstat run_delay tracks
322    /// rq->clock, which follows the guest TSC and is not steal-adjusted,
323    /// so the off-host window inflates it for tasks waiting across it).
324    /// Hard-required `u32`
325    /// (old sidecars re-generate; sidecar data is disposable). EXCLUDED
326    /// from `sidecar_variant_hash`: a budget change is a different
327    /// measurement, separated downstream by the Dimension, not the
328    /// identity bucket.
329    pub vcpus: u32,
330    /// See [`Self::vcpus`].
331    pub cpu_budget: u32,
332    /// Host-side vCPU scheduling dilation for this run — the
333    /// `HostVcpuSchedstat::dilation` ratio `1 + Σrun_delay/Σon_cpu`
334    /// over the vCPU host threads. `> 1.0`
335    /// quantifies how much the HOST time-sliced the guest's vCPUs; the
336    /// direct EVIDENCE behind the `cpu_budget < vcpus` overcommit marker
337    /// (a run can overcommit on paper yet see D≈1.0 if the host was
338    /// otherwise idle). `None` when no vCPU ran or the host lacks
339    /// `CONFIG_SCHEDSTATS` (every schedstat line reads `0 0 0`) — kept
340    /// distinguishable from a genuine `D == 1.0`.
341    ///
342    /// `#[serde(default)]`: sidecars written before this field existed
343    /// deserialize with `None` (disposable schema; a re-run repopulates).
344    #[serde(default, skip_serializing_if = "Option::is_none")]
345    pub host_dilation: Option<f64>,
346    /// Denomination of this sidecar's workload-throughput rate metrics.
347    /// `#[serde(default)]` = [`ThroughputDenomination::Wall`] for
348    /// pre-marker sidecars; the writer stamps the current
349    /// [`ThroughputDenomination::CpuSec`]. Keys row pairing and group
350    /// averaging in the compare pipeline so cross-denomination values are
351    /// never silently compared (see the enum doc).
352    #[serde(default)]
353    pub throughput_denomination: ThroughputDenomination,
354    /// Ordered stimulus events published by the guest step executor
355    /// while the scenario ran.
356    pub stimulus_events: Vec<StimulusEvent>,
357    /// WorkSpec type label used for post-hoc filtering and A/B comparison
358    /// (distinct from the `WorkType` enum — this is the text name).
359    pub work_type: String,
360    /// Per-BPF-program verifier statistics captured from the VM's
361    /// scheduler (when one was loaded). Empty when no scheduler
362    /// programs were inspected. Writer always emits as
363    /// `"verifier_stats": []` in that case; reader-side, this `Vec`
364    /// field is hard-required (non-`Option` fields fail deserialize
365    /// on absence). See the module-level doc.
366    pub verifier_stats: Vec<crate::monitor::bpf_prog::ProgVerifierStats>,
367    /// Aggregate per-vCPU KVM stats read after VM exit. `None` when
368    /// the VM did not run (host-only tests) or KVM stats were
369    /// unavailable. Writer always emits (`"kvm_stats": null` on
370    /// absence); reader-side, serde's native `Option<T>` deserialize
371    /// tolerates absence — see the module-level doc.
372    pub kvm_stats: Option<crate::vmm::KvmStatsTotals>,
373    /// Effective sysctls active during this test run, recorded as raw
374    /// `sysctl.key=value` cmdline strings. Writer always emits as
375    /// `"sysctls": []` when none; reader-side, this `Vec` field is
376    /// hard-required (non-`Option` fields fail deserialize on
377    /// absence). See the module-level doc.
378    pub sysctls: Vec<String>,
379    /// Effective kernel command-line args active during this test run.
380    /// Writer always emits as `"kargs": []` when none; reader-side,
381    /// this `Vec` field is hard-required (non-`Option` fields fail
382    /// deserialize on absence). See the module-level doc.
383    pub kargs: Vec<String>,
384    /// Kernel version of the VM under test (from cache metadata,
385    /// e.g. `"6.14.2"`). Populated from the cache entry's
386    /// `metadata.json` version field, with fallback to the kernel
387    /// source tree's `include/config/kernel.release` when
388    /// `KTSTR_KERNEL` points at a raw source path rather than a
389    /// cache key; `None` for host-only tests or when neither
390    /// source yields a version string. The host's running kernel
391    /// release is carried separately in `host.kernel_release`.
392    /// Writer always emits (`"kernel_version": null` on absence);
393    /// reader-side, serde's native `Option<T>` deserialize tolerates
394    /// absence — see the module-level doc for the full asymmetric
395    /// contract.
396    pub kernel_version: Option<String>,
397    /// Kernel SOURCE TREE git HEAD short hex (7 chars via
398    /// `oid::to_hex_with_len(7)`), with `-dirty` suffix appended
399    /// when HEAD-vs-index or index-vs-worktree changes are
400    /// observed. Probes via `gix::open` against the kernel
401    /// directory resolved from `KTSTR_KERNEL` (not `gix::discover`
402    /// — the kernel dir is explicit, not walked-up). Captured by
403    /// `detect_kernel_commit` at sidecar-write time.
404    ///
405    /// Distinct from sibling fields:
406    /// - [`SidecarResult::kernel_version`] — release string read
407    ///   from cache metadata or `include/config/kernel.release`,
408    ///   e.g. `"6.14.2"`. Two runs of `6.14.2` from a clean
409    ///   tree and a `-dirty` worktree at the same HEAD share
410    ///   `kernel_version` but differ on `kernel_commit`.
411    /// - [`SidecarResult::project_commit`] — ktstr framework
412    ///   HEAD captured from the test process's cwd. Tracks
413    ///   "what version of the harness produced this sidecar?"
414    ///   independently of the kernel under test.
415    /// - [`SidecarResult::scheduler_commit`] — userspace
416    ///   scheduler binary's commit (currently always `None`).
417    ///
418    /// `None` when:
419    /// - `KTSTR_KERNEL` is unset or empty;
420    /// - the resolved `KernelId` is `Version` / `CacheKey` whose
421    ///   underlying source is `Tarball` / `Git` (no source tree
422    ///   on disk to probe);
423    /// - the resolved kernel directory is not a git repository
424    ///   (`gix::open` fails);
425    /// - HEAD cannot be read (unborn HEAD on a fresh `git init`
426    ///   with zero commits);
427    /// - any other gix probe failure — metadata, not a gate.
428    ///
429    /// Writer always emits (`"kernel_commit": null` on absence);
430    /// reader-side, serde's native `Option<T>` deserialize tolerates
431    /// absence — see the module-level doc for the full asymmetric
432    /// contract. Excluded from `sidecar_variant_hash` for the same
433    /// cross-host grouping reason `scheduler_commit` and
434    /// `project_commit` are excluded: two runs of the same semantic
435    /// variant on different kernel-source HEADs must still bucket
436    /// together so `perf-delta` can diff them; the commit-drift
437    /// detection inspects this field directly via the
438    /// `--kernel-commit` filter.
439    pub kernel_commit: Option<String>,
440    /// ISO 8601 timestamp of when this test run started.
441    pub timestamp: String,
442    /// Unique identifier for the test run. Composed as
443    /// `{run_id_timestamp}-{counter}` — the `YYYYMMDDTHHMMSSZ`
444    /// process-start stamp followed by a process-local monotonic
445    /// counter. Every sidecar produced in one `cargo ktstr test`
446    /// invocation shares the same timestamp prefix; the counter
447    /// distinguishes concurrent gauntlet variants within that
448    /// invocation. Distinct from the run DIRECTORY name (keyed
449    /// `{kernel}-{project_commit}`, see [`sidecar_dir`]) — the
450    /// directory groups runs by what they tested, the `run_id`
451    /// groups sidecars by which process emitted them.
452    pub run_id: String,
453    /// Host context — static-ish runtime state (CPU model,
454    /// memory size, THP policy, kernel release, host cmdline,
455    /// scheduler tunables). Populated by production sidecar
456    /// writers.
457    ///
458    /// `None` causes:
459    /// - **test-fixture path**: not the production sidecar
460    ///   writer (production writers always populate `host`).
461    /// - **pre-enrichment archive**: sidecar predates the
462    ///   host-context landing — re-run the test to regenerate
463    ///   under the current schema (no migration shim exists
464    ///   per the pre-1.0 disposable-data contract).
465    ///
466    /// Deliberately excluded from the variant hash so
467    /// gauntlet variants on different hosts collapse into the same
468    /// hash bucket.
469    ///
470    /// No serde attributes: writer always emits (`"host": null` when
471    /// `None`); reader-side, serde's native `Option<T>` deserialize
472    /// tolerates absence (a missing key parses as `None`). The
473    /// asymmetric contract is crate-wide — see the module-level doc.
474    /// Pre-1.0, sidecar data is disposable, so regenerate by
475    /// re-running the test rather than carrying a compat shim for
476    /// older JSON; the reader-side tolerance exists so an in-flight
477    /// schema rename of an `Option` field does not break parsing of
478    /// older sidecars during the same producer-version, not as a
479    /// long-term migration story.
480    pub host: Option<crate::host_context::HostContext>,
481    /// Wall-clock milliseconds spent in
482    /// `KtstrVm::collect_results` — the host-side
483    /// teardown window from BSP exit through SHM drain (mirrors
484    /// [`VmResult::cleanup_duration`](crate::vmm::VmResult::cleanup_duration);
485    /// `Duration` is converted to `u64` ms here because every other
486    /// timing field on this struct that lands in a sidecar-comparison
487    /// CLI uses integer ms or seconds, and JSON has no native
488    /// `Duration`). `None` when the run was killed by the watchdog
489    /// before `collect_results` returned, or for the `host_only` /
490    /// host-only-stub paths that never boot a VM. Writer always emits
491    /// (`"cleanup_duration_ms": null` on absence); reader-side,
492    /// serde's native `Option<T>` deserialize tolerates absence — see
493    /// the module-level doc for the full asymmetric contract.
494    pub cleanup_duration_ms: Option<u64>,
495    /// Provenance tag for this sidecar — distinguishes a developer's
496    /// local run from a CI run so cross-environment comparisons in
497    /// `perf-delta` can narrow on (or contrast across) the run
498    /// environment without inferring it from `host`.
499    ///
500    /// Recorded by `detect_run_source` at sidecar-write time:
501    /// - `Some("ci")` when `KTSTR_CI_ENV` is set non-empty (CI runner
502    ///   scripts export it before invoking the test binary; local
503    ///   runs never set it).
504    /// - `Some("local")` otherwise — the default for any sidecar
505    ///   produced by a developer-driven invocation.
506    /// - The third documented value (`"archive"`) is NEVER written
507    ///   here: a sidecar cannot know it will later be archived. The
508    ///   stats CLI applies the `"archive"` tag at LOAD time when its
509    ///   `--dir` flag points at a non-default pool root, overriding
510    ///   whatever was on disk via `apply_archive_source_override`.
511    ///
512    /// `Option<String>` (rather than an enum) keeps the schema
513    /// extensible without a serde-version bump if a future producer
514    /// wants a new tag (e.g. `"benchmark"`); the consumer side
515    /// treats unknown values the same as known ones — they are
516    /// strings the operator can pass via `--run-source` to filter on.
517    /// Writer always emits (`"run_source": null` on absence);
518    /// reader-side, serde's native `Option<T>` deserialize tolerates
519    /// absence — see the module-level doc for the full asymmetric
520    /// contract. Excluded from `sidecar_variant_hash` for the same
521    /// cross-host grouping reason `host` is excluded — two runs of
522    /// the same semantic variant from different environments must
523    /// still bucket together so `perf-delta` can pair them; `--run-source`
524    /// is the explicit knob for source-aware narrowing.
525    ///
526    /// Field name `run_source` (renamed from `source`) disambiguates
527    /// from [`crate::cache::KernelSource`] / `KernelMetadata.source`
528    /// — those describe the kernel build's input (tarball / git /
529    /// local), this describes the run-environment provenance.
530    ///
531    /// **On-disk JSON key changed from `"source"` to `"run_source"`
532    /// in the field rename.** No `#[serde(alias = "source")]` is
533    /// in place: archived sidecars written before the rename carry
534    /// the `"source"` key, which the current schema treats as an
535    /// unknown field. Because `SidecarResult`'s derive does NOT
536    /// set `deny_unknown_fields`, the deserialize does not fail
537    /// outright — instead serde silently DROPS the stale `"source"`
538    /// payload and lands `run_source = None` (since `Option<T>`'s
539    /// "tolerate absence" rule kicks in for the missing
540    /// `"run_source"` field). The data is lost, not preserved. This
541    /// is deliberate per the project's pre-1.0 disposable-data
542    /// contract: re-running tests regenerates sidecars under the
543    /// new key rather than carrying compat shims forward. Consumers
544    /// who need the run-source classification on archived JSON
545    /// must either rename the key in-place before deserialize, or
546    /// re-run the test to regenerate the sidecar with the new
547    /// schema. Tooling that runs against the renamed schema and
548    /// observes a `None` `run_source` cannot distinguish "sidecar
549    /// pre-dates the field" from "sidecar pre-dates the rename and
550    /// lost its tag" — both lower-bound at `None` for filter
551    /// purposes.
552    pub run_source: Option<String>,
553    /// Per-test [`crate::test_support::PerfDeltaAssertion`]s declared on the
554    /// entry, serialized so `cargo ktstr perf-delta --noise-adjust`'s host-side
555    /// compare can enforce them across commits (the entry registry in the parent
556    /// process describes only HEAD's tests, not a baseline/cached sidecar's
557    /// commit, so the declaration must travel WITH the run). Empty when the test
558    /// declared none. Inert here — a normal `cargo ktstr test` writes them but
559    /// never gates on them; only the `--noise-adjust` compare consults them (the
560    /// scalar compare warns that declared gates were skipped).
561    ///
562    /// Writer always emits (`"perf_delta_assertions": []` on absence); reader-
563    /// side this `Vec` field is hard-required (non-`Option` fails deserialize on
564    /// absence) — see the module-level doc for the full contract.
565    pub perf_delta_assertions: Vec<PerfDeltaAssertionRecord>,
566}
567
568/// Owned, serialized mirror of [`crate::test_support::PerfDeltaAssertion`]. The
569/// public declaration type uses `&'static str` (so it stays const/E0493-safe on
570/// the entry) and therefore cannot `Deserialize` into an owned value; this
571/// `String`-backed record is the sidecar carrier the perf-delta compare reads.
572/// `pub` because it is a field of the `pub` [`SidecarResult`] (constructed
573/// across the workspace, including by the `cargo-ktstr` binary crate); the
574/// author-facing declaration type is [`crate::test_support::PerfDeltaAssertion`].
575#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
576pub struct PerfDeltaAssertionRecord {
577    /// Registry metric name this assertion gates (see `stats list-metrics`).
578    pub metric: String,
579    /// Pinned regression direction, or `None` to inherit the registry polarity.
580    pub direction: Option<crate::test_support::Polarity>,
581    /// Relative-regression override (percent), or `None` for `default_rel`.
582    pub max_regression_pct: Option<f64>,
583    /// Absolute-materiality override, or `None` for `default_abs`.
584    pub min_abs: Option<f64>,
585    /// Phase scope (`step_index`), or `None` to gate the aggregate value.
586    pub phase: Option<u16>,
587}
588
589impl From<&crate::test_support::PerfDeltaAssertion> for PerfDeltaAssertionRecord {
590    fn from(a: &crate::test_support::PerfDeltaAssertion) -> Self {
591        Self {
592            metric: a.metric().to_string(),
593            direction: a.direction(),
594            max_regression_pct: a.max_regression_pct(),
595            min_abs: a.min_abs(),
596            phase: a.phase(),
597        }
598    }
599}
600
601impl SidecarResult {
602    /// Convenience accessor mirroring
603    /// [`crate::assert::AssertResult::is_pass`]. SidecarResult is the
604    /// wire-format mirror of an AssertResult; this method exposes the
605    /// same is_pass / is_fail / is_skip / is_inconclusive vocabulary
606    /// so consumers can swap between the two without re-learning
607    /// field names.
608    ///
609    /// Returns true only when the run reached a real Pass — neither
610    /// skipped, inconclusive, nor failed. The triple-conjunct guard
611    /// matches AssertResult's `Fail > Inconclusive > Pass > Skip`
612    /// dominance under the strict 4-state mutex this struct encodes.
613    /// CI gates that want "ship-on-pass" semantics call this method
614    /// and only this method.
615    ///
616    /// Part of the `is_pass` / `is_fail` / `is_inconclusive` /
617    /// `is_skip` vocabulary uniform across the verdict surfaces:
618    /// [`crate::assert::AssertResult::is_pass`] / `Self::is_pass` /
619    /// [`crate::assert::Outcome::is_pass`] / `MonitorVerdict::is_pass`
620    /// (in the `monitor` module, which is `pub(crate)`) /
621    /// `Verdict::is_pass` (re-exported at [`crate::assert::Verdict`]) /
622    /// `GauntletRow::is_pass` (in the `stats` module, which is
623    /// `pub(crate)`).
624    pub fn is_pass(&self) -> bool {
625        self.passed && !self.skipped && !self.inconclusive
626    }
627    /// Convenience accessor mirroring
628    /// [`crate::assert::AssertResult::is_fail`]. The four-state
629    /// encoding uses three stored bits `(passed, skipped,
630    /// inconclusive)` in strict mutual exclusion (at most one
631    /// set); Fail is the all-false derived state, no dedicated
632    /// bit. `is_fail` reads "none of the three bits are set",
633    /// which under `Fail > Inconclusive > Pass > Skip` dominance
634    /// correctly resolves a mixed Fail+Inconclusive stream as
635    /// Fail.
636    pub fn is_fail(&self) -> bool {
637        !self.passed && !self.skipped && !self.inconclusive
638    }
639    /// Convenience accessor mirroring
640    /// [`crate::assert::AssertResult::is_skip`].
641    pub fn is_skip(&self) -> bool {
642        self.skipped
643    }
644    /// Convenience accessor mirroring
645    /// [`crate::assert::AssertResult::is_inconclusive`]. True when
646    /// the run could not be evaluated (zero-denominator ratio gate);
647    /// false on real Pass, real Fail, or Skip. CI gates that gate
648    /// on "did we get a real verdict?" should test
649    /// `r.is_pass() || r.is_fail()` and treat both `is_skip()` and
650    /// `is_inconclusive()` as "couldn't measure".
651    pub fn is_inconclusive(&self) -> bool {
652        self.inconclusive
653    }
654}
655
656#[cfg(test)]
657impl SidecarResult {
658    /// Populated [`SidecarResult`] for unit tests. Every field has a
659    /// reasonable default so call sites only spell out what they want
660    /// to vary via struct-update syntax:
661    ///
662    /// ```ignore
663    /// let sc = SidecarResult {
664    ///     test_name: "my_test".to_string(),
665    ///     passed: false,
666    ///     ..SidecarResult::test_fixture()
667    /// };
668    /// ```
669    ///
670    /// Defaults model a passing EEVDF run on a minimal `1n1l1c1t`
671    /// topology with no payload and no VM telemetry: `test_name="t"`,
672    /// `topology="1n1l1c1t"`, `scheduler="eevdf"`, `work_type="SpinWait"`,
673    /// `passed=true`, `skipped=false`, `inconclusive=false`, every
674    /// [`Option`] `None`, every [`Vec`] empty, `stats` is
675    /// `ScenarioStats::default()`, and both `timestamp`/`run_id` are
676    /// empty strings.
677    ///
678    /// **Prefer this over local `base = || SidecarResult { ... }`
679    /// closures.** A local closure duplicates the default set and
680    /// drifts the moment [`SidecarResult`] grows a field; this fixture
681    /// is the single place those defaults live.
682    ///
683    /// **Hash-stability tests must not rely on these defaults for
684    /// hash-participating fields** (`topology`, `scheduler`, `payload`,
685    /// `work_type`, `sysctls`, `kargs`). Tests that pin
686    /// a [`sidecar_variant_hash`] output against a literal constant
687    /// must spell every hash-participating field out explicitly so a
688    /// future change to these defaults cannot silently shift the
689    /// pinned value.
690    pub(crate) fn test_fixture() -> SidecarResult {
691        SidecarResult {
692            test_name: "t".to_string(),
693            perf_delta_assertions: Vec::new(),
694            topology: "1n1l1c1t".to_string(),
695            scheduler: "eevdf".to_string(),
696            scheduler_commit: None,
697            resolve_source: None,
698            project_commit: None,
699            payload: None,
700            metrics: Vec::new(),
701            passed: true,
702            skipped: false,
703            inconclusive: false,
704            expected_failure: false,
705            stats: crate::assert::ScenarioStats::default(),
706            monitor: None,
707            periodic_fired: 0,
708            periodic_target: 0,
709            vcpus: 1,
710            cpu_budget: 1,
711            host_dilation: None,
712            throughput_denomination: ThroughputDenomination::CpuSec,
713            stimulus_events: Vec::new(),
714            work_type: "SpinWait".to_string(),
715            verifier_stats: Vec::new(),
716            kvm_stats: None,
717            sysctls: Vec::new(),
718            kargs: Vec::new(),
719            kernel_version: None,
720            kernel_commit: None,
721            timestamp: String::new(),
722            run_id: String::new(),
723            host: None,
724            cleanup_duration_ms: None,
725            run_source: None,
726        }
727    }
728}
729
730/// Predicate: is `path` a ktstr sidecar JSON filename?
731///
732/// True iff the path's extension is `json` AND the path's
733/// FILENAME COMPONENT (`Path::file_name`) contains `.ktstr.` —
734/// matching the on-disk shape produced by [`write_sidecar`]
735/// (`<test>-<variant_hash>.ktstr.json`). Both gates are required:
736/// bare `*.json` files (cargo cache, stray fixtures) and non-json
737/// files whose name happens to contain `.ktstr.` (e.g. a log)
738/// are excluded.
739///
740/// The filename-component check (rather than full-path string)
741/// is load-bearing: a parent directory like
742/// `target/foo.ktstr.bar/extra.json` would falsely match a
743/// whole-path `contains(".ktstr.")` while NOT being a sidecar.
744/// `Path::file_name()` returns only the trailing component, so
745/// `.ktstr.` in any ancestor segment cannot trigger the predicate.
746///
747/// Single source of truth for "is this file a sidecar?" — used
748/// by [`collect_sidecars_with_errors`]'s parsing walker and by
749/// the explain-sidecar file-count walker
750/// (`crate::cli::stats_cmds::explain_sidecar::count_sidecar_files`). Both
751/// walkers MUST agree on the predicate so `walked` (count) and
752/// `valid + errors` (parse outcomes) reconcile against each
753/// other; a divergence would let a file count toward `walked`
754/// without contributing to either bucket, manifesting as a
755/// silent-drop count that has no source.
756pub(crate) fn is_sidecar_filename(path: &std::path::Path) -> bool {
757    path.extension().and_then(|e| e.to_str()) == Some("json")
758        && path
759            .file_name()
760            .and_then(|n| n.to_str())
761            .is_some_and(|n| n.contains(".ktstr."))
762}
763
764/// Scan a directory for ktstr sidecar JSON files. Recurses one level
765/// into subdirectories to handle per-job gauntlet layouts.
766///
767/// Convenience wrapper over [`collect_sidecars_with_errors`] for
768/// single-directory callers that only need the parsed sidecars and not
769/// the per-file parse-failure list. Emits ONE aggregated
770/// [`warn_skipped_sidecars`] summary for that directory when stale
771/// sidecars were dropped. Multi-directory walkers must NOT use this in a
772/// loop (it would print one summary per directory) — they call
773/// [`collect_sidecars_with_errors`] per directory and aggregate the counts
774/// into a single summary (see `collect_pool`).
775pub(crate) fn collect_sidecars(dir: &std::path::Path) -> Vec<SidecarResult> {
776    let (sidecars, parse_errors, _io_errors) = collect_sidecars_with_errors(dir);
777    warn_skipped_sidecars(dir, parse_errors.len());
778    sidecars
779}
780
781/// Emit a single aggregated summary for the stale/unparseable sidecars a
782/// walk skipped, or nothing when `skipped == 0`. Sidecars written before a
783/// schema field was added fail to deserialize and are dropped (sidecar data
784/// is disposable — re-running regenerates it); this collapses what would
785/// otherwise be one `eprintln!` per file into one line. Per-file detail
786/// stays available through [`collect_sidecars_with_errors`]'s parse-error
787/// Vec, which `cargo ktstr stats explain-sidecar` renders.
788///
789/// `pub(crate)` so multi-directory walkers outside this module (e.g.
790/// `stats::analyze::sorted_run_entries`) can accumulate `parse_errors.len()`
791/// across run directories and emit ONE pool-wide summary rather than one
792/// per directory.
793pub(crate) fn warn_skipped_sidecars(dir: &std::path::Path, skipped: usize) {
794    if skipped > 0 {
795        eprintln!(
796            "ktstr_test: skipped {skipped} stale sidecar(s) under {} (older \
797             schema — re-run the affected tests to regenerate; \
798             `cargo ktstr stats explain-sidecar --run <run>` shows per-file \
799             detail)",
800            dir.display(),
801        );
802    }
803}
804
805/// Per-file parse-failure record returned by
806/// [`collect_sidecars_with_errors`] and threaded through
807/// `crate::cli::WalkStats::errors` to the renderers.
808///
809/// Named-field struct (rather than a `(PathBuf, String,
810/// Option<String>)` tuple) so call sites read fields by name —
811/// pattern-matching `for err in errors` and accessing
812/// `err.path` / `err.raw_error` / `err.enriched_message`
813/// resists the tuple-position-swap class of bug where positional
814/// fields could destructure in either order without compiler help.
815pub(crate) struct SidecarParseError {
816    /// On-disk path of the sidecar JSON that failed to parse.
817    pub path: std::path::PathBuf,
818    /// Verbatim serde-error string. Kept raw for
819    /// grep-friendly parse-error tracking and surfaced through
820    /// the JSON channel as the `error` key.
821    pub raw_error: String,
822    /// Operator-facing remediation prose computed by
823    /// [`enriched_parse_error_message`]. `Some(...)` for known
824    /// schema-drift cases (currently the `host` missing-field
825    /// pattern), `None` otherwise. Surfaced through the JSON
826    /// channel as `enriched_message`.
827    pub enriched_message: Option<String>,
828}
829
830/// Per-file IO-failure record returned by
831/// [`collect_sidecars_with_errors`] and threaded through
832/// `crate::cli::WalkStats::io_errors` to the renderers.
833///
834/// Captures files where the filename predicate matched but
835/// `std::fs::read_to_string` failed before parsing could begin —
836/// permission denied, mid-rotate truncation, broken symlink,
837/// etc. Distinct from [`SidecarParseError`] (which represents
838/// "file read OK but JSON parse failed"); separating the two
839/// lets dashboard consumers triage filesystem incidents apart
840/// from schema drift.
841///
842/// Named-field struct mirroring [`SidecarParseError`]'s shape so
843/// the renderer side can iterate by field name without tuple-
844/// position fragility. No `enriched_message` field — there is no
845/// remediation catalog for IO failures (causes vary per host:
846/// fix permissions, fix the filesystem, retry the test).
847pub(crate) struct SidecarIoError {
848    /// On-disk path the predicate matched as a sidecar candidate.
849    pub path: std::path::PathBuf,
850    /// Verbatim `std::io::Error` Display string. Surfaced through
851    /// the JSON channel as the `error` key on
852    /// `crate::cli::WalkIoError` entries and through the text
853    /// channel as the `error: ...` line under the `io errors`
854    /// trailing block.
855    pub raw_error: String,
856}
857
858/// Test-only re-export of [`enriched_parse_error_message`] so
859/// `cli::tests` can verify the enrichment-pattern logic
860/// directly against synthetic error strings. The helper itself
861/// stays private so production code routes through
862/// [`collect_sidecars_with_errors`].
863#[cfg(test)]
864pub(crate) fn enriched_parse_error_message_for_test(
865    path: &std::path::Path,
866    raw_error: &str,
867) -> Option<String> {
868    enriched_parse_error_message(path, raw_error)
869}
870
871/// Compute the operator-prose enrichment for a serde parse-error
872/// message, when one applies. Today the only enriched case is the
873/// `host` missing-field schema-drift diagnostic; the function
874/// returns `None` for any other shape so consumers can branch on
875/// "enrichment exists" without re-implementing the match.
876///
877/// Pulled out of [`collect_sidecars_with_errors`]'s parse path so the
878/// enrichment prose is computed in one place and stored in the returned
879/// [`SidecarParseError`]'s `enriched_message` field — parse failures are
880/// surfaced only through that Vec, not a separate stderr channel.
881///
882/// Matching on the Display text is deliberate: serde's typed-error
883/// surface for `missing field "X"` is not stable across
884/// serde_json versions, but the rendered message is — a
885/// forward-compat regression-resilient check costs one string
886/// search.
887fn enriched_parse_error_message(path: &std::path::Path, raw_error: &str) -> Option<String> {
888    let is_missing_host = raw_error.contains("missing field") && raw_error.contains("`host`");
889    if is_missing_host {
890        Some(format!(
891            "ktstr_test: skipping {}: {raw_error} — the `host` field \
892             was added to SidecarResult; pre-1.0 policy is \
893             disposable-sidecar: re-run the test to regenerate this \
894             file under the current schema (no migration shim exists)",
895            path.display(),
896        ))
897    } else {
898        None
899    }
900}
901
902/// Scan a directory for ktstr sidecar JSON files, returning the
903/// parsed sidecars, a [`SidecarParseError`] record (named fields
904/// `path`, `raw_error`, `enriched_message`) for every file that
905/// passed the filename predicate but failed to deserialize, and a
906/// [`SidecarIoError`] record (named fields `path`, `raw_error`)
907/// for every file that passed the predicate but whose
908/// `read_to_string` failed before parsing could begin. Recurses
909/// one level into subdirectories to handle per-job gauntlet
910/// layouts.
911///
912/// Parse failures are captured ONLY in the returned parse-errors vec —
913/// this walker no longer logs per file. Each failure is a
914/// [`SidecarParseError`] record (named fields `path`, `raw_error`,
915/// `enriched_message`) for structured callers (`explain-sidecar`'s walker
916/// output). Both raw and enriched are exposed so dashboard consumers can
917/// pick: raw for parse-error grepping, enriched for human-facing
918/// remediation prose. Callers that only need the sidecars aggregate
919/// `parse_errors.len()` and emit one [`warn_skipped_sidecars`] summary
920/// (see [`collect_sidecars`] / `collect_pool`) rather than one line per
921/// file.
922///
923/// IO failures (third return) get a single eprintln line plus a
924/// structured [`SidecarIoError`] record. Distinguished from
925/// parse failures so dashboard consumers can triage filesystem
926/// incidents (permission denied, mid-rotate truncation, broken
927/// symlink) apart from schema drift. With this third channel,
928/// every predicate-matching file lands in exactly one of the
929/// three returned vecs — the prior implicit
930/// `walked - valid - parse_errors.len()` silent-drop count is
931/// now zero by construction.
932///
933/// Callers that don't need structured errors should use
934/// [`collect_sidecars`].
935pub(crate) fn collect_sidecars_with_errors(
936    dir: &std::path::Path,
937) -> (
938    Vec<SidecarResult>,
939    Vec<SidecarParseError>,
940    Vec<SidecarIoError>,
941) {
942    let mut sidecars = Vec::new();
943    let mut parse_errors: Vec<SidecarParseError> = Vec::new();
944    let mut io_errors: Vec<SidecarIoError> = Vec::new();
945    let entries = match std::fs::read_dir(dir) {
946        Ok(e) => e,
947        Err(e) => {
948            tracing::warn!(
949                dir = %dir.display(),
950                error = %e,
951                "ktstr_test: collect_sidecars_with_errors cannot read root dir",
952            );
953            return (sidecars, parse_errors, io_errors);
954        }
955    };
956    let mut subdirs = Vec::new();
957    let try_load = |path: &std::path::Path,
958                    out: &mut Vec<SidecarResult>,
959                    parse_errs: &mut Vec<SidecarParseError>,
960                    io_errs: &mut Vec<SidecarIoError>| {
961        if !is_sidecar_filename(path) {
962            return;
963        }
964        let data = match std::fs::read_to_string(path) {
965            Ok(d) => d,
966            Err(e) => {
967                let raw = e.to_string();
968                eprintln!("ktstr_test: cannot read {}: {raw}", path.display());
969                io_errs.push(SidecarIoError {
970                    path: path.to_path_buf(),
971                    raw_error: raw,
972                });
973                return;
974            }
975        };
976        match serde_json::from_str::<SidecarResult>(&data) {
977            Ok(sc) => out.push(sc),
978            Err(e) => {
979                let raw = e.to_string();
980                let enriched = enriched_parse_error_message(path, &raw);
981                // Capture (do not log) the per-file skip: callers emit one
982                // aggregated `warn_skipped_sidecars` summary so a directory
983                // of stale sidecars produces a single line, not a flood.
984                // `cargo ktstr stats explain-sidecar --run <run>` renders the
985                // per-file detail (raw + enriched remediation) from this Vec.
986                parse_errs.push(SidecarParseError {
987                    path: path.to_path_buf(),
988                    raw_error: raw,
989                    enriched_message: enriched,
990                });
991            }
992        }
993    };
994    for entry in entries {
995        let entry = match entry {
996            Ok(e) => e,
997            Err(e) => {
998                tracing::warn!(
999                    dir = %dir.display(),
1000                    error = %e,
1001                    "ktstr_test: skipping unreadable DirEntry while collecting sidecars",
1002                );
1003                continue;
1004            }
1005        };
1006        let path = entry.path();
1007        if path.is_dir() {
1008            subdirs.push(path);
1009            continue;
1010        }
1011        try_load(&path, &mut sidecars, &mut parse_errors, &mut io_errors);
1012    }
1013    for sub in subdirs {
1014        let sub_entries = match std::fs::read_dir(&sub) {
1015            Ok(e) => e,
1016            Err(e) => {
1017                tracing::warn!(
1018                    subdir = %sub.display(),
1019                    error = %e,
1020                    "ktstr_test: skipping unreadable subdirectory while collecting sidecars",
1021                );
1022                continue;
1023            }
1024        };
1025        for entry in sub_entries {
1026            let entry = match entry {
1027                Ok(e) => e,
1028                Err(e) => {
1029                    tracing::warn!(
1030                        subdir = %sub.display(),
1031                        error = %e,
1032                        "ktstr_test: skipping unreadable DirEntry in sidecar subdirectory",
1033                    );
1034                    continue;
1035                }
1036            };
1037            try_load(
1038                &entry.path(),
1039                &mut sidecars,
1040                &mut parse_errors,
1041                &mut io_errors,
1042            );
1043        }
1044    }
1045    (sidecars, parse_errors, io_errors)
1046}
1047
1048/// Pool every sidecar JSON under every run directory at `root`.
1049///
1050/// Walks each immediate subdirectory of `root` (one per run, named
1051/// `{kernel}-{project_commit}` by [`sidecar_dir`] where
1052/// `{project_commit}` is the project tree's HEAD short hex with
1053/// `-dirty` suffix when the worktree differs from HEAD) and
1054/// concatenates the sidecars each one yields via
1055/// `collect_sidecars_with_errors` (per directory, so the per-directory
1056/// stale-sidecar skip counts aggregate into one pool-wide summary). The
1057/// result is a flat
1058/// `Vec<SidecarResult>` covering every recorded run on disk —
1059/// `cargo ktstr perf-delta`'s pool-driven sourcing reads it
1060/// once, applies the typed `--a-*` / `--b-*` filters in memory,
1061/// and partitions the survivors into A/B sides.
1062///
1063/// `root` is typically [`runs_root`]; pass an alternate path when
1064/// comparing archived sidecar trees copied off a CI host (the
1065/// `--dir` escape hatch on `perf-delta`).
1066///
1067/// Returns an empty Vec when `root` does not exist or contains no
1068/// run directories. Per-run failure (a corrupt sidecar, a partial
1069/// directory) is counted and skipped — pool-collection never aborts
1070/// on a single bad file, and emits ONE aggregated
1071/// `warn_skipped_sidecars` summary for the whole walk rather than a
1072/// per-file line.
1073///
1074/// Performance: this is a full filesystem walk over `root`. On a
1075/// host with many archived runs (dozens to hundreds), each
1076/// invocation re-reads every sidecar JSON. The cost is acceptable
1077/// for the current operator workflow (one comparison per
1078/// session) but is taskifyable if it becomes a hot path — a
1079/// directory-name fast-path could skip runs whose
1080/// `{kernel}-{project_commit}` prefix does not match the active
1081/// `--a-kernel` / `--b-kernel` filter.
1082pub fn collect_pool(root: &std::path::Path) -> Vec<SidecarResult> {
1083    let entries = match std::fs::read_dir(root) {
1084        Ok(e) => e,
1085        Err(e) => {
1086            tracing::warn!(
1087                root = %root.display(),
1088                error = %e,
1089                "ktstr_test: collect_pool cannot read root; returning empty pool",
1090            );
1091            return Vec::new();
1092        }
1093    };
1094    let mut pool = Vec::new();
1095    let mut skipped = 0usize;
1096    for entry in entries {
1097        let entry = match entry {
1098            Ok(e) => e,
1099            Err(e) => {
1100                tracing::warn!(
1101                    root = %root.display(),
1102                    error = %e,
1103                    "ktstr_test: skipping unreadable DirEntry while collecting pool",
1104                );
1105                continue;
1106            }
1107        };
1108        let path = entry.path();
1109        if path.is_dir() {
1110            // `collect_sidecars_with_errors` already handles "one level of
1111            // subdirectories for per-job gauntlet layouts" inside each run
1112            // directory, so the two-level `{root}/{run_dir}/{job_subdir}`
1113            // shape works without a third walker level. Use the
1114            // error-returning variant (not `collect_sidecars`, which emits
1115            // its own per-directory summary) so the skip counts aggregate
1116            // into ONE pool-wide summary below.
1117            let (sidecars, parse_errors, _io_errors) = collect_sidecars_with_errors(&path);
1118            pool.extend(sidecars);
1119            skipped += parse_errors.len();
1120        }
1121    }
1122    warn_skipped_sidecars(root, skipped);
1123    pool
1124}
1125
1126/// BPF verifier complexity limit (BPF_COMPLEXITY_LIMIT_INSNS).
1127const VERIFIER_INSN_LIMIT: u32 = 1_000_000;
1128
1129/// Percentage of the verifier limit that triggers a warning.
1130const VERIFIER_WARN_PCT: f64 = 75.0;
1131
1132/// Aggregate BPF verifier stats across sidecars into a summary table.
1133///
1134/// verified_insns is deterministic for a given binary, so per-program
1135/// values are deduplicated (max across observations). Flags programs
1136/// using >=75% of the 1M verifier complexity limit.
1137pub(crate) fn format_verifier_stats(sidecars: &[SidecarResult]) -> String {
1138    use std::collections::BTreeMap;
1139
1140    let mut by_name: BTreeMap<&str, u32> = BTreeMap::new();
1141    for sc in sidecars {
1142        for info in &sc.verifier_stats {
1143            let entry = by_name.entry(&info.name).or_insert(0);
1144            *entry = (*entry).max(info.verified_insns);
1145        }
1146    }
1147
1148    if by_name.is_empty() {
1149        return String::new();
1150    }
1151
1152    let mut out = String::from("\n=== BPF VERIFIER STATS ===\n\n");
1153    out.push_str(&format!(
1154        "  {:<24} {:>12} {:>8}\n",
1155        "program", "verified", "limit%"
1156    ));
1157    out.push_str(&format!("  {:-<24} {:-<12} {:-<8}\n", "", "", ""));
1158
1159    let mut warnings = Vec::new();
1160    let mut total: u64 = 0;
1161
1162    for (&name, &verified_insns) in &by_name {
1163        let pct = (verified_insns as f64 / VERIFIER_INSN_LIMIT as f64) * 100.0;
1164        let flag = if pct >= VERIFIER_WARN_PCT { " !" } else { "" };
1165        out.push_str(&format!(
1166            "  {:<24} {:>12} {:>7.1}%{flag}\n",
1167            name, verified_insns, pct,
1168        ));
1169        if pct >= VERIFIER_WARN_PCT {
1170            warnings.push(format!(
1171                "  {name}: {pct:.1}% of 1M limit ({verified_insns} verified insns)",
1172            ));
1173        }
1174        total += verified_insns as u64;
1175    }
1176
1177    out.push_str(&format!("\n  total verified insns: {total}\n"));
1178
1179    if !warnings.is_empty() {
1180        out.push_str("\nWARNING: programs near verifier complexity limit:\n");
1181        for w in &warnings {
1182            out.push_str(w);
1183            out.push('\n');
1184        }
1185    }
1186
1187    out
1188}
1189
1190/// Per-test BPF callback profile from monitor prog_stats_deltas.
1191///
1192/// Shows per-program invocation count, total CPU time, and average
1193/// nanoseconds per call. Each test's profile is printed independently.
1194pub(crate) fn format_callback_profile(sidecars: &[SidecarResult]) -> String {
1195    let mut out = String::new();
1196
1197    for sc in sidecars {
1198        let deltas = match sc
1199            .monitor
1200            .as_ref()
1201            .and_then(|m| m.prog_stats_deltas.as_ref())
1202        {
1203            Some(d) if !d.is_empty() => d,
1204            _ => continue,
1205        };
1206
1207        if out.is_empty() {
1208            out.push_str("\n=== BPF CALLBACK PROFILE ===\n");
1209        }
1210        out.push_str(&format!("\n  {} ({}):\n", sc.test_name, sc.topology));
1211        out.push_str(&format!(
1212            "    {:<24} {:>12} {:>14} {:>12}\n",
1213            "program", "cnt", "total_ns", "avg_ns"
1214        ));
1215        out.push_str(&format!(
1216            "    {:-<24} {:-<12} {:-<14} {:-<12}\n",
1217            "", "", "", ""
1218        ));
1219        for d in deltas {
1220            out.push_str(&format!(
1221                "    {:<24} {:>12} {:>14} {:>12.0}\n",
1222                d.name, d.cnt, d.nsecs, d.nsecs_per_call,
1223            ));
1224        }
1225    }
1226
1227    out
1228}
1229
1230/// Aggregate KVM stats across sidecars into a compact summary.
1231///
1232/// Averages each stat across all tests that returned `Some(KvmStatsTotals)`.
1233/// Tests without KVM stats (non-VM tests, old kernels) are excluded
1234/// from the denominator.
1235pub(crate) fn format_kvm_stats(sidecars: &[SidecarResult]) -> String {
1236    let with_stats: Vec<&crate::vmm::KvmStatsTotals> = sidecars
1237        .iter()
1238        .filter_map(|sc| sc.kvm_stats.as_ref())
1239        .collect();
1240
1241    if with_stats.is_empty() {
1242        return String::new();
1243    }
1244
1245    let n_vms = with_stats.len();
1246
1247    // Compute cross-VM averages for each stat.
1248    let vm_avg = |name: &str| -> u64 {
1249        let sum: u64 = with_stats.iter().map(|d| d.avg(name)).sum();
1250        sum / n_vms as u64
1251    };
1252
1253    let exits = vm_avg("exits");
1254    let halt = vm_avg("halt_exits");
1255    let halt_wait_ns = vm_avg("halt_wait_ns");
1256    let preempted = vm_avg("preemption_reported");
1257    let signal = vm_avg("signal_exits");
1258    let hypercalls = vm_avg("hypercalls");
1259
1260    // Halt poll efficiency across all vCPUs and VMs.
1261    let total_poll_ok: u64 = with_stats
1262        .iter()
1263        .map(|d| d.sum("halt_successful_poll"))
1264        .sum();
1265    let total_poll_try: u64 = with_stats
1266        .iter()
1267        .map(|d| d.sum("halt_attempted_poll"))
1268        .sum();
1269
1270    if exits == 0 {
1271        return String::new();
1272    }
1273
1274    let halt_wait_ms = halt_wait_ns as f64 / 1_000_000.0;
1275    let poll_pct = if total_poll_try > 0 {
1276        (total_poll_ok as f64 / total_poll_try as f64) * 100.0
1277    } else {
1278        0.0
1279    };
1280
1281    let mut out = format!("\n=== KVM STATS (avg across {n_vms} VMs) ===\n\n");
1282    out.push_str(&format!(
1283        "  exits/vcpu  {:>7}   halt/vcpu     {:>5}   halt_wait_ms {:>7.1}\n",
1284        exits, halt, halt_wait_ms,
1285    ));
1286    out.push_str(&format!(
1287        "  poll_ok%    {:>6.1}%   preempted/vcpu {:>4}   signal/vcpu  {:>7}\n",
1288        poll_pct, preempted, signal,
1289    ));
1290    if hypercalls > 0 {
1291        out.push_str(&format!("  hypercalls/vcpu {:>4}\n", hypercalls));
1292    }
1293
1294    // Trust warnings.
1295    if preempted > 0 {
1296        let total: u64 = with_stats
1297            .iter()
1298            .map(|d| d.sum("preemption_reported"))
1299            .sum();
1300        out.push_str(&format!(
1301            "\n  WARNING: {total} host preemptions detected \
1302             -- timing results may be unreliable\n",
1303        ));
1304    }
1305
1306    out
1307}
1308
1309/// Resolve the sidecar output directory for the current test process.
1310///
1311/// Override: `KTSTR_SIDECAR_DIR` (used as-is when non-empty). When
1312/// the override is set, `serialize_and_write_sidecar` ALSO skips
1313/// the per-directory pre-clear so any pre-existing sidecars in
1314/// the operator-chosen directory are preserved verbatim — see
1315/// `sidecar_dir_override`.
1316///
1317/// Default: `{CARGO_TARGET_DIR or "target"}/ktstr/{kernel}-{project_commit}/`,
1318/// where `{kernel}` is the version detected from `KTSTR_KERNEL`'s
1319/// metadata (or `"unknown"` when no kernel is set / detection fails)
1320/// and `{project_commit}` is the project-tree HEAD short hex from
1321/// `detect_project_commit` (with `-dirty` suffix when the worktree
1322/// differs from HEAD), or `"unknown"` when the test process is not
1323/// running inside a git repository or the probe fails. Every sidecar
1324/// written from the same `cargo ktstr test` invocation lands in the
1325/// same directory; two runs sharing the same kernel + project commit
1326/// (e.g. re-running the same suite without committing changes) reuse
1327/// the same directory, with the second run pre-clearing any
1328/// `*.ktstr.json` files left by the first via
1329/// `reset_run_dir_for_session` — the directory is a last-writer-wins
1330/// snapshot keyed on (kernel, project commit), not an append-only
1331/// archive of every invocation.
1332pub fn sidecar_dir() -> PathBuf {
1333    sidecar_dir_override().unwrap_or_else(resolve_default_sidecar_dir)
1334}
1335
1336/// Compute the default-path sidecar directory:
1337/// `{runs_root}/{kernel}-{project_commit}` where `{kernel}` and
1338/// `{project_commit}` come from [`detect_kernel_version`] and
1339/// [`detect_project_commit`] respectively, with `"unknown"`
1340/// substituted via [`format_run_dirname`] when either probe
1341/// returns `None`. Emits the one-shot
1342/// [`warn_unknown_project_commit_once`] stderr warning when the
1343/// project commit probe falls back to `"unknown"` (operators in
1344/// this state lose the per-commit run-directory discriminator).
1345///
1346/// Shared by [`sidecar_dir`] and the default-path branch of
1347/// [`serialize_and_write_sidecar`] so both call sites resolve the
1348/// same kernel/commit/warn/format chain through one place.
1349/// `serialize_and_write_sidecar` cannot call [`sidecar_dir`]
1350/// directly because it needs a single-read of
1351/// [`sidecar_dir_override`] (gated against the env-var flipping
1352/// mid-call between the dir-resolve and the pre-clear gate); the
1353/// helper supplies the default-branch body so the override read
1354/// stays at one site.
1355fn resolve_default_sidecar_dir() -> PathBuf {
1356    let kernel = detect_kernel_version();
1357    let commit = detect_project_commit();
1358    if commit.is_none() {
1359        warn_unknown_project_commit_once();
1360    }
1361    runs_root().join(format_run_dirname(kernel.as_deref(), commit.as_deref()))
1362}
1363
1364/// Build the run-directory leaf name from optional kernel and commit
1365/// components. `None` collapses to the literal `"unknown"` sentinel
1366/// in either slot, so a non-git cwd produces `"{kernel}-unknown"`
1367/// and a missing kernel produces `"unknown-{project_commit}"`. Pure
1368/// function over the two inputs — no I/O — so unit tests can pin
1369/// every shape (clean, dirty, missing-kernel, missing-commit, both
1370/// missing) without driving the [`detect_kernel_version`] /
1371/// [`detect_project_commit`] OnceLocks.
1372///
1373/// SENTINEL ASYMMETRY: the on-disk dirname uses `"unknown"` for
1374/// missing values, but the in-memory [`SidecarResult::project_commit`]
1375/// / [`SidecarResult::kernel_version`] fields stay `None` (`null`
1376/// in JSON). a `project_commit` filter for a specific commit
1377/// will NOT match a sidecar whose `project_commit` is `None` —
1378/// omit the filter to include `None`-commit rows. The asymmetry
1379/// is deliberate: the dirname needs a filesystem-safe sentinel,
1380/// while the JSON field preserves the original probe outcome for
1381/// downstream tooling that distinguishes "no probe ran" from
1382/// "probe ran but found nothing."
1383fn format_run_dirname(kernel: Option<&str>, commit: Option<&str>) -> String {
1384    let kernel = kernel.unwrap_or("unknown");
1385    let commit = commit.unwrap_or("unknown");
1386    format!("{kernel}-{commit}")
1387}
1388
1389/// Resolve the parent directory that holds all test-run subdirectories.
1390///
1391/// Resolution order:
1392/// 1. [`crate::KTSTR_RUNS_ROOT_ENV`] (absolute) — the `cargo ktstr`
1393///    orchestrator stamps this once at startup so its footer / `stats`
1394///    / `replay` reads AND the child test processes' sidecar writes
1395///    resolve the SAME directory regardless of CWD. This is the
1396///    primary path under `cargo ktstr`.
1397/// 2. `{cargo target directory}/ktstr` — the raw `cargo test` /
1398///    `cargo nextest run` fallback, with the target directory resolved
1399///    by `cargo_target_dir` (which asks cargo, so a
1400///    `.cargo/config [build] target-dir` is honored, not just
1401///    `CARGO_TARGET_DIR`). Only this unpinned path pays the
1402///    one-per-process `cargo metadata` spawn; the orchestrator pins the
1403///    absolute override above and never reaches it.
1404///
1405/// Used by `cargo ktstr stats` / `replay` and the post-run footer to
1406/// enumerate runs without reconstructing a specific run key.
1407pub fn runs_root() -> PathBuf {
1408    if let Some(root) = std::env::var_os(crate::KTSTR_RUNS_ROOT_ENV).filter(|v| !v.is_empty()) {
1409        return PathBuf::from(root);
1410    }
1411    cargo_target_dir().join("ktstr")
1412}
1413
1414/// The cargo target directory for the unpinned [`runs_root`] fallback.
1415///
1416/// Asks cargo via `cargo metadata --no-deps` and reads
1417/// `target_directory`, so the answer reflects `CARGO_TARGET_DIR`, a
1418/// `.cargo/config [build] target-dir`, AND the workspace location — a
1419/// bare `CARGO_TARGET_DIR`-or-`"target"` read honors the env var but
1420/// silently ignores a config `target-dir`, landing sidecars under a
1421/// `target/ktstr` that no other tool writes to. Memoized in a
1422/// `OnceLock`: the answer depends only on the process's CWD + config
1423/// (stable for a suite run) and the subprocess spawn is far too costly
1424/// to repeat per sidecar write. Only the raw `cargo test` / nextest
1425/// path reaches here; the `cargo ktstr` orchestrator pins
1426/// [`crate::KTSTR_RUNS_ROOT_ENV`] and short-circuits [`runs_root`]
1427/// before this runs.
1428///
1429/// On any `cargo metadata` failure (cargo absent, non-zero exit,
1430/// unparsable JSON, missing `target_directory`) it falls back to the
1431/// bare behavior: `CARGO_TARGET_DIR` when set non-empty, else the
1432/// CWD-relative `"target"`.
1433fn cargo_target_dir() -> PathBuf {
1434    static TARGET_DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
1435    if let Some(cached) = TARGET_DIR.get() {
1436        return cached.clone();
1437    }
1438    let resolved = std::env::current_dir()
1439        .ok()
1440        .and_then(|cwd| cargo_metadata_target_dir(&cwd))
1441        .unwrap_or_else(|| {
1442            std::env::var("CARGO_TARGET_DIR")
1443                .ok()
1444                .filter(|d| !d.is_empty())
1445                .map(PathBuf::from)
1446                .unwrap_or_else(|| PathBuf::from("target"))
1447        });
1448    let _ = TARGET_DIR.set(resolved.clone());
1449    resolved
1450}
1451
1452/// Run `cargo metadata --no-deps` in `dir` and return its
1453/// `target_directory`. `None` on any failure (cargo absent, non-zero
1454/// exit, unparsable JSON, missing key) so the caller can fall back.
1455/// The subprocess CWD is set on the `Command` from `dir`, never via a
1456/// process-wide `chdir`, so tests drive it against a tempdir-workspace
1457/// without racing the ambient CWD of concurrent tests.
1458fn cargo_metadata_target_dir(dir: &std::path::Path) -> Option<PathBuf> {
1459    let output = std::process::Command::new("cargo")
1460        .args(["metadata", "--format-version", "1", "--no-deps"])
1461        .current_dir(dir)
1462        .output()
1463        .ok()?;
1464    if !output.status.success() {
1465        return None;
1466    }
1467    let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?;
1468    value.get("target_directory")?.as_str().map(PathBuf::from)
1469}
1470
1471/// Predicate: is `entry` a candidate run directory under
1472/// [`runs_root`]?
1473///
1474/// True iff `entry`'s path is a directory AND its filename does
1475/// NOT begin with a `.` byte. The dotfile filter excludes the
1476/// flock sentinel subdirectory ([`crate::flock::LOCK_DIR_NAME`] =
1477/// `.locks`) plus any other operator-created or filesystem-
1478/// reserved dotfile directories from run-listing walkers
1479/// ([`newest_run_dir`] here, `sorted_run_entries` in
1480/// `crate::stats`) so the lock infrastructure does not pollute
1481/// `cargo ktstr stats list` output or claim the "most recent
1482/// run" bucket. Checking the first byte directly via
1483/// `as_encoded_bytes` is OS-string-safe (no UTF-8 round-trip)
1484/// and short-circuits cleanly on non-UTF-8 names that would
1485/// confuse a `to_str().starts_with('.')` chain.
1486///
1487/// Single source of truth for "is this a run-dir entry?" — both
1488/// run-listing call sites must pipe through this predicate so a
1489/// future relocation of `.locks/` (or any other added reserved
1490/// dotfile) updates one place.
1491pub(crate) fn is_run_directory(entry: &std::fs::DirEntry) -> bool {
1492    let path = entry.path();
1493    if !path.is_dir() {
1494        return false;
1495    }
1496    path.file_name()
1497        .and_then(|n| n.as_encoded_bytes().first().copied())
1498        .is_none_or(|b| b != b'.')
1499}
1500
1501/// Find the most recently modified run directory under [`runs_root`].
1502///
1503/// Used by `cargo ktstr stats last-run` when neither `--dir`,
1504/// `KTSTR_SIDECAR_DIR`, nor `--kernel` is set: the stats command
1505/// doesn't itself run a kernel, so it can't reconstruct the
1506/// `{kernel}-{project_commit}` key that the test process used.
1507/// Picking the newest subdirectory by mtime mirrors "show me the
1508/// report from my last test run."
1509///
1510/// Dotfile-prefixed entries (notably the flock sentinel
1511/// subdirectory `.locks/`) are excluded via `is_run_directory`
1512/// so the lock infrastructure cannot claim the "most recent
1513/// run" bucket — `.locks/`'s mtime tracks per-write flock
1514/// activity and would otherwise eclipse the actual newest run
1515/// dir on every default-path sidecar write.
1516pub fn newest_run_dir() -> Option<PathBuf> {
1517    let root = runs_root();
1518    let entries = std::fs::read_dir(&root).ok()?;
1519    entries
1520        .filter_map(|e| e.ok())
1521        .filter(is_run_directory)
1522        .max_by_key(|e| e.metadata().and_then(|m| m.modified()).ok())
1523        .map(|e| e.path())
1524}
1525
1526/// One failed test's on-disk artifacts within a single run directory,
1527/// for the `cargo ktstr test` post-run footer.
1528///
1529/// `scheduler` / `topology` come from a FAILING variant's
1530/// `.ktstr.json` sidecar and are `None` when the test failed BEFORE
1531/// writing one — e.g. a scheduler BPF-load failure that produced only
1532/// a placeholder `.failure-dump.json` via
1533/// [`crate::test_support::eval`] and never reached [`write_sidecar`].
1534/// Each `Option` path is `Some` only when that artifact exists AND
1535/// was written in the current run (the mtime gate in
1536/// [`summarize_run_artifacts`]).
1537pub(crate) struct FailedTest {
1538    /// Bare test function name (the artifact filename prefix).
1539    pub(crate) test_name: String,
1540    /// Scheduler under test, from a FAILING variant's `.ktstr.json`
1541    /// sidecar; `None` when no variant sidecar recorded a failure (a
1542    /// dump-only pre-sidecar failure).
1543    pub(crate) scheduler: Option<String>,
1544    /// Topology label, from the same failing variant as `scheduler`;
1545    /// `None` under the same condition. For a gauntlet test with
1546    /// multiple failing variants this is a representative one; the
1547    /// full per-variant set is in `stats_sidecars`.
1548    pub(crate) topology: Option<String>,
1549    /// Every primary failure dump for this test, sorted. This includes
1550    /// attempt-qualified nextest retry archives as well as the final
1551    /// attempt's canonical path.
1552    pub(crate) failure_dumps: Vec<PathBuf>,
1553    /// Every auto-repro failure dump for this test, sorted, including
1554    /// attempt-qualified nextest retry archives.
1555    pub(crate) repro_failure_dumps: Vec<PathBuf>,
1556    /// Every `{test}-{variant_hash}.ktstr.json` stats sidecar for
1557    /// this test, sorted — one per gauntlet variant (distinct
1558    /// variant hashes coexist). Empty for a dump-only failure.
1559    pub(crate) stats_sidecars: Vec<PathBuf>,
1560    /// `{test}-{variant_hash}.wprof.pb`.
1561    pub(crate) wprof: Option<PathBuf>,
1562    /// `{test}-{variant_hash}.repro.wprof.pb` (auto-repro retry).
1563    pub(crate) repro_wprof: Option<PathBuf>,
1564    /// True when ANY of this test's variant sidecars is `is_fail()`,
1565    /// so `cargo ktstr replay --filter <name>` (which selects from
1566    /// `is_fail` sidecars — see `replay.rs::select_failed_names`)
1567    /// will re-run it. False for dump-only failures (no sidecar), for
1568    /// which replay's pool selection finds nothing.
1569    pub(crate) replayable: bool,
1570}
1571
1572/// Per-run-directory artifact summary for the post-run footer.
1573pub(crate) struct RunDirSummary {
1574    /// The `{runs_root}/{kernel}-{project_commit}` run directory.
1575    pub(crate) dir: PathBuf,
1576    /// Failed tests in this dir, ordered by `test_name`.
1577    pub(crate) failed: Vec<FailedTest>,
1578    /// Count of `.ktstr.json` stats sidecars written this run
1579    /// (every executed VM test that reached [`write_sidecar`],
1580    /// pass or fail).
1581    pub(crate) stats_sidecars: usize,
1582    /// Count of `.wprof.pb` traces written this run (excludes the
1583    /// `.repro.wprof.pb` auto-repro variant).
1584    pub(crate) wprof_traces: usize,
1585    /// Tests skipped because THIS HOST cannot run them, as
1586    /// `(test_name, class)` where `class` is the host-insufficiency
1587    /// tag written by [`write_host_skip_marker`]
1588    /// (`topology_insufficient` / `resource_contention` /
1589    /// `perf_mode_unavailable`). Ordered by `test_name`. The footer
1590    /// groups these by class so an operator sees, per class, how many
1591    /// tests this host could not run and which.
1592    pub(crate) host_skips: Vec<(String, String)>,
1593    /// Tests whose auto-repro probe pipeline had a problem, as
1594    /// `(test_name, reason)` written by [`write_probe_health_marker`]
1595    /// (trigger failed to load/attach, kprobes attached 0 while
1596    /// functions were traceable, or probes attached but captured 0
1597    /// events). Ordered by `test_name`; one footer line each.
1598    pub(crate) probe_issues: Vec<(String, String)>,
1599    /// `expect_err` tests whose inversion was satisfied by a scheduler
1600    /// LOAD/startup failure (not the intended runtime error), written by
1601    /// [`write_expect_err_load_marker`]. Ordered by `test_name`. The
1602    /// footer names them on one line so a suite that is silently green
1603    /// because no scheduler could load becomes visible.
1604    pub(crate) expect_err_load: Vec<String>,
1605}
1606
1607/// The per-test artifact shapes a run directory holds.
1608enum RunArtifactKind {
1609    FailureDump,
1610    ReproFailureDump,
1611    StatsSidecar,
1612    Wprof,
1613    ReproWprof,
1614    /// `{test}-{hash}.host-skip.json` — a host-insufficiency skip
1615    /// marker ([`write_host_skip_marker`]). Its JSON body carries the
1616    /// skip `class`.
1617    HostSkip,
1618    /// `{test}-{hash}.probe-health.json` — an auto-repro probe-pipeline
1619    /// problem marker ([`write_probe_health_marker`]). Its JSON body
1620    /// carries a short `reason`.
1621    ProbeHealth,
1622    /// `{test}-{hash}.expect-err-load.json` — an `expect_err` inversion
1623    /// that was satisfied by a scheduler LOAD/startup failure rather than
1624    /// the runtime scheduler error the test intends
1625    /// ([`write_expect_err_load_marker`]). The presence of the marker is
1626    /// the signal; its JSON body carries only `test_name`. Surfaced so a
1627    /// suite that looks green on a kernel where no scheduler can load
1628    /// (every `expect_err` test's inversion satisfied by the load failure,
1629    /// not the intended stall/crash) becomes visible.
1630    ExpectErrLoad,
1631}
1632
1633/// Split a `{test}-{16-hex variant hash}` stem into `(test, hash)`.
1634///
1635/// Test function names are Rust identifiers (never contain `-`), so the
1636/// LAST `-` is the variant-hash separator. Falls back to `(stem, 0)`
1637/// when the trailing token is not a valid 16-hex hash — so a NON-variant
1638/// dump (a stale pre-variant-keying file, or a future writer that omits
1639/// the hash) still classifies by its full prefix instead of vanishing
1640/// (the "no silent drops" rule). The mtime gate already excludes stale
1641/// prior-run files; the fallback removes the silent-drop risk entirely.
1642fn split_variant_stem(stem: &str) -> (&str, u64) {
1643    if let Some((test, hash)) = stem.rsplit_once('-')
1644        && hash.len() == 16
1645        && let Ok(h) = u64::from_str_radix(hash, 16)
1646    {
1647        (test, h)
1648    } else {
1649        (stem, 0)
1650    }
1651}
1652
1653/// Parse a run-directory filename into `(test_name, variant_hash, kind)`.
1654///
1655/// Returns `None` for filenames that are not a recognized per-test
1656/// artifact — `.ktstr.json.tmp.<pid>.<run_id>` atomic-write staging
1657/// residue, stray non-ktstr files, or a `.ktstr.json` whose stem
1658/// lacks the `-{16-hex variant hash}` suffix [`write_sidecar`]
1659/// always appends.
1660///
1661/// The `variant_hash` lets the footer correlate each artifact with the
1662/// SAME-variant sidecar (a gauntlet test's per-preset dumps + sidecars
1663/// carry distinct hashes): a failure dump whose variant has no parsed
1664/// sidecar is a per-variant pre-sidecar failure even when a sibling
1665/// preset passed. failure-dump / wprof names fall back to `(stem, 0)`
1666/// when un-hashed (see [`split_variant_stem`]); a `.ktstr.json` sidecar
1667/// is ALWAYS variant-keyed by [`write_sidecar`], so a non-hashed one is
1668/// malformed and is dropped (`None`).
1669///
1670/// Failure dumps may include an attempt archive suffix:
1671/// `{test}-{hash}.attempt-N.failure-dump.json` or
1672/// `{test}-{hash}.repro.attempt-N.failure-dump.json`. The attempt is
1673/// intentionally not part of variant identity: every retry is the
1674/// same nextest test variant and is correlated with the same stats
1675/// sidecar hash.
1676fn classify_run_artifact(name: &str) -> Option<(&str, u64, RunArtifactKind)> {
1677    if let Some(stem) = name.strip_suffix(".host-skip.json") {
1678        let (test, hash) = split_variant_stem(stem);
1679        return Some((test, hash, RunArtifactKind::HostSkip));
1680    }
1681    if let Some(stem) = name.strip_suffix(".probe-health.json") {
1682        let (test, hash) = split_variant_stem(stem);
1683        return Some((test, hash, RunArtifactKind::ProbeHealth));
1684    }
1685    if let Some(stem) = name.strip_suffix(".expect-err-load.json") {
1686        let (test, hash) = split_variant_stem(stem);
1687        return Some((test, hash, RunArtifactKind::ExpectErrLoad));
1688    }
1689    if let Some(stem) = name.strip_suffix(".failure-dump.json") {
1690        let stem = match stem.rsplit_once(".attempt-") {
1691            Some((base, attempt))
1692                if attempt
1693                    .parse::<u32>()
1694                    .ok()
1695                    .is_some_and(|attempt| attempt > 0) =>
1696            {
1697                base
1698            }
1699            _ => stem,
1700        };
1701        let (stem, kind) = match stem.strip_suffix(".repro") {
1702            Some(base) => (base, RunArtifactKind::ReproFailureDump),
1703            None => (stem, RunArtifactKind::FailureDump),
1704        };
1705        let (test, hash) = split_variant_stem(stem);
1706        return Some((test, hash, kind));
1707    }
1708    if let Some(stem) = name.strip_suffix(".repro.wprof.pb") {
1709        let (test, hash) = split_variant_stem(stem);
1710        return Some((test, hash, RunArtifactKind::ReproWprof));
1711    }
1712    if let Some(stem) = name.strip_suffix(".wprof.pb") {
1713        let (test, hash) = split_variant_stem(stem);
1714        return Some((test, hash, RunArtifactKind::Wprof));
1715    }
1716    if let Some(stem) = name.strip_suffix(".ktstr.json") {
1717        // A sidecar is ALWAYS `{test}-{16-hex}` ({:016x} in
1718        // serialize_and_write_sidecar). A stem without a valid hash
1719        // suffix is a hand-named / malformed file — drop it (unlike the
1720        // dump arms, there's no un-hashed-sidecar writer to be lenient
1721        // for).
1722        let (test, hash) = stem.rsplit_once('-')?;
1723        if hash.len() == 16
1724            && let Ok(h) = u64::from_str_radix(hash, 16)
1725        {
1726            return Some((test, h, RunArtifactKind::StatsSidecar));
1727        }
1728    }
1729    None
1730}
1731
1732/// Summarize the per-test artifacts a single run directory holds,
1733/// counting only files written at or after `since`.
1734///
1735/// The mtime gate is the freshness boundary: a run directory is
1736/// keyed `{kernel}-{project_commit}` (see [`sidecar_dir`]), so
1737/// re-running the same suite reuses the directory, and
1738/// [`reset_run_dir_for_session`] wipes only `*.ktstr.json` — stale
1739/// `*.failure-dump.json` / `*.wprof.pb` from an earlier run linger.
1740/// Filtering on `mtime >= since` (where `since` is captured before
1741/// the nextest build+run begins, so genuine artifacts — written
1742/// after the build — sort comfortably after it) keeps a stale dump
1743/// from a prior run from surfacing as a current failure.
1744///
1745/// Returns `None` when the directory holds no fresh artifacts (it
1746/// belongs to an earlier run, or cannot be read).
1747fn summarize_one_run_dir(
1748    dir: &std::path::Path,
1749    since: std::time::SystemTime,
1750) -> Option<RunDirSummary> {
1751    use std::collections::{BTreeMap, BTreeSet};
1752    #[derive(Default)]
1753    struct Acc {
1754        // Every per-variant dump, including nextest attempt archives.
1755        // The fail signal below keys off `dump_hashes`; these vectors
1756        // retain all concrete evidence paths for the footer.
1757        failure_dumps: Vec<PathBuf>,
1758        repro_failure_dumps: Vec<PathBuf>,
1759        wprof: Option<PathBuf>,
1760        repro_wprof: Option<PathBuf>,
1761        // EVERY variant's stats sidecar (distinct variant-hash
1762        // filenames coexist), so a passing variant cannot mask a
1763        // failing sibling.
1764        stats_sidecars: Vec<PathBuf>,
1765        // OR of `is_fail` across all of this name's variant sidecars.
1766        // Post-finalize the sidecar carries the FINAL (post-inversion)
1767        // verdict, so a passing expect_err / expect_auto_repro test
1768        // reads `false` here even though its scenario failed.
1769        any_fail: bool,
1770        // Variant hashes whose stats sidecar PARSED, and variant hashes
1771        // that left a failure dump. The gate is PER-VARIANT: a dump whose
1772        // variant has no parsed sidecar is a pre-sidecar failure
1773        // (scheduler load / VM boot crash) for THAT preset and flags
1774        // FAILED even when a sibling preset's sidecar parsed; a dump whose
1775        // variant DID parse a (final, non-failing) sidecar is an
1776        // expected-failure run whose dump must NOT flag — the sidecar's
1777        // finalized verdict already classified it. (A gauntlet test's
1778        // per-preset dumps + sidecars carry distinct variant hashes.)
1779        parsed_sidecar_hashes: BTreeSet<u64>,
1780        dump_hashes: BTreeSet<u64>,
1781        // (scheduler, topology) of the FIRST failing variant seen,
1782        // for the FAILED block header; `None` when no variant sidecar
1783        // parsed as a failure (a dump-only pre-sidecar failure).
1784        fail_variant: Option<(String, String)>,
1785        // Host-insufficiency skip class from a `.host-skip.json` marker
1786        // (whichever variant scanned last — a plain skip has one).
1787        host_skip_class: Option<String>,
1788        // Short auto-repro probe-pipeline problem reason from a
1789        // `.probe-health.json` marker (last variant scanned wins).
1790        probe_health_reason: Option<String>,
1791        // Presence of an `.expect-err-load.json` marker: this test's
1792        // `expect_err` inversion was satisfied by a scheduler load
1793        // failure (any variant sets it — a gauntlet test that load-failed
1794        // on one preset still surfaces).
1795        expect_err_load: bool,
1796    }
1797    let entries = std::fs::read_dir(dir).ok()?;
1798    let mut by_test: BTreeMap<String, Acc> = BTreeMap::new();
1799    let mut stats_sidecars = 0usize;
1800    let mut wprof_traces = 0usize;
1801    for entry in entries.flatten() {
1802        let Ok(meta) = entry.metadata() else {
1803            continue;
1804        };
1805        if !meta.is_file() {
1806            continue;
1807        }
1808        match meta.modified() {
1809            Ok(m) if m >= since => {}
1810            _ => continue,
1811        }
1812        let path = entry.path();
1813        let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
1814            continue;
1815        };
1816        let Some((test, variant_hash, kind)) = classify_run_artifact(name) else {
1817            continue;
1818        };
1819        let acc = by_test.entry(test.to_string()).or_default();
1820        match kind {
1821            RunArtifactKind::FailureDump => {
1822                acc.dump_hashes.insert(variant_hash);
1823                acc.failure_dumps.push(path);
1824            }
1825            RunArtifactKind::ReproFailureDump => {
1826                acc.dump_hashes.insert(variant_hash);
1827                acc.repro_failure_dumps.push(path);
1828            }
1829            RunArtifactKind::Wprof => {
1830                wprof_traces += 1;
1831                acc.wprof = Some(path);
1832            }
1833            RunArtifactKind::ReproWprof => acc.repro_wprof = Some(path),
1834            RunArtifactKind::StatsSidecar => {
1835                stats_sidecars += 1;
1836                // Accumulate EVERY variant (never overwrite by bare
1837                // name) and OR the fail signal, so one gauntlet
1838                // variant's pass cannot mask a failing sibling.
1839                match std::fs::read_to_string(&path)
1840                    .ok()
1841                    .and_then(|s| serde_json::from_str::<SidecarResult>(&s).ok())
1842                {
1843                    Some(sc) => {
1844                        acc.parsed_sidecar_hashes.insert(variant_hash);
1845                        if sc.is_fail() {
1846                            acc.any_fail = true;
1847                            if acc.fail_variant.is_none() {
1848                                acc.fail_variant = Some((sc.scheduler, sc.topology));
1849                            }
1850                        }
1851                    }
1852                    None => {
1853                        // Counted in `stats_sidecars` but
1854                        // unclassifiable. Warn so the count and the
1855                        // failed list cannot silently disagree (a
1856                        // corrupt `is_fail` sidecar would otherwise be
1857                        // swallowed).
1858                        tracing::warn!(
1859                            path = %path.display(),
1860                            "ktstr footer: unreadable/unparseable stats sidecar — \
1861                             counted but not classified",
1862                        );
1863                    }
1864                }
1865                acc.stats_sidecars.push(path);
1866            }
1867            RunArtifactKind::HostSkip => {
1868                // Read the marker's `class` field. A malformed / absent
1869                // body leaves the class unset — the marker's presence
1870                // alone is not enough to name a class, so it is dropped
1871                // rather than rendered as an unlabelled skip.
1872                if let Some(class) = std::fs::read_to_string(&path)
1873                    .ok()
1874                    .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
1875                    .and_then(|v| v.get("class").and_then(|c| c.as_str()).map(String::from))
1876                {
1877                    acc.host_skip_class = Some(class);
1878                }
1879            }
1880            RunArtifactKind::ProbeHealth => {
1881                if let Some(reason) = std::fs::read_to_string(&path)
1882                    .ok()
1883                    .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
1884                    .and_then(|v| v.get("reason").and_then(|r| r.as_str()).map(String::from))
1885                {
1886                    acc.probe_health_reason = Some(reason);
1887                }
1888            }
1889            RunArtifactKind::ExpectErrLoad => {
1890                // Presence alone is the signal — the body carries only
1891                // `test_name` (already the map key). A malformed body does
1892                // not suppress it: the marker exists because the writer
1893                // observed an `expect_err` inversion satisfied by a load
1894                // failure, and that observability must not hinge on a
1895                // parse.
1896                acc.expect_err_load = true;
1897            }
1898        }
1899    }
1900    if by_test.is_empty() {
1901        return None;
1902    }
1903    let mut failed = Vec::new();
1904    let mut host_skips = Vec::new();
1905    let mut probe_issues = Vec::new();
1906    let mut expect_err_load = Vec::new();
1907    for (test_name, mut acc) in by_test {
1908        // Host-insufficiency skip, probe-pipeline problem, and an
1909        // expect_err inversion satisfied by a load failure are all
1910        // orthogonal to the FAILED gate below (a host skip is not a
1911        // failure, a probe problem rides an otherwise-passing auto-repro
1912        // run, and an expect_err-load test's FINAL verdict is a PASS so it
1913        // never reaches `failed`), so collect them BEFORE the failure
1914        // `continue`.
1915        if let Some(class) = acc.host_skip_class.take() {
1916            host_skips.push((test_name.clone(), class));
1917        }
1918        if let Some(reason) = acc.probe_health_reason.take() {
1919            probe_issues.push((test_name.clone(), reason));
1920        }
1921        if acc.expect_err_load {
1922            expect_err_load.push(test_name.clone());
1923        }
1924        // A test FAILED this run if ANY of its variant sidecars records
1925        // `is_fail` (the FINAL post-inversion verdict), OR it left a
1926        // failure dump WITHOUT any parsed sidecar. A dump with no sidecar
1927        // is a pre-sidecar failure (scheduler load / VM boot) that never
1928        // reached `write_sidecar` — it must still flag. But a dump
1929        // alongside a parsed, non-failing sidecar is an expected-failure
1930        // run (expect_err / expect_auto_repro) whose induced-crash dump
1931        // must NOT flag: the sidecar's finalized verdict is authoritative.
1932        // The `is_fail` aggregate covers in-VM assertion failures,
1933        // including one failing gauntlet variant among passing siblings.
1934        // Per-variant dump gate: a failure dump whose variant has NO
1935        // parsed sidecar is a pre-sidecar failure for that preset and
1936        // flags — even when a SIBLING preset's sidecar parsed. (If every
1937        // dump-variant has a parsed sidecar, dump_hashes ⊆ parsed and the
1938        // sidecars' finalized verdicts are authoritative.) Closes the
1939        // mixed-gauntlet masking the old test-name-granularity gate had.
1940        let dump_only_failure = !acc.dump_hashes.is_subset(&acc.parsed_sidecar_hashes);
1941        if !acc.any_fail && !dump_only_failure {
1942            continue;
1943        }
1944        let (scheduler, topology) = match acc.fail_variant {
1945            Some((sch, topo)) => (Some(sch), Some(topo)),
1946            None => (None, None),
1947        };
1948        // Sort so the rendered footer is deterministic regardless of
1949        // `read_dir` order.
1950        acc.failure_dumps.sort();
1951        acc.repro_failure_dumps.sort();
1952        acc.stats_sidecars.sort();
1953        failed.push(FailedTest {
1954            test_name,
1955            scheduler,
1956            topology,
1957            failure_dumps: acc.failure_dumps,
1958            repro_failure_dumps: acc.repro_failure_dumps,
1959            stats_sidecars: acc.stats_sidecars,
1960            wprof: acc.wprof,
1961            repro_wprof: acc.repro_wprof,
1962            replayable: acc.any_fail,
1963        });
1964    }
1965    Some(RunDirSummary {
1966        dir: dir.to_path_buf(),
1967        failed,
1968        stats_sidecars,
1969        wprof_traces,
1970        host_skips,
1971        probe_issues,
1972        expect_err_load,
1973    })
1974}
1975
1976/// Summarize the artifacts every run directory directly under
1977/// `runs_root` holds, keeping only files written at or after
1978/// `since`. Each [`RunDirSummary`] names its failed tests and the
1979/// concrete artifact path for each, so the `cargo ktstr test`
1980/// footer can point an operator at the exact file for the exact
1981/// test that failed rather than a directory + glob legend.
1982///
1983/// `since` is the wall-clock instant captured before the nextest
1984/// build+run; the mtime gate it drives is what excludes stale
1985/// artifacts left in a reused run directory (see
1986/// [`summarize_one_run_dir`]). Directories are returned sorted by
1987/// path so multi-kernel gauntlet output renders deterministically.
1988pub(crate) fn summarize_run_artifacts(
1989    runs_root: &std::path::Path,
1990    since: std::time::SystemTime,
1991) -> Vec<RunDirSummary> {
1992    let Ok(entries) = std::fs::read_dir(runs_root) else {
1993        return Vec::new();
1994    };
1995    let mut out: Vec<RunDirSummary> = entries
1996        .flatten()
1997        .filter(is_run_directory)
1998        .filter_map(|e| summarize_one_run_dir(&e.path(), since))
1999        .collect();
2000    out.sort_by(|a, b| a.dir.cmp(&b.dir));
2001    out
2002}
2003
2004/// Render the `cargo ktstr test` post-run footer: for each run
2005/// directory written at or after `since`, name every FAILED test
2006/// and the concrete path to each of its artifacts (failure dump,
2007/// auto-repro dump, stats sidecar, wprof trace), plus a per-dir
2008/// count of stats sidecars and wprof traces.
2009///
2010/// Returns the empty string when no run directory under `runs_root`
2011/// holds fresh artifacts — a host-only run (no VM tests) writes no
2012/// sidecars, so there is nothing to point at and the caller emits
2013/// no footer.
2014///
2015/// A test is listed FAILED when it left a failure dump (real or
2016/// placeholder) or an `is_fail` stats sidecar. This is NOT an
2017/// exhaustive failure list: a failure that writes neither — a
2018/// `builder.build()` / `vm.run()` error, a pre-build host error
2019/// (kvm probe, kernel/scheduler resolve, validation), a host panic,
2020/// or an unparseable guest result — leaves no on-disk artifact and
2021/// no entry here. The caller (`cargo_ktstr::run_cargo`) treats
2022/// nextest's own exit status as the authoritative pass/fail signal
2023/// and notes, when nextest reports failures, that any failure
2024/// without an entry left no artifact.
2025///
2026/// This replaces a directory + `*.glob` legend that carried no
2027/// test attribution: a reused run directory mixes artifacts from
2028/// many tests (and, before the mtime gate, prior runs), so a glob
2029/// legend pointed an operator at the directory and left them to
2030/// guess which `*.failure-dump.json` belonged to the test that
2031/// just failed.
2032pub fn format_run_artifact_footer(
2033    runs_root: &std::path::Path,
2034    since: std::time::SystemTime,
2035) -> String {
2036    let summaries = summarize_run_artifacts(runs_root, since);
2037    if summaries.is_empty() {
2038        return String::new();
2039    }
2040    let mut out = String::new();
2041    out.push_str("\ncargo ktstr: test outputs\n");
2042    for s in &summaries {
2043        out.push_str(&format!("  {}\n", s.dir.display()));
2044        for f in &s.failed {
2045            // scheduler/topology are set together (both from one
2046            // failing variant) or both absent (dump-only) — see
2047            // `summarize_one_run_dir`; no mixed arm is reachable.
2048            let variant = match (&f.scheduler, &f.topology) {
2049                (Some(sch), Some(topo)) => format!("  [{sch} {topo}]"),
2050                _ => String::new(),
2051            };
2052            out.push_str(&format!("    FAILED  {}{variant}\n", f.test_name));
2053            for p in &f.failure_dumps {
2054                out.push_str(&format!("      {:<13} {}\n", "failure dump", p.display()));
2055            }
2056            for p in &f.repro_failure_dumps {
2057                out.push_str(&format!("      {:<13} {}\n", "repro dump", p.display()));
2058            }
2059            for p in &f.stats_sidecars {
2060                out.push_str(&format!("      {:<13} {}\n", "stats", p.display()));
2061            }
2062            if let Some(p) = &f.wprof {
2063                out.push_str(&format!("      {:<13} {}\n", "wprof", p.display()));
2064            }
2065            if let Some(p) = &f.repro_wprof {
2066                out.push_str(&format!("      {:<13} {}\n", "repro wprof", p.display()));
2067            }
2068            if f.replayable {
2069                out.push_str(&format!(
2070                    "      {:<13} cargo ktstr replay --filter {} --exec\n",
2071                    "replay", f.test_name,
2072                ));
2073            }
2074        }
2075        out.push_str(&render_host_skips(&s.host_skips));
2076        out.push_str(&render_probe_issues(&s.probe_issues));
2077        out.push_str(&render_expect_err_load(&s.expect_err_load));
2078        // Fold the "run stats-last-run for the analysis" discoverability
2079        // hint into the existing count line, but only when this run
2080        // actually wrote a stats sidecar — pointing at an empty analysis
2081        // would be noise. The gauntlet analysis blob is no longer
2082        // auto-printed after a test run; this line is how an operator
2083        // finds it.
2084        if s.stats_sidecars > 0 {
2085            out.push_str(&format!(
2086                "    ({} stats sidecar(s), {} wprof trace(s) written this run) \
2087                 — run `cargo ktstr stats last-run` for the gauntlet analysis\n",
2088                s.stats_sidecars, s.wprof_traces,
2089            ));
2090        } else {
2091            out.push_str(&format!(
2092                "    ({} stats sidecar(s), {} wprof trace(s) written this run)\n",
2093                s.stats_sidecars, s.wprof_traces,
2094            ));
2095        }
2096    }
2097    out
2098}
2099
2100/// Render the host-topology-skip block: one line per skip class, each
2101/// naming its count and the tests this host could not run. Returns the
2102/// empty string when there are no host skips (the common case), so the
2103/// footer stays silent unless this host actually forced a skip.
2104///
2105/// `skips` is `(test_name, class)` collected from the run dir's
2106/// `.host-skip.json` markers; classes are grouped (BTreeMap →
2107/// class-alphabetical, deterministic) and each group's tests are listed
2108/// in the input order (already `test_name`-sorted by
2109/// `summarize_one_run_dir`'s BTreeMap walk).
2110fn render_host_skips(skips: &[(String, String)]) -> String {
2111    if skips.is_empty() {
2112        return String::new();
2113    }
2114    let mut by_class: std::collections::BTreeMap<&str, Vec<&str>> =
2115        std::collections::BTreeMap::new();
2116    for (test, class) in skips {
2117        by_class.entry(class).or_default().push(test);
2118    }
2119    let mut out = String::from("    host-skipped (this host cannot run):\n");
2120    for (class, tests) in by_class {
2121        out.push_str(&format!(
2122            "      {} ({}): {}\n",
2123            class,
2124            tests.len(),
2125            tests.join(", "),
2126        ));
2127    }
2128    out
2129}
2130
2131/// Render the auto-repro probe-health block: one line per test whose
2132/// probe pipeline had a problem, with its short reason. Empty string
2133/// when every probe pipeline was healthy, so the footer says nothing on
2134/// a clean run.
2135fn render_probe_issues(issues: &[(String, String)]) -> String {
2136    if issues.is_empty() {
2137        return String::new();
2138    }
2139    let mut out = String::from("    probe pipeline problems:\n");
2140    for (test, reason) in issues {
2141        out.push_str(&format!("      {test}: {reason}\n"));
2142    }
2143    out
2144}
2145
2146/// Render the `expect_err`-satisfied-by-load-failure block: one line
2147/// naming every test whose `expect_err` inversion passed because the
2148/// scheduler FAILED TO LOAD/attach, not because it hit the runtime error
2149/// the test intends. Empty string when there are none, so the footer is
2150/// silent on a normal run.
2151///
2152/// This is an ADVISORY, not a failure: each named test still PASSED
2153/// (`expect_err` inverted the error). But a whole suite in this block is
2154/// the tell that no scheduler could load on this kernel — the suite is
2155/// green without having exercised anything. See
2156/// [`write_expect_err_load_marker`].
2157fn render_expect_err_load(tests: &[String]) -> String {
2158    if tests.is_empty() {
2159        return String::new();
2160    }
2161    format!(
2162        "    expect_err satisfied by scheduler load failure (not a runtime error): {}\n",
2163        tests.join(", "),
2164    )
2165}
2166
2167/// Detect the kernel version associated with the current test run.
2168///
2169/// Routes through [`crate::ktstr_kernel_env`] for the raw env value
2170/// and [`crate::kernel_path::KernelId`] for variant dispatch so the
2171/// three [`crate::kernel_path::KernelId`] variants are honoured symmetrically:
2172///
2173/// - `KernelId::Path(dir)`: read `metadata.json` (cache entry
2174///   layout) or `include/config/kernel.release` (source tree
2175///   layout). Unchanged from the previous behaviour.
2176/// - `KernelId::Version(ver)`: the user asked for a specific
2177///   version — return it directly. No cache access needed; a
2178///   version string IS a version string.
2179/// - `KernelId::CacheKey(key)`: look up the cache entry and
2180///   return `entry.metadata.version`. The previous code path
2181///   silently treated the key as a directory name and read
2182///   `<cwd>/<key>/metadata.json`, which never matched — producing
2183///   `None` + `sidecar_dir()` using the `"unknown"` fallback even
2184///   though the cache metadata already carried the version.
2185///
2186/// Returns `None` when the env var is unset, or when the env
2187/// resolves to a variant whose underlying source doesn't yield a
2188/// version string (e.g. a Path whose metadata.json / kernel.release
2189/// are both absent, or a CacheKey with no cache hit).
2190pub(crate) fn detect_kernel_version() -> Option<String> {
2191    use crate::kernel_path::KernelId;
2192    let raw = crate::ktstr_kernel_env()?;
2193    match KernelId::parse(&raw) {
2194        KernelId::Path(_) => {
2195            let p = std::path::Path::new(&raw);
2196            let meta_path = p.join("metadata.json");
2197            if let Ok(data) = std::fs::read_to_string(&meta_path)
2198                && let Ok(meta) = serde_json::from_str::<crate::cache::KernelMetadata>(&data)
2199            {
2200                return meta.version;
2201            }
2202            let ver_path = p.join("include/config/kernel.release");
2203            if let Ok(v) = std::fs::read_to_string(ver_path) {
2204                let v = v.trim();
2205                if !v.is_empty() {
2206                    return Some(v.to_string());
2207                }
2208            }
2209            None
2210        }
2211        KernelId::Version(ver) => Some(ver),
2212        KernelId::CacheKey(key) => {
2213            let cache = crate::cache::CacheDir::new().ok()?;
2214            let entry = cache.lookup(&key)?;
2215            entry.metadata.version
2216        }
2217        // Multi-kernel specs in KTSTR_KERNEL never reach this
2218        // function in production — `find_kernel`'s env reader bails
2219        // before sidecar writing happens. This arm is defensive: if
2220        // the env value is somehow a range or git spec, return
2221        // `None` rather than guessing one endpoint, and the sidecar
2222        // record will leave `kernel_version` as null.
2223        KernelId::Range { .. }
2224        | KernelId::Git { .. }
2225        | KernelId::Package { .. }
2226        | KernelId::Distro { .. } => None,
2227    }
2228}
2229
2230/// Detect the ktstr project's git HEAD at sidecar-write time.
2231///
2232/// Walks up from the test process's current working directory via
2233/// `gix::discover` to find an enclosing repository, then reads HEAD
2234/// short-hex (7 chars via `oid::to_hex_with_len(7)`) and appends
2235/// `-dirty` when index-vs-HEAD or worktree-vs-index changes are
2236/// observed. Submodules are ignored
2237/// (`Submodule::Given { ignore: All }`).
2238///
2239/// Dirt-detection runs through the shared [`repo_is_dirty`]
2240/// helper (peel HEAD to its tree, diff tree-vs-index, then
2241/// `status()` for worktree-vs-index, submodules skipped); see its
2242/// doc for cascade details. The cascade is similar in spirit to
2243/// [`crate::fetch::local_source`]'s dirt probe but deliberately
2244/// diverges in missing-index handling: the sidecar path silently
2245/// degrades a missing index leg to "treat as clean" so metadata
2246/// probes never gate sidecar writes, whereas `local_source`'s
2247/// cache-key path treats every leg as load-bearing. The HASH
2248/// REPRESENTATION also DIFFERS: `fetch::local_source` DROPS the
2249/// short hash entirely on dirty (returns `None`) because the
2250/// commit no longer describes the build input the cache key
2251/// embeds — publishing a stale hash there would misidentify the
2252/// build. This helper KEEPS the hash with a `-dirty` suffix
2253/// instead because the sidecar's `project_commit` is a debugging
2254/// breadcrumb (operator-readable identity, not a cache-key input);
2255/// the hash plus dirty flag carries strictly more information
2256/// than `None` for the operator's "which ktstr commit did this
2257/// sidecar come from?" question.
2258///
2259/// Returns `None` when:
2260/// - `current_dir()` cannot be resolved (process has no valid
2261///   cwd — extremely rare; happens only for processes whose cwd
2262///   was rmdir'd while alive);
2263/// - cwd is not inside any git repository (`gix::discover` fails);
2264/// - HEAD cannot be read (an unborn HEAD on a fresh `git init`
2265///   with zero commits, or a corrupt repository).
2266///
2267/// Returns `Some(short_hash)` (without the `-dirty` suffix) when
2268/// the HEAD read succeeds but a downstream dirt-detection call
2269/// fails — including a missing index, an unreadable working tree,
2270/// or `head_tree()` failure. Each failed leg degrades to "treat
2271/// as clean" rather than aborting the probe, because metadata
2272/// must not gate sidecar writes.
2273///
2274/// `None` is the documented fallback — sidecar writing must not
2275/// abort because of a metadata probe failure. Stats tooling that
2276/// reads `project_commit` already tolerates `None` rows by
2277/// treating them as wildcards (no `--project-commit` filter narrowing
2278/// applies).
2279///
2280/// `gix::discover` is preferred over `gix::open` because tests can
2281/// be launched from a subdirectory of the repo (e.g.
2282/// `cd src && cargo test`); `discover` walks parents until it
2283/// finds the `.git` marker, while `open` requires the exact root
2284/// path. The walk is cheap — a few stat() calls bounded by the
2285/// depth of the cwd inside the repo.
2286///
2287/// `env!("CARGO_MANIFEST_DIR")` is deliberately NOT used here:
2288/// `env!` resolves at compile time and bakes the build-host's
2289/// absolute manifest path into the binary's read-only data
2290/// segment, leaking the build environment into every published
2291/// artifact. Resolving cwd at runtime instead means the recorded
2292/// commit reflects the project tree the test was launched FROM —
2293/// for a scheduler crate using ktstr as a dev-dependency, this is
2294/// the scheduler crate's commit, not ktstr's. That is the more
2295/// accurate semantic anyway: "what code produced this sidecar"
2296/// depends on the cwd at test launch (which crate is exercising
2297/// ktstr), not the build host.
2298#[doc(hidden)]
2299pub fn detect_project_commit() -> Option<String> {
2300    // Explicit override: an orchestrator (perf-delta) that checked the
2301    // project tree out WITHOUT a `.git` — a plain gix checkout of a baseline
2302    // commit into a temp dir — passes the commit label via
2303    // KTSTR_PROJECT_COMMIT_ENV so the sidecar records it verbatim instead of
2304    // a `gix::discover` that would resolve to the wrong repo (or none). It is
2305    // also set on the HEAD run so the recorded `project_commit` equals the
2306    // exact label perf-delta filters the pool on, closing the -dirty-suffix
2307    // mismatch between the filter (`short_hash`) and this recorder. Empty is
2308    // treated as unset. Mirrors the KTSTR_KERNEL_COMMIT_ENV override.
2309    if let Ok(explicit) = std::env::var(crate::KTSTR_PROJECT_COMMIT_ENV)
2310        && !explicit.is_empty()
2311    {
2312        return Some(explicit);
2313    }
2314    // Per-process memoization of the SUCCESS case only.
2315    //
2316    // The cwd is stable for the lifetime of a test process (no
2317    // caller mutates it), and the project tree's HEAD plus dirty
2318    // state cannot change underneath us without an explicit user
2319    // action that's outside the scope of any individual sidecar
2320    // write. Gauntlet runs invoke this function once per sidecar —
2321    // thousands of times per process — so caching the resolved
2322    // hash collapses every post-first successful call to a
2323    // `Clone`. The probe itself does ~3 syscalls (gix discover +
2324    // head_id + status) which dominate the sidecar-write critical
2325    // path; eliminating that cost on the hot path is the only
2326    // meaningful perf win available here.
2327    //
2328    // FAILURE IS NOT CACHED: a `None` probe outcome (no git repo
2329    // discoverable from cwd, unborn HEAD, transient FS / gix open
2330    // failure) does NOT seed the OnceLock. A FIRST call from a
2331    // momentarily-broken context (e.g. a test that swapped CWD via
2332    // some indirect path before ever calling
2333    // `detect_project_commit`, or a transient I/O hiccup during
2334    // `gix::discover`) would otherwise lock in `None` for the
2335    // rest of the process — every subsequent sidecar would land
2336    // under `target/ktstr/{kernel}-unknown/` even though the
2337    // commit IS resolvable from a healthy cwd. Retrying on failure
2338    // costs the same ~3 syscalls the success case pays once; the
2339    // re-probe only fires while the answer is still unknown.
2340    //
2341    // CACHE DOES NOT INVALIDATE on success: a user who commits /
2342    // amends / resets the project tree mid-run and expects the
2343    // new HEAD to surface in subsequent sidecars will see stale
2344    // values. This is acceptable — the
2345    // project tree is treated as stable-enough for a single suite
2346    // run; callers mutating the tree during a run own the
2347    // consequences.
2348    static PROJECT_COMMIT: std::sync::OnceLock<String> = std::sync::OnceLock::new();
2349    if let Some(cached) = PROJECT_COMMIT.get() {
2350        return Some(cached.clone());
2351    }
2352    let cwd = std::env::current_dir().ok()?;
2353    let probed = detect_commit_at(&cwd)?;
2354    // `set` on a hot OnceLock is a no-op `Err` — safe to ignore.
2355    // First successful caller wins; a second concurrent caller's
2356    // identical hash discards harmlessly.
2357    let _ = PROJECT_COMMIT.set(probed.clone());
2358    Some(probed)
2359}
2360
2361/// Path-taking core of [`detect_project_commit`]. Factored out so
2362/// unit tests can drive the full branch matrix (clean repo, dirty
2363/// repo, non-git directory, unborn HEAD, concurrent calls) against
2364/// `gix::init`-built fixtures in tempdirs without mutating the
2365/// process-wide `current_dir`. The public entry point reads `cwd`
2366/// once and delegates here.
2367///
2368/// `gix::discover` walks parents until it finds a `.git` marker —
2369/// tests can be launched from a subdirectory of the repo (e.g.
2370/// `cd src && cargo test`); the parent walk handles that, where
2371/// `gix::open` would require the exact root. The
2372/// open-vs-discover distinction is the ONLY difference between
2373/// this function and [`detect_kernel_commit`]; the post-open
2374/// "read HEAD, format short hex, append `-dirty` on dirt" body
2375/// lives in the shared [`commit_with_dirty_suffix`] helper.
2376fn detect_commit_at(path: &std::path::Path) -> Option<String> {
2377    let repo = gix::discover(path).ok()?;
2378    commit_with_dirty_suffix(&repo)
2379}
2380
2381/// Shared post-open body for [`detect_commit_at`] and
2382/// [`detect_kernel_commit`]: read `repo.head_id()`, format the
2383/// 7-char short hex, and append `-dirty` when [`repo_is_dirty`]
2384/// returns `Some(true)`.
2385///
2386/// Returns `None` when `head_id()` fails (unborn HEAD on a fresh
2387/// `gix::init` with zero commits, or a corrupt repository) — the
2388/// short-hex cannot be formed.
2389///
2390/// Returns `Some(short_hash)` (without `-dirty`) when the HEAD
2391/// read succeeds but the [`repo_is_dirty`] probe returns `None`
2392/// (HEAD-tree peel failure). This matches the documented "treat
2393/// as clean on probe failure" degradation: metadata probes must
2394/// not gate sidecar writes, so a probe failure flows through as
2395/// "clean" rather than aborting.
2396///
2397/// `to_hex_with_len(7)` produces a `HexDisplay` that formats 7
2398/// hex chars without the 40-char intermediate `format!("{}")`
2399/// allocation. `Id` derefs to `oid` (gix-hash) which owns the
2400/// method.
2401///
2402/// CALL SITES diverge ONLY on the open mode (`gix::discover` for
2403/// the project commit, `gix::open` for the kernel commit). The
2404/// helper takes a `&Repository` so each caller picks the open
2405/// strategy that matches its semantics: project commit walks
2406/// parents (cwd may be inside a subdir of the repo); kernel
2407/// commit demands the explicit root (the kernel directory is
2408/// not walked-up to avoid resolving the parent ktstr repo).
2409fn commit_with_dirty_suffix(repo: &gix::Repository) -> Option<String> {
2410    let head = repo.head_id().ok()?;
2411    let short_hash = head.to_hex_with_len(7).to_string();
2412    if repo_is_dirty(repo).unwrap_or(false) {
2413        Some(format!("{short_hash}-dirty"))
2414    } else {
2415        Some(short_hash)
2416    }
2417}
2418
2419/// Probe whether a gix repository's working tree differs from its
2420/// HEAD commit, ignoring submodules.
2421///
2422/// Returns `Some(true)` when the index differs from the HEAD tree
2423/// or the worktree differs from the index for any tracked file;
2424/// `Some(false)` when neither leg observed a difference; `None`
2425/// when the HEAD-tree peel itself failed (HEAD points at something
2426/// that cannot be read as a tree).
2427///
2428/// Callers in [`detect_commit_at`] / [`detect_kernel_commit`]
2429/// degrade `None` to "treat as clean" via `unwrap_or(false)` so
2430/// metadata probes never gate sidecar writes.
2431///
2432/// PROBE LEGS:
2433/// - tree-vs-index: peel HEAD to its tree, then `tree_index_status`
2434///   diff against the on-disk index. `repo.index()` returning Err
2435///   (missing index — partially-checked-out clones, or fresh
2436///   `git init` before the first commit) silently leaves the
2437///   index-dirty leg false. `index_or_empty()` is deliberately
2438///   NOT used because it would substitute an empty index and the
2439///   diff would flag every tracked file as "deleted from index",
2440///   tripping false-dirty.
2441/// - index-vs-worktree: `repo.status()` configured with
2442///   `Submodule::Given { ignore: All }` so submodule worktree
2443///   state is skipped. Short-circuited when the tree-vs-index leg
2444///   already flipped dirty: the result only needs one positive
2445///   signal, so a known-dirty index makes the worktree walk
2446///   redundant. Matches the equivalent short-circuit in
2447///   [`crate::fetch::local_source`].
2448///
2449/// FAILURE DEGRADATION: any individual leg failure (missing index,
2450/// `repo.status()` failure, `into_index_worktree_iter()` failure)
2451/// silently degrades that leg to "no signal" rather than aborting.
2452/// The function only returns `None` when the HEAD-tree peel
2453/// fails, because at that point neither leg can run at all.
2454///
2455/// `pub` (not `pub(crate)`) because `cargo-ktstr.rs` is a
2456/// separate `[[bin]]` crate that consumes `ktstr` as an
2457/// external dependency and needs this helper to compute the
2458/// `-dirty` suffix in
2459/// the baseline/HEAD commit in `cargo ktstr perf-delta`. Hidden
2460/// from rustdoc via `#[doc(hidden)]` because it is a probe-
2461/// style helper without a stable API contract — external
2462/// consumers should not depend on it.
2463#[doc(hidden)]
2464pub fn repo_is_dirty(repo: &gix::Repository) -> Option<bool> {
2465    let head_tree_id = repo.head_tree().ok()?.id;
2466
2467    let mut index_dirty = false;
2468    if let Ok(index) = repo.index() {
2469        let _ = repo.tree_index_status(
2470            &head_tree_id,
2471            &index,
2472            None,
2473            gix::status::tree_index::TrackRenames::Disabled,
2474            |_, _, _| {
2475                index_dirty = true;
2476                Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Break(()))
2477            },
2478        );
2479    }
2480
2481    let worktree_dirty = if index_dirty {
2482        false
2483    } else {
2484        repo.status(gix::progress::Discard)
2485            .ok()
2486            .and_then(|s| {
2487                s.index_worktree_rewrites(None)
2488                    .index_worktree_submodules(gix::status::Submodule::Given {
2489                        ignore: gix::submodule::config::Ignore::All,
2490                        check_dirty: false,
2491                    })
2492                    .index_worktree_options_mut(|opts| {
2493                        configure_dirty_status_options(opts);
2494                    })
2495                    .into_index_worktree_iter(Vec::new())
2496                    .ok()
2497                    .map(crate::git_status::consume_has_any)
2498            })
2499            .unwrap_or(false)
2500    };
2501
2502    Some(index_dirty || worktree_dirty)
2503}
2504
2505/// Configure the tracked-file-only sidecar probe while reusing the shared
2506/// gix worker policy.
2507fn configure_dirty_status_options(options: &mut gix::status::index_worktree::Options) {
2508    options.dirwalk_options = None;
2509    crate::git_status::configure_index_worktree_parallelism(options);
2510}
2511
2512/// Detect the kernel SOURCE TREE's git HEAD at sidecar-write time.
2513///
2514/// `kernel_dir` is the explicit kernel source directory — typically
2515/// resolved from `KTSTR_KERNEL` for `KernelId::Path`, or from the
2516/// cache entry's `KernelSource::Local::source_tree_path` when
2517/// `KTSTR_KERNEL` is a Version / CacheKey whose underlying build
2518/// recorded a local tree. Uses `gix::open(kernel_dir)` (NOT
2519/// `gix::discover`) because the kernel directory is explicit, not
2520/// walked-up: the parent walk that `discover` performs would
2521/// resolve to whichever ancestor `.git` it found first, which
2522/// might be the ktstr project's repo when `kernel_dir` is a
2523/// non-git subdirectory inside it. `open` requires `kernel_dir`
2524/// itself to be the repo root, which is the documented invariant
2525/// for kernel checkouts.
2526///
2527/// Reads HEAD short-hex (7 chars via `oid::to_hex_with_len(7)`)
2528/// and appends `-dirty` when index-vs-HEAD or worktree-vs-index
2529/// changes are observed. Dirt-detection runs through the shared
2530/// [`repo_is_dirty`] helper (submodules skipped via
2531/// `Submodule::Given { ignore: All }`); see its doc for cascade
2532/// details. The cascade matches [`detect_project_commit`] and is
2533/// similar in spirit to [`crate::fetch::local_source`] but
2534/// deliberately diverges in missing-index handling: the sidecar
2535/// path silently degrades a missing index leg to "treat as
2536/// clean" so metadata probes never gate sidecar writes, whereas
2537/// `local_source`'s cache-key path treats every leg as
2538/// load-bearing. Same "treat as clean on probe failure"
2539/// degradation rules apply otherwise: a missing index, an
2540/// unreadable worktree, or `head_tree()` failure each fall
2541/// through as "clean" rather than aborting the probe — metadata
2542/// must not gate sidecar writes.
2543///
2544/// HASH REPRESENTATION matches [`detect_project_commit`]: keeps
2545/// the hash with `-dirty` appended (operator-readable identity).
2546/// Distinct from [`crate::fetch::local_source`], which DROPS the
2547/// hash on dirty because the commit no longer describes the
2548/// build INPUT for cache-key purposes.
2549///
2550/// Returns `None` when:
2551/// - `kernel_dir` is not a git repository (`gix::open` fails);
2552/// - HEAD cannot be read (unborn HEAD on a fresh `git init` with
2553///   zero commits, or a corrupt repository).
2554///
2555/// Returns `Some(short_hash)` (without the `-dirty` suffix) when
2556/// the HEAD read succeeds but a downstream dirt-detection call
2557/// fails — including a missing index, an unreadable working
2558/// tree, or `head_tree()` failure. Each failed leg degrades to
2559/// "treat as clean" rather than aborting the probe, because
2560/// metadata must not gate sidecar writes.
2561///
2562/// `pub` (not `pub(crate)`) + `#[doc(hidden)]` because cargo-ktstr is a
2563/// separate `[[bin]]` crate that consumes `ktstr` as a dependency. Its
2564/// resolved-kernel provenance helper falls back to this walk for raw source
2565/// trees and legacy cache entries. Hidden from rustdoc — a probe helper with
2566/// no stable API contract.
2567#[doc(hidden)]
2568pub fn detect_kernel_commit(kernel_dir: &std::path::Path) -> Option<String> {
2569    // Per-process, path-keyed memoization of the SUCCESS case
2570    // only. Same rationale as `detect_project_commit`: gauntlet
2571    // runs invoke this function once per sidecar — thousands of
2572    // times — and the kernel tree's HEAD plus dirty state cannot
2573    // change underneath us mid-suite without an explicit user
2574    // action outside any sidecar's control. The path key handles
2575    // the fixture-test case where unit tests rotate through
2576    // synthetic `tempfile::TempDir` kernel paths in the same
2577    // process; each distinct path memoizes independently.
2578    //
2579    // `Mutex<HashMap>` rather than `OnceLock` because the input
2580    // is parameterized on `kernel_dir` — a `OnceLock` collapses
2581    // every input to one cached result, which would conflate
2582    // different kernel directories into a single value.
2583    // Contention is bounded: post-warm reads are O(1) hash
2584    // lookups against a near-empty map (in production typically
2585    // ONE kernel per process), and the mutex is held only for
2586    // the duration of the lookup + insert.
2587    //
2588    // FAILURE IS NOT CACHED: a `None` probe outcome (kernel_dir
2589    // is not a git repo, unborn HEAD, transient `gix::open`
2590    // failure) does NOT seed the cache. Caching `None` would lock
2591    // in `unknown` for every subsequent sidecar even after the
2592    // condition resolves (e.g. a kernel directory that becomes a
2593    // valid checkout mid-suite, or a flaky FS that recovers).
2594    // Re-probing on failure costs the same gix-open + dirt-walk
2595    // the success case pays once; the re-probe only fires while
2596    // the answer is still unknown for that path.
2597    //
2598    // Mutex poisoning recovery: a panic mid-probe could poison
2599    // the lock; acquiring via
2600    // [`crate::sync::MutexExt::lock_unpoisoned`] returns the
2601    // guard regardless of poison state so a future caller doesn't
2602    // fail catastrophically. The cached map is just a HashMap of
2603    // owned strings; no invariant beyond "key→value mapping" can
2604    // be broken by an interrupted probe.
2605    use std::collections::HashMap;
2606    use std::path::PathBuf;
2607    use std::sync::{Mutex, OnceLock};
2608    static KERNEL_COMMIT_CACHE: OnceLock<Mutex<HashMap<PathBuf, String>>> = OnceLock::new();
2609    // Canonicalize the cache key so two paths that resolve to the
2610    // same on-disk directory share one entry. Without this, a
2611    // symlinked alias (`./linux` symlinked to `/abs/.../linux`)
2612    // and the resolved target would each populate their own slot,
2613    // re-running the gix-open + dirt-walk on every alias and
2614    // defeating the memoization. `canonicalize` resolves symlinks,
2615    // collapses `..` / `.`, and yields the absolute path the
2616    // kernel actually lives at. Falls back to the raw path on
2617    // canonicalize failure (e.g. caller passed a non-existent
2618    // `kernel_dir`) — gix::open will fail downstream and re-probe
2619    // each call until the path becomes resolvable.
2620    let cache_key = kernel_dir
2621        .canonicalize()
2622        .unwrap_or_else(|_| kernel_dir.to_path_buf());
2623    let cache = KERNEL_COMMIT_CACHE.get_or_init(|| Mutex::new(HashMap::new()));
2624    {
2625        let guard = cache.lock_unpoisoned();
2626        if let Some(cached) = guard.get(&cache_key) {
2627            return Some(cached.clone());
2628        }
2629    }
2630    // `gix::open` (NOT `gix::discover`) — `kernel_dir` must BE the
2631    // repo root. Without this the parent walk could resolve to the
2632    // ktstr project's own `.git` when `kernel_dir` is a non-git
2633    // subdirectory inside the ktstr checkout. The
2634    // open-vs-discover distinction is the ONLY difference between
2635    // this function and [`detect_commit_at`]; the post-open
2636    // "read HEAD, format short hex, append `-dirty` on dirt" body
2637    // lives in the shared [`commit_with_dirty_suffix`] helper.
2638    //
2639    // Open against `kernel_dir` (the caller-supplied path) rather
2640    // than `cache_key`. The two paths point at the same on-disk
2641    // repo by construction (canonicalize resolves to the same
2642    // place), so gix opens the same repository either way; passing
2643    // the original keeps any user-facing diagnostics (gix's
2644    // internal error chain) consistent with the input shape.
2645    let result = gix::open(kernel_dir)
2646        .ok()
2647        .and_then(|repo| commit_with_dirty_suffix(&repo));
2648    if let Some(ref hash) = result {
2649        let mut guard = cache.lock_unpoisoned();
2650        // First successful caller wins; a concurrent caller's
2651        // identical hash would overwrite harmlessly because
2652        // success is deterministic for a given (canonicalized
2653        // path, HEAD, dirty state) tuple.
2654        guard.insert(cache_key, hash.clone());
2655    }
2656    result
2657}
2658
2659/// Environment variable CI runners set to mark sidecars they produce
2660/// as `"ci"`-source. Any non-empty value flips the tag; empty string
2661/// is treated as unset so a defensively-cleared variable does not
2662/// accidentally classify a developer run as CI.
2663///
2664/// Read at sidecar-write time by [`detect_run_source`]; matches the
2665/// `KTSTR_KERNEL` / `KTSTR_CACHE_DIR` env-name convention so the
2666/// full set of ktstr-controlled env vars is `KTSTR_*`-prefixed.
2667pub const KTSTR_CI_ENV: &str = "KTSTR_CI";
2668
2669/// Tag value written to [`SidecarResult::run_source`] for sidecars
2670/// produced under [`KTSTR_CI_ENV`].
2671pub const SIDECAR_RUN_SOURCE_CI: &str = "ci";
2672
2673/// Tag value written to [`SidecarResult::run_source`] for sidecars
2674/// produced without [`KTSTR_CI_ENV`] — the developer-machine
2675/// default.
2676pub const SIDECAR_RUN_SOURCE_LOCAL: &str = "local";
2677
2678/// Tag value applied to [`SidecarResult::run_source`] /
2679/// [`GauntletRow::run_source`](crate::stats::GauntletRow::run_source)
2680/// at LOAD time when the consumer pulls sidecars from a non-default
2681/// pool root via `cargo ktstr stats show-host --dir` /
2682/// `cargo ktstr stats list-values --dir`. NEVER written by
2683/// [`write_sidecar`] — the writer cannot know the file will later
2684/// be moved off-host. See [`apply_archive_source_override`].
2685pub const SIDECAR_RUN_SOURCE_ARCHIVE: &str = "archive";
2686
2687/// Read [`KTSTR_CI_ENV`] and classify the run as `"ci"` (when the
2688/// env var is set non-empty) or `"local"` (the default for any
2689/// developer-driven invocation). Empty-string env values count as
2690/// unset — see [`KTSTR_CI_ENV`] for rationale.
2691///
2692/// Returns `Some(_)` unconditionally because every sidecar producer
2693/// is, by construction, either local or CI; an `Option` return
2694/// keeps the field shape symmetric with the other nullable
2695/// `SidecarResult` fields and reserves room for a future "unknown"
2696/// arm without a serde-version bump.
2697pub(crate) fn detect_run_source() -> Option<String> {
2698    match std::env::var(KTSTR_CI_ENV) {
2699        Ok(v) if !v.is_empty() => Some(SIDECAR_RUN_SOURCE_CI.to_string()),
2700        _ => Some(SIDECAR_RUN_SOURCE_LOCAL.to_string()),
2701    }
2702}
2703
2704/// Override every sidecar's `run_source` field to
2705/// [`SIDECAR_RUN_SOURCE_ARCHIVE`] when the consumer pulled the pool
2706/// from a non-default root via `--dir`. Called at the boundary
2707/// between [`collect_pool`] and the downstream stats pipeline so
2708/// on-disk values stay untouched while the in-memory pool reflects
2709/// the operator's intent: "these sidecars were copied off another
2710/// host; treat them as archives, not as the local-machine record."
2711///
2712/// Mutation strategy is in-place rewrite of the entire `run_source`
2713/// field — the `"local"` / `"ci"` distinction is meaningful on the
2714/// PRODUCING host but irrelevant once the sidecars have been
2715/// moved off, where the only useful classification is "archived
2716/// elsewhere." Operators who need to retain the producer-side
2717/// distinction inside an archive bucket can keep `--dir`
2718/// untargeted (read from the default root) and let the on-disk
2719/// values pass through.
2720pub(crate) fn apply_archive_source_override(pool: &mut [SidecarResult]) {
2721    for sc in pool {
2722        sc.run_source = Some(SIDECAR_RUN_SOURCE_ARCHIVE.to_string());
2723    }
2724}
2725
2726/// Resolve the kernel source-tree path for [`detect_kernel_commit`]
2727/// from the [`crate::KTSTR_KERNEL_ENV`] env var.
2728///
2729/// Routes through [`crate::ktstr_kernel_env`] for the raw env
2730/// value and [`crate::kernel_path::KernelId`] for variant
2731/// dispatch:
2732///
2733/// - `KernelId::Path(p)`: probes the path's `metadata.json` first
2734///   — `cargo-ktstr`'s `--kernel /path/to/linux` resolver routes
2735///   clean source trees through the cache pipeline (see
2736///   [`crate::cli::resolve_kernel_dir_to_entry`]) and exports the
2737///   CACHE ENTRY directory through `KTSTR_KERNEL`, not the
2738///   literal source tree. When `metadata.json` parses and carries
2739///   a `KernelSource::Local::source_tree_path`, that path is the
2740///   underlying source tree and is returned. When parsing fails
2741///   (the path IS the source tree, the dirty-tree path that
2742///   skipped the cache store), falls back to using the raw env
2743///   value verbatim — that path is itself the source tree.
2744/// - `KernelId::Version(ver)`: looks for a Local cache entry
2745///   whose `metadata.version == ver` carrying a
2746///   `source_tree_path`. The tarball-shaped key (`{ver}-tarball-
2747///   {arch}-kc{suffix}`) is checked first because it is the
2748///   most-common form a Version-shaped env points at; on miss
2749///   (or hit yielding `Tarball` / `Git` source, both of which
2750///   are transient with no on-disk tree to probe), the function
2751///   falls back to scanning every valid cache entry for a Local
2752///   match on version. Without this fallback,
2753///   a cache populated by `kernel build --kernel
2754///   /path/to/linux` (a Local entry with source_tree_path) is
2755///   never found by a sidecar writer that has
2756///   `KTSTR_KERNEL=6.14.2`, even though the local tree is
2757///   exactly what the kernel_commit field needs to probe.
2758/// - `KernelId::CacheKey(k)`: uses `k` verbatim — the cache key
2759///   already carries every detail (source-type prefix, arch,
2760///   kconfig hash). On hit, returns
2761///   `KernelSource::Local::source_tree_path` if set, else
2762///   `None` (Tarball / Git entries are transient and have no
2763///   persisted source tree).
2764/// - `KernelId::Range { .. }` / `KernelId::Git { .. }`:
2765///   multi-kernel specs in `KTSTR_KERNEL` never reach this
2766///   helper in production (find_kernel's env reader bails
2767///   before sidecar writing). Defensive: returns `None`.
2768///
2769/// Returns `None` when the env var is unset, when no source
2770/// tree path is recoverable, or when the cache lookup fails.
2771#[cfg(test)]
2772fn resolve_kernel_source_dir() -> Option<std::path::PathBuf> {
2773    source_dir_for(&crate::ktstr_kernel_env()?)
2774}
2775
2776/// Resolve a `KTSTR_KERNEL` identifier string to the on-disk SOURCE
2777/// tree whose git HEAD is the kernel's commit (or `None` for transient
2778/// Range/Git specs or an unrecoverable cache lookup).
2779///
2780/// `pub` + `#[doc(hidden)]` because the cargo-ktstr `[[bin]]` is a separate
2781/// crate. [`kernel_commit_for_resolved`] uses this for raw source trees and
2782/// legacy Local cache entries after taking the cache-metadata fast path.
2783#[doc(hidden)]
2784pub fn source_dir_for(raw: &str) -> Option<std::path::PathBuf> {
2785    use crate::kernel_path::KernelId;
2786    let id = KernelId::parse(raw);
2787    match id {
2788        KernelId::Path(_) => {
2789            let p = std::path::Path::new(raw);
2790            // Cache-entry layout: `metadata.json` carries the
2791            // `KernelSource::Local::source_tree_path` recorded at
2792            // build time. Source-tree layout (dirty path that
2793            // skipped cache store): no metadata, so the env value
2794            // IS the source tree. The shared helper handles both.
2795            crate::cache::recover_local_source_tree(p)
2796                .or_else(|| Some(std::path::PathBuf::from(raw)))
2797        }
2798        KernelId::Version(_) | KernelId::CacheKey(_) => {
2799            let cache = crate::cache::CacheDir::new().ok()?;
2800            resolve_kernel_source_dir_with_cache(&id, &cache)
2801        }
2802        KernelId::Range { .. }
2803        | KernelId::Git { .. }
2804        | KernelId::Package { .. }
2805        | KernelId::Distro { .. } => None,
2806    }
2807}
2808
2809/// Resolve the commit which built one already-resolved kernel directory.
2810///
2811/// A normal cargo-ktstr resolution returns a kernel cache-entry directory.
2812/// Its `metadata.json` already records the source commit captured when the
2813/// kernel was built, so that value is both more authoritative and much
2814/// cheaper than walking the current source worktree again. In particular,
2815/// concurrent CI lanes no longer each scan the same Linux checkout before
2816/// starting nextest.
2817///
2818/// Local cache metadata written before `git_hash` was populated retains its
2819/// historical fallback: probe the recorded `source_tree_path`. A raw source
2820/// path (no valid cache metadata) likewise uses [`source_dir_for`] followed by
2821/// [`detect_kernel_commit`]. Valid non-local metadata without a commit has no
2822/// on-disk source to probe and returns `None`.
2823///
2824/// `pub` + `#[doc(hidden)]` lets the cargo-ktstr binary and sidecar writer use
2825/// the exact same provenance decision when producing/consuming the shared
2826/// [`crate::KTSTR_KERNEL_COMMIT_ENV`] map.
2827#[doc(hidden)]
2828pub fn kernel_commit_for_resolved(raw: &str) -> Option<String> {
2829    let resolved = std::path::Path::new(raw);
2830    if let Ok(bytes) = std::fs::read(resolved.join("metadata.json"))
2831        && let Ok(metadata) = serde_json::from_slice::<crate::cache::KernelMetadata>(&bytes)
2832    {
2833        return match metadata.source {
2834            crate::cache::KernelSource::Local {
2835                git_hash: Some(hash),
2836                ..
2837            }
2838            | crate::cache::KernelSource::Git {
2839                git_hash: Some(hash),
2840                ..
2841            } => Some(hash),
2842            crate::cache::KernelSource::Local {
2843                source_tree_path: Some(source),
2844                git_hash: None,
2845            } => detect_kernel_commit(&source),
2846            crate::cache::KernelSource::Local { .. }
2847            | crate::cache::KernelSource::Git { .. }
2848            | crate::cache::KernelSource::Tarball
2849            | crate::cache::KernelSource::DistroPackage { .. }
2850            | crate::cache::KernelSource::LocalPackage { .. } => None,
2851        };
2852    }
2853    source_dir_for(raw).and_then(|source| detect_kernel_commit(&source))
2854}
2855
2856/// Pure helper for [`source_dir_for`] that takes the
2857/// parsed `KernelId` and an opened `CacheDir`, returning the source
2858/// tree path if recoverable.
2859///
2860/// Split out from [`source_dir_for`] so tests can pin a
2861/// `CacheDir` at a tempdir root without mutating env vars (which
2862/// would race other tests reading `KTSTR_KERNEL` /
2863/// `KTSTR_CACHE_DIR`).
2864///
2865/// Lookup order for [`crate::kernel_path::KernelId::Version`]:
2866/// 1. Tarball-shaped cache key (`{ver}-tarball-{arch}-kc{suffix}`),
2867///    direct lookup. Returns `Some` only if the entry is a
2868///    `KernelSource::Local` carrying a `source_tree_path`.
2869/// 2. Fallback scan: every valid cache entry whose
2870///    `metadata.version == ver`. First match with
2871///    `KernelSource::Local::source_tree_path` set wins. Handles
2872///    the case where the user built `--kernel /path/to/linux`
2873///    (a Local cache entry without the tarball cache-key prefix)
2874///    but later set `KTSTR_KERNEL=6.14.2` for the test run —
2875///    without this fallback, the local source tree would be
2876///    invisible to the sidecar writer.
2877///
2878/// `KernelSource::Tarball` and `KernelSource::Git` entries are
2879/// skipped at every step because their source trees are transient
2880/// (deleted by the cache pipeline after build), so probing them
2881/// for a `kernel_commit` would always fail.
2882///
2883/// For [`crate::kernel_path::KernelId::CacheKey`], performs a single direct lookup —
2884/// the cache key already encodes every detail (source-type
2885/// prefix, arch, kconfig hash) so no fallback scan is needed.
2886fn resolve_kernel_source_dir_with_cache(
2887    id: &crate::kernel_path::KernelId,
2888    cache: &crate::cache::CacheDir,
2889) -> Option<std::path::PathBuf> {
2890    use crate::kernel_path::KernelId;
2891    match id {
2892        KernelId::Version(ver) => {
2893            let arch = std::env::consts::ARCH;
2894            let tarball_key = format!("{ver}-tarball-{arch}-kc{}", crate::cache_key_suffix());
2895            if let Some(entry) = cache.lookup(&tarball_key)
2896                && let crate::cache::KernelSource::Local {
2897                    source_tree_path: Some(p),
2898                    ..
2899                } = &entry.metadata.source
2900            {
2901                return Some(p.clone());
2902            }
2903            let entries = cache.list().ok()?;
2904            for listed in entries {
2905                let crate::cache::ListedEntry::Valid(entry) = listed else {
2906                    continue;
2907                };
2908                if entry.metadata.version.as_deref() != Some(ver.as_str()) {
2909                    continue;
2910                }
2911                if let crate::cache::KernelSource::Local {
2912                    source_tree_path: Some(p),
2913                    ..
2914                } = &entry.metadata.source
2915                {
2916                    return Some(p.clone());
2917                }
2918            }
2919            None
2920        }
2921        KernelId::CacheKey(k) => {
2922            let entry = cache.lookup(k)?;
2923            match entry.metadata.source {
2924                crate::cache::KernelSource::Local {
2925                    source_tree_path: Some(ref p),
2926                    ..
2927                } => Some(p.clone()),
2928                _ => None,
2929            }
2930        }
2931        // Path / Range / Git callers do not reach this helper —
2932        // resolve_kernel_source_dir handles them inline. Defensive
2933        // None covers any future caller that adds a new arm.
2934        _ => None,
2935    }
2936}
2937
2938/// The kernel commit recorded in a sidecar: the env fast-path first,
2939/// then resolved-kernel provenance metadata or the raw-source fallback.
2940///
2941/// cargo-ktstr pre-probes every resolved kernel's HEAD once and exports
2942/// a `dir=commit;...` map in [`crate::KTSTR_KERNEL_COMMIT_ENV`], keyed
2943/// by the dir it also exports as `KTSTR_KERNEL`. This process looks
2944/// itself up by its own [`crate::ktstr_kernel_env`] value — string-equal
2945/// to the map key by construction, since cargo-ktstr built both from the
2946/// same resolved dir. A hit skips `detect_kernel_commit`'s gix HEAD +
2947/// dirty-walk, which is memoized per process but NOT across the per-test
2948/// nextest processes (so without the map each of N processes re-pays
2949/// it).
2950///
2951/// Keying on `ktstr_kernel_env()` (the raw `KTSTR_KERNEL`) is deliberate —
2952/// that is exactly the key cargo-ktstr used. The map's commit VALUE matches
2953/// this function's own fallback because both call
2954/// [`kernel_commit_for_resolved`].
2955///
2956/// Miss / empty commit → the shared provenance resolver. Optimization only.
2957fn kernel_commit_for_sidecar() -> Option<String> {
2958    let self_dir = crate::ktstr_kernel_env()?;
2959    if let Ok(raw) = std::env::var(crate::KTSTR_KERNEL_COMMIT_ENV) {
2960        for seg in raw.split(';') {
2961            if let Some((dir, commit)) = seg.rsplit_once('=')
2962                && dir == self_dir
2963                && !commit.is_empty()
2964            {
2965                return Some(commit.to_string());
2966            }
2967        }
2968    }
2969    kernel_commit_for_resolved(&self_dir)
2970}
2971
2972/// Compute a stable 64-bit discriminator over the fields that
2973/// distinguish gauntlet variants of the same test. Used to suffix
2974/// the sidecar filename so concurrent variants do not clobber each
2975/// other's output.
2976///
2977/// Uses [`siphasher::sip::SipHasher13`] with zero keys for the same
2978/// cross-toolchain stability reason as the other zero-keyed
2979/// SipHasher13 sites (`build.rs`, `runtime.rs` `content_hash`) —
2980/// the discriminator
2981/// must be the same across Rust toolchain versions or downstream
2982/// tooling that groups variants by filename breaks.
2983///
2984/// # Host-state collision caveat
2985///
2986/// The hash is over test-identity fields (topology, scheduler,
2987/// payload, work_type, sysctls, kargs) — NOT over
2988/// [`crate::host_context::HostContext`], NOT over `scheduler_commit`, NOT over
2989/// `project_commit`, NOT over `kernel_commit`, NOT over
2990/// `run_source`, NOT over `resolve_source`, and NOT over
2991/// `cpu_budget` / `vcpus`. The
2992/// [`crate::host_context::HostContext`] exclusion is pinned by
2993/// `sidecar_variant_hash_excludes_host_context`; the
2994/// `scheduler_commit` exclusion by
2995/// `sidecar_variant_hash_excludes_scheduler_commit`; the
2996/// `project_commit` exclusion by
2997/// `sidecar_variant_hash_excludes_project_commit`; the
2998/// `kernel_commit` exclusion by
2999/// `sidecar_variant_hash_excludes_kernel_commit`; the
3000/// `run_source` exclusion by
3001/// `sidecar_variant_hash_excludes_run_source`; the
3002/// `resolve_source` exclusion by
3003/// `sidecar_variant_hash_excludes_resolve_source`; the
3004/// `cpu_budget` / `vcpus` exclusion by
3005/// `sidecar_variant_hash_excludes_cpu_budget`. All seven are
3006/// deliberate for the same cross-host grouping reason — a
3007/// gauntlet rebuilt against a different userspace scheduler
3008/// commit, a bumped ktstr checkout, a kernel source tree at a
3009/// different HEAD, a different CI runner / developer machine, a
3010/// run that resolved its scheduler via a different discovery
3011/// path, or a run that confined its vCPUs to a different
3012/// host-CPU budget must still bucket with the same-named variant so
3013/// `compare_partitions` can diff two runs of the "same" test
3014/// without the commit hash, run-source tag, or budget shattering
3015/// them into one-row-per-commit islands. `cpu_budget` / `vcpus`
3016/// are instead surfaced as the [`crate::stats::Dimension::CpuBudget`]
3017/// pairing axis, which separates cross-budget runs at compare time
3018/// rather than at the identity bucket. Callers that want to detect
3019/// a commit drift or compare across run environments inspect
3020/// [`SidecarResult::scheduler_commit`] /
3021/// [`SidecarResult::project_commit`] /
3022/// [`SidecarResult::kernel_commit`] /
3023/// [`SidecarResult::run_source`] /
3024/// [`SidecarResult::resolve_source`] directly (via
3025/// `--project-commit` / `--kernel-commit` / `--run-source` /
3026/// `--resolve-source` on `perf-delta`); the filename stays stable
3027/// across commits and run environments by design.
3028///
3029/// The corollary of the HostContext exclusion: if the host's
3030/// observable state mutates mid-suite — NUMA hotplug, hugepage
3031/// reconfiguration, a `sysctl -w` from a parallel process — two
3032/// runs of the same test will produce the same sidecar filename
3033/// and the later write clobbers the earlier. ktstr treats host
3034/// state as stable-enough for a single suite run; callers
3035/// mutating host state during a run own the ordering themselves
3036/// (e.g. by writing to a different `KTSTR_SIDECAR_DIR` per host
3037/// snapshot).
3038/// The single canonical-JSON + siphash site for the variant hash.
3039///
3040/// [`sidecar_variant_hash`] (from a written [`SidecarResult`]) and
3041/// [`variant_hash_from_parts`] (from a test entry + resolved topology +
3042/// work_type, before any sidecar exists) both route through this so the
3043/// two derivations can never drift. `sysctls`/`kargs` are sorted here
3044/// for order-independence.
3045fn variant_hash_of(
3046    topology: &str,
3047    scheduler: &str,
3048    payload: Option<&str>,
3049    work_type: &str,
3050    sysctls: &[String],
3051    kargs: &[String],
3052) -> u64 {
3053    use siphasher::sip::SipHasher13;
3054    use std::hash::Hasher;
3055    let mut sorted_sysctls = sysctls.to_vec();
3056    sorted_sysctls.sort();
3057    let mut sorted_kargs = kargs.to_vec();
3058    sorted_kargs.sort();
3059    let canonical = serde_json::json!({
3060        "topology": topology,
3061        "scheduler": scheduler,
3062        "payload": payload,
3063        "work_type": work_type,
3064        "sysctls": sorted_sysctls,
3065        "kargs": sorted_kargs,
3066    });
3067    let bytes = serde_json::to_vec(&canonical).expect("json serialization cannot fail for strings");
3068    let mut h = SipHasher13::new_with_keys(0, 0);
3069    h.write(&bytes);
3070    h.finish()
3071}
3072
3073pub(crate) fn sidecar_variant_hash(sidecar: &SidecarResult) -> u64 {
3074    variant_hash_of(
3075        &sidecar.topology,
3076        &sidecar.scheduler,
3077        sidecar.payload.as_deref(),
3078        &sidecar.work_type,
3079        &sidecar.sysctls,
3080        &sidecar.kargs,
3081    )
3082}
3083
3084/// The variant hash for a test entry's run at a given resolved topology
3085/// and `work_type`, computed BEFORE any sidecar exists — the
3086/// failure-dump path (and the Ctx/VmResult `variant_hash` stamp) need
3087/// the identity at VM-build time. Mirrors [`write_sidecar`]'s field
3088/// derivation (topology = the resolved topology, scheduler/sysctls/kargs
3089/// = [`scheduler_fingerprint`], payload = `entry.payload`) so the dump
3090/// filename carries the SAME variant hash the sidecar will. Pinned
3091/// equal to [`sidecar_variant_hash`] by a roundtrip test.
3092pub(crate) fn variant_hash_from_parts(
3093    entry: &KtstrTestEntry,
3094    resolved_topology: &crate::vmm::topology::Topology,
3095    work_type: &str,
3096) -> u64 {
3097    let fp = scheduler_fingerprint(entry);
3098    variant_hash_of(
3099        &resolved_topology.to_string(),
3100        &fp.scheduler,
3101        entry.payload.map(|p| p.name),
3102        work_type,
3103        &fp.sysctls,
3104        &fp.kargs,
3105    )
3106}
3107
3108/// Entry-derived scheduler metadata that every sidecar carries
3109/// regardless of pass/fail/skip.
3110///
3111/// Both write paths ([`write_sidecar`] and [`write_skip_sidecar`])
3112/// thread the same materialized fields through to their
3113/// `SidecarResult` constructors; keeping the derivation in a
3114/// named struct (rather than a 4-tuple) means a new
3115/// scheduler-level field shows up as a named field at both
3116/// writer sites and in every call-site binding, instead of as
3117/// an additional anonymous tuple slot that readers have to
3118/// remember the ordering of.
3119///
3120/// `pub(crate)` rather than `pub`: the intermediate struct is a
3121/// write-path detail, not a public API surface. No serde — this
3122/// is not a persisted shape, just a grouped return value.
3123///
3124/// Derives `Debug` for `assert_eq!` diagnostics, `Clone` so tests
3125/// can materialize a fixture once and reuse it across assertions,
3126/// and `PartialEq`/`Eq` so tests can compare whole fingerprints
3127/// in one statement rather than destructuring and asserting on
3128/// each field.
3129#[derive(Debug, Clone, PartialEq, Eq)]
3130pub(crate) struct SchedulerFingerprint {
3131    /// Pretty scheduler name (matches `SidecarResult::scheduler`),
3132    /// e.g. `"eevdf"` or a scheduler-kind payload's declared name.
3133    pub(crate) scheduler: String,
3134    /// Best-effort userspace scheduler commit; `None` for every
3135    /// current variant per
3136    /// [`crate::test_support::SchedulerSpec::scheduler_commit`].
3137    pub(crate) scheduler_commit: Option<String>,
3138    /// Formatted `sysctl.<key>=<value>` lines derived from the
3139    /// scheduler's declared `sysctls()`.
3140    pub(crate) sysctls: Vec<String>,
3141    /// Kernel command-line args declared by the scheduler,
3142    /// forwarded verbatim.
3143    pub(crate) kargs: Vec<String>,
3144}
3145
3146/// Materialize the [`SchedulerFingerprint`] for a test entry.
3147///
3148/// A change to the sidecar schema (e.g. a new scheduler-level
3149/// field) extends this function + [`SchedulerFingerprint`] in
3150/// one place and every writer picks it up automatically.
3151fn scheduler_fingerprint(entry: &KtstrTestEntry) -> SchedulerFingerprint {
3152    let scheduler = entry.scheduler.name.to_string();
3153    // `SchedulerSpec::scheduler_commit()` returns `None` for every
3154    // variant (Eevdf, Discover, Path, KernelBuiltin) — the commit
3155    // string is not carried in the static spec; it comes from the
3156    // sidecar's run-time git probe instead. This call is here only
3157    // to surface the slot in the fingerprint so a future spec
3158    // variant carrying a commit would flow through automatically.
3159    let scheduler_commit = entry
3160        .scheduler
3161        .binary
3162        .scheduler_commit()
3163        .map(|s| s.to_string());
3164    let sysctls: Vec<String> = entry
3165        .scheduler
3166        .sysctls
3167        .iter()
3168        .map(|s| format!("sysctl.{}={}", s.key(), s.value()))
3169        .collect();
3170    let kargs: Vec<String> = entry
3171        .scheduler
3172        .kargs
3173        .iter()
3174        .map(|s| s.to_string())
3175        .collect();
3176    SchedulerFingerprint {
3177        scheduler,
3178        scheduler_commit,
3179        sysctls,
3180        kargs,
3181    }
3182}
3183
3184/// Compute the per-variant sidecar path and serialize + write the
3185/// result to disk.
3186///
3187/// Gauntlet variants of the same test differ by work_type, flags
3188/// (via scheduler args → sysctls/kargs), scheduler, and topology. A
3189/// filename of just `{test_name}.ktstr.json` causes variants to
3190/// overwrite each other, erasing all but the last-written result.
3191/// `sidecar_variant_hash` hashes the discriminating fields into a
3192/// short stable suffix so each variant gets its own sidecar file.
3193///
3194/// On an orchestrated run-epoch mismatch,
3195/// [`acquire_run_dir_publication_lock`] removes any pre-existing
3196/// `*.ktstr.json` files before publishing the new sentinel so the
3197/// run is a clean snapshot rather than a mosaic of prior invocations.
3198/// Matching-epoch writes append concurrently.
3199///
3200/// Pre-clear is SKIPPED when `KTSTR_SIDECAR_DIR` is set: the
3201/// operator chose that directory and owns its contents — silent
3202/// data loss is not acceptable on an explicit override. When the
3203/// override is unset (the default-path branch),
3204/// `std::fs::create_dir_all` materializes the directory BEFORE
3205/// pre-clear runs so the helper's canonicalize step always sees
3206/// an existing on-disk path; without this ordering, a missing
3207/// dir on the very first call would key the cache against the
3208/// raw path while a later call (after the dir exists) would key
3209/// against the canonicalized absolute path, splitting the cache
3210/// and causing the second call to re-fire pre-clear and wipe the
3211/// first call's sidecars.
3212///
3213/// CROSS-PROCESS PUBLICATION: on the default path (override unset),
3214/// writers with a matching [`crate::KTSTR_RUN_EPOCH_ENV`] sentinel
3215/// acquire advisory `LOCK_SH` on
3216/// `{runs_root}/.locks/{key}.lock`, recheck the token under that
3217/// lock, and retain SH through their atomic rename. Matching writers
3218/// therefore publish concurrently. A first writer or epoch change
3219/// acquires `LOCK_EX`, rechecks, then performs wipe -> sentinel ->
3220/// first publication. Since EX conflicts with every matching
3221/// writer's SH, an epoch reset cannot cross the validated-token ->
3222/// rename interval. The override path skips the lock for the same
3223/// reason it skips pre-clear: operator-chosen directories are owned
3224/// by the operator, so we do not place a `.locks/` sibling inside
3225/// (or above) their custom layout.
3226///
3227/// A later peer whose sentinel already matches bypasses pre-clear
3228/// and takes only the shared publication rail. Without that sentinel
3229/// a later peer's pre-clear would delete an earlier peer's freshly
3230/// written sidecar — silent stats loss. Raw `cargo nextest run` sets
3231/// no token, so its peers retain the serialized wipe-everything
3232/// behavior.
3233///
3234/// PER-FILE ATOMICITY (both branches): the JSON is written to a
3235/// `<final>.tmp.<pid>.<run_id>` sibling and then `rename(2)`'d into
3236/// place. POSIX `rename` is atomic for same-directory destinations,
3237/// so a peer reader (`collect_sidecars`) never observes a partial
3238/// JSON payload — either the old contents stay or the new contents
3239/// replace them in one filesystem step. Two concurrent writers that
3240/// both target the same `{test_name}-{variant_hash}.ktstr.json`
3241/// (override path: two CI jobs sharing one operator-chosen dir;
3242/// default path: a torn-write window inside the flock body that the
3243/// flock would otherwise have to cover) cannot leave a half-written
3244/// JSON behind — last-rename-wins, both files are individually
3245/// well-formed. The `.tmp.<pid>.<run_id>` discriminator on the
3246/// staging name keeps two writers from racing on the same staging
3247/// path even when their final destinations collide. The flock on
3248/// the default path remains load-bearing for the pre-clear leg
3249/// (atomic write only protects the write itself, not the
3250/// `read_dir + remove_file` walk that pre-clear runs).
3251///
3252/// `label` is a caller-supplied noun for the context message ("skip
3253/// sidecar" / "sidecar") so the error chain points at the right call
3254/// site.
3255fn serialize_and_write_sidecar(sidecar: &SidecarResult, label: &str) -> anyhow::Result<()> {
3256    // Read the override ONCE. The two branches below carry the
3257    // result through structurally so neither leg re-reads
3258    // `KTSTR_SIDECAR_DIR` — preventing the override from flipping
3259    // mid-call (which would otherwise let an external mutation
3260    // between the dir resolve and the pre-clear gate either skip
3261    // the wipe on a default-path dir or fire a wipe on an
3262    // operator-chosen one).
3263    let (dir, do_pre_clear) = match sidecar_dir_override() {
3264        Some(path) => (path, false),
3265        None => (resolve_default_sidecar_dir(), true),
3266    };
3267    // Materialize the directory FIRST so `pre_clear_run_dir_once`
3268    // can canonicalize a path that exists on disk. Without this,
3269    // the very first invocation in a process resolves the cache
3270    // key against the raw relative path (canonicalize fails on a
3271    // missing dir, falls back to raw); subsequent invocations
3272    // resolve against the canonicalized absolute path because the
3273    // dir now exists. Two distinct keys for the same logical dir
3274    // → second invocation re-fires pre-clear and wipes the first
3275    // invocation's sidecars. Materializing pre-pre-clear closes
3276    // the relative-vs-absolute split.
3277    std::fs::create_dir_all(&dir)
3278        .with_context(|| format!("create sidecar dir {}", dir.display()))?;
3279    let variant_hash = sidecar_variant_hash(sidecar);
3280    let path = dir.join(format!(
3281        "{}-{:016x}.ktstr.json",
3282        sidecar.test_name, variant_hash
3283    ));
3284    let json = serde_json::to_string_pretty(sidecar)
3285        .with_context(|| format!("serialize {label} for '{}'", sidecar.test_name))?;
3286    // Atomic write: stage into a `.tmp.<pid>.<run_id>` sibling and
3287    // rename(2) into the final path. `rename` is atomic for
3288    // same-directory destinations on every filesystem ktstr supports
3289    // (ext4, btrfs, xfs, tmpfs, overlayfs); a peer reader never
3290    // observes a partial payload. The staging name carries the pid
3291    // AND the unique sidecar `run_id` so two writers in the same
3292    // process targeting identical final paths (e.g. two threads in
3293    // the budget-test stdout-capture path) cannot stomp each other's
3294    // staging file before either rename lands. On rename failure the
3295    // staging file is removed so a partial sidecar does not survive
3296    // as garbage in the run dir; rename success consumes the staging
3297    // entry and there is nothing to clean up.
3298    let pid = std::process::id();
3299    let staging = dir.join(format!(
3300        "{}-{:016x}.ktstr.json.tmp.{pid}.{}",
3301        sidecar.test_name, variant_hash, sidecar.run_id,
3302    ));
3303
3304    // Serialize before admission to keep both the shared publication
3305    // interval and the exclusive epoch-reset interval limited to filesystem
3306    // mutation. On the orchestrated path, a matching epoch takes LOCK_SH:
3307    // all peer writers publish concurrently, while an epoch reset's LOCK_EX
3308    // cannot cross the validated-token -> rename interval. A missing or
3309    // mismatched epoch takes LOCK_EX, rechecks under that lock, then performs
3310    // wipe -> sentinel before publishing. Raw no-token runs retain the
3311    // historical exclusive, pre-clear-once behavior. The override branch
3312    // skips both coordination and pre-clear.
3313    let session_token = if do_pre_clear {
3314        run_session_token()
3315    } else {
3316        None
3317    };
3318    let _publication_lock = if do_pre_clear {
3319        Some(acquire_run_dir_publication_lock(
3320            &dir,
3321            session_token.as_deref(),
3322        )?)
3323    } else {
3324        None
3325    };
3326    if let Err(error) = std::fs::write(&staging, &json) {
3327        // `write` may have created or partially filled the staging inode
3328        // before reporting an error. Never leave that residue for the rest
3329        // of the epoch; the next reset is not guaranteed to happen soon.
3330        let _ = std::fs::remove_file(&staging);
3331        return Err(anyhow::Error::from(error)
3332            .context(format!("write {label} staging {}", staging.display())));
3333    }
3334    if let Err(e) = std::fs::rename(&staging, &path) {
3335        // Best-effort cleanup of the staged payload; ignore the
3336        // unlink error so the rename failure is what surfaces
3337        // (the rename error names the actual problem).
3338        let _ = std::fs::remove_file(&staging);
3339        return Err(anyhow::Error::from(e).context(format!(
3340            "rename {label} staging {} -> {}",
3341            staging.display(),
3342            path.display(),
3343        )));
3344    }
3345    LAST_SIDECAR_PATH.with(|p| *p.borrow_mut() = Some(path.clone()));
3346    Ok(())
3347}
3348
3349thread_local! {
3350    /// Absolute path of the most recent sidecar this thread wrote (via
3351    /// [`serialize_and_write_sidecar`]). The dispatch run loop
3352    /// ([`crate::test_support::eval::run_ktstr_test_inner`]) reads and
3353    /// clears it after the run to finalize the persisted verdict to the
3354    /// test's FINAL (post-inversion) outcome. nextest is process-per-test
3355    /// so a run writes one sidecar; a value left from an earlier phase is
3356    /// overwritten by the current write, so the take always yields this
3357    /// run's sidecar.
3358    static LAST_SIDECAR_PATH: std::cell::RefCell<Option<PathBuf>> =
3359        const { std::cell::RefCell::new(None) };
3360}
3361
3362/// Take (read + clear) the path of the sidecar most recently written on
3363/// this thread, or `None` when no sidecar was written this run (an
3364/// early bail before any write). See [`LAST_SIDECAR_PATH`].
3365///
3366/// MUST be drained exactly once per run — `run_ktstr_test_inner` does
3367/// this after each dispatch. The thread-local persists across calls in
3368/// a process, so a caller that writes a sidecar WITHOUT a following take
3369/// would leave a stale path for the next take to consume; in practice
3370/// only `run_ktstr_test_inner` pairs a write with a take, and a stale
3371/// path points at a dropped tempdir so the finalize read fails benignly.
3372pub(crate) fn take_last_sidecar_path() -> Option<PathBuf> {
3373    LAST_SIDECAR_PATH.with(|p| p.borrow_mut().take())
3374}
3375
3376/// Overwrite a written sidecar's verdict bits with the test's FINAL
3377/// (post-inversion) `(passed, skipped, inconclusive)` outcome — see
3378/// [`crate::test_support::dispatch::Verdict::sidecar_bits`] — and set
3379/// [`SidecarResult::expected_failure`] when an actual scenario
3380/// failure/inconclusive was inverted to a pass/skip. Rewrites the file
3381/// atomically (temp + rename) under the same publication rail as the
3382/// primary write. Matching orchestrated epochs take SH; a stale token
3383/// returns without touching the path, raw no-token runs take EX, and
3384/// explicit sidecar-directory overrides remain unlocked.
3385///
3386/// A no-op when the final verdict already matches what was persisted (an
3387/// ordinary pass/fail/skip — no `expect_err`/`expect_auto_repro`
3388/// inversion). Best-effort: a read/parse/serialize/write error is
3389/// surfaced on stderr and swallowed so the raw sidecar stands (the
3390/// footer then falls back to it) rather than failing the run.
3391pub(crate) fn finalize_sidecar_verdict(
3392    path: &std::path::Path,
3393    passed: bool,
3394    skipped: bool,
3395    inconclusive: bool,
3396) {
3397    let coordinated = sidecar_dir_override().is_none();
3398    let session_token = if coordinated {
3399        run_session_token()
3400    } else {
3401        None
3402    };
3403    finalize_sidecar_verdict_inner(
3404        path,
3405        passed,
3406        skipped,
3407        inconclusive,
3408        coordinated,
3409        session_token.as_deref(),
3410        || {},
3411    );
3412}
3413
3414fn finalize_sidecar_verdict_inner<F>(
3415    path: &std::path::Path,
3416    passed: bool,
3417    skipped: bool,
3418    inconclusive: bool,
3419    coordinated: bool,
3420    session_token: Option<&str>,
3421    before_rename: F,
3422) where
3423    F: FnOnce(),
3424{
3425    // Finalization is a second publication of the same sidecar. Keep it on
3426    // the same epoch rail as the primary write: an orchestrated finalizer
3427    // takes SH and rechecks its exact token, a raw no-token finalizer takes
3428    // EX, and an explicit override remains unlocked. A token mismatch means
3429    // another invocation reset the directory after this run's primary write;
3430    // returning before even reading the old path prevents a stale finalizer
3431    // from recreating a file on the far side of that reset.
3432    let _publication_lock = if coordinated {
3433        let Some(dir) = path.parent() else {
3434            eprintln!(
3435                "ktstr: finalize_sidecar_verdict: sidecar has no parent {}",
3436                path.display()
3437            );
3438            return;
3439        };
3440        match acquire_run_dir_finalize_lock(dir, session_token) {
3441            Ok(Some(lock)) => Some(lock),
3442            Ok(None) => return,
3443            Err(error) => {
3444                eprintln!(
3445                    "ktstr: finalize_sidecar_verdict: admission for {} failed: {error:#}",
3446                    path.display()
3447                );
3448                return;
3449            }
3450        }
3451    } else {
3452        None
3453    };
3454
3455    let Ok(json) = std::fs::read_to_string(path) else {
3456        return;
3457    };
3458    let Ok(mut sc) = serde_json::from_str::<SidecarResult>(&json) else {
3459        eprintln!(
3460            "ktstr: finalize_sidecar_verdict: unparseable sidecar {}",
3461            path.display()
3462        );
3463        return;
3464    };
3465    // The run's telemetry is failure-mode-dominated when its scenario
3466    // actually failed/was-inconclusive but the final verdict is a
3467    // pass/skip (an inversion) — `perf-delta` excludes such rows.
3468    let raw_failed = sc.is_fail() || sc.is_inconclusive();
3469    let expected_failure = raw_failed && (passed || skipped);
3470    if sc.passed == passed
3471        && sc.skipped == skipped
3472        && sc.inconclusive == inconclusive
3473        && sc.expected_failure == expected_failure
3474    {
3475        return;
3476    }
3477    sc.passed = passed;
3478    sc.skipped = skipped;
3479    sc.inconclusive = inconclusive;
3480    sc.expected_failure = expected_failure;
3481    let Ok(out) = serde_json::to_string_pretty(&sc) else {
3482        return;
3483    };
3484    // Stage with a `.ktstr.json.tmp.…` suffix (append, NOT
3485    // `with_extension`, which would drop `.json`) so a hard-crash orphan
3486    // — write succeeded but rename did not — is reaped by
3487    // the epoch-reset sweep via `is_sidecar_staging_filename`, the
3488    // same way the primary write's staging file is.
3489    let pid = std::process::id();
3490    let mut staging = path.as_os_str().to_owned();
3491    staging.push(format!(".tmp.finalize.{pid}"));
3492    let staging = std::path::PathBuf::from(staging);
3493    if std::fs::write(&staging, &out).is_err() {
3494        let _ = std::fs::remove_file(&staging);
3495        return;
3496    }
3497    before_rename();
3498    if std::fs::rename(&staging, path).is_err() {
3499        let _ = std::fs::remove_file(&staging);
3500    }
3501}
3502
3503/// Remove every failure-dump artifact for this exact test variant,
3504/// including nextest retry archives.
3505///
3506/// Called when a run's FINAL outcome is a pass/skip but it wrote NO
3507/// sidecar — the run crashed before the guest produced a parseable
3508/// result (e.g. an `expect_err` test with a host-triggered BPF crash),
3509/// so [`finalize_sidecar_verdict`] had nothing to finalize. The freeze
3510/// coordinator wrote the dump unconditionally; without a sidecar to mark
3511/// the pass, the footer's dump-only trigger
3512/// ([`summarize_one_run_dir`] flags a dump with no parsed sidecar) would
3513/// surface this PASSING test as FAILED. Removing the dump keeps the
3514/// footer consistent with nextest's pass. Best-effort: a missing dump
3515/// (the normal clean-pass case) is fine. A genuine pre-sidecar failure
3516/// (final = Fail) does NOT call this, so its dump still flags.
3517pub(crate) fn suppress_failure_dumps(test_name: &str, variant_hash: u64) {
3518    let dir = sidecar_dir();
3519    let Ok(entries) = std::fs::read_dir(&dir) else {
3520        return;
3521    };
3522    for entry in entries.flatten() {
3523        let path = entry.path();
3524        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3525            continue;
3526        };
3527        let Some((candidate, hash, kind)) = classify_run_artifact(name) else {
3528            continue;
3529        };
3530        if candidate == test_name
3531            && hash == variant_hash
3532            && matches!(
3533                kind,
3534                RunArtifactKind::FailureDump | RunArtifactKind::ReproFailureDump
3535            )
3536        {
3537            let _ = std::fs::remove_file(path);
3538        }
3539    }
3540}
3541
3542/// How many periodic-snapshot files a FAILING attempt keeps per dump
3543/// base (primary and `.repro.` are bounded independently).
3544///
3545/// The freeze coordinator writes one `{base}.snapshot.periodic_NNN.json`
3546/// per fired boundary (up to [`crate::scenario::snapshot::MAX_STORED_SNAPSHOTS`] = 64),
3547/// each multi-megabyte. NOTHING on disk reads them back: `eval` and the
3548/// cast-analysis pipeline both consume the in-process
3549/// [`crate::scenario::snapshot::SnapshotBridge`] (which FIFO-caps at 64
3550/// on its own), never these files — they exist solely as CI forensic
3551/// artifacts for human inspection. The documented floor for *meaningful*
3552/// periodic coverage is `num_snapshots >= 2` (see
3553/// `crate::assert::temporal` / `crate::stats::metric_id`); keeping the
3554/// most-recent 8 leaves a 4x margin of trajectory approaching the
3555/// failure while bounding a 64-deep pile to 8 files. The most recent
3556/// samples (highest `periodic_NNN` index) are retained because they
3557/// bracket the failure; the periodic tag is zero-padded 3-digit, so
3558/// lexical index order is chronological.
3559pub(crate) const RETAINED_PERIODIC_SNAPSHOTS: usize = 8;
3560
3561/// Filename prefix shared by every on-disk snapshot for one test
3562/// variant: `{test_name}-{variant_hash:016x}`. The freeze coordinator
3563/// derives snapshot names from the failure-dump base by stripping
3564/// `.failure-dump` (see `vmm::freeze_coord::snapshot::snapshot_tagged_path`),
3565/// so the primary base yields `{prefix}.snapshot.{tag}.json` and the
3566/// repro base yields `{prefix}.repro.snapshot.{tag}.json`. The trailing
3567/// `-{hash:016x}` makes the prefix an exact variant boundary — `foo`'s
3568/// prefix cannot match `foobar`'s files, and a sibling gauntlet preset
3569/// (different hash) is untouched.
3570fn snapshot_base_prefix(test_name: &str, variant_hash: u64) -> String {
3571    format!("{test_name}-{variant_hash:016x}")
3572}
3573
3574/// `true` when `name` is any snapshot file (`.snapshot.{tag}.json`,
3575/// primary or `.repro.` base) for the variant identified by `prefix`.
3576fn is_snapshot_file_for(name: &str, prefix: &str) -> bool {
3577    let Some(rest) = name.strip_prefix(prefix) else {
3578        return false;
3579    };
3580    let rest = rest.strip_prefix(".repro").unwrap_or(rest);
3581    rest.starts_with(".snapshot.") && rest.ends_with(".json")
3582}
3583
3584/// `Some(index)` when `name` is a PERIODIC snapshot file for `prefix`,
3585/// returning the zero-padded 3-digit `periodic_NNN` index for recency
3586/// ordering. `None` for a non-matching file or a non-periodic snapshot
3587/// tag (e.g. `.snapshot.mid_run.json`, `.snapshot.early-degraded.json`),
3588/// which are single-shot and never bounded. The 3-digit / all-ASCII-digit
3589/// gate matches the coordinator's `format!("periodic_{:03}", idx)`
3590/// emission exactly, so a user-chosen `Op::CaptureSnapshot { name:
3591/// "periodic_kaslr" }` tag cannot be mistaken for a periodic sample.
3592fn periodic_snapshot_index_for(name: &str, prefix: &str) -> Option<u32> {
3593    let rest = name.strip_prefix(prefix)?;
3594    let rest = rest.strip_prefix(".repro").unwrap_or(rest);
3595    let tag = rest.strip_prefix(".snapshot.")?.strip_suffix(".json")?;
3596    let digits = tag.strip_prefix("periodic_")?;
3597    if digits.len() == 3 && digits.bytes().all(|b| b.is_ascii_digit()) {
3598        digits.parse().ok()
3599    } else {
3600        None
3601    }
3602}
3603
3604/// Remove EVERY on-disk snapshot file for this exact test variant
3605/// (primary + `.repro.` bases, periodic and one-shot tags alike).
3606///
3607/// Two callers, both best-effort (a missing dir or file is fine, and
3608/// nothing here ever fails a test):
3609/// - **Pass/skip finalize** ([`crate::test_support::eval`]): a run whose
3610///   FINAL verdict is pass/skip needs no periodic forensics, so the whole
3611///   pile is dropped before the process exits (and thus before CI uploads
3612///   artifacts). This is the dominant reclaim — most e2e tests pass, and
3613///   each was writing dozens of multi-MB snapshots that nothing consumed.
3614/// - **Attempt-prepare reap**: called before boot so a new attempt starts
3615///   from a clean snapshot slate. A prior attempt that fired MORE
3616///   boundaries — or was SIGKILL'd by the watchdog mid-run — leaves stale
3617///   higher-index snapshots the coordinator's tag-keyed overwrite would
3618///   not reclaim; this reaps them. It is the crash-safety mechanism: a
3619///   killed process cannot clean up after itself, so the NEXT attempt's
3620///   prepare does it (mirrors [`super::eval`]'s `prepare_failure_dump_path`).
3621pub(crate) fn cleanup_snapshots(test_name: &str, variant_hash: u64) {
3622    let dir = sidecar_dir();
3623    let prefix = snapshot_base_prefix(test_name, variant_hash);
3624    let Ok(entries) = std::fs::read_dir(&dir) else {
3625        return;
3626    };
3627    for entry in entries.flatten() {
3628        let path = entry.path();
3629        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3630            continue;
3631        };
3632        if is_snapshot_file_for(name, &prefix) {
3633            let _ = std::fs::remove_file(&path);
3634        }
3635    }
3636}
3637
3638/// Bound the periodic snapshots for a FAILING attempt to the most recent
3639/// `keep` per dump base, deleting the older ones. Primary and `.repro.`
3640/// bases are bounded independently (each keeps its own `keep`). One-shot
3641/// snapshot tags (`mid_run`, degraded) are never touched — they are
3642/// single files, not an unbounded periodic series. Best-effort; never
3643/// fails a test.
3644///
3645/// The retained failure DUMP is the primary evidence and is untouched
3646/// here; these periodic samples are supplementary trajectory, so the
3647/// most-recent `keep` (bracketing the failure) suffice. See
3648/// [`RETAINED_PERIODIC_SNAPSHOTS`] for the floor rationale.
3649pub(crate) fn bound_periodic_snapshots(test_name: &str, variant_hash: u64, keep: usize) {
3650    let dir = sidecar_dir();
3651    let prefix = snapshot_base_prefix(test_name, variant_hash);
3652    let Ok(entries) = std::fs::read_dir(&dir) else {
3653        return;
3654    };
3655    // Bucket by base so the repro series does not evict the primary's.
3656    let mut primary: Vec<(u32, PathBuf)> = Vec::new();
3657    let mut repro: Vec<(u32, PathBuf)> = Vec::new();
3658    for entry in entries.flatten() {
3659        let path = entry.path();
3660        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3661            continue;
3662        };
3663        let Some(idx) = periodic_snapshot_index_for(name, &prefix) else {
3664            continue;
3665        };
3666        let is_repro = name
3667            .strip_prefix(prefix.as_str())
3668            .is_some_and(|rest| rest.starts_with(".repro."));
3669        if is_repro {
3670            repro.push((idx, path));
3671        } else {
3672            primary.push((idx, path));
3673        }
3674    }
3675    for mut set in [primary, repro] {
3676        if set.len() <= keep {
3677            continue;
3678        }
3679        // Ascending by index == chronological; drop the oldest overflow,
3680        // keep the newest `keep`.
3681        set.sort_by_key(|(idx, _)| *idx);
3682        let drop_count = set.len() - keep;
3683        for (_, path) in set.into_iter().take(drop_count) {
3684            let _ = std::fs::remove_file(&path);
3685        }
3686    }
3687}
3688
3689/// `Some(path)` when `KTSTR_SIDECAR_DIR` is set non-empty,
3690/// returning the override path verbatim; `None` when the env
3691/// var is unset or empty (default-path branch). Single source
3692/// of truth for the override read so [`sidecar_dir`] and
3693/// [`serialize_and_write_sidecar`] (which gates pre-clear on
3694/// the override's presence) share one env-read site rather
3695/// than each calling `std::env::var` independently.
3696///
3697/// The `is_empty()` filter is deliberate: a defensively-cleared
3698/// `KTSTR_SIDECAR_DIR=""` must NOT be treated as an override
3699/// (joining an empty path onto the run-root would silently
3700/// alias the runs-root itself, contaminating the listing).
3701/// Empty-string aliases unset, matching the
3702/// `if let Ok(d) ... && !d.is_empty()` predicate the function
3703/// replaced.
3704///
3705/// `serialize_and_write_sidecar` interprets `Some(_)` as the
3706/// "operator chose this dir, do not pre-clear" gate — silent
3707/// data loss is unacceptable on an explicit override (the
3708/// override is for users who want exact control over where
3709/// sidecars land: test isolation, archival capture, custom CI
3710/// layouts).
3711fn sidecar_dir_override() -> Option<PathBuf> {
3712    std::env::var(crate::KTSTR_SIDECAR_DIR_ENV)
3713        .ok()
3714        .filter(|d| !d.is_empty())
3715        .map(PathBuf::from)
3716}
3717
3718/// Emit a one-shot stderr warning when [`detect_project_commit`]
3719/// resolves to `None` and the run directory therefore lands at
3720/// `{kernel}-unknown`. Operators in this state lose the
3721/// `{project_commit}` discriminator on the run-directory name —
3722/// every non-git invocation at the same kernel collides on a
3723/// single directory, with the latest run pre-clearing the
3724/// previous one's sidecars. The warning surfaces this loss-of-isolation
3725/// risk so the operator can either set `KTSTR_SIDECAR_DIR` to
3726/// disambiguate per-run, or place the project tree under git
3727/// so each run carries its own commit hash.
3728///
3729/// `OnceLock<()>` gates the warning to fire EXACTLY ONCE per
3730/// process: every gauntlet variant resolves a sidecar directory
3731/// independently (via [`sidecar_dir`] and
3732/// [`serialize_and_write_sidecar`]), so without the gate the
3733/// operator would see thousands of duplicate warnings interleaved
3734/// with test output. Called via [`resolve_default_sidecar_dir`] —
3735/// which is the shared default-path body that both [`sidecar_dir`]
3736/// and [`serialize_and_write_sidecar`] funnel through — so the
3737/// warning fires only on the default-path branch. The override
3738/// branch in either caller returns before
3739/// [`resolve_default_sidecar_dir`] is reached, so an operator who
3740/// set `KTSTR_SIDECAR_DIR` to disambiguate non-git runs does not
3741/// see a misleading "commit unknown" warning that does not apply
3742/// to their effective directory layout.
3743///
3744/// Implementation is split into a public-facing wrapper
3745/// (this function) that owns the process-global `OnceLock` and
3746/// targets stderr, and a pure inner helper
3747/// [`warn_unknown_project_commit_inner`] that takes the
3748/// `&OnceLock<()>` gate and the `&mut dyn Write` sink as
3749/// parameters. The split lets tests drive the warning logic
3750/// against a local `OnceLock` and a `Vec<u8>` sink without
3751/// fighting the process-global gate or the global stderr fd —
3752/// the wrapper's behavior is what the inner does, just with
3753/// the static gate and stderr supplied.
3754fn warn_unknown_project_commit_once() {
3755    static WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
3756    let mut sink = std::io::stderr();
3757    warn_unknown_project_commit_inner(&WARNED, &mut sink);
3758}
3759
3760/// Pure helper for [`warn_unknown_project_commit_once`]: gate the
3761/// warning on `gate` and write the warning text to `sink` exactly
3762/// once across the gate's lifetime. Both parameters are taken by
3763/// reference so call sites supply ownership semantics that match
3764/// their gating story:
3765/// - The production wrapper passes a `'static` `OnceLock<()>` so
3766///   the gate spans the whole process and a stderr handle so the
3767///   warning lands in the operator's terminal.
3768/// - Tests pass a local `OnceLock<()>` so each test gets a fresh
3769///   gate (no cross-test contamination via a process-global)
3770///   and a `Vec<u8>` sink so the test can read back the emitted
3771///   bytes and assert on the warning text.
3772///
3773/// Errors from `writeln!` are ignored via `let _ =`: a metadata
3774/// probe warning must not gate sidecar writes. This DEPARTS from
3775/// the previous `eprintln!` semantics (which panic on stderr
3776/// write failure per the std docs) — here we drop the write
3777/// error silently because a metadata probe warning must not gate
3778/// sidecar writes.
3779fn warn_unknown_project_commit_inner(
3780    gate: &std::sync::OnceLock<()>,
3781    sink: &mut dyn std::io::Write,
3782) {
3783    gate.get_or_init(|| {
3784        let _ = writeln!(
3785            sink,
3786            "ktstr: WARNING: project commit unavailable (cwd not in a git \
3787             repo, or HEAD unreadable); runs at this kernel overwrite \
3788             each other in target/ktstr/{{kernel}}-unknown/. Set \
3789             KTSTR_SIDECAR_DIR=<unique-path> per run, or run from inside a \
3790             git repo with at least one commit."
3791        );
3792    });
3793}
3794
3795/// Direct-test wrapper for the process-local pre-clear behavior.
3796///
3797/// Production orchestrated writes initialize/reset epochs through
3798/// [`acquire_run_dir_publication_lock`]. Raw no-token writes use the
3799/// token-stable inner helper below while holding EX.
3800#[cfg(test)]
3801fn pre_clear_run_dir_once(dir: &std::path::Path) {
3802    let session_token = run_session_token();
3803    pre_clear_run_dir_once_for_session(dir, session_token.as_deref());
3804}
3805
3806/// Token-stable raw/pre-clear implementation.
3807///
3808/// The serializer captures the epoch once so an unrelated environment
3809/// mutation cannot change identity between admission and sentinel publication.
3810/// Direct tests use the wrapper above, which snapshots the environment at
3811/// entry.
3812fn pre_clear_run_dir_once_for_session(dir: &std::path::Path, session_token: Option<&str>) {
3813    use std::collections::HashSet;
3814    use std::path::PathBuf;
3815    use std::sync::{Mutex, OnceLock};
3816    static PRE_CLEARED: OnceLock<Mutex<HashSet<PathBuf>>> = OnceLock::new();
3817    // Canonicalize so two spellings of the same on-disk dir share
3818    // one cache entry. Falls back to the raw path when canonicalize
3819    // fails (the directory may not exist yet on the very first
3820    // write, in which case the raw path keys the entry; subsequent
3821    // calls with the same raw path also miss canonicalize the
3822    // same way and share the entry).
3823    let cache_key = dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf());
3824    let cache = PRE_CLEARED.get_or_init(|| Mutex::new(HashSet::new()));
3825    let mut guard = cache.lock_unpoisoned();
3826    if guard.contains(&cache_key) {
3827        return;
3828    }
3829    // First time this directory has been seen — wipe sidecars while
3830    // the cache mutex is still held. Releasing the guard before the
3831    // read_dir walk would open a TOCTOU window: a sibling thread that
3832    // observes the now-cached entry would skip its own pre-clear,
3833    // proceed to write a sidecar, and the original thread's walk
3834    // (running after the drop) would then delete that sibling's
3835    // freshly-written file. The walk is one read_dir + a bounded
3836    // number of `*.ktstr.json` removals, so holding the lock across
3837    // it is brief; concurrent calls against DIFFERENT directories
3838    // serialize through this critical section but each does a small,
3839    // bounded amount of I/O, which is acceptable for a metadata
3840    // probe call pattern. The cache insert happens AFTER the wipe
3841    // completes (rather than before) so a panic mid-wipe does not
3842    // poison the cache with an entry whose wipe never actually ran.
3843    // The mutex itself enforces serialization across threads; the
3844    // entry only records "wipe completed for this dir" and must
3845    // never be observed without the wipe having succeeded. `guard`
3846    // is dropped at end-of-scope so the lock release happens after
3847    // the loop completes.
3848    if let Some(token) = session_token
3849        && session_sentinel_matches(dir, token)
3850    {
3851        // A peer test process in THIS session already cleared the dir
3852        // (the sentinel records the session token under the flock);
3853        // its and the other peers' current-session sidecars must
3854        // survive, so skip the wipe entirely. See CONCURRENT WRITERS.
3855        guard.insert(cache_key);
3856        return;
3857    }
3858    // Raw no-token pre-clear is deliberately best-effort. Token-bearing
3859    // orchestrated initialization does not use this once-gated wrapper; its
3860    // publication caller propagates reset and sentinel failures.
3861    let _ = reset_run_dir_for_session(dir, session_token);
3862    // Record the raw/helper attempt only after it returns. Its historical
3863    // best-effort contract deliberately treats filesystem errors as a
3864    // completed attempt; the orchestrated token-bearing path bypasses this
3865    // process-local cache and propagates those errors instead.
3866    guard.insert(cache_key);
3867    drop(guard);
3868}
3869
3870/// Wipe one run directory and, when present, publish its exact epoch token.
3871///
3872/// Production callers hold the run-dir exclusive flock. Unlike
3873/// the process-local pre-clear helper, this deliberately has no
3874/// process-local once gate: an observed token mismatch is authoritative even
3875/// when this process wrote to the same directory earlier.
3876fn reset_run_dir_for_session(
3877    dir: &std::path::Path,
3878    session_token: Option<&str>,
3879) -> anyhow::Result<()> {
3880    reset_run_dir_for_session_with_remover(dir, session_token, |path| std::fs::remove_file(path))
3881}
3882
3883/// Testable implementation of [`reset_run_dir_for_session`].
3884///
3885/// A token-bearing caller uses the sentinel as an authoritative statement
3886/// that the old epoch was completely removed. Every enumeration, metadata,
3887/// and unlink failure must therefore abort before sentinel publication. Raw
3888/// no-token callers preserve their historical best-effort behavior by
3889/// explicitly discarding this result in [`pre_clear_run_dir_once_for_session`].
3890fn reset_run_dir_for_session_with_remover<F>(
3891    dir: &std::path::Path,
3892    session_token: Option<&str>,
3893    mut remove_file: F,
3894) -> anyhow::Result<()>
3895where
3896    F: FnMut(&std::path::Path) -> std::io::Result<()>,
3897{
3898    let Some(token) = session_token else {
3899        // Preserve the raw `cargo nextest run` contract exactly: unreadable
3900        // directories/entries, metadata failures, and individual unlink
3901        // failures are ignored while the sweep continues wherever possible.
3902        if let Ok(entries) = std::fs::read_dir(dir) {
3903            for entry in entries.flatten() {
3904                let path = entry.path();
3905                if path.is_file()
3906                    && (is_sidecar_filename(&path) || is_sidecar_staging_filename(&path))
3907                {
3908                    let _ = remove_file(&path);
3909                }
3910            }
3911        }
3912        return Ok(());
3913    };
3914
3915    let entries = std::fs::read_dir(dir)
3916        .with_context(|| format!("enumerate run directory for epoch reset {}", dir.display()))?;
3917    for entry in entries {
3918        let entry =
3919            entry.with_context(|| format!("enumerate run directory entry {}", dir.display()))?;
3920        let path = entry.path();
3921        let metadata = std::fs::metadata(&path)
3922            .with_context(|| format!("inspect run directory entry {}", path.display()))?;
3923        if !metadata.is_file() {
3924            continue;
3925        }
3926        // Two file shapes are reaped here (current-session peers
3927        // were already spared by the sentinel skip above, so a
3928        // file reaching this point is prior-session or orphaned
3929        // residue):
3930        // - `<test>-<hash>.ktstr.json` — sidecars from a PRIOR
3931        //   session sharing this `{kernel}-{project_commit}` key.
3932        // - `<test>-<hash>.ktstr.json.tmp.<pid>.<run_id>` —
3933        //   orphaned staging from a writer that died between
3934        //   `write` and `rename` in `serialize_and_write_sidecar`
3935        //   (`is_sidecar_filename` excludes these — the extension
3936        //   is `<run_id>`, not `json` — so the staging sweep is
3937        //   what reaps them). The flock makes reaping an in-flight
3938        //   stage impossible: a live peer holds the lock we hold.
3939        if is_sidecar_filename(&path) || is_sidecar_staging_filename(&path) {
3940            remove_file(&path)
3941                .with_context(|| format!("remove prior-epoch artifact {}", path.display()))?;
3942        }
3943    }
3944    // Record this session's token so peer processes skip re-wiping.
3945    // Token-bearing orchestrated initialization treats this write as
3946    // authoritative: publishing a sidecar without its epoch would collapse
3947    // the herd back into repeated EX resets. Written AFTER the wipe so a
3948    // crash mid-wipe leaves no stale sentinel falsely claiming completion.
3949    let sentinel = dir.join(SESSION_SENTINEL);
3950    std::fs::write(&sentinel, token)
3951        .with_context(|| format!("publish run epoch sentinel {}", sentinel.display()))?;
3952    Ok(())
3953}
3954
3955/// Whether the on-disk sentinel records `token` byte-for-byte.
3956fn session_sentinel_matches(dir: &std::path::Path, token: &str) -> bool {
3957    std::fs::read_to_string(dir.join(SESSION_SENTINEL)).is_ok_and(|recorded| recorded == token)
3958}
3959
3960/// Filename of the per-run-directory session sentinel that records
3961/// the [`crate::KTSTR_RUN_EPOCH_ENV`] token of the session that last
3962/// cleared the dir. A dotfile so every sidecar reader ignores it
3963/// (`is_sidecar_filename` requires a `.json` extension and
3964/// `classify_run_artifact` matches none of its suffixes), and it
3965/// lives in the run dir itself (which the caller already
3966/// `create_dir_all`'d) rather than the `.locks/` sibling.
3967const SESSION_SENTINEL: &str = ".ktstr_run_epoch";
3968
3969/// Read the `cargo ktstr test` session token from
3970/// [`crate::KTSTR_RUN_EPOCH_ENV`] — an opaque per-invocation value
3971/// the orchestrator stamps once before nextest spawns, inherited by
3972/// every child test process.
3973///
3974/// `None` when the variable is unset or empty (raw `cargo nextest
3975/// run` — no orchestrator); the publication helper then serializes
3976/// the process-local pre-clear behavior.
3977/// `Some` lets pre-clear record/match the `.ktstr_run_epoch`
3978/// sentinel so a later peer process skips re-wiping a dir this
3979/// session already cleared, sparing the peers' sidecars.
3980fn run_session_token() -> Option<String> {
3981    std::env::var(crate::KTSTR_RUN_EPOCH_ENV)
3982        .ok()
3983        .filter(|v| !v.is_empty())
3984}
3985
3986/// Predicate: is `path` an atomic-write staging file produced by
3987/// [`serialize_and_write_sidecar`]?
3988///
3989/// True iff the filename matches the `<test>-<hash>.ktstr.json.tmp.…`
3990/// shape — `is_sidecar_filename` rejects these because the
3991/// extension is `<run_id>` rather than `json`, so a separate
3992/// predicate is needed for the epoch-reset sweep
3993/// that reaps orphaned staging files. Filename-component check
3994/// (rather than full-path string) for the same load-bearing reason
3995/// `is_sidecar_filename` uses `Path::file_name()`: a `.ktstr.json.tmp.`
3996/// substring inside an ancestor segment must not match.
3997fn is_sidecar_staging_filename(path: &std::path::Path) -> bool {
3998    path.file_name()
3999        .and_then(|n| n.to_str())
4000        .is_some_and(|n| n.contains(".ktstr.json.tmp."))
4001}
4002
4003/// Wall-clock timeout for [`acquire_run_dir_publication_lock`].
4004/// Matching writers normally take SH immediately; EX is held only
4005/// for an epoch reset and its first atomic publication. A holder
4006/// that does not release within 30 s has stalled, and surfacing that
4007/// as an actionable error beats hanging the test run indefinitely.
4008/// The timeout is asymmetric with the cache-store 300 s timeout
4009/// because this rail protects only bounded run-directory metadata I/O.
4010const RUN_DIR_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
4011
4012/// Compute the per-run-key flock sentinel path for `dir`.
4013///
4014/// Layout: `{dir.parent()}/.locks/{dir.file_name()}.lock`. When
4015/// `dir = {runs_root}/{key}` (the production default-path shape),
4016/// this resolves to `{runs_root}/.locks/{key}.lock`. Sourced from
4017/// [`crate::flock::LOCK_DIR_NAME`] so a relocation of the lock
4018/// subdirectory updates one place across both this surface and
4019/// the cache module.
4020///
4021/// Returns `None` when `dir` has no parent (root) or no
4022/// `file_name` component (current dir, root) — neither case is
4023/// reachable on the production default path
4024/// ([`runs_root`] always returns a non-root multi-component
4025/// path), but the function is total over the input domain so a
4026/// future caller passing an unusual path surfaces a clean `None`
4027/// rather than panicking on `unwrap`.
4028///
4029/// Pure function over the input path — no I/O. The caller is
4030/// responsible for materializing the parent `.locks/`
4031/// subdirectory before opening the lockfile —
4032/// [`crate::flock::acquire_flock_with_timeout`] handles that
4033/// lazily.
4034fn run_dir_lock_path(dir: &std::path::Path) -> Option<PathBuf> {
4035    let parent = dir.parent()?;
4036    let leaf = dir.file_name()?;
4037    let mut filename = std::ffi::OsString::from(leaf);
4038    filename.push(".lock");
4039    Some(parent.join(crate::flock::LOCK_DIR_NAME).join(filename))
4040}
4041
4042/// Acquire publication ownership for a verdict rewrite.
4043///
4044/// Unlike a primary write, finalization must never initialize or reset an
4045/// epoch: its sidecar belongs to the epoch that published it. A matching token
4046/// retains SH through rename; a mismatch returns `Ok(None)` without touching
4047/// the file. Raw no-token finalization retains the historical EX rail.
4048fn acquire_run_dir_finalize_lock(
4049    dir: &std::path::Path,
4050    session_token: Option<&str>,
4051) -> anyhow::Result<Option<std::os::fd::OwnedFd>> {
4052    let Some(token) = session_token else {
4053        return acquire_run_dir_flock_with_timeout(dir, RUN_DIR_LOCK_TIMEOUT).map(Some);
4054    };
4055    let shared = acquire_run_dir_flock_mode_with_timeout(
4056        dir,
4057        crate::flock::FlockMode::Shared,
4058        RUN_DIR_LOCK_TIMEOUT,
4059    )?;
4060    if session_sentinel_matches(dir, token) {
4061        Ok(Some(shared))
4062    } else {
4063        drop(shared);
4064        Ok(None)
4065    }
4066}
4067
4068/// Acquire the run-directory publication rail.
4069///
4070/// A matching orchestrated epoch takes `LOCK_SH`, validates the sentinel while
4071/// holding it, and returns the fd for the caller to retain through atomic
4072/// rename. Peer writers for that epoch therefore publish concurrently. An
4073/// epoch reset requires `LOCK_EX`, so it cannot cross the token-validation to
4074/// rename interval.
4075///
4076/// A missing or mismatched epoch makes one non-blocking `LOCK_EX` election.
4077/// Losers wait for SH so a same-epoch initializer wakes the whole herd; a
4078/// genuinely different epoch proceeds to EX, rechecks, then performs
4079/// wipe -> sentinel exactly once. Raw no-token callers retain the historical
4080/// exclusive pre-clear-once path.
4081fn acquire_run_dir_publication_lock(
4082    dir: &std::path::Path,
4083    session_token: Option<&str>,
4084) -> anyhow::Result<(std::os::fd::OwnedFd, crate::flock::FlockMode)> {
4085    acquire_run_dir_publication_lock_with_timeout(dir, session_token, RUN_DIR_LOCK_TIMEOUT)
4086}
4087
4088/// Timeout-parametrizable inner of [`acquire_run_dir_publication_lock`].
4089fn acquire_run_dir_publication_lock_with_timeout(
4090    dir: &std::path::Path,
4091    session_token: Option<&str>,
4092    timeout: std::time::Duration,
4093) -> anyhow::Result<(std::os::fd::OwnedFd, crate::flock::FlockMode)> {
4094    use crate::flock::FlockMode;
4095
4096    let Some(token) = session_token else {
4097        let fd = acquire_run_dir_flock_with_timeout(dir, timeout)?;
4098        pre_clear_run_dir_once_for_session(dir, None);
4099        return Ok((fd, FlockMode::Exclusive));
4100    };
4101
4102    loop {
4103        // Unlocked read is only a fast-path hint. The matching result is
4104        // authoritative only after the shared acquire and recheck below.
4105        if session_sentinel_matches(dir, token) {
4106            let shared = acquire_run_dir_flock_mode_with_timeout(dir, FlockMode::Shared, timeout)?;
4107            if session_sentinel_matches(dir, token) {
4108                return Ok((shared, FlockMode::Shared));
4109            }
4110            drop(shared);
4111        }
4112
4113        // On an uninitialized directory exactly one member of a same-epoch
4114        // herd wins this non-blocking EX attempt. Losers wait for SH instead
4115        // of queueing for EX: once the winner publishes the sentinel and
4116        // releases, every loser wakes as a concurrent matching writer.
4117        if let Some(exclusive) = try_run_dir_flock(dir, FlockMode::Exclusive)? {
4118            if session_sentinel_matches(dir, token) {
4119                drop(exclusive);
4120                continue;
4121            }
4122            reset_run_dir_for_session(dir, Some(token))?;
4123            return Ok((exclusive, FlockMode::Exclusive));
4124        }
4125
4126        let shared = acquire_run_dir_flock_mode_with_timeout(dir, FlockMode::Shared, timeout)?;
4127        if session_sentinel_matches(dir, token) {
4128            return Ok((shared, FlockMode::Shared));
4129        }
4130        drop(shared);
4131
4132        // The incompatible holder was a different epoch's shared writer, not
4133        // an initializer. Wait for reset ownership, then recheck because a
4134        // same-epoch peer may have initialized while we were parked.
4135        let exclusive = acquire_run_dir_flock_with_timeout(dir, timeout)?;
4136        if session_sentinel_matches(dir, token) {
4137            drop(exclusive);
4138            continue;
4139        }
4140        reset_run_dir_for_session(dir, Some(token))?;
4141        return Ok((exclusive, FlockMode::Exclusive));
4142    }
4143}
4144
4145/// Test-parametrizable exclusive wrapper over
4146/// [`acquire_run_dir_flock_mode_with_timeout`].
4147///
4148/// Resolves the per-run-key lockfile path via [`run_dir_lock_path`]
4149/// then delegates to [`crate::flock::acquire_flock_with_timeout`],
4150/// which handles parent-directory creation, the poll loop, the
4151/// `tracing::debug!` contention log, and the formatted timeout
4152/// error. The `context` argument names the run directory and the
4153/// `remediation` argument supplies the operator-facing recovery
4154/// hint about peer cargo ktstr test processes that the shared
4155/// helper appends to the timeout error.
4156///
4157/// Returns `Err` on:
4158/// - `run_dir_lock_path(dir)` returning `None` (no parent / no
4159///   file_name — production default path always satisfies both,
4160///   so this is a defensive arm),
4161/// - any error from [`crate::flock::acquire_flock_with_timeout`]
4162///   (parent directory creation failure, `try_flock` error, or
4163///   wall-clock `timeout` elapsing).
4164///
4165/// Returns `Ok(OwnedFd)` on successful acquire. Caller drops the
4166/// fd to release the kernel-side flock; the OFD-bound semantics
4167/// of `flock(2)` mean no explicit unlock call is required —
4168/// `OwnedFd::drop` runs `close(2)` which releases the lock when
4169/// no other fd refers to the same OFD (the fresh `try_flock`
4170/// open guarantees uniqueness).
4171fn acquire_run_dir_flock_with_timeout(
4172    dir: &std::path::Path,
4173    timeout: std::time::Duration,
4174) -> anyhow::Result<std::os::fd::OwnedFd> {
4175    acquire_run_dir_flock_mode_with_timeout(dir, crate::flock::FlockMode::Exclusive, timeout)
4176}
4177
4178/// Mode-parametrizable run-directory flock acquire used by publication.
4179fn acquire_run_dir_flock_mode_with_timeout(
4180    dir: &std::path::Path,
4181    mode: crate::flock::FlockMode,
4182    timeout: std::time::Duration,
4183) -> anyhow::Result<std::os::fd::OwnedFd> {
4184    let lock_path = run_dir_lock_path(dir).ok_or_else(|| {
4185        anyhow::anyhow!(
4186            "cannot derive run-dir lock path from {} (no parent or no file_name component)",
4187            dir.display(),
4188        )
4189    })?;
4190    let context = format!("run-dir {}", dir.display());
4191    crate::flock::acquire_flock_with_timeout(
4192        &lock_path,
4193        mode,
4194        timeout,
4195        &context,
4196        Some(
4197            "A peer cargo ktstr test process is writing sidecars to the \
4198             same {kernel}-{project_commit} directory; wait for it to \
4199             finish or kill it, then retry.",
4200        ),
4201    )
4202}
4203
4204/// One non-blocking acquire against the run-directory lockfile.
4205fn try_run_dir_flock(
4206    dir: &std::path::Path,
4207    mode: crate::flock::FlockMode,
4208) -> anyhow::Result<Option<std::os::fd::OwnedFd>> {
4209    let lock_path = run_dir_lock_path(dir).ok_or_else(|| {
4210        anyhow::anyhow!(
4211            "cannot derive run-dir lock path from {} (no parent or no file_name component)",
4212            dir.display(),
4213        )
4214    })?;
4215    if let Some(parent) = lock_path.parent() {
4216        std::fs::create_dir_all(parent)
4217            .with_context(|| format!("create lock subdirectory {}", parent.display()))?;
4218    }
4219    crate::flock::try_flock(&lock_path, mode)
4220}
4221
4222/// Emit a minimal sidecar for a PRE-VM-BOOT skip path.
4223///
4224/// Stats tooling enumerates sidecars to compute pass/skip/fail
4225/// rates; when a test bails before `run_ktstr_test_inner` reaches
4226/// the VM-run site that calls [`write_sidecar`], the skip is
4227/// invisible to post-run analysis — it shows up as a missing
4228/// result rather than a recorded skip.
4229///
4230/// This helper writes a sidecar flagged `skipped: true, passed: true`
4231/// with empty VM telemetry (no monitor, no stimulus events, no
4232/// verifier stats, no kvm stats, no payload metrics). Stats tooling
4233/// that subtracts skipped runs from the pass count treats the entry
4234/// correctly.
4235///
4236/// # Distinction from in-VM `AssertResult::skip` paths
4237///
4238/// There are TWO classes of skip, each with its own sidecar writer:
4239///
4240/// 1. **Pre-VM-boot skips** route through this helper
4241///    (`write_skip_sidecar`). Examples:
4242///    - `performance_mode` gated off via `KTSTR_NO_PERF_MODE`
4243///      (see `run_ktstr_test_inner`),
4244///    - `ResourceContention` at `builder.build()` or `vm.run()`
4245///      (all-slots-busy / transient host-resource contention — the
4246///      VM never booted).
4247///
4248///    These paths write a MINIMAL sidecar: empty VM telemetry,
4249///    `skipped: true`, and BOTH `payload` and `work_type` resolved
4250///    exactly as a run of this config would (the entry's declared
4251///    payload and [`crate::test_support::args::current_work_type`]) so
4252///    the skip shares the run's variant identity — a later run of the
4253///    same config overwrites this skip's sidecar instead of coexisting
4254///    with it. There is no VmResult to drain because the VM didn't boot.
4255///
4256/// 2. **In-VM `AssertResult::skip` returns** — e.g. the
4257///    empty-cpuset skip in `scenario::run_scenario`
4258///    (`AssertResult::skip("not enough CPUs/LLCs")`), or the
4259///    `need >= 4 CPUs` checks in `scenario::dynamic::*` — route
4260///    through [`write_sidecar`] at `run_ktstr_test_inner`'s end.
4261///    The guest VM fully booted, ran through scenario setup,
4262///    discovered the topology couldn't accommodate the test, and
4263///    returned early. The resulting sidecar carries REAL VM
4264///    telemetry (monitor, kvm_stats, verifier_stats) alongside
4265///    `skipped: true` — not a blind spot, just a richer record
4266///    than what this helper emits.
4267///
4268/// The asymmetry is intentional: pre-VM-boot skips have no
4269/// telemetry to record, while in-VM skips do. Stats tooling that
4270/// wants to uniformly discount skipped runs filters on
4271/// [`SidecarResult::skipped == true`] regardless of which writer
4272/// produced the entry — both set the field identically.
4273///
4274/// Returns `Err` when the sidecar directory cannot be created, the
4275/// JSON cannot be serialized, or the file write fails. Callers that
4276/// ignore the Result accept the risk of stats-tooling blind spots on
4277/// this run.
4278pub(crate) fn write_skip_sidecar(
4279    entry: &KtstrTestEntry,
4280    resolved_topology: &crate::vmm::topology::Topology,
4281) -> anyhow::Result<()> {
4282    let SchedulerFingerprint {
4283        scheduler,
4284        scheduler_commit,
4285        sysctls,
4286        kargs,
4287    } = scheduler_fingerprint(entry);
4288    let sidecar = SidecarResult {
4289        test_name: entry.name.to_string(),
4290        perf_delta_assertions: entry
4291            .perf_delta_assertions
4292            .iter()
4293            .map(|&a| a.into())
4294            .collect(),
4295        // The RESOLVED topology a run of this preset would boot
4296        // (resolve_vm_topology(entry, topo)), NOT the declared
4297        // entry.topology — for a topology gauntlet each preset boots a
4298        // distinct topology, so recording the declared value would make
4299        // every preset share one variant_hash and clobber. For a plain
4300        // test (no override) resolved == declared. The skip and the run
4301        // of one preset thus share a variant_hash (the run path records
4302        // the same resolved topology), so a flaky test that skips on one
4303        // attempt and runs on the retry writes one sidecar.
4304        topology: resolved_topology.to_string(),
4305        scheduler,
4306        scheduler_commit,
4307        // A skip resolves no scheduler binary (no run), so there is no
4308        // discovery path to record.
4309        resolve_source: None,
4310        project_commit: detect_project_commit(),
4311        // A skip never runs the payload. Still record the declared
4312        // payload name so stats tooling can attribute the skip to
4313        // the payload-gauntlet variant rather than losing the
4314        // association.
4315        payload: entry.payload.map(|p| p.name.to_string()),
4316        metrics: Vec::new(),
4317        passed: false,
4318        skipped: true,
4319        inconclusive: false,
4320        expected_failure: false,
4321        stats: Default::default(),
4322        monitor: None,
4323        // A skip never ran the VM, so no periodic captures fired.
4324        periodic_fired: 0,
4325        periodic_target: 0,
4326        // A skip never booted the VM, so it has no measured budget. 0/0
4327        // maps to None on the GauntletRow's cpu_budget dim (skips carry no
4328        // budget identity; the skipped=true flag, not a sentinel field
4329        // value, marks them).
4330        vcpus: 0,
4331        cpu_budget: 0,
4332        // A skip never booted the VM, so no vCPU schedstat was sampled.
4333        host_dilation: None,
4334        // Era marker: even a skip row carries the current denomination so a
4335        // pool never mixes eras within one marker value.
4336        throughput_denomination: ThroughputDenomination::CpuSec,
4337        stimulus_events: Vec::new(),
4338        // A skip never ran the workload, but it carries the SAME
4339        // work_type a run of this config would (current_work_type reads
4340        // the per-variant --ktstr-work-type arg, identical across nextest
4341        // retry attempts). That keeps the skip's variant_hash equal to
4342        // the run's, so a flaky test that skips on one attempt and runs
4343        // on the retry writes one sidecar (the retry overwrites the skip)
4344        // rather than two coexisting files the footer would both flag.
4345        // Skips stay identified by skipped=true, not by a work_type
4346        // sentinel (see the variant-hash + skipped-bool contract above).
4347        work_type: super::args::current_work_type(),
4348        verifier_stats: Vec::new(),
4349        kvm_stats: None,
4350        sysctls,
4351        kargs,
4352        kernel_version: detect_kernel_version(),
4353        kernel_commit: kernel_commit_for_sidecar(),
4354        timestamp: now_iso8601(),
4355        run_id: generate_run_id(),
4356        host: Some(crate::host_context::collect_host_context()),
4357        // Skip paths never reach `collect_results`, so cleanup
4358        // duration is undefined. Emit `null` per the sidecar's
4359        // symmetric serialize/deserialize contract.
4360        cleanup_duration_ms: None,
4361        run_source: detect_run_source(),
4362    };
4363    serialize_and_write_sidecar(&sidecar, "skip sidecar")
4364}
4365
4366/// Best-effort write of a small per-test run marker JSON to the current
4367/// sidecar dir as `{test}-{variant_hash:016x}.{suffix}`. Shared by the
4368/// host-skip and probe-health markers the footer scans
4369/// ([`summarize_one_run_dir`]). Not variant-clashing with the sidecar /
4370/// dump artifacts (distinct suffix). Failures log at warn and are
4371/// swallowed — a missing marker only costs the footer one advisory line,
4372/// never a run's correctness. NOT atomically staged: the body is a
4373/// single small object a partial read would fail to parse (dropped by
4374/// the reader), and the mtime gate excludes any prior-run residue.
4375fn write_run_marker(entry_name: &str, variant_hash: u64, suffix: &str, body: &serde_json::Value) {
4376    let dir = sidecar_dir();
4377    if let Err(e) = std::fs::create_dir_all(&dir) {
4378        tracing::warn!(error = %e, path = %dir.display(), "ktstr: create dir for {suffix} marker");
4379        return;
4380    }
4381    let path = dir.join(format!("{entry_name}-{variant_hash:016x}.{suffix}"));
4382    match serde_json::to_string_pretty(body) {
4383        Ok(json) => {
4384            if let Err(e) = std::fs::write(&path, json) {
4385                tracing::warn!(error = %e, path = %path.display(), "ktstr: write {suffix} marker");
4386            }
4387        }
4388        Err(e) => tracing::warn!(error = %e, "ktstr: serialize {suffix} marker for '{entry_name}'"),
4389    }
4390}
4391
4392/// Record a `.host-skip.json` marker for a test this HOST cannot run.
4393/// `class` is the host-insufficiency tag
4394/// (`topology_insufficient` / `resource_contention` /
4395/// `perf_mode_unavailable`); the footer groups by it. Keyed by the same
4396/// variant hash the skip's `.ktstr.json` sidecar uses so a gauntlet's
4397/// per-preset skips do not clobber.
4398pub(crate) fn write_host_skip_marker(
4399    entry: &KtstrTestEntry,
4400    resolved_topology: &crate::vmm::topology::Topology,
4401    class: &str,
4402) {
4403    let variant_hash =
4404        variant_hash_from_parts(entry, resolved_topology, &super::args::current_work_type());
4405    write_run_marker(
4406        entry.name,
4407        variant_hash,
4408        "host-skip.json",
4409        &serde_json::json!({ "test_name": entry.name, "class": class }),
4410    );
4411}
4412
4413/// Record a `.probe-health.json` marker naming a per-test auto-repro
4414/// probe-pipeline problem with a short `reason`, for the footer's probe
4415/// block. `variant_hash` matches the run's other per-test artifacts.
4416pub(crate) fn write_probe_health_marker(entry_name: &str, variant_hash: u64, reason: &str) {
4417    write_run_marker(
4418        entry_name,
4419        variant_hash,
4420        "probe-health.json",
4421        &serde_json::json!({ "test_name": entry_name, "reason": reason }),
4422    );
4423}
4424
4425/// Record an `.expect-err-load.json` marker for an `expect_err` test
4426/// whose inversion was satisfied by a scheduler LOAD/startup failure
4427/// (the scheduler never attached — [`crate::verifier::AttachOutcome::Died`]
4428/// / `NotAttached`) rather than the runtime error the test intends. The
4429/// footer names these so a suite that is silently green because no
4430/// scheduler could load on this kernel becomes visible. `variant_hash`
4431/// matches the run's other per-test artifacts.
4432pub(crate) fn write_expect_err_load_marker(entry_name: &str, variant_hash: u64) {
4433    write_run_marker(
4434        entry_name,
4435        variant_hash,
4436        "expect-err-load.json",
4437        &serde_json::json!({ "test_name": entry_name }),
4438    );
4439}
4440
4441/// Write a sidecar JSON file for post-run analysis.
4442///
4443/// Output goes to the current run's sidecar directory
4444/// (`KTSTR_SIDECAR_DIR` override, or
4445/// `{CARGO_TARGET_DIR or "target"}/ktstr/{kernel}-{project_commit}/`,
4446/// where `{project_commit}` is the project HEAD short hex with
4447/// `-dirty` when the worktree differs).
4448///
4449/// `payload_metrics` is the accumulated per-invocation output from
4450/// `ctx.payload(X).run()` / `.spawn().wait()` calls made in the
4451/// test body. Empty vec when the test body never called
4452/// `Ctx::payload` (scheduler-only tests, host-only probes).
4453///
4454/// Returns `Err` when the sidecar directory cannot be created, the
4455/// JSON cannot be serialized, or the file write fails. Callers that
4456/// ignore the Result accept the risk of stats-tooling blind spots on
4457/// this run.
4458pub(crate) fn write_sidecar(
4459    entry: &KtstrTestEntry,
4460    vm_result: &vmm::VmResult,
4461    stimulus_events: &[StimulusEvent],
4462    check_result: &AssertResult,
4463    work_type: &str,
4464    payload_metrics: &[PayloadMetrics],
4465    resolved_topology: &crate::vmm::topology::Topology,
4466) -> anyhow::Result<()> {
4467    let SchedulerFingerprint {
4468        scheduler,
4469        scheduler_commit,
4470        sysctls,
4471        kargs,
4472    } = scheduler_fingerprint(entry);
4473    let sidecar = SidecarResult {
4474        test_name: entry.name.to_string(),
4475        perf_delta_assertions: entry
4476            .perf_delta_assertions
4477            .iter()
4478            .map(|&a| a.into())
4479            .collect(),
4480        // The RESOLVED topology this run booted (resolve_vm_topology
4481        // result), NOT the declared entry.topology — a topology gauntlet
4482        // boots a distinct topology per preset, so the declared value
4483        // would collapse every preset to one variant_hash. resolved ==
4484        // declared for a plain test (no override).
4485        topology: resolved_topology.to_string(),
4486        scheduler,
4487        scheduler_commit,
4488        // Scheduler-resolution provenance, carried on VmResult from the
4489        // host eval path (run_ktstr_test_inner_impl resolves the binary
4490        // once and stamps the source), mirroring how vcpus / cpu_budget
4491        // ride VmResult to this stamp.
4492        resolve_source: vm_result.resolve_source.clone(),
4493        project_commit: detect_project_commit(),
4494        payload: entry.payload.map(|p| p.name.to_string()),
4495        metrics: payload_metrics.to_vec(),
4496        passed: check_result.is_pass(),
4497        skipped: check_result.is_skip(),
4498        inconclusive: check_result.is_inconclusive(),
4499        // Raw scenario verdict at write time; the dispatch-layer
4500        // finalize (finalize_sidecar_verdict) overwrites these bits with
4501        // the post-inversion outcome and sets expected_failure.
4502        expected_failure: false,
4503        stats: check_result.stats.clone(),
4504        monitor: vm_result.monitor.as_ref().map(|m| m.summary.clone()),
4505        periodic_fired: vm_result.periodic_fired,
4506        periodic_target: vm_result.periodic_target,
4507        vcpus: vm_result.vcpus,
4508        cpu_budget: vm_result.cpu_budget,
4509        // Measured host dilation (evidence for the overcommit marker),
4510        // derived from the run's raw vCPU-thread schedstat totals.
4511        host_dilation: vm_result.host_vcpu_schedstat.and_then(|s| s.dilation()),
4512        // Era marker: this build's throughput rates are CPU-second
4513        // denominated (see ThroughputDenomination).
4514        throughput_denomination: ThroughputDenomination::CpuSec,
4515        stimulus_events: stimulus_events.to_vec(),
4516        work_type: work_type.to_string(),
4517        verifier_stats: vm_result.verifier_stats.clone(),
4518        kvm_stats: vm_result.kvm_stats.clone(),
4519        sysctls,
4520        kargs,
4521        kernel_version: detect_kernel_version(),
4522        kernel_commit: kernel_commit_for_sidecar(),
4523        timestamp: now_iso8601(),
4524        run_id: generate_run_id(),
4525        host: Some(crate::host_context::collect_host_context()),
4526        cleanup_duration_ms: vm_result.cleanup_duration.map(|d| d.as_millis() as u64),
4527        run_source: detect_run_source(),
4528    };
4529    serialize_and_write_sidecar(&sidecar, "sidecar")
4530}
4531
4532#[cfg(test)]
4533mod tests;