ktstr/test_support/sidecar/mod.rs
1//! Per-run sidecar JSON — the durable record of a ktstr test outcome.
2//!
3//! Every test (pass, fail, or skip) writes a [`SidecarResult`] to a
4//! JSON file under the run's sidecar directory; downstream analysis
5//! (`cargo ktstr stats`, CI dashboards) aggregates those files to
6//! compute pass/fail rates, verifier stats, callback profiles, and
7//! KVM stats across gauntlet variants.
8//!
9//! Responsibilities owned by this module:
10//! - [`SidecarResult`]: the on-disk schema. Writer-side: every field
11//! is always emitted — `null` for `None`, `[]` for empty `Vec` —
12//! with no `skip_serializing_if` and no `serde(default)`. Reader-
13//! side: serde's native `Option<T>` deserialize tolerates absence
14//! (a missing key parses as `None`); non-`Option` fields (e.g.
15//! `test_name`, `passed`, `stats`) are hard-required and a missing
16//! key fails deserialize. The contract is intentionally asymmetric
17//! so a future producer that drops an `Option` field still parses
18//! on older readers, while the current writer guarantees full
19//! round-trip symmetry. Pre-1.0: old sidecar JSON is disposable;
20//! regenerate by re-running the test rather than relying on the
21//! reader-side tolerance for migration.
22//! - [`collect_sidecars`]: load every `*.ktstr.json` under a directory
23//! (one level of subdirectories for per-job gauntlet layouts).
24//! - [`write_sidecar`] / [`write_skip_sidecar`]: serialize one run to
25//! disk; variant-hash the discriminating fields so gauntlet variants
26//! don't clobber each other.
27//! - [`sidecar_dir`], [`runs_root`], [`newest_run_dir`]: resolve where
28//! sidecars live (env override, or
29//! `{target}/ktstr/{kernel}-{project_commit}` where
30//! `{project_commit}` is the project tree's HEAD short hex from
31//! [`detect_project_commit`], suffixed `-dirty` when the
32//! worktree differs).
33//! - [`format_run_dirname`]: render the
34//! `{kernel}-{project_commit}` leaf name from the resolved
35//! kernel + commit slots, substituting the literal `unknown`
36//! when either probe returned `None` so the dirname stays
37//! filesystem-safe (see the unknown-commit collision
38//! semantics in the runs guide).
39//! - [`is_run_directory`]: predicate consumed by run-listing
40//! walkers ([`newest_run_dir`] here, `sorted_run_entries` in
41//! `crate::stats`). Filters non-directories and dotfile
42//! subdirectories (notably the `.locks/` flock-sentinel
43//! subdirectory) so the lock infrastructure cannot pollute
44//! `cargo ktstr stats list` output or claim the "most recent
45//! run" bucket.
46//! - [`reset_run_dir_for_session`]: shallow-wipe `*.ktstr.json`
47//! files when the run epoch changes so a re-run at the same
48//! `{kernel}-{project_commit}` key produces a last-writer-wins
49//! snapshot rather than an append-only archive. Raw no-token
50//! calls retain a process-local once gate.
51//! - [`acquire_run_dir_publication_lock`]: cross-process publication
52//! rail on `{runs_root}/.locks/{key}.lock`. Writers whose run-epoch
53//! sentinel already matches take `LOCK_SH` and publish concurrently;
54//! the first writer or an epoch transition takes `LOCK_EX` for
55//! wipe -> sentinel -> first publication. Holding SH through atomic
56//! rename prevents an EX reset from crossing either the primary write
57//! or its final-verdict rewrite.
58//! The override branch (operator-chosen
59//! `KTSTR_SIDECAR_DIR`) skips the flock for the same reason
60//! it skips pre-clear: the operator owns the directory's
61//! contents.
62//! - [`warn_unknown_project_commit_once`]: one-shot stderr warning
63//! on first sidecar write when `detect_project_commit` returns
64//! `None` (test process not in a git repo) so concurrent or
65//! successive non-git runs colliding on `{kernel}-unknown`
66//! surface the disambiguation hint
67//! (`KTSTR_SIDECAR_DIR=…` or place the tree under git) at
68//! first invocation rather than as a silent collision.
69//! - [`format_verifier_stats`], [`format_callback_profile`],
70//! [`format_kvm_stats`]: human-readable summaries from a
71//! `Vec<SidecarResult>` for CLI output.
72//! - [`detect_kernel_version`]: read the kernel version from
73//! `KTSTR_KERNEL` cache metadata for sidecar-dir naming and the
74//! `kernel_version` field, with fallback to
75//! `include/config/kernel.release` in the kernel source tree
76//! when the cache metadata is absent or does not carry a
77//! version (e.g. a raw source-tree path set in `KTSTR_KERNEL`
78//! rather than a cache key).
79//! - [`detect_kernel_commit`]: read the kernel SOURCE TREE's git
80//! HEAD short hex (with `-dirty` suffix when worktree differs
81//! from the index or HEAD differs from the index) for the
82//! `kernel_commit` field. Distinct from `kernel_version`
83//! (release string from `kernel.release`) and `project_commit`
84//! (ktstr framework HEAD): this records "what kernel commit
85//! produced this run" so two runs of the same `kernel_version`
86//! but different WIP source trees compare distinctly.
87
88use std::path::PathBuf;
89
90use anyhow::Context;
91
92use crate::assert::{AssertResult, ScenarioStats};
93use crate::monitor::MonitorSummary;
94use crate::sync::MutexExt;
95use crate::test_support::PayloadMetrics;
96use crate::timeline::StimulusEvent;
97use crate::vmm;
98
99use super::entry::KtstrTestEntry;
100use super::timefmt::{generate_run_id, now_iso8601};
101
102/// Which time base denominates this sidecar's workload-THROUGHPUT rate
103/// metrics (`taobench_*_ops_per_cpu_sec` /
104/// `schbench_*loops_per_cpu_sec` / `iteration_rate`, and any future
105/// throughput rate). Some rate key names predate the denomination change, so
106/// a number from a wall-era sidecar is NOT comparable to a CPU-era sidecar —
107/// the compare
108/// pipeline keys its row pairing and its group-averaging on this marker so
109/// cross-denomination values are never silently folded or diffed (see
110/// `PairingKey::from_row` and the `denomination_mismatches` counter in
111/// `CompareReport`).
112///
113/// Event-frequency metrics (IRQ/sec, per-second RPS samples) and latency
114/// metrics are wall-based in BOTH eras — a frequency's meaning is real time —
115/// so they are outside this marker's scope.
116///
117/// `#[serde(default)]` on the carrying field + `Default = Wall`: a sidecar
118/// written before the marker existed is by definition wall-era.
119#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
120#[serde(rename_all = "snake_case")]
121pub enum ThroughputDenomination {
122 /// Throughput rates are work / wall-second (pre-marker sidecars).
123 #[default]
124 Wall,
125 /// Throughput rates are work / CPU-second received
126 /// (`CLOCK_THREAD_CPUTIME_ID`) — the current denomination.
127 CpuSec,
128}
129
130impl ThroughputDenomination {
131 /// Stable string form used in pairing keys and diagnostics.
132 pub fn as_str(self) -> &'static str {
133 match self {
134 ThroughputDenomination::Wall => "wall",
135 ThroughputDenomination::CpuSec => "cpu_sec",
136 }
137 }
138}
139
140/// Test result sidecar written to KTSTR_SIDECAR_DIR for post-run analysis.
141#[derive(Debug, serde::Serialize, serde::Deserialize)]
142pub struct SidecarResult {
143 /// Fully qualified test name (matches `KtstrTestEntry::name`,
144 /// the bare function name without the `ktstr/` nextest prefix).
145 pub test_name: String,
146 /// Rendered topology label (e.g. `1n2l4c1t`) for the variant this
147 /// sidecar describes.
148 pub topology: String,
149 /// Scheduler name (matches `Scheduler::name`); `"eevdf"` for
150 /// tests run without an scx scheduler.
151 pub scheduler: String,
152 /// Best-effort git commit of the scheduler binary used for this
153 /// run. Currently ALWAYS `None` for every `SchedulerSpec`
154 /// variant — no variant today has a reliable commit source.
155 /// The field is reserved on the schema so stats tooling can
156 /// enrich it once a reliable source exists (e.g. a
157 /// `--version` probe or ELF-note read on the resolved
158 /// scheduler binary). See
159 /// [`crate::test_support::SchedulerSpec::scheduler_commit`]
160 /// for the full per-variant rationale.
161 ///
162 /// Writer always emits (`"scheduler_commit": null` on absence).
163 /// Reader-side: serde's native `Option<T>` deserialize tolerates
164 /// absence (a missing key parses as `None`); see the module-level
165 /// doc for the full asymmetric contract that governs every
166 /// nullable on this struct.
167 pub scheduler_commit: Option<String>,
168 /// How the userspace scheduler binary was resolved for this run —
169 /// the snake_case [`crate::test_support::ResolveSource::as_str`] tag
170 /// (`"path"`, `"env_var"`, `"path_lookup"`, `"auto_built"`,
171 /// `"not_found"`). Historical sidecars may carry retired tags
172 /// (`"sibling_dir"`, `"target_debug"`, `"target_release"`) from the
173 /// pre-built-fallback cascade the resolver no longer runs; the stats
174 /// CLI reads `resolve_source` as an opaque string, so those still
175 /// filter and display. Provenance, not identity: distinct from
176 /// [`SidecarResult::scheduler_commit`] (the binary's git commit) —
177 /// this records the resolution PATH, so the stats CLI can answer "was
178 /// this run's scheduler auto-built from the workspace HEAD, or
179 /// resolved from a possibly-stale `$PATH` binary?".
180 /// `"auto_built"` is the only tag whose source commit is known to
181 /// match the workspace tree; every other tag carries the stale-binary
182 /// hazard documented on the [`crate::test_support::ResolveSource`]
183 /// variant.
184 ///
185 /// Writer always emits (`"resolve_source": null` on absence — the
186 /// skip-sidecar path resolves no binary). Reader-side: serde's native
187 /// `Option<T>` deserialize tolerates absence (a missing key parses as
188 /// `None`); see the module-level doc for the full asymmetric
189 /// contract. Excluded from `sidecar_variant_hash` for the same
190 /// cross-host grouping reason as `scheduler_commit` / `run_source`:
191 /// two runs of the same semantic variant resolved via different
192 /// discovery paths must still bucket together.
193 pub resolve_source: Option<String>,
194 /// Best-effort git HEAD of the ktstr project tree at sidecar-
195 /// write time. Captured by `detect_project_commit` via
196 /// `gix::discover` from the test process's current working
197 /// directory; walks up to find the enclosing repo and reads
198 /// HEAD short-hex, suffixing `-dirty` when index-vs-HEAD or
199 /// worktree-vs-index changes are observed (submodules ignored,
200 /// matching the [`crate::fetch::local_source`] dirty-detection
201 /// pattern). `None` when cwd is not inside any git repo, or
202 /// when the gix probe fails for any reason — this is metadata,
203 /// not a gate, so probe failure must not abort the run.
204 ///
205 /// Distinct from [`SidecarResult::scheduler_commit`]: that
206 /// field tracks the userspace scheduler binary's commit
207 /// (currently always `None` per its own doc); this field
208 /// tracks the ktstr framework / test-runner commit, so the
209 /// stats CLI can answer "which version of the harness produced
210 /// this sidecar?" without inspecting the scheduler.
211 ///
212 /// Writer always emits (`"project_commit": null` on absence).
213 /// Reader-side: serde's native `Option<T>` deserialize tolerates
214 /// absence (a missing key parses as `None`) — see the module-
215 /// level doc for the full asymmetric contract. Excluded from
216 /// `sidecar_variant_hash` for the same cross-host grouping
217 /// reason `scheduler_commit` is excluded: two runs of the same
218 /// semantic variant on different ktstr commits must still bucket
219 /// together so `perf-delta` can diff them; the commit-drift
220 /// detection inspects this field directly via `--project-commit`
221 /// / `--a-project-commit` / `--b-project-commit`.
222 pub project_commit: Option<String>,
223 /// Binary payload name (matches `Payload::name` when
224 /// `entry.payload` is set). `None` when the test declared no
225 /// binary payload. Writer always emits (`"payload": null` on
226 /// absence); reader-side, serde's native `Option<T>` deserialize
227 /// tolerates absence — see the module-level doc for the full
228 /// asymmetric contract.
229 pub payload: Option<String>,
230 /// Per-payload extracted metrics collected from `ctx.payload(X).run()`
231 /// / `.spawn().wait()` call sites during the test body.
232 ///
233 /// One [`PayloadMetrics`] per invocation, in the order the calls
234 /// ran. Empty when no payload calls were made (scheduler-only
235 /// tests, or a binary-only test where the body bailed before
236 /// running the payload). Writer always emits (`"metrics": []` in
237 /// that case); reader-side, this `Vec` field is hard-required —
238 /// non-`Option` fields fail deserialize on absence. See the
239 /// module-level doc for the full contract.
240 pub metrics: Vec<PayloadMetrics>,
241 /// True when the run is a real pass — every assertion that
242 /// ran produced a positive verdict. Mirrors
243 /// [`crate::assert::AssertResult::is_pass`]. Mutually
244 /// exclusive with [`Self::skipped`] and [`Self::inconclusive`]:
245 /// the three bits `(passed, skipped, inconclusive)` form a
246 /// strict 4-state encoding where at most one is set per
247 /// record. The fourth state — Fail — is the all-false case
248 /// (no dedicated bit; [`Self::is_fail`] derives it). A real
249 /// pass requires `!skipped && !inconclusive` AND at least one
250 /// observed assertion (the empty / all-skip case routes
251 /// through [`Self::skipped`] instead).
252 pub passed: bool,
253 /// True when the run was skipped (e.g. topology mismatch,
254 /// missing resource, in-VM `AssertResult::skip` return).
255 /// Mutually exclusive with [`Self::passed`] (Pass requires a
256 /// real assertion; an all-skip stream is Skip, not Pass) and
257 /// with [`Self::inconclusive`]. Stats tooling subtracts
258 /// `skipped` runs from "pass count" so non-executions are not
259 /// reported as passes.
260 pub skipped: bool,
261 /// True when at least one assertion was [`Outcome::Inconclusive`](crate::assert::Outcome::Inconclusive) —
262 /// the run ran but a zero-denominator ratio gate could not be
263 /// evaluated (e.g. zero iterations across all workers under a
264 /// `max_migration_ratio` check). Mutually exclusive with
265 /// [`Self::passed`] and [`Self::skipped`]; in the
266 /// `Fail > Inconclusive > Pass > Skip` lattice, Inconclusive
267 /// dominates Pass/Skip but loses to Fail, so a run with both
268 /// Inconclusive and Fail outcomes records `inconclusive = false,
269 /// passed = false` (Fail wins) — `inconclusive = true` requires
270 /// `!is_fail() && !is_pass() && !is_skip()`.
271 ///
272 /// Distinct from `passed = false` (Fail) and `skipped = true`
273 /// (precondition unmet) so CI gates and stats tooling can
274 /// triage zero-denominator runs as "workload didn't produce
275 /// the signal the assertion needed" rather than misclassifying
276 /// them as silent passes (prior to the [`Outcome::Inconclusive`](crate::assert::Outcome::Inconclusive)
277 /// variant the zero-denominator case fell out as Pass) or as
278 /// hard failures.
279 pub inconclusive: bool,
280 /// True when the persisted verdict (`passed`/`skipped`/
281 /// `inconclusive`) is the POST-inversion FINAL outcome of a run
282 /// whose underlying scenario actually failed — i.e. an
283 /// `expect_err` / `expect_auto_repro` test whose induced failure was
284 /// inverted to a pass. Set by the sidecar finalize
285 /// (`finalize_sidecar_verdict`) after dispatch resolves the
286 /// verdict; `false` for an ordinary pass/skip/fail.
287 ///
288 /// The verdict bits carry the FINAL outcome so the footer, `stats`
289 /// analysis, and `replay` match nextest's exit code. This flag
290 /// preserves the one fact that overwrite loses: that the run's
291 /// telemetry is failure-mode-dominated (a deliberately short /
292 /// stalled run). `perf-delta` ORs it into its exclusion guard so
293 /// an inverted-to-pass row is still kept OUT of the regression math
294 /// (its induced-crash telemetry is not real scheduler behavior).
295 pub expected_failure: bool,
296 /// Aggregate per-cgroup statistics merged across every worker.
297 pub stats: ScenarioStats,
298 /// Monitor summary. `None` means the monitor loop did not run
299 /// (host-only tests, early VM failure) or sample collection
300 /// produced no valid data. Writer always emits (`"monitor": null`
301 /// on absence); reader-side, serde's native `Option<T>`
302 /// deserialize tolerates absence — see the module-level doc.
303 pub monitor: Option<MonitorSummary>,
304 /// Periodic-capture coverage for this run: how many periodic snapshot
305 /// boundaries actually fired (`periodic_fired`) out of the configured
306 /// `num_snapshots` target (`periodic_target`). Carried verbatim from
307 /// [`crate::prelude::VmResult`] so cross-run tooling can read the
308 /// coverage off the persisted sidecar (previously only the in-memory
309 /// result exposed it). `0`/`0` for runs with no periodic captures
310 /// configured. Hard-required `u32` fields — old sidecars predating
311 /// them re-generate on the next run (sidecar data is disposable).
312 pub periodic_fired: u32,
313 /// See [`Self::periodic_fired`].
314 pub periodic_target: u32,
315 /// Guest vCPU count and the effective host-CPU budget the vCPU threads
316 /// ran on, carried verbatim from [`crate::prelude::VmResult`]. Drive
317 /// the `cpu-budget` comparison Dimension (cross-budget runs are not
318 /// paired — confining 32 vCPUs to 4 host CPUs measures something else)
319 /// and the overcommit marker: `cpu_budget < vcpus` means the host
320 /// time-sliced the guest's vCPUs, confounding the timing metrics
321 /// (wake-latency / off-CPU / run-delay — schedstat run_delay tracks
322 /// rq->clock, which follows the guest TSC and is not steal-adjusted,
323 /// so the off-host window inflates it for tasks waiting across it).
324 /// Hard-required `u32`
325 /// (old sidecars re-generate; sidecar data is disposable). EXCLUDED
326 /// from `sidecar_variant_hash`: a budget change is a different
327 /// measurement, separated downstream by the Dimension, not the
328 /// identity bucket.
329 pub vcpus: u32,
330 /// See [`Self::vcpus`].
331 pub cpu_budget: u32,
332 /// Host-side vCPU scheduling dilation for this run — the
333 /// `HostVcpuSchedstat::dilation` ratio `1 + Σrun_delay/Σon_cpu`
334 /// over the vCPU host threads. `> 1.0`
335 /// quantifies how much the HOST time-sliced the guest's vCPUs; the
336 /// direct EVIDENCE behind the `cpu_budget < vcpus` overcommit marker
337 /// (a run can overcommit on paper yet see D≈1.0 if the host was
338 /// otherwise idle). `None` when no vCPU ran or the host lacks
339 /// `CONFIG_SCHEDSTATS` (every schedstat line reads `0 0 0`) — kept
340 /// distinguishable from a genuine `D == 1.0`.
341 ///
342 /// `#[serde(default)]`: sidecars written before this field existed
343 /// deserialize with `None` (disposable schema; a re-run repopulates).
344 #[serde(default, skip_serializing_if = "Option::is_none")]
345 pub host_dilation: Option<f64>,
346 /// Denomination of this sidecar's workload-throughput rate metrics.
347 /// `#[serde(default)]` = [`ThroughputDenomination::Wall`] for
348 /// pre-marker sidecars; the writer stamps the current
349 /// [`ThroughputDenomination::CpuSec`]. Keys row pairing and group
350 /// averaging in the compare pipeline so cross-denomination values are
351 /// never silently compared (see the enum doc).
352 #[serde(default)]
353 pub throughput_denomination: ThroughputDenomination,
354 /// Ordered stimulus events published by the guest step executor
355 /// while the scenario ran.
356 pub stimulus_events: Vec<StimulusEvent>,
357 /// WorkSpec type label used for post-hoc filtering and A/B comparison
358 /// (distinct from the `WorkType` enum — this is the text name).
359 pub work_type: String,
360 /// Per-BPF-program verifier statistics captured from the VM's
361 /// scheduler (when one was loaded). Empty when no scheduler
362 /// programs were inspected. Writer always emits as
363 /// `"verifier_stats": []` in that case; reader-side, this `Vec`
364 /// field is hard-required (non-`Option` fields fail deserialize
365 /// on absence). See the module-level doc.
366 pub verifier_stats: Vec<crate::monitor::bpf_prog::ProgVerifierStats>,
367 /// Aggregate per-vCPU KVM stats read after VM exit. `None` when
368 /// the VM did not run (host-only tests) or KVM stats were
369 /// unavailable. Writer always emits (`"kvm_stats": null` on
370 /// absence); reader-side, serde's native `Option<T>` deserialize
371 /// tolerates absence — see the module-level doc.
372 pub kvm_stats: Option<crate::vmm::KvmStatsTotals>,
373 /// Effective sysctls active during this test run, recorded as raw
374 /// `sysctl.key=value` cmdline strings. Writer always emits as
375 /// `"sysctls": []` when none; reader-side, this `Vec` field is
376 /// hard-required (non-`Option` fields fail deserialize on
377 /// absence). See the module-level doc.
378 pub sysctls: Vec<String>,
379 /// Effective kernel command-line args active during this test run.
380 /// Writer always emits as `"kargs": []` when none; reader-side,
381 /// this `Vec` field is hard-required (non-`Option` fields fail
382 /// deserialize on absence). See the module-level doc.
383 pub kargs: Vec<String>,
384 /// Kernel version of the VM under test (from cache metadata,
385 /// e.g. `"6.14.2"`). Populated from the cache entry's
386 /// `metadata.json` version field, with fallback to the kernel
387 /// source tree's `include/config/kernel.release` when
388 /// `KTSTR_KERNEL` points at a raw source path rather than a
389 /// cache key; `None` for host-only tests or when neither
390 /// source yields a version string. The host's running kernel
391 /// release is carried separately in `host.kernel_release`.
392 /// Writer always emits (`"kernel_version": null` on absence);
393 /// reader-side, serde's native `Option<T>` deserialize tolerates
394 /// absence — see the module-level doc for the full asymmetric
395 /// contract.
396 pub kernel_version: Option<String>,
397 /// Kernel SOURCE TREE git HEAD short hex (7 chars via
398 /// `oid::to_hex_with_len(7)`), with `-dirty` suffix appended
399 /// when HEAD-vs-index or index-vs-worktree changes are
400 /// observed. Probes via `gix::open` against the kernel
401 /// directory resolved from `KTSTR_KERNEL` (not `gix::discover`
402 /// — the kernel dir is explicit, not walked-up). Captured by
403 /// `detect_kernel_commit` at sidecar-write time.
404 ///
405 /// Distinct from sibling fields:
406 /// - [`SidecarResult::kernel_version`] — release string read
407 /// from cache metadata or `include/config/kernel.release`,
408 /// e.g. `"6.14.2"`. Two runs of `6.14.2` from a clean
409 /// tree and a `-dirty` worktree at the same HEAD share
410 /// `kernel_version` but differ on `kernel_commit`.
411 /// - [`SidecarResult::project_commit`] — ktstr framework
412 /// HEAD captured from the test process's cwd. Tracks
413 /// "what version of the harness produced this sidecar?"
414 /// independently of the kernel under test.
415 /// - [`SidecarResult::scheduler_commit`] — userspace
416 /// scheduler binary's commit (currently always `None`).
417 ///
418 /// `None` when:
419 /// - `KTSTR_KERNEL` is unset or empty;
420 /// - the resolved `KernelId` is `Version` / `CacheKey` whose
421 /// underlying source is `Tarball` / `Git` (no source tree
422 /// on disk to probe);
423 /// - the resolved kernel directory is not a git repository
424 /// (`gix::open` fails);
425 /// - HEAD cannot be read (unborn HEAD on a fresh `git init`
426 /// with zero commits);
427 /// - any other gix probe failure — metadata, not a gate.
428 ///
429 /// Writer always emits (`"kernel_commit": null` on absence);
430 /// reader-side, serde's native `Option<T>` deserialize tolerates
431 /// absence — see the module-level doc for the full asymmetric
432 /// contract. Excluded from `sidecar_variant_hash` for the same
433 /// cross-host grouping reason `scheduler_commit` and
434 /// `project_commit` are excluded: two runs of the same semantic
435 /// variant on different kernel-source HEADs must still bucket
436 /// together so `perf-delta` can diff them; the commit-drift
437 /// detection inspects this field directly via the
438 /// `--kernel-commit` filter.
439 pub kernel_commit: Option<String>,
440 /// ISO 8601 timestamp of when this test run started.
441 pub timestamp: String,
442 /// Unique identifier for the test run. Composed as
443 /// `{run_id_timestamp}-{counter}` — the `YYYYMMDDTHHMMSSZ`
444 /// process-start stamp followed by a process-local monotonic
445 /// counter. Every sidecar produced in one `cargo ktstr test`
446 /// invocation shares the same timestamp prefix; the counter
447 /// distinguishes concurrent gauntlet variants within that
448 /// invocation. Distinct from the run DIRECTORY name (keyed
449 /// `{kernel}-{project_commit}`, see [`sidecar_dir`]) — the
450 /// directory groups runs by what they tested, the `run_id`
451 /// groups sidecars by which process emitted them.
452 pub run_id: String,
453 /// Host context — static-ish runtime state (CPU model,
454 /// memory size, THP policy, kernel release, host cmdline,
455 /// scheduler tunables). Populated by production sidecar
456 /// writers.
457 ///
458 /// `None` causes:
459 /// - **test-fixture path**: not the production sidecar
460 /// writer (production writers always populate `host`).
461 /// - **pre-enrichment archive**: sidecar predates the
462 /// host-context landing — re-run the test to regenerate
463 /// under the current schema (no migration shim exists
464 /// per the pre-1.0 disposable-data contract).
465 ///
466 /// Deliberately excluded from the variant hash so
467 /// gauntlet variants on different hosts collapse into the same
468 /// hash bucket.
469 ///
470 /// No serde attributes: writer always emits (`"host": null` when
471 /// `None`); reader-side, serde's native `Option<T>` deserialize
472 /// tolerates absence (a missing key parses as `None`). The
473 /// asymmetric contract is crate-wide — see the module-level doc.
474 /// Pre-1.0, sidecar data is disposable, so regenerate by
475 /// re-running the test rather than carrying a compat shim for
476 /// older JSON; the reader-side tolerance exists so an in-flight
477 /// schema rename of an `Option` field does not break parsing of
478 /// older sidecars during the same producer-version, not as a
479 /// long-term migration story.
480 pub host: Option<crate::host_context::HostContext>,
481 /// Wall-clock milliseconds spent in
482 /// `KtstrVm::collect_results` — the host-side
483 /// teardown window from BSP exit through SHM drain (mirrors
484 /// [`VmResult::cleanup_duration`](crate::vmm::VmResult::cleanup_duration);
485 /// `Duration` is converted to `u64` ms here because every other
486 /// timing field on this struct that lands in a sidecar-comparison
487 /// CLI uses integer ms or seconds, and JSON has no native
488 /// `Duration`). `None` when the run was killed by the watchdog
489 /// before `collect_results` returned, or for the `host_only` /
490 /// host-only-stub paths that never boot a VM. Writer always emits
491 /// (`"cleanup_duration_ms": null` on absence); reader-side,
492 /// serde's native `Option<T>` deserialize tolerates absence — see
493 /// the module-level doc for the full asymmetric contract.
494 pub cleanup_duration_ms: Option<u64>,
495 /// Provenance tag for this sidecar — distinguishes a developer's
496 /// local run from a CI run so cross-environment comparisons in
497 /// `perf-delta` can narrow on (or contrast across) the run
498 /// environment without inferring it from `host`.
499 ///
500 /// Recorded by `detect_run_source` at sidecar-write time:
501 /// - `Some("ci")` when `KTSTR_CI_ENV` is set non-empty (CI runner
502 /// scripts export it before invoking the test binary; local
503 /// runs never set it).
504 /// - `Some("local")` otherwise — the default for any sidecar
505 /// produced by a developer-driven invocation.
506 /// - The third documented value (`"archive"`) is NEVER written
507 /// here: a sidecar cannot know it will later be archived. The
508 /// stats CLI applies the `"archive"` tag at LOAD time when its
509 /// `--dir` flag points at a non-default pool root, overriding
510 /// whatever was on disk via `apply_archive_source_override`.
511 ///
512 /// `Option<String>` (rather than an enum) keeps the schema
513 /// extensible without a serde-version bump if a future producer
514 /// wants a new tag (e.g. `"benchmark"`); the consumer side
515 /// treats unknown values the same as known ones — they are
516 /// strings the operator can pass via `--run-source` to filter on.
517 /// Writer always emits (`"run_source": null` on absence);
518 /// reader-side, serde's native `Option<T>` deserialize tolerates
519 /// absence — see the module-level doc for the full asymmetric
520 /// contract. Excluded from `sidecar_variant_hash` for the same
521 /// cross-host grouping reason `host` is excluded — two runs of
522 /// the same semantic variant from different environments must
523 /// still bucket together so `perf-delta` can pair them; `--run-source`
524 /// is the explicit knob for source-aware narrowing.
525 ///
526 /// Field name `run_source` (renamed from `source`) disambiguates
527 /// from [`crate::cache::KernelSource`] / `KernelMetadata.source`
528 /// — those describe the kernel build's input (tarball / git /
529 /// local), this describes the run-environment provenance.
530 ///
531 /// **On-disk JSON key changed from `"source"` to `"run_source"`
532 /// in the field rename.** No `#[serde(alias = "source")]` is
533 /// in place: archived sidecars written before the rename carry
534 /// the `"source"` key, which the current schema treats as an
535 /// unknown field. Because `SidecarResult`'s derive does NOT
536 /// set `deny_unknown_fields`, the deserialize does not fail
537 /// outright — instead serde silently DROPS the stale `"source"`
538 /// payload and lands `run_source = None` (since `Option<T>`'s
539 /// "tolerate absence" rule kicks in for the missing
540 /// `"run_source"` field). The data is lost, not preserved. This
541 /// is deliberate per the project's pre-1.0 disposable-data
542 /// contract: re-running tests regenerates sidecars under the
543 /// new key rather than carrying compat shims forward. Consumers
544 /// who need the run-source classification on archived JSON
545 /// must either rename the key in-place before deserialize, or
546 /// re-run the test to regenerate the sidecar with the new
547 /// schema. Tooling that runs against the renamed schema and
548 /// observes a `None` `run_source` cannot distinguish "sidecar
549 /// pre-dates the field" from "sidecar pre-dates the rename and
550 /// lost its tag" — both lower-bound at `None` for filter
551 /// purposes.
552 pub run_source: Option<String>,
553 /// Per-test [`crate::test_support::PerfDeltaAssertion`]s declared on the
554 /// entry, serialized so `cargo ktstr perf-delta --noise-adjust`'s host-side
555 /// compare can enforce them across commits (the entry registry in the parent
556 /// process describes only HEAD's tests, not a baseline/cached sidecar's
557 /// commit, so the declaration must travel WITH the run). Empty when the test
558 /// declared none. Inert here — a normal `cargo ktstr test` writes them but
559 /// never gates on them; only the `--noise-adjust` compare consults them (the
560 /// scalar compare warns that declared gates were skipped).
561 ///
562 /// Writer always emits (`"perf_delta_assertions": []` on absence); reader-
563 /// side this `Vec` field is hard-required (non-`Option` fails deserialize on
564 /// absence) — see the module-level doc for the full contract.
565 pub perf_delta_assertions: Vec<PerfDeltaAssertionRecord>,
566}
567
568/// Owned, serialized mirror of [`crate::test_support::PerfDeltaAssertion`]. The
569/// public declaration type uses `&'static str` (so it stays const/E0493-safe on
570/// the entry) and therefore cannot `Deserialize` into an owned value; this
571/// `String`-backed record is the sidecar carrier the perf-delta compare reads.
572/// `pub` because it is a field of the `pub` [`SidecarResult`] (constructed
573/// across the workspace, including by the `cargo-ktstr` binary crate); the
574/// author-facing declaration type is [`crate::test_support::PerfDeltaAssertion`].
575#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
576pub struct PerfDeltaAssertionRecord {
577 /// Registry metric name this assertion gates (see `stats list-metrics`).
578 pub metric: String,
579 /// Pinned regression direction, or `None` to inherit the registry polarity.
580 pub direction: Option<crate::test_support::Polarity>,
581 /// Relative-regression override (percent), or `None` for `default_rel`.
582 pub max_regression_pct: Option<f64>,
583 /// Absolute-materiality override, or `None` for `default_abs`.
584 pub min_abs: Option<f64>,
585 /// Phase scope (`step_index`), or `None` to gate the aggregate value.
586 pub phase: Option<u16>,
587}
588
589impl From<&crate::test_support::PerfDeltaAssertion> for PerfDeltaAssertionRecord {
590 fn from(a: &crate::test_support::PerfDeltaAssertion) -> Self {
591 Self {
592 metric: a.metric().to_string(),
593 direction: a.direction(),
594 max_regression_pct: a.max_regression_pct(),
595 min_abs: a.min_abs(),
596 phase: a.phase(),
597 }
598 }
599}
600
601impl SidecarResult {
602 /// Convenience accessor mirroring
603 /// [`crate::assert::AssertResult::is_pass`]. SidecarResult is the
604 /// wire-format mirror of an AssertResult; this method exposes the
605 /// same is_pass / is_fail / is_skip / is_inconclusive vocabulary
606 /// so consumers can swap between the two without re-learning
607 /// field names.
608 ///
609 /// Returns true only when the run reached a real Pass — neither
610 /// skipped, inconclusive, nor failed. The triple-conjunct guard
611 /// matches AssertResult's `Fail > Inconclusive > Pass > Skip`
612 /// dominance under the strict 4-state mutex this struct encodes.
613 /// CI gates that want "ship-on-pass" semantics call this method
614 /// and only this method.
615 ///
616 /// Part of the `is_pass` / `is_fail` / `is_inconclusive` /
617 /// `is_skip` vocabulary uniform across the verdict surfaces:
618 /// [`crate::assert::AssertResult::is_pass`] / `Self::is_pass` /
619 /// [`crate::assert::Outcome::is_pass`] / `MonitorVerdict::is_pass`
620 /// (in the `monitor` module, which is `pub(crate)`) /
621 /// `Verdict::is_pass` (re-exported at [`crate::assert::Verdict`]) /
622 /// `GauntletRow::is_pass` (in the `stats` module, which is
623 /// `pub(crate)`).
624 pub fn is_pass(&self) -> bool {
625 self.passed && !self.skipped && !self.inconclusive
626 }
627 /// Convenience accessor mirroring
628 /// [`crate::assert::AssertResult::is_fail`]. The four-state
629 /// encoding uses three stored bits `(passed, skipped,
630 /// inconclusive)` in strict mutual exclusion (at most one
631 /// set); Fail is the all-false derived state, no dedicated
632 /// bit. `is_fail` reads "none of the three bits are set",
633 /// which under `Fail > Inconclusive > Pass > Skip` dominance
634 /// correctly resolves a mixed Fail+Inconclusive stream as
635 /// Fail.
636 pub fn is_fail(&self) -> bool {
637 !self.passed && !self.skipped && !self.inconclusive
638 }
639 /// Convenience accessor mirroring
640 /// [`crate::assert::AssertResult::is_skip`].
641 pub fn is_skip(&self) -> bool {
642 self.skipped
643 }
644 /// Convenience accessor mirroring
645 /// [`crate::assert::AssertResult::is_inconclusive`]. True when
646 /// the run could not be evaluated (zero-denominator ratio gate);
647 /// false on real Pass, real Fail, or Skip. CI gates that gate
648 /// on "did we get a real verdict?" should test
649 /// `r.is_pass() || r.is_fail()` and treat both `is_skip()` and
650 /// `is_inconclusive()` as "couldn't measure".
651 pub fn is_inconclusive(&self) -> bool {
652 self.inconclusive
653 }
654}
655
656#[cfg(test)]
657impl SidecarResult {
658 /// Populated [`SidecarResult`] for unit tests. Every field has a
659 /// reasonable default so call sites only spell out what they want
660 /// to vary via struct-update syntax:
661 ///
662 /// ```ignore
663 /// let sc = SidecarResult {
664 /// test_name: "my_test".to_string(),
665 /// passed: false,
666 /// ..SidecarResult::test_fixture()
667 /// };
668 /// ```
669 ///
670 /// Defaults model a passing EEVDF run on a minimal `1n1l1c1t`
671 /// topology with no payload and no VM telemetry: `test_name="t"`,
672 /// `topology="1n1l1c1t"`, `scheduler="eevdf"`, `work_type="SpinWait"`,
673 /// `passed=true`, `skipped=false`, `inconclusive=false`, every
674 /// [`Option`] `None`, every [`Vec`] empty, `stats` is
675 /// `ScenarioStats::default()`, and both `timestamp`/`run_id` are
676 /// empty strings.
677 ///
678 /// **Prefer this over local `base = || SidecarResult { ... }`
679 /// closures.** A local closure duplicates the default set and
680 /// drifts the moment [`SidecarResult`] grows a field; this fixture
681 /// is the single place those defaults live.
682 ///
683 /// **Hash-stability tests must not rely on these defaults for
684 /// hash-participating fields** (`topology`, `scheduler`, `payload`,
685 /// `work_type`, `sysctls`, `kargs`). Tests that pin
686 /// a [`sidecar_variant_hash`] output against a literal constant
687 /// must spell every hash-participating field out explicitly so a
688 /// future change to these defaults cannot silently shift the
689 /// pinned value.
690 pub(crate) fn test_fixture() -> SidecarResult {
691 SidecarResult {
692 test_name: "t".to_string(),
693 perf_delta_assertions: Vec::new(),
694 topology: "1n1l1c1t".to_string(),
695 scheduler: "eevdf".to_string(),
696 scheduler_commit: None,
697 resolve_source: None,
698 project_commit: None,
699 payload: None,
700 metrics: Vec::new(),
701 passed: true,
702 skipped: false,
703 inconclusive: false,
704 expected_failure: false,
705 stats: crate::assert::ScenarioStats::default(),
706 monitor: None,
707 periodic_fired: 0,
708 periodic_target: 0,
709 vcpus: 1,
710 cpu_budget: 1,
711 host_dilation: None,
712 throughput_denomination: ThroughputDenomination::CpuSec,
713 stimulus_events: Vec::new(),
714 work_type: "SpinWait".to_string(),
715 verifier_stats: Vec::new(),
716 kvm_stats: None,
717 sysctls: Vec::new(),
718 kargs: Vec::new(),
719 kernel_version: None,
720 kernel_commit: None,
721 timestamp: String::new(),
722 run_id: String::new(),
723 host: None,
724 cleanup_duration_ms: None,
725 run_source: None,
726 }
727 }
728}
729
730/// Predicate: is `path` a ktstr sidecar JSON filename?
731///
732/// True iff the path's extension is `json` AND the path's
733/// FILENAME COMPONENT (`Path::file_name`) contains `.ktstr.` —
734/// matching the on-disk shape produced by [`write_sidecar`]
735/// (`<test>-<variant_hash>.ktstr.json`). Both gates are required:
736/// bare `*.json` files (cargo cache, stray fixtures) and non-json
737/// files whose name happens to contain `.ktstr.` (e.g. a log)
738/// are excluded.
739///
740/// The filename-component check (rather than full-path string)
741/// is load-bearing: a parent directory like
742/// `target/foo.ktstr.bar/extra.json` would falsely match a
743/// whole-path `contains(".ktstr.")` while NOT being a sidecar.
744/// `Path::file_name()` returns only the trailing component, so
745/// `.ktstr.` in any ancestor segment cannot trigger the predicate.
746///
747/// Single source of truth for "is this file a sidecar?" — used
748/// by [`collect_sidecars_with_errors`]'s parsing walker and by
749/// the explain-sidecar file-count walker
750/// (`crate::cli::stats_cmds::explain_sidecar::count_sidecar_files`). Both
751/// walkers MUST agree on the predicate so `walked` (count) and
752/// `valid + errors` (parse outcomes) reconcile against each
753/// other; a divergence would let a file count toward `walked`
754/// without contributing to either bucket, manifesting as a
755/// silent-drop count that has no source.
756pub(crate) fn is_sidecar_filename(path: &std::path::Path) -> bool {
757 path.extension().and_then(|e| e.to_str()) == Some("json")
758 && path
759 .file_name()
760 .and_then(|n| n.to_str())
761 .is_some_and(|n| n.contains(".ktstr."))
762}
763
764/// Scan a directory for ktstr sidecar JSON files. Recurses one level
765/// into subdirectories to handle per-job gauntlet layouts.
766///
767/// Convenience wrapper over [`collect_sidecars_with_errors`] for
768/// single-directory callers that only need the parsed sidecars and not
769/// the per-file parse-failure list. Emits ONE aggregated
770/// [`warn_skipped_sidecars`] summary for that directory when stale
771/// sidecars were dropped. Multi-directory walkers must NOT use this in a
772/// loop (it would print one summary per directory) — they call
773/// [`collect_sidecars_with_errors`] per directory and aggregate the counts
774/// into a single summary (see `collect_pool`).
775pub(crate) fn collect_sidecars(dir: &std::path::Path) -> Vec<SidecarResult> {
776 let (sidecars, parse_errors, _io_errors) = collect_sidecars_with_errors(dir);
777 warn_skipped_sidecars(dir, parse_errors.len());
778 sidecars
779}
780
781/// Emit a single aggregated summary for the stale/unparseable sidecars a
782/// walk skipped, or nothing when `skipped == 0`. Sidecars written before a
783/// schema field was added fail to deserialize and are dropped (sidecar data
784/// is disposable — re-running regenerates it); this collapses what would
785/// otherwise be one `eprintln!` per file into one line. Per-file detail
786/// stays available through [`collect_sidecars_with_errors`]'s parse-error
787/// Vec, which `cargo ktstr stats explain-sidecar` renders.
788///
789/// `pub(crate)` so multi-directory walkers outside this module (e.g.
790/// `stats::analyze::sorted_run_entries`) can accumulate `parse_errors.len()`
791/// across run directories and emit ONE pool-wide summary rather than one
792/// per directory.
793pub(crate) fn warn_skipped_sidecars(dir: &std::path::Path, skipped: usize) {
794 if skipped > 0 {
795 eprintln!(
796 "ktstr_test: skipped {skipped} stale sidecar(s) under {} (older \
797 schema — re-run the affected tests to regenerate; \
798 `cargo ktstr stats explain-sidecar --run <run>` shows per-file \
799 detail)",
800 dir.display(),
801 );
802 }
803}
804
805/// Per-file parse-failure record returned by
806/// [`collect_sidecars_with_errors`] and threaded through
807/// `crate::cli::WalkStats::errors` to the renderers.
808///
809/// Named-field struct (rather than a `(PathBuf, String,
810/// Option<String>)` tuple) so call sites read fields by name —
811/// pattern-matching `for err in errors` and accessing
812/// `err.path` / `err.raw_error` / `err.enriched_message`
813/// resists the tuple-position-swap class of bug where positional
814/// fields could destructure in either order without compiler help.
815pub(crate) struct SidecarParseError {
816 /// On-disk path of the sidecar JSON that failed to parse.
817 pub path: std::path::PathBuf,
818 /// Verbatim serde-error string. Kept raw for
819 /// grep-friendly parse-error tracking and surfaced through
820 /// the JSON channel as the `error` key.
821 pub raw_error: String,
822 /// Operator-facing remediation prose computed by
823 /// [`enriched_parse_error_message`]. `Some(...)` for known
824 /// schema-drift cases (currently the `host` missing-field
825 /// pattern), `None` otherwise. Surfaced through the JSON
826 /// channel as `enriched_message`.
827 pub enriched_message: Option<String>,
828}
829
830/// Per-file IO-failure record returned by
831/// [`collect_sidecars_with_errors`] and threaded through
832/// `crate::cli::WalkStats::io_errors` to the renderers.
833///
834/// Captures files where the filename predicate matched but
835/// `std::fs::read_to_string` failed before parsing could begin —
836/// permission denied, mid-rotate truncation, broken symlink,
837/// etc. Distinct from [`SidecarParseError`] (which represents
838/// "file read OK but JSON parse failed"); separating the two
839/// lets dashboard consumers triage filesystem incidents apart
840/// from schema drift.
841///
842/// Named-field struct mirroring [`SidecarParseError`]'s shape so
843/// the renderer side can iterate by field name without tuple-
844/// position fragility. No `enriched_message` field — there is no
845/// remediation catalog for IO failures (causes vary per host:
846/// fix permissions, fix the filesystem, retry the test).
847pub(crate) struct SidecarIoError {
848 /// On-disk path the predicate matched as a sidecar candidate.
849 pub path: std::path::PathBuf,
850 /// Verbatim `std::io::Error` Display string. Surfaced through
851 /// the JSON channel as the `error` key on
852 /// `crate::cli::WalkIoError` entries and through the text
853 /// channel as the `error: ...` line under the `io errors`
854 /// trailing block.
855 pub raw_error: String,
856}
857
858/// Test-only re-export of [`enriched_parse_error_message`] so
859/// `cli::tests` can verify the enrichment-pattern logic
860/// directly against synthetic error strings. The helper itself
861/// stays private so production code routes through
862/// [`collect_sidecars_with_errors`].
863#[cfg(test)]
864pub(crate) fn enriched_parse_error_message_for_test(
865 path: &std::path::Path,
866 raw_error: &str,
867) -> Option<String> {
868 enriched_parse_error_message(path, raw_error)
869}
870
871/// Compute the operator-prose enrichment for a serde parse-error
872/// message, when one applies. Today the only enriched case is the
873/// `host` missing-field schema-drift diagnostic; the function
874/// returns `None` for any other shape so consumers can branch on
875/// "enrichment exists" without re-implementing the match.
876///
877/// Pulled out of [`collect_sidecars_with_errors`]'s parse path so the
878/// enrichment prose is computed in one place and stored in the returned
879/// [`SidecarParseError`]'s `enriched_message` field — parse failures are
880/// surfaced only through that Vec, not a separate stderr channel.
881///
882/// Matching on the Display text is deliberate: serde's typed-error
883/// surface for `missing field "X"` is not stable across
884/// serde_json versions, but the rendered message is — a
885/// forward-compat regression-resilient check costs one string
886/// search.
887fn enriched_parse_error_message(path: &std::path::Path, raw_error: &str) -> Option<String> {
888 let is_missing_host = raw_error.contains("missing field") && raw_error.contains("`host`");
889 if is_missing_host {
890 Some(format!(
891 "ktstr_test: skipping {}: {raw_error} — the `host` field \
892 was added to SidecarResult; pre-1.0 policy is \
893 disposable-sidecar: re-run the test to regenerate this \
894 file under the current schema (no migration shim exists)",
895 path.display(),
896 ))
897 } else {
898 None
899 }
900}
901
902/// Scan a directory for ktstr sidecar JSON files, returning the
903/// parsed sidecars, a [`SidecarParseError`] record (named fields
904/// `path`, `raw_error`, `enriched_message`) for every file that
905/// passed the filename predicate but failed to deserialize, and a
906/// [`SidecarIoError`] record (named fields `path`, `raw_error`)
907/// for every file that passed the predicate but whose
908/// `read_to_string` failed before parsing could begin. Recurses
909/// one level into subdirectories to handle per-job gauntlet
910/// layouts.
911///
912/// Parse failures are captured ONLY in the returned parse-errors vec —
913/// this walker no longer logs per file. Each failure is a
914/// [`SidecarParseError`] record (named fields `path`, `raw_error`,
915/// `enriched_message`) for structured callers (`explain-sidecar`'s walker
916/// output). Both raw and enriched are exposed so dashboard consumers can
917/// pick: raw for parse-error grepping, enriched for human-facing
918/// remediation prose. Callers that only need the sidecars aggregate
919/// `parse_errors.len()` and emit one [`warn_skipped_sidecars`] summary
920/// (see [`collect_sidecars`] / `collect_pool`) rather than one line per
921/// file.
922///
923/// IO failures (third return) get a single eprintln line plus a
924/// structured [`SidecarIoError`] record. Distinguished from
925/// parse failures so dashboard consumers can triage filesystem
926/// incidents (permission denied, mid-rotate truncation, broken
927/// symlink) apart from schema drift. With this third channel,
928/// every predicate-matching file lands in exactly one of the
929/// three returned vecs — the prior implicit
930/// `walked - valid - parse_errors.len()` silent-drop count is
931/// now zero by construction.
932///
933/// Callers that don't need structured errors should use
934/// [`collect_sidecars`].
935pub(crate) fn collect_sidecars_with_errors(
936 dir: &std::path::Path,
937) -> (
938 Vec<SidecarResult>,
939 Vec<SidecarParseError>,
940 Vec<SidecarIoError>,
941) {
942 let mut sidecars = Vec::new();
943 let mut parse_errors: Vec<SidecarParseError> = Vec::new();
944 let mut io_errors: Vec<SidecarIoError> = Vec::new();
945 let entries = match std::fs::read_dir(dir) {
946 Ok(e) => e,
947 Err(e) => {
948 tracing::warn!(
949 dir = %dir.display(),
950 error = %e,
951 "ktstr_test: collect_sidecars_with_errors cannot read root dir",
952 );
953 return (sidecars, parse_errors, io_errors);
954 }
955 };
956 let mut subdirs = Vec::new();
957 let try_load = |path: &std::path::Path,
958 out: &mut Vec<SidecarResult>,
959 parse_errs: &mut Vec<SidecarParseError>,
960 io_errs: &mut Vec<SidecarIoError>| {
961 if !is_sidecar_filename(path) {
962 return;
963 }
964 let data = match std::fs::read_to_string(path) {
965 Ok(d) => d,
966 Err(e) => {
967 let raw = e.to_string();
968 eprintln!("ktstr_test: cannot read {}: {raw}", path.display());
969 io_errs.push(SidecarIoError {
970 path: path.to_path_buf(),
971 raw_error: raw,
972 });
973 return;
974 }
975 };
976 match serde_json::from_str::<SidecarResult>(&data) {
977 Ok(sc) => out.push(sc),
978 Err(e) => {
979 let raw = e.to_string();
980 let enriched = enriched_parse_error_message(path, &raw);
981 // Capture (do not log) the per-file skip: callers emit one
982 // aggregated `warn_skipped_sidecars` summary so a directory
983 // of stale sidecars produces a single line, not a flood.
984 // `cargo ktstr stats explain-sidecar --run <run>` renders the
985 // per-file detail (raw + enriched remediation) from this Vec.
986 parse_errs.push(SidecarParseError {
987 path: path.to_path_buf(),
988 raw_error: raw,
989 enriched_message: enriched,
990 });
991 }
992 }
993 };
994 for entry in entries {
995 let entry = match entry {
996 Ok(e) => e,
997 Err(e) => {
998 tracing::warn!(
999 dir = %dir.display(),
1000 error = %e,
1001 "ktstr_test: skipping unreadable DirEntry while collecting sidecars",
1002 );
1003 continue;
1004 }
1005 };
1006 let path = entry.path();
1007 if path.is_dir() {
1008 subdirs.push(path);
1009 continue;
1010 }
1011 try_load(&path, &mut sidecars, &mut parse_errors, &mut io_errors);
1012 }
1013 for sub in subdirs {
1014 let sub_entries = match std::fs::read_dir(&sub) {
1015 Ok(e) => e,
1016 Err(e) => {
1017 tracing::warn!(
1018 subdir = %sub.display(),
1019 error = %e,
1020 "ktstr_test: skipping unreadable subdirectory while collecting sidecars",
1021 );
1022 continue;
1023 }
1024 };
1025 for entry in sub_entries {
1026 let entry = match entry {
1027 Ok(e) => e,
1028 Err(e) => {
1029 tracing::warn!(
1030 subdir = %sub.display(),
1031 error = %e,
1032 "ktstr_test: skipping unreadable DirEntry in sidecar subdirectory",
1033 );
1034 continue;
1035 }
1036 };
1037 try_load(
1038 &entry.path(),
1039 &mut sidecars,
1040 &mut parse_errors,
1041 &mut io_errors,
1042 );
1043 }
1044 }
1045 (sidecars, parse_errors, io_errors)
1046}
1047
1048/// Pool every sidecar JSON under every run directory at `root`.
1049///
1050/// Walks each immediate subdirectory of `root` (one per run, named
1051/// `{kernel}-{project_commit}` by [`sidecar_dir`] where
1052/// `{project_commit}` is the project tree's HEAD short hex with
1053/// `-dirty` suffix when the worktree differs from HEAD) and
1054/// concatenates the sidecars each one yields via
1055/// `collect_sidecars_with_errors` (per directory, so the per-directory
1056/// stale-sidecar skip counts aggregate into one pool-wide summary). The
1057/// result is a flat
1058/// `Vec<SidecarResult>` covering every recorded run on disk —
1059/// `cargo ktstr perf-delta`'s pool-driven sourcing reads it
1060/// once, applies the typed `--a-*` / `--b-*` filters in memory,
1061/// and partitions the survivors into A/B sides.
1062///
1063/// `root` is typically [`runs_root`]; pass an alternate path when
1064/// comparing archived sidecar trees copied off a CI host (the
1065/// `--dir` escape hatch on `perf-delta`).
1066///
1067/// Returns an empty Vec when `root` does not exist or contains no
1068/// run directories. Per-run failure (a corrupt sidecar, a partial
1069/// directory) is counted and skipped — pool-collection never aborts
1070/// on a single bad file, and emits ONE aggregated
1071/// `warn_skipped_sidecars` summary for the whole walk rather than a
1072/// per-file line.
1073///
1074/// Performance: this is a full filesystem walk over `root`. On a
1075/// host with many archived runs (dozens to hundreds), each
1076/// invocation re-reads every sidecar JSON. The cost is acceptable
1077/// for the current operator workflow (one comparison per
1078/// session) but is taskifyable if it becomes a hot path — a
1079/// directory-name fast-path could skip runs whose
1080/// `{kernel}-{project_commit}` prefix does not match the active
1081/// `--a-kernel` / `--b-kernel` filter.
1082pub fn collect_pool(root: &std::path::Path) -> Vec<SidecarResult> {
1083 let entries = match std::fs::read_dir(root) {
1084 Ok(e) => e,
1085 Err(e) => {
1086 tracing::warn!(
1087 root = %root.display(),
1088 error = %e,
1089 "ktstr_test: collect_pool cannot read root; returning empty pool",
1090 );
1091 return Vec::new();
1092 }
1093 };
1094 let mut pool = Vec::new();
1095 let mut skipped = 0usize;
1096 for entry in entries {
1097 let entry = match entry {
1098 Ok(e) => e,
1099 Err(e) => {
1100 tracing::warn!(
1101 root = %root.display(),
1102 error = %e,
1103 "ktstr_test: skipping unreadable DirEntry while collecting pool",
1104 );
1105 continue;
1106 }
1107 };
1108 let path = entry.path();
1109 if path.is_dir() {
1110 // `collect_sidecars_with_errors` already handles "one level of
1111 // subdirectories for per-job gauntlet layouts" inside each run
1112 // directory, so the two-level `{root}/{run_dir}/{job_subdir}`
1113 // shape works without a third walker level. Use the
1114 // error-returning variant (not `collect_sidecars`, which emits
1115 // its own per-directory summary) so the skip counts aggregate
1116 // into ONE pool-wide summary below.
1117 let (sidecars, parse_errors, _io_errors) = collect_sidecars_with_errors(&path);
1118 pool.extend(sidecars);
1119 skipped += parse_errors.len();
1120 }
1121 }
1122 warn_skipped_sidecars(root, skipped);
1123 pool
1124}
1125
1126/// BPF verifier complexity limit (BPF_COMPLEXITY_LIMIT_INSNS).
1127const VERIFIER_INSN_LIMIT: u32 = 1_000_000;
1128
1129/// Percentage of the verifier limit that triggers a warning.
1130const VERIFIER_WARN_PCT: f64 = 75.0;
1131
1132/// Aggregate BPF verifier stats across sidecars into a summary table.
1133///
1134/// verified_insns is deterministic for a given binary, so per-program
1135/// values are deduplicated (max across observations). Flags programs
1136/// using >=75% of the 1M verifier complexity limit.
1137pub(crate) fn format_verifier_stats(sidecars: &[SidecarResult]) -> String {
1138 use std::collections::BTreeMap;
1139
1140 let mut by_name: BTreeMap<&str, u32> = BTreeMap::new();
1141 for sc in sidecars {
1142 for info in &sc.verifier_stats {
1143 let entry = by_name.entry(&info.name).or_insert(0);
1144 *entry = (*entry).max(info.verified_insns);
1145 }
1146 }
1147
1148 if by_name.is_empty() {
1149 return String::new();
1150 }
1151
1152 let mut out = String::from("\n=== BPF VERIFIER STATS ===\n\n");
1153 out.push_str(&format!(
1154 " {:<24} {:>12} {:>8}\n",
1155 "program", "verified", "limit%"
1156 ));
1157 out.push_str(&format!(" {:-<24} {:-<12} {:-<8}\n", "", "", ""));
1158
1159 let mut warnings = Vec::new();
1160 let mut total: u64 = 0;
1161
1162 for (&name, &verified_insns) in &by_name {
1163 let pct = (verified_insns as f64 / VERIFIER_INSN_LIMIT as f64) * 100.0;
1164 let flag = if pct >= VERIFIER_WARN_PCT { " !" } else { "" };
1165 out.push_str(&format!(
1166 " {:<24} {:>12} {:>7.1}%{flag}\n",
1167 name, verified_insns, pct,
1168 ));
1169 if pct >= VERIFIER_WARN_PCT {
1170 warnings.push(format!(
1171 " {name}: {pct:.1}% of 1M limit ({verified_insns} verified insns)",
1172 ));
1173 }
1174 total += verified_insns as u64;
1175 }
1176
1177 out.push_str(&format!("\n total verified insns: {total}\n"));
1178
1179 if !warnings.is_empty() {
1180 out.push_str("\nWARNING: programs near verifier complexity limit:\n");
1181 for w in &warnings {
1182 out.push_str(w);
1183 out.push('\n');
1184 }
1185 }
1186
1187 out
1188}
1189
1190/// Per-test BPF callback profile from monitor prog_stats_deltas.
1191///
1192/// Shows per-program invocation count, total CPU time, and average
1193/// nanoseconds per call. Each test's profile is printed independently.
1194pub(crate) fn format_callback_profile(sidecars: &[SidecarResult]) -> String {
1195 let mut out = String::new();
1196
1197 for sc in sidecars {
1198 let deltas = match sc
1199 .monitor
1200 .as_ref()
1201 .and_then(|m| m.prog_stats_deltas.as_ref())
1202 {
1203 Some(d) if !d.is_empty() => d,
1204 _ => continue,
1205 };
1206
1207 if out.is_empty() {
1208 out.push_str("\n=== BPF CALLBACK PROFILE ===\n");
1209 }
1210 out.push_str(&format!("\n {} ({}):\n", sc.test_name, sc.topology));
1211 out.push_str(&format!(
1212 " {:<24} {:>12} {:>14} {:>12}\n",
1213 "program", "cnt", "total_ns", "avg_ns"
1214 ));
1215 out.push_str(&format!(
1216 " {:-<24} {:-<12} {:-<14} {:-<12}\n",
1217 "", "", "", ""
1218 ));
1219 for d in deltas {
1220 out.push_str(&format!(
1221 " {:<24} {:>12} {:>14} {:>12.0}\n",
1222 d.name, d.cnt, d.nsecs, d.nsecs_per_call,
1223 ));
1224 }
1225 }
1226
1227 out
1228}
1229
1230/// Aggregate KVM stats across sidecars into a compact summary.
1231///
1232/// Averages each stat across all tests that returned `Some(KvmStatsTotals)`.
1233/// Tests without KVM stats (non-VM tests, old kernels) are excluded
1234/// from the denominator.
1235pub(crate) fn format_kvm_stats(sidecars: &[SidecarResult]) -> String {
1236 let with_stats: Vec<&crate::vmm::KvmStatsTotals> = sidecars
1237 .iter()
1238 .filter_map(|sc| sc.kvm_stats.as_ref())
1239 .collect();
1240
1241 if with_stats.is_empty() {
1242 return String::new();
1243 }
1244
1245 let n_vms = with_stats.len();
1246
1247 // Compute cross-VM averages for each stat.
1248 let vm_avg = |name: &str| -> u64 {
1249 let sum: u64 = with_stats.iter().map(|d| d.avg(name)).sum();
1250 sum / n_vms as u64
1251 };
1252
1253 let exits = vm_avg("exits");
1254 let halt = vm_avg("halt_exits");
1255 let halt_wait_ns = vm_avg("halt_wait_ns");
1256 let preempted = vm_avg("preemption_reported");
1257 let signal = vm_avg("signal_exits");
1258 let hypercalls = vm_avg("hypercalls");
1259
1260 // Halt poll efficiency across all vCPUs and VMs.
1261 let total_poll_ok: u64 = with_stats
1262 .iter()
1263 .map(|d| d.sum("halt_successful_poll"))
1264 .sum();
1265 let total_poll_try: u64 = with_stats
1266 .iter()
1267 .map(|d| d.sum("halt_attempted_poll"))
1268 .sum();
1269
1270 if exits == 0 {
1271 return String::new();
1272 }
1273
1274 let halt_wait_ms = halt_wait_ns as f64 / 1_000_000.0;
1275 let poll_pct = if total_poll_try > 0 {
1276 (total_poll_ok as f64 / total_poll_try as f64) * 100.0
1277 } else {
1278 0.0
1279 };
1280
1281 let mut out = format!("\n=== KVM STATS (avg across {n_vms} VMs) ===\n\n");
1282 out.push_str(&format!(
1283 " exits/vcpu {:>7} halt/vcpu {:>5} halt_wait_ms {:>7.1}\n",
1284 exits, halt, halt_wait_ms,
1285 ));
1286 out.push_str(&format!(
1287 " poll_ok% {:>6.1}% preempted/vcpu {:>4} signal/vcpu {:>7}\n",
1288 poll_pct, preempted, signal,
1289 ));
1290 if hypercalls > 0 {
1291 out.push_str(&format!(" hypercalls/vcpu {:>4}\n", hypercalls));
1292 }
1293
1294 // Trust warnings.
1295 if preempted > 0 {
1296 let total: u64 = with_stats
1297 .iter()
1298 .map(|d| d.sum("preemption_reported"))
1299 .sum();
1300 out.push_str(&format!(
1301 "\n WARNING: {total} host preemptions detected \
1302 -- timing results may be unreliable\n",
1303 ));
1304 }
1305
1306 out
1307}
1308
1309/// Resolve the sidecar output directory for the current test process.
1310///
1311/// Override: `KTSTR_SIDECAR_DIR` (used as-is when non-empty). When
1312/// the override is set, `serialize_and_write_sidecar` ALSO skips
1313/// the per-directory pre-clear so any pre-existing sidecars in
1314/// the operator-chosen directory are preserved verbatim — see
1315/// `sidecar_dir_override`.
1316///
1317/// Default: `{CARGO_TARGET_DIR or "target"}/ktstr/{kernel}-{project_commit}/`,
1318/// where `{kernel}` is the version detected from `KTSTR_KERNEL`'s
1319/// metadata (or `"unknown"` when no kernel is set / detection fails)
1320/// and `{project_commit}` is the project-tree HEAD short hex from
1321/// `detect_project_commit` (with `-dirty` suffix when the worktree
1322/// differs from HEAD), or `"unknown"` when the test process is not
1323/// running inside a git repository or the probe fails. Every sidecar
1324/// written from the same `cargo ktstr test` invocation lands in the
1325/// same directory; two runs sharing the same kernel + project commit
1326/// (e.g. re-running the same suite without committing changes) reuse
1327/// the same directory, with the second run pre-clearing any
1328/// `*.ktstr.json` files left by the first via
1329/// `reset_run_dir_for_session` — the directory is a last-writer-wins
1330/// snapshot keyed on (kernel, project commit), not an append-only
1331/// archive of every invocation.
1332pub fn sidecar_dir() -> PathBuf {
1333 sidecar_dir_override().unwrap_or_else(resolve_default_sidecar_dir)
1334}
1335
1336/// Compute the default-path sidecar directory:
1337/// `{runs_root}/{kernel}-{project_commit}` where `{kernel}` and
1338/// `{project_commit}` come from [`detect_kernel_version`] and
1339/// [`detect_project_commit`] respectively, with `"unknown"`
1340/// substituted via [`format_run_dirname`] when either probe
1341/// returns `None`. Emits the one-shot
1342/// [`warn_unknown_project_commit_once`] stderr warning when the
1343/// project commit probe falls back to `"unknown"` (operators in
1344/// this state lose the per-commit run-directory discriminator).
1345///
1346/// Shared by [`sidecar_dir`] and the default-path branch of
1347/// [`serialize_and_write_sidecar`] so both call sites resolve the
1348/// same kernel/commit/warn/format chain through one place.
1349/// `serialize_and_write_sidecar` cannot call [`sidecar_dir`]
1350/// directly because it needs a single-read of
1351/// [`sidecar_dir_override`] (gated against the env-var flipping
1352/// mid-call between the dir-resolve and the pre-clear gate); the
1353/// helper supplies the default-branch body so the override read
1354/// stays at one site.
1355fn resolve_default_sidecar_dir() -> PathBuf {
1356 let kernel = detect_kernel_version();
1357 let commit = detect_project_commit();
1358 if commit.is_none() {
1359 warn_unknown_project_commit_once();
1360 }
1361 runs_root().join(format_run_dirname(kernel.as_deref(), commit.as_deref()))
1362}
1363
1364/// Build the run-directory leaf name from optional kernel and commit
1365/// components. `None` collapses to the literal `"unknown"` sentinel
1366/// in either slot, so a non-git cwd produces `"{kernel}-unknown"`
1367/// and a missing kernel produces `"unknown-{project_commit}"`. Pure
1368/// function over the two inputs — no I/O — so unit tests can pin
1369/// every shape (clean, dirty, missing-kernel, missing-commit, both
1370/// missing) without driving the [`detect_kernel_version`] /
1371/// [`detect_project_commit`] OnceLocks.
1372///
1373/// SENTINEL ASYMMETRY: the on-disk dirname uses `"unknown"` for
1374/// missing values, but the in-memory [`SidecarResult::project_commit`]
1375/// / [`SidecarResult::kernel_version`] fields stay `None` (`null`
1376/// in JSON). a `project_commit` filter for a specific commit
1377/// will NOT match a sidecar whose `project_commit` is `None` —
1378/// omit the filter to include `None`-commit rows. The asymmetry
1379/// is deliberate: the dirname needs a filesystem-safe sentinel,
1380/// while the JSON field preserves the original probe outcome for
1381/// downstream tooling that distinguishes "no probe ran" from
1382/// "probe ran but found nothing."
1383fn format_run_dirname(kernel: Option<&str>, commit: Option<&str>) -> String {
1384 let kernel = kernel.unwrap_or("unknown");
1385 let commit = commit.unwrap_or("unknown");
1386 format!("{kernel}-{commit}")
1387}
1388
1389/// Resolve the parent directory that holds all test-run subdirectories.
1390///
1391/// Resolution order:
1392/// 1. [`crate::KTSTR_RUNS_ROOT_ENV`] (absolute) — the `cargo ktstr`
1393/// orchestrator stamps this once at startup so its footer / `stats`
1394/// / `replay` reads AND the child test processes' sidecar writes
1395/// resolve the SAME directory regardless of CWD. This is the
1396/// primary path under `cargo ktstr`.
1397/// 2. `{cargo target directory}/ktstr` — the raw `cargo test` /
1398/// `cargo nextest run` fallback, with the target directory resolved
1399/// by `cargo_target_dir` (which asks cargo, so a
1400/// `.cargo/config [build] target-dir` is honored, not just
1401/// `CARGO_TARGET_DIR`). Only this unpinned path pays the
1402/// one-per-process `cargo metadata` spawn; the orchestrator pins the
1403/// absolute override above and never reaches it.
1404///
1405/// Used by `cargo ktstr stats` / `replay` and the post-run footer to
1406/// enumerate runs without reconstructing a specific run key.
1407pub fn runs_root() -> PathBuf {
1408 if let Some(root) = std::env::var_os(crate::KTSTR_RUNS_ROOT_ENV).filter(|v| !v.is_empty()) {
1409 return PathBuf::from(root);
1410 }
1411 cargo_target_dir().join("ktstr")
1412}
1413
1414/// The cargo target directory for the unpinned [`runs_root`] fallback.
1415///
1416/// Asks cargo via `cargo metadata --no-deps` and reads
1417/// `target_directory`, so the answer reflects `CARGO_TARGET_DIR`, a
1418/// `.cargo/config [build] target-dir`, AND the workspace location — a
1419/// bare `CARGO_TARGET_DIR`-or-`"target"` read honors the env var but
1420/// silently ignores a config `target-dir`, landing sidecars under a
1421/// `target/ktstr` that no other tool writes to. Memoized in a
1422/// `OnceLock`: the answer depends only on the process's CWD + config
1423/// (stable for a suite run) and the subprocess spawn is far too costly
1424/// to repeat per sidecar write. Only the raw `cargo test` / nextest
1425/// path reaches here; the `cargo ktstr` orchestrator pins
1426/// [`crate::KTSTR_RUNS_ROOT_ENV`] and short-circuits [`runs_root`]
1427/// before this runs.
1428///
1429/// On any `cargo metadata` failure (cargo absent, non-zero exit,
1430/// unparsable JSON, missing `target_directory`) it falls back to the
1431/// bare behavior: `CARGO_TARGET_DIR` when set non-empty, else the
1432/// CWD-relative `"target"`.
1433fn cargo_target_dir() -> PathBuf {
1434 static TARGET_DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
1435 if let Some(cached) = TARGET_DIR.get() {
1436 return cached.clone();
1437 }
1438 let resolved = std::env::current_dir()
1439 .ok()
1440 .and_then(|cwd| cargo_metadata_target_dir(&cwd))
1441 .unwrap_or_else(|| {
1442 std::env::var("CARGO_TARGET_DIR")
1443 .ok()
1444 .filter(|d| !d.is_empty())
1445 .map(PathBuf::from)
1446 .unwrap_or_else(|| PathBuf::from("target"))
1447 });
1448 let _ = TARGET_DIR.set(resolved.clone());
1449 resolved
1450}
1451
1452/// Run `cargo metadata --no-deps` in `dir` and return its
1453/// `target_directory`. `None` on any failure (cargo absent, non-zero
1454/// exit, unparsable JSON, missing key) so the caller can fall back.
1455/// The subprocess CWD is set on the `Command` from `dir`, never via a
1456/// process-wide `chdir`, so tests drive it against a tempdir-workspace
1457/// without racing the ambient CWD of concurrent tests.
1458fn cargo_metadata_target_dir(dir: &std::path::Path) -> Option<PathBuf> {
1459 let output = std::process::Command::new("cargo")
1460 .args(["metadata", "--format-version", "1", "--no-deps"])
1461 .current_dir(dir)
1462 .output()
1463 .ok()?;
1464 if !output.status.success() {
1465 return None;
1466 }
1467 let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?;
1468 value.get("target_directory")?.as_str().map(PathBuf::from)
1469}
1470
1471/// Predicate: is `entry` a candidate run directory under
1472/// [`runs_root`]?
1473///
1474/// True iff `entry`'s path is a directory AND its filename does
1475/// NOT begin with a `.` byte. The dotfile filter excludes the
1476/// flock sentinel subdirectory ([`crate::flock::LOCK_DIR_NAME`] =
1477/// `.locks`) plus any other operator-created or filesystem-
1478/// reserved dotfile directories from run-listing walkers
1479/// ([`newest_run_dir`] here, `sorted_run_entries` in
1480/// `crate::stats`) so the lock infrastructure does not pollute
1481/// `cargo ktstr stats list` output or claim the "most recent
1482/// run" bucket. Checking the first byte directly via
1483/// `as_encoded_bytes` is OS-string-safe (no UTF-8 round-trip)
1484/// and short-circuits cleanly on non-UTF-8 names that would
1485/// confuse a `to_str().starts_with('.')` chain.
1486///
1487/// Single source of truth for "is this a run-dir entry?" — both
1488/// run-listing call sites must pipe through this predicate so a
1489/// future relocation of `.locks/` (or any other added reserved
1490/// dotfile) updates one place.
1491pub(crate) fn is_run_directory(entry: &std::fs::DirEntry) -> bool {
1492 let path = entry.path();
1493 if !path.is_dir() {
1494 return false;
1495 }
1496 path.file_name()
1497 .and_then(|n| n.as_encoded_bytes().first().copied())
1498 .is_none_or(|b| b != b'.')
1499}
1500
1501/// Find the most recently modified run directory under [`runs_root`].
1502///
1503/// Used by `cargo ktstr stats last-run` when neither `--dir`,
1504/// `KTSTR_SIDECAR_DIR`, nor `--kernel` is set: the stats command
1505/// doesn't itself run a kernel, so it can't reconstruct the
1506/// `{kernel}-{project_commit}` key that the test process used.
1507/// Picking the newest subdirectory by mtime mirrors "show me the
1508/// report from my last test run."
1509///
1510/// Dotfile-prefixed entries (notably the flock sentinel
1511/// subdirectory `.locks/`) are excluded via `is_run_directory`
1512/// so the lock infrastructure cannot claim the "most recent
1513/// run" bucket — `.locks/`'s mtime tracks per-write flock
1514/// activity and would otherwise eclipse the actual newest run
1515/// dir on every default-path sidecar write.
1516pub fn newest_run_dir() -> Option<PathBuf> {
1517 let root = runs_root();
1518 let entries = std::fs::read_dir(&root).ok()?;
1519 entries
1520 .filter_map(|e| e.ok())
1521 .filter(is_run_directory)
1522 .max_by_key(|e| e.metadata().and_then(|m| m.modified()).ok())
1523 .map(|e| e.path())
1524}
1525
1526/// One failed test's on-disk artifacts within a single run directory,
1527/// for the `cargo ktstr test` post-run footer.
1528///
1529/// `scheduler` / `topology` come from a FAILING variant's
1530/// `.ktstr.json` sidecar and are `None` when the test failed BEFORE
1531/// writing one — e.g. a scheduler BPF-load failure that produced only
1532/// a placeholder `.failure-dump.json` via
1533/// [`crate::test_support::eval`] and never reached [`write_sidecar`].
1534/// Each `Option` path is `Some` only when that artifact exists AND
1535/// was written in the current run (the mtime gate in
1536/// [`summarize_run_artifacts`]).
1537pub(crate) struct FailedTest {
1538 /// Bare test function name (the artifact filename prefix).
1539 pub(crate) test_name: String,
1540 /// Scheduler under test, from a FAILING variant's `.ktstr.json`
1541 /// sidecar; `None` when no variant sidecar recorded a failure (a
1542 /// dump-only pre-sidecar failure).
1543 pub(crate) scheduler: Option<String>,
1544 /// Topology label, from the same failing variant as `scheduler`;
1545 /// `None` under the same condition. For a gauntlet test with
1546 /// multiple failing variants this is a representative one; the
1547 /// full per-variant set is in `stats_sidecars`.
1548 pub(crate) topology: Option<String>,
1549 /// Every primary failure dump for this test, sorted. This includes
1550 /// attempt-qualified nextest retry archives as well as the final
1551 /// attempt's canonical path.
1552 pub(crate) failure_dumps: Vec<PathBuf>,
1553 /// Every auto-repro failure dump for this test, sorted, including
1554 /// attempt-qualified nextest retry archives.
1555 pub(crate) repro_failure_dumps: Vec<PathBuf>,
1556 /// Every `{test}-{variant_hash}.ktstr.json` stats sidecar for
1557 /// this test, sorted — one per gauntlet variant (distinct
1558 /// variant hashes coexist). Empty for a dump-only failure.
1559 pub(crate) stats_sidecars: Vec<PathBuf>,
1560 /// `{test}-{variant_hash}.wprof.pb`.
1561 pub(crate) wprof: Option<PathBuf>,
1562 /// `{test}-{variant_hash}.repro.wprof.pb` (auto-repro retry).
1563 pub(crate) repro_wprof: Option<PathBuf>,
1564 /// True when ANY of this test's variant sidecars is `is_fail()`,
1565 /// so `cargo ktstr replay --filter <name>` (which selects from
1566 /// `is_fail` sidecars — see `replay.rs::select_failed_names`)
1567 /// will re-run it. False for dump-only failures (no sidecar), for
1568 /// which replay's pool selection finds nothing.
1569 pub(crate) replayable: bool,
1570}
1571
1572/// Per-run-directory artifact summary for the post-run footer.
1573pub(crate) struct RunDirSummary {
1574 /// The `{runs_root}/{kernel}-{project_commit}` run directory.
1575 pub(crate) dir: PathBuf,
1576 /// Failed tests in this dir, ordered by `test_name`.
1577 pub(crate) failed: Vec<FailedTest>,
1578 /// Count of `.ktstr.json` stats sidecars written this run
1579 /// (every executed VM test that reached [`write_sidecar`],
1580 /// pass or fail).
1581 pub(crate) stats_sidecars: usize,
1582 /// Count of `.wprof.pb` traces written this run (excludes the
1583 /// `.repro.wprof.pb` auto-repro variant).
1584 pub(crate) wprof_traces: usize,
1585 /// Tests skipped because THIS HOST cannot run them, as
1586 /// `(test_name, class)` where `class` is the host-insufficiency
1587 /// tag written by [`write_host_skip_marker`]
1588 /// (`topology_insufficient` / `resource_contention` /
1589 /// `perf_mode_unavailable`). Ordered by `test_name`. The footer
1590 /// groups these by class so an operator sees, per class, how many
1591 /// tests this host could not run and which.
1592 pub(crate) host_skips: Vec<(String, String)>,
1593 /// Tests whose auto-repro probe pipeline had a problem, as
1594 /// `(test_name, reason)` written by [`write_probe_health_marker`]
1595 /// (trigger failed to load/attach, kprobes attached 0 while
1596 /// functions were traceable, or probes attached but captured 0
1597 /// events). Ordered by `test_name`; one footer line each.
1598 pub(crate) probe_issues: Vec<(String, String)>,
1599 /// `expect_err` tests whose inversion was satisfied by a scheduler
1600 /// LOAD/startup failure (not the intended runtime error), written by
1601 /// [`write_expect_err_load_marker`]. Ordered by `test_name`. The
1602 /// footer names them on one line so a suite that is silently green
1603 /// because no scheduler could load becomes visible.
1604 pub(crate) expect_err_load: Vec<String>,
1605}
1606
1607/// The per-test artifact shapes a run directory holds.
1608enum RunArtifactKind {
1609 FailureDump,
1610 ReproFailureDump,
1611 StatsSidecar,
1612 Wprof,
1613 ReproWprof,
1614 /// `{test}-{hash}.host-skip.json` — a host-insufficiency skip
1615 /// marker ([`write_host_skip_marker`]). Its JSON body carries the
1616 /// skip `class`.
1617 HostSkip,
1618 /// `{test}-{hash}.probe-health.json` — an auto-repro probe-pipeline
1619 /// problem marker ([`write_probe_health_marker`]). Its JSON body
1620 /// carries a short `reason`.
1621 ProbeHealth,
1622 /// `{test}-{hash}.expect-err-load.json` — an `expect_err` inversion
1623 /// that was satisfied by a scheduler LOAD/startup failure rather than
1624 /// the runtime scheduler error the test intends
1625 /// ([`write_expect_err_load_marker`]). The presence of the marker is
1626 /// the signal; its JSON body carries only `test_name`. Surfaced so a
1627 /// suite that looks green on a kernel where no scheduler can load
1628 /// (every `expect_err` test's inversion satisfied by the load failure,
1629 /// not the intended stall/crash) becomes visible.
1630 ExpectErrLoad,
1631}
1632
1633/// Split a `{test}-{16-hex variant hash}` stem into `(test, hash)`.
1634///
1635/// Test function names are Rust identifiers (never contain `-`), so the
1636/// LAST `-` is the variant-hash separator. Falls back to `(stem, 0)`
1637/// when the trailing token is not a valid 16-hex hash — so a NON-variant
1638/// dump (a stale pre-variant-keying file, or a future writer that omits
1639/// the hash) still classifies by its full prefix instead of vanishing
1640/// (the "no silent drops" rule). The mtime gate already excludes stale
1641/// prior-run files; the fallback removes the silent-drop risk entirely.
1642fn split_variant_stem(stem: &str) -> (&str, u64) {
1643 if let Some((test, hash)) = stem.rsplit_once('-')
1644 && hash.len() == 16
1645 && let Ok(h) = u64::from_str_radix(hash, 16)
1646 {
1647 (test, h)
1648 } else {
1649 (stem, 0)
1650 }
1651}
1652
1653/// Parse a run-directory filename into `(test_name, variant_hash, kind)`.
1654///
1655/// Returns `None` for filenames that are not a recognized per-test
1656/// artifact — `.ktstr.json.tmp.<pid>.<run_id>` atomic-write staging
1657/// residue, stray non-ktstr files, or a `.ktstr.json` whose stem
1658/// lacks the `-{16-hex variant hash}` suffix [`write_sidecar`]
1659/// always appends.
1660///
1661/// The `variant_hash` lets the footer correlate each artifact with the
1662/// SAME-variant sidecar (a gauntlet test's per-preset dumps + sidecars
1663/// carry distinct hashes): a failure dump whose variant has no parsed
1664/// sidecar is a per-variant pre-sidecar failure even when a sibling
1665/// preset passed. failure-dump / wprof names fall back to `(stem, 0)`
1666/// when un-hashed (see [`split_variant_stem`]); a `.ktstr.json` sidecar
1667/// is ALWAYS variant-keyed by [`write_sidecar`], so a non-hashed one is
1668/// malformed and is dropped (`None`).
1669///
1670/// Failure dumps may include an attempt archive suffix:
1671/// `{test}-{hash}.attempt-N.failure-dump.json` or
1672/// `{test}-{hash}.repro.attempt-N.failure-dump.json`. The attempt is
1673/// intentionally not part of variant identity: every retry is the
1674/// same nextest test variant and is correlated with the same stats
1675/// sidecar hash.
1676fn classify_run_artifact(name: &str) -> Option<(&str, u64, RunArtifactKind)> {
1677 if let Some(stem) = name.strip_suffix(".host-skip.json") {
1678 let (test, hash) = split_variant_stem(stem);
1679 return Some((test, hash, RunArtifactKind::HostSkip));
1680 }
1681 if let Some(stem) = name.strip_suffix(".probe-health.json") {
1682 let (test, hash) = split_variant_stem(stem);
1683 return Some((test, hash, RunArtifactKind::ProbeHealth));
1684 }
1685 if let Some(stem) = name.strip_suffix(".expect-err-load.json") {
1686 let (test, hash) = split_variant_stem(stem);
1687 return Some((test, hash, RunArtifactKind::ExpectErrLoad));
1688 }
1689 if let Some(stem) = name.strip_suffix(".failure-dump.json") {
1690 let stem = match stem.rsplit_once(".attempt-") {
1691 Some((base, attempt))
1692 if attempt
1693 .parse::<u32>()
1694 .ok()
1695 .is_some_and(|attempt| attempt > 0) =>
1696 {
1697 base
1698 }
1699 _ => stem,
1700 };
1701 let (stem, kind) = match stem.strip_suffix(".repro") {
1702 Some(base) => (base, RunArtifactKind::ReproFailureDump),
1703 None => (stem, RunArtifactKind::FailureDump),
1704 };
1705 let (test, hash) = split_variant_stem(stem);
1706 return Some((test, hash, kind));
1707 }
1708 if let Some(stem) = name.strip_suffix(".repro.wprof.pb") {
1709 let (test, hash) = split_variant_stem(stem);
1710 return Some((test, hash, RunArtifactKind::ReproWprof));
1711 }
1712 if let Some(stem) = name.strip_suffix(".wprof.pb") {
1713 let (test, hash) = split_variant_stem(stem);
1714 return Some((test, hash, RunArtifactKind::Wprof));
1715 }
1716 if let Some(stem) = name.strip_suffix(".ktstr.json") {
1717 // A sidecar is ALWAYS `{test}-{16-hex}` ({:016x} in
1718 // serialize_and_write_sidecar). A stem without a valid hash
1719 // suffix is a hand-named / malformed file — drop it (unlike the
1720 // dump arms, there's no un-hashed-sidecar writer to be lenient
1721 // for).
1722 let (test, hash) = stem.rsplit_once('-')?;
1723 if hash.len() == 16
1724 && let Ok(h) = u64::from_str_radix(hash, 16)
1725 {
1726 return Some((test, h, RunArtifactKind::StatsSidecar));
1727 }
1728 }
1729 None
1730}
1731
1732/// Summarize the per-test artifacts a single run directory holds,
1733/// counting only files written at or after `since`.
1734///
1735/// The mtime gate is the freshness boundary: a run directory is
1736/// keyed `{kernel}-{project_commit}` (see [`sidecar_dir`]), so
1737/// re-running the same suite reuses the directory, and
1738/// [`reset_run_dir_for_session`] wipes only `*.ktstr.json` — stale
1739/// `*.failure-dump.json` / `*.wprof.pb` from an earlier run linger.
1740/// Filtering on `mtime >= since` (where `since` is captured before
1741/// the nextest build+run begins, so genuine artifacts — written
1742/// after the build — sort comfortably after it) keeps a stale dump
1743/// from a prior run from surfacing as a current failure.
1744///
1745/// Returns `None` when the directory holds no fresh artifacts (it
1746/// belongs to an earlier run, or cannot be read).
1747fn summarize_one_run_dir(
1748 dir: &std::path::Path,
1749 since: std::time::SystemTime,
1750) -> Option<RunDirSummary> {
1751 use std::collections::{BTreeMap, BTreeSet};
1752 #[derive(Default)]
1753 struct Acc {
1754 // Every per-variant dump, including nextest attempt archives.
1755 // The fail signal below keys off `dump_hashes`; these vectors
1756 // retain all concrete evidence paths for the footer.
1757 failure_dumps: Vec<PathBuf>,
1758 repro_failure_dumps: Vec<PathBuf>,
1759 wprof: Option<PathBuf>,
1760 repro_wprof: Option<PathBuf>,
1761 // EVERY variant's stats sidecar (distinct variant-hash
1762 // filenames coexist), so a passing variant cannot mask a
1763 // failing sibling.
1764 stats_sidecars: Vec<PathBuf>,
1765 // OR of `is_fail` across all of this name's variant sidecars.
1766 // Post-finalize the sidecar carries the FINAL (post-inversion)
1767 // verdict, so a passing expect_err / expect_auto_repro test
1768 // reads `false` here even though its scenario failed.
1769 any_fail: bool,
1770 // Variant hashes whose stats sidecar PARSED, and variant hashes
1771 // that left a failure dump. The gate is PER-VARIANT: a dump whose
1772 // variant has no parsed sidecar is a pre-sidecar failure
1773 // (scheduler load / VM boot crash) for THAT preset and flags
1774 // FAILED even when a sibling preset's sidecar parsed; a dump whose
1775 // variant DID parse a (final, non-failing) sidecar is an
1776 // expected-failure run whose dump must NOT flag — the sidecar's
1777 // finalized verdict already classified it. (A gauntlet test's
1778 // per-preset dumps + sidecars carry distinct variant hashes.)
1779 parsed_sidecar_hashes: BTreeSet<u64>,
1780 dump_hashes: BTreeSet<u64>,
1781 // (scheduler, topology) of the FIRST failing variant seen,
1782 // for the FAILED block header; `None` when no variant sidecar
1783 // parsed as a failure (a dump-only pre-sidecar failure).
1784 fail_variant: Option<(String, String)>,
1785 // Host-insufficiency skip class from a `.host-skip.json` marker
1786 // (whichever variant scanned last — a plain skip has one).
1787 host_skip_class: Option<String>,
1788 // Short auto-repro probe-pipeline problem reason from a
1789 // `.probe-health.json` marker (last variant scanned wins).
1790 probe_health_reason: Option<String>,
1791 // Presence of an `.expect-err-load.json` marker: this test's
1792 // `expect_err` inversion was satisfied by a scheduler load
1793 // failure (any variant sets it — a gauntlet test that load-failed
1794 // on one preset still surfaces).
1795 expect_err_load: bool,
1796 }
1797 let entries = std::fs::read_dir(dir).ok()?;
1798 let mut by_test: BTreeMap<String, Acc> = BTreeMap::new();
1799 let mut stats_sidecars = 0usize;
1800 let mut wprof_traces = 0usize;
1801 for entry in entries.flatten() {
1802 let Ok(meta) = entry.metadata() else {
1803 continue;
1804 };
1805 if !meta.is_file() {
1806 continue;
1807 }
1808 match meta.modified() {
1809 Ok(m) if m >= since => {}
1810 _ => continue,
1811 }
1812 let path = entry.path();
1813 let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
1814 continue;
1815 };
1816 let Some((test, variant_hash, kind)) = classify_run_artifact(name) else {
1817 continue;
1818 };
1819 let acc = by_test.entry(test.to_string()).or_default();
1820 match kind {
1821 RunArtifactKind::FailureDump => {
1822 acc.dump_hashes.insert(variant_hash);
1823 acc.failure_dumps.push(path);
1824 }
1825 RunArtifactKind::ReproFailureDump => {
1826 acc.dump_hashes.insert(variant_hash);
1827 acc.repro_failure_dumps.push(path);
1828 }
1829 RunArtifactKind::Wprof => {
1830 wprof_traces += 1;
1831 acc.wprof = Some(path);
1832 }
1833 RunArtifactKind::ReproWprof => acc.repro_wprof = Some(path),
1834 RunArtifactKind::StatsSidecar => {
1835 stats_sidecars += 1;
1836 // Accumulate EVERY variant (never overwrite by bare
1837 // name) and OR the fail signal, so one gauntlet
1838 // variant's pass cannot mask a failing sibling.
1839 match std::fs::read_to_string(&path)
1840 .ok()
1841 .and_then(|s| serde_json::from_str::<SidecarResult>(&s).ok())
1842 {
1843 Some(sc) => {
1844 acc.parsed_sidecar_hashes.insert(variant_hash);
1845 if sc.is_fail() {
1846 acc.any_fail = true;
1847 if acc.fail_variant.is_none() {
1848 acc.fail_variant = Some((sc.scheduler, sc.topology));
1849 }
1850 }
1851 }
1852 None => {
1853 // Counted in `stats_sidecars` but
1854 // unclassifiable. Warn so the count and the
1855 // failed list cannot silently disagree (a
1856 // corrupt `is_fail` sidecar would otherwise be
1857 // swallowed).
1858 tracing::warn!(
1859 path = %path.display(),
1860 "ktstr footer: unreadable/unparseable stats sidecar — \
1861 counted but not classified",
1862 );
1863 }
1864 }
1865 acc.stats_sidecars.push(path);
1866 }
1867 RunArtifactKind::HostSkip => {
1868 // Read the marker's `class` field. A malformed / absent
1869 // body leaves the class unset — the marker's presence
1870 // alone is not enough to name a class, so it is dropped
1871 // rather than rendered as an unlabelled skip.
1872 if let Some(class) = std::fs::read_to_string(&path)
1873 .ok()
1874 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
1875 .and_then(|v| v.get("class").and_then(|c| c.as_str()).map(String::from))
1876 {
1877 acc.host_skip_class = Some(class);
1878 }
1879 }
1880 RunArtifactKind::ProbeHealth => {
1881 if let Some(reason) = std::fs::read_to_string(&path)
1882 .ok()
1883 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
1884 .and_then(|v| v.get("reason").and_then(|r| r.as_str()).map(String::from))
1885 {
1886 acc.probe_health_reason = Some(reason);
1887 }
1888 }
1889 RunArtifactKind::ExpectErrLoad => {
1890 // Presence alone is the signal — the body carries only
1891 // `test_name` (already the map key). A malformed body does
1892 // not suppress it: the marker exists because the writer
1893 // observed an `expect_err` inversion satisfied by a load
1894 // failure, and that observability must not hinge on a
1895 // parse.
1896 acc.expect_err_load = true;
1897 }
1898 }
1899 }
1900 if by_test.is_empty() {
1901 return None;
1902 }
1903 let mut failed = Vec::new();
1904 let mut host_skips = Vec::new();
1905 let mut probe_issues = Vec::new();
1906 let mut expect_err_load = Vec::new();
1907 for (test_name, mut acc) in by_test {
1908 // Host-insufficiency skip, probe-pipeline problem, and an
1909 // expect_err inversion satisfied by a load failure are all
1910 // orthogonal to the FAILED gate below (a host skip is not a
1911 // failure, a probe problem rides an otherwise-passing auto-repro
1912 // run, and an expect_err-load test's FINAL verdict is a PASS so it
1913 // never reaches `failed`), so collect them BEFORE the failure
1914 // `continue`.
1915 if let Some(class) = acc.host_skip_class.take() {
1916 host_skips.push((test_name.clone(), class));
1917 }
1918 if let Some(reason) = acc.probe_health_reason.take() {
1919 probe_issues.push((test_name.clone(), reason));
1920 }
1921 if acc.expect_err_load {
1922 expect_err_load.push(test_name.clone());
1923 }
1924 // A test FAILED this run if ANY of its variant sidecars records
1925 // `is_fail` (the FINAL post-inversion verdict), OR it left a
1926 // failure dump WITHOUT any parsed sidecar. A dump with no sidecar
1927 // is a pre-sidecar failure (scheduler load / VM boot) that never
1928 // reached `write_sidecar` — it must still flag. But a dump
1929 // alongside a parsed, non-failing sidecar is an expected-failure
1930 // run (expect_err / expect_auto_repro) whose induced-crash dump
1931 // must NOT flag: the sidecar's finalized verdict is authoritative.
1932 // The `is_fail` aggregate covers in-VM assertion failures,
1933 // including one failing gauntlet variant among passing siblings.
1934 // Per-variant dump gate: a failure dump whose variant has NO
1935 // parsed sidecar is a pre-sidecar failure for that preset and
1936 // flags — even when a SIBLING preset's sidecar parsed. (If every
1937 // dump-variant has a parsed sidecar, dump_hashes ⊆ parsed and the
1938 // sidecars' finalized verdicts are authoritative.) Closes the
1939 // mixed-gauntlet masking the old test-name-granularity gate had.
1940 let dump_only_failure = !acc.dump_hashes.is_subset(&acc.parsed_sidecar_hashes);
1941 if !acc.any_fail && !dump_only_failure {
1942 continue;
1943 }
1944 let (scheduler, topology) = match acc.fail_variant {
1945 Some((sch, topo)) => (Some(sch), Some(topo)),
1946 None => (None, None),
1947 };
1948 // Sort so the rendered footer is deterministic regardless of
1949 // `read_dir` order.
1950 acc.failure_dumps.sort();
1951 acc.repro_failure_dumps.sort();
1952 acc.stats_sidecars.sort();
1953 failed.push(FailedTest {
1954 test_name,
1955 scheduler,
1956 topology,
1957 failure_dumps: acc.failure_dumps,
1958 repro_failure_dumps: acc.repro_failure_dumps,
1959 stats_sidecars: acc.stats_sidecars,
1960 wprof: acc.wprof,
1961 repro_wprof: acc.repro_wprof,
1962 replayable: acc.any_fail,
1963 });
1964 }
1965 Some(RunDirSummary {
1966 dir: dir.to_path_buf(),
1967 failed,
1968 stats_sidecars,
1969 wprof_traces,
1970 host_skips,
1971 probe_issues,
1972 expect_err_load,
1973 })
1974}
1975
1976/// Summarize the artifacts every run directory directly under
1977/// `runs_root` holds, keeping only files written at or after
1978/// `since`. Each [`RunDirSummary`] names its failed tests and the
1979/// concrete artifact path for each, so the `cargo ktstr test`
1980/// footer can point an operator at the exact file for the exact
1981/// test that failed rather than a directory + glob legend.
1982///
1983/// `since` is the wall-clock instant captured before the nextest
1984/// build+run; the mtime gate it drives is what excludes stale
1985/// artifacts left in a reused run directory (see
1986/// [`summarize_one_run_dir`]). Directories are returned sorted by
1987/// path so multi-kernel gauntlet output renders deterministically.
1988pub(crate) fn summarize_run_artifacts(
1989 runs_root: &std::path::Path,
1990 since: std::time::SystemTime,
1991) -> Vec<RunDirSummary> {
1992 let Ok(entries) = std::fs::read_dir(runs_root) else {
1993 return Vec::new();
1994 };
1995 let mut out: Vec<RunDirSummary> = entries
1996 .flatten()
1997 .filter(is_run_directory)
1998 .filter_map(|e| summarize_one_run_dir(&e.path(), since))
1999 .collect();
2000 out.sort_by(|a, b| a.dir.cmp(&b.dir));
2001 out
2002}
2003
2004/// Render the `cargo ktstr test` post-run footer: for each run
2005/// directory written at or after `since`, name every FAILED test
2006/// and the concrete path to each of its artifacts (failure dump,
2007/// auto-repro dump, stats sidecar, wprof trace), plus a per-dir
2008/// count of stats sidecars and wprof traces.
2009///
2010/// Returns the empty string when no run directory under `runs_root`
2011/// holds fresh artifacts — a host-only run (no VM tests) writes no
2012/// sidecars, so there is nothing to point at and the caller emits
2013/// no footer.
2014///
2015/// A test is listed FAILED when it left a failure dump (real or
2016/// placeholder) or an `is_fail` stats sidecar. This is NOT an
2017/// exhaustive failure list: a failure that writes neither — a
2018/// `builder.build()` / `vm.run()` error, a pre-build host error
2019/// (kvm probe, kernel/scheduler resolve, validation), a host panic,
2020/// or an unparseable guest result — leaves no on-disk artifact and
2021/// no entry here. The caller (`cargo_ktstr::run_cargo`) treats
2022/// nextest's own exit status as the authoritative pass/fail signal
2023/// and notes, when nextest reports failures, that any failure
2024/// without an entry left no artifact.
2025///
2026/// This replaces a directory + `*.glob` legend that carried no
2027/// test attribution: a reused run directory mixes artifacts from
2028/// many tests (and, before the mtime gate, prior runs), so a glob
2029/// legend pointed an operator at the directory and left them to
2030/// guess which `*.failure-dump.json` belonged to the test that
2031/// just failed.
2032pub fn format_run_artifact_footer(
2033 runs_root: &std::path::Path,
2034 since: std::time::SystemTime,
2035) -> String {
2036 let summaries = summarize_run_artifacts(runs_root, since);
2037 if summaries.is_empty() {
2038 return String::new();
2039 }
2040 let mut out = String::new();
2041 out.push_str("\ncargo ktstr: test outputs\n");
2042 for s in &summaries {
2043 out.push_str(&format!(" {}\n", s.dir.display()));
2044 for f in &s.failed {
2045 // scheduler/topology are set together (both from one
2046 // failing variant) or both absent (dump-only) — see
2047 // `summarize_one_run_dir`; no mixed arm is reachable.
2048 let variant = match (&f.scheduler, &f.topology) {
2049 (Some(sch), Some(topo)) => format!(" [{sch} {topo}]"),
2050 _ => String::new(),
2051 };
2052 out.push_str(&format!(" FAILED {}{variant}\n", f.test_name));
2053 for p in &f.failure_dumps {
2054 out.push_str(&format!(" {:<13} {}\n", "failure dump", p.display()));
2055 }
2056 for p in &f.repro_failure_dumps {
2057 out.push_str(&format!(" {:<13} {}\n", "repro dump", p.display()));
2058 }
2059 for p in &f.stats_sidecars {
2060 out.push_str(&format!(" {:<13} {}\n", "stats", p.display()));
2061 }
2062 if let Some(p) = &f.wprof {
2063 out.push_str(&format!(" {:<13} {}\n", "wprof", p.display()));
2064 }
2065 if let Some(p) = &f.repro_wprof {
2066 out.push_str(&format!(" {:<13} {}\n", "repro wprof", p.display()));
2067 }
2068 if f.replayable {
2069 out.push_str(&format!(
2070 " {:<13} cargo ktstr replay --filter {} --exec\n",
2071 "replay", f.test_name,
2072 ));
2073 }
2074 }
2075 out.push_str(&render_host_skips(&s.host_skips));
2076 out.push_str(&render_probe_issues(&s.probe_issues));
2077 out.push_str(&render_expect_err_load(&s.expect_err_load));
2078 // Fold the "run stats-last-run for the analysis" discoverability
2079 // hint into the existing count line, but only when this run
2080 // actually wrote a stats sidecar — pointing at an empty analysis
2081 // would be noise. The gauntlet analysis blob is no longer
2082 // auto-printed after a test run; this line is how an operator
2083 // finds it.
2084 if s.stats_sidecars > 0 {
2085 out.push_str(&format!(
2086 " ({} stats sidecar(s), {} wprof trace(s) written this run) \
2087 — run `cargo ktstr stats last-run` for the gauntlet analysis\n",
2088 s.stats_sidecars, s.wprof_traces,
2089 ));
2090 } else {
2091 out.push_str(&format!(
2092 " ({} stats sidecar(s), {} wprof trace(s) written this run)\n",
2093 s.stats_sidecars, s.wprof_traces,
2094 ));
2095 }
2096 }
2097 out
2098}
2099
2100/// Render the host-topology-skip block: one line per skip class, each
2101/// naming its count and the tests this host could not run. Returns the
2102/// empty string when there are no host skips (the common case), so the
2103/// footer stays silent unless this host actually forced a skip.
2104///
2105/// `skips` is `(test_name, class)` collected from the run dir's
2106/// `.host-skip.json` markers; classes are grouped (BTreeMap →
2107/// class-alphabetical, deterministic) and each group's tests are listed
2108/// in the input order (already `test_name`-sorted by
2109/// `summarize_one_run_dir`'s BTreeMap walk).
2110fn render_host_skips(skips: &[(String, String)]) -> String {
2111 if skips.is_empty() {
2112 return String::new();
2113 }
2114 let mut by_class: std::collections::BTreeMap<&str, Vec<&str>> =
2115 std::collections::BTreeMap::new();
2116 for (test, class) in skips {
2117 by_class.entry(class).or_default().push(test);
2118 }
2119 let mut out = String::from(" host-skipped (this host cannot run):\n");
2120 for (class, tests) in by_class {
2121 out.push_str(&format!(
2122 " {} ({}): {}\n",
2123 class,
2124 tests.len(),
2125 tests.join(", "),
2126 ));
2127 }
2128 out
2129}
2130
2131/// Render the auto-repro probe-health block: one line per test whose
2132/// probe pipeline had a problem, with its short reason. Empty string
2133/// when every probe pipeline was healthy, so the footer says nothing on
2134/// a clean run.
2135fn render_probe_issues(issues: &[(String, String)]) -> String {
2136 if issues.is_empty() {
2137 return String::new();
2138 }
2139 let mut out = String::from(" probe pipeline problems:\n");
2140 for (test, reason) in issues {
2141 out.push_str(&format!(" {test}: {reason}\n"));
2142 }
2143 out
2144}
2145
2146/// Render the `expect_err`-satisfied-by-load-failure block: one line
2147/// naming every test whose `expect_err` inversion passed because the
2148/// scheduler FAILED TO LOAD/attach, not because it hit the runtime error
2149/// the test intends. Empty string when there are none, so the footer is
2150/// silent on a normal run.
2151///
2152/// This is an ADVISORY, not a failure: each named test still PASSED
2153/// (`expect_err` inverted the error). But a whole suite in this block is
2154/// the tell that no scheduler could load on this kernel — the suite is
2155/// green without having exercised anything. See
2156/// [`write_expect_err_load_marker`].
2157fn render_expect_err_load(tests: &[String]) -> String {
2158 if tests.is_empty() {
2159 return String::new();
2160 }
2161 format!(
2162 " expect_err satisfied by scheduler load failure (not a runtime error): {}\n",
2163 tests.join(", "),
2164 )
2165}
2166
2167/// Detect the kernel version associated with the current test run.
2168///
2169/// Routes through [`crate::ktstr_kernel_env`] for the raw env value
2170/// and [`crate::kernel_path::KernelId`] for variant dispatch so the
2171/// three [`crate::kernel_path::KernelId`] variants are honoured symmetrically:
2172///
2173/// - `KernelId::Path(dir)`: read `metadata.json` (cache entry
2174/// layout) or `include/config/kernel.release` (source tree
2175/// layout). Unchanged from the previous behaviour.
2176/// - `KernelId::Version(ver)`: the user asked for a specific
2177/// version — return it directly. No cache access needed; a
2178/// version string IS a version string.
2179/// - `KernelId::CacheKey(key)`: look up the cache entry and
2180/// return `entry.metadata.version`. The previous code path
2181/// silently treated the key as a directory name and read
2182/// `<cwd>/<key>/metadata.json`, which never matched — producing
2183/// `None` + `sidecar_dir()` using the `"unknown"` fallback even
2184/// though the cache metadata already carried the version.
2185///
2186/// Returns `None` when the env var is unset, or when the env
2187/// resolves to a variant whose underlying source doesn't yield a
2188/// version string (e.g. a Path whose metadata.json / kernel.release
2189/// are both absent, or a CacheKey with no cache hit).
2190pub(crate) fn detect_kernel_version() -> Option<String> {
2191 use crate::kernel_path::KernelId;
2192 let raw = crate::ktstr_kernel_env()?;
2193 match KernelId::parse(&raw) {
2194 KernelId::Path(_) => {
2195 let p = std::path::Path::new(&raw);
2196 let meta_path = p.join("metadata.json");
2197 if let Ok(data) = std::fs::read_to_string(&meta_path)
2198 && let Ok(meta) = serde_json::from_str::<crate::cache::KernelMetadata>(&data)
2199 {
2200 return meta.version;
2201 }
2202 let ver_path = p.join("include/config/kernel.release");
2203 if let Ok(v) = std::fs::read_to_string(ver_path) {
2204 let v = v.trim();
2205 if !v.is_empty() {
2206 return Some(v.to_string());
2207 }
2208 }
2209 None
2210 }
2211 KernelId::Version(ver) => Some(ver),
2212 KernelId::CacheKey(key) => {
2213 let cache = crate::cache::CacheDir::new().ok()?;
2214 let entry = cache.lookup(&key)?;
2215 entry.metadata.version
2216 }
2217 // Multi-kernel specs in KTSTR_KERNEL never reach this
2218 // function in production — `find_kernel`'s env reader bails
2219 // before sidecar writing happens. This arm is defensive: if
2220 // the env value is somehow a range or git spec, return
2221 // `None` rather than guessing one endpoint, and the sidecar
2222 // record will leave `kernel_version` as null.
2223 KernelId::Range { .. }
2224 | KernelId::Git { .. }
2225 | KernelId::Package { .. }
2226 | KernelId::Distro { .. } => None,
2227 }
2228}
2229
2230/// Detect the ktstr project's git HEAD at sidecar-write time.
2231///
2232/// Walks up from the test process's current working directory via
2233/// `gix::discover` to find an enclosing repository, then reads HEAD
2234/// short-hex (7 chars via `oid::to_hex_with_len(7)`) and appends
2235/// `-dirty` when index-vs-HEAD or worktree-vs-index changes are
2236/// observed. Submodules are ignored
2237/// (`Submodule::Given { ignore: All }`).
2238///
2239/// Dirt-detection runs through the shared [`repo_is_dirty`]
2240/// helper (peel HEAD to its tree, diff tree-vs-index, then
2241/// `status()` for worktree-vs-index, submodules skipped); see its
2242/// doc for cascade details. The cascade is similar in spirit to
2243/// [`crate::fetch::local_source`]'s dirt probe but deliberately
2244/// diverges in missing-index handling: the sidecar path silently
2245/// degrades a missing index leg to "treat as clean" so metadata
2246/// probes never gate sidecar writes, whereas `local_source`'s
2247/// cache-key path treats every leg as load-bearing. The HASH
2248/// REPRESENTATION also DIFFERS: `fetch::local_source` DROPS the
2249/// short hash entirely on dirty (returns `None`) because the
2250/// commit no longer describes the build input the cache key
2251/// embeds — publishing a stale hash there would misidentify the
2252/// build. This helper KEEPS the hash with a `-dirty` suffix
2253/// instead because the sidecar's `project_commit` is a debugging
2254/// breadcrumb (operator-readable identity, not a cache-key input);
2255/// the hash plus dirty flag carries strictly more information
2256/// than `None` for the operator's "which ktstr commit did this
2257/// sidecar come from?" question.
2258///
2259/// Returns `None` when:
2260/// - `current_dir()` cannot be resolved (process has no valid
2261/// cwd — extremely rare; happens only for processes whose cwd
2262/// was rmdir'd while alive);
2263/// - cwd is not inside any git repository (`gix::discover` fails);
2264/// - HEAD cannot be read (an unborn HEAD on a fresh `git init`
2265/// with zero commits, or a corrupt repository).
2266///
2267/// Returns `Some(short_hash)` (without the `-dirty` suffix) when
2268/// the HEAD read succeeds but a downstream dirt-detection call
2269/// fails — including a missing index, an unreadable working tree,
2270/// or `head_tree()` failure. Each failed leg degrades to "treat
2271/// as clean" rather than aborting the probe, because metadata
2272/// must not gate sidecar writes.
2273///
2274/// `None` is the documented fallback — sidecar writing must not
2275/// abort because of a metadata probe failure. Stats tooling that
2276/// reads `project_commit` already tolerates `None` rows by
2277/// treating them as wildcards (no `--project-commit` filter narrowing
2278/// applies).
2279///
2280/// `gix::discover` is preferred over `gix::open` because tests can
2281/// be launched from a subdirectory of the repo (e.g.
2282/// `cd src && cargo test`); `discover` walks parents until it
2283/// finds the `.git` marker, while `open` requires the exact root
2284/// path. The walk is cheap — a few stat() calls bounded by the
2285/// depth of the cwd inside the repo.
2286///
2287/// `env!("CARGO_MANIFEST_DIR")` is deliberately NOT used here:
2288/// `env!` resolves at compile time and bakes the build-host's
2289/// absolute manifest path into the binary's read-only data
2290/// segment, leaking the build environment into every published
2291/// artifact. Resolving cwd at runtime instead means the recorded
2292/// commit reflects the project tree the test was launched FROM —
2293/// for a scheduler crate using ktstr as a dev-dependency, this is
2294/// the scheduler crate's commit, not ktstr's. That is the more
2295/// accurate semantic anyway: "what code produced this sidecar"
2296/// depends on the cwd at test launch (which crate is exercising
2297/// ktstr), not the build host.
2298#[doc(hidden)]
2299pub fn detect_project_commit() -> Option<String> {
2300 // Explicit override: an orchestrator (perf-delta) that checked the
2301 // project tree out WITHOUT a `.git` — a plain gix checkout of a baseline
2302 // commit into a temp dir — passes the commit label via
2303 // KTSTR_PROJECT_COMMIT_ENV so the sidecar records it verbatim instead of
2304 // a `gix::discover` that would resolve to the wrong repo (or none). It is
2305 // also set on the HEAD run so the recorded `project_commit` equals the
2306 // exact label perf-delta filters the pool on, closing the -dirty-suffix
2307 // mismatch between the filter (`short_hash`) and this recorder. Empty is
2308 // treated as unset. Mirrors the KTSTR_KERNEL_COMMIT_ENV override.
2309 if let Ok(explicit) = std::env::var(crate::KTSTR_PROJECT_COMMIT_ENV)
2310 && !explicit.is_empty()
2311 {
2312 return Some(explicit);
2313 }
2314 // Per-process memoization of the SUCCESS case only.
2315 //
2316 // The cwd is stable for the lifetime of a test process (no
2317 // caller mutates it), and the project tree's HEAD plus dirty
2318 // state cannot change underneath us without an explicit user
2319 // action that's outside the scope of any individual sidecar
2320 // write. Gauntlet runs invoke this function once per sidecar —
2321 // thousands of times per process — so caching the resolved
2322 // hash collapses every post-first successful call to a
2323 // `Clone`. The probe itself does ~3 syscalls (gix discover +
2324 // head_id + status) which dominate the sidecar-write critical
2325 // path; eliminating that cost on the hot path is the only
2326 // meaningful perf win available here.
2327 //
2328 // FAILURE IS NOT CACHED: a `None` probe outcome (no git repo
2329 // discoverable from cwd, unborn HEAD, transient FS / gix open
2330 // failure) does NOT seed the OnceLock. A FIRST call from a
2331 // momentarily-broken context (e.g. a test that swapped CWD via
2332 // some indirect path before ever calling
2333 // `detect_project_commit`, or a transient I/O hiccup during
2334 // `gix::discover`) would otherwise lock in `None` for the
2335 // rest of the process — every subsequent sidecar would land
2336 // under `target/ktstr/{kernel}-unknown/` even though the
2337 // commit IS resolvable from a healthy cwd. Retrying on failure
2338 // costs the same ~3 syscalls the success case pays once; the
2339 // re-probe only fires while the answer is still unknown.
2340 //
2341 // CACHE DOES NOT INVALIDATE on success: a user who commits /
2342 // amends / resets the project tree mid-run and expects the
2343 // new HEAD to surface in subsequent sidecars will see stale
2344 // values. This is acceptable — the
2345 // project tree is treated as stable-enough for a single suite
2346 // run; callers mutating the tree during a run own the
2347 // consequences.
2348 static PROJECT_COMMIT: std::sync::OnceLock<String> = std::sync::OnceLock::new();
2349 if let Some(cached) = PROJECT_COMMIT.get() {
2350 return Some(cached.clone());
2351 }
2352 let cwd = std::env::current_dir().ok()?;
2353 let probed = detect_commit_at(&cwd)?;
2354 // `set` on a hot OnceLock is a no-op `Err` — safe to ignore.
2355 // First successful caller wins; a second concurrent caller's
2356 // identical hash discards harmlessly.
2357 let _ = PROJECT_COMMIT.set(probed.clone());
2358 Some(probed)
2359}
2360
2361/// Path-taking core of [`detect_project_commit`]. Factored out so
2362/// unit tests can drive the full branch matrix (clean repo, dirty
2363/// repo, non-git directory, unborn HEAD, concurrent calls) against
2364/// `gix::init`-built fixtures in tempdirs without mutating the
2365/// process-wide `current_dir`. The public entry point reads `cwd`
2366/// once and delegates here.
2367///
2368/// `gix::discover` walks parents until it finds a `.git` marker —
2369/// tests can be launched from a subdirectory of the repo (e.g.
2370/// `cd src && cargo test`); the parent walk handles that, where
2371/// `gix::open` would require the exact root. The
2372/// open-vs-discover distinction is the ONLY difference between
2373/// this function and [`detect_kernel_commit`]; the post-open
2374/// "read HEAD, format short hex, append `-dirty` on dirt" body
2375/// lives in the shared [`commit_with_dirty_suffix`] helper.
2376fn detect_commit_at(path: &std::path::Path) -> Option<String> {
2377 let repo = gix::discover(path).ok()?;
2378 commit_with_dirty_suffix(&repo)
2379}
2380
2381/// Shared post-open body for [`detect_commit_at`] and
2382/// [`detect_kernel_commit`]: read `repo.head_id()`, format the
2383/// 7-char short hex, and append `-dirty` when [`repo_is_dirty`]
2384/// returns `Some(true)`.
2385///
2386/// Returns `None` when `head_id()` fails (unborn HEAD on a fresh
2387/// `gix::init` with zero commits, or a corrupt repository) — the
2388/// short-hex cannot be formed.
2389///
2390/// Returns `Some(short_hash)` (without `-dirty`) when the HEAD
2391/// read succeeds but the [`repo_is_dirty`] probe returns `None`
2392/// (HEAD-tree peel failure). This matches the documented "treat
2393/// as clean on probe failure" degradation: metadata probes must
2394/// not gate sidecar writes, so a probe failure flows through as
2395/// "clean" rather than aborting.
2396///
2397/// `to_hex_with_len(7)` produces a `HexDisplay` that formats 7
2398/// hex chars without the 40-char intermediate `format!("{}")`
2399/// allocation. `Id` derefs to `oid` (gix-hash) which owns the
2400/// method.
2401///
2402/// CALL SITES diverge ONLY on the open mode (`gix::discover` for
2403/// the project commit, `gix::open` for the kernel commit). The
2404/// helper takes a `&Repository` so each caller picks the open
2405/// strategy that matches its semantics: project commit walks
2406/// parents (cwd may be inside a subdir of the repo); kernel
2407/// commit demands the explicit root (the kernel directory is
2408/// not walked-up to avoid resolving the parent ktstr repo).
2409fn commit_with_dirty_suffix(repo: &gix::Repository) -> Option<String> {
2410 let head = repo.head_id().ok()?;
2411 let short_hash = head.to_hex_with_len(7).to_string();
2412 if repo_is_dirty(repo).unwrap_or(false) {
2413 Some(format!("{short_hash}-dirty"))
2414 } else {
2415 Some(short_hash)
2416 }
2417}
2418
2419/// Probe whether a gix repository's working tree differs from its
2420/// HEAD commit, ignoring submodules.
2421///
2422/// Returns `Some(true)` when the index differs from the HEAD tree
2423/// or the worktree differs from the index for any tracked file;
2424/// `Some(false)` when neither leg observed a difference; `None`
2425/// when the HEAD-tree peel itself failed (HEAD points at something
2426/// that cannot be read as a tree).
2427///
2428/// Callers in [`detect_commit_at`] / [`detect_kernel_commit`]
2429/// degrade `None` to "treat as clean" via `unwrap_or(false)` so
2430/// metadata probes never gate sidecar writes.
2431///
2432/// PROBE LEGS:
2433/// - tree-vs-index: peel HEAD to its tree, then `tree_index_status`
2434/// diff against the on-disk index. `repo.index()` returning Err
2435/// (missing index — partially-checked-out clones, or fresh
2436/// `git init` before the first commit) silently leaves the
2437/// index-dirty leg false. `index_or_empty()` is deliberately
2438/// NOT used because it would substitute an empty index and the
2439/// diff would flag every tracked file as "deleted from index",
2440/// tripping false-dirty.
2441/// - index-vs-worktree: `repo.status()` configured with
2442/// `Submodule::Given { ignore: All }` so submodule worktree
2443/// state is skipped. Short-circuited when the tree-vs-index leg
2444/// already flipped dirty: the result only needs one positive
2445/// signal, so a known-dirty index makes the worktree walk
2446/// redundant. Matches the equivalent short-circuit in
2447/// [`crate::fetch::local_source`].
2448///
2449/// FAILURE DEGRADATION: any individual leg failure (missing index,
2450/// `repo.status()` failure, `into_index_worktree_iter()` failure)
2451/// silently degrades that leg to "no signal" rather than aborting.
2452/// The function only returns `None` when the HEAD-tree peel
2453/// fails, because at that point neither leg can run at all.
2454///
2455/// `pub` (not `pub(crate)`) because `cargo-ktstr.rs` is a
2456/// separate `[[bin]]` crate that consumes `ktstr` as an
2457/// external dependency and needs this helper to compute the
2458/// `-dirty` suffix in
2459/// the baseline/HEAD commit in `cargo ktstr perf-delta`. Hidden
2460/// from rustdoc via `#[doc(hidden)]` because it is a probe-
2461/// style helper without a stable API contract — external
2462/// consumers should not depend on it.
2463#[doc(hidden)]
2464pub fn repo_is_dirty(repo: &gix::Repository) -> Option<bool> {
2465 let head_tree_id = repo.head_tree().ok()?.id;
2466
2467 let mut index_dirty = false;
2468 if let Ok(index) = repo.index() {
2469 let _ = repo.tree_index_status(
2470 &head_tree_id,
2471 &index,
2472 None,
2473 gix::status::tree_index::TrackRenames::Disabled,
2474 |_, _, _| {
2475 index_dirty = true;
2476 Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Break(()))
2477 },
2478 );
2479 }
2480
2481 let worktree_dirty = if index_dirty {
2482 false
2483 } else {
2484 repo.status(gix::progress::Discard)
2485 .ok()
2486 .and_then(|s| {
2487 s.index_worktree_rewrites(None)
2488 .index_worktree_submodules(gix::status::Submodule::Given {
2489 ignore: gix::submodule::config::Ignore::All,
2490 check_dirty: false,
2491 })
2492 .index_worktree_options_mut(|opts| {
2493 configure_dirty_status_options(opts);
2494 })
2495 .into_index_worktree_iter(Vec::new())
2496 .ok()
2497 .map(crate::git_status::consume_has_any)
2498 })
2499 .unwrap_or(false)
2500 };
2501
2502 Some(index_dirty || worktree_dirty)
2503}
2504
2505/// Configure the tracked-file-only sidecar probe while reusing the shared
2506/// gix worker policy.
2507fn configure_dirty_status_options(options: &mut gix::status::index_worktree::Options) {
2508 options.dirwalk_options = None;
2509 crate::git_status::configure_index_worktree_parallelism(options);
2510}
2511
2512/// Detect the kernel SOURCE TREE's git HEAD at sidecar-write time.
2513///
2514/// `kernel_dir` is the explicit kernel source directory — typically
2515/// resolved from `KTSTR_KERNEL` for `KernelId::Path`, or from the
2516/// cache entry's `KernelSource::Local::source_tree_path` when
2517/// `KTSTR_KERNEL` is a Version / CacheKey whose underlying build
2518/// recorded a local tree. Uses `gix::open(kernel_dir)` (NOT
2519/// `gix::discover`) because the kernel directory is explicit, not
2520/// walked-up: the parent walk that `discover` performs would
2521/// resolve to whichever ancestor `.git` it found first, which
2522/// might be the ktstr project's repo when `kernel_dir` is a
2523/// non-git subdirectory inside it. `open` requires `kernel_dir`
2524/// itself to be the repo root, which is the documented invariant
2525/// for kernel checkouts.
2526///
2527/// Reads HEAD short-hex (7 chars via `oid::to_hex_with_len(7)`)
2528/// and appends `-dirty` when index-vs-HEAD or worktree-vs-index
2529/// changes are observed. Dirt-detection runs through the shared
2530/// [`repo_is_dirty`] helper (submodules skipped via
2531/// `Submodule::Given { ignore: All }`); see its doc for cascade
2532/// details. The cascade matches [`detect_project_commit`] and is
2533/// similar in spirit to [`crate::fetch::local_source`] but
2534/// deliberately diverges in missing-index handling: the sidecar
2535/// path silently degrades a missing index leg to "treat as
2536/// clean" so metadata probes never gate sidecar writes, whereas
2537/// `local_source`'s cache-key path treats every leg as
2538/// load-bearing. Same "treat as clean on probe failure"
2539/// degradation rules apply otherwise: a missing index, an
2540/// unreadable worktree, or `head_tree()` failure each fall
2541/// through as "clean" rather than aborting the probe — metadata
2542/// must not gate sidecar writes.
2543///
2544/// HASH REPRESENTATION matches [`detect_project_commit`]: keeps
2545/// the hash with `-dirty` appended (operator-readable identity).
2546/// Distinct from [`crate::fetch::local_source`], which DROPS the
2547/// hash on dirty because the commit no longer describes the
2548/// build INPUT for cache-key purposes.
2549///
2550/// Returns `None` when:
2551/// - `kernel_dir` is not a git repository (`gix::open` fails);
2552/// - HEAD cannot be read (unborn HEAD on a fresh `git init` with
2553/// zero commits, or a corrupt repository).
2554///
2555/// Returns `Some(short_hash)` (without the `-dirty` suffix) when
2556/// the HEAD read succeeds but a downstream dirt-detection call
2557/// fails — including a missing index, an unreadable working
2558/// tree, or `head_tree()` failure. Each failed leg degrades to
2559/// "treat as clean" rather than aborting the probe, because
2560/// metadata must not gate sidecar writes.
2561///
2562/// `pub` (not `pub(crate)`) + `#[doc(hidden)]` because cargo-ktstr is a
2563/// separate `[[bin]]` crate that consumes `ktstr` as a dependency. Its
2564/// resolved-kernel provenance helper falls back to this walk for raw source
2565/// trees and legacy cache entries. Hidden from rustdoc — a probe helper with
2566/// no stable API contract.
2567#[doc(hidden)]
2568pub fn detect_kernel_commit(kernel_dir: &std::path::Path) -> Option<String> {
2569 // Per-process, path-keyed memoization of the SUCCESS case
2570 // only. Same rationale as `detect_project_commit`: gauntlet
2571 // runs invoke this function once per sidecar — thousands of
2572 // times — and the kernel tree's HEAD plus dirty state cannot
2573 // change underneath us mid-suite without an explicit user
2574 // action outside any sidecar's control. The path key handles
2575 // the fixture-test case where unit tests rotate through
2576 // synthetic `tempfile::TempDir` kernel paths in the same
2577 // process; each distinct path memoizes independently.
2578 //
2579 // `Mutex<HashMap>` rather than `OnceLock` because the input
2580 // is parameterized on `kernel_dir` — a `OnceLock` collapses
2581 // every input to one cached result, which would conflate
2582 // different kernel directories into a single value.
2583 // Contention is bounded: post-warm reads are O(1) hash
2584 // lookups against a near-empty map (in production typically
2585 // ONE kernel per process), and the mutex is held only for
2586 // the duration of the lookup + insert.
2587 //
2588 // FAILURE IS NOT CACHED: a `None` probe outcome (kernel_dir
2589 // is not a git repo, unborn HEAD, transient `gix::open`
2590 // failure) does NOT seed the cache. Caching `None` would lock
2591 // in `unknown` for every subsequent sidecar even after the
2592 // condition resolves (e.g. a kernel directory that becomes a
2593 // valid checkout mid-suite, or a flaky FS that recovers).
2594 // Re-probing on failure costs the same gix-open + dirt-walk
2595 // the success case pays once; the re-probe only fires while
2596 // the answer is still unknown for that path.
2597 //
2598 // Mutex poisoning recovery: a panic mid-probe could poison
2599 // the lock; acquiring via
2600 // [`crate::sync::MutexExt::lock_unpoisoned`] returns the
2601 // guard regardless of poison state so a future caller doesn't
2602 // fail catastrophically. The cached map is just a HashMap of
2603 // owned strings; no invariant beyond "key→value mapping" can
2604 // be broken by an interrupted probe.
2605 use std::collections::HashMap;
2606 use std::path::PathBuf;
2607 use std::sync::{Mutex, OnceLock};
2608 static KERNEL_COMMIT_CACHE: OnceLock<Mutex<HashMap<PathBuf, String>>> = OnceLock::new();
2609 // Canonicalize the cache key so two paths that resolve to the
2610 // same on-disk directory share one entry. Without this, a
2611 // symlinked alias (`./linux` symlinked to `/abs/.../linux`)
2612 // and the resolved target would each populate their own slot,
2613 // re-running the gix-open + dirt-walk on every alias and
2614 // defeating the memoization. `canonicalize` resolves symlinks,
2615 // collapses `..` / `.`, and yields the absolute path the
2616 // kernel actually lives at. Falls back to the raw path on
2617 // canonicalize failure (e.g. caller passed a non-existent
2618 // `kernel_dir`) — gix::open will fail downstream and re-probe
2619 // each call until the path becomes resolvable.
2620 let cache_key = kernel_dir
2621 .canonicalize()
2622 .unwrap_or_else(|_| kernel_dir.to_path_buf());
2623 let cache = KERNEL_COMMIT_CACHE.get_or_init(|| Mutex::new(HashMap::new()));
2624 {
2625 let guard = cache.lock_unpoisoned();
2626 if let Some(cached) = guard.get(&cache_key) {
2627 return Some(cached.clone());
2628 }
2629 }
2630 // `gix::open` (NOT `gix::discover`) — `kernel_dir` must BE the
2631 // repo root. Without this the parent walk could resolve to the
2632 // ktstr project's own `.git` when `kernel_dir` is a non-git
2633 // subdirectory inside the ktstr checkout. The
2634 // open-vs-discover distinction is the ONLY difference between
2635 // this function and [`detect_commit_at`]; the post-open
2636 // "read HEAD, format short hex, append `-dirty` on dirt" body
2637 // lives in the shared [`commit_with_dirty_suffix`] helper.
2638 //
2639 // Open against `kernel_dir` (the caller-supplied path) rather
2640 // than `cache_key`. The two paths point at the same on-disk
2641 // repo by construction (canonicalize resolves to the same
2642 // place), so gix opens the same repository either way; passing
2643 // the original keeps any user-facing diagnostics (gix's
2644 // internal error chain) consistent with the input shape.
2645 let result = gix::open(kernel_dir)
2646 .ok()
2647 .and_then(|repo| commit_with_dirty_suffix(&repo));
2648 if let Some(ref hash) = result {
2649 let mut guard = cache.lock_unpoisoned();
2650 // First successful caller wins; a concurrent caller's
2651 // identical hash would overwrite harmlessly because
2652 // success is deterministic for a given (canonicalized
2653 // path, HEAD, dirty state) tuple.
2654 guard.insert(cache_key, hash.clone());
2655 }
2656 result
2657}
2658
2659/// Environment variable CI runners set to mark sidecars they produce
2660/// as `"ci"`-source. Any non-empty value flips the tag; empty string
2661/// is treated as unset so a defensively-cleared variable does not
2662/// accidentally classify a developer run as CI.
2663///
2664/// Read at sidecar-write time by [`detect_run_source`]; matches the
2665/// `KTSTR_KERNEL` / `KTSTR_CACHE_DIR` env-name convention so the
2666/// full set of ktstr-controlled env vars is `KTSTR_*`-prefixed.
2667pub const KTSTR_CI_ENV: &str = "KTSTR_CI";
2668
2669/// Tag value written to [`SidecarResult::run_source`] for sidecars
2670/// produced under [`KTSTR_CI_ENV`].
2671pub const SIDECAR_RUN_SOURCE_CI: &str = "ci";
2672
2673/// Tag value written to [`SidecarResult::run_source`] for sidecars
2674/// produced without [`KTSTR_CI_ENV`] — the developer-machine
2675/// default.
2676pub const SIDECAR_RUN_SOURCE_LOCAL: &str = "local";
2677
2678/// Tag value applied to [`SidecarResult::run_source`] /
2679/// [`GauntletRow::run_source`](crate::stats::GauntletRow::run_source)
2680/// at LOAD time when the consumer pulls sidecars from a non-default
2681/// pool root via `cargo ktstr stats show-host --dir` /
2682/// `cargo ktstr stats list-values --dir`. NEVER written by
2683/// [`write_sidecar`] — the writer cannot know the file will later
2684/// be moved off-host. See [`apply_archive_source_override`].
2685pub const SIDECAR_RUN_SOURCE_ARCHIVE: &str = "archive";
2686
2687/// Read [`KTSTR_CI_ENV`] and classify the run as `"ci"` (when the
2688/// env var is set non-empty) or `"local"` (the default for any
2689/// developer-driven invocation). Empty-string env values count as
2690/// unset — see [`KTSTR_CI_ENV`] for rationale.
2691///
2692/// Returns `Some(_)` unconditionally because every sidecar producer
2693/// is, by construction, either local or CI; an `Option` return
2694/// keeps the field shape symmetric with the other nullable
2695/// `SidecarResult` fields and reserves room for a future "unknown"
2696/// arm without a serde-version bump.
2697pub(crate) fn detect_run_source() -> Option<String> {
2698 match std::env::var(KTSTR_CI_ENV) {
2699 Ok(v) if !v.is_empty() => Some(SIDECAR_RUN_SOURCE_CI.to_string()),
2700 _ => Some(SIDECAR_RUN_SOURCE_LOCAL.to_string()),
2701 }
2702}
2703
2704/// Override every sidecar's `run_source` field to
2705/// [`SIDECAR_RUN_SOURCE_ARCHIVE`] when the consumer pulled the pool
2706/// from a non-default root via `--dir`. Called at the boundary
2707/// between [`collect_pool`] and the downstream stats pipeline so
2708/// on-disk values stay untouched while the in-memory pool reflects
2709/// the operator's intent: "these sidecars were copied off another
2710/// host; treat them as archives, not as the local-machine record."
2711///
2712/// Mutation strategy is in-place rewrite of the entire `run_source`
2713/// field — the `"local"` / `"ci"` distinction is meaningful on the
2714/// PRODUCING host but irrelevant once the sidecars have been
2715/// moved off, where the only useful classification is "archived
2716/// elsewhere." Operators who need to retain the producer-side
2717/// distinction inside an archive bucket can keep `--dir`
2718/// untargeted (read from the default root) and let the on-disk
2719/// values pass through.
2720pub(crate) fn apply_archive_source_override(pool: &mut [SidecarResult]) {
2721 for sc in pool {
2722 sc.run_source = Some(SIDECAR_RUN_SOURCE_ARCHIVE.to_string());
2723 }
2724}
2725
2726/// Resolve the kernel source-tree path for [`detect_kernel_commit`]
2727/// from the [`crate::KTSTR_KERNEL_ENV`] env var.
2728///
2729/// Routes through [`crate::ktstr_kernel_env`] for the raw env
2730/// value and [`crate::kernel_path::KernelId`] for variant
2731/// dispatch:
2732///
2733/// - `KernelId::Path(p)`: probes the path's `metadata.json` first
2734/// — `cargo-ktstr`'s `--kernel /path/to/linux` resolver routes
2735/// clean source trees through the cache pipeline (see
2736/// [`crate::cli::resolve_kernel_dir_to_entry`]) and exports the
2737/// CACHE ENTRY directory through `KTSTR_KERNEL`, not the
2738/// literal source tree. When `metadata.json` parses and carries
2739/// a `KernelSource::Local::source_tree_path`, that path is the
2740/// underlying source tree and is returned. When parsing fails
2741/// (the path IS the source tree, the dirty-tree path that
2742/// skipped the cache store), falls back to using the raw env
2743/// value verbatim — that path is itself the source tree.
2744/// - `KernelId::Version(ver)`: looks for a Local cache entry
2745/// whose `metadata.version == ver` carrying a
2746/// `source_tree_path`. The tarball-shaped key (`{ver}-tarball-
2747/// {arch}-kc{suffix}`) is checked first because it is the
2748/// most-common form a Version-shaped env points at; on miss
2749/// (or hit yielding `Tarball` / `Git` source, both of which
2750/// are transient with no on-disk tree to probe), the function
2751/// falls back to scanning every valid cache entry for a Local
2752/// match on version. Without this fallback,
2753/// a cache populated by `kernel build --kernel
2754/// /path/to/linux` (a Local entry with source_tree_path) is
2755/// never found by a sidecar writer that has
2756/// `KTSTR_KERNEL=6.14.2`, even though the local tree is
2757/// exactly what the kernel_commit field needs to probe.
2758/// - `KernelId::CacheKey(k)`: uses `k` verbatim — the cache key
2759/// already carries every detail (source-type prefix, arch,
2760/// kconfig hash). On hit, returns
2761/// `KernelSource::Local::source_tree_path` if set, else
2762/// `None` (Tarball / Git entries are transient and have no
2763/// persisted source tree).
2764/// - `KernelId::Range { .. }` / `KernelId::Git { .. }`:
2765/// multi-kernel specs in `KTSTR_KERNEL` never reach this
2766/// helper in production (find_kernel's env reader bails
2767/// before sidecar writing). Defensive: returns `None`.
2768///
2769/// Returns `None` when the env var is unset, when no source
2770/// tree path is recoverable, or when the cache lookup fails.
2771#[cfg(test)]
2772fn resolve_kernel_source_dir() -> Option<std::path::PathBuf> {
2773 source_dir_for(&crate::ktstr_kernel_env()?)
2774}
2775
2776/// Resolve a `KTSTR_KERNEL` identifier string to the on-disk SOURCE
2777/// tree whose git HEAD is the kernel's commit (or `None` for transient
2778/// Range/Git specs or an unrecoverable cache lookup).
2779///
2780/// `pub` + `#[doc(hidden)]` because the cargo-ktstr `[[bin]]` is a separate
2781/// crate. [`kernel_commit_for_resolved`] uses this for raw source trees and
2782/// legacy Local cache entries after taking the cache-metadata fast path.
2783#[doc(hidden)]
2784pub fn source_dir_for(raw: &str) -> Option<std::path::PathBuf> {
2785 use crate::kernel_path::KernelId;
2786 let id = KernelId::parse(raw);
2787 match id {
2788 KernelId::Path(_) => {
2789 let p = std::path::Path::new(raw);
2790 // Cache-entry layout: `metadata.json` carries the
2791 // `KernelSource::Local::source_tree_path` recorded at
2792 // build time. Source-tree layout (dirty path that
2793 // skipped cache store): no metadata, so the env value
2794 // IS the source tree. The shared helper handles both.
2795 crate::cache::recover_local_source_tree(p)
2796 .or_else(|| Some(std::path::PathBuf::from(raw)))
2797 }
2798 KernelId::Version(_) | KernelId::CacheKey(_) => {
2799 let cache = crate::cache::CacheDir::new().ok()?;
2800 resolve_kernel_source_dir_with_cache(&id, &cache)
2801 }
2802 KernelId::Range { .. }
2803 | KernelId::Git { .. }
2804 | KernelId::Package { .. }
2805 | KernelId::Distro { .. } => None,
2806 }
2807}
2808
2809/// Resolve the commit which built one already-resolved kernel directory.
2810///
2811/// A normal cargo-ktstr resolution returns a kernel cache-entry directory.
2812/// Its `metadata.json` already records the source commit captured when the
2813/// kernel was built, so that value is both more authoritative and much
2814/// cheaper than walking the current source worktree again. In particular,
2815/// concurrent CI lanes no longer each scan the same Linux checkout before
2816/// starting nextest.
2817///
2818/// Local cache metadata written before `git_hash` was populated retains its
2819/// historical fallback: probe the recorded `source_tree_path`. A raw source
2820/// path (no valid cache metadata) likewise uses [`source_dir_for`] followed by
2821/// [`detect_kernel_commit`]. Valid non-local metadata without a commit has no
2822/// on-disk source to probe and returns `None`.
2823///
2824/// `pub` + `#[doc(hidden)]` lets the cargo-ktstr binary and sidecar writer use
2825/// the exact same provenance decision when producing/consuming the shared
2826/// [`crate::KTSTR_KERNEL_COMMIT_ENV`] map.
2827#[doc(hidden)]
2828pub fn kernel_commit_for_resolved(raw: &str) -> Option<String> {
2829 let resolved = std::path::Path::new(raw);
2830 if let Ok(bytes) = std::fs::read(resolved.join("metadata.json"))
2831 && let Ok(metadata) = serde_json::from_slice::<crate::cache::KernelMetadata>(&bytes)
2832 {
2833 return match metadata.source {
2834 crate::cache::KernelSource::Local {
2835 git_hash: Some(hash),
2836 ..
2837 }
2838 | crate::cache::KernelSource::Git {
2839 git_hash: Some(hash),
2840 ..
2841 } => Some(hash),
2842 crate::cache::KernelSource::Local {
2843 source_tree_path: Some(source),
2844 git_hash: None,
2845 } => detect_kernel_commit(&source),
2846 crate::cache::KernelSource::Local { .. }
2847 | crate::cache::KernelSource::Git { .. }
2848 | crate::cache::KernelSource::Tarball
2849 | crate::cache::KernelSource::DistroPackage { .. }
2850 | crate::cache::KernelSource::LocalPackage { .. } => None,
2851 };
2852 }
2853 source_dir_for(raw).and_then(|source| detect_kernel_commit(&source))
2854}
2855
2856/// Pure helper for [`source_dir_for`] that takes the
2857/// parsed `KernelId` and an opened `CacheDir`, returning the source
2858/// tree path if recoverable.
2859///
2860/// Split out from [`source_dir_for`] so tests can pin a
2861/// `CacheDir` at a tempdir root without mutating env vars (which
2862/// would race other tests reading `KTSTR_KERNEL` /
2863/// `KTSTR_CACHE_DIR`).
2864///
2865/// Lookup order for [`crate::kernel_path::KernelId::Version`]:
2866/// 1. Tarball-shaped cache key (`{ver}-tarball-{arch}-kc{suffix}`),
2867/// direct lookup. Returns `Some` only if the entry is a
2868/// `KernelSource::Local` carrying a `source_tree_path`.
2869/// 2. Fallback scan: every valid cache entry whose
2870/// `metadata.version == ver`. First match with
2871/// `KernelSource::Local::source_tree_path` set wins. Handles
2872/// the case where the user built `--kernel /path/to/linux`
2873/// (a Local cache entry without the tarball cache-key prefix)
2874/// but later set `KTSTR_KERNEL=6.14.2` for the test run —
2875/// without this fallback, the local source tree would be
2876/// invisible to the sidecar writer.
2877///
2878/// `KernelSource::Tarball` and `KernelSource::Git` entries are
2879/// skipped at every step because their source trees are transient
2880/// (deleted by the cache pipeline after build), so probing them
2881/// for a `kernel_commit` would always fail.
2882///
2883/// For [`crate::kernel_path::KernelId::CacheKey`], performs a single direct lookup —
2884/// the cache key already encodes every detail (source-type
2885/// prefix, arch, kconfig hash) so no fallback scan is needed.
2886fn resolve_kernel_source_dir_with_cache(
2887 id: &crate::kernel_path::KernelId,
2888 cache: &crate::cache::CacheDir,
2889) -> Option<std::path::PathBuf> {
2890 use crate::kernel_path::KernelId;
2891 match id {
2892 KernelId::Version(ver) => {
2893 let arch = std::env::consts::ARCH;
2894 let tarball_key = format!("{ver}-tarball-{arch}-kc{}", crate::cache_key_suffix());
2895 if let Some(entry) = cache.lookup(&tarball_key)
2896 && let crate::cache::KernelSource::Local {
2897 source_tree_path: Some(p),
2898 ..
2899 } = &entry.metadata.source
2900 {
2901 return Some(p.clone());
2902 }
2903 let entries = cache.list().ok()?;
2904 for listed in entries {
2905 let crate::cache::ListedEntry::Valid(entry) = listed else {
2906 continue;
2907 };
2908 if entry.metadata.version.as_deref() != Some(ver.as_str()) {
2909 continue;
2910 }
2911 if let crate::cache::KernelSource::Local {
2912 source_tree_path: Some(p),
2913 ..
2914 } = &entry.metadata.source
2915 {
2916 return Some(p.clone());
2917 }
2918 }
2919 None
2920 }
2921 KernelId::CacheKey(k) => {
2922 let entry = cache.lookup(k)?;
2923 match entry.metadata.source {
2924 crate::cache::KernelSource::Local {
2925 source_tree_path: Some(ref p),
2926 ..
2927 } => Some(p.clone()),
2928 _ => None,
2929 }
2930 }
2931 // Path / Range / Git callers do not reach this helper —
2932 // resolve_kernel_source_dir handles them inline. Defensive
2933 // None covers any future caller that adds a new arm.
2934 _ => None,
2935 }
2936}
2937
2938/// The kernel commit recorded in a sidecar: the env fast-path first,
2939/// then resolved-kernel provenance metadata or the raw-source fallback.
2940///
2941/// cargo-ktstr pre-probes every resolved kernel's HEAD once and exports
2942/// a `dir=commit;...` map in [`crate::KTSTR_KERNEL_COMMIT_ENV`], keyed
2943/// by the dir it also exports as `KTSTR_KERNEL`. This process looks
2944/// itself up by its own [`crate::ktstr_kernel_env`] value — string-equal
2945/// to the map key by construction, since cargo-ktstr built both from the
2946/// same resolved dir. A hit skips `detect_kernel_commit`'s gix HEAD +
2947/// dirty-walk, which is memoized per process but NOT across the per-test
2948/// nextest processes (so without the map each of N processes re-pays
2949/// it).
2950///
2951/// Keying on `ktstr_kernel_env()` (the raw `KTSTR_KERNEL`) is deliberate —
2952/// that is exactly the key cargo-ktstr used. The map's commit VALUE matches
2953/// this function's own fallback because both call
2954/// [`kernel_commit_for_resolved`].
2955///
2956/// Miss / empty commit → the shared provenance resolver. Optimization only.
2957fn kernel_commit_for_sidecar() -> Option<String> {
2958 let self_dir = crate::ktstr_kernel_env()?;
2959 if let Ok(raw) = std::env::var(crate::KTSTR_KERNEL_COMMIT_ENV) {
2960 for seg in raw.split(';') {
2961 if let Some((dir, commit)) = seg.rsplit_once('=')
2962 && dir == self_dir
2963 && !commit.is_empty()
2964 {
2965 return Some(commit.to_string());
2966 }
2967 }
2968 }
2969 kernel_commit_for_resolved(&self_dir)
2970}
2971
2972/// Compute a stable 64-bit discriminator over the fields that
2973/// distinguish gauntlet variants of the same test. Used to suffix
2974/// the sidecar filename so concurrent variants do not clobber each
2975/// other's output.
2976///
2977/// Uses [`siphasher::sip::SipHasher13`] with zero keys for the same
2978/// cross-toolchain stability reason as the other zero-keyed
2979/// SipHasher13 sites (`build.rs`, `runtime.rs` `content_hash`) —
2980/// the discriminator
2981/// must be the same across Rust toolchain versions or downstream
2982/// tooling that groups variants by filename breaks.
2983///
2984/// # Host-state collision caveat
2985///
2986/// The hash is over test-identity fields (topology, scheduler,
2987/// payload, work_type, sysctls, kargs) — NOT over
2988/// [`crate::host_context::HostContext`], NOT over `scheduler_commit`, NOT over
2989/// `project_commit`, NOT over `kernel_commit`, NOT over
2990/// `run_source`, NOT over `resolve_source`, and NOT over
2991/// `cpu_budget` / `vcpus`. The
2992/// [`crate::host_context::HostContext`] exclusion is pinned by
2993/// `sidecar_variant_hash_excludes_host_context`; the
2994/// `scheduler_commit` exclusion by
2995/// `sidecar_variant_hash_excludes_scheduler_commit`; the
2996/// `project_commit` exclusion by
2997/// `sidecar_variant_hash_excludes_project_commit`; the
2998/// `kernel_commit` exclusion by
2999/// `sidecar_variant_hash_excludes_kernel_commit`; the
3000/// `run_source` exclusion by
3001/// `sidecar_variant_hash_excludes_run_source`; the
3002/// `resolve_source` exclusion by
3003/// `sidecar_variant_hash_excludes_resolve_source`; the
3004/// `cpu_budget` / `vcpus` exclusion by
3005/// `sidecar_variant_hash_excludes_cpu_budget`. All seven are
3006/// deliberate for the same cross-host grouping reason — a
3007/// gauntlet rebuilt against a different userspace scheduler
3008/// commit, a bumped ktstr checkout, a kernel source tree at a
3009/// different HEAD, a different CI runner / developer machine, a
3010/// run that resolved its scheduler via a different discovery
3011/// path, or a run that confined its vCPUs to a different
3012/// host-CPU budget must still bucket with the same-named variant so
3013/// `compare_partitions` can diff two runs of the "same" test
3014/// without the commit hash, run-source tag, or budget shattering
3015/// them into one-row-per-commit islands. `cpu_budget` / `vcpus`
3016/// are instead surfaced as the [`crate::stats::Dimension::CpuBudget`]
3017/// pairing axis, which separates cross-budget runs at compare time
3018/// rather than at the identity bucket. Callers that want to detect
3019/// a commit drift or compare across run environments inspect
3020/// [`SidecarResult::scheduler_commit`] /
3021/// [`SidecarResult::project_commit`] /
3022/// [`SidecarResult::kernel_commit`] /
3023/// [`SidecarResult::run_source`] /
3024/// [`SidecarResult::resolve_source`] directly (via
3025/// `--project-commit` / `--kernel-commit` / `--run-source` /
3026/// `--resolve-source` on `perf-delta`); the filename stays stable
3027/// across commits and run environments by design.
3028///
3029/// The corollary of the HostContext exclusion: if the host's
3030/// observable state mutates mid-suite — NUMA hotplug, hugepage
3031/// reconfiguration, a `sysctl -w` from a parallel process — two
3032/// runs of the same test will produce the same sidecar filename
3033/// and the later write clobbers the earlier. ktstr treats host
3034/// state as stable-enough for a single suite run; callers
3035/// mutating host state during a run own the ordering themselves
3036/// (e.g. by writing to a different `KTSTR_SIDECAR_DIR` per host
3037/// snapshot).
3038/// The single canonical-JSON + siphash site for the variant hash.
3039///
3040/// [`sidecar_variant_hash`] (from a written [`SidecarResult`]) and
3041/// [`variant_hash_from_parts`] (from a test entry + resolved topology +
3042/// work_type, before any sidecar exists) both route through this so the
3043/// two derivations can never drift. `sysctls`/`kargs` are sorted here
3044/// for order-independence.
3045fn variant_hash_of(
3046 topology: &str,
3047 scheduler: &str,
3048 payload: Option<&str>,
3049 work_type: &str,
3050 sysctls: &[String],
3051 kargs: &[String],
3052) -> u64 {
3053 use siphasher::sip::SipHasher13;
3054 use std::hash::Hasher;
3055 let mut sorted_sysctls = sysctls.to_vec();
3056 sorted_sysctls.sort();
3057 let mut sorted_kargs = kargs.to_vec();
3058 sorted_kargs.sort();
3059 let canonical = serde_json::json!({
3060 "topology": topology,
3061 "scheduler": scheduler,
3062 "payload": payload,
3063 "work_type": work_type,
3064 "sysctls": sorted_sysctls,
3065 "kargs": sorted_kargs,
3066 });
3067 let bytes = serde_json::to_vec(&canonical).expect("json serialization cannot fail for strings");
3068 let mut h = SipHasher13::new_with_keys(0, 0);
3069 h.write(&bytes);
3070 h.finish()
3071}
3072
3073pub(crate) fn sidecar_variant_hash(sidecar: &SidecarResult) -> u64 {
3074 variant_hash_of(
3075 &sidecar.topology,
3076 &sidecar.scheduler,
3077 sidecar.payload.as_deref(),
3078 &sidecar.work_type,
3079 &sidecar.sysctls,
3080 &sidecar.kargs,
3081 )
3082}
3083
3084/// The variant hash for a test entry's run at a given resolved topology
3085/// and `work_type`, computed BEFORE any sidecar exists — the
3086/// failure-dump path (and the Ctx/VmResult `variant_hash` stamp) need
3087/// the identity at VM-build time. Mirrors [`write_sidecar`]'s field
3088/// derivation (topology = the resolved topology, scheduler/sysctls/kargs
3089/// = [`scheduler_fingerprint`], payload = `entry.payload`) so the dump
3090/// filename carries the SAME variant hash the sidecar will. Pinned
3091/// equal to [`sidecar_variant_hash`] by a roundtrip test.
3092pub(crate) fn variant_hash_from_parts(
3093 entry: &KtstrTestEntry,
3094 resolved_topology: &crate::vmm::topology::Topology,
3095 work_type: &str,
3096) -> u64 {
3097 let fp = scheduler_fingerprint(entry);
3098 variant_hash_of(
3099 &resolved_topology.to_string(),
3100 &fp.scheduler,
3101 entry.payload.map(|p| p.name),
3102 work_type,
3103 &fp.sysctls,
3104 &fp.kargs,
3105 )
3106}
3107
3108/// Entry-derived scheduler metadata that every sidecar carries
3109/// regardless of pass/fail/skip.
3110///
3111/// Both write paths ([`write_sidecar`] and [`write_skip_sidecar`])
3112/// thread the same materialized fields through to their
3113/// `SidecarResult` constructors; keeping the derivation in a
3114/// named struct (rather than a 4-tuple) means a new
3115/// scheduler-level field shows up as a named field at both
3116/// writer sites and in every call-site binding, instead of as
3117/// an additional anonymous tuple slot that readers have to
3118/// remember the ordering of.
3119///
3120/// `pub(crate)` rather than `pub`: the intermediate struct is a
3121/// write-path detail, not a public API surface. No serde — this
3122/// is not a persisted shape, just a grouped return value.
3123///
3124/// Derives `Debug` for `assert_eq!` diagnostics, `Clone` so tests
3125/// can materialize a fixture once and reuse it across assertions,
3126/// and `PartialEq`/`Eq` so tests can compare whole fingerprints
3127/// in one statement rather than destructuring and asserting on
3128/// each field.
3129#[derive(Debug, Clone, PartialEq, Eq)]
3130pub(crate) struct SchedulerFingerprint {
3131 /// Pretty scheduler name (matches `SidecarResult::scheduler`),
3132 /// e.g. `"eevdf"` or a scheduler-kind payload's declared name.
3133 pub(crate) scheduler: String,
3134 /// Best-effort userspace scheduler commit; `None` for every
3135 /// current variant per
3136 /// [`crate::test_support::SchedulerSpec::scheduler_commit`].
3137 pub(crate) scheduler_commit: Option<String>,
3138 /// Formatted `sysctl.<key>=<value>` lines derived from the
3139 /// scheduler's declared `sysctls()`.
3140 pub(crate) sysctls: Vec<String>,
3141 /// Kernel command-line args declared by the scheduler,
3142 /// forwarded verbatim.
3143 pub(crate) kargs: Vec<String>,
3144}
3145
3146/// Materialize the [`SchedulerFingerprint`] for a test entry.
3147///
3148/// A change to the sidecar schema (e.g. a new scheduler-level
3149/// field) extends this function + [`SchedulerFingerprint`] in
3150/// one place and every writer picks it up automatically.
3151fn scheduler_fingerprint(entry: &KtstrTestEntry) -> SchedulerFingerprint {
3152 let scheduler = entry.scheduler.name.to_string();
3153 // `SchedulerSpec::scheduler_commit()` returns `None` for every
3154 // variant (Eevdf, Discover, Path, KernelBuiltin) — the commit
3155 // string is not carried in the static spec; it comes from the
3156 // sidecar's run-time git probe instead. This call is here only
3157 // to surface the slot in the fingerprint so a future spec
3158 // variant carrying a commit would flow through automatically.
3159 let scheduler_commit = entry
3160 .scheduler
3161 .binary
3162 .scheduler_commit()
3163 .map(|s| s.to_string());
3164 let sysctls: Vec<String> = entry
3165 .scheduler
3166 .sysctls
3167 .iter()
3168 .map(|s| format!("sysctl.{}={}", s.key(), s.value()))
3169 .collect();
3170 let kargs: Vec<String> = entry
3171 .scheduler
3172 .kargs
3173 .iter()
3174 .map(|s| s.to_string())
3175 .collect();
3176 SchedulerFingerprint {
3177 scheduler,
3178 scheduler_commit,
3179 sysctls,
3180 kargs,
3181 }
3182}
3183
3184/// Compute the per-variant sidecar path and serialize + write the
3185/// result to disk.
3186///
3187/// Gauntlet variants of the same test differ by work_type, flags
3188/// (via scheduler args → sysctls/kargs), scheduler, and topology. A
3189/// filename of just `{test_name}.ktstr.json` causes variants to
3190/// overwrite each other, erasing all but the last-written result.
3191/// `sidecar_variant_hash` hashes the discriminating fields into a
3192/// short stable suffix so each variant gets its own sidecar file.
3193///
3194/// On an orchestrated run-epoch mismatch,
3195/// [`acquire_run_dir_publication_lock`] removes any pre-existing
3196/// `*.ktstr.json` files before publishing the new sentinel so the
3197/// run is a clean snapshot rather than a mosaic of prior invocations.
3198/// Matching-epoch writes append concurrently.
3199///
3200/// Pre-clear is SKIPPED when `KTSTR_SIDECAR_DIR` is set: the
3201/// operator chose that directory and owns its contents — silent
3202/// data loss is not acceptable on an explicit override. When the
3203/// override is unset (the default-path branch),
3204/// `std::fs::create_dir_all` materializes the directory BEFORE
3205/// pre-clear runs so the helper's canonicalize step always sees
3206/// an existing on-disk path; without this ordering, a missing
3207/// dir on the very first call would key the cache against the
3208/// raw path while a later call (after the dir exists) would key
3209/// against the canonicalized absolute path, splitting the cache
3210/// and causing the second call to re-fire pre-clear and wipe the
3211/// first call's sidecars.
3212///
3213/// CROSS-PROCESS PUBLICATION: on the default path (override unset),
3214/// writers with a matching [`crate::KTSTR_RUN_EPOCH_ENV`] sentinel
3215/// acquire advisory `LOCK_SH` on
3216/// `{runs_root}/.locks/{key}.lock`, recheck the token under that
3217/// lock, and retain SH through their atomic rename. Matching writers
3218/// therefore publish concurrently. A first writer or epoch change
3219/// acquires `LOCK_EX`, rechecks, then performs wipe -> sentinel ->
3220/// first publication. Since EX conflicts with every matching
3221/// writer's SH, an epoch reset cannot cross the validated-token ->
3222/// rename interval. The override path skips the lock for the same
3223/// reason it skips pre-clear: operator-chosen directories are owned
3224/// by the operator, so we do not place a `.locks/` sibling inside
3225/// (or above) their custom layout.
3226///
3227/// A later peer whose sentinel already matches bypasses pre-clear
3228/// and takes only the shared publication rail. Without that sentinel
3229/// a later peer's pre-clear would delete an earlier peer's freshly
3230/// written sidecar — silent stats loss. Raw `cargo nextest run` sets
3231/// no token, so its peers retain the serialized wipe-everything
3232/// behavior.
3233///
3234/// PER-FILE ATOMICITY (both branches): the JSON is written to a
3235/// `<final>.tmp.<pid>.<run_id>` sibling and then `rename(2)`'d into
3236/// place. POSIX `rename` is atomic for same-directory destinations,
3237/// so a peer reader (`collect_sidecars`) never observes a partial
3238/// JSON payload — either the old contents stay or the new contents
3239/// replace them in one filesystem step. Two concurrent writers that
3240/// both target the same `{test_name}-{variant_hash}.ktstr.json`
3241/// (override path: two CI jobs sharing one operator-chosen dir;
3242/// default path: a torn-write window inside the flock body that the
3243/// flock would otherwise have to cover) cannot leave a half-written
3244/// JSON behind — last-rename-wins, both files are individually
3245/// well-formed. The `.tmp.<pid>.<run_id>` discriminator on the
3246/// staging name keeps two writers from racing on the same staging
3247/// path even when their final destinations collide. The flock on
3248/// the default path remains load-bearing for the pre-clear leg
3249/// (atomic write only protects the write itself, not the
3250/// `read_dir + remove_file` walk that pre-clear runs).
3251///
3252/// `label` is a caller-supplied noun for the context message ("skip
3253/// sidecar" / "sidecar") so the error chain points at the right call
3254/// site.
3255fn serialize_and_write_sidecar(sidecar: &SidecarResult, label: &str) -> anyhow::Result<()> {
3256 // Read the override ONCE. The two branches below carry the
3257 // result through structurally so neither leg re-reads
3258 // `KTSTR_SIDECAR_DIR` — preventing the override from flipping
3259 // mid-call (which would otherwise let an external mutation
3260 // between the dir resolve and the pre-clear gate either skip
3261 // the wipe on a default-path dir or fire a wipe on an
3262 // operator-chosen one).
3263 let (dir, do_pre_clear) = match sidecar_dir_override() {
3264 Some(path) => (path, false),
3265 None => (resolve_default_sidecar_dir(), true),
3266 };
3267 // Materialize the directory FIRST so `pre_clear_run_dir_once`
3268 // can canonicalize a path that exists on disk. Without this,
3269 // the very first invocation in a process resolves the cache
3270 // key against the raw relative path (canonicalize fails on a
3271 // missing dir, falls back to raw); subsequent invocations
3272 // resolve against the canonicalized absolute path because the
3273 // dir now exists. Two distinct keys for the same logical dir
3274 // → second invocation re-fires pre-clear and wipes the first
3275 // invocation's sidecars. Materializing pre-pre-clear closes
3276 // the relative-vs-absolute split.
3277 std::fs::create_dir_all(&dir)
3278 .with_context(|| format!("create sidecar dir {}", dir.display()))?;
3279 let variant_hash = sidecar_variant_hash(sidecar);
3280 let path = dir.join(format!(
3281 "{}-{:016x}.ktstr.json",
3282 sidecar.test_name, variant_hash
3283 ));
3284 let json = serde_json::to_string_pretty(sidecar)
3285 .with_context(|| format!("serialize {label} for '{}'", sidecar.test_name))?;
3286 // Atomic write: stage into a `.tmp.<pid>.<run_id>` sibling and
3287 // rename(2) into the final path. `rename` is atomic for
3288 // same-directory destinations on every filesystem ktstr supports
3289 // (ext4, btrfs, xfs, tmpfs, overlayfs); a peer reader never
3290 // observes a partial payload. The staging name carries the pid
3291 // AND the unique sidecar `run_id` so two writers in the same
3292 // process targeting identical final paths (e.g. two threads in
3293 // the budget-test stdout-capture path) cannot stomp each other's
3294 // staging file before either rename lands. On rename failure the
3295 // staging file is removed so a partial sidecar does not survive
3296 // as garbage in the run dir; rename success consumes the staging
3297 // entry and there is nothing to clean up.
3298 let pid = std::process::id();
3299 let staging = dir.join(format!(
3300 "{}-{:016x}.ktstr.json.tmp.{pid}.{}",
3301 sidecar.test_name, variant_hash, sidecar.run_id,
3302 ));
3303
3304 // Serialize before admission to keep both the shared publication
3305 // interval and the exclusive epoch-reset interval limited to filesystem
3306 // mutation. On the orchestrated path, a matching epoch takes LOCK_SH:
3307 // all peer writers publish concurrently, while an epoch reset's LOCK_EX
3308 // cannot cross the validated-token -> rename interval. A missing or
3309 // mismatched epoch takes LOCK_EX, rechecks under that lock, then performs
3310 // wipe -> sentinel before publishing. Raw no-token runs retain the
3311 // historical exclusive, pre-clear-once behavior. The override branch
3312 // skips both coordination and pre-clear.
3313 let session_token = if do_pre_clear {
3314 run_session_token()
3315 } else {
3316 None
3317 };
3318 let _publication_lock = if do_pre_clear {
3319 Some(acquire_run_dir_publication_lock(
3320 &dir,
3321 session_token.as_deref(),
3322 )?)
3323 } else {
3324 None
3325 };
3326 if let Err(error) = std::fs::write(&staging, &json) {
3327 // `write` may have created or partially filled the staging inode
3328 // before reporting an error. Never leave that residue for the rest
3329 // of the epoch; the next reset is not guaranteed to happen soon.
3330 let _ = std::fs::remove_file(&staging);
3331 return Err(anyhow::Error::from(error)
3332 .context(format!("write {label} staging {}", staging.display())));
3333 }
3334 if let Err(e) = std::fs::rename(&staging, &path) {
3335 // Best-effort cleanup of the staged payload; ignore the
3336 // unlink error so the rename failure is what surfaces
3337 // (the rename error names the actual problem).
3338 let _ = std::fs::remove_file(&staging);
3339 return Err(anyhow::Error::from(e).context(format!(
3340 "rename {label} staging {} -> {}",
3341 staging.display(),
3342 path.display(),
3343 )));
3344 }
3345 LAST_SIDECAR_PATH.with(|p| *p.borrow_mut() = Some(path.clone()));
3346 Ok(())
3347}
3348
3349thread_local! {
3350 /// Absolute path of the most recent sidecar this thread wrote (via
3351 /// [`serialize_and_write_sidecar`]). The dispatch run loop
3352 /// ([`crate::test_support::eval::run_ktstr_test_inner`]) reads and
3353 /// clears it after the run to finalize the persisted verdict to the
3354 /// test's FINAL (post-inversion) outcome. nextest is process-per-test
3355 /// so a run writes one sidecar; a value left from an earlier phase is
3356 /// overwritten by the current write, so the take always yields this
3357 /// run's sidecar.
3358 static LAST_SIDECAR_PATH: std::cell::RefCell<Option<PathBuf>> =
3359 const { std::cell::RefCell::new(None) };
3360}
3361
3362/// Take (read + clear) the path of the sidecar most recently written on
3363/// this thread, or `None` when no sidecar was written this run (an
3364/// early bail before any write). See [`LAST_SIDECAR_PATH`].
3365///
3366/// MUST be drained exactly once per run — `run_ktstr_test_inner` does
3367/// this after each dispatch. The thread-local persists across calls in
3368/// a process, so a caller that writes a sidecar WITHOUT a following take
3369/// would leave a stale path for the next take to consume; in practice
3370/// only `run_ktstr_test_inner` pairs a write with a take, and a stale
3371/// path points at a dropped tempdir so the finalize read fails benignly.
3372pub(crate) fn take_last_sidecar_path() -> Option<PathBuf> {
3373 LAST_SIDECAR_PATH.with(|p| p.borrow_mut().take())
3374}
3375
3376/// Overwrite a written sidecar's verdict bits with the test's FINAL
3377/// (post-inversion) `(passed, skipped, inconclusive)` outcome — see
3378/// [`crate::test_support::dispatch::Verdict::sidecar_bits`] — and set
3379/// [`SidecarResult::expected_failure`] when an actual scenario
3380/// failure/inconclusive was inverted to a pass/skip. Rewrites the file
3381/// atomically (temp + rename) under the same publication rail as the
3382/// primary write. Matching orchestrated epochs take SH; a stale token
3383/// returns without touching the path, raw no-token runs take EX, and
3384/// explicit sidecar-directory overrides remain unlocked.
3385///
3386/// A no-op when the final verdict already matches what was persisted (an
3387/// ordinary pass/fail/skip — no `expect_err`/`expect_auto_repro`
3388/// inversion). Best-effort: a read/parse/serialize/write error is
3389/// surfaced on stderr and swallowed so the raw sidecar stands (the
3390/// footer then falls back to it) rather than failing the run.
3391pub(crate) fn finalize_sidecar_verdict(
3392 path: &std::path::Path,
3393 passed: bool,
3394 skipped: bool,
3395 inconclusive: bool,
3396) {
3397 let coordinated = sidecar_dir_override().is_none();
3398 let session_token = if coordinated {
3399 run_session_token()
3400 } else {
3401 None
3402 };
3403 finalize_sidecar_verdict_inner(
3404 path,
3405 passed,
3406 skipped,
3407 inconclusive,
3408 coordinated,
3409 session_token.as_deref(),
3410 || {},
3411 );
3412}
3413
3414fn finalize_sidecar_verdict_inner<F>(
3415 path: &std::path::Path,
3416 passed: bool,
3417 skipped: bool,
3418 inconclusive: bool,
3419 coordinated: bool,
3420 session_token: Option<&str>,
3421 before_rename: F,
3422) where
3423 F: FnOnce(),
3424{
3425 // Finalization is a second publication of the same sidecar. Keep it on
3426 // the same epoch rail as the primary write: an orchestrated finalizer
3427 // takes SH and rechecks its exact token, a raw no-token finalizer takes
3428 // EX, and an explicit override remains unlocked. A token mismatch means
3429 // another invocation reset the directory after this run's primary write;
3430 // returning before even reading the old path prevents a stale finalizer
3431 // from recreating a file on the far side of that reset.
3432 let _publication_lock = if coordinated {
3433 let Some(dir) = path.parent() else {
3434 eprintln!(
3435 "ktstr: finalize_sidecar_verdict: sidecar has no parent {}",
3436 path.display()
3437 );
3438 return;
3439 };
3440 match acquire_run_dir_finalize_lock(dir, session_token) {
3441 Ok(Some(lock)) => Some(lock),
3442 Ok(None) => return,
3443 Err(error) => {
3444 eprintln!(
3445 "ktstr: finalize_sidecar_verdict: admission for {} failed: {error:#}",
3446 path.display()
3447 );
3448 return;
3449 }
3450 }
3451 } else {
3452 None
3453 };
3454
3455 let Ok(json) = std::fs::read_to_string(path) else {
3456 return;
3457 };
3458 let Ok(mut sc) = serde_json::from_str::<SidecarResult>(&json) else {
3459 eprintln!(
3460 "ktstr: finalize_sidecar_verdict: unparseable sidecar {}",
3461 path.display()
3462 );
3463 return;
3464 };
3465 // The run's telemetry is failure-mode-dominated when its scenario
3466 // actually failed/was-inconclusive but the final verdict is a
3467 // pass/skip (an inversion) — `perf-delta` excludes such rows.
3468 let raw_failed = sc.is_fail() || sc.is_inconclusive();
3469 let expected_failure = raw_failed && (passed || skipped);
3470 if sc.passed == passed
3471 && sc.skipped == skipped
3472 && sc.inconclusive == inconclusive
3473 && sc.expected_failure == expected_failure
3474 {
3475 return;
3476 }
3477 sc.passed = passed;
3478 sc.skipped = skipped;
3479 sc.inconclusive = inconclusive;
3480 sc.expected_failure = expected_failure;
3481 let Ok(out) = serde_json::to_string_pretty(&sc) else {
3482 return;
3483 };
3484 // Stage with a `.ktstr.json.tmp.…` suffix (append, NOT
3485 // `with_extension`, which would drop `.json`) so a hard-crash orphan
3486 // — write succeeded but rename did not — is reaped by
3487 // the epoch-reset sweep via `is_sidecar_staging_filename`, the
3488 // same way the primary write's staging file is.
3489 let pid = std::process::id();
3490 let mut staging = path.as_os_str().to_owned();
3491 staging.push(format!(".tmp.finalize.{pid}"));
3492 let staging = std::path::PathBuf::from(staging);
3493 if std::fs::write(&staging, &out).is_err() {
3494 let _ = std::fs::remove_file(&staging);
3495 return;
3496 }
3497 before_rename();
3498 if std::fs::rename(&staging, path).is_err() {
3499 let _ = std::fs::remove_file(&staging);
3500 }
3501}
3502
3503/// Remove every failure-dump artifact for this exact test variant,
3504/// including nextest retry archives.
3505///
3506/// Called when a run's FINAL outcome is a pass/skip but it wrote NO
3507/// sidecar — the run crashed before the guest produced a parseable
3508/// result (e.g. an `expect_err` test with a host-triggered BPF crash),
3509/// so [`finalize_sidecar_verdict`] had nothing to finalize. The freeze
3510/// coordinator wrote the dump unconditionally; without a sidecar to mark
3511/// the pass, the footer's dump-only trigger
3512/// ([`summarize_one_run_dir`] flags a dump with no parsed sidecar) would
3513/// surface this PASSING test as FAILED. Removing the dump keeps the
3514/// footer consistent with nextest's pass. Best-effort: a missing dump
3515/// (the normal clean-pass case) is fine. A genuine pre-sidecar failure
3516/// (final = Fail) does NOT call this, so its dump still flags.
3517pub(crate) fn suppress_failure_dumps(test_name: &str, variant_hash: u64) {
3518 let dir = sidecar_dir();
3519 let Ok(entries) = std::fs::read_dir(&dir) else {
3520 return;
3521 };
3522 for entry in entries.flatten() {
3523 let path = entry.path();
3524 let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3525 continue;
3526 };
3527 let Some((candidate, hash, kind)) = classify_run_artifact(name) else {
3528 continue;
3529 };
3530 if candidate == test_name
3531 && hash == variant_hash
3532 && matches!(
3533 kind,
3534 RunArtifactKind::FailureDump | RunArtifactKind::ReproFailureDump
3535 )
3536 {
3537 let _ = std::fs::remove_file(path);
3538 }
3539 }
3540}
3541
3542/// How many periodic-snapshot files a FAILING attempt keeps per dump
3543/// base (primary and `.repro.` are bounded independently).
3544///
3545/// The freeze coordinator writes one `{base}.snapshot.periodic_NNN.json`
3546/// per fired boundary (up to [`crate::scenario::snapshot::MAX_STORED_SNAPSHOTS`] = 64),
3547/// each multi-megabyte. NOTHING on disk reads them back: `eval` and the
3548/// cast-analysis pipeline both consume the in-process
3549/// [`crate::scenario::snapshot::SnapshotBridge`] (which FIFO-caps at 64
3550/// on its own), never these files — they exist solely as CI forensic
3551/// artifacts for human inspection. The documented floor for *meaningful*
3552/// periodic coverage is `num_snapshots >= 2` (see
3553/// `crate::assert::temporal` / `crate::stats::metric_id`); keeping the
3554/// most-recent 8 leaves a 4x margin of trajectory approaching the
3555/// failure while bounding a 64-deep pile to 8 files. The most recent
3556/// samples (highest `periodic_NNN` index) are retained because they
3557/// bracket the failure; the periodic tag is zero-padded 3-digit, so
3558/// lexical index order is chronological.
3559pub(crate) const RETAINED_PERIODIC_SNAPSHOTS: usize = 8;
3560
3561/// Filename prefix shared by every on-disk snapshot for one test
3562/// variant: `{test_name}-{variant_hash:016x}`. The freeze coordinator
3563/// derives snapshot names from the failure-dump base by stripping
3564/// `.failure-dump` (see `vmm::freeze_coord::snapshot::snapshot_tagged_path`),
3565/// so the primary base yields `{prefix}.snapshot.{tag}.json` and the
3566/// repro base yields `{prefix}.repro.snapshot.{tag}.json`. The trailing
3567/// `-{hash:016x}` makes the prefix an exact variant boundary — `foo`'s
3568/// prefix cannot match `foobar`'s files, and a sibling gauntlet preset
3569/// (different hash) is untouched.
3570fn snapshot_base_prefix(test_name: &str, variant_hash: u64) -> String {
3571 format!("{test_name}-{variant_hash:016x}")
3572}
3573
3574/// `true` when `name` is any snapshot file (`.snapshot.{tag}.json`,
3575/// primary or `.repro.` base) for the variant identified by `prefix`.
3576fn is_snapshot_file_for(name: &str, prefix: &str) -> bool {
3577 let Some(rest) = name.strip_prefix(prefix) else {
3578 return false;
3579 };
3580 let rest = rest.strip_prefix(".repro").unwrap_or(rest);
3581 rest.starts_with(".snapshot.") && rest.ends_with(".json")
3582}
3583
3584/// `Some(index)` when `name` is a PERIODIC snapshot file for `prefix`,
3585/// returning the zero-padded 3-digit `periodic_NNN` index for recency
3586/// ordering. `None` for a non-matching file or a non-periodic snapshot
3587/// tag (e.g. `.snapshot.mid_run.json`, `.snapshot.early-degraded.json`),
3588/// which are single-shot and never bounded. The 3-digit / all-ASCII-digit
3589/// gate matches the coordinator's `format!("periodic_{:03}", idx)`
3590/// emission exactly, so a user-chosen `Op::CaptureSnapshot { name:
3591/// "periodic_kaslr" }` tag cannot be mistaken for a periodic sample.
3592fn periodic_snapshot_index_for(name: &str, prefix: &str) -> Option<u32> {
3593 let rest = name.strip_prefix(prefix)?;
3594 let rest = rest.strip_prefix(".repro").unwrap_or(rest);
3595 let tag = rest.strip_prefix(".snapshot.")?.strip_suffix(".json")?;
3596 let digits = tag.strip_prefix("periodic_")?;
3597 if digits.len() == 3 && digits.bytes().all(|b| b.is_ascii_digit()) {
3598 digits.parse().ok()
3599 } else {
3600 None
3601 }
3602}
3603
3604/// Remove EVERY on-disk snapshot file for this exact test variant
3605/// (primary + `.repro.` bases, periodic and one-shot tags alike).
3606///
3607/// Two callers, both best-effort (a missing dir or file is fine, and
3608/// nothing here ever fails a test):
3609/// - **Pass/skip finalize** ([`crate::test_support::eval`]): a run whose
3610/// FINAL verdict is pass/skip needs no periodic forensics, so the whole
3611/// pile is dropped before the process exits (and thus before CI uploads
3612/// artifacts). This is the dominant reclaim — most e2e tests pass, and
3613/// each was writing dozens of multi-MB snapshots that nothing consumed.
3614/// - **Attempt-prepare reap**: called before boot so a new attempt starts
3615/// from a clean snapshot slate. A prior attempt that fired MORE
3616/// boundaries — or was SIGKILL'd by the watchdog mid-run — leaves stale
3617/// higher-index snapshots the coordinator's tag-keyed overwrite would
3618/// not reclaim; this reaps them. It is the crash-safety mechanism: a
3619/// killed process cannot clean up after itself, so the NEXT attempt's
3620/// prepare does it (mirrors [`super::eval`]'s `prepare_failure_dump_path`).
3621pub(crate) fn cleanup_snapshots(test_name: &str, variant_hash: u64) {
3622 let dir = sidecar_dir();
3623 let prefix = snapshot_base_prefix(test_name, variant_hash);
3624 let Ok(entries) = std::fs::read_dir(&dir) else {
3625 return;
3626 };
3627 for entry in entries.flatten() {
3628 let path = entry.path();
3629 let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3630 continue;
3631 };
3632 if is_snapshot_file_for(name, &prefix) {
3633 let _ = std::fs::remove_file(&path);
3634 }
3635 }
3636}
3637
3638/// Bound the periodic snapshots for a FAILING attempt to the most recent
3639/// `keep` per dump base, deleting the older ones. Primary and `.repro.`
3640/// bases are bounded independently (each keeps its own `keep`). One-shot
3641/// snapshot tags (`mid_run`, degraded) are never touched — they are
3642/// single files, not an unbounded periodic series. Best-effort; never
3643/// fails a test.
3644///
3645/// The retained failure DUMP is the primary evidence and is untouched
3646/// here; these periodic samples are supplementary trajectory, so the
3647/// most-recent `keep` (bracketing the failure) suffice. See
3648/// [`RETAINED_PERIODIC_SNAPSHOTS`] for the floor rationale.
3649pub(crate) fn bound_periodic_snapshots(test_name: &str, variant_hash: u64, keep: usize) {
3650 let dir = sidecar_dir();
3651 let prefix = snapshot_base_prefix(test_name, variant_hash);
3652 let Ok(entries) = std::fs::read_dir(&dir) else {
3653 return;
3654 };
3655 // Bucket by base so the repro series does not evict the primary's.
3656 let mut primary: Vec<(u32, PathBuf)> = Vec::new();
3657 let mut repro: Vec<(u32, PathBuf)> = Vec::new();
3658 for entry in entries.flatten() {
3659 let path = entry.path();
3660 let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
3661 continue;
3662 };
3663 let Some(idx) = periodic_snapshot_index_for(name, &prefix) else {
3664 continue;
3665 };
3666 let is_repro = name
3667 .strip_prefix(prefix.as_str())
3668 .is_some_and(|rest| rest.starts_with(".repro."));
3669 if is_repro {
3670 repro.push((idx, path));
3671 } else {
3672 primary.push((idx, path));
3673 }
3674 }
3675 for mut set in [primary, repro] {
3676 if set.len() <= keep {
3677 continue;
3678 }
3679 // Ascending by index == chronological; drop the oldest overflow,
3680 // keep the newest `keep`.
3681 set.sort_by_key(|(idx, _)| *idx);
3682 let drop_count = set.len() - keep;
3683 for (_, path) in set.into_iter().take(drop_count) {
3684 let _ = std::fs::remove_file(&path);
3685 }
3686 }
3687}
3688
3689/// `Some(path)` when `KTSTR_SIDECAR_DIR` is set non-empty,
3690/// returning the override path verbatim; `None` when the env
3691/// var is unset or empty (default-path branch). Single source
3692/// of truth for the override read so [`sidecar_dir`] and
3693/// [`serialize_and_write_sidecar`] (which gates pre-clear on
3694/// the override's presence) share one env-read site rather
3695/// than each calling `std::env::var` independently.
3696///
3697/// The `is_empty()` filter is deliberate: a defensively-cleared
3698/// `KTSTR_SIDECAR_DIR=""` must NOT be treated as an override
3699/// (joining an empty path onto the run-root would silently
3700/// alias the runs-root itself, contaminating the listing).
3701/// Empty-string aliases unset, matching the
3702/// `if let Ok(d) ... && !d.is_empty()` predicate the function
3703/// replaced.
3704///
3705/// `serialize_and_write_sidecar` interprets `Some(_)` as the
3706/// "operator chose this dir, do not pre-clear" gate — silent
3707/// data loss is unacceptable on an explicit override (the
3708/// override is for users who want exact control over where
3709/// sidecars land: test isolation, archival capture, custom CI
3710/// layouts).
3711fn sidecar_dir_override() -> Option<PathBuf> {
3712 std::env::var(crate::KTSTR_SIDECAR_DIR_ENV)
3713 .ok()
3714 .filter(|d| !d.is_empty())
3715 .map(PathBuf::from)
3716}
3717
3718/// Emit a one-shot stderr warning when [`detect_project_commit`]
3719/// resolves to `None` and the run directory therefore lands at
3720/// `{kernel}-unknown`. Operators in this state lose the
3721/// `{project_commit}` discriminator on the run-directory name —
3722/// every non-git invocation at the same kernel collides on a
3723/// single directory, with the latest run pre-clearing the
3724/// previous one's sidecars. The warning surfaces this loss-of-isolation
3725/// risk so the operator can either set `KTSTR_SIDECAR_DIR` to
3726/// disambiguate per-run, or place the project tree under git
3727/// so each run carries its own commit hash.
3728///
3729/// `OnceLock<()>` gates the warning to fire EXACTLY ONCE per
3730/// process: every gauntlet variant resolves a sidecar directory
3731/// independently (via [`sidecar_dir`] and
3732/// [`serialize_and_write_sidecar`]), so without the gate the
3733/// operator would see thousands of duplicate warnings interleaved
3734/// with test output. Called via [`resolve_default_sidecar_dir`] —
3735/// which is the shared default-path body that both [`sidecar_dir`]
3736/// and [`serialize_and_write_sidecar`] funnel through — so the
3737/// warning fires only on the default-path branch. The override
3738/// branch in either caller returns before
3739/// [`resolve_default_sidecar_dir`] is reached, so an operator who
3740/// set `KTSTR_SIDECAR_DIR` to disambiguate non-git runs does not
3741/// see a misleading "commit unknown" warning that does not apply
3742/// to their effective directory layout.
3743///
3744/// Implementation is split into a public-facing wrapper
3745/// (this function) that owns the process-global `OnceLock` and
3746/// targets stderr, and a pure inner helper
3747/// [`warn_unknown_project_commit_inner`] that takes the
3748/// `&OnceLock<()>` gate and the `&mut dyn Write` sink as
3749/// parameters. The split lets tests drive the warning logic
3750/// against a local `OnceLock` and a `Vec<u8>` sink without
3751/// fighting the process-global gate or the global stderr fd —
3752/// the wrapper's behavior is what the inner does, just with
3753/// the static gate and stderr supplied.
3754fn warn_unknown_project_commit_once() {
3755 static WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
3756 let mut sink = std::io::stderr();
3757 warn_unknown_project_commit_inner(&WARNED, &mut sink);
3758}
3759
3760/// Pure helper for [`warn_unknown_project_commit_once`]: gate the
3761/// warning on `gate` and write the warning text to `sink` exactly
3762/// once across the gate's lifetime. Both parameters are taken by
3763/// reference so call sites supply ownership semantics that match
3764/// their gating story:
3765/// - The production wrapper passes a `'static` `OnceLock<()>` so
3766/// the gate spans the whole process and a stderr handle so the
3767/// warning lands in the operator's terminal.
3768/// - Tests pass a local `OnceLock<()>` so each test gets a fresh
3769/// gate (no cross-test contamination via a process-global)
3770/// and a `Vec<u8>` sink so the test can read back the emitted
3771/// bytes and assert on the warning text.
3772///
3773/// Errors from `writeln!` are ignored via `let _ =`: a metadata
3774/// probe warning must not gate sidecar writes. This DEPARTS from
3775/// the previous `eprintln!` semantics (which panic on stderr
3776/// write failure per the std docs) — here we drop the write
3777/// error silently because a metadata probe warning must not gate
3778/// sidecar writes.
3779fn warn_unknown_project_commit_inner(
3780 gate: &std::sync::OnceLock<()>,
3781 sink: &mut dyn std::io::Write,
3782) {
3783 gate.get_or_init(|| {
3784 let _ = writeln!(
3785 sink,
3786 "ktstr: WARNING: project commit unavailable (cwd not in a git \
3787 repo, or HEAD unreadable); runs at this kernel overwrite \
3788 each other in target/ktstr/{{kernel}}-unknown/. Set \
3789 KTSTR_SIDECAR_DIR=<unique-path> per run, or run from inside a \
3790 git repo with at least one commit."
3791 );
3792 });
3793}
3794
3795/// Direct-test wrapper for the process-local pre-clear behavior.
3796///
3797/// Production orchestrated writes initialize/reset epochs through
3798/// [`acquire_run_dir_publication_lock`]. Raw no-token writes use the
3799/// token-stable inner helper below while holding EX.
3800#[cfg(test)]
3801fn pre_clear_run_dir_once(dir: &std::path::Path) {
3802 let session_token = run_session_token();
3803 pre_clear_run_dir_once_for_session(dir, session_token.as_deref());
3804}
3805
3806/// Token-stable raw/pre-clear implementation.
3807///
3808/// The serializer captures the epoch once so an unrelated environment
3809/// mutation cannot change identity between admission and sentinel publication.
3810/// Direct tests use the wrapper above, which snapshots the environment at
3811/// entry.
3812fn pre_clear_run_dir_once_for_session(dir: &std::path::Path, session_token: Option<&str>) {
3813 use std::collections::HashSet;
3814 use std::path::PathBuf;
3815 use std::sync::{Mutex, OnceLock};
3816 static PRE_CLEARED: OnceLock<Mutex<HashSet<PathBuf>>> = OnceLock::new();
3817 // Canonicalize so two spellings of the same on-disk dir share
3818 // one cache entry. Falls back to the raw path when canonicalize
3819 // fails (the directory may not exist yet on the very first
3820 // write, in which case the raw path keys the entry; subsequent
3821 // calls with the same raw path also miss canonicalize the
3822 // same way and share the entry).
3823 let cache_key = dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf());
3824 let cache = PRE_CLEARED.get_or_init(|| Mutex::new(HashSet::new()));
3825 let mut guard = cache.lock_unpoisoned();
3826 if guard.contains(&cache_key) {
3827 return;
3828 }
3829 // First time this directory has been seen — wipe sidecars while
3830 // the cache mutex is still held. Releasing the guard before the
3831 // read_dir walk would open a TOCTOU window: a sibling thread that
3832 // observes the now-cached entry would skip its own pre-clear,
3833 // proceed to write a sidecar, and the original thread's walk
3834 // (running after the drop) would then delete that sibling's
3835 // freshly-written file. The walk is one read_dir + a bounded
3836 // number of `*.ktstr.json` removals, so holding the lock across
3837 // it is brief; concurrent calls against DIFFERENT directories
3838 // serialize through this critical section but each does a small,
3839 // bounded amount of I/O, which is acceptable for a metadata
3840 // probe call pattern. The cache insert happens AFTER the wipe
3841 // completes (rather than before) so a panic mid-wipe does not
3842 // poison the cache with an entry whose wipe never actually ran.
3843 // The mutex itself enforces serialization across threads; the
3844 // entry only records "wipe completed for this dir" and must
3845 // never be observed without the wipe having succeeded. `guard`
3846 // is dropped at end-of-scope so the lock release happens after
3847 // the loop completes.
3848 if let Some(token) = session_token
3849 && session_sentinel_matches(dir, token)
3850 {
3851 // A peer test process in THIS session already cleared the dir
3852 // (the sentinel records the session token under the flock);
3853 // its and the other peers' current-session sidecars must
3854 // survive, so skip the wipe entirely. See CONCURRENT WRITERS.
3855 guard.insert(cache_key);
3856 return;
3857 }
3858 // Raw no-token pre-clear is deliberately best-effort. Token-bearing
3859 // orchestrated initialization does not use this once-gated wrapper; its
3860 // publication caller propagates reset and sentinel failures.
3861 let _ = reset_run_dir_for_session(dir, session_token);
3862 // Record the raw/helper attempt only after it returns. Its historical
3863 // best-effort contract deliberately treats filesystem errors as a
3864 // completed attempt; the orchestrated token-bearing path bypasses this
3865 // process-local cache and propagates those errors instead.
3866 guard.insert(cache_key);
3867 drop(guard);
3868}
3869
3870/// Wipe one run directory and, when present, publish its exact epoch token.
3871///
3872/// Production callers hold the run-dir exclusive flock. Unlike
3873/// the process-local pre-clear helper, this deliberately has no
3874/// process-local once gate: an observed token mismatch is authoritative even
3875/// when this process wrote to the same directory earlier.
3876fn reset_run_dir_for_session(
3877 dir: &std::path::Path,
3878 session_token: Option<&str>,
3879) -> anyhow::Result<()> {
3880 reset_run_dir_for_session_with_remover(dir, session_token, |path| std::fs::remove_file(path))
3881}
3882
3883/// Testable implementation of [`reset_run_dir_for_session`].
3884///
3885/// A token-bearing caller uses the sentinel as an authoritative statement
3886/// that the old epoch was completely removed. Every enumeration, metadata,
3887/// and unlink failure must therefore abort before sentinel publication. Raw
3888/// no-token callers preserve their historical best-effort behavior by
3889/// explicitly discarding this result in [`pre_clear_run_dir_once_for_session`].
3890fn reset_run_dir_for_session_with_remover<F>(
3891 dir: &std::path::Path,
3892 session_token: Option<&str>,
3893 mut remove_file: F,
3894) -> anyhow::Result<()>
3895where
3896 F: FnMut(&std::path::Path) -> std::io::Result<()>,
3897{
3898 let Some(token) = session_token else {
3899 // Preserve the raw `cargo nextest run` contract exactly: unreadable
3900 // directories/entries, metadata failures, and individual unlink
3901 // failures are ignored while the sweep continues wherever possible.
3902 if let Ok(entries) = std::fs::read_dir(dir) {
3903 for entry in entries.flatten() {
3904 let path = entry.path();
3905 if path.is_file()
3906 && (is_sidecar_filename(&path) || is_sidecar_staging_filename(&path))
3907 {
3908 let _ = remove_file(&path);
3909 }
3910 }
3911 }
3912 return Ok(());
3913 };
3914
3915 let entries = std::fs::read_dir(dir)
3916 .with_context(|| format!("enumerate run directory for epoch reset {}", dir.display()))?;
3917 for entry in entries {
3918 let entry =
3919 entry.with_context(|| format!("enumerate run directory entry {}", dir.display()))?;
3920 let path = entry.path();
3921 let metadata = std::fs::metadata(&path)
3922 .with_context(|| format!("inspect run directory entry {}", path.display()))?;
3923 if !metadata.is_file() {
3924 continue;
3925 }
3926 // Two file shapes are reaped here (current-session peers
3927 // were already spared by the sentinel skip above, so a
3928 // file reaching this point is prior-session or orphaned
3929 // residue):
3930 // - `<test>-<hash>.ktstr.json` — sidecars from a PRIOR
3931 // session sharing this `{kernel}-{project_commit}` key.
3932 // - `<test>-<hash>.ktstr.json.tmp.<pid>.<run_id>` —
3933 // orphaned staging from a writer that died between
3934 // `write` and `rename` in `serialize_and_write_sidecar`
3935 // (`is_sidecar_filename` excludes these — the extension
3936 // is `<run_id>`, not `json` — so the staging sweep is
3937 // what reaps them). The flock makes reaping an in-flight
3938 // stage impossible: a live peer holds the lock we hold.
3939 if is_sidecar_filename(&path) || is_sidecar_staging_filename(&path) {
3940 remove_file(&path)
3941 .with_context(|| format!("remove prior-epoch artifact {}", path.display()))?;
3942 }
3943 }
3944 // Record this session's token so peer processes skip re-wiping.
3945 // Token-bearing orchestrated initialization treats this write as
3946 // authoritative: publishing a sidecar without its epoch would collapse
3947 // the herd back into repeated EX resets. Written AFTER the wipe so a
3948 // crash mid-wipe leaves no stale sentinel falsely claiming completion.
3949 let sentinel = dir.join(SESSION_SENTINEL);
3950 std::fs::write(&sentinel, token)
3951 .with_context(|| format!("publish run epoch sentinel {}", sentinel.display()))?;
3952 Ok(())
3953}
3954
3955/// Whether the on-disk sentinel records `token` byte-for-byte.
3956fn session_sentinel_matches(dir: &std::path::Path, token: &str) -> bool {
3957 std::fs::read_to_string(dir.join(SESSION_SENTINEL)).is_ok_and(|recorded| recorded == token)
3958}
3959
3960/// Filename of the per-run-directory session sentinel that records
3961/// the [`crate::KTSTR_RUN_EPOCH_ENV`] token of the session that last
3962/// cleared the dir. A dotfile so every sidecar reader ignores it
3963/// (`is_sidecar_filename` requires a `.json` extension and
3964/// `classify_run_artifact` matches none of its suffixes), and it
3965/// lives in the run dir itself (which the caller already
3966/// `create_dir_all`'d) rather than the `.locks/` sibling.
3967const SESSION_SENTINEL: &str = ".ktstr_run_epoch";
3968
3969/// Read the `cargo ktstr test` session token from
3970/// [`crate::KTSTR_RUN_EPOCH_ENV`] — an opaque per-invocation value
3971/// the orchestrator stamps once before nextest spawns, inherited by
3972/// every child test process.
3973///
3974/// `None` when the variable is unset or empty (raw `cargo nextest
3975/// run` — no orchestrator); the publication helper then serializes
3976/// the process-local pre-clear behavior.
3977/// `Some` lets pre-clear record/match the `.ktstr_run_epoch`
3978/// sentinel so a later peer process skips re-wiping a dir this
3979/// session already cleared, sparing the peers' sidecars.
3980fn run_session_token() -> Option<String> {
3981 std::env::var(crate::KTSTR_RUN_EPOCH_ENV)
3982 .ok()
3983 .filter(|v| !v.is_empty())
3984}
3985
3986/// Predicate: is `path` an atomic-write staging file produced by
3987/// [`serialize_and_write_sidecar`]?
3988///
3989/// True iff the filename matches the `<test>-<hash>.ktstr.json.tmp.…`
3990/// shape — `is_sidecar_filename` rejects these because the
3991/// extension is `<run_id>` rather than `json`, so a separate
3992/// predicate is needed for the epoch-reset sweep
3993/// that reaps orphaned staging files. Filename-component check
3994/// (rather than full-path string) for the same load-bearing reason
3995/// `is_sidecar_filename` uses `Path::file_name()`: a `.ktstr.json.tmp.`
3996/// substring inside an ancestor segment must not match.
3997fn is_sidecar_staging_filename(path: &std::path::Path) -> bool {
3998 path.file_name()
3999 .and_then(|n| n.to_str())
4000 .is_some_and(|n| n.contains(".ktstr.json.tmp."))
4001}
4002
4003/// Wall-clock timeout for [`acquire_run_dir_publication_lock`].
4004/// Matching writers normally take SH immediately; EX is held only
4005/// for an epoch reset and its first atomic publication. A holder
4006/// that does not release within 30 s has stalled, and surfacing that
4007/// as an actionable error beats hanging the test run indefinitely.
4008/// The timeout is asymmetric with the cache-store 300 s timeout
4009/// because this rail protects only bounded run-directory metadata I/O.
4010const RUN_DIR_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
4011
4012/// Compute the per-run-key flock sentinel path for `dir`.
4013///
4014/// Layout: `{dir.parent()}/.locks/{dir.file_name()}.lock`. When
4015/// `dir = {runs_root}/{key}` (the production default-path shape),
4016/// this resolves to `{runs_root}/.locks/{key}.lock`. Sourced from
4017/// [`crate::flock::LOCK_DIR_NAME`] so a relocation of the lock
4018/// subdirectory updates one place across both this surface and
4019/// the cache module.
4020///
4021/// Returns `None` when `dir` has no parent (root) or no
4022/// `file_name` component (current dir, root) — neither case is
4023/// reachable on the production default path
4024/// ([`runs_root`] always returns a non-root multi-component
4025/// path), but the function is total over the input domain so a
4026/// future caller passing an unusual path surfaces a clean `None`
4027/// rather than panicking on `unwrap`.
4028///
4029/// Pure function over the input path — no I/O. The caller is
4030/// responsible for materializing the parent `.locks/`
4031/// subdirectory before opening the lockfile —
4032/// [`crate::flock::acquire_flock_with_timeout`] handles that
4033/// lazily.
4034fn run_dir_lock_path(dir: &std::path::Path) -> Option<PathBuf> {
4035 let parent = dir.parent()?;
4036 let leaf = dir.file_name()?;
4037 let mut filename = std::ffi::OsString::from(leaf);
4038 filename.push(".lock");
4039 Some(parent.join(crate::flock::LOCK_DIR_NAME).join(filename))
4040}
4041
4042/// Acquire publication ownership for a verdict rewrite.
4043///
4044/// Unlike a primary write, finalization must never initialize or reset an
4045/// epoch: its sidecar belongs to the epoch that published it. A matching token
4046/// retains SH through rename; a mismatch returns `Ok(None)` without touching
4047/// the file. Raw no-token finalization retains the historical EX rail.
4048fn acquire_run_dir_finalize_lock(
4049 dir: &std::path::Path,
4050 session_token: Option<&str>,
4051) -> anyhow::Result<Option<std::os::fd::OwnedFd>> {
4052 let Some(token) = session_token else {
4053 return acquire_run_dir_flock_with_timeout(dir, RUN_DIR_LOCK_TIMEOUT).map(Some);
4054 };
4055 let shared = acquire_run_dir_flock_mode_with_timeout(
4056 dir,
4057 crate::flock::FlockMode::Shared,
4058 RUN_DIR_LOCK_TIMEOUT,
4059 )?;
4060 if session_sentinel_matches(dir, token) {
4061 Ok(Some(shared))
4062 } else {
4063 drop(shared);
4064 Ok(None)
4065 }
4066}
4067
4068/// Acquire the run-directory publication rail.
4069///
4070/// A matching orchestrated epoch takes `LOCK_SH`, validates the sentinel while
4071/// holding it, and returns the fd for the caller to retain through atomic
4072/// rename. Peer writers for that epoch therefore publish concurrently. An
4073/// epoch reset requires `LOCK_EX`, so it cannot cross the token-validation to
4074/// rename interval.
4075///
4076/// A missing or mismatched epoch makes one non-blocking `LOCK_EX` election.
4077/// Losers wait for SH so a same-epoch initializer wakes the whole herd; a
4078/// genuinely different epoch proceeds to EX, rechecks, then performs
4079/// wipe -> sentinel exactly once. Raw no-token callers retain the historical
4080/// exclusive pre-clear-once path.
4081fn acquire_run_dir_publication_lock(
4082 dir: &std::path::Path,
4083 session_token: Option<&str>,
4084) -> anyhow::Result<(std::os::fd::OwnedFd, crate::flock::FlockMode)> {
4085 acquire_run_dir_publication_lock_with_timeout(dir, session_token, RUN_DIR_LOCK_TIMEOUT)
4086}
4087
4088/// Timeout-parametrizable inner of [`acquire_run_dir_publication_lock`].
4089fn acquire_run_dir_publication_lock_with_timeout(
4090 dir: &std::path::Path,
4091 session_token: Option<&str>,
4092 timeout: std::time::Duration,
4093) -> anyhow::Result<(std::os::fd::OwnedFd, crate::flock::FlockMode)> {
4094 use crate::flock::FlockMode;
4095
4096 let Some(token) = session_token else {
4097 let fd = acquire_run_dir_flock_with_timeout(dir, timeout)?;
4098 pre_clear_run_dir_once_for_session(dir, None);
4099 return Ok((fd, FlockMode::Exclusive));
4100 };
4101
4102 loop {
4103 // Unlocked read is only a fast-path hint. The matching result is
4104 // authoritative only after the shared acquire and recheck below.
4105 if session_sentinel_matches(dir, token) {
4106 let shared = acquire_run_dir_flock_mode_with_timeout(dir, FlockMode::Shared, timeout)?;
4107 if session_sentinel_matches(dir, token) {
4108 return Ok((shared, FlockMode::Shared));
4109 }
4110 drop(shared);
4111 }
4112
4113 // On an uninitialized directory exactly one member of a same-epoch
4114 // herd wins this non-blocking EX attempt. Losers wait for SH instead
4115 // of queueing for EX: once the winner publishes the sentinel and
4116 // releases, every loser wakes as a concurrent matching writer.
4117 if let Some(exclusive) = try_run_dir_flock(dir, FlockMode::Exclusive)? {
4118 if session_sentinel_matches(dir, token) {
4119 drop(exclusive);
4120 continue;
4121 }
4122 reset_run_dir_for_session(dir, Some(token))?;
4123 return Ok((exclusive, FlockMode::Exclusive));
4124 }
4125
4126 let shared = acquire_run_dir_flock_mode_with_timeout(dir, FlockMode::Shared, timeout)?;
4127 if session_sentinel_matches(dir, token) {
4128 return Ok((shared, FlockMode::Shared));
4129 }
4130 drop(shared);
4131
4132 // The incompatible holder was a different epoch's shared writer, not
4133 // an initializer. Wait for reset ownership, then recheck because a
4134 // same-epoch peer may have initialized while we were parked.
4135 let exclusive = acquire_run_dir_flock_with_timeout(dir, timeout)?;
4136 if session_sentinel_matches(dir, token) {
4137 drop(exclusive);
4138 continue;
4139 }
4140 reset_run_dir_for_session(dir, Some(token))?;
4141 return Ok((exclusive, FlockMode::Exclusive));
4142 }
4143}
4144
4145/// Test-parametrizable exclusive wrapper over
4146/// [`acquire_run_dir_flock_mode_with_timeout`].
4147///
4148/// Resolves the per-run-key lockfile path via [`run_dir_lock_path`]
4149/// then delegates to [`crate::flock::acquire_flock_with_timeout`],
4150/// which handles parent-directory creation, the poll loop, the
4151/// `tracing::debug!` contention log, and the formatted timeout
4152/// error. The `context` argument names the run directory and the
4153/// `remediation` argument supplies the operator-facing recovery
4154/// hint about peer cargo ktstr test processes that the shared
4155/// helper appends to the timeout error.
4156///
4157/// Returns `Err` on:
4158/// - `run_dir_lock_path(dir)` returning `None` (no parent / no
4159/// file_name — production default path always satisfies both,
4160/// so this is a defensive arm),
4161/// - any error from [`crate::flock::acquire_flock_with_timeout`]
4162/// (parent directory creation failure, `try_flock` error, or
4163/// wall-clock `timeout` elapsing).
4164///
4165/// Returns `Ok(OwnedFd)` on successful acquire. Caller drops the
4166/// fd to release the kernel-side flock; the OFD-bound semantics
4167/// of `flock(2)` mean no explicit unlock call is required —
4168/// `OwnedFd::drop` runs `close(2)` which releases the lock when
4169/// no other fd refers to the same OFD (the fresh `try_flock`
4170/// open guarantees uniqueness).
4171fn acquire_run_dir_flock_with_timeout(
4172 dir: &std::path::Path,
4173 timeout: std::time::Duration,
4174) -> anyhow::Result<std::os::fd::OwnedFd> {
4175 acquire_run_dir_flock_mode_with_timeout(dir, crate::flock::FlockMode::Exclusive, timeout)
4176}
4177
4178/// Mode-parametrizable run-directory flock acquire used by publication.
4179fn acquire_run_dir_flock_mode_with_timeout(
4180 dir: &std::path::Path,
4181 mode: crate::flock::FlockMode,
4182 timeout: std::time::Duration,
4183) -> anyhow::Result<std::os::fd::OwnedFd> {
4184 let lock_path = run_dir_lock_path(dir).ok_or_else(|| {
4185 anyhow::anyhow!(
4186 "cannot derive run-dir lock path from {} (no parent or no file_name component)",
4187 dir.display(),
4188 )
4189 })?;
4190 let context = format!("run-dir {}", dir.display());
4191 crate::flock::acquire_flock_with_timeout(
4192 &lock_path,
4193 mode,
4194 timeout,
4195 &context,
4196 Some(
4197 "A peer cargo ktstr test process is writing sidecars to the \
4198 same {kernel}-{project_commit} directory; wait for it to \
4199 finish or kill it, then retry.",
4200 ),
4201 )
4202}
4203
4204/// One non-blocking acquire against the run-directory lockfile.
4205fn try_run_dir_flock(
4206 dir: &std::path::Path,
4207 mode: crate::flock::FlockMode,
4208) -> anyhow::Result<Option<std::os::fd::OwnedFd>> {
4209 let lock_path = run_dir_lock_path(dir).ok_or_else(|| {
4210 anyhow::anyhow!(
4211 "cannot derive run-dir lock path from {} (no parent or no file_name component)",
4212 dir.display(),
4213 )
4214 })?;
4215 if let Some(parent) = lock_path.parent() {
4216 std::fs::create_dir_all(parent)
4217 .with_context(|| format!("create lock subdirectory {}", parent.display()))?;
4218 }
4219 crate::flock::try_flock(&lock_path, mode)
4220}
4221
4222/// Emit a minimal sidecar for a PRE-VM-BOOT skip path.
4223///
4224/// Stats tooling enumerates sidecars to compute pass/skip/fail
4225/// rates; when a test bails before `run_ktstr_test_inner` reaches
4226/// the VM-run site that calls [`write_sidecar`], the skip is
4227/// invisible to post-run analysis — it shows up as a missing
4228/// result rather than a recorded skip.
4229///
4230/// This helper writes a sidecar flagged `skipped: true, passed: true`
4231/// with empty VM telemetry (no monitor, no stimulus events, no
4232/// verifier stats, no kvm stats, no payload metrics). Stats tooling
4233/// that subtracts skipped runs from the pass count treats the entry
4234/// correctly.
4235///
4236/// # Distinction from in-VM `AssertResult::skip` paths
4237///
4238/// There are TWO classes of skip, each with its own sidecar writer:
4239///
4240/// 1. **Pre-VM-boot skips** route through this helper
4241/// (`write_skip_sidecar`). Examples:
4242/// - `performance_mode` gated off via `KTSTR_NO_PERF_MODE`
4243/// (see `run_ktstr_test_inner`),
4244/// - `ResourceContention` at `builder.build()` or `vm.run()`
4245/// (all-slots-busy / transient host-resource contention — the
4246/// VM never booted).
4247///
4248/// These paths write a MINIMAL sidecar: empty VM telemetry,
4249/// `skipped: true`, and BOTH `payload` and `work_type` resolved
4250/// exactly as a run of this config would (the entry's declared
4251/// payload and [`crate::test_support::args::current_work_type`]) so
4252/// the skip shares the run's variant identity — a later run of the
4253/// same config overwrites this skip's sidecar instead of coexisting
4254/// with it. There is no VmResult to drain because the VM didn't boot.
4255///
4256/// 2. **In-VM `AssertResult::skip` returns** — e.g. the
4257/// empty-cpuset skip in `scenario::run_scenario`
4258/// (`AssertResult::skip("not enough CPUs/LLCs")`), or the
4259/// `need >= 4 CPUs` checks in `scenario::dynamic::*` — route
4260/// through [`write_sidecar`] at `run_ktstr_test_inner`'s end.
4261/// The guest VM fully booted, ran through scenario setup,
4262/// discovered the topology couldn't accommodate the test, and
4263/// returned early. The resulting sidecar carries REAL VM
4264/// telemetry (monitor, kvm_stats, verifier_stats) alongside
4265/// `skipped: true` — not a blind spot, just a richer record
4266/// than what this helper emits.
4267///
4268/// The asymmetry is intentional: pre-VM-boot skips have no
4269/// telemetry to record, while in-VM skips do. Stats tooling that
4270/// wants to uniformly discount skipped runs filters on
4271/// [`SidecarResult::skipped == true`] regardless of which writer
4272/// produced the entry — both set the field identically.
4273///
4274/// Returns `Err` when the sidecar directory cannot be created, the
4275/// JSON cannot be serialized, or the file write fails. Callers that
4276/// ignore the Result accept the risk of stats-tooling blind spots on
4277/// this run.
4278pub(crate) fn write_skip_sidecar(
4279 entry: &KtstrTestEntry,
4280 resolved_topology: &crate::vmm::topology::Topology,
4281) -> anyhow::Result<()> {
4282 let SchedulerFingerprint {
4283 scheduler,
4284 scheduler_commit,
4285 sysctls,
4286 kargs,
4287 } = scheduler_fingerprint(entry);
4288 let sidecar = SidecarResult {
4289 test_name: entry.name.to_string(),
4290 perf_delta_assertions: entry
4291 .perf_delta_assertions
4292 .iter()
4293 .map(|&a| a.into())
4294 .collect(),
4295 // The RESOLVED topology a run of this preset would boot
4296 // (resolve_vm_topology(entry, topo)), NOT the declared
4297 // entry.topology — for a topology gauntlet each preset boots a
4298 // distinct topology, so recording the declared value would make
4299 // every preset share one variant_hash and clobber. For a plain
4300 // test (no override) resolved == declared. The skip and the run
4301 // of one preset thus share a variant_hash (the run path records
4302 // the same resolved topology), so a flaky test that skips on one
4303 // attempt and runs on the retry writes one sidecar.
4304 topology: resolved_topology.to_string(),
4305 scheduler,
4306 scheduler_commit,
4307 // A skip resolves no scheduler binary (no run), so there is no
4308 // discovery path to record.
4309 resolve_source: None,
4310 project_commit: detect_project_commit(),
4311 // A skip never runs the payload. Still record the declared
4312 // payload name so stats tooling can attribute the skip to
4313 // the payload-gauntlet variant rather than losing the
4314 // association.
4315 payload: entry.payload.map(|p| p.name.to_string()),
4316 metrics: Vec::new(),
4317 passed: false,
4318 skipped: true,
4319 inconclusive: false,
4320 expected_failure: false,
4321 stats: Default::default(),
4322 monitor: None,
4323 // A skip never ran the VM, so no periodic captures fired.
4324 periodic_fired: 0,
4325 periodic_target: 0,
4326 // A skip never booted the VM, so it has no measured budget. 0/0
4327 // maps to None on the GauntletRow's cpu_budget dim (skips carry no
4328 // budget identity; the skipped=true flag, not a sentinel field
4329 // value, marks them).
4330 vcpus: 0,
4331 cpu_budget: 0,
4332 // A skip never booted the VM, so no vCPU schedstat was sampled.
4333 host_dilation: None,
4334 // Era marker: even a skip row carries the current denomination so a
4335 // pool never mixes eras within one marker value.
4336 throughput_denomination: ThroughputDenomination::CpuSec,
4337 stimulus_events: Vec::new(),
4338 // A skip never ran the workload, but it carries the SAME
4339 // work_type a run of this config would (current_work_type reads
4340 // the per-variant --ktstr-work-type arg, identical across nextest
4341 // retry attempts). That keeps the skip's variant_hash equal to
4342 // the run's, so a flaky test that skips on one attempt and runs
4343 // on the retry writes one sidecar (the retry overwrites the skip)
4344 // rather than two coexisting files the footer would both flag.
4345 // Skips stay identified by skipped=true, not by a work_type
4346 // sentinel (see the variant-hash + skipped-bool contract above).
4347 work_type: super::args::current_work_type(),
4348 verifier_stats: Vec::new(),
4349 kvm_stats: None,
4350 sysctls,
4351 kargs,
4352 kernel_version: detect_kernel_version(),
4353 kernel_commit: kernel_commit_for_sidecar(),
4354 timestamp: now_iso8601(),
4355 run_id: generate_run_id(),
4356 host: Some(crate::host_context::collect_host_context()),
4357 // Skip paths never reach `collect_results`, so cleanup
4358 // duration is undefined. Emit `null` per the sidecar's
4359 // symmetric serialize/deserialize contract.
4360 cleanup_duration_ms: None,
4361 run_source: detect_run_source(),
4362 };
4363 serialize_and_write_sidecar(&sidecar, "skip sidecar")
4364}
4365
4366/// Best-effort write of a small per-test run marker JSON to the current
4367/// sidecar dir as `{test}-{variant_hash:016x}.{suffix}`. Shared by the
4368/// host-skip and probe-health markers the footer scans
4369/// ([`summarize_one_run_dir`]). Not variant-clashing with the sidecar /
4370/// dump artifacts (distinct suffix). Failures log at warn and are
4371/// swallowed — a missing marker only costs the footer one advisory line,
4372/// never a run's correctness. NOT atomically staged: the body is a
4373/// single small object a partial read would fail to parse (dropped by
4374/// the reader), and the mtime gate excludes any prior-run residue.
4375fn write_run_marker(entry_name: &str, variant_hash: u64, suffix: &str, body: &serde_json::Value) {
4376 let dir = sidecar_dir();
4377 if let Err(e) = std::fs::create_dir_all(&dir) {
4378 tracing::warn!(error = %e, path = %dir.display(), "ktstr: create dir for {suffix} marker");
4379 return;
4380 }
4381 let path = dir.join(format!("{entry_name}-{variant_hash:016x}.{suffix}"));
4382 match serde_json::to_string_pretty(body) {
4383 Ok(json) => {
4384 if let Err(e) = std::fs::write(&path, json) {
4385 tracing::warn!(error = %e, path = %path.display(), "ktstr: write {suffix} marker");
4386 }
4387 }
4388 Err(e) => tracing::warn!(error = %e, "ktstr: serialize {suffix} marker for '{entry_name}'"),
4389 }
4390}
4391
4392/// Record a `.host-skip.json` marker for a test this HOST cannot run.
4393/// `class` is the host-insufficiency tag
4394/// (`topology_insufficient` / `resource_contention` /
4395/// `perf_mode_unavailable`); the footer groups by it. Keyed by the same
4396/// variant hash the skip's `.ktstr.json` sidecar uses so a gauntlet's
4397/// per-preset skips do not clobber.
4398pub(crate) fn write_host_skip_marker(
4399 entry: &KtstrTestEntry,
4400 resolved_topology: &crate::vmm::topology::Topology,
4401 class: &str,
4402) {
4403 let variant_hash =
4404 variant_hash_from_parts(entry, resolved_topology, &super::args::current_work_type());
4405 write_run_marker(
4406 entry.name,
4407 variant_hash,
4408 "host-skip.json",
4409 &serde_json::json!({ "test_name": entry.name, "class": class }),
4410 );
4411}
4412
4413/// Record a `.probe-health.json` marker naming a per-test auto-repro
4414/// probe-pipeline problem with a short `reason`, for the footer's probe
4415/// block. `variant_hash` matches the run's other per-test artifacts.
4416pub(crate) fn write_probe_health_marker(entry_name: &str, variant_hash: u64, reason: &str) {
4417 write_run_marker(
4418 entry_name,
4419 variant_hash,
4420 "probe-health.json",
4421 &serde_json::json!({ "test_name": entry_name, "reason": reason }),
4422 );
4423}
4424
4425/// Record an `.expect-err-load.json` marker for an `expect_err` test
4426/// whose inversion was satisfied by a scheduler LOAD/startup failure
4427/// (the scheduler never attached — [`crate::verifier::AttachOutcome::Died`]
4428/// / `NotAttached`) rather than the runtime error the test intends. The
4429/// footer names these so a suite that is silently green because no
4430/// scheduler could load on this kernel becomes visible. `variant_hash`
4431/// matches the run's other per-test artifacts.
4432pub(crate) fn write_expect_err_load_marker(entry_name: &str, variant_hash: u64) {
4433 write_run_marker(
4434 entry_name,
4435 variant_hash,
4436 "expect-err-load.json",
4437 &serde_json::json!({ "test_name": entry_name }),
4438 );
4439}
4440
4441/// Write a sidecar JSON file for post-run analysis.
4442///
4443/// Output goes to the current run's sidecar directory
4444/// (`KTSTR_SIDECAR_DIR` override, or
4445/// `{CARGO_TARGET_DIR or "target"}/ktstr/{kernel}-{project_commit}/`,
4446/// where `{project_commit}` is the project HEAD short hex with
4447/// `-dirty` when the worktree differs).
4448///
4449/// `payload_metrics` is the accumulated per-invocation output from
4450/// `ctx.payload(X).run()` / `.spawn().wait()` calls made in the
4451/// test body. Empty vec when the test body never called
4452/// `Ctx::payload` (scheduler-only tests, host-only probes).
4453///
4454/// Returns `Err` when the sidecar directory cannot be created, the
4455/// JSON cannot be serialized, or the file write fails. Callers that
4456/// ignore the Result accept the risk of stats-tooling blind spots on
4457/// this run.
4458pub(crate) fn write_sidecar(
4459 entry: &KtstrTestEntry,
4460 vm_result: &vmm::VmResult,
4461 stimulus_events: &[StimulusEvent],
4462 check_result: &AssertResult,
4463 work_type: &str,
4464 payload_metrics: &[PayloadMetrics],
4465 resolved_topology: &crate::vmm::topology::Topology,
4466) -> anyhow::Result<()> {
4467 let SchedulerFingerprint {
4468 scheduler,
4469 scheduler_commit,
4470 sysctls,
4471 kargs,
4472 } = scheduler_fingerprint(entry);
4473 let sidecar = SidecarResult {
4474 test_name: entry.name.to_string(),
4475 perf_delta_assertions: entry
4476 .perf_delta_assertions
4477 .iter()
4478 .map(|&a| a.into())
4479 .collect(),
4480 // The RESOLVED topology this run booted (resolve_vm_topology
4481 // result), NOT the declared entry.topology — a topology gauntlet
4482 // boots a distinct topology per preset, so the declared value
4483 // would collapse every preset to one variant_hash. resolved ==
4484 // declared for a plain test (no override).
4485 topology: resolved_topology.to_string(),
4486 scheduler,
4487 scheduler_commit,
4488 // Scheduler-resolution provenance, carried on VmResult from the
4489 // host eval path (run_ktstr_test_inner_impl resolves the binary
4490 // once and stamps the source), mirroring how vcpus / cpu_budget
4491 // ride VmResult to this stamp.
4492 resolve_source: vm_result.resolve_source.clone(),
4493 project_commit: detect_project_commit(),
4494 payload: entry.payload.map(|p| p.name.to_string()),
4495 metrics: payload_metrics.to_vec(),
4496 passed: check_result.is_pass(),
4497 skipped: check_result.is_skip(),
4498 inconclusive: check_result.is_inconclusive(),
4499 // Raw scenario verdict at write time; the dispatch-layer
4500 // finalize (finalize_sidecar_verdict) overwrites these bits with
4501 // the post-inversion outcome and sets expected_failure.
4502 expected_failure: false,
4503 stats: check_result.stats.clone(),
4504 monitor: vm_result.monitor.as_ref().map(|m| m.summary.clone()),
4505 periodic_fired: vm_result.periodic_fired,
4506 periodic_target: vm_result.periodic_target,
4507 vcpus: vm_result.vcpus,
4508 cpu_budget: vm_result.cpu_budget,
4509 // Measured host dilation (evidence for the overcommit marker),
4510 // derived from the run's raw vCPU-thread schedstat totals.
4511 host_dilation: vm_result.host_vcpu_schedstat.and_then(|s| s.dilation()),
4512 // Era marker: this build's throughput rates are CPU-second
4513 // denominated (see ThroughputDenomination).
4514 throughput_denomination: ThroughputDenomination::CpuSec,
4515 stimulus_events: stimulus_events.to_vec(),
4516 work_type: work_type.to_string(),
4517 verifier_stats: vm_result.verifier_stats.clone(),
4518 kvm_stats: vm_result.kvm_stats.clone(),
4519 sysctls,
4520 kargs,
4521 kernel_version: detect_kernel_version(),
4522 kernel_commit: kernel_commit_for_sidecar(),
4523 timestamp: now_iso8601(),
4524 run_id: generate_run_id(),
4525 host: Some(crate::host_context::collect_host_context()),
4526 cleanup_duration_ms: vm_result.cleanup_duration.map(|d| d.as_millis() as u64),
4527 run_source: detect_run_source(),
4528 };
4529 serialize_and_write_sidecar(&sidecar, "sidecar")
4530}
4531
4532#[cfg(test)]
4533mod tests;