ktstr/cli/stats_cmds/
explain_sidecar.rs

1//! Per-sidecar `Option`-field absence diagnostic surface.
2//!
3//! Holds [`explain_sidecar`] (the `cargo ktstr stats explain-sidecar` entry
4//! point), the static catalog ([`SIDECAR_NONE_CATALOG`]) cataloging
5//! every `Option<T>` field on
6//! [`crate::test_support::SidecarResult`] with cause prose +
7//! actionable fix wording, the [`NoneClassification`] tag splitting
8//! "expected" from "actionable" gaps, the run-directory walker
9//! ([`walk_run_with_stats`]) plus its file-count helper
10//! ([`count_sidecar_files`]) and per-walk [`WalkStats`], and both
11//! renderers ([`render_explain_sidecar_text`] /
12//! [`render_explain_sidecar_json`] with the schema-versioned
13//! [`ExplainOutput`] / [`WalkStatsJson`] / [`WalkError`] /
14//! [`WalkIoError`] / [`FieldDiagnostic`] shapes).
15
16use std::path::Path;
17
18use anyhow::{Result, bail};
19
20use super::dispatch::suggest_closest_run_key;
21
22/// Whether a `None` value on a [`crate::test_support::SidecarResult`]
23/// `Option` field is the expected steady-state shape (e.g. `payload`
24/// for a scheduler-only test) or signals a recoverable gap an
25/// operator can remediate (e.g. `kernel_commit` from a tarball-cache
26/// kernel that has no on-disk source tree to probe).
27///
28/// Used by [`explain_sidecar`] to label every diagnostic block; the
29/// `JSON` shape exposes this as a `"classification"` string per
30/// field so dashboards can color-code "expected" vs "actionable"
31/// blocks without re-deriving the rule from causes prose.
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33enum NoneClassification {
34    /// `None` is the expected steady state for this field — no
35    /// operator action recovers it (the source data does not
36    /// exist or has not been wired yet).
37    Expected,
38    /// `None` indicates a recoverable gap — re-running the test in
39    /// a different environment (in-repo cwd, non-tarball kernel,
40    /// non-host-only test) would populate the field.
41    Actionable,
42}
43
44impl NoneClassification {
45    /// Stable string token for the JSON `classification` key on
46    /// [`explain_sidecar`]'s machine-readable output. The text
47    /// renderer uses the same token so a human reading the
48    /// terminal output sees the same label they would scrape from
49    /// JSON.
50    fn as_str(self) -> &'static str {
51        match self {
52            Self::Expected => "expected",
53            Self::Actionable => "actionable",
54        }
55    }
56}
57
58/// Catalog entry for one [`crate::test_support::SidecarResult`]
59/// `Option` field — wired into [`SIDECAR_NONE_CATALOG`] so the
60/// diagnostic surface stays in lockstep with the on-disk schema.
61///
62/// Each entry was derived from the rustdoc on the corresponding
63/// field in `src/test_support/sidecar/mod.rs` — see the per-field
64/// references at the catalog site for the source of truth. This
65/// catalog is static (no live probing): explain-sidecar's purpose
66/// is post-hoc archive diagnosis, not live debugging, and dynamic
67/// probes against an absent host (e.g. checking `KTSTR_KERNEL` on
68/// a CI runner that produced an archived sidecar) would return
69/// nonsense.
70struct NoneCatalogEntry {
71    /// Field name as serialized on disk (matches the struct field
72    /// identifier verbatim — serde uses the field name without
73    /// rename attributes for [`crate::test_support::SidecarResult`]).
74    field: &'static str,
75    /// Classification when this field is `None`. See
76    /// [`NoneClassification`].
77    classification: NoneClassification,
78    /// Human-readable cause prose, one entry per documented cause
79    /// from the field's rustdoc. The text renderer prints each
80    /// cause on its own bulleted line; the JSON shape emits them
81    /// as a JSON string array verbatim.
82    causes: &'static [&'static str],
83    /// Operator-actionable remediation, when one applies.
84    /// `Some(...)` for fields where re-running in a different
85    /// configuration would populate the value (e.g.
86    /// `kernel_commit` recovers when `KTSTR_KERNEL` points at a
87    /// local source tree); `None` for fields whose `None` is the
88    /// steady-state shape with no recourse (e.g.
89    /// `scheduler_commit` is reserved on the schema for future
90    /// enrichment).
91    ///
92    /// One fix per entry: this is the most-common-case
93    /// remediation, picked when a field has multiple causes that
94    /// all converge on the same operator action. The current
95    /// shape covers the typical case without forcing a deeper
96    /// data-model change; a per-cause split is possible if the
97    /// catalog grows fields whose causes legitimately diverge
98    /// in their remediation.
99    fix: Option<&'static str>,
100}
101
102/// Static catalog covering every `Option<T>` field on
103/// [`crate::test_support::SidecarResult`]. Order matches the on-
104/// disk schema declaration order so a human diff against
105/// `SidecarResult` reads top-to-bottom.
106///
107/// Causes prose is sourced FROM the per-field rustdoc on
108/// `SidecarResult` — see `src/test_support/sidecar/mod.rs` for the
109/// single source of truth on what each `None` means. A future
110/// schema change that adds, removes, or renames an `Option`
111/// field MUST update this catalog; the
112/// `none_catalog_covers_every_option_field` test enforces
113/// `SIDECAR_NONE_CATALOG.len() == EXPECTED_OPTION_FIELD_COUNT`
114/// (a hand-coded `11`) and asserts the projection helper
115/// enumerates the same field names in the same order. A new
116/// `Option` field on `SidecarResult` requires bumping the
117/// constant, extending [`project_optional_fields`]'s array
118/// (which has compile-checked length `11` and will fail to
119/// compile on a missing entry), and adding a catalog row.
120const SIDECAR_NONE_CATALOG: &[NoneCatalogEntry] = &[
121    NoneCatalogEntry {
122        field: "scheduler_commit",
123        classification: NoneClassification::Expected,
124        causes: &["no SchedulerSpec variant currently exposes a reliable \
125             commit source — reserved on the schema for future \
126             enrichment (e.g. --version probe or ELF-note read on \
127             the resolved scheduler binary)"],
128        fix: None,
129    },
130    NoneCatalogEntry {
131        field: "resolve_source",
132        classification: NoneClassification::Actionable,
133        causes: &[
134            "the sidecar predates the `resolve_source` field (added when \
135             scheduler-resolution provenance began being recorded), so it \
136             falls back to None via serde's tolerate-absence rule",
137            "a pre-VM-boot skip: no scheduler binary was resolved, so there \
138             is no discovery path to record",
139        ],
140        fix: Some(
141            "re-run the test to regenerate the sidecar — a real run records \
142             the scheduler's discovery path",
143        ),
144    },
145    NoneCatalogEntry {
146        field: "project_commit",
147        classification: NoneClassification::Actionable,
148        causes: &[
149            "current_dir() could not be resolved at sidecar-write \
150             time (process cwd was rmdir'd while alive)",
151            "test process cwd was not inside any git repository",
152            "HEAD could not be read (unborn HEAD on a fresh \
153             `git init` with zero commits, or a corrupt repository)",
154        ],
155        fix: Some(
156            "run from inside a git-tracked source tree with at \
157             least one commit",
158        ),
159    },
160    NoneCatalogEntry {
161        field: "payload",
162        classification: NoneClassification::Expected,
163        causes: &["test declared no binary payload (scheduler-only test \
164             or pure-scenario test that never invokes \
165             ctx.payload(...))"],
166        fix: None,
167    },
168    NoneCatalogEntry {
169        field: "monitor",
170        classification: NoneClassification::Actionable,
171        causes: &[
172            "host-only test path: monitor loop never started",
173            "early VM failure: monitor loop terminated before \
174             producing samples",
175            "sample collection produced no valid data",
176        ],
177        fix: None,
178    },
179    NoneCatalogEntry {
180        field: "kvm_stats",
181        classification: NoneClassification::Actionable,
182        causes: &[
183            "host-only test path: VM did not run",
184            "KVM stats were unavailable on this host (e.g. KVM \
185             module not loaded, /dev/kvm permissions, or kernel \
186             missing the stats interface)",
187        ],
188        fix: None,
189    },
190    NoneCatalogEntry {
191        field: "kernel_version",
192        classification: NoneClassification::Actionable,
193        causes: &[
194            "host-only test path: no kernel under test",
195            "neither cache metadata nor `include/config/kernel.release` \
196             yielded a version string",
197        ],
198        fix: None,
199    },
200    NoneCatalogEntry {
201        field: "kernel_commit",
202        classification: NoneClassification::Actionable,
203        causes: &[
204            "KTSTR_KERNEL is unset or empty",
205            "kernel source is a Tarball or Git transient cache \
206             entry (no on-disk source tree to probe)",
207            "resolved kernel directory is not a git repository \
208             (gix::open failed)",
209            "HEAD cannot be read (unborn HEAD on a fresh `git init` \
210             with zero commits)",
211            "gix probe failed for another reason — metadata, not \
212             a gate",
213        ],
214        fix: Some(
215            "set KTSTR_KERNEL to a local kernel source tree that \
216             is a git repository (e.g. a git clone of the kernel)",
217        ),
218    },
219    NoneCatalogEntry {
220        field: "host",
221        classification: NoneClassification::Actionable,
222        causes: &[
223            "test-fixture path: not the production sidecar \
224             writer (production writers always populate `host`)",
225            "pre-enrichment archive: sidecar predates the \
226             host-context landing — re-run the test to \
227             regenerate under the current schema",
228        ],
229        fix: Some(
230            "for pre-enrichment archives, re-run the test to \
231             regenerate under the current schema; test-fixture \
232             sidecars are not production runs and cannot be \
233             recovered by re-running",
234        ),
235    },
236    NoneCatalogEntry {
237        field: "cleanup_duration_ms",
238        classification: NoneClassification::Actionable,
239        causes: &[
240            "host-only / host-only-stub test path: no VM teardown \
241             window to time",
242            "run was killed by the watchdog before \
243             `KtstrVm::collect_results` returned",
244        ],
245        fix: None,
246    },
247    NoneCatalogEntry {
248        field: "run_source",
249        classification: NoneClassification::Actionable,
250        causes: &["pre-rename archive: sidecar carries the old `source` \
251             key which the current schema drops as an unknown \
252             field, leaving `run_source` to fall back to None via \
253             serde's tolerate-absence rule. Re-run the test to \
254             regenerate under the new schema, or rename the key \
255             in-place before deserialize"],
256        fix: Some(
257            "re-run the test to regenerate, or rename the on-disk \
258             `source` key to `run_source`",
259        ),
260    },
261];
262
263/// Project one [`crate::test_support::SidecarResult`] onto its
264/// `Option` fields, returning `(field_name, is_some)` pairs in the
265/// same order as [`SIDECAR_NONE_CATALOG`].
266///
267/// Hand-written rather than derived because:
268/// - Only the 11 `Option<T>` fields are diagnostic surface; the
269///   non-`Option` fields (`test_name`, `passed`, `stats`, etc.)
270///   are always populated by deserialize and would clutter the
271///   output without adding signal.
272/// - The order MUST match the catalog so the field-by-field
273///   lookup in [`render_explain_sidecar_text`] resolves
274///   correctly.
275///
276/// A future schema addition that introduces a new `Option<T>`
277/// field on `SidecarResult` MUST update this projection; the
278/// `[(_, _); 11]` array literal makes the length compile-checked
279/// — adding an entry without updating the length is a compile
280/// error — and the `none_catalog_covers_every_option_field` test
281/// asserts the catalog and projection enumerate the same names
282/// in the same order.
283fn project_optional_fields(sc: &crate::test_support::SidecarResult) -> [(&'static str, bool); 11] {
284    [
285        ("scheduler_commit", sc.scheduler_commit.is_some()),
286        ("resolve_source", sc.resolve_source.is_some()),
287        ("project_commit", sc.project_commit.is_some()),
288        ("payload", sc.payload.is_some()),
289        ("monitor", sc.monitor.is_some()),
290        ("kvm_stats", sc.kvm_stats.is_some()),
291        ("kernel_version", sc.kernel_version.is_some()),
292        ("kernel_commit", sc.kernel_commit.is_some()),
293        ("host", sc.host.is_some()),
294        ("cleanup_duration_ms", sc.cleanup_duration_ms.is_some()),
295        ("run_source", sc.run_source.is_some()),
296    ]
297}
298
299/// File-walk statistics for the run directory under
300/// [`explain_sidecar`]. Used to drive the `walked / valid` header
301/// and the corrupt-sidecars footer; the steady-state invariant is
302/// `walked == valid + errors.len() + io_errors.len()`.
303struct WalkStats {
304    walked: usize,
305    valid: usize,
306    errors: Vec<crate::test_support::SidecarParseError>,
307    io_errors: Vec<crate::test_support::SidecarIoError>,
308}
309
310/// Count `.ktstr.json` files under `run_dir` using
311/// [`crate::test_support::collect_sidecars`]'s walk shape (flat
312/// files plus one level of subdirectories). Pure file-existence
313/// pass — no parsing, no `serde_json` work — used purely to
314/// derive the `walked` count for [`WalkStats`].
315fn count_sidecar_files(run_dir: &Path) -> usize {
316    let mut count = 0usize;
317    let entries = match std::fs::read_dir(run_dir) {
318        Ok(e) => e,
319        Err(_) => return 0,
320    };
321    let mut subdirs = Vec::new();
322    for entry in entries.flatten() {
323        let path = entry.path();
324        if path.is_dir() {
325            subdirs.push(path);
326            continue;
327        }
328        if crate::test_support::is_sidecar_filename(&path) {
329            count += 1;
330        }
331    }
332    for sub in subdirs {
333        if let Ok(entries) = std::fs::read_dir(&sub) {
334            for entry in entries.flatten() {
335                if crate::test_support::is_sidecar_filename(&entry.path()) {
336                    count += 1;
337                }
338            }
339        }
340    }
341    count
342}
343
344/// Load sidecars under `run_dir` and report file-walk statistics.
345fn walk_run_with_stats(run_dir: &Path) -> (Vec<crate::test_support::SidecarResult>, WalkStats) {
346    let walked = count_sidecar_files(run_dir);
347    let (sidecars, errors, io_errors) = crate::test_support::collect_sidecars_with_errors(run_dir);
348    let valid = sidecars.len();
349    (
350        sidecars,
351        WalkStats {
352            walked,
353            valid,
354            errors,
355            io_errors,
356        },
357    )
358}
359
360/// Diagnose `Option`-field absences for a run's sidecars. Mirrors
361/// `show_run_host`'s shape (`--run` + optional `--dir`,
362/// printable string return). See the original rustdoc on the
363/// public surface for the full JSON / text contract and exit-code
364/// policy.
365pub fn explain_sidecar(run: &str, dir: Option<&Path>, json: bool) -> Result<String> {
366    if run.is_empty() {
367        bail!(
368            "run argument must not be empty. The run argument is \
369             joined onto the run-root via `Path::join` and must \
370             contain at least one `Normal` path component — i.e. \
371             must not be empty, `.`, `..`, or absolute (e.g. a \
372             typical run key shape: `6.14-abc1234` or \
373             `6.14-abc1234-dirty`). To point at a different pool \
374             root, use `--dir`. Run `cargo ktstr stats list` to \
375             enumerate available run keys.",
376        );
377    }
378    for component in std::path::Path::new(run).components() {
379        match component {
380            std::path::Component::CurDir
381            | std::path::Component::ParentDir
382            | std::path::Component::RootDir
383            | std::path::Component::Prefix(_) => {
384                bail!(
385                    "run '{run}' contains pool-root-aliasing or \
386                     path-traversal components (`.`, `..`, or absolute \
387                     path). The run argument is joined onto the \
388                     run-root via `Path::join` and must contain only \
389                     `Normal` path components — no `.`, `..`, or \
390                     absolute prefix (e.g. a typical run key shape: \
391                     `6.14-abc1234` or `6.14-abc1234-dirty`; \
392                     multi-component paths like `gauntlet/job-1` are \
393                     also accepted). To point at a different pool \
394                     root, use `--dir`. Run `cargo ktstr stats list` \
395                     to enumerate available run keys.",
396                );
397            }
398            std::path::Component::Normal(_) => {}
399        }
400    }
401    let root: std::path::PathBuf = match dir {
402        Some(d) => d.to_path_buf(),
403        None => crate::test_support::runs_root(),
404    };
405    let run_dir = root.join(run);
406    if !run_dir.exists() {
407        let suggestion = suggest_closest_run_key(run, &root)
408            .map(|name| format!(" Did you mean `{name}`?"))
409            .unwrap_or_default();
410        bail!(
411            "run '{run}' not found under {}.{suggestion} \
412             Run `cargo ktstr stats list` to enumerate available run keys.",
413            root.display(),
414        );
415    }
416    let (sidecars, walk_stats) = walk_run_with_stats(&run_dir);
417    if walk_stats.walked == 0 {
418        bail!(
419            "run '{run}' has no sidecar data (searched {})",
420            run_dir.display(),
421        );
422    }
423    if json {
424        Ok(render_explain_sidecar_json(&sidecars, &walk_stats))
425    } else {
426        Ok(render_explain_sidecar_text(&sidecars, &walk_stats))
427    }
428}
429
430/// Render the per-sidecar text block for [`explain_sidecar`].
431fn render_explain_sidecar_text(
432    sidecars: &[crate::test_support::SidecarResult],
433    walk_stats: &WalkStats,
434) -> String {
435    use std::fmt::Write as _;
436    let mut sorted: Vec<&crate::test_support::SidecarResult> = sidecars.iter().collect();
437    sorted.sort_by(|a, b| {
438        a.test_name
439            .cmp(&b.test_name)
440            .then_with(|| a.run_id.cmp(&b.run_id))
441    });
442    let mut out = String::new();
443    let _ = writeln!(
444        out,
445        "walked {} sidecar file(s), parsed {} valid\n",
446        walk_stats.walked, walk_stats.valid,
447    );
448    for sc in &sorted {
449        let _ = writeln!(out, "test: {}", sc.test_name);
450        let _ = writeln!(out, "  topology: {}", sc.topology);
451        let _ = writeln!(out, "  scheduler: {}", sc.scheduler);
452        let _ = writeln!(out, "  run_id: {}", sc.run_id);
453        let arch = sc
454            .host
455            .as_ref()
456            .and_then(|h| h.arch.as_deref())
457            .unwrap_or("-");
458        let _ = writeln!(out, "  arch: {arch}");
459        // vCPU count + effective host-CPU budget, with the overcommit
460        // marker: cpu_budget < vcpus means the host time-sliced the guest's
461        // vCPU threads, so wake-latency / off-CPU / run-delay timing metrics
462        // are host-contention-confounded. schedstat run_delay accumulates
463        // `rq_clock` deltas (kernel/sched/stats.h sched_info_arrive), and
464        // rq->clock tracks the guest TSC and is NEVER steal-adjusted — the
465        // steal subtraction hits only rq->clock_task (kernel/sched/core.c
466        // update_rq_clock_task) — so the off-host deschedule window INFLATES
467        // run_delay for guest tasks enqueued-and-waiting across it, just
468        // like wake-latency and off-CPU, not "freezes" it. Matches
469        // host_topology::overcommit_warning. The raw metrics are
470        // KEPT (never None'd); this annotates them. The branch order is
471        // skip -> malformed -> overcommit -> plain: cpu_budget == 0 renders
472        // skip irrespective of vcpus (the writer pairs 0/0), and vcpus == 0
473        // with a nonzero budget is a malformed on-disk sidecar the writer
474        // never emits.
475        if sc.cpu_budget == 0 {
476            let _ = writeln!(out, "  cpu_budget: - (skip; VM not booted)");
477        } else if sc.vcpus == 0 {
478            let _ = writeln!(
479                out,
480                "  cpu_budget: {} / 0 vcpus  [malformed: vcpus=0 with a nonzero \
481                 budget; the writer never emits this]",
482                sc.cpu_budget,
483            );
484        } else if sc.cpu_budget < sc.vcpus {
485            // Cite the MEASURED dilation when the run recorded one: paper
486            // overcommit (cpu_budget < vcpus) predicts contention, but D
487            // = 1 + Σrun_delay/Σon_cpu is what the vCPU threads actually
488            // suffered — an on-paper-overcommitted run on an otherwise
489            // idle host can still read D≈1.0. `None` (schedstats-off host
490            // or pre-field sidecar) drops the clause.
491            let dilation_note = match sc.host_dilation {
492                Some(d) => format!(" measured host dilation D={d:.2}x this run;"),
493                None => String::new(),
494            };
495            let _ = writeln!(
496                out,
497                "  cpu_budget: {} / {} vcpus  [OVERCOMMIT: host time-slices the \
498                 guest vCPUs ->{dilation_note} wake-latency / off-CPU / run-delay timing \
499                 metrics are host-contention-confounded; compare the overcommit-invariant \
500                 worst_iterations_per_cpu_sec, not raw timing]",
501                sc.cpu_budget, sc.vcpus,
502            );
503        } else {
504            let _ = writeln!(out, "  cpu_budget: {} / {} vcpus", sc.cpu_budget, sc.vcpus);
505        }
506        // Per-cgroup CPU placement — which host/guest CPUs each cgroup's
507        // workers actually ran on. Surfaced on EVERY run (the scheduler
508        // failure dump only fires on failure); reads the always-written
509        // ScenarioStats.cgroups. Cgroups with no recorded CPU (no worker
510        // reported one) are skipped.
511        for cg in sc.stats.cgroups.iter().filter(|c| !c.cpus_used.is_empty()) {
512            let label = if cg.cgroup_name.is_empty() {
513                "(unnamed)"
514            } else {
515                cg.cgroup_name.as_str()
516            };
517            let cpus = cg
518                .cpus_used
519                .iter()
520                .map(|c| c.to_string())
521                .collect::<Vec<_>>()
522                .join(",");
523            let _ = writeln!(out, "  cgroup {label}: ran on cpus [{cpus}]");
524        }
525        let projected = project_optional_fields(sc);
526        let populated: Vec<&'static str> = projected
527            .iter()
528            .filter(|(_, b)| *b)
529            .map(|(n, _)| *n)
530            .collect();
531        let none_fields: Vec<&'static str> = projected
532            .iter()
533            .filter(|(_, b)| !*b)
534            .map(|(n, _)| *n)
535            .collect();
536        let populated_text = if populated.is_empty() {
537            "<none>".to_string()
538        } else {
539            populated.join(", ")
540        };
541        let _ = writeln!(
542            out,
543            "  populated optional fields ({}): {populated_text}",
544            populated.len(),
545        );
546        if none_fields.is_empty() {
547            let _ = writeln!(out, "  none fields: <all populated>\n");
548            continue;
549        }
550        let _ = writeln!(out, "  none fields ({}):", none_fields.len());
551        for field in none_fields {
552            let entry = SIDECAR_NONE_CATALOG
553                .iter()
554                .find(|e| e.field == field)
555                .expect(
556                    "catalog must cover every projected field — \
557                     guarded by none_catalog_covers_every_option_field",
558                );
559            let _ = writeln!(
560                out,
561                "    {} [{}]",
562                entry.field,
563                entry.classification.as_str(),
564            );
565            for cause in entry.causes {
566                let _ = writeln!(out, "      - {cause}");
567            }
568            if let Some(fix) = entry.fix {
569                let _ = writeln!(out, "      fix: {fix}");
570            }
571        }
572        out.push('\n');
573    }
574    if !walk_stats.errors.is_empty() {
575        let _ = writeln!(out, "corrupt sidecars ({}):", walk_stats.errors.len());
576        for err in &walk_stats.errors {
577            let _ = writeln!(out, "  {}", err.path.display());
578            let _ = writeln!(out, "    error: {}", err.raw_error);
579            if let Some(prose) = &err.enriched_message {
580                let _ = writeln!(out, "    enriched: {prose}");
581            }
582        }
583        out.push('\n');
584    }
585    if !walk_stats.io_errors.is_empty() {
586        let _ = writeln!(out, "io errors ({}):", walk_stats.io_errors.len());
587        for err in &walk_stats.io_errors {
588            let _ = writeln!(out, "  {}", err.path.display());
589            let _ = writeln!(out, "    error: {}", err.raw_error);
590        }
591        out.push('\n');
592    }
593    out
594}
595
596/// JSON schema version stamp emitted on
597/// [`ExplainOutput::_schema_version`]. Bumped on any incompatible
598/// shape change.
599// "2": added `overcommit_runs` (the count of pooled sidecars whose
600// host CPU budget was below their vCPU count, so their timing metrics
601// are host-contention-confounded — the JSON-path mirror of the text
602// per-sidecar OVERCOMMIT marker, for scripted consumers that gate on
603// data quality).
604const EXPLAIN_SIDECAR_SCHEMA_VERSION: &str = "2";
605
606#[derive(serde::Serialize)]
607struct ExplainOutput<'a> {
608    _schema_version: &'a str,
609    _walk: WalkStatsJson<'a>,
610    /// Count of pooled sidecars with `cpu_budget != 0 && cpu_budget <
611    /// vcpus` — host-overcommitted runs whose wake-latency / off-CPU /
612    /// run-delay timing metrics are host-contention-confounded. Skip
613    /// rows (`cpu_budget == 0`) never booted and are excluded.
614    overcommit_runs: usize,
615    fields: std::collections::BTreeMap<&'a str, FieldDiagnostic<'a>>,
616}
617
618#[derive(serde::Serialize)]
619struct WalkStatsJson<'a> {
620    walked: usize,
621    valid: usize,
622    errors: Vec<WalkError<'a>>,
623    io_errors: Vec<WalkIoError<'a>>,
624}
625
626#[derive(serde::Serialize)]
627struct WalkError<'a> {
628    path: String,
629    error: &'a str,
630    enriched_message: Option<&'a str>,
631}
632
633#[derive(serde::Serialize)]
634struct WalkIoError<'a> {
635    path: String,
636    error: &'a str,
637}
638
639#[derive(serde::Serialize)]
640struct FieldDiagnostic<'a> {
641    none_count: usize,
642    some_count: usize,
643    classification: &'a str,
644    causes: &'a [&'a str],
645    fix: Option<&'a str>,
646}
647
648/// Render the aggregate JSON shape for [`explain_sidecar`].
649fn render_explain_sidecar_json(
650    sidecars: &[crate::test_support::SidecarResult],
651    walk_stats: &WalkStats,
652) -> String {
653    let fields: std::collections::BTreeMap<&str, FieldDiagnostic<'_>> = SIDECAR_NONE_CATALOG
654        .iter()
655        .map(|entry| {
656            let none_count = sidecars
657                .iter()
658                .filter(|sc| {
659                    project_optional_fields(sc)
660                        .iter()
661                        .any(|(n, b)| *n == entry.field && !*b)
662                })
663                .count();
664            let some_count = sidecars.len().saturating_sub(none_count);
665            (
666                entry.field,
667                FieldDiagnostic {
668                    none_count,
669                    some_count,
670                    classification: entry.classification.as_str(),
671                    causes: entry.causes,
672                    fix: entry.fix,
673                },
674            )
675        })
676        .collect();
677    let errors: Vec<WalkError<'_>> = walk_stats
678        .errors
679        .iter()
680        .map(|err| WalkError {
681            path: err.path.display().to_string(),
682            error: &err.raw_error,
683            enriched_message: err.enriched_message.as_deref(),
684        })
685        .collect();
686    let io_errors: Vec<WalkIoError<'_>> = walk_stats
687        .io_errors
688        .iter()
689        .map(|err| WalkIoError {
690            path: err.path.display().to_string(),
691            error: &err.raw_error,
692        })
693        .collect();
694    let overcommit_runs = sidecars
695        .iter()
696        .filter(|sc| sc.cpu_budget != 0 && sc.cpu_budget < sc.vcpus)
697        .count();
698    let output = ExplainOutput {
699        _schema_version: EXPLAIN_SIDECAR_SCHEMA_VERSION,
700        _walk: WalkStatsJson {
701            walked: walk_stats.walked,
702            valid: walk_stats.valid,
703            errors,
704            io_errors,
705        },
706        overcommit_runs,
707        fields,
708    };
709    serde_json::to_string_pretty(&output).expect(
710        "static-shape JSON serialization is infallible — every \
711         field in ExplainOutput / WalkStatsJson / WalkError / WalkIoError / \
712         FieldDiagnostic is a primitive, &str, or Vec/BTreeMap \
713         of those — no NaN, no non-string keys, no unsupported \
714         types",
715    )
716}
717
718#[cfg(test)]
719mod tests {
720    use super::super::super::testing::{
721        SIDECAR_VEC_FIELDS, make_test_run, write_corrupt_sidecar, write_sidecar,
722    };
723    use super::*;
724
725    /// Drift guard: every `Option<T>` field on `SidecarResult` must
726    /// have a matching catalog entry in `SIDECAR_NONE_CATALOG`, and
727    /// the projected-fields helper must enumerate the same set.
728    #[test]
729    fn none_catalog_covers_every_option_field() {
730        const EXPECTED_OPTION_FIELD_COUNT: usize = 11;
731        assert_eq!(
732            SIDECAR_NONE_CATALOG.len(),
733            EXPECTED_OPTION_FIELD_COUNT,
734            "SIDECAR_NONE_CATALOG must cover every Option<T> field on \
735             SidecarResult; expected {EXPECTED_OPTION_FIELD_COUNT}, got \
736             {}. A schema change must update the catalog in lockstep.",
737            SIDECAR_NONE_CATALOG.len(),
738        );
739        let sc = crate::test_support::SidecarResult::test_fixture();
740        let projected = project_optional_fields(&sc);
741        assert_eq!(
742            projected.len(),
743            EXPECTED_OPTION_FIELD_COUNT,
744            "project_optional_fields must enumerate every Option<T> \
745             field; expected {EXPECTED_OPTION_FIELD_COUNT}, got {}. Co-update \
746             with the catalog when adding a new Option field.",
747            projected.len(),
748        );
749        for (i, (name, _)) in projected.iter().enumerate() {
750            let catalog = &SIDECAR_NONE_CATALOG[i];
751            assert_eq!(
752                *name, catalog.field,
753                "projected field {i} ({name:?}) must match catalog \
754                 entry at the same index ({:?}) — order drift breaks \
755                 the renderer's catalog-lookup expectation",
756                catalog.field,
757            );
758        }
759    }
760
761    /// Catalog `causes` arrays must be non-empty for every entry.
762    #[test]
763    fn none_catalog_every_entry_has_causes() {
764        for entry in SIDECAR_NONE_CATALOG {
765            assert!(
766                !entry.causes.is_empty(),
767                "catalog entry for {} has no causes — every field's \
768                 None case must document at least one cause",
769                entry.field,
770            );
771        }
772    }
773
774    /// Expected-classified entries (steady-state None) must NOT
775    /// carry a `fix:` — there is no operator action that recovers
776    /// an Expected None, so emitting one would mislead.
777    #[test]
778    fn none_catalog_expected_entries_have_no_fix() {
779        for entry in SIDECAR_NONE_CATALOG {
780            if matches!(entry.classification, NoneClassification::Expected) {
781                assert!(
782                    entry.fix.is_none(),
783                    "Expected-classified field {} must not carry a `fix:` \
784                     — there is no operator action that recovers a \
785                     steady-state None",
786                    entry.field,
787                );
788            }
789        }
790    }
791
792    /// `fix:` assignment policy: must-fix for fields with a single
793    /// concrete recovery action; must-not-fix for Actionable fields
794    /// whose cause set spans multiple unrelated remedies (no single
795    /// operator action covers them).
796    #[test]
797    fn none_catalog_fix_assignments_match_policy() {
798        let by_field: std::collections::HashMap<&'static str, Option<&'static str>> =
799            SIDECAR_NONE_CATALOG
800                .iter()
801                .map(|e| (e.field, e.fix))
802                .collect();
803        let must_fix = [
804            "project_commit",
805            "kernel_commit",
806            "host",
807            "run_source",
808            "resolve_source",
809        ];
810        let must_not_fix = [
811            "scheduler_commit",
812            "payload",
813            "monitor",
814            "kvm_stats",
815            "kernel_version",
816            "cleanup_duration_ms",
817        ];
818        assert_eq!(
819            must_fix.len() + must_not_fix.len(),
820            SIDECAR_NONE_CATALOG.len(),
821            "every catalog entry must be classified as either \
822             must-fix or must-not-fix; expected sum = catalog len \
823             ({}), got must_fix={} + must_not_fix={}",
824            SIDECAR_NONE_CATALOG.len(),
825            must_fix.len(),
826            must_not_fix.len(),
827        );
828        for field in &must_fix {
829            let fix = by_field.get(field).copied().flatten();
830            assert!(
831                fix.is_some(),
832                "field {field} has a single concrete recovery action and must carry a `fix:`",
833            );
834        }
835        for field in &must_not_fix {
836            let fix = by_field.get(field).copied().flatten();
837            assert!(
838                fix.is_none(),
839                "field {field} must NOT carry a `fix:` (multi-cause or \
840                 steady-state None) — got: {fix:?}",
841            );
842        }
843    }
844
845    /// Error path: the named run directory does not exist.
846    #[test]
847    fn explain_sidecar_missing_run_returns_error() {
848        let tmp = tempfile::tempdir().unwrap();
849        let err = explain_sidecar("nonexistent-run", Some(tmp.path()), false).unwrap_err();
850        let msg = format!("{err:#}");
851        assert!(
852            msg.contains("run 'nonexistent-run' not found"),
853            "missing-run error must name the run: {msg}",
854        );
855        assert!(
856            msg.contains("cargo ktstr stats list"),
857            "missing-run error must name the discovery command: {msg}",
858        );
859    }
860
861    /// Error path: run directory exists but is empty.
862    #[test]
863    fn explain_sidecar_empty_run_returns_error() {
864        let tmp = tempfile::tempdir().unwrap();
865        let run_dir = tmp.path().join("run-empty");
866        std::fs::create_dir(&run_dir).unwrap();
867        let err = explain_sidecar("run-empty", Some(tmp.path()), false).unwrap_err();
868        let msg = format!("{err:#}");
869        assert!(
870            msg.contains("no sidecar data"),
871            "empty-run error must use the canonical message: {msg}",
872        );
873        assert!(
874            msg.contains("searched"),
875            "empty-run error must name the searched directory: {msg}",
876        );
877        assert!(
878            msg.contains(&run_dir.display().to_string()),
879            "empty-run error must include the resolved run_dir path \
880             ({}): {msg}",
881            run_dir.display(),
882        );
883    }
884
885    /// All-corrupt run is NOT a hard error — text rendering surfaces
886    /// every parse failure under the trailing `corrupt sidecars`
887    /// block; per-sidecar `test:` blocks must NOT appear.
888    #[test]
889    fn explain_sidecar_all_corrupt_renders_structured_diagnostic() {
890        let tmp = tempfile::tempdir().unwrap();
891        let run_dir = tmp.path().join("run-corrupt");
892        std::fs::create_dir(&run_dir).unwrap();
893        std::fs::write(run_dir.join("a-0000000000000000.ktstr.json"), "not json {").unwrap();
894        std::fs::write(
895            run_dir.join("b-0000000000000000.ktstr.json"),
896            "{\"missing\": \"required-fields\"}",
897        )
898        .unwrap();
899        let out = explain_sidecar("run-corrupt", Some(tmp.path()), false)
900            .expect("all-corrupt is no longer a hard error — must render");
901        assert!(
902            out.contains("walked 2"),
903            "header must name the walked count: {out}",
904        );
905        assert!(
906            out.contains("parsed 0 valid"),
907            "header must distinguish walked-vs-parsed (zero valid): {out}",
908        );
909        assert!(
910            out.contains("corrupt sidecars (2):"),
911            "all-corrupt run must surface the corrupt-sidecars \
912             block listing every parse failure: {out}",
913        );
914        assert!(
915            !out.contains("test:"),
916            "no sidecar parsed — must not emit any per-sidecar \
917             block: {out}",
918        );
919    }
920
921    /// Happy path: one fixture sidecar (every Option None). Text
922    /// output must list ALL eleven fields under "none fields" with
923    /// classifications + at least one cause string per entry.
924    #[test]
925    fn explain_sidecar_text_lists_all_none_fields_for_fixture() {
926        let tmp = tempfile::tempdir().unwrap();
927        let run_dir = tmp.path().join("run-all-none");
928        std::fs::create_dir(&run_dir).unwrap();
929        let sc = crate::test_support::SidecarResult::test_fixture();
930        std::fs::write(
931            run_dir.join("t-0000000000000000.ktstr.json"),
932            serde_json::to_string(&sc).unwrap(),
933        )
934        .unwrap();
935        let out = explain_sidecar("run-all-none", Some(tmp.path()), false).unwrap();
936        assert!(out.contains("walked 1"), "header must report walked: {out}");
937        assert!(out.contains("parsed 1"), "header must report parsed: {out}");
938        assert!(
939            out.contains("none fields (11)"),
940            "fixture has every Option as None — count must be 11: {out}",
941        );
942        for entry in SIDECAR_NONE_CATALOG {
943            assert!(
944                out.contains(entry.field),
945                "output must mention field {}: {out}",
946                entry.field,
947            );
948        }
949        assert!(
950            out.contains("[expected]"),
951            "expected-class fields must surface their tag: {out}",
952        );
953        assert!(
954            out.contains("[actionable]"),
955            "actionable-class fields must surface their tag: {out}",
956        );
957        let project_commit_fix = SIDECAR_NONE_CATALOG
958            .iter()
959            .find(|e| e.field == "project_commit")
960            .and_then(|e| e.fix)
961            .expect("project_commit has a single concrete recovery action and must carry a fix");
962        assert!(
963            out.contains(&format!("fix: {project_commit_fix}")),
964            "project_commit's fix: line must render its catalog \
965             prose verbatim ({project_commit_fix:?}): {out}",
966        );
967        let fix_line_count = out.matches("\n      fix:").count();
968        let expected_fix_count = SIDECAR_NONE_CATALOG
969            .iter()
970            .filter(|e| e.fix.is_some())
971            .count();
972        assert_eq!(
973            fix_line_count, expected_fix_count,
974            "exactly {expected_fix_count} entries carry a fix: in \
975             the catalog; output emitted {fix_line_count}: {out}",
976        );
977    }
978
979    /// JSON shape: aggregate per-field with `none_count`,
980    /// `some_count`, `classification`, `causes`, `fix`. With one
981    /// fixture sidecar (every Option None), every field reports
982    /// none_count=1, some_count=0, and the two sum to _walk.valid.
983    #[test]
984    fn explain_sidecar_json_shape_aggregates_none_counts() {
985        let tmp = tempfile::tempdir().unwrap();
986        let run_dir = tmp.path().join("run-json");
987        std::fs::create_dir(&run_dir).unwrap();
988        let sc = crate::test_support::SidecarResult::test_fixture();
989        std::fs::write(
990            run_dir.join("t-0000000000000000.ktstr.json"),
991            serde_json::to_string(&sc).unwrap(),
992        )
993        .unwrap();
994        let out = explain_sidecar("run-json", Some(tmp.path()), true).unwrap();
995        let parsed: serde_json::Value =
996            serde_json::from_str(&out).expect("json output must round-trip parse");
997        let walk = parsed.get("_walk").expect("must have _walk key");
998        assert_eq!(walk.get("walked").and_then(|v| v.as_u64()), Some(1));
999        assert_eq!(walk.get("valid").and_then(|v| v.as_u64()), Some(1));
1000        let fields = parsed.get("fields").expect("must have fields key");
1001        for entry in SIDECAR_NONE_CATALOG {
1002            let f = fields
1003                .get(entry.field)
1004                .unwrap_or_else(|| panic!("missing field {}", entry.field));
1005            let none_count = f
1006                .get("none_count")
1007                .and_then(|v| v.as_u64())
1008                .unwrap_or_else(|| panic!("missing none_count for {}", entry.field));
1009            let some_count = f
1010                .get("some_count")
1011                .and_then(|v| v.as_u64())
1012                .unwrap_or_else(|| panic!("missing some_count for {}", entry.field));
1013            assert_eq!(
1014                none_count, 1,
1015                "fixture: none_count must be 1 for {}",
1016                entry.field
1017            );
1018            assert_eq!(
1019                some_count, 0,
1020                "fixture: some_count must be 0 for {}",
1021                entry.field
1022            );
1023            assert_eq!(
1024                none_count + some_count,
1025                1,
1026                "sum invariant for {}",
1027                entry.field
1028            );
1029            assert_eq!(
1030                f.get("classification").and_then(|v| v.as_str()),
1031                Some(entry.classification.as_str()),
1032                "classification must round-trip for {}",
1033                entry.field,
1034            );
1035            let causes = f
1036                .get("causes")
1037                .and_then(|v| v.as_array())
1038                .unwrap_or_else(|| panic!("missing causes for {}", entry.field));
1039            assert_eq!(
1040                causes.len(),
1041                entry.causes.len(),
1042                "causes array length must match catalog for {}",
1043                entry.field,
1044            );
1045            let fix_value = f
1046                .get("fix")
1047                .unwrap_or_else(|| panic!("missing fix for {}", entry.field));
1048            match entry.fix {
1049                Some(expected) => {
1050                    assert_eq!(
1051                        fix_value.as_str(),
1052                        Some(expected),
1053                        "fix string must round-trip for {}",
1054                        entry.field,
1055                    );
1056                }
1057                None => {
1058                    assert!(
1059                        fix_value.is_null(),
1060                        "fix must be JSON null for fix=None entry {}: \
1061                         got {fix_value:?}",
1062                        entry.field,
1063                    );
1064                }
1065            }
1066        }
1067    }
1068
1069    /// JSON `overcommit_runs` counts only sidecars whose host CPU
1070    /// budget was below their vCPU count; skip rows (budget 0) and
1071    /// roomy/exact runs are excluded — the JSON-path mirror of the
1072    /// text OVERCOMMIT marker.
1073    #[test]
1074    fn explain_sidecar_json_counts_overcommitted_runs() {
1075        let tmp = tempfile::tempdir().unwrap();
1076        let run_dir = tmp.path().join("run-budget");
1077        std::fs::create_dir(&run_dir).unwrap();
1078        let mk = |name: &str, prefix: &str, budget: u32, vcpus: u32| {
1079            let mut sc = crate::test_support::SidecarResult::test_fixture();
1080            sc.test_name = name.to_string();
1081            sc.cpu_budget = budget;
1082            sc.vcpus = vcpus;
1083            std::fs::write(
1084                run_dir.join(format!("{prefix}-0000000000000000.ktstr.json")),
1085                serde_json::to_string(&sc).unwrap(),
1086            )
1087            .unwrap();
1088        };
1089        mk("over", "a", 4, 16); // overcommit -> counted
1090        mk("exact", "b", 16, 16); // boundary -> not counted
1091        mk("roomy", "c", 32, 16); // roomy -> not counted
1092        mk("skip", "d", 0, 0); // skip -> not counted
1093        let out = explain_sidecar("run-budget", Some(tmp.path()), true).unwrap();
1094        let parsed: serde_json::Value =
1095            serde_json::from_str(&out).expect("json output must round-trip parse");
1096        assert_eq!(
1097            parsed.get("overcommit_runs").and_then(|v| v.as_u64()),
1098            Some(1),
1099            "only the 4/16 run is overcommitted: {out}",
1100        );
1101    }
1102
1103    /// Mixed populated/None: text output splits "populated optional
1104    /// fields (N)" from "none fields (M)" with the right counts.
1105    #[test]
1106    fn explain_sidecar_text_distinguishes_populated_from_none() {
1107        let tmp = tempfile::tempdir().unwrap();
1108        let run_dir = tmp.path().join("run-mixed");
1109        std::fs::create_dir(&run_dir).unwrap();
1110        let mut sc = crate::test_support::SidecarResult::test_fixture();
1111        sc.payload = Some("ipc_pingpong".to_string());
1112        sc.kernel_version = Some("6.14.2".to_string());
1113        sc.run_source = Some("local".to_string());
1114        std::fs::write(
1115            run_dir.join("t-0000000000000000.ktstr.json"),
1116            serde_json::to_string(&sc).unwrap(),
1117        )
1118        .unwrap();
1119        let out = explain_sidecar("run-mixed", Some(tmp.path()), false).unwrap();
1120        assert!(
1121            out.contains("populated optional fields (3)"),
1122            "must report 3 populated: {out}",
1123        );
1124        assert!(
1125            out.contains("payload"),
1126            "populated `payload` must appear: {out}",
1127        );
1128        assert!(out.contains("none fields (8)"), "must report 8 None: {out}",);
1129    }
1130
1131    /// Per-sidecar text output: `arch:` line surfaces under each
1132    /// sidecar's block, sourced from `host.arch`.
1133    #[test]
1134    fn explain_sidecar_text_renders_arch_line() {
1135        let tmp = tempfile::tempdir().unwrap();
1136        let run_dir = tmp.path().join("run-arch");
1137        std::fs::create_dir(&run_dir).unwrap();
1138        let mut sc = crate::test_support::SidecarResult::test_fixture();
1139        sc.host = Some(crate::host_context::HostContext::test_fixture());
1140        std::fs::write(
1141            run_dir.join("t-0000000000000000.ktstr.json"),
1142            serde_json::to_string(&sc).unwrap(),
1143        )
1144        .unwrap();
1145        let out = explain_sidecar("run-arch", Some(tmp.path()), false).unwrap();
1146        assert!(
1147            out.contains("arch: x86_64"),
1148            "host-populated sidecar must surface `arch: x86_64`: {out}",
1149        );
1150    }
1151
1152    /// Per-sidecar text output: when `host` is `None`, `arch:` line
1153    /// still emits with `-` sentinel for uniform shape.
1154    #[test]
1155    fn explain_sidecar_text_arch_line_falls_back_to_dash_when_host_none() {
1156        let tmp = tempfile::tempdir().unwrap();
1157        let run_dir = tmp.path().join("run-arch-none");
1158        std::fs::create_dir(&run_dir).unwrap();
1159        let sc = crate::test_support::SidecarResult::test_fixture();
1160        std::fs::write(
1161            run_dir.join("t-0000000000000000.ktstr.json"),
1162            serde_json::to_string(&sc).unwrap(),
1163        )
1164        .unwrap();
1165        let out = explain_sidecar("run-arch-none", Some(tmp.path()), false).unwrap();
1166        assert!(
1167            out.contains("arch: -"),
1168            "host-None sidecar must surface `arch: -`: {out}",
1169        );
1170    }
1171
1172    /// Per-sidecar text output: the cpu_budget line renders all four
1173    /// branches — skip (budget 0), overcommit (budget < vcpus), normal
1174    /// (budget >= vcpus), and the malformed budget>0/vcpus=0 case — and
1175    /// the overcommit marker names run-delay as confounded (locks in
1176    /// the kernel-grounded "run_delay is steal-inflated" semantics so a
1177    /// regression that dropped it back to "only partial" is caught).
1178    #[test]
1179    fn explain_sidecar_text_renders_cpu_budget_marker_branches() {
1180        let tmp = tempfile::tempdir().unwrap();
1181        let render = |name: &str, budget: u32, vcpus: u32| -> String {
1182            let run_dir = tmp.path().join(name);
1183            std::fs::create_dir(&run_dir).unwrap();
1184            let mut sc = crate::test_support::SidecarResult::test_fixture();
1185            sc.cpu_budget = budget;
1186            sc.vcpus = vcpus;
1187            std::fs::write(
1188                run_dir.join("t-0000000000000000.ktstr.json"),
1189                serde_json::to_string(&sc).unwrap(),
1190            )
1191            .unwrap();
1192            explain_sidecar(name, Some(tmp.path()), false).unwrap()
1193        };
1194
1195        let skip = render("run-skip", 0, 0);
1196        assert!(
1197            skip.contains("cpu_budget: - (skip; VM not booted)"),
1198            "budget 0 must render the skip sentinel: {skip}",
1199        );
1200        assert!(
1201            !skip.contains("OVERCOMMIT"),
1202            "skip row must not be flagged overcommit: {skip}",
1203        );
1204
1205        let over = render("run-over", 4, 16);
1206        assert!(
1207            over.contains("cpu_budget: 4 / 16 vcpus") && over.contains("[OVERCOMMIT"),
1208            "budget < vcpus must render the raw values AND the OVERCOMMIT marker: {over}",
1209        );
1210        assert!(
1211            over.contains("run-delay"),
1212            "OVERCOMMIT marker must list run-delay as confounded (steal-inflated): {over}",
1213        );
1214
1215        let exact = render("run-exact", 16, 16);
1216        assert!(
1217            exact.contains("cpu_budget: 16 / 16 vcpus") && !exact.contains("OVERCOMMIT"),
1218            "budget == vcpus is the boundary: plain line, no marker: {exact}",
1219        );
1220
1221        let roomy = render("run-roomy", 32, 16);
1222        assert!(
1223            roomy.contains("cpu_budget: 32 / 16 vcpus") && !roomy.contains("OVERCOMMIT"),
1224            "budget > vcpus must render plain, no marker: {roomy}",
1225        );
1226
1227        let malformed = render("run-malformed", 8, 0);
1228        assert!(
1229            malformed.contains("[malformed") && !malformed.contains("OVERCOMMIT"),
1230            "budget>0 with vcpus==0 must render the malformed note, not overcommit: {malformed}",
1231        );
1232    }
1233
1234    /// The OVERCOMMIT marker cites the MEASURED host dilation D when the
1235    /// sidecar recorded one, and omits the clause when it did not (host
1236    /// without CONFIG_SCHEDSTATS / pre-field sidecar → `host_dilation`
1237    /// None). Both flavors still render the base OVERCOMMIT marker.
1238    #[test]
1239    fn explain_sidecar_overcommit_cites_measured_dilation() {
1240        let tmp = tempfile::tempdir().unwrap();
1241        let render = |name: &str, dilation: Option<f64>| -> String {
1242            let run_dir = tmp.path().join(name);
1243            std::fs::create_dir(&run_dir).unwrap();
1244            let mut sc = crate::test_support::SidecarResult::test_fixture();
1245            sc.cpu_budget = 4;
1246            sc.vcpus = 16;
1247            sc.host_dilation = dilation;
1248            std::fs::write(
1249                run_dir.join("t-0000000000000000.ktstr.json"),
1250                serde_json::to_string(&sc).unwrap(),
1251            )
1252            .unwrap();
1253            explain_sidecar(name, Some(tmp.path()), false).unwrap()
1254        };
1255
1256        let with_d = render("run-d", Some(3.0));
1257        assert!(
1258            with_d.contains("[OVERCOMMIT") && with_d.contains("measured host dilation D=3.00x"),
1259            "OVERCOMMIT marker must cite the measured D when present: {with_d}",
1260        );
1261
1262        let without_d = render("run-nod", None);
1263        assert!(
1264            without_d.contains("[OVERCOMMIT") && !without_d.contains("measured host dilation"),
1265            "OVERCOMMIT marker must omit the D clause when host_dilation is None: {without_d}",
1266        );
1267    }
1268
1269    /// Per-cgroup CPU placement renders on the text path for any run.
1270    /// A labeled cgroup with a non-empty cpus_used prints a
1271    /// "cgroup <name>: ran on cpus [..]" line; a cgroup that recorded no
1272    /// CPU (empty cpus_used) is skipped.
1273    #[test]
1274    fn explain_sidecar_text_renders_per_cgroup_cpus() {
1275        let tmp = tempfile::tempdir().unwrap();
1276        let run_dir = tmp.path().join("run-cgcpus");
1277        std::fs::create_dir(&run_dir).unwrap();
1278        let mut sc = crate::test_support::SidecarResult::test_fixture();
1279        sc.stats.cgroups = vec![
1280            crate::assert::CgroupStats {
1281                cgroup_name: "cg_a".to_string(),
1282                cpus_used: [0usize, 1].into_iter().collect(),
1283                num_cpus: 2,
1284                ..Default::default()
1285            },
1286            crate::assert::CgroupStats {
1287                cgroup_name: "cg_idle".to_string(),
1288                cpus_used: std::collections::BTreeSet::new(),
1289                ..Default::default()
1290            },
1291        ];
1292        std::fs::write(
1293            run_dir.join("t-0000000000000000.ktstr.json"),
1294            serde_json::to_string(&sc).unwrap(),
1295        )
1296        .unwrap();
1297        let out = explain_sidecar("run-cgcpus", Some(tmp.path()), false).unwrap();
1298        assert!(
1299            out.contains("cgroup cg_a: ran on cpus [0,1]"),
1300            "labeled cgroup must render its cpus_used: {out}",
1301        );
1302        assert!(
1303            !out.contains("cg_idle"),
1304            "a cgroup with empty cpus_used must be skipped: {out}",
1305        );
1306    }
1307
1308    /// Per-sidecar text output: two sidecars in the same run with
1309    /// different None patterns must each get their own block.
1310    #[test]
1311    fn explain_sidecar_text_emits_one_block_per_sidecar() {
1312        let tmp = tempfile::tempdir().unwrap();
1313        let run_dir = tmp.path().join("run-two");
1314        std::fs::create_dir(&run_dir).unwrap();
1315        let mut a = crate::test_support::SidecarResult::test_fixture();
1316        a.test_name = "test_a".to_string();
1317        let mut b = crate::test_support::SidecarResult::test_fixture();
1318        b.test_name = "test_b".to_string();
1319        b.payload = Some("ipc_pingpong".to_string());
1320        std::fs::write(
1321            run_dir.join("a-0000000000000000.ktstr.json"),
1322            serde_json::to_string(&a).unwrap(),
1323        )
1324        .unwrap();
1325        std::fs::write(
1326            run_dir.join("b-0000000000000000.ktstr.json"),
1327            serde_json::to_string(&b).unwrap(),
1328        )
1329        .unwrap();
1330        let out = explain_sidecar("run-two", Some(tmp.path()), false).unwrap();
1331        assert!(out.contains("test: test_a"), "test_a block missing: {out}");
1332        assert!(out.contains("test: test_b"), "test_b block missing: {out}");
1333        assert!(out.contains("walked 2"), "walked count must be 2: {out}");
1334        assert!(out.contains("parsed 2"), "parsed count must be 2: {out}");
1335    }
1336
1337    /// JSON aggregation across multiple sidecars: partial None on a
1338    /// per-field basis surfaces both none_count and some_count.
1339    #[test]
1340    fn explain_sidecar_json_aggregates_partial_none_correctly() {
1341        let tmp = tempfile::tempdir().unwrap();
1342        let run_dir = tmp.path().join("run-partial");
1343        std::fs::create_dir(&run_dir).unwrap();
1344        let a = crate::test_support::SidecarResult::test_fixture();
1345        let mut b = crate::test_support::SidecarResult::test_fixture();
1346        b.payload = Some("ipc_pingpong".to_string());
1347        std::fs::write(
1348            run_dir.join("a-0000000000000000.ktstr.json"),
1349            serde_json::to_string(&a).unwrap(),
1350        )
1351        .unwrap();
1352        std::fs::write(
1353            run_dir.join("b-0000000000000000.ktstr.json"),
1354            serde_json::to_string(&b).unwrap(),
1355        )
1356        .unwrap();
1357        let out = explain_sidecar("run-partial", Some(tmp.path()), true).unwrap();
1358        let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
1359        let payload = parsed
1360            .get("fields")
1361            .and_then(|f| f.get("payload"))
1362            .expect("payload field must be present");
1363        assert_eq!(payload.get("none_count").and_then(|v| v.as_u64()), Some(1));
1364        assert_eq!(payload.get("some_count").and_then(|v| v.as_u64()), Some(1));
1365        let host = parsed
1366            .get("fields")
1367            .and_then(|f| f.get("host"))
1368            .expect("host field must be present");
1369        assert_eq!(host.get("none_count").and_then(|v| v.as_u64()), Some(2));
1370        assert_eq!(host.get("some_count").and_then(|v| v.as_u64()), Some(0));
1371    }
1372
1373    /// Walker counts both valid and corrupt `.ktstr.json` files.
1374    #[test]
1375    fn explain_sidecar_walks_corrupt_files_into_count() {
1376        let tmp = tempfile::tempdir().unwrap();
1377        let run_dir = tmp.path().join("run-mixed-parse");
1378        std::fs::create_dir(&run_dir).unwrap();
1379        let valid = crate::test_support::SidecarResult::test_fixture();
1380        std::fs::write(
1381            run_dir.join("a-0000000000000000.ktstr.json"),
1382            serde_json::to_string(&valid).unwrap(),
1383        )
1384        .unwrap();
1385        std::fs::write(run_dir.join("b-0000000000000000.ktstr.json"), "garbage{").unwrap();
1386        let out = explain_sidecar("run-mixed-parse", Some(tmp.path()), false).unwrap();
1387        assert!(
1388            out.contains("walked 2"),
1389            "walker must visit both files: {out}"
1390        );
1391        assert!(
1392            out.contains("parsed 1"),
1393            "only the valid file parses: {out}"
1394        );
1395    }
1396
1397    /// Walker recurses one level into subdirectories.
1398    #[test]
1399    fn explain_sidecar_walks_one_level_subdirectory() {
1400        let tmp = tempfile::tempdir().unwrap();
1401        let run_dir = tmp.path().join("run-sub");
1402        let sub = run_dir.join("job-x");
1403        std::fs::create_dir_all(&sub).unwrap();
1404        let sc = crate::test_support::SidecarResult::test_fixture();
1405        std::fs::write(
1406            sub.join("t-0000000000000000.ktstr.json"),
1407            serde_json::to_string(&sc).unwrap(),
1408        )
1409        .unwrap();
1410        let out = explain_sidecar("run-sub", Some(tmp.path()), false).unwrap();
1411        assert!(out.contains("walked 1"), "must walk into job-x: {out}");
1412        assert!(
1413            out.contains("parsed 1"),
1414            "must parse the nested file: {out}"
1415        );
1416    }
1417
1418    /// Walker MUST ignore non-`.ktstr.json` files.
1419    #[test]
1420    fn explain_sidecar_ignores_non_ktstr_json() {
1421        let tmp = tempfile::tempdir().unwrap();
1422        let run_dir = tmp.path().join("run-with-other-json");
1423        std::fs::create_dir(&run_dir).unwrap();
1424        let sc = crate::test_support::SidecarResult::test_fixture();
1425        std::fs::write(
1426            run_dir.join("t-0000000000000000.ktstr.json"),
1427            serde_json::to_string(&sc).unwrap(),
1428        )
1429        .unwrap();
1430        std::fs::write(run_dir.join("metadata.json"), "{}").unwrap();
1431        let out = explain_sidecar("run-with-other-json", Some(tmp.path()), false).unwrap();
1432        assert!(
1433            out.contains("walked 1"),
1434            "non-ktstr JSON must not inflate the walked count: {out}",
1435        );
1436    }
1437
1438    /// JSON output must be a single valid JSON document.
1439    #[test]
1440    fn explain_sidecar_json_is_valid_document() {
1441        let tmp = tempfile::tempdir().unwrap();
1442        let run_dir = tmp.path().join("run-roundtrip");
1443        std::fs::create_dir(&run_dir).unwrap();
1444        let sc = crate::test_support::SidecarResult::test_fixture();
1445        std::fs::write(
1446            run_dir.join("t-0000000000000000.ktstr.json"),
1447            serde_json::to_string(&sc).unwrap(),
1448        )
1449        .unwrap();
1450        let out = explain_sidecar("run-roundtrip", Some(tmp.path()), true).unwrap();
1451        let _: serde_json::Value = serde_json::from_str(&out).expect("output must be valid JSON");
1452    }
1453
1454    /// Partial population: 7 of 11 Options populated; report shows
1455    /// "populated optional fields (7)" + "none fields (4)".
1456    #[test]
1457    fn explain_sidecar_text_handles_partial_population() {
1458        let tmp = tempfile::tempdir().unwrap();
1459        let run_dir = tmp.path().join("run-partial-pop");
1460        std::fs::create_dir(&run_dir).unwrap();
1461        let mut sc = crate::test_support::SidecarResult::test_fixture();
1462        sc.scheduler_commit = Some("aaaa111".to_string());
1463        sc.project_commit = Some("bbbb222".to_string());
1464        sc.payload = Some("payload".to_string());
1465        sc.kernel_version = Some("6.14.2".to_string());
1466        sc.kernel_commit = Some("cccc333".to_string());
1467        sc.cleanup_duration_ms = Some(123);
1468        sc.run_source = Some("local".to_string());
1469        std::fs::write(
1470            run_dir.join("t-0000000000000000.ktstr.json"),
1471            serde_json::to_string(&sc).unwrap(),
1472        )
1473        .unwrap();
1474        let out = explain_sidecar("run-partial-pop", Some(tmp.path()), false).unwrap();
1475        assert!(
1476            out.contains("populated optional fields (7)"),
1477            "7 of 11 Options populated must be reflected in the count: {out}",
1478        );
1479        assert!(
1480            out.contains("none fields (4)"),
1481            "4 of 11 Options remain None — must report (4): {out}",
1482        );
1483    }
1484
1485    /// Classification labels are stable strings.
1486    #[test]
1487    fn none_classification_as_str_returns_stable_tokens() {
1488        assert_eq!(NoneClassification::Expected.as_str(), "expected");
1489        assert_eq!(NoneClassification::Actionable.as_str(), "actionable");
1490    }
1491
1492    /// `kernel_commit` rustdoc enumerates 5 None causes; catalog
1493    /// must mirror that.
1494    #[test]
1495    fn kernel_commit_catalog_lists_five_causes() {
1496        let entry = SIDECAR_NONE_CATALOG
1497            .iter()
1498            .find(|e| e.field == "kernel_commit")
1499            .expect("kernel_commit must be in the catalog");
1500        assert_eq!(
1501            entry.causes.len(),
1502            5,
1503            "kernel_commit rustdoc enumerates 5 None causes; catalog \
1504             must mirror that",
1505        );
1506    }
1507
1508    /// Schema version stamp is "2".
1509    #[test]
1510    fn explain_sidecar_schema_version_constant_is_two() {
1511        assert_eq!(EXPLAIN_SIDECAR_SCHEMA_VERSION, "2");
1512    }
1513
1514    /// JSON output stamps `_schema_version` at top level.
1515    #[test]
1516    fn explain_sidecar_json_includes_schema_version() {
1517        let tmp = tempfile::tempdir().unwrap();
1518        let run_dir = tmp.path().join("run-schema");
1519        std::fs::create_dir(&run_dir).unwrap();
1520        let sc = crate::test_support::SidecarResult::test_fixture();
1521        std::fs::write(
1522            run_dir.join("t-0000000000000000.ktstr.json"),
1523            serde_json::to_string(&sc).unwrap(),
1524        )
1525        .unwrap();
1526        let out = explain_sidecar("run-schema", Some(tmp.path()), true).unwrap();
1527        let parsed: serde_json::Value =
1528            serde_json::from_str(&out).expect("json output must round-trip parse");
1529        assert_eq!(
1530            parsed.get("_schema_version").and_then(|v| v.as_str()),
1531            Some(EXPLAIN_SIDECAR_SCHEMA_VERSION),
1532            "JSON output must stamp _schema_version: {out}",
1533        );
1534    }
1535
1536    /// JSON `_walk.errors` is empty array on the all-clean path.
1537    #[test]
1538    fn explain_sidecar_json_walk_errors_empty_when_all_valid() {
1539        let tmp = tempfile::tempdir().unwrap();
1540        let run_dir = tmp.path().join("run-clean-walk");
1541        std::fs::create_dir(&run_dir).unwrap();
1542        let sc = crate::test_support::SidecarResult::test_fixture();
1543        std::fs::write(
1544            run_dir.join("t-0000000000000000.ktstr.json"),
1545            serde_json::to_string(&sc).unwrap(),
1546        )
1547        .unwrap();
1548        let out = explain_sidecar("run-clean-walk", Some(tmp.path()), true).unwrap();
1549        let parsed: serde_json::Value =
1550            serde_json::from_str(&out).expect("json output must round-trip parse");
1551        let errors = parsed
1552            .get("_walk")
1553            .and_then(|w| w.get("errors"))
1554            .and_then(|e| e.as_array())
1555            .expect("_walk.errors must be a JSON array");
1556        assert!(
1557            errors.is_empty(),
1558            "no parse failures — _walk.errors must be empty: {out}",
1559        );
1560    }
1561
1562    /// JSON `_walk.errors` lists `{path, error, enriched_message}`
1563    /// triples for every parse failure. enriched_message is null
1564    /// for generic parse failures.
1565    #[test]
1566    fn explain_sidecar_json_walk_errors_lists_corrupt_files() {
1567        let tmp = tempfile::tempdir().unwrap();
1568        let run_dir = tmp.path().join("run-mixed-errs-json");
1569        std::fs::create_dir(&run_dir).unwrap();
1570        let valid = crate::test_support::SidecarResult::test_fixture();
1571        std::fs::write(
1572            run_dir.join("a-0000000000000000.ktstr.json"),
1573            serde_json::to_string(&valid).unwrap(),
1574        )
1575        .unwrap();
1576        let corrupt_path = run_dir.join("b-0000000000000000.ktstr.json");
1577        std::fs::write(&corrupt_path, "garbage{").unwrap();
1578        let out = explain_sidecar("run-mixed-errs-json", Some(tmp.path()), true).unwrap();
1579        let parsed: serde_json::Value =
1580            serde_json::from_str(&out).expect("json output must round-trip parse");
1581        let walk = parsed.get("_walk").expect("must have _walk key");
1582        assert_eq!(walk.get("walked").and_then(|v| v.as_u64()), Some(2));
1583        assert_eq!(walk.get("valid").and_then(|v| v.as_u64()), Some(1));
1584        let errors = walk
1585            .get("errors")
1586            .and_then(|e| e.as_array())
1587            .expect("_walk.errors must be a JSON array");
1588        assert_eq!(errors.len(), 1);
1589        let entry = &errors[0];
1590        let path = entry.get("path").and_then(|v| v.as_str()).expect("path");
1591        assert_eq!(path, corrupt_path.display().to_string());
1592        let error = entry.get("error").and_then(|v| v.as_str()).expect("error");
1593        assert!(!error.is_empty());
1594        let enriched = entry
1595            .get("enriched_message")
1596            .expect("each error entry must carry an enriched_message key");
1597        assert!(
1598            enriched.is_null(),
1599            "generic parse failure has no schema-drift remediation; \
1600             enriched_message must be JSON null: {enriched:?}",
1601        );
1602    }
1603
1604    /// `enriched_parse_error_message` returns operator-facing prose
1605    /// for the host-missing schema-drift pattern.
1606    #[test]
1607    fn enriched_parse_error_message_returns_prose_for_host_missing_pattern() {
1608        let raw = "missing field `host` at line 1 column 100";
1609        let path = std::path::Path::new("/tmp/example-run/sidecar.ktstr.json");
1610        let enriched = crate::test_support::enriched_parse_error_message_for_test(path, raw)
1611            .expect("host-missing pattern must produce enrichment prose");
1612        assert!(
1613            enriched.contains("host"),
1614            "enrichment must mention host: {enriched}"
1615        );
1616        assert!(
1617            enriched.contains("re-run"),
1618            "enrichment must point at the re-run remediation: {enriched}",
1619        );
1620        assert!(
1621            enriched.contains("disposable-sidecar"),
1622            "enrichment must reference the pre-1.0 disposable-sidecar \
1623             policy: {enriched}",
1624        );
1625        let raw_generic = "expected ident at line 1 column 2";
1626        let no_enrichment =
1627            crate::test_support::enriched_parse_error_message_for_test(path, raw_generic);
1628        assert!(
1629            no_enrichment.is_none(),
1630            "generic parse error must produce no enrichment"
1631        );
1632    }
1633
1634    /// All-corrupt run renders structured JSON with `valid: 0`,
1635    /// every parse failure under `_walk.errors`, and every field's
1636    /// counts at zero.
1637    #[test]
1638    fn explain_sidecar_all_corrupt_json_renders_structured_diagnostic() {
1639        let tmp = tempfile::tempdir().unwrap();
1640        let run_dir = tmp.path().join("run-all-corrupt-json");
1641        std::fs::create_dir(&run_dir).unwrap();
1642        std::fs::write(run_dir.join("a-0000000000000000.ktstr.json"), "{").unwrap();
1643        std::fs::write(run_dir.join("b-0000000000000000.ktstr.json"), "garbage{").unwrap();
1644        let out = explain_sidecar("run-all-corrupt-json", Some(tmp.path()), true)
1645            .expect("all-corrupt JSON must render, not bail");
1646        let parsed: serde_json::Value =
1647            serde_json::from_str(&out).expect("json output must round-trip parse");
1648        let walk = parsed.get("_walk").expect("must have _walk key");
1649        assert_eq!(walk.get("walked").and_then(|v| v.as_u64()), Some(2));
1650        assert_eq!(
1651            walk.get("valid").and_then(|v| v.as_u64()),
1652            Some(0),
1653            "all-corrupt run must report valid=0: {out}",
1654        );
1655        let errors = walk
1656            .get("errors")
1657            .and_then(|e| e.as_array())
1658            .expect("_walk.errors must be present");
1659        assert_eq!(errors.len(), 2);
1660        let fields = parsed
1661            .get("fields")
1662            .and_then(|f| f.as_object())
1663            .expect("fields must be present");
1664        for entry in SIDECAR_NONE_CATALOG {
1665            let f = fields
1666                .get(entry.field)
1667                .unwrap_or_else(|| panic!("field {} must be present", entry.field));
1668            assert_eq!(f.get("none_count").and_then(|v| v.as_u64()), Some(0));
1669            assert_eq!(f.get("some_count").and_then(|v| v.as_u64()), Some(0));
1670        }
1671        assert_eq!(
1672            parsed.get("_schema_version").and_then(|v| v.as_str()),
1673            Some(EXPLAIN_SIDECAR_SCHEMA_VERSION),
1674        );
1675    }
1676
1677    /// Generic parse failures emit `error:` line in corrupt block
1678    /// but NOT `enriched:` line.
1679    #[test]
1680    fn explain_sidecar_text_omits_enriched_line_for_generic_failure() {
1681        let (tmp, run_dir) = make_test_run("run-generic-fail-text");
1682        write_corrupt_sidecar(&run_dir, "a-0000000000000000", "garbage{");
1683        let out = explain_sidecar("run-generic-fail-text", Some(tmp.path()), false).unwrap();
1684        assert!(
1685            out.contains("corrupt sidecars (1):"),
1686            "generic parse failure must surface in the corrupt block: {out}",
1687        );
1688        assert!(
1689            out.contains("    error:"),
1690            "generic parse failure must emit raw `error:` line: {out}",
1691        );
1692        assert!(
1693            !out.contains("    enriched:"),
1694            "generic parse failure has no enrichment — `enriched:` \
1695             line must NOT appear: {out}",
1696        );
1697    }
1698
1699    /// Text output appends trailing `corrupt sidecars (N):` block
1700    /// with positional invariant: header → per-sidecar blocks →
1701    /// trailing corrupt block.
1702    #[test]
1703    fn explain_sidecar_text_appends_corrupt_sidecars_block() {
1704        let tmp = tempfile::tempdir().unwrap();
1705        let run_dir = tmp.path().join("run-text-corrupt");
1706        std::fs::create_dir(&run_dir).unwrap();
1707        let mut valid = crate::test_support::SidecarResult::test_fixture();
1708        valid.test_name = "valid_test".to_string();
1709        std::fs::write(
1710            run_dir.join("a-0000000000000000.ktstr.json"),
1711            serde_json::to_string(&valid).unwrap(),
1712        )
1713        .unwrap();
1714        let corrupt_path = run_dir.join("b-0000000000000000.ktstr.json");
1715        std::fs::write(&corrupt_path, "garbage{").unwrap();
1716        let out = explain_sidecar("run-text-corrupt", Some(tmp.path()), false).unwrap();
1717        assert!(out.contains("corrupt sidecars (1):"));
1718        assert!(out.contains(&corrupt_path.display().to_string()));
1719        assert!(out.contains("    error:"));
1720        let header_pos = out.find("walked 2 sidecar file(s)").unwrap();
1721        let test_block_pos = out.find("test: valid_test").unwrap();
1722        let corrupt_pos = out.find("corrupt sidecars (1):").unwrap();
1723        assert!(
1724            header_pos < test_block_pos,
1725            "header must precede per-sidecar blocks"
1726        );
1727        assert!(
1728            test_block_pos < corrupt_pos,
1729            "per-sidecar blocks must precede trailing corrupt block"
1730        );
1731    }
1732
1733    /// Corrupt-sidecars block is suppressed when zero parse failures.
1734    #[test]
1735    fn explain_sidecar_text_omits_corrupt_block_when_no_errors() {
1736        let (tmp, run_dir) = make_test_run("run-text-clean");
1737        let sc = crate::test_support::SidecarResult::test_fixture();
1738        write_sidecar(&run_dir, "t-0000000000000000", &sc);
1739        let out = explain_sidecar("run-text-clean", Some(tmp.path()), false).unwrap();
1740        assert!(
1741            !out.contains("corrupt sidecars"),
1742            "no parse failures — corrupt-sidecars block must be \
1743             suppressed: {out}",
1744        );
1745    }
1746
1747    /// Vec fields on `SidecarResult` (metrics, stimulus_events, etc.)
1748    /// are hard-required, NOT Option<T>. Catalog must NEVER name a
1749    /// Vec field as None.
1750    #[test]
1751    fn explain_sidecar_does_not_flag_empty_vec_fields_as_none() {
1752        let tmp = tempfile::tempdir().unwrap();
1753        let run_dir = tmp.path().join("run-vecs");
1754        std::fs::create_dir(&run_dir).unwrap();
1755        let mut sc = crate::test_support::SidecarResult::test_fixture();
1756        sc.scheduler_commit = Some("aaaa111".to_string());
1757        sc.project_commit = Some("bbbb222".to_string());
1758        sc.payload = Some("payload".to_string());
1759        sc.kernel_version = Some("6.14.2".to_string());
1760        sc.kernel_commit = Some("cccc333".to_string());
1761        sc.cleanup_duration_ms = Some(123);
1762        sc.run_source = Some("local".to_string());
1763        sc.resolve_source = Some("auto_built".to_string());
1764        sc.monitor = Some(crate::monitor::MonitorSummary::default());
1765        sc.kvm_stats = Some(crate::vmm::KvmStatsTotals::default());
1766        sc.host = Some(crate::host_context::HostContext::test_fixture());
1767        std::fs::write(
1768            run_dir.join("t-0000000000000000.ktstr.json"),
1769            serde_json::to_string(&sc).unwrap(),
1770        )
1771        .unwrap();
1772        let out = explain_sidecar("run-vecs", Some(tmp.path()), false).unwrap();
1773        assert!(
1774            out.contains("none fields: <all populated>"),
1775            "all Options populated — must report no None fields: {out}",
1776        );
1777        for vec_field in SIDECAR_VEC_FIELDS {
1778            assert!(
1779                !out.contains(vec_field),
1780                "Vec field '{vec_field}' is hard-required (not Option) and \
1781                 must never appear in explain-sidecar output: {out}",
1782            );
1783        }
1784    }
1785
1786    /// Pre-rename archive: on-disk `source` key → `run_source` None
1787    /// via serde tolerate-absence; diagnostic must surface with
1788    /// "rename" cause prose.
1789    #[test]
1790    fn explain_sidecar_handles_old_source_key_sidecar() {
1791        let tmp = tempfile::tempdir().unwrap();
1792        let run_dir = tmp.path().join("run-old-source-key");
1793        std::fs::create_dir(&run_dir).unwrap();
1794        let sc = crate::test_support::SidecarResult::test_fixture();
1795        let mut value = serde_json::to_value(&sc).expect("fixture must serialize");
1796        let obj = value.as_object_mut().expect("fixture is an Object");
1797        obj.remove("run_source");
1798        obj.insert(
1799            "source".to_string(),
1800            serde_json::Value::String("archive".to_string()),
1801        );
1802        std::fs::write(
1803            run_dir.join("t-0000000000000000.ktstr.json"),
1804            serde_json::to_string(&value).unwrap(),
1805        )
1806        .unwrap();
1807        let out = explain_sidecar("run-old-source-key", Some(tmp.path()), false).unwrap();
1808        assert!(
1809            out.contains("run_source"),
1810            "explain-sidecar must surface run_source as None for \
1811             pre-rename archive: {out}",
1812        );
1813        assert!(
1814            out.contains("rename"),
1815            "run_source None cause must mention the rename: {out}",
1816        );
1817    }
1818
1819    /// `dir=None` defaults to `runs_root` derived from `CARGO_TARGET_DIR`.
1820    #[test]
1821    fn explain_sidecar_resolves_dir_default_to_runs_root() {
1822        use crate::test_support::test_helpers::{EnvVarGuard, lock_env};
1823        let _lock = lock_env();
1824        let tmp = tempfile::tempdir().unwrap();
1825        let _env_target = EnvVarGuard::set("CARGO_TARGET_DIR", tmp.path());
1826        // Clear the orchestrator's KTSTR_RUNS_ROOT so runs_root() resolves
1827        // via CARGO_TARGET_DIR (the default this test pins).
1828        let _env_runs_root = EnvVarGuard::remove(crate::KTSTR_RUNS_ROOT_ENV);
1829        let _env_sidecar = EnvVarGuard::remove(crate::KTSTR_SIDECAR_DIR_ENV);
1830        let runs_root = tmp.path().join("ktstr");
1831        let run_dir = runs_root.join("run-default-root");
1832        std::fs::create_dir_all(&run_dir).unwrap();
1833        let sc = crate::test_support::SidecarResult::test_fixture();
1834        std::fs::write(
1835            run_dir.join("t-0000000000000000.ktstr.json"),
1836            serde_json::to_string(&sc).unwrap(),
1837        )
1838        .unwrap();
1839        let out = explain_sidecar("run-default-root", None, false)
1840            .expect("dir=None must resolve via runs_root() and succeed");
1841        assert!(out.contains("walked 1"));
1842        assert!(out.contains("parsed 1 valid"));
1843    }
1844
1845    /// 0-byte `.ktstr.json` is a parse failure (serde_json rejects
1846    /// empty input); walker counts it in walked + emits parse error.
1847    #[test]
1848    fn explain_sidecar_handles_zero_byte_file() {
1849        let tmp = tempfile::tempdir().unwrap();
1850        let run_dir = tmp.path().join("run-zero-byte");
1851        std::fs::create_dir(&run_dir).unwrap();
1852        let valid = crate::test_support::SidecarResult::test_fixture();
1853        std::fs::write(
1854            run_dir.join("a-0000000000000000.ktstr.json"),
1855            serde_json::to_string(&valid).unwrap(),
1856        )
1857        .unwrap();
1858        std::fs::write(run_dir.join("b-0000000000000000.ktstr.json"), "").unwrap();
1859        let out = explain_sidecar("run-zero-byte", Some(tmp.path()), false).unwrap();
1860        assert!(out.contains("walked 2"));
1861        assert!(out.contains("parsed 1"));
1862        assert!(
1863            out.contains("corrupt sidecars (1):"),
1864            "zero-byte file must surface in the corrupt-sidecars \
1865             block as a parse failure, not be silently dropped: {out}",
1866        );
1867    }
1868
1869    /// `SidecarResult` does NOT set `deny_unknown_fields`, so a
1870    /// future-schema sidecar must still deserialize cleanly.
1871    #[test]
1872    fn explain_sidecar_tolerates_unknown_extra_fields() {
1873        let tmp = tempfile::tempdir().unwrap();
1874        let run_dir = tmp.path().join("run-extra-fields");
1875        std::fs::create_dir(&run_dir).unwrap();
1876        let sc = crate::test_support::SidecarResult::test_fixture();
1877        let mut value = serde_json::to_value(&sc).expect("fixture must serialize");
1878        let obj = value.as_object_mut().expect("fixture is an Object");
1879        obj.insert(
1880            "future_field".to_string(),
1881            serde_json::Value::String("hypothetical".to_string()),
1882        );
1883        std::fs::write(
1884            run_dir.join("t-0000000000000000.ktstr.json"),
1885            serde_json::to_string(&value).unwrap(),
1886        )
1887        .unwrap();
1888        let out = explain_sidecar("run-extra-fields", Some(tmp.path()), false).unwrap();
1889        assert!(out.contains("walked 1"));
1890        assert!(out.contains("parsed 1 valid"));
1891        assert!(out.contains("test: t"));
1892    }
1893
1894    /// Per-field classification mapping is operator-visible as a
1895    /// stable tag. HashMap dedup guard catches catalog duplicate
1896    /// field names.
1897    #[test]
1898    fn explain_sidecar_classification_accuracy_per_field() {
1899        let by_field: std::collections::HashMap<&'static str, NoneClassification> =
1900            SIDECAR_NONE_CATALOG
1901                .iter()
1902                .map(|e| (e.field, e.classification))
1903                .collect();
1904        assert_eq!(
1905            by_field.len(),
1906            SIDECAR_NONE_CATALOG.len(),
1907            "SIDECAR_NONE_CATALOG must have unique `field` values \
1908             — HashMap collected {} entries, catalog has {}.",
1909            by_field.len(),
1910            SIDECAR_NONE_CATALOG.len(),
1911        );
1912        let expected_pairs: &[(&str, NoneClassification)] = &[
1913            ("scheduler_commit", NoneClassification::Expected),
1914            ("payload", NoneClassification::Expected),
1915            ("project_commit", NoneClassification::Actionable),
1916            ("monitor", NoneClassification::Actionable),
1917            ("kvm_stats", NoneClassification::Actionable),
1918            ("kernel_version", NoneClassification::Actionable),
1919            ("kernel_commit", NoneClassification::Actionable),
1920            ("host", NoneClassification::Actionable),
1921            ("cleanup_duration_ms", NoneClassification::Actionable),
1922            ("run_source", NoneClassification::Actionable),
1923            ("resolve_source", NoneClassification::Actionable),
1924        ];
1925        assert_eq!(
1926            expected_pairs.len(),
1927            SIDECAR_NONE_CATALOG.len(),
1928            "every catalog entry must have a pinned classification",
1929        );
1930        for (field, expected) in expected_pairs {
1931            let actual = by_field
1932                .get(field)
1933                .copied()
1934                .unwrap_or_else(|| panic!("catalog must contain field {field}"));
1935            assert_eq!(
1936                actual, *expected,
1937                "field {field}: classification mismatch — expected \
1938                 {expected:?}, got {actual:?}",
1939            );
1940        }
1941    }
1942
1943    /// IO failures (sidecar predicate matched but read_to_string
1944    /// failed) surface in the trailing `io errors` text block AND
1945    /// `_walk.io_errors` JSON array. Trigger via a directory named
1946    /// like a sidecar (read returns EISDIR).
1947    #[test]
1948    fn explain_sidecar_io_errors_surface_in_text_block_and_json() {
1949        let tmp = tempfile::tempdir().unwrap();
1950        let run_dir = tmp.path().join("run-io-err");
1951        std::fs::create_dir(&run_dir).unwrap();
1952        let sub = run_dir.join("sub");
1953        std::fs::create_dir(&sub).unwrap();
1954        std::fs::create_dir(sub.join("eisdir.ktstr.json")).unwrap();
1955
1956        let text_out = explain_sidecar("run-io-err", Some(tmp.path()), false).unwrap();
1957        assert!(text_out.contains("walked 1"));
1958        assert!(text_out.contains("parsed 0 valid"));
1959        assert!(text_out.contains("io errors (1):"));
1960        assert!(text_out.contains("eisdir.ktstr.json"));
1961        assert!(!text_out.contains("corrupt sidecars"));
1962
1963        let json_out = explain_sidecar("run-io-err", Some(tmp.path()), true).unwrap();
1964        let parsed: serde_json::Value =
1965            serde_json::from_str(&json_out).expect("json output must round-trip parse");
1966        let walk = parsed.get("_walk").expect("must have _walk");
1967        assert_eq!(walk.get("walked").and_then(|v| v.as_u64()), Some(1));
1968        assert_eq!(walk.get("valid").and_then(|v| v.as_u64()), Some(0));
1969        let parse_errs = walk.get("errors").and_then(|e| e.as_array()).unwrap();
1970        assert!(parse_errs.is_empty());
1971        let io_errs = walk.get("io_errors").and_then(|e| e.as_array()).unwrap();
1972        assert_eq!(io_errs.len(), 1);
1973        let entry = &io_errs[0];
1974        let path = entry.get("path").and_then(|v| v.as_str()).unwrap();
1975        assert!(path.ends_with("eisdir.ktstr.json"));
1976        let error = entry.get("error").and_then(|v| v.as_str()).unwrap();
1977        assert!(!error.is_empty());
1978        assert!(
1979            entry.get("enriched_message").is_none(),
1980            "io-error entries must NOT have enriched_message: {json_out}",
1981        );
1982    }
1983
1984    /// `walked == valid + errors.len() + io_errors.len()` — every
1985    /// predicate-matching file lands in exactly one bucket.
1986    #[test]
1987    fn explain_sidecar_walk_counts_reconcile_across_outcomes() {
1988        let tmp = tempfile::tempdir().unwrap();
1989        let run_dir = tmp.path().join("run-mixed-outcomes");
1990        std::fs::create_dir(&run_dir).unwrap();
1991        let valid = crate::test_support::SidecarResult::test_fixture();
1992        std::fs::write(
1993            run_dir.join("a-0000000000000000.ktstr.json"),
1994            serde_json::to_string(&valid).unwrap(),
1995        )
1996        .unwrap();
1997        std::fs::write(run_dir.join("b-0000000000000000.ktstr.json"), "garbage{").unwrap();
1998        let sub = run_dir.join("sub");
1999        std::fs::create_dir(&sub).unwrap();
2000        std::fs::create_dir(sub.join("c-0000000000000000.ktstr.json")).unwrap();
2001
2002        let json_out = explain_sidecar("run-mixed-outcomes", Some(tmp.path()), true).unwrap();
2003        let parsed: serde_json::Value = serde_json::from_str(&json_out).unwrap();
2004        let walk = parsed.get("_walk").unwrap();
2005        let walked = walk.get("walked").and_then(|v| v.as_u64()).unwrap();
2006        let valid_n = walk.get("valid").and_then(|v| v.as_u64()).unwrap();
2007        let parse_errs = walk.get("errors").and_then(|e| e.as_array()).unwrap().len() as u64;
2008        let io_errs = walk
2009            .get("io_errors")
2010            .and_then(|e| e.as_array())
2011            .unwrap()
2012            .len() as u64;
2013        assert_eq!(
2014            walked,
2015            valid_n + parse_errs + io_errs,
2016            "walked must equal valid + errors + io_errors. \
2017             walked={walked}, valid={valid_n}, errors={parse_errs}, \
2018             io_errors={io_errs}",
2019        );
2020        assert_eq!(walked, 3);
2021        assert_eq!(valid_n, 1);
2022        assert_eq!(parse_errs, 1);
2023        assert_eq!(io_errs, 1);
2024    }
2025
2026    /// `_walk.io_errors` is empty array on the all-clean happy path.
2027    #[test]
2028    fn explain_sidecar_json_walk_io_errors_empty_when_no_io_failures() {
2029        let tmp = tempfile::tempdir().unwrap();
2030        let run_dir = tmp.path().join("run-clean-io");
2031        std::fs::create_dir(&run_dir).unwrap();
2032        let sc = crate::test_support::SidecarResult::test_fixture();
2033        std::fs::write(
2034            run_dir.join("t-0000000000000000.ktstr.json"),
2035            serde_json::to_string(&sc).unwrap(),
2036        )
2037        .unwrap();
2038        let out = explain_sidecar("run-clean-io", Some(tmp.path()), true).unwrap();
2039        let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
2040        let io_errs = parsed
2041            .get("_walk")
2042            .and_then(|w| w.get("io_errors"))
2043            .and_then(|e| e.as_array())
2044            .unwrap();
2045        assert!(io_errs.is_empty());
2046    }
2047
2048    /// E2E renderer test: synthetic [`WalkStats`] with one enriched
2049    /// parse error renders both `error:` and `enriched:` lines in
2050    /// the corrupt block, in that order.
2051    #[test]
2052    fn explain_sidecar_text_e2e_enrichment_renders_in_corrupt_block() {
2053        let parse_err = crate::test_support::SidecarParseError {
2054            path: std::path::PathBuf::from("/tmp/example-run/sidecar.ktstr.json"),
2055            raw_error: "missing field `host` at line 1 column 100".to_string(),
2056            enriched_message: Some(
2057                "ktstr_test: skipping /tmp/example-run/sidecar.ktstr.json: \
2058                 missing field `host` ... — re-run the test"
2059                    .to_string(),
2060            ),
2061        };
2062        let walk = WalkStats {
2063            walked: 1,
2064            valid: 0,
2065            errors: vec![parse_err],
2066            io_errors: Vec::new(),
2067        };
2068        let out = render_explain_sidecar_text(&[], &walk);
2069        assert!(out.contains("corrupt sidecars (1):"));
2070        assert!(out.contains("    error: missing field `host`"));
2071        assert!(out.contains("    enriched: "));
2072        let error_pos = out.find("    error: ").unwrap();
2073        let enriched_pos = out.find("    enriched: ").unwrap();
2074        assert!(
2075            error_pos < enriched_pos,
2076            "raw `error:` line must precede `enriched:` line",
2077        );
2078    }
2079
2080    /// JSON-channel mirror: synthetic [`WalkStats`] with one enriched
2081    /// parse error renders enriched_message as JSON string (not null).
2082    #[test]
2083    fn explain_sidecar_json_e2e_enrichment_renders_in_walk_errors() {
2084        let prose = "ktstr_test: skipping path: missing field `host` \
2085                     — re-run the test to regenerate";
2086        let parse_err = crate::test_support::SidecarParseError {
2087            path: std::path::PathBuf::from("/tmp/example-run/sidecar.ktstr.json"),
2088            raw_error: "missing field `host` at line 1 column 100".to_string(),
2089            enriched_message: Some(prose.to_string()),
2090        };
2091        let walk = WalkStats {
2092            walked: 1,
2093            valid: 0,
2094            errors: vec![parse_err],
2095            io_errors: Vec::new(),
2096        };
2097        let out = render_explain_sidecar_json(&[], &walk);
2098        let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
2099        let errors = parsed
2100            .get("_walk")
2101            .and_then(|w| w.get("errors"))
2102            .and_then(|e| e.as_array())
2103            .unwrap();
2104        assert_eq!(errors.len(), 1);
2105        let entry = &errors[0];
2106        let enriched = entry
2107            .get("enriched_message")
2108            .and_then(|v| v.as_str())
2109            .expect("enriched_message must be a JSON string");
2110        assert_eq!(enriched, prose);
2111        let raw = entry.get("error").and_then(|v| v.as_str()).unwrap();
2112        assert!(raw.contains("missing field"));
2113    }
2114
2115    /// `--run` with `..` segments must bail before path resolution.
2116    #[test]
2117    fn explain_sidecar_rejects_parent_dir_traversal_in_run() {
2118        let tmp = tempfile::tempdir().unwrap();
2119        for traversal in ["../escape", "subdir/../../escape"] {
2120            let err = explain_sidecar(traversal, Some(tmp.path()), false)
2121                .expect_err("path-traversal `..` in --run must be rejected");
2122            let msg = format!("{err:#}");
2123            assert!(
2124                msg.contains("path-traversal"),
2125                "rejection message must name the cause for {traversal}: \
2126                 {msg}",
2127            );
2128            assert!(msg.contains(traversal));
2129        }
2130    }
2131
2132    /// Absolute paths in `--run` must bail.
2133    #[test]
2134    fn explain_sidecar_rejects_absolute_path_in_run() {
2135        let tmp = tempfile::tempdir().unwrap();
2136        let err = explain_sidecar("/etc/passwd", Some(tmp.path()), false)
2137            .expect_err("absolute path in --run must be rejected");
2138        let msg = format!("{err:#}");
2139        assert!(msg.contains("path-traversal"));
2140    }
2141
2142    /// Empty `--run` must bail.
2143    #[test]
2144    fn explain_sidecar_rejects_empty_run() {
2145        let tmp = tempfile::tempdir().unwrap();
2146        let err =
2147            explain_sidecar("", Some(tmp.path()), false).expect_err("empty --run must be rejected");
2148        let msg = format!("{err:#}");
2149        assert!(msg.contains("must not be empty"));
2150    }
2151
2152    /// `--run .` must bail (CurDir aliases pool root).
2153    #[test]
2154    fn explain_sidecar_rejects_curdir_run() {
2155        let tmp = tempfile::tempdir().unwrap();
2156        let err =
2157            explain_sidecar(".", Some(tmp.path()), false).expect_err("`.` --run must be rejected");
2158        let msg = format!("{err:#}");
2159        assert!(msg.contains("path-traversal"));
2160    }
2161
2162    /// Bare run keys with Normal-only components pass the traversal
2163    /// validator and reach the not-found gate.
2164    #[test]
2165    fn explain_sidecar_accepts_bare_run_key_after_traversal_check() {
2166        let tmp = tempfile::tempdir().unwrap();
2167        let err = explain_sidecar("6.14-abc1234", Some(tmp.path()), false)
2168            .expect_err("non-existent run must surface the not-found error");
2169        let msg = format!("{err:#}");
2170        assert!(msg.contains("not found"));
2171        assert!(!msg.contains("path-traversal"));
2172    }
2173}