ktstr/stats/
compare.rs

1use super::*;
2
3/// One significant per-metric finding produced by [`compare_rows_by`].
4///
5/// `pairing_key` carries the dynamic identity the row pair joined
6/// on — `scenario` plus every NON-slicing dimension's value. The
7/// table renderer in [`compare_partitions`] decodes the key against
8/// the slicing-dim list to produce a label like
9/// `scenario/topology/work_type` (when topology + work_type are
10/// pairing dims) or just `scenario` (when every other dim slices).
11///
12/// The `scenario` / `topology` / `work_type` fields carry the
13/// matched row's values verbatim for legacy-shape consumers and
14/// test fixtures that pre-date the dimensional-slicing refactor.
15/// New code should read [`Finding::pairing_key`] directly so the
16/// slicing-dim variation stays visible.
17///
18/// `metric` is the registry entry the comparison ran against;
19/// consumers read polarity, display unit, and name through it
20/// directly without re-looking up [`metric_def`].
21#[derive(Debug, Clone, serde::Serialize)]
22pub(crate) struct Finding {
23    pub pairing_key: PairingKey,
24    pub scenario: String,
25    pub topology: String,
26    pub work_type: String,
27    pub metric: &'static MetricDef,
28    pub val_a: f64,
29    pub val_b: f64,
30    pub delta: f64,
31    pub kind: FindingKind,
32}
33
34/// How a significant (past-dual-gate) delta is classified. A metric
35/// becomes a [`Finding`] only after clearing the
36/// dual gate; this says which kind. `Informational` is for a
37/// [`Polarity::Informational`](crate::test_support::Polarity::Informational)
38/// metric (`MetricDef::classify_direction` => `None`): the change is
39/// shown but is NEVER a regression or improvement and never affects the
40/// exit code.
41#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
42pub(crate) enum FindingKind {
43    Regression,
44    Improvement,
45    Informational,
46}
47
48/// A metric present on exactly ONE side of a paired (scenario,
49/// topology, work_type) row — a coverage difference, not a perf delta.
50///
51/// `MetricDef::read` returns `None` for a metric absent on a row and
52/// `Some(v)` (including `Some(0.0)`) when present, so an absent metric
53/// is distinguishable from a genuine zero. A metric present on one side
54/// and absent on the other is NOT a regression/improvement: it never
55/// had a comparable baseline. Recording it here (never gated, never
56/// counted in `regressions`/`improvements`/`informational`) surfaces
57/// the appear/disappear-between-runs case instead of either silently
58/// dropping it or — as the pre-fix `read().unwrap_or(0.0)` did —
59/// mis-flagging it as a directional verdict against a coerced-zero
60/// side: an unbounded relative change when the absent side is the
61/// baseline (rel-gate INFINITY), a bounded one otherwise (e.g. 5 -> 0
62/// gives rel ~1.0) — either clears the gate and yields a phantom
63/// regression or improvement (the direction follows the metric's
64/// polarity, so it inverts between LowerBetter and HigherBetter).
65#[derive(Debug, Clone, serde::Serialize)]
66pub(crate) struct CoverageDiff {
67    pub pairing_key: PairingKey,
68    pub scenario: String,
69    pub topology: String,
70    pub work_type: String,
71    pub metric: &'static MetricDef,
72    /// The side that HAS the metric; the other side is absent.
73    pub present_side: ComparePartition,
74    /// The present side's value (the absent side has none).
75    pub value: f64,
76}
77
78/// Aggregate result of comparing two row sets via [`compare_rows_by`].
79///
80/// `regressions` and `improvements` count significant entries in
81/// `findings`; `unchanged` counts metrics that fell below the dual
82/// gate; `excluded_pairs` counts paired (scenario, topology, work_type)
83/// row pairs where either side is excluded from regression math —
84/// `fail`, `inconclusive`, `skip`, or an inverted `expected_failure`
85/// run (which passes but carries failure-mode-dominated telemetry) all
86/// route here. The field name captures "excluded from regression math"
87/// rather than encoding any of the four excluded states, because the
88/// per-side disposition (which side, which state) is recoverable from
89/// the individual `GauntletRow::is_*` / `expected_failure` accessors
90/// when the operator drills in.
91/// `new_in_b`
92/// counts B-side rows whose key has no match on the A side; the
93/// converse is `removed_from_a`. The filter (when set) applies to
94/// every counter, so excluded rows do not contribute.
95#[derive(Debug, Clone, Default, serde::Serialize)]
96pub(crate) struct CompareReport {
97    pub regressions: u32,
98    pub improvements: u32,
99    /// Significant changes in `Polarity::Informational` metrics — shown
100    /// but never gated (excluded from `regressions`/`improvements` and
101    /// the exit code).
102    pub informational: u32,
103    pub unchanged: u32,
104    pub excluded_pairs: u32,
105    pub new_in_b: u32,
106    pub removed_from_a: u32,
107    /// B-side rows whose pairing key misses on A ONLY because the two sides
108    /// carry different throughput denominations (wall-era vs cpu-era
109    /// sidecars — see `ThroughputDenomination`). The throughput rate keys
110    /// kept their names across the denomination change, so such rows are
111    /// refused rather than compared; this counter makes the refusal loud.
112    /// Also counted in `new_in_b` (the row did go unpaired).
113    pub denomination_mismatches: u32,
114    pub findings: Vec<Finding>,
115    /// Metrics present on exactly one side of a paired row (a metric
116    /// appeared or disappeared between runs A and B). Never gated — not
117    /// counted in `regressions`/`improvements`/`informational` and no
118    /// effect on the exit code; surfaced so a coverage change is
119    /// visible rather than silently dropped or mis-flagged as a
120    /// regression from a zero baseline. See [`CoverageDiff`].
121    pub coverage_diffs: Vec<CoverageDiff>,
122}
123
124/// Which side of an A/B comparison a row belongs to. Typed surface
125/// for the per-phase rows so new code does not propagate the
126/// `"A"` / `"B"` string-literal pattern the scalar-finding path
127/// uses (kept as-is at the existing `"A"` / `"B"` call sites in this
128/// module — `render_side_label`, `zero_match_diagnostic`).
129#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize)]
130pub(crate) enum ComparePartition {
131    A,
132    B,
133}
134
135impl ComparePartition {
136    /// Render the side as the same one-letter label
137    /// `render_side_label` produces for the scalar table headers,
138    /// so the noise per-phase coverage rows and the scalar findings
139    /// table share the same operator-facing side identifier.
140    pub fn as_str(self) -> &'static str {
141        match self {
142            Self::A => "A",
143            Self::B => "B",
144        }
145    }
146}
147
148/// Per-metric threshold policy driving `compare_rows` /
149/// `compare_partitions`.
150///
151/// Resolution priority for a given metric's relative significance
152/// threshold, highest first:
153///
154/// 1. `per_metric_percent[metric_name]` — explicit override for
155///    this metric.
156/// 2. `default_percent` — uniform override across every metric
157///    not listed in the map (equivalent to the old `--threshold N`
158///    CLI flag).
159/// 3. The metric's built-in `default_rel` from the `METRICS`
160///    registry — the "no policy" fallback.
161///
162/// Values in the struct are stored as PERCENT (e.g. `10.0` meaning
163/// 10%), NOT fractions. [`Self::rel_threshold`] does the `/100.0`
164/// conversion so every caller inside `compare_rows` reads a
165/// fraction without re-deriving the division.
166///
167/// Note on the registry-fallback branch: the `default_rel` field
168/// on `MetricDef` is already a FRACTION (e.g. `0.25` for 25%),
169/// not a percent. `rel_threshold` returns it verbatim — it
170/// does NOT divide by 100. Only the override branches
171/// (per-metric map, `default_percent`) do the percent-to-fraction
172/// conversion because their inputs are percents. This asymmetry
173/// is deliberate so callers supplying CLI/file-based overrides
174/// work in human-intuitive percent units while the registry
175/// defaults (which already ship in fraction form) pass through
176/// unchanged.
177///
178/// The struct is `serde::Serialize` / `serde::Deserialize` so
179/// `cargo ktstr perf-delta --policy <path>` can load a
180/// JSON-persisted policy file. Default construction produces an
181/// empty policy that uses every registry default; [`Self::uniform`]
182/// reproduces the old `--threshold N` behaviour without any
183/// per-metric override plumbing at the call site.
184#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
185#[serde(default, deny_unknown_fields)]
186pub struct ComparisonPolicy {
187    /// Uniform override: when `Some(p)`, every metric whose name is
188    /// NOT in [`Self::per_metric_percent`] uses `p / 100.0` as its
189    /// relative threshold. `None` falls through to the registry
190    /// `default_rel`. Stored as percent (e.g. `10.0` for 10%).
191    pub default_percent: Option<f64>,
192    /// Per-metric overrides keyed by metric name. Each value is a
193    /// percent (e.g. `15.0` → 15%). An entry here takes precedence
194    /// over both [`Self::default_percent`] and the registry
195    /// `default_rel`.
196    pub per_metric_percent: BTreeMap<String, f64>,
197}
198
199/// CLI-controlled rendering of the per-phase spread block in
200/// `cargo ktstr perf-delta --noise-adjust`. Bundled as a struct
201/// so the 5-flag clap surface threads through
202/// `compare_partitions_noise` as a single positional rather than
203/// five. Default value renders every phase / every metric / every
204/// paired row — equivalent to passing no phase flags. All 5 flags
205/// require `--noise-adjust` (per-phase output exists only there).
206///
207/// The flags compose via AND on independent axes (block-level
208/// suppression × phase-id × row-significance), with three
209/// mutex constraints enforced at CLI parse time:
210///
211/// - `--no-phases` excludes every other phase flag (the whole
212///   block is suppressed; refining what to render is a
213///   contradiction).
214/// - `--phases-only` excludes `--no-phases` (same reason).
215/// - `--steps-only` excludes `--phase` (one of them collapses
216///   to a single bucket; the other suppresses BASELINE — both
217///   together are confused phrasing).
218///
219/// The 5 flags trigger renderer behaviour ONLY — the
220/// `--noise-adjust` per-phase pass always computes the full set
221/// of `NoisePhaseFinding`s and coverage entries so programmatic
222/// consumers see the unfiltered surface. Filtering is render-time
223/// projection.
224#[derive(Debug, Default, Clone)]
225pub struct PhaseDisplayOptions {
226    /// `--no-phases`: suppress the `--noise-adjust` per-phase
227    /// spread block entirely. The aggregate spread table and
228    /// footer render unchanged; the only effect is hiding the
229    /// per-phase block (and its footer hint). Mutually exclusive
230    /// with every other phase flag at CLI parse time.
231    pub no_phases: bool,
232    /// `--phases-only`: suppress the aggregate spread table and
233    /// the host-context delta; render ONLY the per-phase spread
234    /// block. Useful for narrowing investigation to a phase
235    /// regression when the aggregate rollup is noise. Composes
236    /// with `--steps-only`, `--phase`, and `--phase-threshold`.
237    pub phases_only: bool,
238    /// `--steps-only`: within the per-phase block, suppress
239    /// the BASELINE bucket (`step_index == 0`); render only
240    /// scenario Step buckets. Useful when the BASELINE settle
241    /// window is dominated by scheduler startup transients.
242    /// Mutually exclusive with `--phase`.
243    pub steps_only: bool,
244    /// `--phase <N>`: within the per-phase block, render only
245    /// rows whose `step_index == N`. `0` selects BASELINE;
246    /// `1..=N` selects scenario Step ordinals (1 → Step\[0\],
247    /// 2 → Step\[1\], ...). Integer chosen over label so a label
248    /// rename (`"Step[0]"` → `"Step:0"`) doesn't break operator
249    /// CI invocations. Mutually exclusive with `--steps-only`.
250    pub phase: Option<u16>,
251    /// `--phase-threshold <PCT>`: render-side relative-spread
252    /// gate for the `--noise-adjust` per-phase pass. Suppresses
253    /// paired rows where `|delta-mean| / |a.mean| < PCT / 100.0`;
254    /// a value from a ~zero baseline (`|a.mean| < ZERO_MEAN_EPS`)
255    /// is an unbounded relative change and clears any finite
256    /// threshold. `0.0` shows every paired row; absence falls
257    /// through to the registry's per-metric `default_rel`.
258    /// Independent from `--threshold` — the aggregate and
259    /// per-phase passes have separate filters so an operator can
260    /// widen the per-phase view without widening the aggregate
261    /// view.
262    pub phase_threshold: Option<f64>,
263}
264
265impl PhaseDisplayOptions {
266    /// Resolve the per-phase relative threshold for a given
267    /// metric. Returns the override fraction when
268    /// `phase_threshold` is set, else falls through to the
269    /// `ComparisonPolicy` resolution the scalar pass uses. The
270    /// `metric_name` + `default_rel` shape mirrors
271    /// [`ComparisonPolicy::rel_threshold`] so the two surfaces
272    /// stay symmetric.
273    pub fn rel_threshold(
274        &self,
275        policy: &ComparisonPolicy,
276        metric_name: &str,
277        default_rel: f64,
278    ) -> f64 {
279        match self.phase_threshold {
280            Some(pct) => pct / 100.0,
281            None => policy.rel_threshold(metric_name, default_rel),
282        }
283    }
284
285    /// True when a phase row at the given `step_index` should
286    /// render under the current display flags. Combines the two
287    /// step-axis predicates (`--phase <N>` filter and
288    /// `--steps-only` BASELINE-suppressor) into a single
289    /// row-level decision the renderer applies uniformly across
290    /// the `--noise-adjust` per-phase findings and coverage rows.
291    /// Returns `true` when no relevant flag is set (default
292    /// path: every step renders).
293    pub fn matches_phase(&self, step_index: u16) -> bool {
294        if let Some(want) = self.phase
295            && step_index != want
296        {
297            return false;
298        }
299        if self.steps_only && step_index == 0 {
300            return false;
301        }
302        true
303    }
304
305    /// The `--phase-threshold` relative-spread gate for a
306    /// [`NoisePhaseFinding`]'s verdict: `|b.mean - a.mean| / |a.mean| >=
307    /// phase_threshold / 100`. A move from a ~zero baseline (`|a.mean| <
308    /// ZERO_MEAN_EPS`) is unbounded → shown; both ~zero carries no signal →
309    /// filtered by any positive threshold. Returns `true` when no flag is set.
310    /// The noise row carries per-side means (a [`NoiseVerdict`]), so the gate
311    /// works on the mean delta rather than a single-run row delta.
312    pub(crate) fn passes_noise_spread_threshold(&self, verdict: &NoiseVerdict) -> bool {
313        let Some(pct) = self.phase_threshold else {
314            return true;
315        };
316        let a = verdict.a.mean.abs();
317        let delta = (verdict.b.mean - verdict.a.mean).abs();
318        let rel = if a > ZERO_MEAN_EPS {
319            delta / a
320        } else if delta > ZERO_MEAN_EPS {
321            f64::INFINITY
322        } else {
323            0.0
324        };
325        rel >= pct / 100.0
326    }
327}
328
329impl ComparisonPolicy {
330    /// Empty policy — every metric uses its `METRICS` registry
331    /// default. Equivalent to the old `--threshold None` CLI path.
332    pub fn new() -> Self {
333        Self::default()
334    }
335
336    /// Uniform override: every metric uses `percent / 100.0`.
337    /// Mirrors the old `--threshold N` CLI behaviour; the CLI
338    /// dispatch at `cargo ktstr perf-delta --threshold N`
339    /// constructs a policy via this constructor.
340    pub fn uniform(percent: f64) -> Self {
341        Self {
342            default_percent: Some(percent),
343            per_metric_percent: BTreeMap::new(),
344        }
345    }
346
347    /// Load a JSON-persisted policy from a file. Errors propagate
348    /// the read / parse reason as an `anyhow::Error` with the file
349    /// path in the context chain so a malformed `--policy path.json`
350    /// surfaces an actionable message rather than a generic
351    /// "invalid JSON."
352    ///
353    /// Validates after parsing via [`Self::validate`]: rejects
354    /// negative thresholds (a misconfigured 10 vs -10 would
355    /// invert the dual-gate logic at the `.abs() >= rel_thresh`
356    /// check and silently classify every metric as significant)
357    /// and rejects per-metric keys not registered in `METRICS`
358    /// (a typo like `"wrost_spread"` would otherwise be silently
359    /// ignored — the key simply never matches during resolution
360    /// and the metric falls through to `default_percent`).
361    pub fn load_json(path: &std::path::Path) -> anyhow::Result<Self> {
362        use anyhow::Context;
363        let data = std::fs::read_to_string(path)
364            .with_context(|| format!("read comparison policy from {}", path.display()))?;
365        let policy: ComparisonPolicy = serde_json::from_str(&data)
366            .with_context(|| format!("parse comparison policy from {}", path.display()))?;
367        policy
368            .validate()
369            .with_context(|| format!("validate comparison policy from {}", path.display()))?;
370        Ok(policy)
371    }
372
373    /// Structural validation separate from parsing so both the
374    /// `load_json` path and programmatic constructors (after
375    /// [`Self::uniform`] with a user-supplied percent) can share
376    /// one set of invariants without re-implementing checks at
377    /// each call site. Called automatically by [`Self::load_json`];
378    /// CLI dispatch should call it after constructing via
379    /// [`Self::uniform`] to catch `--threshold -10` at the
380    /// entry point rather than deep inside `compare_rows` where
381    /// the dual-gate math silently misbehaves.
382    ///
383    /// Rejects:
384    /// - Negative `default_percent` (nonsensical — thresholds are
385    ///   absolute-value comparisons).
386    /// - Negative entries in `per_metric_percent`.
387    /// - Per-metric keys not in the `METRICS` registry (silent
388    ///   typos would otherwise fall through to `default_percent`
389    ///   unnoticed).
390    pub fn validate(&self) -> anyhow::Result<()> {
391        if let Some(p) = self.default_percent
392            && p < 0.0
393        {
394            anyhow::bail!(
395                "ComparisonPolicy: default_percent must be non-negative; got {p}. \
396                 Thresholds are absolute-value comparisons — a negative value \
397                 would invert the dual-gate logic and silently classify every \
398                 delta as significant."
399            );
400        }
401        for (name, p) in &self.per_metric_percent {
402            if !METRICS.iter().any(|m| m.name == name) {
403                let known: Vec<&str> = METRICS.iter().map(|m| m.name).collect();
404                anyhow::bail!(
405                    "ComparisonPolicy: per_metric_percent contains unknown \
406                     metric `{name}`. A typo in the key would silently fall \
407                     through to default_percent. Registered metrics: {}",
408                    known.join(", "),
409                );
410            }
411            if *p < 0.0 {
412                anyhow::bail!(
413                    "ComparisonPolicy: per_metric_percent[{name:?}] must be \
414                     non-negative; got {p}",
415                );
416            }
417        }
418        Ok(())
419    }
420
421    /// Resolve the mutually-exclusive `--threshold` / `--policy` CLI
422    /// pair into a policy: `--threshold N` is sugar for a uniform N%
423    /// default (validated for sign); `--policy PATH` loads a
424    /// per-metric JSON policy; neither falls through to the registry
425    /// defaults. Shared by every subcommand that accepts the pair
426    /// (`perf-delta`) so the resolution rules — and
427    /// the "exactly one of the two" contract — live in one place.
428    ///
429    /// Both flags set is rejected with an error. At the CLI call
430    /// sites clap `conflicts_with` makes that unreachable, but this is
431    /// a library entry point and must not panic on its inputs; the
432    /// error is the defence-in-depth backstop.
433    pub fn from_cli_flags(
434        threshold: Option<f64>,
435        policy: Option<&std::path::Path>,
436    ) -> anyhow::Result<Self> {
437        match (threshold, policy) {
438            (Some(t), None) => {
439                let p = Self::uniform(t);
440                p.validate()?;
441                Ok(p)
442            }
443            (None, Some(path)) => Self::load_json(path),
444            (None, None) => Ok(Self::default()),
445            (Some(_), Some(_)) => anyhow::bail!(
446                "--threshold and --policy are mutually exclusive; use --policy \
447                 for per-metric overrides"
448            ),
449        }
450    }
451
452    /// Resolve the relative threshold (as a fraction, e.g. `0.10`
453    /// for 10%) for `metric_name` with `default_rel` as the
454    /// registry-level fallback. Handles the percent→fraction
455    /// conversion so `compare_rows_by` does not need to re-derive
456    /// `p / 100.0` at every call site.
457    pub fn rel_threshold(&self, metric_name: &str, default_rel: f64) -> f64 {
458        if let Some(p) = self.per_metric_percent.get(metric_name) {
459            p / 100.0
460        } else if let Some(p) = self.default_percent {
461            p / 100.0
462        } else {
463            default_rel
464        }
465    }
466}
467
468/// Compare two row sets metric-by-metric, parametrised on
469/// `pairing_dims`.
470///
471/// Pure function: no I/O, no globals. Two rows pair iff their
472/// [`PairingKey`] (scenario + every value for each dimension in
473/// `pairing_dims`) is equal — this is the dimensional-slicing
474/// pipeline's join primitive, with slicing dims EXCLUDED from
475/// `pairing_dims` so rows on the A/B sides that differ on those
476/// dims still pair as long as they agree on every non-slicing
477/// dim. When `filter` is `Some(s)`, a row is included only if
478/// `s` appears as a substring of the joined `"scenario topology
479/// scheduler work_type"` string. The scheduler is
480/// searchable via the substring filter but is not part of the
481/// pairing key by default (only when `Dimension::Scheduler` is
482/// in `pairing_dims`), so the same scenario+topology+work_type
483/// pair compares correctly across different scheduler binaries
484/// when the filter does not constrain it.
485///
486/// Row-pair accounting:
487/// - B-side rows with no A-side match are counted in `new_in_b`.
488/// - A-side rows with no B-side match are counted in `removed_from_a`
489///   (a separate pass over `rows_a`).
490/// - Paired rows where either side has `passed=false` are dropped
491///   from the regression math and counted in `excluded_pairs`: a
492///   failed scenario's metrics reflect the failure mode (short run,
493///   stalled workload, missing samples), not the scheduler's
494///   behavior.
495///
496/// The filter (when set) applies to every counter -- excluded rows
497/// never reach the matching, pass, or metric stages.
498///
499/// `policy` carries the comparison thresholds. See
500/// [`ComparisonPolicy`] for the resolution rules — per-metric
501/// override → `default_percent` → registry `default_rel`. The
502/// absolute gate always uses the metric's `default_abs`. A delta
503/// must clear both gates to count as significant.
504pub(crate) fn compare_rows_by(
505    rows_a: &[GauntletRow],
506    rows_b: &[GauntletRow],
507    pairing_dims: &[Dimension],
508    filter: Option<&str>,
509    policy: &ComparisonPolicy,
510) -> CompareReport {
511    let mut report = CompareReport::default();
512
513    // Build a HashMap<PairingKey, &GauntletRow> from rows_a once so
514    // each row_b lookup is O(1) instead of O(rows_a). `or_insert_with`
515    // preserves first-match semantics from the prior `rows_a.iter().find()`
516    // call: on the rare path where two A-side rows share a key (the
517    // averaging path folds same-key rows into one mean, so a
518    // shared key is not normally reachable), the
519    // earlier-iterated row wins.
520    let mut a_by_key: HashMap<PairingKey, &GauntletRow> = HashMap::with_capacity(rows_a.len());
521    // Denomination-agnostic shadow of the A-side key set: a B row that
522    // misses `a_by_key` but hits here is the SAME logical run split only by
523    // the wall→cpu-sec throughput-denomination change — refused from pairing
524    // (the values are not comparable) but counted so the refusal is loud.
525    let mut a_denom_agnostic: std::collections::HashSet<PairingKey> =
526        std::collections::HashSet::with_capacity(rows_a.len());
527    for row_a in rows_a {
528        let key = PairingKey::from_row(row_a, pairing_dims);
529        a_denom_agnostic.insert(key.denomination_agnostic());
530        a_by_key.entry(key).or_insert(row_a);
531    }
532
533    // Hoist the per-metric relative threshold out of the row×metric
534    // loop. `policy.rel_threshold(m.name, m.default_rel)` is a pure
535    // function of the metric — recomputing it for every row pair was
536    // O(rows_b × METRICS) BTreeMap probes for nothing.
537    let rel_thresholds: Vec<f64> = METRICS
538        .iter()
539        .map(|m| policy.rel_threshold(m.name, m.default_rel))
540        .collect();
541    // Same hoist for the render-suppression predicate: it is a pure
542    // function of the metric (a small fixed-slice membership scan), so
543    // probing it per (row_b x metric) re-ran the scan for nothing.
544    let suppressed: Vec<bool> = METRICS
545        .iter()
546        .map(|m| is_render_suppressed_component(m.name))
547        .collect();
548
549    for row_b in rows_b {
550        // Dynamic pairing key: scenario + every NON-slicing
551        // dimension's value. Two rows pair iff their dynamic keys
552        // match.
553        let key_b = PairingKey::from_row(row_b, pairing_dims);
554        if let Some(f) = filter {
555            // Substring filter joins all identity-bearing fields —
556            // including the SLICING dim values — so an operator
557            // can narrow by any visible field via `-E`.
558            let joined = format!(
559                "{} {} {} {}",
560                row_b.scenario, row_b.topology, row_b.scheduler, row_b.work_type,
561            );
562            if !joined.contains(f) {
563                continue;
564            }
565        }
566        let Some(&row_a) = a_by_key.get(&key_b) else {
567            report.new_in_b += 1;
568            if a_denom_agnostic.contains(&key_b.denomination_agnostic()) {
569                report.denomination_mismatches += 1;
570            }
571            continue;
572        };
573
574        // Drop from regression math when either side is a skip,
575        // inconclusive, failure, or an inverted expected_failure run.
576        // Skips carry no executed metrics
577        // (the run didn't happen); inconclusive runs ran but lacked
578        // signal to evaluate (zero-denominator ratio gate); failures
579        // carry telemetry dominated by the failure mode (short run,
580        // stalled workload), not the scheduler's behavior. An
581        // expected_failure run has `passed == true` but its telemetry
582        // is likewise failure-mode-dominated (short / stalled run), so
583        // it is excluded despite passing —
584        // comparing any of these against a real run produces
585        // meaningless deltas.
586        if row_a.is_fail()
587            || row_b.is_fail()
588            || row_a.is_inconclusive()
589            || row_b.is_inconclusive()
590            || row_a.is_skip()
591            || row_b.is_skip()
592            || row_a.expected_failure
593            || row_b.expected_failure
594        {
595            report.excluded_pairs += 1;
596            continue;
597        }
598
599        push_scalar_findings(
600            &mut report,
601            row_a,
602            row_b,
603            &key_b,
604            &rel_thresholds,
605            &suppressed,
606        );
607    }
608
609    // Second pass: A-side rows whose key has no match on the B side.
610    // Filter applies here too, so rows excluded by the filter never
611    // count as removed. Build a HashSet<PairingKey> from rows_b once
612    // so the existence check is O(1) per row_a; rows_b are inserted
613    // unfiltered to preserve prior behaviour where a row_b that fails
614    // the substring filter still suppresses a same-key row_a's
615    // removed_from_a increment (the substring filter compares against
616    // identity-bearing fields including slicing dims, so two rows
617    // sharing a pairing key can disagree on filter membership).
618    let b_keys: HashSet<PairingKey> = rows_b
619        .iter()
620        .map(|r| PairingKey::from_row(r, pairing_dims))
621        .collect();
622    for row_a in rows_a {
623        let key_a = PairingKey::from_row(row_a, pairing_dims);
624        if let Some(f) = filter {
625            let joined = format!(
626                "{} {} {} {}",
627                row_a.scenario, row_a.topology, row_a.scheduler, row_a.work_type,
628            );
629            if !joined.contains(f) {
630                continue;
631            }
632        }
633        if !b_keys.contains(&key_a) {
634            report.removed_from_a += 1;
635        }
636    }
637
638    report
639}
640
641/// Append the scalar per-metric findings for one matched `(row_a,
642/// row_b)` pair to `report`. Indexed by the `METRICS` enumerate
643/// position: `rel_thresholds[i]` is the hoisted relative threshold
644/// and `suppressed[i]` the hoisted render-suppression flag for the
645/// i-th metric (both built once by [`compare_rows_by`] over the same
646/// `METRICS` order). Bumps `report.unchanged` for sub-dual-gate
647/// deltas and `report.regressions` / `report.improvements` per
648/// metric polarity for the rest, pushing a [`Finding`] for each
649/// significant delta.
650fn push_scalar_findings(
651    report: &mut CompareReport,
652    row_a: &GauntletRow,
653    row_b: &GauntletRow,
654    key_b: &PairingKey,
655    rel_thresholds: &[f64],
656    suppressed: &[bool],
657) {
658    for (i, m) in METRICS.iter().enumerate() {
659        // Rate components are internal plumbing — suppressed from compare
660        // output (they remain in storage for the cross-run re-pool).
661        if suppressed[i] {
662            continue;
663        }
664        // `read` returns `None` for a metric absent on a row and `Some(v)`
665        // (including `Some(0.0)`) when present, so absent is distinguishable
666        // from a genuine zero. A metric present on exactly one side is a
667        // coverage difference, NOT a delta: record it (never gated) and skip
668        // the directional verdict. The pre-fix `unwrap_or(0.0)` coerced an
669        // absent side to 0.0, producing a phantom directional verdict: when
670        // the absent side was the baseline (val_a==0) the rel-gate's INFINITY
671        // branch fired (unbounded), and otherwise rel_delta was bounded (e.g.
672        // 5 -> 0 gives |(-5)/5|=1.0); either cleared both gates and yielded a
673        // phantom regression or improvement (direction per the metric's
674        // polarity) for a metric simply not captured on one side.
675        let (val_a, val_b) = match (m.read(row_a), m.read(row_b)) {
676            (Some(a), Some(b)) => (a, b),
677            (None, None) => continue,
678            (Some(a), None) => {
679                report.coverage_diffs.push(CoverageDiff {
680                    pairing_key: key_b.clone(),
681                    scenario: row_b.scenario.clone(),
682                    topology: row_b.topology.clone(),
683                    work_type: row_b.work_type.clone(),
684                    metric: m,
685                    present_side: ComparePartition::A,
686                    value: a,
687                });
688                continue;
689            }
690            (None, Some(b)) => {
691                report.coverage_diffs.push(CoverageDiff {
692                    pairing_key: key_b.clone(),
693                    scenario: row_b.scenario.clone(),
694                    topology: row_b.topology.clone(),
695                    work_type: row_b.work_type.clone(),
696                    metric: m,
697                    present_side: ComparePartition::B,
698                    value: b,
699                });
700                continue;
701            }
702        };
703        // Both sides negligible (under ZERO_MEAN_EPS, the domain zero epsilon —
704        // not f64::EPSILON, the machine ulp near 1.0): no signal, skip without
705        // counting.
706        if val_a.abs() < ZERO_MEAN_EPS && val_b.abs() < ZERO_MEAN_EPS {
707            continue;
708        }
709
710        let rel_thresh = rel_thresholds[i];
711
712        let delta = val_b - val_a;
713        let rel_delta = if val_a.abs() > ZERO_MEAN_EPS {
714            (delta / val_a).abs()
715        } else {
716            // A non-negligible value (val_b — the both-zero case is skipped
717            // above) appearing from a ~zero baseline is an unbounded relative
718            // change, not "unchanged". INFINITY clears the rel gate so the
719            // absolute gate alone decides whether this delta is significant.
720            f64::INFINITY
721        };
722
723        if delta.abs() < m.default_abs || rel_delta < rel_thresh {
724            report.unchanged += 1;
725            continue;
726        }
727
728        // Verdict: dual-gate already passed above (significant). An
729        // Informational metric (classify_direction => None) is recorded
730        // and displayed but NEVER counted as regression/improvement and
731        // NEVER affects the exit code; a directional metric splits on its
732        // polarity.
733        let kind = match m.classify_direction() {
734            None => {
735                report.informational += 1;
736                FindingKind::Informational
737            }
738            Some(higher_is_worse) => {
739                let is_regression = if higher_is_worse {
740                    delta > 0.0
741                } else {
742                    delta < 0.0
743                };
744                if is_regression {
745                    report.regressions += 1;
746                    FindingKind::Regression
747                } else {
748                    report.improvements += 1;
749                    FindingKind::Improvement
750                }
751            }
752        };
753        report.findings.push(Finding {
754            pairing_key: key_b.clone(),
755            scenario: row_b.scenario.clone(),
756            topology: row_b.topology.clone(),
757            work_type: row_b.work_type.clone(),
758            metric: m,
759            val_a,
760            val_b,
761            delta,
762            kind,
763        });
764    }
765}
766
767/// Emit a stderr warning naming any `-dirty` commit values present
768/// in the partitioned rows so the operator knows the comparison
769/// includes builds whose source tree may not match the recorded
770/// HEAD.
771///
772/// Scans `commit` (project HEAD) and `kernel_commit` (kernel source
773/// tree HEAD) on both sides' rows, dedupes the surviving values,
774/// and emits one warning block listing each distinct dirty value
775/// per dimension. Emits at most one block — silent when no row
776/// carries a `-dirty` suffix on either dimension.
777///
778/// Dirty runs reuse the same sidecar filename as their clean HEAD
779/// (the variant hash excludes `commit` / `kernel_commit` per
780/// `crate::test_support::sidecar`), so re-running the same test
781/// from a dirty tree overwrites the previous record. The warning
782/// surfaces this so an operator can decide whether to commit the
783/// working tree before re-running for a reproducible comparison.
784///
785/// Splits collection from emission via [`render_dirty_warning`] so
786/// unit tests can pin the rendered text without trapping `stderr`.
787fn warn_on_dirty_builds(rows_a: &[GauntletRow], rows_b: &[GauntletRow]) {
788    if let Some(text) = render_dirty_warning(rows_a, rows_b) {
789        eprint!("{text}");
790    }
791}
792
793/// Emit the CPU-budget hazard warning for a comparison, if any.
794/// Pure-render half is [`render_overcommit_warning`]; this only
795/// `eprint!`s it, mirroring [`warn_on_dirty_builds`].
796fn warn_on_overcommit(rows_a: &[GauntletRow], rows_b: &[GauntletRow], pairing_dims: &[Dimension]) {
797    if let Some(text) = render_overcommit_warning(rows_a, rows_b, pairing_dims) {
798        eprint!("{text}");
799    }
800}
801
802/// Build the CPU-budget hazard warning from the filtered compare
803/// sides, or `None` when neither hazard is present.
804///
805/// Two independent hazards, both read from [`GauntletRow::cpu_budget`]
806/// / [`GauntletRow::vcpus`] — the consumers that make those fields
807/// load-bearing on the compare path:
808///
809/// - OVERCOMMIT (`cpu_budget < vcpus`): the host time-sliced that
810///   run's vCPU threads, so its wake-latency / off-CPU / run-delay
811///   timing metrics are host-contention artifacts, not scheduler
812///   signal (see [`crate::vmm::host_topology::overcommit_warning`]).
813///   Always flagged when present on either side: comparing raw timing
814///   from an overcommitted run is the silent-wrong-answer the budget
815///   stamp exists to surface.
816/// - MIXED BUDGET: a single pairing group on a side holds more than
817///   one distinct non-skip budget. [`group_and_average_by`] folds rows
818///   that share a full [`PairingKey`], so this is exactly the set the
819///   averaging fold combines across budgets. It only arises
820///   when [`Dimension::CpuBudget`] is NOT a pairing dim (the operator
821///   sliced on cpu-budget, dropping it from the key); when it IS a
822///   pairing dim, each budget keys its own group and is never folded.
823///   Detection is per pairing group, NOT side-wide: two rows of
824///   different scenarios (or any differing pairing dim) carry different
825///   keys and never average, so a side merely spanning budgets across
826///   distinct groups is not flagged.
827///
828/// Skip rows (budget 0 -> `None` in [`sidecar_to_row`]) carry no
829/// budget identity and are ignored by both checks. Split from
830/// emission so a unit test pins the text and the `None`-when-clean
831/// polarity without trapping stderr, mirroring [`render_dirty_warning`].
832pub(crate) fn render_overcommit_warning(
833    rows_a: &[GauntletRow],
834    rows_b: &[GauntletRow],
835    pairing_dims: &[Dimension],
836) -> Option<String> {
837    use std::collections::BTreeSet;
838    use std::fmt::Write;
839
840    // Side-wide: the distinct overcommitted (budget, vcpus) pairs.
841    let overcommitted = |rows: &[GauntletRow]| -> BTreeSet<(u32, u32)> {
842        let mut over = BTreeSet::new();
843        for r in rows {
844            if let (Some(b), Some(v)) = (r.cpu_budget, r.vcpus)
845                && b < v
846            {
847                over.insert((b, v));
848            }
849        }
850        over
851    };
852
853    // Per pairing group: the union of budgets across groups that hold
854    // >1 distinct budget — exactly the budgets the averaging fold
855    // combines into one mean. Empty when CpuBudget is a pairing dim (each budget keys
856    // its own group, so no group ever holds two).
857    let cpu_budget_is_pairing = pairing_dims.contains(&Dimension::CpuBudget);
858    let mixed_folded = |rows: &[GauntletRow]| -> BTreeSet<u32> {
859        let mut folded = BTreeSet::new();
860        if cpu_budget_is_pairing {
861            return folded;
862        }
863        let mut by_key: std::collections::HashMap<PairingKey, BTreeSet<u32>> =
864            std::collections::HashMap::new();
865        for r in rows {
866            if let Some(b) = r.cpu_budget {
867                by_key
868                    .entry(PairingKey::from_row(r, pairing_dims))
869                    .or_default()
870                    .insert(b);
871            }
872        }
873        for budgets in by_key.values() {
874            if budgets.len() > 1 {
875                folded.extend(budgets.iter().copied());
876            }
877        }
878        folded
879    };
880
881    let over_a = overcommitted(rows_a);
882    let over_b = overcommitted(rows_b);
883    let mixed_a = mixed_folded(rows_a);
884    let mixed_b = mixed_folded(rows_b);
885    let dilation_range = |rows: &[GauntletRow]| -> Option<(f64, f64)> {
886        let mut min = f64::INFINITY;
887        let mut max = f64::NEG_INFINITY;
888        for dilation in rows
889            .iter()
890            .filter(|row| !row.is_skip())
891            .filter_map(|row| row.host_dilation)
892            .filter(|dilation| dilation.is_finite() && *dilation > 0.0)
893        {
894            min = min.min(dilation);
895            max = max.max(dilation);
896        }
897        (max > 1.05).then_some((min, max))
898    };
899    let dilation_a = dilation_range(rows_a);
900    let dilation_b = dilation_range(rows_b);
901
902    if over_a.is_empty()
903        && over_b.is_empty()
904        && mixed_a.is_empty()
905        && mixed_b.is_empty()
906        && dilation_a.is_none()
907        && dilation_b.is_none()
908    {
909        return None;
910    }
911
912    let any_overcommit = !over_a.is_empty() || !over_b.is_empty();
913    let any_dilation = dilation_a.is_some() || dilation_b.is_some();
914    let mut out = String::new();
915    if any_dilation {
916        let _ = writeln!(
917            out,
918            "ktstr: WARNING: measured host dilation exceeds 1.05 in this \
919             comparison. CPU-denominated iteration, event, schedstat, and IRQ \
920             rates exclude host-preempted time; genuinely wall-semantic latency, \
921             off-CPU, gap, PSI, and upstream wall-throughput values remain raw and \
922             must be interpreted with the dilation shown below."
923        );
924    } else if any_overcommit {
925        let _ = writeln!(
926            out,
927            "ktstr: WARNING: CPU-budget hazard in this comparison — a run was \
928             structurally host-overcommitted. CPU-denominated rates are robust, \
929             while wall-semantic latency / run-delay / off-CPU / gap / PSI / upstream wall-throughput \
930             values may be host-contention-confounded."
931        );
932    } else {
933        // Mixed budgets with NO overcommit: no host contention, the hazard is
934        // collapsing two different measurement conditions into one number.
935        let _ = writeln!(
936            out,
937            "ktstr: WARNING: CPU-budget hazard in this comparison — runs of \
938             different CPU budgets share a pairing group, mixing two measurement \
939             conditions. Slice with --cpu-budget, or compare the budget-invariant \
940             worst_iterations_per_cpu_sec metric."
941        );
942    }
943    let mut emit_side = |label: &str, over: &BTreeSet<(u32, u32)>, mixed: &BTreeSet<u32>| {
944        if !over.is_empty() {
945            let list = over
946                .iter()
947                .map(|(b, v)| format!("{b}/{v}"))
948                .collect::<Vec<_>>()
949                .join(", ");
950            let _ = writeln!(
951                out,
952                "  side {label}: host-overcommitted run(s) [budget/vcpus]: {list}"
953            );
954        }
955        if !mixed.is_empty() {
956            let list = mixed
957                .iter()
958                .map(|b| b.to_string())
959                .collect::<Vec<_>>()
960                .join(", ");
961            let _ = writeln!(
962                out,
963                "  side {label}: CPU budgets [{list}] share a pairing group — \
964                 the average fold collapses them into one mean; slice with --cpu-budget so cross-budget runs are \
965                 not compared under one key"
966            );
967        }
968    };
969    emit_side("A", &over_a, &mixed_a);
970    emit_side("B", &over_b, &mixed_b);
971    for (label, range) in [("A", dilation_a), ("B", dilation_b)] {
972        if let Some((min, max)) = range {
973            if (max - min).abs() < f64::EPSILON {
974                let _ = writeln!(out, "  side {label}: measured dilation D={max:.3}");
975            } else {
976                let _ = writeln!(
977                    out,
978                    "  side {label}: measured dilation D={min:.3}..{max:.3}"
979                );
980            }
981        }
982    }
983    Some(out)
984}
985
986/// Build the dirty-builds warning block from row data.
987///
988/// Returns `None` when no row on either side carries a `-dirty`
989/// suffix on either `commit` or `kernel_commit`. Otherwise returns
990/// the full multi-line warning text — the body emitted to stderr by
991/// [`warn_on_dirty_builds`] — terminated with a trailing newline so
992/// the caller can `eprint!` it without further formatting.
993///
994/// Dimensions render in fixed order ("kernel source" before
995/// "project") so the same dirty hashes always produce byte-identical
996/// output across runs; values within each dimension are
997/// `BTreeSet`-deduped so multiple rows sharing one dirty hash list
998/// it once, and multiple distinct dirty hashes on one dimension list
999/// in lex order.
1000pub(crate) fn render_dirty_warning(
1001    rows_a: &[GauntletRow],
1002    rows_b: &[GauntletRow],
1003) -> Option<String> {
1004    use std::collections::BTreeSet;
1005    use std::fmt::Write;
1006
1007    let mut dirty_kernel: BTreeSet<&str> = BTreeSet::new();
1008    let mut dirty_project: BTreeSet<&str> = BTreeSet::new();
1009    for row in rows_a.iter().chain(rows_b.iter()) {
1010        // `ends_with` matches the producer contract: `detect_kernel_commit`
1011        // and `detect_project_commit` (src/test_support/sidecar/mod.rs) append
1012        // `-dirty` as a SUFFIX to the 7-char hex via
1013        // `format!("{short_hash}-dirty")`, so the dirty marker is
1014        // always tail-positioned. `contains` would also match a
1015        // hex hash that legitimately contains the substring `-dirty`
1016        // somewhere in the middle (impossible for the current
1017        // 7-char hex prefix, but a future commit-ish format change
1018        // would let a non-dirty value flag itself dirty under
1019        // `contains`).
1020        if let Some(c) = row.kernel_commit.as_deref()
1021            && c.ends_with("-dirty")
1022        {
1023            dirty_kernel.insert(c);
1024        }
1025        if let Some(c) = row.commit.as_deref()
1026            && c.ends_with("-dirty")
1027        {
1028            dirty_project.insert(c);
1029        }
1030    }
1031
1032    if dirty_kernel.is_empty() && dirty_project.is_empty() {
1033        return None;
1034    }
1035
1036    let mut out = String::new();
1037    writeln!(out, "warning: comparison includes dirty builds:").unwrap();
1038    for v in &dirty_kernel {
1039        writeln!(
1040            out,
1041            "  - kernel source: {v} (working tree may have changed since this run)"
1042        )
1043        .unwrap();
1044    }
1045    for v in &dirty_project {
1046        writeln!(
1047            out,
1048            "  - project: {v} (working tree may have changed since this run)"
1049        )
1050        .unwrap();
1051    }
1052    writeln!(
1053        out,
1054        "  Dirty runs overwrite previous results with the same HEAD."
1055    )
1056    .unwrap();
1057    writeln!(out, "  Commit changes for reproducible-ish comparisons.").unwrap();
1058    Some(out)
1059}
1060
1061/// Render the actionable bail message emitted when one side's filter
1062/// matches zero sidecars in the pool.
1063///
1064/// Beyond the generic "check filters / run `cargo ktstr stats list`"
1065/// redirect, this helper inspects WHY the filter matched nothing and
1066/// adds three operator-actionable hints when applicable:
1067///
1068/// 1. **Dirty-form hint**: when the user passed
1069///    `--project-commit X` (or per-side / kernel-commit equivalent)
1070///    and the pool contains a row whose `commit` (or `kernel_commit`)
1071///    is `X-dirty`, append "Did you mean `--project-commit X-dirty`?".
1072///    A clean-vs-dirty mismatch is the single most common cause of a
1073///    false-zero on the commit dims — `detect_project_commit` /
1074///    `detect_kernel_commit` append `-dirty` whenever HEAD-vs-index
1075///    or index-vs-worktree changes are observed, so an operator who
1076///    expected `abcdef1` but the recorded value is `abcdef1-dirty`
1077///    sees no rows match without realizing why.
1078///
1079/// 2. **Unknown run-source hint**: when the user passed
1080///    `--run-source X` (or per-side equivalent) and `X` is NOT
1081///    among the distinct `run_source` values present in the pool,
1082///    append a hint listing the actual values seen. The schema is
1083///    deliberately extensible (`"benchmark"` and other future tags
1084///    are valid), so this is a hint rather than a hard validator —
1085///    but a typo (`--run-source loca` for `local`, or `--run-source CI`
1086///    for `ci` since the values are case-sensitive) is the most
1087///    common cause of a false-zero on the source dim, and listing
1088///    the distinct values present is more actionable than asking
1089///    the operator to consult the schema doc.
1090///
1091/// 3. **list-values redirect for commit dims**: when the user
1092///    populated any commit dimension (`project_commits` /
1093///    `kernel_commits`), suggest `cargo ktstr stats list-values`
1094///    specifically — that command emits the exact distinct values
1095///    present per dimension, which is more actionable than the
1096///    generic `stats list` which only shows top-level run keys.
1097///
1098/// `side` is `"A"` or `"B"` for diagnostic context. `filter` is the
1099/// per-side `RowFilter`. `rows` is the sidecar-derived row vec
1100/// (post-`sidecar_to_row` mapping, pre-filtering). `pool_len` is
1101/// the raw pool count for the "(N pooled)" diagnostic context.
1102pub(crate) fn zero_match_diagnostic(
1103    side: &str,
1104    filter: &RowFilter,
1105    rows: &[GauntletRow],
1106    pool_len: usize,
1107) -> String {
1108    let mut msg = format!(
1109        "perf-delta: {side} side filter matched 0 sidecars in \
1110         pool ({pool_len} pooled). Check the per-side filters or \
1111         confirm the runs exist with `cargo ktstr stats list`."
1112    );
1113
1114    // Dirty-form hint per commit dimension. Only fires when a
1115    // populated filter value's `-dirty` form is in the pool.
1116    let mut dirty_hints: Vec<String> = Vec::new();
1117    for want in &filter.project_commits {
1118        let dirty = format!("{want}-dirty");
1119        let found = rows
1120            .iter()
1121            .any(|r| r.commit.as_deref() == Some(dirty.as_str()));
1122        if found {
1123            dirty_hints.push(format!(
1124                "no rows match `--project-commit {want}` but `{dirty}` exists in the pool — \
1125                 did you mean `--project-commit {dirty}`?"
1126            ));
1127        }
1128    }
1129    for want in &filter.kernel_commits {
1130        let dirty = format!("{want}-dirty");
1131        let found = rows
1132            .iter()
1133            .any(|r| r.kernel_commit.as_deref() == Some(dirty.as_str()));
1134        if found {
1135            dirty_hints.push(format!(
1136                "no rows match `--kernel-commit {want}` but `{dirty}` exists in the pool — \
1137                 did you mean `--kernel-commit {dirty}`?"
1138            ));
1139        }
1140    }
1141    for hint in dirty_hints {
1142        msg.push_str("\nhint: ");
1143        msg.push_str(&hint);
1144    }
1145
1146    // Unknown-run-source hint. Fires when a `--run-source X` value
1147    // is not present in the pool — typo / wrong casing is the most
1148    // common cause. Schema is intentionally extensible (operators
1149    // can write `"benchmark"` etc.), so this is a hint not a hard
1150    // validator: the bail still fires, the operator still sees the
1151    // distinct values present, and the producer side is free to
1152    // emit any tag.
1153    if !filter.run_sources.is_empty() {
1154        let pool_run_sources: std::collections::BTreeSet<&str> = rows
1155            .iter()
1156            .filter_map(|r| r.run_source.as_deref())
1157            .collect();
1158        let unknowns: Vec<&str> = filter
1159            .run_sources
1160            .iter()
1161            .map(String::as_str)
1162            .filter(|want| !pool_run_sources.contains(*want))
1163            .collect();
1164        if !unknowns.is_empty() {
1165            let mut present: Vec<&str> = pool_run_sources.iter().copied().collect();
1166            present.sort_unstable();
1167            let unknown_list = unknowns
1168                .iter()
1169                .map(|s| format!("`{s}`"))
1170                .collect::<Vec<_>>()
1171                .join(", ");
1172            let present_list = if present.is_empty() {
1173                "(none — every row has `run_source: null`)".to_string()
1174            } else {
1175                present
1176                    .iter()
1177                    .map(|s| format!("`{s}`"))
1178                    .collect::<Vec<_>>()
1179                    .join(", ")
1180            };
1181            msg.push_str(&format!(
1182                "\nhint: --run-source {unknown_list} not found in pool; \
1183                 distinct values present: {present_list}. Values are \
1184                 case-sensitive (`ci` ≠ `CI`)."
1185            ));
1186        }
1187    }
1188
1189    // Unknown-resolve-source hint. Mirrors the run_sources hint for the
1190    // scheduler-resolution-path dimension: fires when a `--resolve-source`
1191    // value is not among the resolve_sources present in the pool.
1192    if !filter.resolve_sources.is_empty() {
1193        let pool_resolve_sources: std::collections::BTreeSet<&str> = rows
1194            .iter()
1195            .filter_map(|r| r.resolve_source.as_deref())
1196            .collect();
1197        let unknowns: Vec<&str> = filter
1198            .resolve_sources
1199            .iter()
1200            .map(String::as_str)
1201            .filter(|want| !pool_resolve_sources.contains(*want))
1202            .collect();
1203        if !unknowns.is_empty() {
1204            let mut present: Vec<&str> = pool_resolve_sources.iter().copied().collect();
1205            present.sort_unstable();
1206            let unknown_list = unknowns
1207                .iter()
1208                .map(|s| format!("`{s}`"))
1209                .collect::<Vec<_>>()
1210                .join(", ");
1211            let present_list = if present.is_empty() {
1212                "(none — every row has `resolve_source: null`)".to_string()
1213            } else {
1214                present
1215                    .iter()
1216                    .map(|s| format!("`{s}`"))
1217                    .collect::<Vec<_>>()
1218                    .join(", ")
1219            };
1220            msg.push_str(&format!(
1221                "\nhint: --resolve-source {unknown_list} not found in pool; \
1222                 distinct values present: {present_list}. Values are \
1223                 case-sensitive (`auto_built` \u{2260} `Auto_Built`)."
1224            ));
1225        }
1226    }
1227
1228    // Unknown-cpu-budget hint. Mirrors the run_sources hint for the
1229    // numeric budget dimension: fires when a `--cpu-budget` value is
1230    // not among the budgets present in the pool (the budgets render
1231    // canonically as decimal via `cpu_budget.to_string()`, so a
1232    // non-canonical input like `032` lists as not-found against the
1233    // canonical present set). Skip rows (`cpu_budget == None`) carry no
1234    // budget and are excluded.
1235    if !filter.cpu_budgets.is_empty() {
1236        let pool_budgets: std::collections::BTreeSet<u32> =
1237            rows.iter().filter_map(|r| r.cpu_budget).collect();
1238        let present_strs: std::collections::BTreeSet<String> =
1239            pool_budgets.iter().map(|b| b.to_string()).collect();
1240        let unknowns: Vec<&str> = filter
1241            .cpu_budgets
1242            .iter()
1243            .map(String::as_str)
1244            .filter(|want| !present_strs.contains(*want))
1245            .collect();
1246        if !unknowns.is_empty() {
1247            let unknown_list = unknowns
1248                .iter()
1249                .map(|s| format!("`{s}`"))
1250                .collect::<Vec<_>>()
1251                .join(", ");
1252            let present_list = if pool_budgets.is_empty() {
1253                "(none — every row is a skip with no recorded budget)".to_string()
1254            } else {
1255                pool_budgets
1256                    .iter()
1257                    .map(|b| format!("`{b}`"))
1258                    .collect::<Vec<_>>()
1259                    .join(", ")
1260            };
1261            msg.push_str(&format!(
1262                "\nhint: --cpu-budget {unknown_list} not found in pool; \
1263                 distinct budgets present: {present_list}."
1264            ));
1265        }
1266    }
1267
1268    // list-values redirect: only fires when the operator narrowed
1269    // on a commit dimension. Generic case (no commit filter) keeps
1270    // the existing `stats list` redirect at the top of the message
1271    // — `list-values` would emit a long per-dimension dump that
1272    // isn't more actionable than `stats list` for a kernel/scheduler
1273    // /topology miss.
1274    let touched_commit_dim =
1275        !filter.project_commits.is_empty() || !filter.kernel_commits.is_empty();
1276    if touched_commit_dim {
1277        msg.push_str(
1278            "\nhint: run `cargo ktstr stats list-values` to see every \
1279             distinct commit value present in the pool — the specific \
1280             value the filter expected may not have a sidecar yet, or \
1281             may differ from what was recorded by \
1282             `detect_project_commit` / `detect_kernel_commit`.",
1283        );
1284    }
1285    msg
1286}
1287
1288/// Resolved inputs for the `perf-delta` render phase.
1289///
1290/// Produced by [`prepare_partitioned_comparison`] — the validation,
1291/// pooling, partitioning, and averaging steps of [`compare_partitions`]
1292/// extracted into an owned bundle so the render half reads from one
1293/// destructure rather than a long flat prelude. Every field carries
1294/// the exact value the prior in-function prelude bound; the render
1295/// half computes labels and headers from these, then runs the four
1296/// print helpers.
1297struct PartitionedComparison {
1298    /// Dimensions on which `filter_a` differs from `filter_b` — the
1299    /// A/B contrast axes. Guaranteed non-empty (the empty case bails).
1300    slicing_dims: Vec<Dimension>,
1301    /// Dimensions NOT in `slicing_dims`, in canonical
1302    /// [`Dimension::ALL`] order — the join axes for pairing.
1303    pairing_dims: Vec<Dimension>,
1304    /// Every sidecar under the runs root (or `--dir` override).
1305    /// Guaranteed non-empty (the empty pool bails).
1306    pool: Vec<crate::test_support::SidecarResult>,
1307    /// `pool` converted to rows, same length and iteration order.
1308    rows: Vec<GauntletRow>,
1309    /// A-side rows fed to [`compare_rows_by`]: averaged mean rows
1310    /// under [`RowPrep::Averaged`], the raw filtered rows under
1311    /// [`RowPrep::PerRunPooled`].
1312    rows_a_for_compare: Vec<GauntletRow>,
1313    /// B-side counterpart of `rows_a_for_compare`.
1314    rows_b_for_compare: Vec<GauntletRow>,
1315    /// A-side averaged groups under [`RowPrep::Averaged`]; `None` under
1316    /// [`RowPrep::PerRunPooled`]. Drives the per-group pass-count block.
1317    avg_a: Option<Vec<AveragedGroup>>,
1318    /// B-side counterpart of `avg_a`.
1319    avg_b: Option<Vec<AveragedGroup>>,
1320    /// Post-typed-filter A-side contributor row count (pre-aggregation)
1321    /// — the "averaged across N runs" header numerator.
1322    pre_agg_a: usize,
1323    /// B-side counterpart of `pre_agg_a`.
1324    pre_agg_b: usize,
1325}
1326
1327/// How [`prepare_partitioned_comparison`] folds each side's rows before
1328/// the compare half consumes them.
1329#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1330enum RowPrep {
1331    /// Fold same-pairing-key rows on each side into one arithmetic-mean
1332    /// row — the default (averaging) compare behavior.
1333    Averaged,
1334    /// Keep every row INCLUDING duplicate pairing keys. The
1335    /// `perf-delta --noise-adjust`
1336    /// path: [`noise_findings`] groups the N runs per key per side into
1337    /// a spread, so N-per-key is the intended input, not an error.
1338    PerRunPooled,
1339}
1340
1341/// Validate, pool, partition, and average the inputs for
1342/// [`compare_partitions`]. Returns the owned [`PartitionedComparison`]
1343/// bundle the render half destructures, or bails with the same
1344/// diagnostics in the same order as the original in-function prelude:
1345/// identical-rows gate, empty-pool gate, then the two zero-match
1346/// gates. The multi-dim slicing warning and the dirty-build /
1347/// overcommit warnings are emitted here so they precede the render
1348/// half's header lines, preserving output order.
1349fn prepare_partitioned_comparison(
1350    filter_a: &RowFilter,
1351    filter_b: &RowFilter,
1352    dir: Option<&std::path::Path>,
1353    prep: RowPrep,
1354) -> anyhow::Result<PartitionedComparison> {
1355    // Validation gate 1: there must be at least one dimension
1356    // on which filter_a differs from filter_b — otherwise the
1357    // operator hasn't expressed a contrast and the function has
1358    // nothing to compare. Empty slicing dims OR identical filters
1359    // are both rejected here with actionable diagnostics so the
1360    // user knows which knob to turn.
1361    let slicing_dims = derive_slicing_dims(filter_a, filter_b);
1362    if slicing_dims.is_empty() {
1363        anyhow::bail!(
1364            "perf-delta: A and B select identical rows. \
1365             Specify at least one per-side filter (e.g. \
1366             --a-kernel 6.14 --b-kernel 6.15) to define what \
1367             dimension separates the two sides."
1368        );
1369    }
1370
1371    // Validation gate 2: warn (not error) when slicing on
1372    // multiple dimensions. The result is still well-defined —
1373    // the comparison joins on remaining pairing dims and
1374    // collapses the slicing-dim cross-product into a single
1375    // A/B contrast — but the operator is asking for a multi-axis
1376    // delta which is harder to interpret. The warning surfaces
1377    // the dim list so they can confirm the cohort shape.
1378    if slicing_dims.len() > 1 {
1379        let dim_names: Vec<&str> = slicing_dims.iter().map(|d| d.name()).collect();
1380        eprintln!(
1381            "warning: perf-delta: slicing on {n} dimensions [{dims}]; \
1382             results compress multiple axes into a single A/B contrast.",
1383            n = slicing_dims.len(),
1384            dims = dim_names.join(", "),
1385        );
1386    }
1387
1388    // Pairing dims = every dimension NOT in the slicing-dim set,
1389    // in canonical [`Dimension::ALL`] order. The dynamic key
1390    // shape `(scenario, *pairing_dims)` matches whatever
1391    // dimensions are currently NOT being contrasted across A
1392    // and B.
1393    let pairing_dims = Dimension::pairing_dims(&slicing_dims);
1394
1395    // Pool every sidecar under the runs root (or the operator's
1396    // --dir override) and convert to rows. The full-scan cost
1397    // is acceptable for the single-comparison-per-session
1398    // workflow.
1399    //
1400    // `--dir`-loaded sidecars get their `source` field rewritten
1401    // to `"archive"` via `apply_archive_source_override` before
1402    // row conversion. The producer-side `"local"` / `"ci"`
1403    // distinction is meaningful on the host that wrote the
1404    // sidecars; once the files have been copied off, the only
1405    // useful classification is "this came from elsewhere", which
1406    // is what `--run-source archive` queries for. Operators who need
1407    // to retain the producer-side distinction read from the
1408    // default root (no `--dir`) so values pass through untouched.
1409    let (root, override_archive) = match dir {
1410        Some(d) => (d.to_path_buf(), true),
1411        None => (crate::test_support::runs_root(), false),
1412    };
1413    let mut pool = crate::test_support::collect_pool(&root);
1414    if override_archive {
1415        crate::test_support::apply_archive_source_override(&mut pool);
1416    }
1417    if pool.is_empty() {
1418        anyhow::bail!(
1419            "perf-delta: no sidecar data found under {}. \
1420             Run `cargo ktstr test` to generate runs, or pass \
1421             --dir to point at an archived sidecar tree.",
1422            root.display(),
1423        );
1424    }
1425    let rows: Vec<GauntletRow> = pool.iter().map(sidecar_to_row).collect();
1426
1427    // Partition: apply each side's filter to the same pool. A
1428    // row may match both sides (e.g. when project_commit is the
1429    // slicing dim, a row whose `project_commit` is in
1430    // `filter_a.project_commits` matches A but NOT B unless
1431    // `filter_b.project_commits` also contains it).
1432    let rows_a = apply_row_filters(&rows, filter_a);
1433    let rows_b = apply_row_filters(&rows, filter_b);
1434    if rows_a.is_empty() {
1435        anyhow::bail!(
1436            "{}",
1437            zero_match_diagnostic("A", filter_a, &rows, pool.len()),
1438        );
1439    }
1440    if rows_b.is_empty() {
1441        anyhow::bail!(
1442            "{}",
1443            zero_match_diagnostic("B", filter_b, &rows, pool.len()),
1444        );
1445    }
1446
1447    warn_on_dirty_builds(&rows_a, &rows_b);
1448    warn_on_overcommit(&rows_a, &rows_b, &pairing_dims);
1449
1450    let pre_agg_a = rows_a.len();
1451    let pre_agg_b = rows_b.len();
1452
1453    // Fold each side's rows per the caller's [`RowPrep`]. `Averaged`
1454    // collapses same-pairing-key rows into one mean row (the default);
1455    // PerRunPooled keeps every row distinct, including duplicate
1456    // pairing keys (N-per-key is the intended noise-adjust input).
1457    let (rows_a_for_compare, rows_b_for_compare, avg_a, avg_b) = match prep {
1458        RowPrep::Averaged => {
1459            let avg_a = group_and_average_by(&rows_a, &pairing_dims);
1460            let avg_b = group_and_average_by(&rows_b, &pairing_dims);
1461            let a_rows: Vec<GauntletRow> = avg_a.iter().map(|r| r.row.clone()).collect();
1462            let b_rows: Vec<GauntletRow> = avg_b.iter().map(|r| r.row.clone()).collect();
1463            (a_rows, b_rows, Some(avg_a), Some(avg_b))
1464        }
1465        RowPrep::PerRunPooled => {
1466            // The noise-spread compare groups the N runs per pairing key
1467            // per side (see `noise_findings`), so N-per-key is the
1468            // intended input. Keep every row including duplicate
1469            // pairing keys — N-per-key is expected here, not an error.
1470            (rows_a, rows_b, None, None)
1471        }
1472    };
1473
1474    Ok(PartitionedComparison {
1475        slicing_dims,
1476        pairing_dims,
1477        pool,
1478        rows,
1479        rows_a_for_compare,
1480        rows_b_for_compare,
1481        avg_a,
1482        avg_b,
1483        pre_agg_a,
1484        pre_agg_b,
1485    })
1486}
1487
1488/// Warning for the SCALAR compare path when compared tests declare
1489/// `perf_delta_assertions` gates it does not evaluate. Declared gates are a
1490/// CI-gating perf assertion; gating on single-run scalar data would flip CI on
1491/// noise, so they are honored ONLY under `perf-delta --noise-adjust` (multi-run,
1492/// Welch + separation). Returning the message instead of a bare no-op keeps the
1493/// gate from silently disappearing on the default `perf-delta` invocation.
1494/// `None` when no compared test declares a gate. Pure (no I/O) so the
1495/// count/message is unit-testable. Counts DISTINCT scenarios carrying a gate.
1496pub(crate) fn scalar_declared_gate_warning(rows_b: &[GauntletRow]) -> Option<String> {
1497    let n = rows_b
1498        .iter()
1499        .filter(|r| !r.perf_delta_assertions.is_empty())
1500        .map(|r| r.scenario.as_str())
1501        .collect::<std::collections::BTreeSet<_>>()
1502        .len();
1503    (n > 0).then(|| {
1504        format!(
1505            "NOTE — {n} compared test(s) declare perf_delta_assertions gate(s) that this \
1506             scalar comparison does NOT evaluate. Declared gates are enforced only under \
1507             `perf-delta --noise-adjust N` (single-run scalar gating would flip CI on \
1508             noise); re-run with --noise-adjust to gate on them."
1509        )
1510    })
1511}
1512
1513/// Compare two filter-defined partitions of the sidecar pool and
1514/// report regressions across slicing dimensions.
1515///
1516/// `filter_a` and `filter_b` are the per-side row filters that
1517/// define the A/B contrast. The dimensions on which the two
1518/// filters DIFFER are the SLICING dimensions; the dimensions on
1519/// which they AGREE (or on which both are unconstrained) are the
1520/// PAIRING dimensions. Two rows pair across the A/B sides iff
1521/// their dynamic [`PairingKey`] (scenario plus every pairing-dim
1522/// value) is equal — so the comparison naturally ignores
1523/// differences on the slicing axes (those ARE the contrast) and
1524/// joins on everything else.
1525///
1526/// `dir` overrides the default `runs_root()` for pool collection.
1527/// Pass `Some(path)` to compare archived sidecar trees copied off
1528/// a CI host; pass `None` to walk `target/ktstr/` (or
1529/// `CARGO_TARGET_DIR/ktstr/`).
1530///
1531/// Validation:
1532/// - Empty slicing-dim set (every dimension is identical between
1533///   A and B): bail with "specify at least one --a-X / --b-X to
1534///   define what to compare". This includes the no-flags-at-all
1535///   case (both filters are the empty default).
1536/// - Identical effective filters with at least one slicing dim is
1537///   a contradiction caught by clap-level construction; the
1538///   downstream check is "every value in filter_a appears in
1539///   filter_b on the same dim and vice versa." We catch that as
1540///   "A and B select identical rows" — symmetric to the empty
1541///   case.
1542/// - More than one slicing dimension prints a warning to stderr
1543///   ("warning: slicing on N dimensions; results compress
1544///   multiple axes into a single A/B contrast") but does NOT
1545///   bail — multi-dim slicing is a deliberate feature for
1546///   comparing e.g. (kernel A + scheduler A) against (kernel B +
1547///   scheduler B).
1548///
1549/// Groups every matching sidecar within each side by pairing key
1550/// and averages the metrics across the group.
1551///
1552/// Returns 1 when the confident regressions fail the operator gate — their
1553/// count reaches `gate.fail_threshold` (default 5) or a `gate.must_fail`
1554/// metric regressed; 0 otherwise. See [`gate_fails`] / [`GateOptions`].
1555pub fn compare_partitions(
1556    filter_a: &RowFilter,
1557    filter_b: &RowFilter,
1558    filter: Option<&str>,
1559    policy: &ComparisonPolicy,
1560    dir: Option<&std::path::Path>,
1561    gate: &GateOptions,
1562) -> anyhow::Result<i32> {
1563    let prepared = prepare_partitioned_comparison(filter_a, filter_b, dir, RowPrep::Averaged)?;
1564    let PartitionedComparison {
1565        slicing_dims,
1566        pairing_dims,
1567        pool,
1568        rows,
1569        rows_a_for_compare,
1570        rows_b_for_compare,
1571        avg_a,
1572        avg_b,
1573        pre_agg_a,
1574        pre_agg_b,
1575    } = &prepared;
1576
1577    let report = compare_rows_by(
1578        rows_a_for_compare,
1579        rows_b_for_compare,
1580        pairing_dims,
1581        filter,
1582        policy,
1583    );
1584
1585    // Side labels derive from the slicing dims' filter values.
1586    // Single slicing dim: e.g. "6.14.2" / "6.15.0". Multi: e.g.
1587    // "6.14.2:scx_rusty" / "6.15.0:scx_alpha". >3 values per dim:
1588    // collapse to "A"/"B" to keep column headers readable.
1589    let label_a = render_side_label(filter_a, slicing_dims, "A");
1590    let label_b = render_side_label(filter_b, slicing_dims, "B");
1591
1592    // Header lines: name the slicing and pairing axes so the
1593    // operator can confirm the comparison shape at a glance.
1594    let slice_names: Vec<&str> = slicing_dims.iter().map(|d| d.name()).collect();
1595    let pair_names: Vec<&str> = pairing_dims.iter().map(|d| d.name()).collect();
1596    println!("slicing dimensions: {}", slice_names.join(", "));
1597    println!(
1598        "pairing on: scenario{}{}",
1599        if pair_names.is_empty() { "" } else { ", " },
1600        pair_names.join(", "),
1601    );
1602    // Declared gates are not evaluated on the scalar path — warn rather than
1603    // silently ignore (they are honored only under `perf-delta --noise-adjust`).
1604    if let Some(warning) = scalar_declared_gate_warning(rows_b_for_compare) {
1605        println!("{warning}");
1606    }
1607
1608    println!(
1609        "{}",
1610        format_average_header(*pre_agg_a, *pre_agg_b, &label_a, &label_b)
1611    );
1612
1613    // Scalar findings table.
1614    print_scalar_findings_table(&report, &label_a, &label_b);
1615
1616    // Scalar summary block — regressions / improvements /
1617    // unchanged + skipped-failed + per-group pass counts +
1618    // new_in_b / removed_from_a.
1619    print_summary_block(&report, avg_a, avg_b, &label_a, &label_b);
1620
1621    // Host-context delta. Same first-Some(host) baseline
1622    // `compare_partitions` uses — picking representative hosts
1623    // off the partitioned sidecars rather than the full pool so
1624    // the delta reflects what actually fed the comparison.
1625    print_host_context_delta(pool, rows, filter_a, filter_b, &label_a, &label_b);
1626
1627    // Operator gate: the significance policy above decided WHICH deltas are
1628    // confident regressions; the gate decides HOW MANY / WHICH-NAMED of them
1629    // fail the run. (--all-metrics is a no-op on this path — the scalar table
1630    // already lists every changed metric and the unchanged COUNT prints in
1631    // the summary; it reveals stable/noisy rows on the --noise-adjust table.)
1632    let regressing: Vec<&str> = report
1633        .findings
1634        .iter()
1635        .filter(|f| f.kind == FindingKind::Regression)
1636        .map(|f| f.metric.name)
1637        .collect();
1638    Ok(if gate_fails(&regressing, gate) { 1 } else { 0 })
1639}
1640
1641/// Operator-level perf-delta failure gate + render options, layered on top
1642/// of the per-metric significance policy (which decides WHICH deltas are
1643/// confident regressions). These decide HOW MANY / WHICH-NAMED confident
1644/// regressions fail the run, and whether stable/noisy rows render.
1645#[derive(Debug, Clone, Default)]
1646pub struct GateOptions {
1647    /// Fail iff at least this many confident regressions occur. `None`
1648    /// means 5 — a handful of regressions is tolerated as run-to-run noise
1649    /// and the run fails only once several metrics regress; pass
1650    /// `--fail-threshold 1` for fail-on-any. `Some(0)` disables the count
1651    /// gate entirely — only [`Self::must_fail`] can then fail the run.
1652    pub fail_threshold: Option<usize>,
1653    /// Metric registry names that fail the run if ANY of them regresses,
1654    /// regardless of the count gate (ORed on top). Caller-validated against
1655    /// the metric registry.
1656    pub must_fail: Vec<String>,
1657    /// Render every compared metric row (stable + noisy included) on the
1658    /// `--noise-adjust` table instead of only the meaningful ones.
1659    /// Display-only — never affects the gate.
1660    pub show_all: bool,
1661}
1662
1663/// Decide whether a perf-delta run FAILS, given the registry names of the
1664/// confident regressions it found. Fails iff the count meets
1665/// [`GateOptions::fail_threshold`] (default 5; `Some(0)` disables the count
1666/// gate), OR any regressing metric is in [`GateOptions::must_fail`] (ORed
1667/// on top). The caller passes the CLASSIFIED regressions, so display-hidden
1668/// (suppressed) rows still feed the gate.
1669pub(crate) fn gate_fails(regressing_metrics: &[&str], gate: &GateOptions) -> bool {
1670    let n = gate.fail_threshold.unwrap_or(5);
1671    let fail_on_count = n >= 1 && regressing_metrics.len() >= n;
1672    let fail_on_must = !gate.must_fail.is_empty()
1673        && regressing_metrics
1674            .iter()
1675            .any(|m| gate.must_fail.iter().any(|w| w.as_str() == *m));
1676    fail_on_count || fail_on_must
1677}
1678
1679#[cfg(test)]
1680mod gate_option_tests {
1681    use super::*;
1682
1683    #[test]
1684    fn default_gate_fails_only_at_five_regressions() {
1685        // Default (None) threshold is 5: a handful of regressions is
1686        // tolerated as run-to-run noise; the run fails once >= 5 regress.
1687        let g = GateOptions::default();
1688        assert!(!gate_fails(&[], &g), "0 regressions passes");
1689        assert!(
1690            !gate_fails(&["a", "b", "c", "d"], &g),
1691            "4 < 5 passes under the default gate"
1692        );
1693        assert!(
1694            gate_fails(&["a", "b", "c", "d", "e"], &g),
1695            "5 >= 5 fails under the default gate"
1696        );
1697    }
1698
1699    #[test]
1700    fn count_threshold_requires_n() {
1701        let g = GateOptions {
1702            fail_threshold: Some(3),
1703            ..Default::default()
1704        };
1705        assert!(!gate_fails(&["a", "b"], &g), "2 < 3 passes");
1706        assert!(gate_fails(&["a", "b", "c"], &g), "3 >= 3 fails");
1707    }
1708
1709    #[test]
1710    fn zero_threshold_disables_count_gate() {
1711        let g = GateOptions {
1712            fail_threshold: Some(0),
1713            ..Default::default()
1714        };
1715        assert!(
1716            !gate_fails(&["a", "b", "c"], &g),
1717            "N=0 never fails on the count"
1718        );
1719    }
1720
1721    #[test]
1722    fn must_fail_fails_regardless_of_count() {
1723        let g = GateOptions {
1724            fail_threshold: Some(0),
1725            must_fail: vec!["worst_p99_wake_latency_us".to_string()],
1726            ..Default::default()
1727        };
1728        assert!(
1729            gate_fails(&["worst_p99_wake_latency_us"], &g),
1730            "a must-fail metric regressing fails even with the count gate off"
1731        );
1732        assert!(
1733            !gate_fails(&["some_other_metric"], &g),
1734            "a non-must-fail regression does not fail with the count gate off"
1735        );
1736    }
1737
1738    #[test]
1739    fn must_fail_is_ored_above_the_count() {
1740        let g = GateOptions {
1741            fail_threshold: Some(10),
1742            must_fail: vec!["avg_dsq_depth".to_string()],
1743            ..Default::default()
1744        };
1745        assert!(
1746            gate_fails(&["avg_dsq_depth"], &g),
1747            "must-fail fires even below the count threshold"
1748        );
1749    }
1750}
1751
1752/// How a metric's noise-adjusted verdict classifies for the gate.
1753#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1754pub(crate) enum NoiseKind {
1755    /// SEPARATED (Welch or disjoint bands) AND MATERIAL in the worsening
1756    /// direction (per polarity) — the only kind that fails the gate.
1757    Regression,
1758    /// Separated AND material in the improving direction.
1759    Improvement,
1760    /// A side realized fewer than 2 samples, so variance / Welch are undefined —
1761    /// verdict untrustworthy; flagged but does NOT fail the gate. A merely
1762    /// high-spread side no longer lands here: `high_spread` is ADVISORY and
1763    /// annotates a reported verdict (e.g. `REGRESSION (noisy spread)`) rather
1764    /// than suppressing it — the fix for the old spread-gate signal inversion.
1765    Noisy,
1766    /// Separated + material change in a directionless (`Polarity::Informational`)
1767    /// metric — shown but NEVER fails the gate (no good/bad direction to regress).
1768    Informational,
1769    /// Not separated, or separated but immaterial (below the registry dual-gate),
1770    /// with both sides having >= 2 samples. Shown in the full metrics table so the
1771    /// operator sees every metric's A/B stats, but never gates. Emitted only when
1772    /// the caller passes `include_stable = true` (the render path); the gate-only
1773    /// path omits it.
1774    Stable,
1775}
1776
1777/// One metric's noise-adjusted finding for a paired scenario.
1778pub(crate) struct NoiseFinding {
1779    /// The pairing-key label ("scenario" plus the pairing dims like
1780    /// topology/work_type, joined with "/"), so groups that share a scenario
1781    /// name but differ in a pairing dim render distinctly.
1782    pub pairing_label: String,
1783    pub metric: &'static MetricDef,
1784    pub verdict: NoiseVerdict,
1785    pub kind: NoiseKind,
1786    /// A whole-run [`crate::test_support::PerfDeltaAssertionRecord`] is declared
1787    /// on this (test, metric) — its `min_abs` / `max_regression_pct` /
1788    /// `direction` override the registry defaults in [`classify_noise`] (a
1789    /// declared value that is out of range on a corrupt/stale sidecar is
1790    /// rejected there and falls back to the registry default, but the row still
1791    /// CARRIES the declared gate). Rendered as a `(declared gate)` verdict
1792    /// annotation so the operator sees the author declared a gate here. `false`
1793    /// = no declared assertion; the registry defaults classified this row.
1794    pub gated_by_assertion: bool,
1795}
1796
1797/// One metric's noise-adjusted finding for a matched PHASE of a paired
1798/// scenario — the per-phase (`--noise-adjust` + per-phase) analog of
1799/// [`NoiseFinding`], carrying the `step_index`/`label` the render prints.
1800/// Emitted only for a `(step_index, metric)` present on BOTH sides across
1801/// the runs. Per-phase SPREAD findings are render-only EXCEPT one carrying
1802/// an author-declared phase-scoped gate (`gated_by_assertion`), which DOES
1803/// contribute to the exit via [`NoiseReport::declared_phase_regressions`].
1804pub(crate) struct NoisePhaseFinding {
1805    /// The pairing-key label (see [`NoiseFinding::pairing_label`]).
1806    pub pairing_label: String,
1807    /// `0` = BASELINE, `1..=N` = scenario Step ordinals (framework convention).
1808    pub step_index: u16,
1809    /// Mirrors [`crate::assert::PhaseBucket::label`] (`"BASELINE"` / `"Step[k]"`).
1810    pub label: String,
1811    pub metric: &'static MetricDef,
1812    pub verdict: NoiseVerdict,
1813    pub kind: NoiseKind,
1814    /// A phase-scoped declared assertion (`phase == Some(step_index)`) drove the
1815    /// gate for this per-phase row — see [`NoiseFinding::gated_by_assertion`].
1816    pub gated_by_assertion: bool,
1817}
1818
1819/// A per-phase metric present on only ONE side of the noise comparison —
1820/// either a metric absent in the other side's matched-`step_index` buckets,
1821/// or a whole one-sided `step_index`. Both collapse into one row because
1822/// a one-sided metric has no band on the absent side either way. Never gated;
1823/// surfaced so a coverage asymmetry is not silently dropped.
1824pub(crate) struct NoisePhaseCoverage {
1825    /// The pairing-key label (see [`NoiseFinding::pairing_label`]).
1826    pub pairing_label: String,
1827    pub step_index: u16,
1828    pub label: String,
1829    /// The one-sided metric, or `None` for a whole one-sided phase that
1830    /// carried no readable (non-suppressed) metric — surfaced (rendered with
1831    /// `—`) so the "phase fired but produced no data on one side" shape is not
1832    /// silently dropped.
1833    pub metric: Option<&'static MetricDef>,
1834    /// The side that carries the metric/phase; the other has none.
1835    pub present_side: ComparePartition,
1836    /// The present side's per-side mean across its runs, or `None` for the
1837    /// metric-less empty-phase-shape row.
1838    pub value: Option<f64>,
1839}
1840
1841/// A declared [`crate::test_support::PerfDeltaAssertionRecord`] that was NEVER
1842/// evaluated — its metric resolves in the registry (guaranteed by
1843/// `KtstrTestEntry::validate`) but produced no comparable value in THIS
1844/// comparison, so [`classify_noise`] never saw it. The runtime analog of the
1845/// validate-time typo check: the author declared a perf gate that silently did
1846/// not fire (metric absent from the captured data, a Rate with a zero pooled
1847/// denominator, or — for a phase-scoped assertion — a step that no matched run
1848/// carried). Surfaced (never gated) so a silently-inert declared gate is not
1849/// mistaken for a passing one.
1850pub(crate) struct NoiseAssertionCoverage {
1851    /// The pairing-key label the unmatched gate was declared under (see
1852    /// [`NoiseFinding::pairing_label`]).
1853    pub pairing_label: String,
1854    /// The declared gate that never evaluated — carries its metric, phase
1855    /// scope, and overridden thresholds for the warning message.
1856    pub assertion: crate::test_support::PerfDeltaAssertionRecord,
1857}
1858
1859/// Result of [`noise_findings`]: the per-(scenario, metric) aggregate findings,
1860/// the per-(scenario, phase, metric) findings + one-sided coverage rows, the
1861/// declared gates that never evaluated, and the number of scenarios paired
1862/// across both sides (for the footer).
1863pub(crate) struct NoiseReport {
1864    pub findings: Vec<NoiseFinding>,
1865    pub phase_findings: Vec<NoisePhaseFinding>,
1866    pub phase_coverage: Vec<NoisePhaseCoverage>,
1867    pub assertion_coverage: Vec<NoiseAssertionCoverage>,
1868    pub paired_scenarios: usize,
1869}
1870
1871impl NoiseReport {
1872    /// Confident AGGREGATE regressions — the gate's exit basis. Per-phase
1873    /// findings are render-only and deliberately excluded (see
1874    /// [`Self::phase_regressions`]).
1875    pub fn regressions(&self) -> usize {
1876        self.findings
1877            .iter()
1878            .filter(|f| f.kind == NoiseKind::Regression)
1879            .count()
1880    }
1881    /// Aggregate (whole-run) regressions on a metric the test author explicitly
1882    /// DECLARED a whole-run [`crate::test_support::PerfDeltaAssertion`] for
1883    /// (`phase: None`, `gated_by_assertion`). Like a declared PHASE gate
1884    /// ([`Self::declared_phase_regressions`]), a declared whole-run gate is an
1885    /// author opt-in that ALWAYS contributes to the exit, orthogonal to the
1886    /// operator's count / must-fail gate — an UNdeclared aggregate regression
1887    /// stays subject to that count gate (`gate_fails`).
1888    pub fn declared_regressions(&self) -> usize {
1889        self.findings
1890            .iter()
1891            .filter(|f| f.kind == NoiseKind::Regression && f.gated_by_assertion)
1892            .count()
1893    }
1894    /// Aggregate metrics flagged untrustworthy — a side had < 2 usable runs.
1895    pub fn noisy(&self) -> usize {
1896        self.findings
1897            .iter()
1898            .filter(|f| f.kind == NoiseKind::Noisy)
1899            .count()
1900    }
1901    /// Confident AGGREGATE improvements — a metric that moved MATERIALLY in the
1902    /// better direction and cleared the significance test. Render-only (an
1903    /// improvement never gates); cited in the summary footer alongside
1904    /// [`Self::regressions`] so the reader sees the composite verdict (what
1905    /// regressed AND what improved), with [`Self::stable`] as the residual.
1906    pub fn improvements(&self) -> usize {
1907        self.findings
1908            .iter()
1909            .filter(|f| f.kind == NoiseKind::Improvement)
1910            .count()
1911    }
1912    /// Aggregate metrics that did NOT move confidently+materially — the expected
1913    /// common outcome, since the noise gate (Welch / disjoint-bands + material
1914    /// dual-gate) is conservative. Cited as a residual COUNT in the footer (the
1915    /// low-value majority), never enumerated.
1916    pub fn stable(&self) -> usize {
1917        self.findings
1918            .iter()
1919            .filter(|f| f.kind == NoiseKind::Stable)
1920            .count()
1921    }
1922    /// Aggregate metrics that changed but carry no better/worse polarity
1923    /// (registry `Informational`) — cited in the footer only when present.
1924    pub fn informational(&self) -> usize {
1925        self.findings
1926            .iter()
1927            .filter(|f| f.kind == NoiseKind::Informational)
1928            .count()
1929    }
1930    /// Confident per-phase regressions — for the footer + tests ONLY, never
1931    /// the exit basis (per-phase SPREAD is render-only). A per-phase regression the author explicitly DECLARED via a
1932    /// phase-scoped [`crate::test_support::PerfDeltaAssertion`] DOES gate — see
1933    /// [`Self::declared_phase_regressions`].
1934    pub fn phase_regressions(&self) -> usize {
1935        self.phase_findings
1936            .iter()
1937            .filter(|f| f.kind == NoiseKind::Regression)
1938            .count()
1939    }
1940    /// Per-phase regressions on a metric the test author explicitly DECLARED a
1941    /// phase-scoped gate for. Unlike the render-only per-phase spread pass, a
1942    /// declared phase gate is an opt-in: the author accepted the narrower
1943    /// phase-window noise, so it contributes to the perf-delta EXIT alongside
1944    /// the aggregate [`Self::regressions`]. A per-phase regression WITHOUT a
1945    /// declared gate stays render-only (a narrow-window flake must not flip CI
1946    /// red on its own).
1947    pub fn declared_phase_regressions(&self) -> usize {
1948        self.phase_findings
1949            .iter()
1950            .filter(|f| f.kind == NoiseKind::Regression && f.gated_by_assertion)
1951            .count()
1952    }
1953    /// Per-phase metrics flagged untrustworthy — a side had < 2 usable runs.
1954    pub fn phase_noisy(&self) -> usize {
1955        self.phase_findings
1956            .iter()
1957            .filter(|f| f.kind == NoiseKind::Noisy)
1958            .count()
1959    }
1960}
1961
1962/// Row-level noise-adjusted compare — the testable core of
1963/// [`compare_partitions_noise`] (which wraps it with sidecar pooling + render),
1964/// mirroring how [`compare_rows_by`] underlies [`compare_partitions`]. Groups
1965/// each side's per-run rows by pairing key — EXCLUDING non-pass runs
1966/// (fail / skip / inconclusive / expected_failure), whose failure-mode
1967/// telemetry would corrupt the spread, exactly as [`compare_rows_by`] excludes
1968/// them — then per metric summarizes the spread and classifies via
1969/// [`noise_verdict`]. Returns one [`NoiseFinding`] per
1970/// (scenario, metric) that is changed, noisy, or — when `include_stable` is
1971/// true — unchanged-and-clean ([`NoiseKind::Stable`], shown in the full metrics
1972/// table but never gating). With `include_stable = false` only changed/noisy
1973/// metrics are returned (the gate-only path). Metrics with no signal on either
1974/// side (both means under [`ZERO_MEAN_EPS`]) and render-suppressed rate
1975/// components are omitted regardless of `include_stable`.
1976pub(crate) fn noise_findings(
1977    rows_a: &[GauntletRow],
1978    rows_b: &[GauntletRow],
1979    pairing_dims: &[Dimension],
1980    spread_threshold_pct: f64,
1981    include_stable: bool,
1982) -> NoiseReport {
1983    use std::collections::{BTreeMap, BTreeSet};
1984    let group = |rows: &[GauntletRow]| {
1985        let mut by_key: BTreeMap<PairingKey, Vec<GauntletRow>> = BTreeMap::new();
1986        for r in rows {
1987            // Exclude non-pass runs from the spread pool, mirroring the
1988            // scalar `compare_rows_by` and the averaged `group_and_average_by`:
1989            // a failed / inconclusive / skipped / expected_failure run's
1990            // telemetry is failure-mode-dominated (zeroed or an outlier), so
1991            // pooling it would corrupt the per-side [min, max] band and mean
1992            // and could produce a confident FALSE gate — the exact
1993            // silent-wrong-verdict class the noise mode exists to prevent.
1994            // A side reduced below 2 real samples by this filter is then
1995            // caught by `noise_verdict`'s n<2 insufficient_samples guard.
1996            if r.is_fail() || r.is_inconclusive() || r.is_skip() || r.expected_failure {
1997                continue;
1998            }
1999            // Derive this run's Rate metrics (e.g. the schedstat rates whose
2000            // components only materialize in ext_metrics at sidecar_to_row
2001            // time) so the spread reads them — the Averaged path derives via
2002            // group_and_average_by -> derive_rate_metrics; the per-run paths
2003            // must derive per row or every schedstat Rate is silently absent
2004            // from the verdict. Per-run derivation (each run's own num/den) is
2005            // the correct band semantics for run-to-run spread.
2006            let mut row = r.clone();
2007            crate::stats::metric::derive_rate_metrics(&mut row.ext_metrics);
2008            by_key
2009                .entry(PairingKey::from_row(&row, pairing_dims))
2010                .or_default()
2011                .push(row);
2012        }
2013        by_key
2014    };
2015    let a_by_key = group(rows_a);
2016    let b_by_key = group(rows_b);
2017
2018    let mut findings = Vec::new();
2019    let mut phase_findings = Vec::new();
2020    let mut phase_coverage = Vec::new();
2021    let mut assertion_coverage = Vec::new();
2022    let mut paired_scenarios = 0usize;
2023    for (key, a_rows) in &a_by_key {
2024        let Some(b_rows) = b_by_key.get(key) else {
2025            continue;
2026        };
2027        paired_scenarios += 1;
2028        // Metrics whose gate WAS evaluated this group, split by scope: whole-run
2029        // (aggregate pass) and phase-scoped `(step_index, metric)` (per-phase
2030        // pass). Diffed against the declared assertions after both passes to
2031        // surface any declared gate that never fired (metric absent from the
2032        // captured data).
2033        let mut consulted: BTreeSet<&'static str> = BTreeSet::new();
2034        let mut consulted_phase: BTreeSet<(u16, &'static str)> = BTreeSet::new();
2035        // Label by the FULL pairing key (scenario + pairing dims like
2036        // topology/work_type), joined via pairing_key.0.join. noise_findings groups by the full pairing key,
2037        // so a scenario run on multiple topologies/work_types forms distinct
2038        // groups; labeling by scenario alone would render them indistinguishably.
2039        let pairing_label = key.label();
2040        for m in METRICS {
2041            if is_render_suppressed_component(m.name) {
2042                continue;
2043            }
2044            let a_vals: Vec<f64> = a_rows.iter().filter_map(|r| m.read(r)).collect();
2045            let b_vals: Vec<f64> = b_rows.iter().filter_map(|r| m.read(r)).collect();
2046            if a_vals.is_empty() || b_vals.is_empty() {
2047                continue;
2048            }
2049            // Rate metrics: the per-run ratios (a_vals/b_vals) give the
2050            // run-to-run band, but the compared centroid is the pooled
2051            // Σnum/Σden (duration-weighted) — the cross-run Rate value the
2052            // registry documents (metric.rs), so --noise-adjust and the scalar
2053            // averaging compare agree on a Rate's central value while the band still measures
2054            // per-run variability. Non-Rate metrics summarize their samples.
2055            let verdict = match m.kind {
2056                MetricKind::Rate {
2057                    numerator,
2058                    denominator,
2059                } => {
2060                    let pooled = |rows: &[GauntletRow]| -> Option<f64> {
2061                        // Sum num/den only over runs that carry BOTH components,
2062                        // so a run missing one cannot skew the pooled ratio (the
2063                        // per-run rate is undefined for it anyway).
2064                        let (num, den) = rows.iter().fold((0.0, 0.0), |(sn, sd), r| {
2065                            match (r.ext_metrics.get(numerator), r.ext_metrics.get(denominator)) {
2066                                (Some(n), Some(d)) => (sn + n, sd + d),
2067                                _ => (sn, sd),
2068                            }
2069                        });
2070                        (den != 0.0).then(|| num / den)
2071                    };
2072                    // A zero pooled denominator means no rate to compare on that
2073                    // side — skip (matches the per-run derivation guard).
2074                    let (Some(a_pooled), Some(b_pooled)) = (pooled(a_rows), pooled(b_rows)) else {
2075                        continue;
2076                    };
2077                    noise_verdict_from(
2078                        SideSummary::of(&a_vals).with_pooled_mean(a_pooled),
2079                        SideSummary::of(&b_vals).with_pooled_mean(b_pooled),
2080                        spread_threshold_pct,
2081                    )
2082                }
2083                _ => noise_verdict(&a_vals, &b_vals, spread_threshold_pct),
2084            };
2085            // This metric produced a comparable verdict, so a declared gate on
2086            // it WAS evaluated — record it so the post-loop diff does not flag it
2087            // as never-evaluated. Recorded here (not at the empty-values / zero-
2088            // denominator `continue`s above) so those absent-data cases DO
2089            // surface as unmatched declared gates.
2090            consulted.insert(m.name);
2091            // Aggregate (whole-run) declared assertion for this test+metric
2092            // (phase: None). HEAD (B) side is authoritative on the declaration.
2093            let assertion = b_rows.first().and_then(|r| {
2094                r.perf_delta_assertions
2095                    .iter()
2096                    .find(|x| x.metric == m.name && x.phase.is_none())
2097            });
2098            let gated_by_assertion = assertion.is_some();
2099            let Some(kind) = classify_noise(&verdict, m, assertion, include_stable) else {
2100                continue;
2101            };
2102            findings.push(NoiseFinding {
2103                pairing_label: pairing_label.clone(),
2104                metric: m,
2105                verdict,
2106                kind,
2107                gated_by_assertion,
2108            });
2109        }
2110        // Per-phase sub-pass: mirror the aggregate spread for each matched
2111        // (step_index, metric), surfacing one-sided phases/metrics as coverage
2112        // rows. Render-only — never contributes to the gate exit.
2113        noise_phase_findings(
2114            a_rows,
2115            b_rows,
2116            &pairing_label,
2117            spread_threshold_pct,
2118            include_stable,
2119            &mut phase_findings,
2120            &mut phase_coverage,
2121            &mut consulted_phase,
2122        );
2123        // Diff the declared gates (HEAD/B authoritative) against what actually
2124        // evaluated this group. A whole-run gate (`phase: None`) matches when its
2125        // metric produced an aggregate verdict; a phase-scoped gate matches when
2126        // its `(phase, metric)` produced a per-phase verdict. Anything left is a
2127        // gate the author declared that silently never fired — surfaced, never
2128        // gated.
2129        if let Some(first) = b_rows.first() {
2130            for a in &first.perf_delta_assertions {
2131                let matched = match a.phase {
2132                    None => consulted.contains(a.metric.as_str()),
2133                    Some(step) => consulted_phase.contains(&(step, a.metric.as_str())),
2134                };
2135                if !matched {
2136                    assertion_coverage.push(NoiseAssertionCoverage {
2137                        pairing_label: pairing_label.clone(),
2138                        assertion: a.clone(),
2139                    });
2140                }
2141            }
2142        }
2143    }
2144    NoiseReport {
2145        findings,
2146        phase_findings,
2147        phase_coverage,
2148        assertion_coverage,
2149        paired_scenarios,
2150    }
2151}
2152
2153/// Classify one metric's [`NoiseVerdict`] into a [`NoiseKind`], shared by the
2154/// aggregate and per-phase noise passes. Returns `None` to omit the row: both
2155/// sides ~zero (no signal), or unchanged/immaterial-and-clean when
2156/// `include_stable` is false (the gate-only path).
2157///
2158/// A `< 2`-sample side (`insufficient_samples`) is a HARD gate that precedes
2159/// significance (Noisy, never a confident regression); the ADVISORY
2160/// `high_spread` flag does NOT gate (that suppression was the signal-inverting
2161/// bug). A row is a confident regression only when SEPARATED (Welch or disjoint
2162/// bands) AND MATERIAL (the registry dual-gate) in the worsening polarity.
2163fn classify_noise(
2164    verdict: &NoiseVerdict,
2165    m: &MetricDef,
2166    assertion: Option<&crate::test_support::PerfDeltaAssertionRecord>,
2167    include_stable: bool,
2168) -> Option<NoiseKind> {
2169    // No signal on either side (both ~zero): skip. Same zero epsilon as the
2170    // spread ratio for one consistent "is this zero".
2171    if verdict.a.mean.abs() < ZERO_MEAN_EPS && verdict.b.mean.abs() < ZERO_MEAN_EPS {
2172        return None;
2173    }
2174    // HARD gate: a side realized < 2 samples, so variance / Welch are undefined.
2175    // Precedes significance — never a confident regression. (The ADVISORY
2176    // high_spread flag, by contrast, does NOT gate.)
2177    if verdict.insufficient_samples {
2178        return Some(NoiseKind::Noisy);
2179    }
2180    // MATERIALITY: mirror the scalar dual-gate (push_scalar_findings ~L899) so
2181    // noise and default modes agree on "is this delta large enough". A
2182    // statistically-separated but trivially-small move stays Stable. --noise-adjust
2183    // conflicts with --threshold/--policy (cli.rs), so the registry defaults ARE
2184    // the resolved thresholds (an empty ComparisonPolicy::rel_threshold returns
2185    // default_rel); reading them from `m` directly is equivalent, no policy needed.
2186    // For a Rate metric, `mean` is the pooled Σnum/Σden centroid (the
2187    // registry-authoritative cross-run value), while the Welch separation arm
2188    // reads `sample_mean` (mean-of-ratios, coherent with `var`). Materiality AND
2189    // the direction label below authoritatively follow `mean`; the two statistics
2190    // can order the sides oppositely only under orders-of-magnitude denominator
2191    // skew between runs of one config (physically implausible), where the pooled
2192    // direction is the correct label anyway.
2193    let a = verdict.a.mean;
2194    let b = verdict.b.mean;
2195    let delta = b - a;
2196    let rel_delta = if a.abs() > ZERO_MEAN_EPS {
2197        (delta / a).abs()
2198    } else if delta.abs() > ZERO_MEAN_EPS {
2199        // Non-negligible move from a ~zero baseline: unbounded relative change,
2200        // so the absolute gate alone decides (mirrors the scalar path).
2201        f64::INFINITY
2202    } else {
2203        0.0
2204    };
2205    // A declared PerfDeltaAssertion OVERRIDES the registry gate for THIS
2206    // (test, metric): an absolute floor (`min_abs`), a relative threshold
2207    // (`max_regression_pct`), and/or a pinned direction. Absent (None) =>
2208    // registry defaults, so the no-assertion path is byte-identical to the
2209    // default gate. A tighter declared threshold turns a default-Stable move
2210    // into a Regression; a pinned direction can assert a registry-Informational
2211    // metric.
2212    //
2213    // `min_abs` / `max_regression_pct` come from a pub serde
2214    // `PerfDeltaAssertionRecord`. `KtstrTestEntry::validate` rejects a negative
2215    // or NaN threshold on the entry-construction path, but a hand-edited or
2216    // stale sidecar could deserialize one — and `delta.abs()` / `rel_delta` are
2217    // non-negative, so a NEGATIVE gate makes `material` unconditionally true (a
2218    // phantom confident regression that flips the exit) while a NaN gate makes
2219    // every `>=` false (silently disabled). Reject out-of-range values here and
2220    // fall back to the registry default — symmetric with the `TargetValue`
2221    // direction guard below, defending the same untrusted deserialization path.
2222    let abs_gate = assertion
2223        .and_then(|x| x.min_abs)
2224        .filter(|v| v.is_finite() && *v >= 0.0)
2225        .unwrap_or(m.default_abs);
2226    let rel_gate = assertion
2227        .and_then(|x| x.max_regression_pct)
2228        .filter(|v| v.is_finite() && *v >= 0.0)
2229        .map(|pct| pct / 100.0)
2230        .unwrap_or(m.default_rel);
2231    let material = delta.abs() >= abs_gate && rel_delta >= rel_gate;
2232    // A declared direction override, else the registry polarity. `TargetValue`
2233    // is rejected at the `PerfDeltaAssertion::with_direction` builder, so a
2234    // validated entry never carries it — but `PerfDeltaAssertionRecord` is a
2235    // pub serde type, so a hand-edited or stale sidecar could deserialize a
2236    // `direction: TargetValue`. Symmetric target-distance gating is
2237    // unimplemented (the polarity path would misread it as increase-is-worse),
2238    // so ignore it here and inherit the registry polarity — matching the
2239    // entry-path guarantee on the sidecar-deserialization path.
2240    let classify = match assertion.and_then(|x| x.direction) {
2241        Some(crate::test_support::Polarity::TargetValue(_)) | None => m.classify_direction(),
2242        Some(p) => p.classify_direction(),
2243    };
2244
2245    Some(if verdict.separated && material {
2246        match classify {
2247            // Directionless metric: a separated + material move with no good/bad
2248            // direction — shown, never fails the gate.
2249            None => NoiseKind::Informational,
2250            Some(higher_is_worse) => {
2251                // Polarity split by the SIGN of the mean delta (b vs a), NOT a
2252                // band position: separation can come from the Welch arm even when
2253                // b.mean sits inside a's [min, max] band.
2254                let worsened = if higher_is_worse { b > a } else { b < a };
2255                if worsened {
2256                    NoiseKind::Regression
2257                } else {
2258                    NoiseKind::Improvement
2259                }
2260            }
2261        }
2262    } else if include_stable {
2263        NoiseKind::Stable
2264    } else {
2265        return None; // unchanged/immaterial + clean: omit (gate-only path)
2266    })
2267}
2268
2269/// Push one one-sided per-phase metric to `coverage` — a metric present in a
2270/// matched-`step_index` bucket on one side only. `value` is the present
2271/// side's per-side mean across its runs (the absent side has none).
2272fn push_noise_phase_coverage(
2273    coverage: &mut Vec<NoisePhaseCoverage>,
2274    pairing_label: &str,
2275    step_index: u16,
2276    label: &str,
2277    metric: &'static MetricDef,
2278    present_side: ComparePartition,
2279    vals: &[f64],
2280) {
2281    if vals.is_empty() {
2282        return;
2283    }
2284    let value = vals.iter().sum::<f64>() / vals.len() as f64;
2285    coverage.push(NoisePhaseCoverage {
2286        pairing_label: pairing_label.to_string(),
2287        step_index,
2288        label: label.to_string(),
2289        metric: Some(metric),
2290        present_side,
2291        value: Some(value),
2292    });
2293}
2294
2295/// Surface every non-suppressed metric of a whole one-sided `step_index` (a
2296/// phase present on only one side's runs) as [`NoisePhaseCoverage`] rows, so a
2297/// scenario-shape asymmetry is not silently dropped.
2298fn push_noise_unpaired_step(
2299    coverage: &mut Vec<NoisePhaseCoverage>,
2300    pairing_label: &str,
2301    step_index: u16,
2302    side: ComparePartition,
2303    buckets: &[&crate::assert::PhaseBucket],
2304) {
2305    let label = buckets[0].label.clone();
2306    let names: std::collections::BTreeSet<&str> = buckets
2307        .iter()
2308        .flat_map(|p| p.metrics.keys())
2309        .map(String::as_str)
2310        .collect();
2311    let before = coverage.len();
2312    for name in names {
2313        if is_render_suppressed_component(name) {
2314            continue;
2315        }
2316        let Some(m) = metric_def(name) else {
2317            continue;
2318        };
2319        let vals: Vec<f64> = buckets
2320            .iter()
2321            .filter_map(|p| p.metrics.get(name).copied())
2322            .collect();
2323        push_noise_phase_coverage(coverage, pairing_label, step_index, &label, m, side, &vals);
2324    }
2325    if coverage.len() == before {
2326        // A one-sided phase with no readable (non-suppressed) metric — a
2327        // synthesized capture-free step can carry an empty metrics map. Surface
2328        // the empty shape (metric/value None -> rendered with `—`) so it is not
2329        // silently dropped.
2330        coverage.push(NoisePhaseCoverage {
2331            pairing_label: pairing_label.to_string(),
2332            step_index,
2333            label,
2334            metric: None,
2335            present_side: side,
2336            value: None,
2337        });
2338    }
2339}
2340
2341/// Per-phase noise sub-pass for one matched `(a_rows, b_rows)` pair, over the
2342/// N per-run [`crate::assert::PhaseBucket`]s per side. For each matched
2343/// `step_index` it walks the union of metric names
2344/// and emits a [`NoisePhaseFinding`] per `(step, metric)` present on BOTH sides
2345/// (spread verdict via the same machinery as the aggregate pass, incl. the
2346/// pooled-`Σnum/Σden` Rate centroid — here summed WITHIN the phase from
2347/// `bucket.metrics`), or a [`NoisePhaseCoverage`] for a one-sided metric or a
2348/// whole one-sided `step_index`. Skips the pair unless BOTH sides have at least
2349/// one run carrying phases (single-phase scenarios). Render-only: nothing here
2350/// contributes to the gate exit.
2351#[allow(clippy::too_many_arguments)]
2352fn noise_phase_findings(
2353    a_rows: &[GauntletRow],
2354    b_rows: &[GauntletRow],
2355    pairing_label: &str,
2356    spread_threshold_pct: f64,
2357    include_stable: bool,
2358    findings: &mut Vec<NoisePhaseFinding>,
2359    coverage: &mut Vec<NoisePhaseCoverage>,
2360    consulted_phase: &mut std::collections::BTreeSet<(u16, &'static str)>,
2361) {
2362    use std::collections::BTreeSet;
2363    // Single-phase scenarios carry empty phases on every run; skip the per-phase
2364    // view if either side has no run with phases.
2365    let has_phases = |rows: &[GauntletRow]| rows.iter().any(|r| !r.phases.is_empty());
2366    if !has_phases(a_rows) || !has_phases(b_rows) {
2367        return;
2368    }
2369    // A nested fn (not a closure) so the elided output lifetime ties the
2370    // borrowed &PhaseBucket to `rows` — a closure can't express that linkage.
2371    fn by_step(
2372        rows: &[GauntletRow],
2373    ) -> std::collections::BTreeMap<u16, Vec<&crate::assert::PhaseBucket>> {
2374        let mut m: std::collections::BTreeMap<u16, Vec<&crate::assert::PhaseBucket>> =
2375            std::collections::BTreeMap::new();
2376        for r in rows {
2377            for p in &r.phases {
2378                m.entry(p.step_index).or_default().push(p);
2379            }
2380        }
2381        m
2382    }
2383    let a_by_step = by_step(a_rows);
2384    let b_by_step = by_step(b_rows);
2385    let steps: BTreeSet<u16> = a_by_step.keys().chain(b_by_step.keys()).copied().collect();
2386    for step_index in steps {
2387        match (a_by_step.get(&step_index), b_by_step.get(&step_index)) {
2388            (Some(a_buckets), Some(b_buckets)) => {
2389                let label = a_buckets[0].label.clone();
2390                let names: BTreeSet<&str> = a_buckets
2391                    .iter()
2392                    .chain(b_buckets.iter())
2393                    .flat_map(|p| p.metrics.keys())
2394                    .map(String::as_str)
2395                    .collect();
2396                for name in names {
2397                    if is_render_suppressed_component(name) {
2398                        continue;
2399                    }
2400                    let Some(m) = metric_def(name) else {
2401                        continue;
2402                    };
2403                    let a_vals: Vec<f64> = a_buckets
2404                        .iter()
2405                        .filter_map(|p| p.metrics.get(name).copied())
2406                        .collect();
2407                    let b_vals: Vec<f64> = b_buckets
2408                        .iter()
2409                        .filter_map(|p| p.metrics.get(name).copied())
2410                        .collect();
2411                    match (a_vals.is_empty(), b_vals.is_empty()) {
2412                        (true, true) => continue,
2413                        // Present on only one matched side: no band on the other
2414                        // — coverage, not a delta.
2415                        (false, true) => {
2416                            push_noise_phase_coverage(
2417                                coverage,
2418                                pairing_label,
2419                                step_index,
2420                                &label,
2421                                m,
2422                                ComparePartition::A,
2423                                &a_vals,
2424                            );
2425                            continue;
2426                        }
2427                        (true, false) => {
2428                            push_noise_phase_coverage(
2429                                coverage,
2430                                pairing_label,
2431                                step_index,
2432                                &label,
2433                                m,
2434                                ComparePartition::B,
2435                                &b_vals,
2436                            );
2437                            continue;
2438                        }
2439                        (false, false) => {}
2440                    }
2441                    let verdict = match m.kind {
2442                        MetricKind::Rate {
2443                            numerator,
2444                            denominator,
2445                        } => {
2446                            // Per-phase Rate: pooled Σnum/Σden WITHIN the phase
2447                            // from bucket.metrics (phase-derivable rates carry
2448                            // both components per phase), band from per-run ratios.
2449                            let pooled = |buckets: &[&crate::assert::PhaseBucket]| -> Option<f64> {
2450                                let (num, den) = buckets.iter().fold((0.0, 0.0), |(sn, sd), p| {
2451                                    match (p.metrics.get(numerator), p.metrics.get(denominator)) {
2452                                        (Some(n), Some(d)) => (sn + n, sd + d),
2453                                        _ => (sn, sd),
2454                                    }
2455                                });
2456                                (den != 0.0).then(|| num / den)
2457                            };
2458                            let (Some(a_pooled), Some(b_pooled)) =
2459                                (pooled(a_buckets), pooled(b_buckets))
2460                            else {
2461                                continue;
2462                            };
2463                            noise_verdict_from(
2464                                SideSummary::of(&a_vals).with_pooled_mean(a_pooled),
2465                                SideSummary::of(&b_vals).with_pooled_mean(b_pooled),
2466                                spread_threshold_pct,
2467                            )
2468                        }
2469                        _ => noise_verdict(&a_vals, &b_vals, spread_threshold_pct),
2470                    };
2471                    // This (step, metric) produced a comparable verdict, so a
2472                    // phase-scoped gate on it WAS evaluated (see the aggregate
2473                    // pass's `consulted`).
2474                    consulted_phase.insert((step_index, m.name));
2475                    // Phase-scoped declared assertion for this test+metric+step.
2476                    let assertion = b_rows.first().and_then(|r| {
2477                        r.perf_delta_assertions
2478                            .iter()
2479                            .find(|x| x.metric == m.name && x.phase == Some(step_index))
2480                    });
2481                    let gated_by_assertion = assertion.is_some();
2482                    let Some(kind) = classify_noise(&verdict, m, assertion, include_stable) else {
2483                        continue;
2484                    };
2485                    findings.push(NoisePhaseFinding {
2486                        pairing_label: pairing_label.to_string(),
2487                        step_index,
2488                        label: label.clone(),
2489                        metric: m,
2490                        verdict,
2491                        kind,
2492                        gated_by_assertion,
2493                    });
2494                }
2495            }
2496            (Some(a_buckets), None) => {
2497                push_noise_unpaired_step(
2498                    coverage,
2499                    pairing_label,
2500                    step_index,
2501                    ComparePartition::A,
2502                    a_buckets,
2503                );
2504            }
2505            (None, Some(b_buckets)) => {
2506                push_noise_unpaired_step(
2507                    coverage,
2508                    pairing_label,
2509                    step_index,
2510                    ComparePartition::B,
2511                    b_buckets,
2512                );
2513            }
2514            (None, None) => {}
2515        }
2516    }
2517}
2518
2519/// Summarize a side's loaded runs by why each is or isn't comparable, using the
2520/// SAME exclusions [`noise_findings`] applies (skip / fail / inconclusive /
2521/// expected-failure). Returns `(comparable_count, human_summary)` so an empty
2522/// comparison can explain WHY — e.g. every run was skipped — instead of the bare
2523/// "nothing to compare". The `comparable_count` equals the number of rows
2524/// `noise_findings` would keep (a row is comparable iff none of the four
2525/// exclusions hold), so a zero here is exactly why the side produced no findings.
2526pub(crate) fn summarize_side_runs(rows: &[GauntletRow]) -> (usize, String) {
2527    let (mut skipped, mut failed, mut inconclusive, mut xfail, mut comparable) =
2528        (0usize, 0usize, 0usize, 0usize, 0usize);
2529    for r in rows {
2530        if r.is_skip() {
2531            skipped += 1;
2532        } else if r.is_fail() {
2533            failed += 1;
2534        } else if r.is_inconclusive() {
2535            inconclusive += 1;
2536        } else if r.expected_failure {
2537            xfail += 1;
2538        } else {
2539            comparable += 1;
2540        }
2541    }
2542    let mut excluded = Vec::new();
2543    if skipped > 0 {
2544        excluded.push(format!("{skipped} skipped"));
2545    }
2546    if failed > 0 {
2547        excluded.push(format!("{failed} failed"));
2548    }
2549    if inconclusive > 0 {
2550        excluded.push(format!("{inconclusive} inconclusive"));
2551    }
2552    if xfail > 0 {
2553        excluded.push(format!("{xfail} expected-failure"));
2554    }
2555    let breakdown = if excluded.is_empty() {
2556        "none excluded".to_string()
2557    } else {
2558        excluded.join(", ")
2559    };
2560    (
2561        comparable,
2562        format!(
2563            "{} run(s): {comparable} comparable ({breakdown})",
2564            rows.len()
2565        ),
2566    )
2567}
2568
2569/// Noise-adjusted variant of [`compare_partitions`]: instead of averaging each
2570/// side's runs into one mean and gating on a fixed threshold, it keeps every run
2571/// ([`RowPrep::PerRunPooled`]), summarizes each side per metric, and decides
2572/// whether the two sides are distinguishable given their run-to-run variability
2573/// (see [`noise_findings`] for the row-level core + [`noise_verdict`] for the
2574/// per-metric decision). Used by `perf-delta --noise-adjust N`, which produces N
2575/// runs per side. A metric is a CONFIDENT REGRESSION (fed to the operator
2576/// failure gate below) when it is SEPARATED (a two-sided Welch t-test rejects equal means at
2577/// `NOISE_ALPHA`, OR the `[min, max]` bands are fully disjoint) AND MATERIAL (the
2578/// mean delta clears both the registry `default_abs` and `default_rel`, the same
2579/// dual-gate as the scalar path) in the worsening direction (per polarity). A
2580/// side that realized fewer than 2 runs is flagged `NOISY` and never gates; a
2581/// per-side relative spread over `spread_threshold_pct` is an ADVISORY
2582/// `(noisy spread)` annotation that NEVER suppresses a verdict. By default prints
2583/// only the MEANINGFUL rows (confident regression / improvement / informational),
2584/// each with its side's `mean [min-max] spread%` and verdict; stable (unchanged +
2585/// clean) and noisy (<2-run) rows are hidden unless `--all-metrics`, and a
2586/// one-line summary prints when every row is suppressed. The footer leads with an
2587/// overall verdict word ([`overall_verdict`]: `STABLE` unless the regressed or
2588/// improved count clears the significance cutoff — sub-cutoff moves are flagged
2589/// but likely noise) followed by the composite counts (regressed / improved /
2590/// stable / under-sampled, plus informational when present); and, when every
2591/// changed (non-stable) metric had a side with <2 usable runs, an explicit
2592/// inconclusive note.
2593///
2594/// When the scenarios carry phases, a per-phase spread block follows the
2595/// aggregate table (via [`format_noise_phase_findings_lines`]): the same
2596/// spread verdict per matched `(step_index, metric)`, plus a coverage table for
2597/// one-sided phases/metrics. Like the aggregate table, the per-phase block shows
2598/// only MEANINGFUL rows by default (regression / improvement / informational) and
2599/// hides stable / noisy rows unless `--all-metrics` (`gate.show_all`); whenever the
2600/// spread rows are all suppressed it surfaces a one-line hint naming
2601/// `--all-metrics` (even alongside a coverage table), so the suppressed rows are
2602/// never silently gone. The coverage (one-sided) table is itself never
2603/// suppressed. `phase_opts` controls the per-phase
2604/// render only (`--no-phases` / `--phases-only` / `--steps-only` / `--phase` /
2605/// `--phase-threshold`); under `--phases-only` the aggregate table is
2606/// suppressed. Per-phase SPREAD findings are RENDER-ONLY — classified and
2607/// colored but not gating — EXCEPT a
2608/// per-phase regression the author explicitly declared a phase-scoped gate for,
2609/// which DOES contribute to the exit alongside the operator gate on the aggregate
2610/// confident regressions ([`gate_fails`]: their count reaches `fail_threshold`
2611/// [default 5], or a `must_fail` metric regressed).
2612/// The footer appends per-phase counts only when per-phase data exists and no
2613/// phase filter is active.
2614///
2615/// When no scenario pairs, the aggregate note breaks down each side's loaded
2616/// runs via [`summarize_side_runs`] — naming skipped / failed / inconclusive
2617/// runs — so an all-skipped comparison (e.g. a non-`performance_mode` test under
2618/// perf-delta's `KTSTR_PERF_ONLY`) is explained, not silently reported as
2619/// "nothing to compare".
2620pub fn compare_partitions_noise(
2621    filter_a: &RowFilter,
2622    filter_b: &RowFilter,
2623    dir: Option<&std::path::Path>,
2624    spread_threshold_pct: f64,
2625    phase_opts: &PhaseDisplayOptions,
2626    gate: &GateOptions,
2627) -> anyhow::Result<i32> {
2628    // Keep every per-run row INCLUDING duplicate pairing keys so the
2629    // run-to-run spread is observable: noise_findings groups the N runs
2630    // per key per side. PerRunPooled keeps every per-run row including
2631    // duplicate pairing keys — the N-per-key spread is the intended input.
2632    let prepared = prepare_partitioned_comparison(filter_a, filter_b, dir, RowPrep::PerRunPooled)?;
2633    let label_a = render_side_label(filter_a, &prepared.slicing_dims, "A");
2634    let label_b = render_side_label(filter_b, &prepared.slicing_dims, "B");
2635
2636    let report = noise_findings(
2637        &prepared.rows_a_for_compare,
2638        &prepared.rows_b_for_compare,
2639        &prepared.pairing_dims,
2640        spread_threshold_pct,
2641        // Include Stable (unchanged + clean) metrics so the rendered table
2642        // shows the full comparison, not just changed/noisy rows.
2643        true,
2644    );
2645
2646    println!(
2647        "perf-delta --noise-adjust: {label_b} vs {label_a} (advisory noisy-spread threshold {spread_threshold_pct:.2}%)"
2648    );
2649    // Aggregate spread table — suppressed under --phases-only (renders ONLY
2650    // the per-phase block).
2651    if !phase_opts.phases_only {
2652        if report.findings.is_empty() {
2653            // Distinguish "nothing paired" from "paired but every metric omitted"
2654            // so the message never contradicts the paired-scenario footer below.
2655            // A metric is omitted when both sides' means are ~zero, it read on
2656            // only one side, or it is a render-suppressed rate component.
2657            if report.paired_scenarios == 0 {
2658                // Explain WHY nothing paired instead of a bare "nothing to
2659                // compare": prepare_partitioned_comparison already bailed if a
2660                // side loaded ZERO rows, so both sides have >=1 run here — the
2661                // pairing failure is either all-excluded runs (skip/fail/etc.)
2662                // or a genuine scenario mismatch. Break down each side so a
2663                // skipped-on-both-sides run (the common perf-delta case: a
2664                // non-performance_mode test under KTSTR_PERF_ONLY) is named.
2665                let (a_ok, a_desc) = summarize_side_runs(&prepared.rows_a_for_compare);
2666                let (b_ok, b_desc) = summarize_side_runs(&prepared.rows_b_for_compare);
2667                println!(
2668                    "perf-delta --noise-adjust: no comparable runs to pair across the two runs."
2669                );
2670                println!("  {label_a} — {a_desc}");
2671                println!("  {label_b} — {b_desc}");
2672                if a_ok == 0 || b_ok == 0 {
2673                    println!(
2674                        "  A skipped run carries no metrics: perf-delta runs with \
2675                         KTSTR_PERF_ONLY, which skips any test not marked \
2676                         #[ktstr_test(performance_mode = true)]; host-gated skips land here \
2677                         too. A failed / inconclusive run is excluded from the spread math."
2678                    );
2679                } else {
2680                    println!(
2681                        "  Both sides produced comparable runs but share no scenario / topology \
2682                         / work_type — the two selections have no common test to contrast."
2683                    );
2684                }
2685            } else {
2686                println!(
2687                    "perf-delta --noise-adjust: no metric to display — every compared metric \
2688                     was unchanged at zero, present on only one side, or render-suppressed"
2689                );
2690            }
2691        } else {
2692            print!(
2693                "{}",
2694                format_noise_findings_table(&report.findings, &label_a, &label_b, gate.show_all)
2695            );
2696        }
2697    }
2698    // Per-phase spread block — render-only (never gates), honoring --no-phases /
2699    // --phase / --steps-only / --phase-threshold. Under --phases-only it is the
2700    // ONLY table, so emit an explicit note rather than a silent blank.
2701    let phase_lines = format_noise_phase_findings_lines(
2702        &report.phase_findings,
2703        &report.phase_coverage,
2704        phase_opts,
2705        &label_a,
2706        &label_b,
2707        gate.show_all,
2708    );
2709    if phase_lines.is_empty() && phase_opts.phases_only {
2710        println!(
2711            "perf-delta --noise-adjust: no per-phase noise data to show (no matched \
2712             multi-phase scenario at the selected step, or every per-phase row was \
2713             filtered by --phase / --steps-only / --phase-threshold)"
2714        );
2715    }
2716    for line in phase_lines {
2717        println!("{line}");
2718    }
2719    // Declared gates that never evaluated (metric absent from the compared
2720    // data). Rendered regardless of --phases-only — a silently-inert declared
2721    // gate is important whether or not the aggregate table is shown. Never
2722    // gates the exit.
2723    for line in format_noise_assertion_coverage_lines(&report.assertion_coverage) {
2724        println!("{line}");
2725    }
2726    let regressions = report.regressions();
2727    // Per-phase regressions the author explicitly DECLARED a phase-scoped gate
2728    // for — these gate the exit alongside the aggregate regressions (spread-only
2729    // per-phase findings stay render-only). Computed unconditionally so the exit
2730    // is identical whether or not the phase footer is shown.
2731    let declared_phase_regressions = report.declared_phase_regressions();
2732    // The aggregate summary footer describes the (hidden) aggregate spread, so
2733    // suppress it under --phases-only — which renders ONLY the per-phase
2734    // block.
2735    // The exit still gates on gate_fails(aggregate regressions) +
2736    // `declared_phase_regressions` (computed above) when the footer is hidden.
2737    if !phase_opts.phases_only {
2738        let noisy = report.noisy();
2739        // Composite verdict counts: regressed / improved LEAD (the table shows
2740        // each with its magnitude), stable is the expected residual cited as a
2741        // count only, and informational (changed but no polarity) is cited only
2742        // when present. All render-only — the exit reads gate_fails + declared
2743        // gates, never these.
2744        let improvements = report.improvements();
2745        let stable = report.stable();
2746        let informational = report.informational();
2747        let info_clause = if informational > 0 {
2748            format!(", {informational} informational")
2749        } else {
2750            String::new()
2751        };
2752        // Overall verdict word: STABLE unless a direction clears the cutoff (see
2753        // overall_verdict). Sub-cutoff moves are flagged in the counts above but
2754        // do not shift the verdict off STABLE.
2755        let verdict = overall_verdict(&report, gate);
2756        let stable_note = if verdict == "STABLE" && regressions + improvements > 0 {
2757            " (moves are below the significance cutoff, more likely noise than signal)"
2758        } else {
2759            ""
2760        };
2761        // Per-phase footer counts are shown ONLY when there IS per-phase data
2762        // AND no phase filter is active (no --no-phases / --phase / --steps-only
2763        // / --phase-threshold) — so the counts always match the fully-rendered
2764        // per-phase table, and a single-phase run (no per-phase view) shows no
2765        // confusing 0/0 per-phase clause. Uses the same !any-flag-set
2766        // discipline as the aggregate paired-scenario hint.
2767        let has_phase_data = !report.phase_findings.is_empty() || !report.phase_coverage.is_empty();
2768        let show_phase_footer = has_phase_data
2769            && !phase_opts.no_phases
2770            && phase_opts.phase.is_none()
2771            && !phase_opts.steps_only
2772            && phase_opts.phase_threshold.is_none();
2773        // A REGRESSED verdict with ZERO aggregate regressions can only come from a
2774        // declared PER-PHASE gate (a declared whole-run regression and --must-fail
2775        // both require an aggregate Regression, so regressions >= 1 there). When
2776        // the phase footer below is hidden (a phase filter is active) name the
2777        // source so `overall REGRESSED: 0 regressed` is not self-contradictory;
2778        // when the phase footer shows, its "(declared-gated, exit-affecting)"
2779        // clause already explains it, so skip the redundant note.
2780        let verdict_source =
2781            if verdict.starts_with("REGRESSED") && regressions == 0 && !show_phase_footer {
2782                " (regression is a declared per-phase gate)"
2783            } else {
2784                ""
2785            };
2786        let phase_footer = if show_phase_footer {
2787            // Per-phase regressions are render-only EXCEPT the declared-gated
2788            // ones, which gate the exit — call that out so the count is not read
2789            // as fully render-only when a declared phase gate fired.
2790            let declared_note = if declared_phase_regressions > 0 {
2791                format!(" ({declared_phase_regressions} declared-gated, exit-affecting)")
2792            } else {
2793                " (render-only)".to_string()
2794            };
2795            format!(
2796                "; {} per-phase regression(s){declared_note}, {} per-phase under-sampled (<2 runs)",
2797                report.phase_regressions(),
2798                report.phase_noisy(),
2799            )
2800        } else {
2801            String::new()
2802        };
2803        // Declared gates that never evaluated — a single-line summary of the
2804        // warning block above (shown whenever any gate went unevaluated).
2805        let unevaluated_gates = report.assertion_coverage.len();
2806        let gate_footer = if unevaluated_gates > 0 {
2807            format!("; {unevaluated_gates} declared gate(s) not evaluated")
2808        } else {
2809            String::new()
2810        };
2811        println!(
2812            "perf-delta --noise-adjust: {} paired scenario(s); overall {verdict}: \
2813             {regressions} regressed, {improvements} improved, {stable} stable{info_clause}, \
2814             {noisy} under-sampled (<2 runs){stable_note}{verdict_source}{phase_footer}{gate_footer}",
2815            report.paired_scenarios,
2816        );
2817    }
2818    // Inconclusive: every CHANGED aggregate metric (excluding Stable rows, which
2819    // never gate) had a side with < 2 usable runs — no trustworthy signal either
2820    // way. Surfaced prominently for CI logs, but noise FLAGS, not FAILS, so the
2821    // exit stays 0 unless a confident AGGREGATE regression fired. Suppressed
2822    // under --phases-only (the aggregate table is hidden there).
2823    if !phase_opts.phases_only {
2824        let changed: Vec<&NoiseFinding> = report
2825            .findings
2826            .iter()
2827            .filter(|f| f.kind != NoiseKind::Stable)
2828            .collect();
2829        if !changed.is_empty() && changed.iter().all(|f| f.kind == NoiseKind::Noisy) {
2830            println!(
2831                "perf-delta --noise-adjust: NOTE -- every changed metric had a side with <2 usable \
2832                 runs; raise --noise-adjust N (or investigate why per-side runs failed) for a \
2833                 trustworthy verdict"
2834            );
2835        }
2836    }
2837    // Exit gates on AGGREGATE regressions plus DECLARED phase regressions.
2838    // Spread-only per-phase findings stay render-only (parity with the scalar
2839    // per-phase pass — a narrow-window phase flake must not flip CI red), but a
2840    // phase-scoped gate the author explicitly declared is an opt-in and DOES
2841    // gate (matches the `PerfDeltaAssertion::phase` doc).
2842    // Operator gate: the count / must-fail gate applies to the UNdeclared
2843    // aggregate confident regressions; an author-DECLARED gate — whole-run OR
2844    // phase-scoped — always fails (its own opt-in, orthogonal to the operator's
2845    // count gate).
2846    Ok(noise_exit_code(&report, gate))
2847}
2848
2849/// Exit code for the noise-adjusted compare. Fails (`1`) when EITHER the
2850/// operator gate ([`gate_fails`]) trips on the aggregate confident regressions,
2851/// OR any author-DECLARED regression is present — whole-run
2852/// ([`NoiseReport::declared_regressions`]) or phase-scoped
2853/// ([`NoiseReport::declared_phase_regressions`]). A declared assertion is a
2854/// per-test opt-in that ALWAYS gates on its metric, independent of the
2855/// operator's count / must-fail gate, so a single declared regression fails
2856/// even below `--fail-threshold`. Extracted so the exit decision is
2857/// unit-testable.
2858pub(crate) fn noise_exit_code(report: &NoiseReport, gate: &GateOptions) -> i32 {
2859    let regressing: Vec<&str> = report
2860        .findings
2861        .iter()
2862        .filter(|f| f.kind == NoiseKind::Regression)
2863        .map(|f| f.metric.name)
2864        .collect();
2865    if gate_fails(&regressing, gate)
2866        || report.declared_regressions() > 0
2867        || report.declared_phase_regressions() > 0
2868    {
2869        1
2870    } else {
2871        0
2872    }
2873}
2874
2875/// Overall run verdict word for the `--noise-adjust` summary, derived from the
2876/// confident move counts against the significance cutoff (`--fail-threshold`,
2877/// default 5). Sub-cutoff moves in EITHER direction are FLAGGED (shown in the
2878/// table, counted in the footer) but do NOT shift the verdict off `STABLE` —
2879/// below the cutoff a move is more likely noise than signal, which is the whole
2880/// point of the conservative noise gate. A direction only reads into the verdict
2881/// once it clears the cutoff, and both can hold at once (`REGRESSED + IMPROVED`).
2882///
2883/// The regressed side reuses the exit decision ([`noise_exit_code`]) so a
2884/// `--must-fail` or declared gate that fails the run also reads `REGRESSED` even
2885/// below the count cutoff; the improved side has no gate, so it uses the count
2886/// cutoff alone. Display-only — never the exit basis.
2887pub(crate) fn overall_verdict(report: &NoiseReport, gate: &GateOptions) -> &'static str {
2888    verdict_label(
2889        noise_exit_code(report, gate) == 1,
2890        report.improvements(),
2891        gate.fail_threshold,
2892    )
2893}
2894
2895/// Pure verdict classifier: `regressed` is the run's regression-fail decision;
2896/// `improvements` is the confident-improvement count; `fail_threshold` is the
2897/// significance cutoff (`None` = 5, `Some(0)` disables count significance).
2898/// Split out from [`overall_verdict`] so the STABLE-below-cutoff policy is
2899/// unit-testable without building a full report.
2900pub(crate) fn verdict_label(
2901    regressed: bool,
2902    improvements: usize,
2903    fail_threshold: Option<usize>,
2904) -> &'static str {
2905    let n = fail_threshold.unwrap_or(5);
2906    let improved = n >= 1 && improvements >= n;
2907    match (regressed, improved) {
2908        (true, true) => "REGRESSED + IMPROVED",
2909        (true, false) => "REGRESSED",
2910        (false, true) => "IMPROVED",
2911        (false, false) => "STABLE",
2912    }
2913}
2914
2915/// Compose a noise verdict cell's text: the base label plus any parenthesized,
2916/// comma-joined annotations. Shared by the aggregate and per-phase tables so
2917/// both annotate identically. `noisy spread` (advisory `high_spread`, suppressed
2918/// on a Noisy <2-runs row where it is redundant) and `declared gate` (the row
2919/// carries a declared [`crate::test_support::PerfDeltaAssertion`] — its
2920/// overrides drive the gate, or fall back to the registry defaults if rejected
2921/// as out-of-range on a corrupt sidecar) can co-occur → `REGRESSION (noisy
2922/// spread, declared gate)`.
2923fn compose_noise_verdict_text(
2924    base: &str,
2925    high_spread: bool,
2926    kind: NoiseKind,
2927    gated_by_assertion: bool,
2928) -> String {
2929    let mut annotations: Vec<&str> = Vec::new();
2930    if high_spread && kind != NoiseKind::Noisy {
2931        annotations.push("noisy spread");
2932    }
2933    if gated_by_assertion {
2934        annotations.push("declared gate");
2935    }
2936    if annotations.is_empty() {
2937        base.to_string()
2938    } else {
2939        format!("{base} ({})", annotations.join(", "))
2940    }
2941}
2942
2943/// Render the per-metric noise-adjusted findings as a table for
2944/// `perf-delta --noise-adjust`: one row per (scenario, metric) with each
2945/// side's `mean [min-max] spread%` and the colored verdict. Includes
2946/// [`NoiseKind::Stable`] rows so the operator sees every compared metric,
2947/// not only the changed ones. Pure (returns the rendered string with a
2948/// trailing newline) so the row/verdict mapping is unit-testable without
2949/// capturing stdout.
2950pub(crate) fn format_noise_findings_table(
2951    findings: &[NoiseFinding],
2952    label_a: &str,
2953    label_b: &str,
2954    show_all: bool,
2955) -> String {
2956    use comfy_table::{Cell, Color};
2957    // Default view shows only MEANINGFUL rows (confident regression /
2958    // improvement / informational). Stable (unchanged / immaterial) and
2959    // Noisy (<2 usable runs) rows are hidden unless `show_all`; their COUNTS
2960    // still print in the caller's footer, and the gate reads the full
2961    // classified set, so this suppression is display-only.
2962    let visible: Vec<&NoiseFinding> = findings
2963        .iter()
2964        .filter(|f| show_all || !matches!(f.kind, NoiseKind::Stable | NoiseKind::Noisy))
2965        .collect();
2966    if visible.is_empty() {
2967        if findings.is_empty() {
2968            return String::new();
2969        }
2970        let noisy = findings
2971            .iter()
2972            .filter(|f| f.kind == NoiseKind::Noisy)
2973            .count();
2974        return format!(
2975            "perf-delta --noise-adjust: {} metric(s) compared, none meaningfully changed \
2976             ({noisy} under-sampled); re-run with --all-metrics to see them\n",
2977            findings.len(),
2978        );
2979    }
2980    let mut table = crate::cli::new_table();
2981    table.set_header(vec![
2982        "TEST / METRIC".to_string(),
2983        format!("{label_a} (A: mean [min-max] spread%)"),
2984        format!("{label_b} (B: mean [min-max] spread%)"),
2985        "VERDICT".to_string(),
2986    ]);
2987    for f in visible {
2988        let (base, color) = match f.kind {
2989            NoiseKind::Regression => ("REGRESSION", Color::Red),
2990            NoiseKind::Improvement => ("improvement", Color::Green),
2991            NoiseKind::Noisy => ("NOISY (<2 runs)", Color::Yellow),
2992            NoiseKind::Informational => ("informational", Color::Blue),
2993            NoiseKind::Stable => ("stable", Color::Grey),
2994        };
2995        // Verdict annotations (parenthesized, comma-joined). `noisy spread` is
2996        // ADVISORY (high_spread) — flags a noisy side without changing the
2997        // classification, redundant on a Noisy <2-runs row so omitted there.
2998        // `declared gate` marks a row that CARRIES a declared PerfDeltaAssertion
2999        // (its overrides drive the gate, or fall back to registry defaults if
3000        // rejected as out-of-range on a corrupt sidecar), so the operator can
3001        // tell an author-declared gate from a pure registry-default one.
3002        let verdict_text =
3003            compose_noise_verdict_text(base, f.verdict.high_spread, f.kind, f.gated_by_assertion);
3004        let v = &f.verdict;
3005        table.add_row(vec![
3006            Cell::new(format!("{} / {}", f.pairing_label, f.metric.name)),
3007            Cell::new(format!(
3008                "{:.1} [{:.1}-{:.1}] {:.2}%",
3009                v.a.mean, v.a.min, v.a.max, v.a.spread_pct
3010            )),
3011            Cell::new(format!(
3012                "{:.1} [{:.1}-{:.1}] {:.2}%",
3013                v.b.mean, v.b.min, v.b.max, v.b.spread_pct
3014            )),
3015            Cell::new(verdict_text).fg(color),
3016        ]);
3017    }
3018    format!("{table}\n")
3019}
3020
3021/// One-line description of a declared gate's overrides for the
3022/// not-evaluated warning: the thresholds it WOULD have applied. All-`None`
3023/// (a bare `PerfDeltaAssertion::new(metric)`) renders `registry defaults` —
3024/// a presence-checked gate that inherits the registry `default_abs`/
3025/// `default_rel`/polarity.
3026fn describe_declared_gate(a: &crate::test_support::PerfDeltaAssertionRecord) -> String {
3027    let mut parts: Vec<String> = Vec::new();
3028    if let Some(pct) = a.max_regression_pct {
3029        parts.push(format!("max_regression_pct={pct}"));
3030    }
3031    if let Some(abs) = a.min_abs {
3032        parts.push(format!("min_abs={abs}"));
3033    }
3034    if let Some(dir) = a.direction {
3035        parts.push(format!("direction={dir:?}"));
3036    }
3037    if parts.is_empty() {
3038        "registry defaults".to_string()
3039    } else {
3040        parts.join(", ")
3041    }
3042}
3043
3044/// Render the declared perf gates that never evaluated (a metric absent from
3045/// the compared data) as warning lines for `perf-delta --noise-adjust`: a
3046/// TEST | METRIC | PHASE | DECLARED GATE table, so an author whose declared
3047/// [`crate::test_support::PerfDeltaAssertion`] silently did not fire sees it
3048/// rather than mistaking a not-evaluated gate for a passing one. Never gates
3049/// the exit (a gate that could not evaluate is not a regression). Pure —
3050/// returns the lines (empty when there are none) so the mapping is
3051/// unit-testable without capturing stdout.
3052pub(crate) fn format_noise_assertion_coverage_lines(
3053    coverage: &[NoiseAssertionCoverage],
3054) -> Vec<String> {
3055    use comfy_table::{Cell, Color};
3056    let mut lines = Vec::new();
3057    if coverage.is_empty() {
3058        return lines;
3059    }
3060    let mut rows: Vec<&NoiseAssertionCoverage> = coverage.iter().collect();
3061    rows.sort_by(|a, b| {
3062        a.pairing_label
3063            .cmp(&b.pairing_label)
3064            .then_with(|| a.assertion.metric.cmp(&b.assertion.metric))
3065            .then_with(|| a.assertion.phase.cmp(&b.assertion.phase))
3066    });
3067    lines.push(String::new());
3068    lines.push(
3069        "declared perf gate(s) NOT evaluated — the metric was absent from the compared \
3070         data (workload no longer emits it, a one-sided/failed run, or a Rate with no \
3071         samples), so the declared gate silently did not fire:"
3072            .to_string(),
3073    );
3074    let mut table = crate::cli::new_table();
3075    table.set_header(vec![
3076        "TEST".to_string(),
3077        "METRIC".to_string(),
3078        "PHASE".to_string(),
3079        "DECLARED GATE".to_string(),
3080    ]);
3081    for c in rows {
3082        let phase = match c.assertion.phase {
3083            None => "aggregate".to_string(),
3084            Some(k) => k.to_string(),
3085        };
3086        table.add_row(vec![
3087            Cell::new(&c.pairing_label),
3088            Cell::new(&c.assertion.metric).fg(Color::Yellow),
3089            Cell::new(phase),
3090            Cell::new(describe_declared_gate(&c.assertion)),
3091        ]);
3092    }
3093    lines.push(format!("{table}"));
3094    lines
3095}
3096
3097/// Render the per-phase noise-adjusted findings as lines for
3098/// `perf-delta --noise-adjust`: a per-phase spread table (PHASE | TEST /
3099/// METRIC | A mean[min-max] spread% | B ... | VERDICT) plus a one-sided
3100/// coverage table (SIDE | TEST | PHASE | METRIC | VALUE). The TEST column
3101/// carries the full pairing-key label (scenario plus every pairing dim),
3102/// matching the scalar compare path — a scenario shared across topologies
3103/// renders as distinct rows. Mirrors
3104/// [`format_noise_findings_table`], honoring
3105/// [`PhaseDisplayOptions`] (no_phases / phase / steps_only / phase_threshold).
3106/// Render-only: these rows never gate. Pure — returns the lines (empty when
3107/// suppressed / no per-phase data) so the row/verdict mapping is unit-testable
3108/// without capturing stdout.
3109pub(crate) fn format_noise_phase_findings_lines(
3110    phase_findings: &[NoisePhaseFinding],
3111    phase_coverage: &[NoisePhaseCoverage],
3112    phase_opts: &PhaseDisplayOptions,
3113    label_a: &str,
3114    label_b: &str,
3115    show_all: bool,
3116) -> Vec<String> {
3117    use comfy_table::{Cell, Color};
3118    let mut lines = Vec::new();
3119    if phase_opts.no_phases {
3120        return lines;
3121    }
3122    // Rows passing the phase-axis filters (`--phase` / `--steps-only`) and the
3123    // `--phase-threshold` spread gate, BEFORE the meaningful-only display filter.
3124    // Retained so the collapse summary below can report how many rows the default
3125    // view suppressed.
3126    let phase_filtered: Vec<&NoisePhaseFinding> = phase_findings
3127        .iter()
3128        .filter(|f| phase_opts.matches_phase(f.step_index))
3129        .filter(|f| phase_opts.passes_noise_spread_threshold(&f.verdict))
3130        .collect();
3131    // Default view shows only MEANINGFUL rows (regression / improvement /
3132    // informational); Stable + Noisy rows are hidden unless `show_all`
3133    // (`--all-metrics`), mirroring the aggregate table
3134    // ([`format_noise_findings_table`]). Display-only: the footer still reports
3135    // the per-phase regression / under-sampled COUNTS from the unfiltered report,
3136    // and the exit gate reads the unfiltered findings — so suppressing rows here
3137    // changes neither the counts nor the pass/fail.
3138    let mut findings: Vec<&NoisePhaseFinding> = phase_filtered
3139        .iter()
3140        .copied()
3141        .filter(|f| show_all || !matches!(f.kind, NoiseKind::Stable | NoiseKind::Noisy))
3142        .collect();
3143    let mut coverage: Vec<&NoisePhaseCoverage> = phase_coverage
3144        .iter()
3145        .filter(|c| phase_opts.matches_phase(c.step_index))
3146        .collect();
3147    let had_findings = !findings.is_empty();
3148    // Spread rows existed but were all suppressed as stable/noisy in the default
3149    // view: a one-line hint (naming `--all-metrics`, wording matched to the
3150    // aggregate collapse) keeps the suppression discoverable — surfaced whether
3151    // the block is otherwise empty OR a coverage table follows, so the suppressed
3152    // rows are never silently gone. Only in the default view: under `show_all`
3153    // nothing is suppressed, so an empty result there means there was genuinely no
3154    // per-phase spread data.
3155    let suppressed_hint: Option<String> =
3156        if findings.is_empty() && !phase_filtered.is_empty() && !show_all {
3157            let noisy = phase_filtered
3158                .iter()
3159                .filter(|f| f.kind == NoiseKind::Noisy)
3160                .count();
3161            Some(format!(
3162                "perf-delta --noise-adjust: {} per-phase metric(s) compared, none meaningfully \
3163                 changed ({noisy} under-sampled); re-run with --all-metrics to see them",
3164                phase_filtered.len(),
3165            ))
3166        } else {
3167            None
3168        };
3169    if findings.is_empty() && coverage.is_empty() {
3170        // Nothing else to render — the hint (if any) is the whole block.
3171        lines.extend(suppressed_hint);
3172        return lines;
3173    }
3174    lines.push(String::new());
3175    if had_findings {
3176        // "per-phase spread:" heads the findings table ONLY — a coverage-only
3177        // section (no spread rows) gets its own header below, so the label
3178        // never mislabels a table.
3179        lines.push("per-phase spread:".to_string());
3180        // step_index-first (BASELINE..Step[N] time order), then pairing label,
3181        // then metric — a stable, top-down-by-phase-boundary order.
3182        findings.sort_by(|a, b| {
3183            a.step_index
3184                .cmp(&b.step_index)
3185                .then_with(|| a.pairing_label.cmp(&b.pairing_label))
3186                .then_with(|| a.metric.name.cmp(b.metric.name))
3187        });
3188        let mut table = crate::cli::new_table();
3189        table.set_header(vec![
3190            "PHASE".to_string(),
3191            "TEST / METRIC".to_string(),
3192            format!("{label_a} (A: mean [min-max] spread%)"),
3193            format!("{label_b} (B: mean [min-max] spread%)"),
3194            "VERDICT".to_string(),
3195        ]);
3196        for f in findings {
3197            let (base, color) = match f.kind {
3198                NoiseKind::Regression => ("REGRESSION", Color::Red),
3199                NoiseKind::Improvement => ("improvement", Color::Green),
3200                NoiseKind::Noisy => ("NOISY (<2 runs)", Color::Yellow),
3201                NoiseKind::Informational => ("informational", Color::Blue),
3202                NoiseKind::Stable => ("stable", Color::Grey),
3203            };
3204            // Same annotation composition as the aggregate table (advisory
3205            // `noisy spread` + `declared gate`).
3206            let verdict_text = compose_noise_verdict_text(
3207                base,
3208                f.verdict.high_spread,
3209                f.kind,
3210                f.gated_by_assertion,
3211            );
3212            let v = &f.verdict;
3213            table.add_row(vec![
3214                Cell::new(format!("{}: {}", f.step_index, f.label)),
3215                Cell::new(format!("{} / {}", f.pairing_label, f.metric.name)),
3216                Cell::new(format!(
3217                    "{:.1} [{:.1}-{:.1}] {:.2}%",
3218                    v.a.mean, v.a.min, v.a.max, v.a.spread_pct
3219                )),
3220                Cell::new(format!(
3221                    "{:.1} [{:.1}-{:.1}] {:.2}%",
3222                    v.b.mean, v.b.min, v.b.max, v.b.spread_pct
3223                )),
3224                Cell::new(verdict_text).fg(color),
3225            ]);
3226        }
3227        lines.push(table.to_string());
3228    } else {
3229        // Spread rows were all suppressed but a coverage table follows: surface
3230        // the suppression hint before it so `--all-metrics` stays discoverable.
3231        lines.extend(suppressed_hint);
3232    }
3233    if !coverage.is_empty() {
3234        // Separate from the spread table above only when one was rendered; the
3235        // section-leading blank already precedes a coverage-only block.
3236        if had_findings {
3237            lines.push(String::new());
3238        }
3239        lines.push("per-phase coverage asymmetry (one-sided metrics):".to_string());
3240        coverage.sort_by(|a, b| {
3241            a.step_index
3242                .cmp(&b.step_index)
3243                .then_with(|| a.present_side.as_str().cmp(b.present_side.as_str()))
3244                .then_with(|| a.pairing_label.cmp(&b.pairing_label))
3245                .then_with(|| a.metric.map(|m| m.name).cmp(&b.metric.map(|m| m.name)))
3246        });
3247        let mut table = crate::cli::new_table();
3248        table.set_header(vec!["SIDE", "TEST", "PHASE", "METRIC", "VALUE"]);
3249        for c in coverage {
3250            // A whole one-sided phase with no readable metric renders `—` in the
3251            // METRIC + VALUE columns.
3252            let metric_cell = c.metric.map(|m| m.name).unwrap_or("—");
3253            // Bare {:.2} with NO display_unit — matching the noise aggregate
3254            // findings table,
3255            // so a unit-carrying metric renders consistently across all of them.
3256            let value_cell = match c.value {
3257                Some(v) => format!("{v:.2}"),
3258                None => "—".to_string(),
3259            };
3260            table.add_row(vec![
3261                Cell::new(c.present_side.as_str()),
3262                Cell::new(c.pairing_label.as_str()),
3263                Cell::new(format!("{}: {}", c.step_index, c.label)),
3264                Cell::new(metric_cell),
3265                Cell::new(value_cell),
3266            ]);
3267        }
3268        lines.push(table.to_string());
3269    }
3270    lines
3271}
3272
3273/// Render the scalar findings table for `perf-delta`.
3274///
3275/// Extracted from [`compare_partitions`] verbatim; the
3276/// `--phases-only` gate stays at the call site so this prints
3277/// unconditionally when invoked.
3278fn print_scalar_findings_table(report: &CompareReport, label_a: &str, label_b: &str) {
3279    use comfy_table::{Cell, Color};
3280    let mut table = crate::cli::new_table();
3281    table.set_header(vec!["TEST", "METRIC", label_a, label_b, "DELTA", "VERDICT"]);
3282    for f in &report.findings {
3283        let (verdict_text, verdict_color) = match f.kind {
3284            FindingKind::Regression => ("REGRESSION", Color::Red),
3285            FindingKind::Improvement => ("improvement", Color::Green),
3286            // Directionless metric: shown, never gated. Neutral color.
3287            FindingKind::Informational => ("informational", Color::Blue),
3288        };
3289        // PairingKey's first slot is scenario; subsequent slots
3290        // are the pairing-dim values in canonical order. Joining
3291        // with `/` produces a label whose shape mirrors the
3292        // pairing-dim count — so a comparison that pairs on
3293        // (topology, work_type) renders a `scenario/topology/work_type`
3294        // label, while a comparison that slices on most dims
3295        // renders a shorter identifier. The operator can always
3296        // cross-reference the "pairing on:" header line above to
3297        // see what each segment means.
3298        let label = f.pairing_key.label();
3299        table.add_row(vec![
3300            Cell::new(label),
3301            Cell::new(f.metric.name),
3302            Cell::new(format!("{:.2}", f.val_a)),
3303            Cell::new(format!("{:.2}", f.val_b)),
3304            Cell::new(format!("{:+.2}{}", f.delta, f.metric.display_unit)),
3305            Cell::new(verdict_text).fg(verdict_color),
3306        ]);
3307    }
3308    println!("{table}");
3309}
3310
3311/// Render the scalar summary block for `perf-delta` —
3312/// regressions / improvements / unchanged + skipped-failed +
3313/// per-group pass counts + new_in_b / removed_from_a. All lines
3314/// describe the scalar findings table; the `--phases-only` gate
3315/// stays at the call site so this prints unconditionally when
3316/// invoked.
3317fn print_summary_block(
3318    report: &CompareReport,
3319    avg_a: &Option<Vec<AveragedGroup>>,
3320    avg_b: &Option<Vec<AveragedGroup>>,
3321    label_a: &str,
3322    label_b: &str,
3323) {
3324    println!();
3325    println!(
3326        "summary: {} regressions, {} improvements, {} informational, {} unchanged",
3327        report.regressions, report.improvements, report.informational, report.unchanged,
3328    );
3329    if report.excluded_pairs > 0 {
3330        println!(
3331            "  {} pairing-key row pair(s) excluded from regression math because one \
3332             or both sides was excluded (failed, inconclusive, skipped, or an inverted expected-failure run)",
3333            report.excluded_pairs,
3334        );
3335    }
3336    if let (Some(avg_a), Some(avg_b)) = (avg_a, avg_b) {
3337        let block = format_per_group_pass_counts(avg_a, avg_b, label_a, label_b);
3338        if !block.is_empty() {
3339            print!("{block}");
3340        }
3341    }
3342    if report.new_in_b > 0 {
3343        println!(
3344            "  {} row(s) new in '{}' (no matching key in '{}')",
3345            report.new_in_b, label_b, label_a,
3346        );
3347    }
3348    if report.denomination_mismatches > 0 {
3349        println!(
3350            "  {} of those differ from '{}' ONLY by throughput denomination \
3351             (wall-era vs cpu-era sidecars): the throughput rate keys kept their \
3352             names across the wall→CPU-second change, so cross-era values are \
3353             refused, never compared — re-run the older side to regenerate",
3354            report.denomination_mismatches, label_a,
3355        );
3356    }
3357    if report.removed_from_a > 0 {
3358        println!(
3359            "  {} row(s) removed from '{}' (no matching key in '{}')",
3360            report.removed_from_a, label_a, label_b,
3361        );
3362    }
3363    for line in format_coverage_diff_lines(report, label_a, label_b) {
3364        println!("{line}");
3365    }
3366}
3367
3368/// Render the coverage-diff lines (metrics present on exactly one side of a
3369/// paired row) for [`print_summary_block`]. Pure (returns the lines, empty
3370/// when there are no coverage diffs) so the present/absent label mapping by
3371/// [`ComparePartition`] is unit-testable without capturing stdout.
3372pub(crate) fn format_coverage_diff_lines(
3373    report: &CompareReport,
3374    label_a: &str,
3375    label_b: &str,
3376) -> Vec<String> {
3377    if report.coverage_diffs.is_empty() {
3378        return Vec::new();
3379    }
3380    let mut lines = vec![format!(
3381        "  {} metric(s) present on only one side (coverage difference, \
3382         not a regression):",
3383        report.coverage_diffs.len(),
3384    )];
3385    for cd in &report.coverage_diffs {
3386        // present_side names the side that HAS the metric; the other is absent.
3387        let (present, absent) = match cd.present_side {
3388            ComparePartition::A => (label_a, label_b),
3389            ComparePartition::B => (label_b, label_a),
3390        };
3391        lines.push(format!(
3392            "    {} / {} = {:.2} in '{}', absent in '{}'",
3393            cd.pairing_key.label(),
3394            cd.metric.name,
3395            cd.value,
3396            present,
3397            absent,
3398        ));
3399    }
3400    lines
3401}
3402
3403/// Print the host-context delta for `perf-delta`. Same
3404/// first-Some(host) baseline `compare_partitions` uses — picking
3405/// representative hosts off the partitioned sidecars rather than
3406/// the full pool so the delta reflects what actually fed the
3407/// comparison.
3408fn print_host_context_delta(
3409    pool: &[crate::test_support::SidecarResult],
3410    rows: &[GauntletRow],
3411    filter_a: &RowFilter,
3412    filter_b: &RowFilter,
3413    label_a: &str,
3414    label_b: &str,
3415) {
3416    // Zip the pool with the pre-computed `rows` (built once above
3417    // via `pool.iter().map(sidecar_to_row).collect()`) so the
3418    // per-side filter reuses the existing row instead of calling
3419    // `sidecar_to_row` a second and third time. `pool` and `rows`
3420    // are the same length and same iteration order by construction.
3421    let sidecars_a: Vec<&crate::test_support::SidecarResult> = pool
3422        .iter()
3423        .zip(rows.iter())
3424        .filter(|(_, r)| filter_a.matches(r))
3425        .map(|(s, _)| s)
3426        .collect();
3427    let sidecars_b: Vec<&crate::test_support::SidecarResult> = pool
3428        .iter()
3429        .zip(rows.iter())
3430        .filter(|(_, r)| filter_b.matches(r))
3431        .map(|(s, _)| s)
3432        .collect();
3433    let host_a = sidecars_a.iter().find_map(|s| s.host.as_ref());
3434    let host_b = sidecars_b.iter().find_map(|s| s.host.as_ref());
3435    print!("{}", format_host_delta(host_a, host_b, label_a, label_b));
3436}
3437
3438/// Render the host-context delta section of `perf-delta`
3439/// as a block of text ready to `print!`. Extracted as a pure
3440/// function of `(Option<&HostContext>, Option<&HostContext>, &str,
3441/// &str)` so the five match arms can be unit-tested without
3442/// fixturing a real run directory.
3443///
3444/// The returned string is either empty (when both sides have no
3445/// host data — nothing to print) or ends with a newline so callers
3446/// can chain further output. Single-side cases print a clear
3447/// "captured in X only, delta unavailable" message rather than
3448/// silently suppressing the section — a mixed-tooling-version run
3449/// comparison should surface the asymmetry.
3450/// Format the one-line averaging-mode header that prints above
3451/// the comparison table.
3452///
3453/// Pure function of (`pre_agg_a`, `pre_agg_b`, `a`, `b`) so the
3454/// exact-string contract — the operator-visible "averaged across
3455/// N runs (A) and M runs (B)" surface — can be unit-tested
3456/// without capturing stdout from `compare_partitions`.
3457///
3458/// `pre_agg_a` / `pre_agg_b` are the post-typed-filter contributor
3459/// row counts (i.e. the number of sidecar rows that fed
3460/// [`group_and_average_by`]), NOT the post-aggregation unique-key
3461/// counts. The two answer different operator questions; the
3462/// header surfaces the contributor count because that's the
3463/// "how many trials got folded?" intuition the averaging fold
3464/// is actually delivering.
3465pub(crate) fn format_average_header(
3466    pre_agg_a: usize,
3467    pre_agg_b: usize,
3468    a: &str,
3469    b: &str,
3470) -> String {
3471    format!("averaged across {pre_agg_a} runs ({a}) and {pre_agg_b} runs ({b})")
3472}
3473
3474/// Format the per-group `passes_observed/total_observed` block
3475/// that prints below the summary line.
3476///
3477/// Pure function of (`avg_a`, `avg_b`, `a`, `b`) so the rendered
3478/// surface — one line per (scenario, topology, work_type) group
3479/// present on either side, with `N/M` per side and `-` for any
3480/// side that lacks the group — can be unit-tested without
3481/// capturing stdout. Returns the trailing-newline-terminated
3482/// block, or empty string when neither side has groups.
3483///
3484/// Line shape:
3485/// `  scenario/topology/work_type: {a}=N/M {b}=N/M`
3486///
3487/// The leading two-space indent matches the sibling
3488/// `summary:` block's continuation lines (e.g.
3489/// `"  N (scenario, topology, work_type) row pair(s) skipped..."`)
3490/// so the per-group block reads as a continuation of the same
3491/// summary section. A blank line separates this block from the
3492/// preceding `summary:` line for readability.
3493///
3494/// Groups present on only one side render `-` for the missing
3495/// side (also counted in `compare_rows`' `new_in_b` /
3496/// `removed_from_a` upstream — the per-group block surfaces the
3497/// asymmetry by name so the operator can see *which* groups went
3498/// missing without cross-referencing the summary counters).
3499pub(crate) fn format_per_group_pass_counts(
3500    avg_a: &[AveragedGroup],
3501    avg_b: &[AveragedGroup],
3502    a: &str,
3503    b: &str,
3504) -> String {
3505    type SummaryKey<'a> = (&'a str, &'a str, &'a str);
3506    type SummaryValue<'a> = (Option<&'a AveragedGroup>, Option<&'a AveragedGroup>);
3507    let mut keys: BTreeMap<SummaryKey<'_>, SummaryValue<'_>> = BTreeMap::new();
3508    for ar in avg_a {
3509        let k = (
3510            ar.row.scenario.as_str(),
3511            ar.row.topology.as_str(),
3512            ar.row.work_type.as_str(),
3513        );
3514        keys.entry(k).or_insert((None, None)).0 = Some(ar);
3515    }
3516    for br in avg_b {
3517        let k = (
3518            br.row.scenario.as_str(),
3519            br.row.topology.as_str(),
3520            br.row.work_type.as_str(),
3521        );
3522        keys.entry(k).or_insert((None, None)).1 = Some(br);
3523    }
3524    if keys.is_empty() {
3525        return String::new();
3526    }
3527    let mut out = String::new();
3528    out.push('\n');
3529    out.push_str(
3530        "per-group pass counts (passes/total + skip/inconc/fail breakdown when non-zero):\n",
3531    );
3532    for ((scn, topo, wt), (ka, kb)) in keys.into_iter() {
3533        let fmt_side = |r: Option<&AveragedGroup>| -> String {
3534            let Some(x) = r else {
3535                return "-".to_string();
3536            };
3537            // Mirror format_dimension_summary's 4-state breakdown —
3538            // operators reading per-group lines must be able to
3539            // distinguish skip / inconclusive / fail buckets, not
3540            // see them collapsed into the (total - pass) denominator
3541            // gap. Skip silently rendering buckets that are zero so
3542            // the common-case "all passed" line stays terse.
3543            let mut s = format!("{}/{}", x.passes_observed, x.total_observed);
3544            let mut extras: Vec<String> = Vec::with_capacity(3);
3545            if x.skips_observed > 0 {
3546                extras.push(format!("{} skip", x.skips_observed));
3547            }
3548            if x.inconclusives_observed > 0 {
3549                extras.push(format!("{} inc", x.inconclusives_observed));
3550            }
3551            if x.failures_observed > 0 {
3552                extras.push(format!("{} fail", x.failures_observed));
3553            }
3554            if !extras.is_empty() {
3555                s.push_str(&format!(" ({})", extras.join(", ")));
3556            }
3557            s
3558        };
3559        out.push_str(&format!(
3560            "  {scn}/{topo}/{wt}: {a}={pa} {b}={pb}\n",
3561            pa = fmt_side(ka),
3562            pb = fmt_side(kb),
3563        ));
3564    }
3565    out
3566}
3567
3568pub(crate) fn format_host_delta(
3569    host_a: Option<&crate::host_context::HostContext>,
3570    host_b: Option<&crate::host_context::HostContext>,
3571    a: &str,
3572    b: &str,
3573) -> String {
3574    match (host_a, host_b) {
3575        (Some(ha), Some(hb)) => {
3576            let delta = ha.diff(hb);
3577            if delta.is_empty() {
3578                // Identical hosts: surface arch when both sides
3579                // carry it so the operator sees WHAT is identical
3580                // (the two runs share x86_64 vs both being aarch64
3581                // is the operator's question). When
3582                // either side leaves arch as `None` (pre-host-
3583                // context-landing archive, or arch probe failed
3584                // on at least one side), fall through to the
3585                // bare "identical" message — emitting a partial
3586                // hint would mislead the reader into thinking
3587                // the silent side disagreed.
3588                match (ha.arch.as_deref(), hb.arch.as_deref()) {
3589                    (Some(arch_a), Some(arch_b)) if arch_a == arch_b => {
3590                        format!("\nhost: identical between '{a}' and '{b}' (arch: {arch_a})\n",)
3591                    }
3592                    _ => format!("\nhost: identical between '{a}' and '{b}'\n"),
3593                }
3594            } else {
3595                format!("\nhost delta ('{a}' → '{b}'):\n{delta}")
3596            }
3597        }
3598        (Some(_), None) => {
3599            format!("\nhost: captured in '{a}' only, delta unavailable\n")
3600        }
3601        (None, Some(_)) => {
3602            format!("\nhost: captured in '{b}' only, delta unavailable\n")
3603        }
3604        (None, None) => String::new(),
3605    }
3606}