ktstr/assert/
run_metrics.rs

1use super::*;
2
3/// Aggregated statistics across all cgroups in a scenario.
4#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize, crate::Claim)]
5pub struct ScenarioStats {
6    /// Per-cgroup stats, one entry per cgroup.
7    pub cgroups: Vec<CgroupStats>,
8    /// Sum of workers across all cgroups.
9    pub total_workers: usize,
10    /// Sum of per-cgroup distinct CPU counts (not deduplicated across cgroups).
11    pub total_cpus: usize,
12    /// Sum of migration counts across all cgroups.
13    pub total_migrations: u64,
14    /// Worst spread across any cgroup (highest).
15    pub worst_spread: f64,
16    /// Worst gap across any cgroup (highest, ms). Paired with
17    /// `worst_gap_cpu` — both come from the same cgroup.
18    pub worst_gap_ms: u64,
19    /// CPU where the worst gap occurred across all cgroups. Paired
20    /// with `worst_gap_ms` — both come from the same cgroup.
21    pub worst_gap_cpu: usize,
22    /// Worst migration ratio across any cgroup (highest).
23    pub worst_migration_ratio: f64,
24    /// Sum of iteration counts across all cgroups.
25    pub total_iterations: u64,
26    // worst_page_locality is NO LONGER a typed field: it is
27    // `crate::stats::MetricKind::WorstLowest` (NumaLocal/NumaTotal), re-pooled
28    // None-aware post-merge by `populate_run_distribution_metrics` from the
29    // per-phase NUMA carriers (the lowest per-cgroup page_locality, a measured
30    // 0.0 winning) — the `worst_page_locality` method shared with
31    // `run_metric`. The deleted typed field folded via `fold_lowest_nonzero`,
32    // which SKIPPED a measured 0.0 and reported a better-than-worst cross-run
33    // value; the ext re-pool fixes both the field and the read surfaces at once.
34    // worst_cross_node_migration_ratio is NO LONGER a typed field: it is
35    // `crate::stats::MetricKind::WorstCrossNodeRatio`, re-pooled post-merge by
36    // `populate_run_distribution_metrics` from the per-phase NUMA carriers (the
37    // MAX per-cgroup cross-node churn ratio over the latest residency total) via
38    // the `worst_cross_node_migration_ratio` method shared with `run_metric`. The
39    // deleted typed `Gauge(Last)` field/GauntletRow column was merge-max-folded
40    // within-run but cross-run averaged each run's value over `passes_observed`
41    // (folding a NUMA-less passing run's 0.0 sentinel into the mean), AND
42    // diverged from `run_metric` (which already re-derived from the per-phase
43    // carriers) — so the sidecar and the in-test read gave different values on
44    // multi-phase runs. The ext re-pool writes no key for a never-measured run,
45    // so the MEAN divides only by runs that measured NUMA. A CHURN ratio
46    // (cumulative migration EVENTS / final-snapshot resident pages), so it can
47    // exceed 1.0; NOT a bounded `[0,1]` fraction.
48    // worst_wake_latency_tail_ratio is NO LONGER a typed field: it is
49    // `crate::stats::MetricKind::WakeLatencyTailRatio`, re-selected into
50    // `ext_metrics` post-merge by `populate_run_distribution_metrics` (max
51    // over the per-cgroup `CgroupStats::wake_latency_tail_ratio` values,
52    // floor-gated below WAKE_LATENCY_TAIL_RATIO_MIN_ITERATIONS); `MetricDef::read`
53    // surfaces it via the ext fallback.
54    /// Extensible metrics for the generic comparison pipeline.
55    /// Populated from per-cgroup ext_metrics (worst value across cgroups).
56    pub ext_metrics: BTreeMap<String, f64>,
57    /// Per-phase metric buckets in step-index order. A scenario
58    /// with N Steps populates `N + 1` entries: phase 0 is the
59    /// BASELINE settle window before Step 0 fires, phases
60    /// 1..=N align with Step 0..Step N-1 in scenario order
61    /// (1-indexed Steps so the BASELINE encoding doesn't collide
62    /// with first-Step's index).
63    ///
64    /// Empty when the scenario produced no periodic captures
65    /// (Default::default() yields `vec![]`). The existing
66    /// flat-bucket scalars on this struct are independent of the
67    /// per-phase view — they remain the "all phases merged"
68    /// reading, unchanged in semantics by the introduction of
69    /// `phases`.
70    ///
71    /// **Auto-populated by the framework**: scenarios that fire
72    /// periodic captures (via
73    /// [`crate::test_support::KtstrTestEntry::num_snapshots`] or
74    /// [`crate::scenario::ops::Op::CaptureSnapshot`]) have this
75    /// field populated automatically inside
76    /// `crate::test_support::eval`'s `evaluate_vm_result` —
77    /// test code never needs to call
78    /// [`crate::assert::build_phase_buckets`] manually. The auto-
79    /// populate path drains the snapshot bridge from the
80    /// [`crate::vmm::VmResult`] returned by the framework and folds
81    /// the per-sample readings through
82    /// `crate::stats::aggregate_samples_for_phase` per metric.
83    /// Single-phase scenarios that fire no captures leave this
84    /// `vec![]`; the flat-bucket scalars on this struct cover the
85    /// single-phase case.
86    ///
87    /// See [`PhaseBucket`] for the per-phase shape.
88    #[serde(default)]
89    pub phases: Vec<PhaseBucket>,
90}
91
92impl ScenarioStats {
93    /// Look up the phase bucket for a [`Phase`] — `Phase::BASELINE` for the
94    /// pre-first-Step settle window, `Phase::step(k)` for the test author's
95    /// 0-indexed Step k. The typed `Phase` keeps the 1-indexed encoding (Step 0
96    /// lives at the underlying `step_index = 1`) invisible at the call site.
97    ///
98    /// Returns `None` when no bucket with that phase exists (single-phase
99    /// scenario, the scenario didn't reach that Step, or a phase past the last).
100    pub fn phase(&self, phase: Phase) -> Option<&PhaseBucket> {
101        self.phases.iter().find(|p| p.step_index == phase.as_u16())
102    }
103
104    /// Shortcut: look up a single metric value in a specific
105    /// phase by phase-index. Returns `None` when:
106    /// (a) the phase is absent (no bucket with `step_index` in
107    ///     [`Self::phases`]),
108    /// (b) the phase exists but had no finite samples for that
109    ///     metric, OR
110    /// (c) `metric` is a dynamic [`crate::stats::MetricId`] key (a
111    ///     scheduler-runtime / payload string) not present in this phase's
112    ///     stores. A built-in [`crate::stats::BuiltinMetric`] cannot be a
113    ///     typo — that is a compile error — so this case needs a dynamic key.
114    ///
115    /// Two stores are checked: [`PhaseBucket::metrics`] (via
116    /// [`PhaseBucket::get`]) and, failing that, the cross-cgroup phase
117    /// SUM of a per-cgroup Counter ([`PhaseBucket::cgroup_counter_total`])
118    /// for `"total_migrations"` / `"total_iterations"` / `"total_cpu_time_ns"` —
119    /// registered `Counter`s with no per-sample source, so they live ONLY in the
120    /// per-cgroup carriers; without this fallback the pooled lookup
121    /// returned a silent `None` while the per-cgroup
122    /// [`Self::phase_cgroup_metric`] surfaced the value. Symmetric with
123    /// [`crate::vmm::VmResult::phase_metric`].
124    ///
125    /// Sentinel-free: `Some(0.0)` means the reducer produced a
126    /// real zero from finite samples, NOT "missing data". See
127    /// [`PhaseBucket::metrics`] for the registry source. When
128    /// debugging an unexpected `None` on a dynamic key, check
129    /// [`crate::stats::MetricId::def`]`().is_some()` to tell an unregistered
130    /// key from absent data (built-in ids always resolve).
131    ///
132    /// Pass `Phase::BASELINE` for the settle window or `Phase::step(k)` for the
133    /// test author's 0-indexed Step k — the typed `Phase` hides the 1-indexed
134    /// encoding (see [`Self::phase`]).
135    pub fn phase_metric(
136        &self,
137        phase: Phase,
138        metric: impl Into<crate::stats::MetricId>,
139    ) -> Option<f64> {
140        let metric = metric.into();
141        self.phase(phase).and_then(|p| {
142            p.get(metric.as_str())
143                .or_else(|| p.cgroup_counter_total(metric.as_str()))
144        })
145    }
146
147    /// Cross-cgroup balance: the ratio of the busiest cell's per-worker
148    /// throughput to the quietest's — `max / min` over each cgroup's
149    /// [`CgroupStats::iterations_per_worker`]. The bread-and-butter
150    /// scheduler-fairness assertion (every balance test hand-rolls this
151    /// `max/min` over `self.cgroups` today).
152    ///
153    /// No-worker cgroups (`iterations_per_worker() == None`) are SKIPPED: a
154    /// 0-worker cell is a config condition, not a balance signal. Returns
155    /// `None` when fewer than two cgroups have workers (a ratio needs two);
156    /// check the cgroup count separately if every declared cell must have
157    /// workers. A cell that ran workers but completed zero iterations
158    /// (measured `Some(0.0)`) drives the ratio to `f64::INFINITY` so
159    /// starvation SURFACES rather than vanishing — matching the
160    /// `None`-vs-`Some(0.0)` discipline of
161    /// [`CgroupStats::iterations_per_worker`]. For an explicit starvation
162    /// gate, check `min > 0` over the same cgroups separately.
163    ///
164    /// Whole-run aggregate: this reads `self.cgroups`, which sums over all
165    /// phases. For a single phase's per-cgroup balance in a multi-phase
166    /// scenario, read each cgroup's per-phase throughput (`total_iterations`
167    /// / `num_workers`) from [`crate::vmm::VmResult::phase_cgroup`] — the
168    /// per-phase per-cgroup carriers now landed in
169    /// [`PhaseBucket::per_cgroup`].
170    pub fn cgroup_balance_ratio(&self) -> Option<f64> {
171        let mut min = f64::INFINITY;
172        let mut max = 0.0_f64;
173        let mut n = 0usize;
174        for cg in &self.cgroups {
175            if let Some(rate) = cg.iterations_per_worker() {
176                min = min.min(rate);
177                max = max.max(rate);
178                n += 1;
179            }
180        }
181        if n < 2 {
182            return None;
183        }
184        if min == 0.0 {
185            // A with-worker cell did zero work: starvation. Surface it as an
186            // infinite ratio rather than a NaN (0/0) or a hidden None.
187            return Some(f64::INFINITY);
188        }
189        Some(max / min)
190    }
191
192    /// Per-cgroup analog of [`Self::phase_metric`]: look up `metric` for a named
193    /// `cgroup` in a [`Phase`] (`Phase::BASELINE` / `Phase::step(k)`), via that
194    /// cgroup's per-phase carrier ([`PhaseCgroupStats::get`]), falling back to
195    /// [`PhaseCgroupStats::cgroup_counter`] for the per-cgroup Counters
196    /// `total_migrations`/`total_iterations`/`total_cpu_time_ns`. `None` when the
197    /// phase has no bucket, no carrier for `cgroup`, the carrier carried no finite
198    /// value for the metric, OR the metric is an unregistered dynamic key (a
199    /// built-in id is typo-proof; an unregistered [`crate::stats::MetricId`] has no
200    /// [`crate::stats::MetricId::def`], same as [`Self::phase_metric`]). The
201    /// N-cgroups-to-N-queryable-sets surface on the AssertResult-holding path (the
202    /// in-VM `post_vm` path uses [`crate::vmm::VmResult::phase_cgroup_metric`]).
203    pub fn phase_cgroup_metric(
204        &self,
205        phase: Phase,
206        cgroup: &str,
207        metric: impl Into<crate::stats::MetricId>,
208    ) -> Option<f64> {
209        let metric = metric.into();
210        self.phase(phase)
211            .and_then(|p| p.per_cgroup.get(cgroup))
212            .and_then(|pc| {
213                pc.get(metric.as_str())
214                    .or_else(|| pc.cgroup_counter(metric.as_str()))
215            })
216    }
217
218    /// True iff the scenario produced at least one Step-phase
219    /// bucket (any phase with `step_index >= 1`). False when
220    /// `phases` is empty OR contains only `BASELINE` (the
221    /// pre-first-Step settle window).
222    ///
223    /// Use this to fail a phase-aware assertion BEFORE calling
224    /// [`Self::phase`] with a `Phase::step(k)` on a scenario that
225    /// silently never advanced past BASELINE: a test that declared
226    /// no `Step`s, OR a scenario that bailed in setup before any
227    /// `Step` ran, would otherwise see [`Self::phase`] return
228    /// `None` for every Step and the test would either panic on
229    /// `.expect(...)` or pass vacuously.
230    ///
231    /// ```ignore
232    /// anyhow::ensure!(
233    ///     r.stats.has_steps(),
234    ///     "scenario produced no Step-phase buckets — \
235    ///      declare a Step or read Phase::BASELINE",
236    /// );
237    /// let throughput = r.stats.phase_metric(Phase::step(0), "throughput");
238    /// ```
239    pub fn has_steps(&self) -> bool {
240        self.phases.iter().any(|p| p.step_index >= 1)
241    }
242
243    /// Run-level value for a metric by registry name, for the
244    /// ext-sourced metric family that carries no typed
245    /// `ScenarioStats` field.
246    ///
247    /// Resolves [`Self::ext_metrics`] — the run-level map the
248    /// framework fills post-merge with every metric whose value has no
249    /// typed struct field: the pooled wake-latency / run-delay
250    /// distributions and worst-cgroup iteration efficiencies
251    /// (the `MetricKind::Distribution` / `MetricKind::WorstLowest`
252    /// registry kinds — `worst_p99_wake_latency_us`, `worst_run_delay_us`,
253    /// `worst_iterations_per_cpu_sec`, …), the canonical pooled
254    /// `iteration_rate`,
255    /// the per-thread-group `system_time_ns` / `user_time_ns`, and
256    /// `avg_imbalance_ratio` / `avg_dsq_depth`. This is the
257    /// run-level analogue of [`Self::phase_metric`] for that family:
258    /// code holding the run's [`AssertResult`] reads
259    /// `r.stats.run_metric("worst_run_delay_us")` instead of reaching
260    /// into the raw `ext_metrics` map by string key (`ScenarioStats` is
261    /// the [`AssertResult::stats`] field — the value a test body, or a
262    /// callback that builds an `AssertResult` via `collect_all` /
263    /// `execute_scenario`, holds). A `post_vm` callback instead receives
264    /// a `VmResult`, which has NO `stats` field and no run-level
265    /// Distribution surface — compare those cross-run via `cargo ktstr
266    /// perf-delta`.
267    ///
268    /// The ext family is populated only by the `#[ktstr_test]` eval
269    /// flow's post-merge producer
270    /// ([`populate_run_distribution_metrics`]). An `AssertResult` built
271    /// by a DIRECT host assertion (`assert_not_stuck` /
272    /// `AssertPlan::assert_cgroup`, which never run that producer)
273    /// carries the per-cgroup values on [`Self::cgroups`] but none of
274    /// these run-level roll-ups, so `run_metric` returns `None` for them
275    /// on that path — read the per-cgroup `CgroupStats` field directly
276    /// (e.g. `r.stats.cgroups[i].p99_wake_latency_us`) there.
277    ///
278    /// Sentinel-free, matching [`Self::phase_metric`]: `None` means
279    /// the metric is absent from this run (no contributing cgroup or
280    /// carrier, or a name not present in the map); `Some(0.0)` is a
281    /// real measured zero. Check [`crate::stats::MetricId::def`] on a dynamic
282    /// key to tell an unregistered key from genuinely-absent data (built-in
283    /// ids always resolve). (The map also carries any
284    /// user-defined extensible-metric keys, plus the framework-internal
285    /// Rate-component Counters — `total_iterations_pooled` /
286    /// `total_cpu_time_sec`, the numerator/denominator plumbing behind
287    /// `iteration_rate` — both of which resolve
288    /// here too; prefer the derived rate over its raw components.)
289    ///
290    /// RESOLVED here None-aware, ahead of the ext lookup, via a typed dispatch
291    /// (`typed_sentinel_metric`): the cross-cgroup metrics `worst_spread`,
292    /// `worst_migration_ratio`, `worst_gap_ms`, `total_migrations`,
293    /// `total_iterations`, `worst_page_locality`,
294    /// `worst_cross_node_migration_ratio`. The dispatch RE-DERIVES each from the
295    /// per-cgroup (`self.cgroups`) / per-phase (`self.phases[].per_cgroup`, the
296    /// NUMA metrics) carriers — `None` when no carrier measured it, `Some(0.0)`
297    /// for a measured zero — preserving the sentinel-free contract. The 5
298    /// non-NUMA metrics are 0.0-sentinel typed struct fields (a not-measured
299    /// carrier coerces to 0.0, indistinguishable from a measured zero, so the
300    /// re-derivation recovers the distinction); the two NUMA roll-ups
301    /// (`worst_page_locality`, `worst_cross_node_migration_ratio`) have no struct
302    /// field and re-pool purely from the per-phase NUMA carriers.
303    /// (`worst_wake_latency_tail_ratio` resolves via the ext lookup as the
304    /// `WakeLatencyTailRatio` key.)
305    ///
306    /// NOT resolved here (not in `ext_metrics`, no typed dispatch):
307    /// - the monitor-sourced run-level metrics (`max_imbalance_ratio`,
308    ///   `max_dsq_depth`, `stuck_count`, `total_fallback`,
309    ///   `total_keep_last`), which `ScenarioStats` does not hold
310    ///   run-level — read those per-phase via [`Self::phase_metric`].
311    ///
312    /// So this resolves the ext-sourced family AND the typed cross-cgroup fields
313    /// (via the dispatch); only the monitor-sourced run-level metrics remain
314    /// unresolved here (`ScenarioStats` holds them only per-phase).
315    pub fn run_metric(&self, metric: impl Into<crate::stats::MetricId>) -> Option<f64> {
316        let id = metric.into();
317        // The sentinel-prone cross-cgroup run-level metrics re-derive None-aware
318        // from the per-cgroup / per-phase carriers, recovering the never-measured
319        // vs measured-zero distinction the 0.0-coerced struct fields (or, for
320        // worst_page_locality, the absent field) cannot carry. A non-sentinel id
321        // returns `None` from the dispatch and falls through to the ext-metrics map.
322        if let crate::stats::MetricId::Builtin(b) = &id
323            && let Some(resolved) = self.typed_sentinel_metric(*b)
324        {
325            return resolved;
326        }
327        self.ext_metrics.get(id.as_str()).copied()
328    }
329
330    /// Re-derive a typed run-level metric None-aware from the per-cgroup /
331    /// per-phase carriers — recovering the never-measured (`None`) vs
332    /// measured-zero (`Some(0.0)`) distinction a bare 0.0-sentinel `f64` cannot
333    /// carry. The 0.0-sentinel typed struct fields (`worst_spread`,
334    /// `worst_migration_ratio`, ...) coerce a not-measured carrier to 0.0 in
335    /// `scenario_stats_for_cgroup` (indistinguishable from a measured zero);
336    /// `worst_page_locality` is no longer a struct field at all — it re-pools
337    /// purely from the per-phase NUMA carriers (see below).
338    ///
339    /// Returns `None` when `m` is NOT one of the typed sentinel ids — the caller
340    /// then falls through to the ext-metrics lookup. `Some(None)` when `m` IS a
341    /// sentinel id but no contributing carrier exists (loud-absent). `Some(Some(v))`
342    /// for the re-derived value (a measured zero stays `Some(0.0)`).
343    ///
344    /// The 5 non-NUMA metrics source from `self.cgroups`, whose `Option` / counter
345    /// carriers preserve the measured-vs-unmeasured state. The 2 NUMA roll-ups
346    /// (`worst_page_locality`, `worst_cross_node_migration_ratio` — neither a
347    /// struct field) source from `self.phases[].per_cgroup`, for different
348    /// reasons: `page_locality` is structurally 0.0 on the `cgroup_stats` path
349    /// (it needs an expected-node set the reports-only builder lacks), so
350    /// `self.cgroups` cannot source it; `cross_node_migration_ratio` IS populated
351    /// on `cgroup_stats`, but only as a single pre-folded scalar that cannot carry
352    /// the cross-phase fold (SUM the per-phase migration-counter deltas over the
353    /// LATEST residency total), so it too re-pools from the per-phase
354    /// `PhaseCgroupStats` NUMA counters — which also single-sources it with the
355    /// ext/sidecar value. On a phase-less direct-assertion `AssertResult` (no NUMA
356    /// capture) both NUMA roll-ups read `None` (correct loud-absent).
357    fn typed_sentinel_metric(&self, m: crate::stats::BuiltinMetric) -> Option<Option<f64>> {
358        use crate::stats::BuiltinMetric as B;
359        Some(match m {
360            // worst_spread: highest spread over cgroups that measured it
361            // (CgroupStats::spread is already Option — None when no worker had
362            // measurable wall time). None iff no cgroup measured spread.
363            B::WorstSpread => self
364                .cgroups
365                .iter()
366                .filter_map(|c| c.spread)
367                .reduce(f64::max),
368            // worst_migration_ratio: highest migration ratio over cgroups that
369            // ran iterations (a 0.0 with total_iterations>0 is measured; with
370            // ==0 it is the rate-over-zero sentinel — never-measured).
371            B::WorstMigrationRatio => self
372                .cgroups
373                .iter()
374                .filter(|c| c.total_iterations > 0)
375                .map(|c| c.migration_ratio)
376                .reduce(f64::max),
377            // total_migrations / total_iterations: cross-cgroup SUMs. 0 is a real
378            // measured sum (cgroups ran, zero events); never-measured = no cgroups.
379            B::TotalMigrations => (!self.cgroups.is_empty()).then(|| {
380                self.cgroups
381                    .iter()
382                    .map(|c| c.total_migrations)
383                    .fold(0u64, u64::saturating_add) as f64
384            }),
385            B::TotalIterations => (!self.cgroups.is_empty()).then(|| {
386                self.cgroups
387                    .iter()
388                    .map(|c| c.total_iterations)
389                    .fold(0u64, u64::saturating_add) as f64
390            }),
391            // worst_gap_ms: longest gap over cgroups with workers (gap 0 with
392            // workers = measured "no gap observed"; never-measured = no workers).
393            B::WorstGapMs => self
394                .cgroups
395                .iter()
396                .filter(|c| c.num_workers > 0)
397                .map(|c| c.max_gap_ms as f64)
398                .reduce(f64::max),
399            // worst_page_locality: LOWEST per-cgroup locality (the worst cell) over
400            // cgroups that measured NUMA residency — a measured 0.0 (all pages
401            // off-node) is the worst and WINS the lowest fold. Shared with the
402            // ext/sidecar re-pool via worst_page_locality() so the typed read and
403            // the cross-run comparison value agree (the same fix replaces the
404            // fold_lowest_nonzero sentinel-skip on BOTH surfaces).
405            B::WorstPageLocality => self.worst_page_locality(),
406            // worst_cross_node_migration_ratio: highest cross-node churn ratio over
407            // cgroups that measured NUMA. Shared with the ext/sidecar re-pool via
408            // worst_cross_node_migration_ratio() so the typed read and the cross-run
409            // comparison value agree (the divergence the typed field had).
410            B::WorstCrossNodeMigrationRatio => self.worst_cross_node_migration_ratio(),
411            _ => return None,
412        })
413    }
414
415    /// Aggregate each cgroup's NUMA carriers across `self.phases` into
416    /// `(latest_local, latest_total, summed_migrated)`. The cross-phase fold is
417    /// ASYMMETRIC by counter class: `numa_pages_local` / `numa_pages_total` are
418    /// current-residency GAUGE snapshots (the live `/proc/.../numa_maps`
419    /// residency, recomputed each read), so the LATEST MEASURED phase's value is the run-end placement
420    /// — SUMMING across phases would multiply residency by the phase count (a
421    /// silent over-count); `cross_node_migrated` is a per-phase migration-counter
422    /// DELTA over disjoint intervals, so it SUMS to the run total. The shared
423    /// `latest_total` denominator serves both ratios (page_locality and
424    /// cross_node share one page total), matching `cgroup_stats`, which reads the
425    /// final-snapshot residency. One entry per cgroup that appeared in any phase.
426    ///
427    /// LATEST is None-aware: a later phase carrier that measured NO NUMA pages
428    /// (`numa_pages_total == 0` — an empty backdrop slice, a non-NUMA `WorkType`
429    /// phase, or a zero-worker carrier) does NOT overwrite an earlier MEASURED
430    /// residency. A bare overwrite would zero out the earlier snapshot, and the
431    /// `total > 0` filter in `worst_page_locality()` would then drop the cgroup
432    /// from the worst-pool — silently reporting a better-than-worst run-level
433    /// locality (the same measured-vs-unmeasured discipline the cross-cgroup fold
434    /// applies, extended to the cross-phase axis).
435    fn numa_agg_per_cgroup(&self) -> Vec<(u64, u64, u64)> {
436        let mut by_cg: std::collections::BTreeMap<&str, (u64, u64, u64)> =
437            std::collections::BTreeMap::new();
438        // self.phases is in step (chronological) order, so a later MEASURED
439        // phase's residency snapshot overwrites the earlier (LATEST wins); a
440        // not-measured (total == 0) phase leaves the prior snapshot intact. The
441        // migration deltas accumulate unconditionally (a 0 delta adds nothing).
442        for phase in &self.phases {
443            for (name, pc) in &phase.per_cgroup {
444                let e = by_cg.entry(name.as_str()).or_insert((0, 0, 0));
445                if pc.numa_pages_total > 0 {
446                    e.0 = pc.numa_pages_local;
447                    e.1 = pc.numa_pages_total;
448                }
449                // saturating: guest-runtime migration counter pooled across
450                // phases; never wrap the derived cross-node-migration ratio.
451                e.2 = e.2.saturating_add(pc.cross_node_migrated);
452            }
453        }
454        by_cg.into_values().collect()
455    }
456
457    /// The run-level WORST (lowest) per-cgroup page-locality fraction, re-pooled
458    /// None-aware from the per-phase NUMA carriers: the lowest
459    /// `page_locality_of(latest_local, latest_total)` over cgroups that measured
460    /// NUMA residency (`numa_pages_total > 0`) — a measured 0.0 (all pages
461    /// off-node) WINS the lowest; an all-unmeasured cohort yields `None`. Shared
462    /// by `run_metric`'s `WorstPageLocality` dispatch AND
463    /// `populate_run_distribution_metrics`'s ext/sidecar re-pool so the typed
464    /// read and the cross-run comparison value are byte-identical.
465    fn worst_page_locality(&self) -> Option<f64> {
466        self.numa_agg_per_cgroup()
467            .into_iter()
468            .filter(|&(_, total, _)| total > 0)
469            .map(|(local, total, _)| super::reductions::page_locality_of(local, total))
470            .reduce(f64::min)
471    }
472
473    /// The run-level WORST (highest) per-cgroup cross-node migration-churn ratio,
474    /// re-pooled from the per-phase NUMA carriers: the MAX
475    /// `cross_node_migration_ratio_of(summed_migrated, latest_total)` over cgroups
476    /// that measured NUMA residency (`numa_pages_total > 0`) — an all-unmeasured
477    /// cohort yields `None`. The polarity twin of `worst_page_locality` (LowerBetter
478    /// → highest-wins). Shared by `run_metric`'s `WorstCrossNodeMigrationRatio`
479    /// dispatch AND `populate_run_distribution_metrics`'s ext/sidecar re-pool so the
480    /// typed read and the cross-run comparison value are byte-identical (the typed
481    /// `Gauge(Last)` field diverged from this on multi-phase runs).
482    fn worst_cross_node_migration_ratio(&self) -> Option<f64> {
483        self.numa_agg_per_cgroup()
484            .into_iter()
485            .filter(|&(_, total, _)| total > 0)
486            .map(|(_, total, migrated)| {
487                super::reductions::cross_node_migration_ratio_of(migrated, total)
488            })
489            .reduce(f64::max)
490    }
491}
492
493/// Registry metric names that already have a typed `GauntletRow` field — the
494/// typed accessor populates them at `sidecar_to_row` time and
495/// `MetricDef::read` prefers the accessor over `ext_metrics`, so writing the
496/// same key into `ext_metrics` would create unread sidecar bloat AND
497/// double-source the run-level value. For `stuck_count` the typed whole-run
498/// count (`MonitorSummary::stuck_count`, windowed over the full sample
499/// stream) is authoritative; the per-phase fold sum shares the
500/// `is_cpu_stuck` predicate but is a lower-or-equal (`<=`),
501/// partition-dependent quantity (it drops cross-boundary + out-of-phase
502/// windows, so it falls strictly below once any of those is stuck), so
503/// injecting the ext copy would shadow the authoritative typed value with
504/// a redundant — and, once a dropped window is stuck, divergent —
505/// number. Both run-level ext-metrics populators consult this — the
506/// SampleSeries
507/// path ([`populate_run_ext_metrics`]) and the phase-fold path
508/// ([`populate_run_ext_metrics_from_phases`]) — so only ext-metrics-only
509/// registry entries are written and a typed-backed metric's run-level value
510/// always comes from its accessor. `max_imbalance_ratio` is included because
511/// its accessor reads the typed `GauntletRow.imbalance_ratio` (whole-run
512/// MonitorSummary); its per-phase monitor fold feeds rendering only.
513const TYPED_FIELD_NAMES: &[&str] = &[
514    "max_dsq_depth",
515    "max_imbalance_ratio",
516    "total_fallback",
517    "total_keep_last",
518    "stuck_count",
519    "total_iterations",
520    "total_migrations",
521];
522
523/// Sibling of [`populate_run_ext_metrics`] that mines per-phase
524/// metrics back into the run-level `ext_metrics` map. Closes the
525/// gap for registered metrics whose values live in
526/// `PhaseBucket.metrics` but never reach `ext_metrics` via the
527/// SampleSeries path (their `read_sample` returns `None`):
528/// `avg_imbalance_ratio` (sourced from MonitorSample windowing
529/// inside [`build_phase_buckets`]), `iteration_rate` (sourced from matching
530/// guest per-cgroup iteration and CPU-time carriers),
531/// and `system_time_ns` / `user_time_ns` (per-thread-group CPU-time
532/// deltas injected by `phase_group_cpu_delta` inside
533/// `buckets_from_grouped`). The fold is generic over every key
534/// present on any phase, so it carries any such phase-only metric (the
535/// ext-metrics-only set whose `read_sample` returns `None`). Keys with a
536/// typed `GauntletRow` field (`TYPED_FIELD_NAMES`) are SKIPPED: their
537/// run-level value comes from the typed accessor (which wins on read), so
538/// re-injecting them here would double-source the run aggregate — the
539/// hazard the const's doc describes. Their per-phase `PhaseBucket` value
540/// still feeds per-phase rendering.
541///
542/// Per-phase reduction dispatch is described on [`PhaseBucket`];
543/// the cross-phase fold here uses `sample_count` as the weight so
544/// Gauge(Avg) keys get the weighted mean (the correct cross-phase
545/// semantic for typical-load metrics) while other kinds fold per
546/// their natural reduction. Existing keys in `target` are not
547/// overwritten — `read_sample` path values win when both produced
548/// an entry.
549///
550/// Without this fill, `cargo ktstr perf-delta` silently misses
551/// these phase-only metrics (avg_imbalance_ratio, iteration_rate,
552/// system_time_ns, user_time_ns) in flat-row output because
553/// `MetricDef::read` falls back to ext_metrics and finds nothing.
554pub fn populate_run_ext_metrics_from_phases(
555    phases: &[PhaseBucket],
556    target: &mut std::collections::BTreeMap<String, f64>,
557) {
558    // No early-return on empty `phases`: the derive_rate_metrics post-pass
559    // below must still run over whatever components populate_run_ext_metrics
560    // already inserted into `target` (the empty-phases case), so a run-level
561    // Rate is re-derived rather than silently dropped. The loops below are
562    // no-ops when `phases` is empty.
563    // Collect every metric key that appears on any phase.
564    let mut keys: std::collections::BTreeSet<&String> = std::collections::BTreeSet::new();
565    for phase in phases {
566        for key in phase.metrics.keys() {
567            keys.insert(key);
568        }
569    }
570    for key in keys {
571        if target.contains_key(key) {
572            continue;
573        }
574        let Some(def) = crate::stats::metric_def(key) else {
575            continue;
576        };
577        // Derived metrics (every `is_derived()`: Rate / Distribution / WorstLowest /
578        // WakeLatencyTailRatio / WorstCrossNodeRatio / PerPhase / PerRunDistribution)
579        // are produced
580        // from their pooled components, not folded as per-phase values: skip
581        // here. A Rate re-derives after the loop (Σnum/Σdenom over the folded
582        // components); the distributional kinds (Distribution / WorstLowest /
583        // WakeLatencyTailRatio / WorstCrossNodeRatio) are re-pooled run-level by
584        // `populate_run_distribution_metrics` (and never appear in
585        // phase.metrics anyway); PerPhase is re-derived by `derive_phase_metrics`.
586        // Folding a ready-made derived value would lose
587        // the re-pool, and routing one into aggregate_samples_weighted within
588        // a run is not its producer path.
589        if def.kind.is_derived() {
590            continue;
591        }
592        // Typed-backed keys (those in TYPED_FIELD_NAMES — a typed GauntletRow
593        // accessor that wins on read) must NOT be re-injected into ext_metrics
594        // from the phase fold: the ext copy would be unread bloat and, for
595        // stuck_count (whose per-phase fold sum is `<=` the typed whole-run
596        // count, strictly below once a cross-boundary/out-of-phase window is
597        // stuck — they share the is_cpu_stuck predicate but the run-level
598        // count windows the full stream), a redundant-or-divergent value,
599        // not a guaranteed duplicate. Their per-phase
600        // PhaseBucket value still feeds rendering; the run-level value stays
601        // the typed path. Mirrors the sibling populate_run_ext_metrics.
602        // (Without this, folding max_imbalance_ratio + stuck_count onto
603        // captured buckets would leak both into ext_metrics on the common
604        // path.)
605        if TYPED_FIELD_NAMES.contains(&key.as_str()) {
606            continue;
607        }
608        // avg_nr_running is NOT typed-backed (no GauntletRow accessor), but its
609        // authoritative run-level value is MonitorSummary::avg_nr_running, folded
610        // by fold_run_level_ext. It is the one monitor-summary-fold key that ALSO
611        // appears in per-phase bucket.metrics (fold_monitor_into_bucket writes it
612        // for rendering). Skip it here so the per-phase re-pool never claims the
613        // run-level key: VmResult::run_metric runs this re-pool BEFORE
614        // fold_run_level_ext, and the fold's `or_insert` would then no-op,
615        // silently replacing the whole-run value with the per-phase weighted
616        // mean. Its per-phase PhaseBucket value still feeds rendering +
617        // change-detection. (TYPED_FIELD_NAMES is the typed-accessor analogue of
618        // this same skip rationale.)
619        if key == "avg_nr_running" {
620            continue;
621        }
622        // Per-phase (value, sample_count) for the kind-aware fold.
623        // A phase that doesn't carry the key contributes nothing.
624        // Lock-step shape enforced by the (f64, usize) pair type.
625        // `sample_count.max(1)` is load-bearing for Gauge(Avg) keys: a
626        // synthesized zero-capture phase carrying a
627        // capture-independent Gauge(Avg) value at sample_count==0 gets
628        // weight 1 (one phase observation) rather than being zero-weighted
629        // out of the run-level mean. The floor is a no-op for
630        // Counter/DeltaSum keys, which sum with weights ignored (see
631        // aggregate_finite): iteration_rate's guest-carrier components
632        // total_iterations_pooled / total_cpu_time_sec are such Counters, so
633        // a zero-capture step's work is INCLUDED in the re-pooled rate
634        // (iteration_rate itself is a Rate, skipped above and re-derived
635        // below). A regression dropping
636        // the floor would silently re-drop a zero-capture step's Gauge(Avg)
637        // value from the sidecar aggregate.
638        let pairs: Vec<(f64, usize)> = phases
639            .iter()
640            .filter_map(|phase| {
641                phase
642                    .metrics
643                    .get(key)
644                    .copied()
645                    .map(|v| (v, phase.sample_count.max(1)))
646            })
647            .collect();
648        if pairs.is_empty() {
649            continue;
650        }
651        // PerPhaseDeltaSum folds cross-PHASE by SUM: the run-level value is the
652        // sum of the disjoint per-phase OBSERVED CPU-time deltas (a lower bound
653        // on total run CPU time — excludes head / tail / inter-phase-gap
654        // windows; see the MetricKind::PerPhaseDeltaSum doc). It is NOT routed
655        // through aggregate_samples_weighted, whose PerPhaseDeltaSum arm
656        // implements the CROSS-RUN unweighted mean — mean-folding the phases
657        // here would double-count duration (the bug this kind fixes). Weights
658        // (phase sample_count) are irrelevant to a sum.
659        if def.kind == crate::stats::MetricKind::PerPhaseDeltaSum {
660            let sum: f64 = pairs.iter().map(|(v, _)| v).sum();
661            target.insert(key.clone(), sum);
662            continue;
663        }
664        if let Some(reduced) = crate::stats::aggregate_samples_weighted(&pairs, def.kind) {
665            target.insert(key.clone(), reduced);
666        }
667    }
668    // Re-derive Rate metrics from the now-folded components so the run
669    // rate is Σnumerator / Σdenominator (the components folded by their
670    // own kinds above — a Counter numerator summed across phases).
671    crate::stats::derive_rate_metrics(target);
672}
673
674/// Run the FULL run-level `ext_metrics` population sequence into `stats` — the
675/// single source of truth shared by the eval layer (`evaluate_vm_result`) and
676/// [`crate::vmm::VmResult::run_metric`], so the two produce byte-identical
677/// run-level ext maps for the same run. Reads `samples` + `stats.phases` +
678/// `stats.cgroups`, writes `stats.ext_metrics`, in the canonical order:
679/// 1. `derive_phase_metrics` — derive per-cgroup and pooled phase scalars,
680///    including the CPU-denominated `iteration_rate` components.
681/// 2. [`populate_run_ext_metrics`] — the `read_sample`-wired registry family
682///    (over every freeze in `samples`), then the whole-run wall + IRQ rates.
683/// 3. [`populate_run_ext_metrics_from_phases`] — the phase-only ext metrics
684///    whose `read_sample` is `None` (`avg_imbalance_ratio`, `iteration_rate`,
685///    `system_time_ns`, `user_time_ns`, the per-CPU IRQ spatial maxes, and the
686///    per-cgroup PSI-irq spatial maxes).
687/// 4. [`populate_run_pooled_iteration_rate`] — add any whole-run
688///    cgroup carriers to the canonical `iteration_rate` pool.
689/// 5. [`populate_run_pooled_taobench`] — the whole-run taobench throughput + hit Rates
690///    pooled cross-cgroup (from `stats.cgroups[].taobench_whole`).
691/// 6. [`populate_run_pooled_taobench_distribution`] — the taobench whole-run
692///    open-loop serve-latency `*_whole` percentiles (union of the per-phase
693///    per-cgroup serve `PlatStats` histograms, percentile re-derived over the union).
694/// 7. [`populate_run_pooled_schbench`] — the schbench whole-run loop Counter +
695///    role-separate run-delay gate-Rates (from
696///    `stats.phases[].per_cgroup[].schbench` raw pairs, summed over phases+cgroups).
697/// 8. [`populate_run_pooled_schbench_distribution`] — the schbench whole-run
698///    latency/rps `*_whole` percentiles (union of the per-phase per-cgroup
699///    `PlatStats` histograms, percentile re-derived over the union).
700/// 9. [`populate_run_distribution_metrics`] — the `Distribution` / `WorstLowest`
701///    / `WakeLatencyTailRatio` / `WorstCrossNodeRatio` re-pools (from
702///    `stats.phases[].per_cgroup` raw samples + `stats.cgroups`).
703///
704/// ORDER IS LOAD-BEARING: step 1 must precede step 3 so the canonical
705/// CPU-denominated phase rate exists before the run fold. Step 2 must also
706/// precede step 3 so whole-run IRQ-counter deltas land before step 3's
707/// `contains_key` skip; steps 4-9 fold the per-cgroup roll-up after the
708/// per-phase families.
709///
710/// `stats.phases` may be pre- or post-derived. Step 1 is idempotent and
711/// deliberately owns this derivation so callers cannot accidentally omit the
712/// guest CPU-time components required by `iteration_rate`.
713pub fn populate_run_ext_all(
714    stats: &mut ScenarioStats,
715    samples: &crate::scenario::sample::SampleSeries,
716) {
717    // CPU-denominated phase rates require the guest per-cgroup carriers. Derive
718    // them before the phase fold so both the per-phase and run-level views use
719    // the same delivered-CPU currency.
720    derive_phase_metrics(&mut stats.phases);
721    populate_run_ext_metrics(samples, &mut stats.ext_metrics);
722    populate_run_ext_metrics_from_phases(&stats.phases, &mut stats.ext_metrics);
723    populate_run_pooled_iteration_rate(stats);
724    populate_run_pooled_taobench(stats);
725    populate_run_pooled_taobench_distribution(stats);
726    populate_run_pooled_schbench(stats);
727    populate_run_pooled_schbench_distribution(stats);
728    populate_run_distribution_metrics(stats);
729}
730
731/// Inject the canonical run-level pooled `iteration_rate` Rate's two Counter
732/// components into `stats.ext_metrics`, summed across the cgroups that have
733/// measured on-CPU time — the cross-cgroup re-pool axis. Rather than routing
734/// the per-cgroup efficiency through `AssertResult::merge`'s worst-by-polarity
735/// `ext_metrics` fold (which picks the WORST cgroup's value, not Σ, and has
736/// no derive post-pass), this reads the already-merged `stats.cgroups` vec
737/// directly: `iteration_rate` = Σ`total_iterations` /
738/// Σ(`total_cpu_time_ns`/1e9) over cgroups with `total_cpu_time_ns > 0` — the
739/// per-cgroup [`CgroupStats::iterations_per_cpu_sec`] re-pooled, NOT a mean of
740/// per-cgroup ratios, NOT the worst single cgroup.
741///
742/// MUST run at the eval layer AFTER the cgroup-bearing merges (every merge that
743/// contributes a [`CgroupStats`], so `stats.cgroups` holds every per-cgroup
744/// entry) and BEFORE the sidecar write. The trailing monitor-verdict merge at
745/// the eval layer merges an `inconclusive()` carrying empty `stats` (no cgroups,
746/// no ext keys), so it is safe to run after this. If component injection ever
747/// moved BEFORE a cgroup-bearing merge, that worst-by-polarity fold would
748/// min/max these Counter keys into single-cgroup scalars, silently corrupting
749/// the pooled sum.
750///
751/// A cgroup with `total_cpu_time_ns == 0` (schedstat unavailable, or
752/// `num_workers == 0`) is EXCLUDED from BOTH sums — mirroring the per-cgroup
753/// [`CgroupStats::iterations_per_cpu_sec`] None-on-zero (`total_cpu_time_ns >
754/// 0` implies `num_workers > 0`, so the one predicate covers both). Crediting
755/// an unmeasured cgroup's iterations against the measured cgroups' CPU-seconds
756/// would overstate cohort efficiency — the silent-wrong-answer this gate
757/// prevents. Both components are inserted both-or-neither (the
758/// `derive_rate_metrics` co-location invariant), only when the summed MEASURED
759/// on-CPU time is > 0 (every cgroup unmeasured ⇒ no rate). The ns→s `/1e9` is
760/// applied ONCE here on the summed ns (not per-cgroup, to avoid repeated float
761/// rounding), since `derive_rate_metrics` is a bare num/den.
762/// `total_iterations_pooled` is a DISTINCT ext-only key, not the typed
763/// `total_iterations` (skipped from ext_metrics; it folds cross-RUN as a MEAN
764/// — a display average — while a Rate numerator must SUM-fold so Σnum/Σdenom
765/// re-pools, so one shared key cannot carry both folds). Because it sums only
766/// MEASURED cgroups, it is ≤ the merge-summed typed `total_iterations` (which
767/// includes any zero-cpu-time cgroups), and equals it unless an excluded
768/// zero-cpu-time cgroup carried iterations>0.
769pub fn populate_run_pooled_iteration_rate(stats: &mut ScenarioStats) {
770    // Exclude cgroups with no measured on-CPU time from BOTH sums (mirrors the
771    // per-cgroup None-on-zero): crediting an unmeasured cgroup's iterations
772    // against the measured cgroups' CPU-seconds would overstate efficiency.
773    // saturating fold: pool the guest-runtime cpu-time-ns / iteration counters
774    // across cgroups; a plain `.sum()` would debug-panic / release-wrap on a
775    // corrupt/hostile component, silently corrupting iteration_rate.
776    let summed_ns: u64 = stats
777        .cgroups
778        .iter()
779        .filter(|c| c.total_cpu_time_ns > 0)
780        .map(|c| c.total_cpu_time_ns)
781        .fold(0u64, u64::saturating_add);
782    if summed_ns == 0 {
783        return;
784    }
785    let summed_iters: u64 = stats
786        .cgroups
787        .iter()
788        .filter(|c| c.total_cpu_time_ns > 0)
789        .map(|c| c.total_iterations)
790        .fold(0u64, u64::saturating_add);
791    stats
792        .ext_metrics
793        .insert("total_iterations_pooled".to_string(), summed_iters as f64);
794    stats
795        .ext_metrics
796        .insert("total_cpu_time_sec".to_string(), summed_ns as f64 / 1e9);
797    crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
798}
799
800/// Inject the whole-run taobench engine's CPU-throughput + hit Rate components into
801/// `stats.ext_metrics`, pooled across the run's `WorkType::Taobench` cgroups.
802/// Each cgroup carries its workers' merged whole-run aggregate
803/// ([`crate::assert::CgroupStats::taobench_whole`]); this folds those across
804/// cgroups (Σ ops, MAX wall window, Σ client CPU — per `TaobenchStats::merge`)
805/// and writes the `total_taobench_*` Counter components (`ops`, `fast_ops`,
806/// `slow_ops`, `wall_sec`, `cpu_sec`, plus the command-time `get_cmds` /
807/// `get_hits`), from which `crate::stats::derive_rate_metrics` derives the
808/// CPU-SECOND-denominated `taobench_total_ops_per_cpu_sec_whole`,
809/// `taobench_fast_ops_per_cpu_sec_whole`,
810/// `taobench_slow_ops_per_cpu_sec_whole` (Σops / Σclient
811/// CPU-sec — dilation-safe; wall reconstructs as `rate / D` or exactly as
812/// `Σops / total_taobench_wall_sec`), the response-time
813/// `taobench_hit_fraction` (Σfast/Σcompleted), and the command-time
814/// `taobench_command_hit_rate` (Σhits/Σcmds). The whole-run Rate keys are
815/// registered METRICS, so — unlike the per-phase
816/// `taobench_*_ops_per_cpu_sec`
817/// (`MetricKind::PerPhase`, invisible to the whole-run cross-run fold) — they
818/// reach the perf-delta `--noise-adjust` spread analysis. The open-loop
819/// serve-latency distribution is a SEPARATE pool
820/// ([`populate_run_pooled_taobench_distribution`], the `*_us_whole` keys).
821///
822/// MUST run post-`merge` (after every cgroup-bearing merge has populated
823/// `stats.cgroups`), exactly like [`populate_run_pooled_iteration_rate`]:
824/// an earlier run would pool over an incomplete cgroup set. A run with no
825/// Taobench cgroup writes nothing (the pool is `None`) — the keys stay absent,
826/// keeping a non-taobench run distinct from a measured zero.
827///
828/// Numerators are retained whenever a Taobench carrier exists. Wall and CPU
829/// denominators are inserted independently only when positive, so a missing
830/// wall window cannot suppress a valid CPU-denominated rate (and vice versa).
831/// The three throughput Rates derive only with measured client CPU time;
832/// `taobench_hit_fraction` =
833/// `total_taobench_fast_ops` / `total_taobench_ops` derives iff ops completed
834/// (`total_taobench_ops > 0`) and `taobench_command_hit_rate` =
835/// `total_taobench_get_hits` / `total_taobench_get_cmds` iff lookups issued
836/// (`get_cmds > 0`); `derive_rate_metrics` skips a zero-denominator rate, so a
837/// window-but-no-ops run gets qps=0 keys but no false hit fraction / hit rate.
838/// The ns→s `/1e9` is applied ONCE here (not in `derive_rate_metrics`, a bare
839/// num/den), mirroring `total_cpu_time_sec`. Cross-RUN the components SUM-fold
840/// (Counter), so each Rate re-pools as Σnumerator / Σdenominator over the cohort
841/// — the aggregate throughput / hit rate, not a mean of per-run values.
842pub fn populate_run_pooled_taobench(stats: &mut ScenarioStats) {
843    use crate::stats::{
844        TOTAL_TAOBENCH_FAST_OPS, TOTAL_TAOBENCH_GET_CMDS, TOTAL_TAOBENCH_GET_HITS,
845        TOTAL_TAOBENCH_OPS, TOTAL_TAOBENCH_SLOW_OPS, TOTAL_TAOBENCH_WALL_SEC,
846    };
847    // Pool the per-cgroup whole-run aggregates across the run's Taobench cgroups:
848    // Σ ops, MAX wall window (shared by the concurrent cohorts). `None` when no
849    // cgroup ran a Taobench worker.
850    let pooled = stats
851        .cgroups
852        .iter()
853        .filter_map(|c| c.taobench_whole.as_ref())
854        .fold(
855            None,
856            |acc: Option<crate::workload::taobench::run::TaobenchStats>, t| {
857                Some(match acc {
858                    Some(mut a) => {
859                        a.merge(t);
860                        a
861                    }
862                    None => *t,
863                })
864            },
865        );
866    let Some(w) = pooled else {
867        return;
868    };
869    let c = &w;
870    stats
871        .ext_metrics
872        .insert(TOTAL_TAOBENCH_OPS.to_string(), c.total_ops() as f64);
873    stats
874        .ext_metrics
875        .insert(TOTAL_TAOBENCH_FAST_OPS.to_string(), c.fast_ops as f64);
876    stats
877        .ext_metrics
878        .insert(TOTAL_TAOBENCH_SLOW_OPS.to_string(), c.slow_ops as f64);
879    if c.elapsed_ns > 0 {
880        stats.ext_metrics.insert(
881            TOTAL_TAOBENCH_WALL_SEC.to_string(),
882            c.elapsed_ns as f64 / 1e9,
883        );
884    }
885    // CPU-second window: Σ client-thread CLOCK_THREAD_CPUTIME_ID across the
886    // pooled cgroups — the `taobench_*_ops_per_cpu_sec_whole` Rate DENOMINATOR (the
887    // wall window above stays as a component: it is window evidence, not a
888    // throughput rate, and preserves the exact wall reconstruction
889    // `Σops / Σwall` alongside the `rate / D` approximation). Inserted only
890    // when measured (>0) so the CPU rates read absent — never a false rate —
891    // on a pre-field carrier.
892    if c.cpu_time_ns > 0 {
893        stats.ext_metrics.insert(
894            crate::stats::TOTAL_TAOBENCH_CPU_SEC.to_string(),
895            c.cpu_time_ns as f64 / 1e9,
896        );
897    }
898    // Command-time hit components: hits = cmds − misses (request-time). The Rate
899    // taobench_command_hit_rate = Σhits / Σcmds re-derives via derive_rate_metrics
900    // (skipped, hence absent, when no lookups issued). Diverges from the
901    // response-time taobench_hit_fraction under open-loop arrival.
902    stats
903        .ext_metrics
904        .insert(TOTAL_TAOBENCH_GET_CMDS.to_string(), c.get_cmds as f64);
905    stats.ext_metrics.insert(
906        TOTAL_TAOBENCH_GET_HITS.to_string(),
907        c.get_cmds.saturating_sub(c.get_misses) as f64,
908    );
909    crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
910}
911
912/// Inject the taobench WHOLE-RUN open-loop serve-latency percentiles into
913/// `stats.ext_metrics` as the `taobench_serve_*_us_whole` keys, re-pooled
914/// run-level by UNIONING the per-phase per-cgroup serve `PlatStats` histograms
915/// (`stats.phases[].per_cgroup[].taobench.serve_lat`) across every step-attributed
916/// phase and every cgroup, then re-deriving each percentile / min / max over the
917/// merged histogram (`PlatStats::combine` is an associative bucket-count add, so
918/// the merged histogram is the faithful pooled sample set and the re-derived
919/// percentile is the percentile OF the union — NOT a mean of per-source
920/// percentiles). The taobench analog of
921/// [`populate_run_pooled_schbench_distribution`], with the same source: the
922/// BASELINE (epoch 0) and inter-step-gap (`u32::MAX`) epochs are excluded (they
923/// are dropped from `stats.phases` by `expand_backdrop_phase_buckets`), so this is
924/// the steady-state serve distribution over the measured steps — a faithful
925/// PerRunDistribution, distinct from the standalone driver's full-run histogram.
926///
927/// Runs in [`populate_run_ext_all`] (post-merge); reads the per-phase carriers
928/// (disjoint from the taobench counter pool) and writes distinct `*_whole` keys,
929/// so it is order-independent. Keys are written only when the merged histogram
930/// has samples (`sample_count() > 0`) — a closed-loop run (no serve samples)
931/// reads ABSENT, never a false 0. The keys are `MetricKind::PerRunDistribution`:
932/// noise-compared per-run by `crate::stats::noise_findings`, NEVER cross-run
933/// folded (a percentile of a union is not a mean of per-run percentiles, and the
934/// per-phase histograms are dropped at the cross-run boundary).
935pub fn populate_run_pooled_taobench_distribution(stats: &mut ScenarioStats) {
936    use crate::stats::{
937        TAOBENCH_SERVE_MAX_US_WHOLE, TAOBENCH_SERVE_MIN_US_WHOLE, TAOBENCH_SERVE_P50_US_WHOLE,
938        TAOBENCH_SERVE_P90_US_WHOLE, TAOBENCH_SERVE_P99_US_WHOLE, TAOBENCH_SERVE_P999_US_WHOLE,
939    };
940    use crate::workload::schbench::plat::{Pct, PlatStats};
941
942    // Union the per-phase per-cgroup serve histograms across the whole run.
943    let mut serve = PlatStats::default();
944    for phase in &stats.phases {
945        for pc in phase.per_cgroup.values() {
946            if let Some(t) = pc.taobench.as_ref() {
947                serve.combine(&t.serve_lat);
948            }
949        }
950    }
951    if serve.sample_count() == 0 {
952        return;
953    }
954    let q = serve.percentiles();
955    stats.ext_metrics.insert(
956        TAOBENCH_SERVE_P50_US_WHOLE.to_string(),
957        q.value_at(Pct::P50) as f64,
958    );
959    stats.ext_metrics.insert(
960        TAOBENCH_SERVE_P90_US_WHOLE.to_string(),
961        q.value_at(Pct::P90) as f64,
962    );
963    stats.ext_metrics.insert(
964        TAOBENCH_SERVE_P99_US_WHOLE.to_string(),
965        q.value_at(Pct::P99) as f64,
966    );
967    stats.ext_metrics.insert(
968        TAOBENCH_SERVE_P999_US_WHOLE.to_string(),
969        q.value_at(Pct::P999) as f64,
970    );
971    stats
972        .ext_metrics
973        .insert(TAOBENCH_SERVE_MIN_US_WHOLE.to_string(), q.min as f64);
974    stats
975        .ext_metrics
976        .insert(TAOBENCH_SERVE_MAX_US_WHOLE.to_string(), q.max as f64);
977}
978
979/// Inject the schbench whole-run Class-3 metrics — the loop Counter and the
980/// role-separate run-delay gate-Rate components — into `stats.ext_metrics`,
981/// summed across EVERY phase and EVERY cgroup from the per-phase
982/// `SchbenchPhaseStats` raw pairs (`stats.phases[].per_cgroup[].schbench`). The
983/// raw `(run_delay_ns, pcount)` pairs and `loop_count` are integer and
984/// associative, so summing across phases+cgroups gives the run-level totals; the
985/// two `*_run_delay_ns_per_sched` Rates then re-derive Σrun_delay/Σpcount (the
986/// sample-weighted per-schedule mean — NOT a mean of per-run means). The MESSAGE
987/// and WORKER thread roles pool SEPARATELY (different per-schedule wait
988/// populations — never cross-pool).
989///
990/// Runs in [`populate_run_ext_all`] (post-merge, after
991/// [`populate_run_pooled_taobench`]); reads the per-phase carriers (a disjoint
992/// source from the iterations/taobench pools) and writes distinct
993/// `total_schbench_*` / `schbench_*_run_delay_ns_per_sched` keys, so it is
994/// order-independent. A run with no schbench carrier writes nothing (keys stay
995/// absent — a non-schbench run is distinct from a measured zero).
996///
997/// Both-or-neither PER ROLE: each role's two Counter components are inserted only
998/// when that role was scheduled (`pcount > 0`), so `derive_rate_metrics` yields
999/// the role's gate-Rate iff it ran (never a 0/0); the two roles are independent
1000/// (a worker-only run emits only the worker Rate). `total_schbench_loops` is
1001/// always written when any schbench carrier ran (0 is a measured zero). The
1002/// per-phase `sched_delay_msg/worker_us` is the SAME Σrun_delay_ns/Σpcount
1003/// per-schedule mean at phase scope (NOT schbench's native mean-of-per-thread-
1004/// means, a separate whole-run SchbenchResult stat) and stays PerPhase
1005/// display-only — only these Rates gate; no double-count. Cross-RUN the
1006/// components SUM-fold (Counter), so each Rate re-pools Σrun_delay/Σpcount.
1007pub fn populate_run_pooled_schbench(stats: &mut ScenarioStats) {
1008    use crate::stats::{
1009        TOTAL_SCHBENCH_LOOPS, TOTAL_SCHBENCH_MSG_PCOUNT, TOTAL_SCHBENCH_MSG_RUN_DELAY_NS,
1010        TOTAL_SCHBENCH_WORKER_PCOUNT, TOTAL_SCHBENCH_WORKER_RUN_DELAY_NS,
1011    };
1012    let mut msg_run_delay_ns: u64 = 0;
1013    let mut msg_pcount: u64 = 0;
1014    let mut worker_run_delay_ns: u64 = 0;
1015    let mut worker_pcount: u64 = 0;
1016    let mut loops: u64 = 0;
1017    let mut worker_cpu_ns: u64 = 0;
1018    let mut any = false;
1019    for phase in &stats.phases {
1020        for pc in phase.per_cgroup.values() {
1021            if let Some(s) = pc.schbench.as_ref() {
1022                any = true;
1023                // saturating: guest-runtime run-delay-ns / pcount / loop /
1024                // cpu counters pooled across phases+cgroups (matches the
1025                // already-saturating SchbenchPhaseStats::merge); never wrap a
1026                // gate-Rate.
1027                msg_run_delay_ns = msg_run_delay_ns.saturating_add(s.msg_run_delay_ns);
1028                msg_pcount = msg_pcount.saturating_add(s.msg_pcount);
1029                worker_run_delay_ns = worker_run_delay_ns.saturating_add(s.worker_run_delay_ns);
1030                worker_pcount = worker_pcount.saturating_add(s.worker_pcount);
1031                loops = loops.saturating_add(s.loop_count);
1032                worker_cpu_ns = worker_cpu_ns.saturating_add(s.worker_cpu_ns);
1033            }
1034        }
1035    }
1036    if !any {
1037        return;
1038    }
1039    stats
1040        .ext_metrics
1041        .insert(TOTAL_SCHBENCH_LOOPS.to_string(), loops as f64);
1042    // Whole-run worker CPU-second window — the CPU-SECOND-denominated
1043    // `schbench_total_loops_per_cpu_sec` Rate denominator (Σloops / Σworker
1044    // CPU-sec; message-thread CPU excluded, mirroring schbench's Σ worker
1045    // runtimes). Dilation-safe; wall reconstructs as `rate / D` via the
1046    // sidecar's `host_dilation`. Inserted only when measured (>0) so the
1047    // Rate reads absent on a pre-field carrier, never a false rate.
1048    if worker_cpu_ns > 0 {
1049        stats.ext_metrics.insert(
1050            crate::stats::TOTAL_SCHBENCH_WORKER_CPU_SEC.to_string(),
1051            worker_cpu_ns as f64 / 1e9,
1052        );
1053    }
1054    if msg_pcount > 0 {
1055        stats.ext_metrics.insert(
1056            TOTAL_SCHBENCH_MSG_RUN_DELAY_NS.to_string(),
1057            msg_run_delay_ns as f64,
1058        );
1059        stats
1060            .ext_metrics
1061            .insert(TOTAL_SCHBENCH_MSG_PCOUNT.to_string(), msg_pcount as f64);
1062    }
1063    if worker_pcount > 0 {
1064        stats.ext_metrics.insert(
1065            TOTAL_SCHBENCH_WORKER_RUN_DELAY_NS.to_string(),
1066            worker_run_delay_ns as f64,
1067        );
1068        stats.ext_metrics.insert(
1069            TOTAL_SCHBENCH_WORKER_PCOUNT.to_string(),
1070            worker_pcount as f64,
1071        );
1072    }
1073    crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
1074}
1075
1076/// Inject the schbench whole-run DISTRIBUTIONAL metrics (the wakeup /
1077/// request latency percentiles + min/max and the achieved-rps percentiles) into
1078/// `stats.ext_metrics` as the `*_whole` keys, re-pooled run-level by UNIONING the
1079/// per-phase per-cgroup `PlatStats` histograms
1080/// (`stats.phases[].per_cgroup[].schbench.{wakeup,request,rps}`) across EVERY
1081/// phase and EVERY cgroup, then re-deriving each percentile / min / max over the
1082/// merged histogram. `PlatStats::combine` is an associative bucket-count add, so
1083/// the merged histogram is the FAITHFUL union and the re-derived percentile is
1084/// the percentile OF the pooled sample set — NOT a mean of per-phase / per-cgroup
1085/// percentiles (the percentile operator is non-linear). This is the schbench
1086/// histogram analog of [`populate_run_distribution_metrics`]'s raw-sample union.
1087///
1088/// Runs in [`populate_run_ext_all`] (post-merge, after
1089/// [`populate_run_pooled_schbench`]); reads the per-phase carriers (disjoint from
1090/// the iterations / taobench / schbench loop/run-delay pools) and writes distinct
1091/// `*_whole` keys, so it is order-independent. Each stream's keys are written
1092/// only when its merged histogram has samples (`sample_count() > 0`) — a stream
1093/// with no samples (e.g. a sub-1s run with no rps samples) reads ABSENT, never a
1094/// false 0 (mirrors the per-phase `write_schbench_scalars` gating and the
1095/// carrier-less graceful degradation of [`populate_run_distribution_metrics`]).
1096///
1097/// The `*_whole` keys are `crate::stats::MetricKind::PerRunDistribution`:
1098/// noise-compared per-run by `crate::stats::noise_findings` (each run's own p99),
1099/// NEVER cross-RUN folded (a percentile of a union is not a mean of per-run
1100/// percentiles, and the per-phase histograms are dropped at the cross-run
1101/// boundary), so they are gated out of the cross-RUN ext fold and the within-run
1102/// reducers (`is_derived`). Distinct names from the per-phase percentile keys
1103/// (one registry name = one kind), produced solely here.
1104pub fn populate_run_pooled_schbench_distribution(stats: &mut ScenarioStats) {
1105    use crate::stats::{
1106        SCHBENCH_REQUEST_MAX_US_WHOLE, SCHBENCH_REQUEST_MIN_US_WHOLE,
1107        SCHBENCH_REQUEST_P50_US_WHOLE, SCHBENCH_REQUEST_P90_US_WHOLE,
1108        SCHBENCH_REQUEST_P99_US_WHOLE, SCHBENCH_REQUEST_P999_US_WHOLE, SCHBENCH_RPS_MAX_WHOLE,
1109        SCHBENCH_RPS_MIN_WHOLE, SCHBENCH_RPS_P20_WHOLE, SCHBENCH_RPS_P50_WHOLE,
1110        SCHBENCH_RPS_P90_WHOLE, SCHBENCH_WAKEUP_MAX_US_WHOLE, SCHBENCH_WAKEUP_MIN_US_WHOLE,
1111        SCHBENCH_WAKEUP_P50_US_WHOLE, SCHBENCH_WAKEUP_P90_US_WHOLE, SCHBENCH_WAKEUP_P99_US_WHOLE,
1112        SCHBENCH_WAKEUP_P999_US_WHOLE,
1113    };
1114    use crate::workload::schbench::plat::{Pct, PlatStats};
1115
1116    // Union the per-stream histograms across ALL phases+cgroups (combine =
1117    // associative bucket-count add → the faithful pooled histogram).
1118    let mut wakeup = PlatStats::default();
1119    let mut request = PlatStats::default();
1120    let mut rps = PlatStats::default();
1121    for phase in &stats.phases {
1122        for pc in phase.per_cgroup.values() {
1123            if let Some(s) = pc.schbench.as_ref() {
1124                wakeup.combine(&s.wakeup);
1125                request.combine(&s.request);
1126                rps.combine(&s.rps);
1127            }
1128        }
1129    }
1130    // Latency streams: 4 percentiles + min/max, re-derived over the union, µs.
1131    if wakeup.sample_count() > 0 {
1132        let q = wakeup.percentiles();
1133        stats.ext_metrics.insert(
1134            SCHBENCH_WAKEUP_P50_US_WHOLE.to_string(),
1135            q.value_at(Pct::P50) as f64,
1136        );
1137        stats.ext_metrics.insert(
1138            SCHBENCH_WAKEUP_P90_US_WHOLE.to_string(),
1139            q.value_at(Pct::P90) as f64,
1140        );
1141        stats.ext_metrics.insert(
1142            SCHBENCH_WAKEUP_P99_US_WHOLE.to_string(),
1143            q.value_at(Pct::P99) as f64,
1144        );
1145        stats.ext_metrics.insert(
1146            SCHBENCH_WAKEUP_P999_US_WHOLE.to_string(),
1147            q.value_at(Pct::P999) as f64,
1148        );
1149        stats
1150            .ext_metrics
1151            .insert(SCHBENCH_WAKEUP_MIN_US_WHOLE.to_string(), q.min as f64);
1152        stats
1153            .ext_metrics
1154            .insert(SCHBENCH_WAKEUP_MAX_US_WHOLE.to_string(), q.max as f64);
1155    }
1156    if request.sample_count() > 0 {
1157        let q = request.percentiles();
1158        stats.ext_metrics.insert(
1159            SCHBENCH_REQUEST_P50_US_WHOLE.to_string(),
1160            q.value_at(Pct::P50) as f64,
1161        );
1162        stats.ext_metrics.insert(
1163            SCHBENCH_REQUEST_P90_US_WHOLE.to_string(),
1164            q.value_at(Pct::P90) as f64,
1165        );
1166        stats.ext_metrics.insert(
1167            SCHBENCH_REQUEST_P99_US_WHOLE.to_string(),
1168            q.value_at(Pct::P99) as f64,
1169        );
1170        stats.ext_metrics.insert(
1171            SCHBENCH_REQUEST_P999_US_WHOLE.to_string(),
1172            q.value_at(Pct::P999) as f64,
1173        );
1174        stats
1175            .ext_metrics
1176            .insert(SCHBENCH_REQUEST_MIN_US_WHOLE.to_string(), q.min as f64);
1177        stats
1178            .ext_metrics
1179            .insert(SCHBENCH_REQUEST_MAX_US_WHOLE.to_string(), q.max as f64);
1180    }
1181    // RPS stream: PLIST_FOR_RPS = 20/50/90 + min/max (the schbench rps table).
1182    if rps.sample_count() > 0 {
1183        let r = rps.percentiles();
1184        stats.ext_metrics.insert(
1185            SCHBENCH_RPS_P20_WHOLE.to_string(),
1186            r.value_at(Pct::P20) as f64,
1187        );
1188        stats.ext_metrics.insert(
1189            SCHBENCH_RPS_P50_WHOLE.to_string(),
1190            r.value_at(Pct::P50) as f64,
1191        );
1192        stats.ext_metrics.insert(
1193            SCHBENCH_RPS_P90_WHOLE.to_string(),
1194            r.value_at(Pct::P90) as f64,
1195        );
1196        stats
1197            .ext_metrics
1198            .insert(SCHBENCH_RPS_MIN_WHOLE.to_string(), r.min as f64);
1199        stats
1200            .ext_metrics
1201            .insert(SCHBENCH_RPS_MAX_WHOLE.to_string(), r.max as f64);
1202    }
1203}
1204
1205/// Populate run-level DERIVED distributional metrics into
1206/// `stats.ext_metrics`: every registered `MetricKind::Distribution`,
1207/// `MetricKind::WorstLowest`, `MetricKind::WakeLatencyTailRatio`, and
1208/// `MetricKind::WorstCrossNodeRatio`. This is the SOLE
1209/// within-run producer of those metrics' values — they carry no per-phase
1210/// sample slice and no cross-cgroup merge fold, and their registry accessors
1211/// are `|_| None`, so `MetricDef::read` reads the value
1212/// written here from `ext_metrics`.
1213///
1214/// DISTRIBUTION (the 5 wake / run-delay aggregates): pools the RAW sample
1215/// vectors held in `stats.phases[].per_cgroup` across EVERY phase and EVERY
1216/// cgroup into one combined set, then recomputes the percentile / CV / mean
1217/// / extreme over it — the statistic of the union, NOT a max or mean of
1218/// per-cgroup reductions (the percentile of a union is not the max of
1219/// per-source percentiles). The ns→µs scale is applied ONCE here (the
1220/// carriers store raw ns, per [`PhaseCgroupStats::run_delays_ns`]). The wake
1221/// pool is population-WEIGHTED: each phase carrier's samples carry weight
1222/// `wake_sample_total / wake_latencies_ns.len()`, so a phase whose reservoir
1223/// hit the cap contributes by true population, not capped length (the
1224/// cross-PHASE de-skew) — reduced via the weighted percentile / moments.
1225/// The run-delay pool is unweighted (per-worker, never reservoir-capped, so
1226/// length IS population). Below the wake cap every weight is 1.0, so the
1227/// weighted P99 / median / mean / worst are byte-identical to the unweighted
1228/// concat; the weighted CV matches only within ~1e-9 (it sums the mean in f64
1229/// where the unweighted path sums in u64 — a weighted variance cannot keep the
1230/// u64 sum).
1231///
1232/// CARRIER-LESS FOLD (graceful degradation): a cgroup whose raw samples are
1233/// NOT in the pool — a backdrop epoch that fell on BASELINE or the
1234/// inter-step gap (no paired host bucket, so no carrier) or a cgroup whose
1235/// carrier was stripped/empty (`strip_phase_cgroup_samples`) — is NOT
1236/// dropped. Its
1237/// surviving per-cgroup [`CgroupStats`] reduction folds worst-wins (max — every
1238/// Distribution metric is `LowerBetter`, registry-gated) into the pooled value.
1239/// The CgroupStats reductions are never stripped — `stats.cgroups[]` is the
1240/// already-reduced `cgroup_stats(reports)` output, a SEPARATE reduction path
1241/// from the per-phase carriers — so a carrier-less cgroup always has a source.
1242/// When EVERY carrier is empty (a fully-stripped run) the pool is empty and the
1243/// result degenerates to the max over every cgroup's reduction — the pre-Item-7
1244/// cross-cgroup max. NOTE the value CLASS of a folded cgroup differs from a
1245/// pooled one for the P99 / Median / Mean / CV reductions: a pooled cgroup
1246/// contributes to the percentile of the union; a carrier-less cgroup
1247/// contributes its per-cgroup reduction worst-wins (a worst-cgroup proxy, not
1248/// pooled). For the `SampleReduction::Worst` reduction the two COINCIDE
1249/// (max-of-union == max-of-per-cgroup-maxes), so the carrier-less fold is exact
1250/// there, not a proxy. A second asymmetry specific to CV (from the population
1251/// weighting): the POOLED CV divides variance/mean by Σ per-sample weights (the
1252/// reconstructed population), while a carrier-less cgroup's folded CV is
1253/// [`cgroup_stats`]'s UNWEIGHTED CV (`n = all_latencies.len()`). The two
1254/// coincide below the cap (all weights 1.0) and diverge above it; the mix is
1255/// sound — a carrier-less cgroup has no per-phase weight data to
1256/// population-weight (its carrier is absent by definition), and both feed the
1257/// same LowerBetter worst-wins max. Backdrop step-phase carriers now join
1258/// the pool directly (per-epoch expansion in `collect_handles`); only the
1259/// carrier-less cases above fold worst-wins.
1260///
1261/// WORSTLOWEST (the 2 iteration efficiencies): the lowest (worst) cgroup's
1262/// efficiency, computed per-cgroup from the `stats.cgroups[]` COUNTERS via
1263/// [`CgroupStats::iterations_per_worker`] / [`CgroupStats::iterations_per_cpu_sec`]
1264/// and the None-aware lowest-wins fold (a measured `Some(0.0)` — starvation
1265/// — wins; a no-data `None` is skipped; an all-`None` cohort writes no key,
1266/// preserving absence as a missing ext entry rather than a `0.0`). The
1267/// counters survive stripping, so WorstLowest needs no fallback branch.
1268///
1269/// Runs post-merge at the eval layer beside
1270/// [`populate_run_pooled_iteration_rate`], AFTER the per-cgroup
1271/// carriers are folded into `stats.phases` and BEFORE the sidecar write, so
1272/// `stats.phases[].per_cgroup` is fully merged and `stats.cgroups` is the
1273/// final per-cgroup roll-up.
1274pub fn populate_run_distribution_metrics(stats: &mut ScenarioStats) {
1275    // Pool the per-phase per-cgroup raw sample vectors across every phase and
1276    // cgroup ONCE for the Distribution PRIMARY path, then sort so the
1277    // percentile reductions can index directly. `wake_latencies_ns` is
1278    // per-WAKEUP (reservoir-capped at MAX_WAKE_SAMPLES on the carrier because
1279    // it can reach 100k); `run_delays_ns` is per-WORKER (one sample/worker, not
1280    // capped), so the run-delay pool is total-workers × phases — genuinely
1281    // small. The wake pool is NOT intrinsically small: it is the union of the
1282    // per-carrier wake vectors, num_carriers × MAX_WAKE_SAMPLES worst case, so
1283    // its size is bounded by the upstream 16 MiB bulk-frame cap on the arriving
1284    // carriers (strip_phase_cgroup_samples is the overflow lever) rather than by
1285    // being tiny — no OOM risk, no cap needed here. Both are transient: reduced
1286    // to scalars here, never re-serialized.
1287    // Wake samples carry a per-sample population WEIGHT (`wake_sample_total /
1288    // reservoir len`) so a >cap phase contributes in proportion to its true
1289    // population, not its guest-capped length (the cross-PHASE de-skew). Run-delay
1290    // samples are per-worker and never reservoir-capped (no `*_sample_total`), so
1291    // their length IS their population — pooled unweighted.
1292    let mut wake_pool: Vec<(u64, f64)> = Vec::new();
1293    // Distinct timer-latency pool, population-WEIGHTED like
1294    // wake_pool (reservoir-capped, so a >cap phase carries weight
1295    // timer_sample_total/len for the cross-phase de-skew).
1296    let mut timer_pool: Vec<(u64, f64)> = Vec::new();
1297    let mut run_delay_pool: Vec<u64> = Vec::new();
1298    // Names of cgroups that contributed NON-EMPTY samples to each pool. A
1299    // cgroup absent here — a backdrop epoch that fell on BASELINE / the
1300    // inter-step gap (no paired host bucket, so no carrier) or a
1301    // stripped/empty carrier — is NOT dropped from the run-level
1302    // Distribution: the re-pool folds its surviving per-cgroup CgroupStats
1303    // reduction worst-wins (see `populate_run_distribution_metrics_from`).
1304    // Backdrop step-phase carriers now join the pool directly (per-epoch
1305    // expansion in collect_handles), so a step-matched backdrop epoch pools
1306    // rather than worst-wins-folds.
1307    //
1308    // The fallback dedup keys on cgroup NAME (a `stats.cgroups` entry whose
1309    // name is in `*_carriers` is pooled, not reduction-folded), which assumes
1310    // carrier-bearing and carrier-less cgroup names are DISJOINT. That holds
1311    // WITHIN one step's collect (cgroupfs path uniqueness — two live cgroups
1312    // cannot share a name, mkdir would EEXIST — and a single collect_handles
1313    // call attaches carriers to all its handles or none). It does NOT hold
1314    // across STEPS: `AssertResult::merge` extends `stats.cgroups` per
1315    // (handle, step), so a name that carried samples at step k recurs at step
1316    // k+1, and the step-(k+1) entry is skipped by this dedup (its name is in
1317    // `*_carriers`). That only OMITS a contribution, never vanishes the metric
1318    // (the step-k pool still produces it). A skipped step-(k+1) entry whose
1319    // carrier is merely EMPTY (collected no samples) is harmless: its per-cgroup
1320    // reduction is the trivial zero a worst-wins f64::max ignores. The only
1321    // LOSSY case is a step-(k+1) entry STRIPPED of live samples while step k
1322    // survives, and that cannot arise today: `strip_phase_cgroup_samples` strips
1323    // RUN-WIDE (every phase at once), so a run is never partially stripped per
1324    // step. A backdrop name now enters `*_carriers` (pooled once via its
1325    // per-epoch expansion) so it is skipped from the reduction-fold — and a
1326    // backdrop and a step-local cgroup cannot share a live name (cgroupfs
1327    // mkdir EEXIST; a backdrop is live the whole scenario), so each
1328    // stats.cgroups entry still contributes via exactly one of {pool,
1329    // reduction-fold} — no double count.
1330    let mut wake_carriers: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
1331    let mut timer_carriers: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
1332    let mut run_delay_carriers: std::collections::BTreeSet<&str> =
1333        std::collections::BTreeSet::new();
1334    for phase in &stats.phases {
1335        for (cgname, pcg) in &phase.per_cgroup {
1336            if !pcg.wake_latencies_ns.is_empty() {
1337                // Per-sample weight = true population / surviving reservoir size.
1338                // A ≤cap carrier has len == wake_sample_total → weight 1.0, so the
1339                // pool is value-for-value with the unweighted concat; a >cap
1340                // carrier's capped samples each stand for `total/len > 1` true
1341                // wakes, restoring the cross-phase population proportion.
1342                //
1343                // INVARIANT: `reservoir_push` bumps wake_sample_total on EVERY
1344                // wakeup but pushes into the reservoir only up to MAX_WAKE_SAMPLES,
1345                // and both the carrier merge and `phase_cgroup_stats` SUM the two,
1346                // so wake_sample_total >= len always (== len below the cap). A
1347                // carrier violating that — samples present but a zeroed/under-count
1348                // total — would yield weight < 1 and silently UNDER-weight (at
1349                // weight 0, DROP) its samples. Clamp the numerator to len so a
1350                // malformed carrier degrades to unit weight (reservoir treated as
1351                // its own population) instead of dropping data; debug_assert the
1352                // invariant so a real counting bug surfaces in dev.
1353                let len = pcg.wake_latencies_ns.len() as u64;
1354                debug_assert!(
1355                    pcg.wake_sample_total >= len,
1356                    "wake_sample_total ({}) < reservoir len ({}): malformed carrier",
1357                    pcg.wake_sample_total,
1358                    len,
1359                );
1360                let w = pcg.wake_sample_total.max(len) as f64 / len as f64;
1361                wake_pool.extend(pcg.wake_latencies_ns.iter().map(|&v| (v, w)));
1362                wake_carriers.insert(cgname.as_str());
1363            }
1364            if !pcg.timer_latencies_ns.is_empty() {
1365                // Population-weighted exactly like the wake pool: a >cap phase's
1366                // capped samples each stand for timer_sample_total/len true
1367                // wakes, restoring the cross-phase population proportion.
1368                let len = pcg.timer_latencies_ns.len() as u64;
1369                debug_assert!(
1370                    pcg.timer_sample_total >= len,
1371                    "timer_sample_total ({}) < reservoir len ({}): malformed carrier",
1372                    pcg.timer_sample_total,
1373                    len,
1374                );
1375                let w = pcg.timer_sample_total.max(len) as f64 / len as f64;
1376                timer_pool.extend(pcg.timer_latencies_ns.iter().map(|&v| (v, w)));
1377                timer_carriers.insert(cgname.as_str());
1378            }
1379            if !pcg.run_delays_ns.is_empty() {
1380                run_delay_pool.extend_from_slice(&pcg.run_delays_ns);
1381                run_delay_carriers.insert(cgname.as_str());
1382            }
1383        }
1384    }
1385    wake_pool.sort_unstable_by_key(|&(v, _)| v);
1386    timer_pool.sort_unstable_by_key(|&(v, _)| v);
1387    run_delay_pool.sort_unstable();
1388    populate_run_distribution_metrics_from(
1389        &mut stats.ext_metrics,
1390        crate::stats::METRICS.iter().filter_map(|m| {
1391            matches!(
1392                m.kind,
1393                crate::stats::MetricKind::Distribution { .. }
1394                    | crate::stats::MetricKind::WorstLowest { .. }
1395                    | crate::stats::MetricKind::WakeLatencyTailRatio
1396            )
1397            .then_some((m.name, m.kind))
1398        }),
1399        &wake_pool,
1400        &wake_carriers,
1401        &timer_pool,
1402        &timer_carriers,
1403        &run_delay_pool,
1404        &run_delay_carriers,
1405        &stats.cgroups,
1406        stats.total_iterations,
1407    );
1408    // worst_page_locality (WorstLowest{NumaLocal,NumaTotal}) re-pools from the
1409    // per-phase NUMA carriers, which the _from helper above cannot reach (it
1410    // takes only stats.cgroups, and the reports-only CgroupStats hardcodes
1411    // page_locality 0.0). Single-sourced with run_metric's WorstPageLocality via
1412    // worst_page_locality(): the lowest per-cgroup page_locality over cgroups
1413    // that measured NUMA residency (a measured 0.0 — all off-node — winning); an
1414    // all-unmeasured cohort writes no key (absence preserved as a missing ext
1415    // entry, never a 0.0 sentinel). numa_agg_per_cgroup()'s borrow ends before
1416    // the insert.
1417    if let Some(v) = stats.worst_page_locality() {
1418        stats
1419            .ext_metrics
1420            .insert("worst_page_locality".to_string(), v);
1421    }
1422    // worst_cross_node_migration_ratio (WorstCrossNodeRatio) re-pools from the same
1423    // per-phase NUMA carriers — the MAX per-cgroup churn ratio over the latest
1424    // residency total — single-sourced with run_metric's WorstCrossNodeMigrationRatio
1425    // via worst_cross_node_migration_ratio(); an all-unmeasured cohort writes no key
1426    // (absence preserved). Not handled by the _from helper above (it has only
1427    // stats.cgroups, and the metric is excluded from that filter).
1428    if let Some(v) = stats.worst_cross_node_migration_ratio() {
1429        stats
1430            .ext_metrics
1431            .insert("worst_cross_node_migration_ratio".to_string(), v);
1432    }
1433}
1434
1435/// Inner of [`populate_run_distribution_metrics`] taking the metric specs
1436/// `(name, kind)` and the pre-pooled+SORTED sample sets explicitly, so the
1437/// re-pool math is unit-testable without registered metrics (the
1438/// `derive_rate_metrics_from` precedent). `wake_pool` / `run_delay_pool` are
1439/// the cross-phase+cross-cgroup raw-ns unions (ascending); `*_carriers` name
1440/// the cgroups that contributed samples to each pool; `cgroups` supplies the
1441/// WorstLowest counters and the per-cgroup reductions that carrier-less
1442/// cgroups (backdrop / stripped) fold into the Distribution result.
1443#[allow(clippy::too_many_arguments)]
1444pub(crate) fn populate_run_distribution_metrics_from<'a>(
1445    target: &mut std::collections::BTreeMap<String, f64>,
1446    metrics: impl Iterator<Item = (&'a str, crate::stats::MetricKind)>,
1447    wake_pool: &[(u64, f64)],
1448    wake_carriers: &std::collections::BTreeSet<&str>,
1449    timer_pool: &[(u64, f64)],
1450    timer_carriers: &std::collections::BTreeSet<&str>,
1451    run_delay_pool: &[u64],
1452    run_delay_carriers: &std::collections::BTreeSet<&str>,
1453    cgroups: &[CgroupStats],
1454    run_total_iterations: u64,
1455) {
1456    use crate::stats::{MetricKind, SampleSource, WorstLowestDenominator, WorstLowestNumerator};
1457    for (name, kind) in metrics {
1458        let value: Option<f64> = match kind {
1459            MetricKind::Distribution { source, reduction } => {
1460                // Pool the carried samples (the thesis: percentile of the
1461                // UNION), then fold worst-wins (max — Distribution is
1462                // LowerBetter, registry-gated) the surviving per-cgroup
1463                // reduction of every cgroup WITHOUT a carrier-with-samples for
1464                // this source (a backdrop, or a stripped/empty carrier), so no
1465                // cgroup is dropped from the run-level distribution. When EVERY
1466                // carrier is empty (fully stripped) the pool is empty and this
1467                // degenerates to the max over every cgroup — the pre-Item-7
1468                // cross-cgroup max.
1469                //
1470                // Pool reduction is per-source: WakeLatencyNs is population-WEIGHTED
1471                // (each phase's guest-capped samples carry weight
1472                // wake_sample_total/len, so a >cap phase contributes by true
1473                // population not capped length — the cross-PHASE de-skew, via
1474                // reduce_weighted_sorted_distribution); RunDelayNs is unweighted
1475                // (per-worker, never reservoir-capped, so length IS population, via
1476                // reduce_sorted_distribution).
1477                //
1478                // CONTRACT (now uniform with WorstLowest and WakeLatencyTailRatio
1479                // below): a cohort with NO measurement for this source anywhere
1480                // (empty carrier pool AND every non-carrier cgroup not-measured)
1481                // yields ABSENCE (None), not Some(0.0). A percentile / mean over
1482                // zero samples is undefined, not a measured zero — folding its
1483                // 0.0 sentinel into the cross-run mean would, for the LowerBetter
1484                // wake/timer/run-delay metrics, falsely drag the mean toward
1485                // "perfect". The non-carrier fold below is gated on
1486                // `cg.measured_for(source)` so an unmeasured cgroup contributes
1487                // nothing; a non-carrier cgroup that DID measure (a genuine
1488                // measured zero, e.g. workers that queued for no time) still
1489                // contributes its real value. This matches WorstLowest (None when
1490                // every iterations_per_*() is None) and WakeLatencyTailRatio (None
1491                // when no cgroup has a tail) — every distributional kind now emits
1492                // None for a no-measurement run.
1493                let (mut v, carriers): (Option<f64>, &std::collections::BTreeSet<&str>) =
1494                    match source {
1495                        SampleSource::WakeLatencyNs => (
1496                            (!wake_pool.is_empty())
1497                                .then(|| reduce_weighted_sorted_distribution(wake_pool, reduction)),
1498                            wake_carriers,
1499                        ),
1500                        SampleSource::TimerLatencyNs => (
1501                            (!timer_pool.is_empty()).then(|| {
1502                                reduce_weighted_sorted_distribution(timer_pool, reduction)
1503                            }),
1504                            timer_carriers,
1505                        ),
1506                        SampleSource::RunDelayNs => (
1507                            (!run_delay_pool.is_empty())
1508                                .then(|| reduce_sorted_distribution(run_delay_pool, reduction)),
1509                            run_delay_carriers,
1510                        ),
1511                    };
1512                for cg in cgroups {
1513                    if !carriers.contains(cg.cgroup_name.as_str()) && cg.measured_for(source) {
1514                        let r = distribution_cgroup_reduction(cg, source, reduction);
1515                        v = Some(v.map_or(r, |acc| acc.max(r)));
1516                    }
1517                }
1518                v
1519            }
1520            // The iterations-efficiency WorstLowest selectors (numerator
1521            // Iterations) re-pool from the `stats.cgroups` counters here; the
1522            // denominator picks the per-cgroup efficiency method. The
1523            // page-locality selector (numerator NumaLocal) re-pools instead from
1524            // the per-phase NUMA carriers in populate_run_distribution_metrics
1525            // (this fn has only stats.cgroups, not stats.phases), so it is
1526            // skipped here (the NumaLocal arm below returns None).
1527            //
1528            // In a MULTI-STEP scenario `AssertResult::merge` extends
1529            // `stats.cgroups` per (handle, step), so the same cgroup name
1530            // appears once per step; this selects the lowest single
1531            // (handle, step) entry, NOT a per-name whole-run efficiency. That
1532            // preserves the deleted `fold_lowest_some` granularity exactly and
1533            // mirrors `populate_run_pooled_iteration_rate`, which sums
1534            // over the same per-(handle, step) entries.
1535            MetricKind::WorstLowest {
1536                numerator: WorstLowestNumerator::Iterations,
1537                denominator,
1538            } => {
1539                let mut worst: Option<f64> = None;
1540                for cg in cgroups {
1541                    let per_cg = match denominator {
1542                        WorstLowestDenominator::NumWorkers => cg.iterations_per_worker(),
1543                        WorstLowestDenominator::CpuTimeNs => cg.iterations_per_cpu_sec(),
1544                        // NumaTotal pairs only with the NumaLocal numerator
1545                        // (handled in the NumaLocal arm below) — never with
1546                        // Iterations.
1547                        WorstLowestDenominator::NumaTotal => None,
1548                    };
1549                    // Lowest-wins, None-aware (the semantic the deleted
1550                    // `fold_lowest_some` carried in `AssertResult::merge`): a
1551                    // measured `Some(0.0)` (starvation) wins the worst bucket;
1552                    // a `None` is skipped.
1553                    if let Some(v) = per_cg
1554                        && worst.is_none_or(|w| v < w)
1555                    {
1556                        worst = Some(v);
1557                    }
1558                }
1559                worst
1560            }
1561            // page-locality (numerator NumaLocal): re-pooled from the per-phase
1562            // NUMA carriers in populate_run_distribution_metrics, not here.
1563            MetricKind::WorstLowest {
1564                numerator: WorstLowestNumerator::NumaLocal,
1565                ..
1566            } => None,
1567            // Worst-cgroup wake-latency tail amplification: the MAX over each
1568            // cgroup's own p99/median ratio (`CgroupStats::wake_latency_tail_ratio`).
1569            // Emit NO key below the min-iterations noise floor (low-N ratios are
1570            // single-outlier noise, not a distributional signal — gated HERE at
1571            // the producer, NOT via a meaned-iteration accessor on the
1572            // aggregated row), and none when no cgroup carried a measurable tail
1573            // (every per-cgroup ratio 0.0, i.e. no median wake latency anywhere).
1574            // Absence then stays distinct from a measured value and no
1575            // sub-threshold run enters the cross-RUN mean. `wake_latency_tail_ratio`
1576            // returns 0.0 for a cgroup with no wake samples (median <= 0), which
1577            // a max-wins fold over the r > 0.0 reals correctly skips.
1578            MetricKind::WakeLatencyTailRatio => {
1579                if run_total_iterations < crate::stats::WAKE_LATENCY_TAIL_RATIO_MIN_ITERATIONS {
1580                    None
1581                } else {
1582                    let mut worst: Option<f64> = None;
1583                    for cg in cgroups {
1584                        let r = cg.wake_latency_tail_ratio();
1585                        if r > 0.0 {
1586                            worst = Some(worst.map_or(r, |w| w.max(r)));
1587                        }
1588                    }
1589                    worst
1590                }
1591            }
1592            _ => None,
1593        };
1594        // Insert only a real, FINITE value: an absent key (all-None
1595        // WorstLowest cohort, or no cgroups at all) stays distinct from a
1596        // measured 0.0, matching the None-vs-Some(0.0) contract the typed
1597        // Option carried. The is_finite guard is a no-op for every
1598        // registry-valid metric (reduce_sorted_distribution reduces non-empty
1599        // pools with CV guarded to 0.0 on zero mean; WorstLowest reuses
1600        // iterations_per_worker()/iterations_per_cpu_sec() which return None on
1601        // a zero denominator), but it MATTERS for the registry-impossible
1602        // cross-source arm of distribution_cgroup_reduction: that arm returns
1603        // NaN, and when a Distribution has no pool (every carrier stripped) the
1604        // carrier-less fold can carry that NaN to `v`. An inserted NaN would
1605        // fail the ENTIRE serde_json sidecar write (serde_json rejects
1606        // non-finite), losing ALL run telemetry — so the guard degrades a
1607        // misauthored metric to ABSENCE here rather than risking that write
1608        // failure downstream.
1609        if let Some(v) = value.filter(|v| v.is_finite()) {
1610            target.insert(name.to_string(), v);
1611        }
1612    }
1613}
1614
1615/// Reduce a NON-EMPTY ascending-sorted raw-ns sample pool to one
1616/// [`crate::stats::SampleReduction`] value, ns→µs once. Mirrors the
1617/// per-cgroup reductions [`cgroup_stats`] computes (p99 / median via
1618/// [`percentile`], CV with `n = pool.len()`, mean, max) so the run-level
1619/// re-pool reproduces them over the COMBINED cross-cgroup set — to within
1620/// FP tolerance for CV / mean, not bit-exactly: this sums over the
1621/// ASCENDING-sorted pool while `cgroup_stats` sums over the unsorted
1622/// arrival order, so the float results differ by ~1e-15 (the parity test
1623/// `repool_distribution_value_for_value_with_cgroup_stats` uses a 1e-9
1624/// bound). Same "distribution-equivalent, not byte-identical" framing as
1625/// the `wake_latencies_ns` carrier doc.
1626pub(crate) fn reduce_sorted_distribution(
1627    sorted: &[u64],
1628    reduction: crate::stats::SampleReduction,
1629) -> f64 {
1630    use crate::stats::SampleReduction;
1631    match reduction {
1632        SampleReduction::P99 => percentile(sorted, 0.99) as f64 / 1000.0,
1633        SampleReduction::P999 => percentile(sorted, 0.999) as f64 / 1000.0,
1634        SampleReduction::Median => percentile(sorted, 0.5) as f64 / 1000.0,
1635        SampleReduction::Cv => {
1636            let n = sorted.len() as f64;
1637            let mean_ns = sorted.iter().sum::<u64>() as f64 / n;
1638            if mean_ns > 0.0 {
1639                let variance = sorted
1640                    .iter()
1641                    .map(|&v| (v as f64 - mean_ns).powi(2))
1642                    .sum::<f64>()
1643                    / n;
1644                variance.sqrt() / mean_ns
1645            } else {
1646                0.0
1647            }
1648        }
1649        // Divide ONCE on the summed/maxed ns (the carriers store raw ns):
1650        // mean(ns)/1000 == mean(ns/1000) and max(ns)/1000 == max(ns/1000).
1651        // Sum in f64 (not u64-then-cast) to match cgroup_stats's f64 run-delay
1652        // accumulation and PhaseCgroupStats::run_delay_summary — overflow-safe
1653        // (an f64 sum saturates toward +inf; a u64 sum would panic in debug /
1654        // silently wrap in release on a pathological pool), value identical
1655        // within the 1e-9 parity bound. (The Cv arm's mean_ns above keeps the u64 sum
1656        // because cgroup_stats's CV also u64-sums all_latencies — matching it is
1657        // exact-parity-preserving there.)
1658        SampleReduction::Mean => {
1659            sorted.iter().map(|&v| v as f64).sum::<f64>() / sorted.len() as f64 / 1000.0
1660        }
1661        // Sorted ascending, so the last element is the max.
1662        SampleReduction::Worst => *sorted.last().expect("non-empty by caller") as f64 / 1000.0,
1663    }
1664}
1665
1666/// Weighted nearest-rank percentile over a value-sorted `(value, weight)` pool —
1667/// the weighted sibling of [`percentile`]. Matches `percentile`'s convention
1668/// (the value at 1-indexed rank `ceil(W * p)`, `W` = total weight, floored at
1669/// rank 1) so with UNIT weights (every weight `1.0`) it returns byte-identically:
1670/// cumulative weight after `k` elements is `k`, `ceil(W*p) == ceil(n*p)`, and the
1671/// `.max(1.0)` floor mirrors `percentile`'s `saturating_sub(1)`, so the crossing
1672/// element is `percentile`'s `sorted[ceil(n*p)-1]` for p>0 and `sorted[0]` at
1673/// p=0. Used by the run-level wake re-pool to weight each phase's samples by
1674/// true population.
1675pub(crate) fn weighted_percentile(sorted: &[(u64, f64)], p: f64) -> u64 {
1676    if sorted.is_empty() {
1677        return 0;
1678    }
1679    debug_assert!(
1680        sorted.windows(2).all(|w| w[0].0 <= w[1].0),
1681        "weighted_percentile() requires value-sorted input",
1682    );
1683    let total: f64 = sorted.iter().map(|&(_, w)| w).sum();
1684    // Nearest-rank target, floored at 1 so `p == 0.0` maps to the first element
1685    // (mirrors percentile's saturating_sub(1) flooring rank 0 to index 0).
1686    let target = (total * p).ceil().max(1.0);
1687    let mut cum = 0.0;
1688    for &(v, w) in sorted {
1689        cum += w;
1690        if cum >= target {
1691            return v;
1692        }
1693    }
1694    sorted.last().map(|&(v, _)| v).unwrap_or(0)
1695}
1696
1697/// Weighted sibling of [`reduce_sorted_distribution`] for the wake-latency
1698/// re-pool: each `(value, weight)` carries a per-sample weight of
1699/// `wake_sample_total / reservoir_len`, so a >cap phase (reservoir-capped on the
1700/// guest) contributes in proportion to its TRUE population, not its capped
1701/// length — removing the cross-PHASE length-skew. With UNIT weights (every phase
1702/// ≤cap, so `len == wake_sample_total`) it reduces byte-identically to
1703/// [`reduce_sorted_distribution`] for P99 / Median / Mean / Worst; the Cv arm
1704/// differs only by the f64-vs-u64 mean sum. For the small fixed pool the parity
1705/// test uses, that gap is ~1e-15 (within its 1e-9 bound), but it grows ~n·ε with
1706/// pool size — a cross-phase pool can reach millions of samples (~1e-9–1e-8 on a
1707/// high-CV pool), so a LARGE-pool parity test must not assume a universal 1e-15.
1708/// A weighted variance cannot keep the u64 sum.  Exhaustive over SampleReduction,
1709/// mirroring [`reduce_sorted_distribution`], so a new variant fails the build.
1710///
1711/// The Cv / Mean `total_w <= 0.0` guards and [`weighted_percentile`]'s
1712/// all-weight-zero fall-through are degenerate-input belts: the capture-path
1713/// caller [`populate_run_distribution_metrics`] clamps every per-sample weight to
1714/// a floor of 1.0, so `total_w >= len >= 1` there and those branches are
1715/// unreachable on the production path.
1716pub(crate) fn reduce_weighted_sorted_distribution(
1717    sorted: &[(u64, f64)],
1718    reduction: crate::stats::SampleReduction,
1719) -> f64 {
1720    use crate::stats::SampleReduction;
1721    match reduction {
1722        SampleReduction::P99 => weighted_percentile(sorted, 0.99) as f64 / 1000.0,
1723        SampleReduction::P999 => weighted_percentile(sorted, 0.999) as f64 / 1000.0,
1724        SampleReduction::Median => weighted_percentile(sorted, 0.5) as f64 / 1000.0,
1725        SampleReduction::Cv => {
1726            let total_w: f64 = sorted.iter().map(|&(_, w)| w).sum();
1727            if total_w <= 0.0 {
1728                return 0.0;
1729            }
1730            let mean_ns = sorted.iter().map(|&(v, w)| v as f64 * w).sum::<f64>() / total_w;
1731            if mean_ns > 0.0 {
1732                let variance = sorted
1733                    .iter()
1734                    .map(|&(v, w)| w * (v as f64 - mean_ns).powi(2))
1735                    .sum::<f64>()
1736                    / total_w;
1737                variance.sqrt() / mean_ns
1738            } else {
1739                0.0
1740            }
1741        }
1742        SampleReduction::Mean => {
1743            let total_w: f64 = sorted.iter().map(|&(_, w)| w).sum();
1744            if total_w <= 0.0 {
1745                return 0.0;
1746            }
1747            sorted.iter().map(|&(v, w)| v as f64 * w).sum::<f64>() / total_w / 1000.0
1748        }
1749        // Max value present, weight-invariant — last element of the value-sorted pool.
1750        SampleReduction::Worst => sorted.last().map(|&(v, _)| v).unwrap_or(0) as f64 / 1000.0,
1751    }
1752}
1753
1754/// One cgroup's surviving [`CgroupStats`] reduction for a
1755/// [`crate::stats::MetricKind::Distribution`] (source, reduction) pair — the
1756/// value folded worst-wins into the run-level distribution for a cgroup whose
1757/// raw samples are NOT in the pool (a backdrop, or a stripped/empty carrier).
1758/// Worst-wins is `f64::max` (every Distribution metric is `LowerBetter`,
1759/// enforced by `every_metric_has_kind_consistent_with_naming`).
1760///
1761/// Per-source match, EXHAUSTIVE over SampleReduction (no `_` catch-all,
1762/// mirroring reduce_sorted_distribution) so a new SampleSource or
1763/// SampleReduction variant fails the build until a reduction field is wired.
1764/// The cross-source reductions (a wake source asking for a run-delay reduction,
1765/// or vice versa) are registry-impossible (no CgroupStats field exists), so
1766/// they debug_assert in tests and, in release, return `f64::NAN` rather than
1767/// 0.0 — NaN is IGNORED by the caller's `f64::max` worst-wins fold, and if it
1768/// still reaches `populate_run_distribution_metrics`'s insert (a pool-less
1769/// Distribution whose every carrier-less cgroup hits this arm) the is_finite
1770/// insert guard drops it to absence. Either way a registry-authoring mistake
1771/// drops the bogus contribution instead of folding a 0.0 that a LowerBetter
1772/// metric would read as "perfect".
1773fn distribution_cgroup_reduction(
1774    cg: &CgroupStats,
1775    source: crate::stats::SampleSource,
1776    reduction: crate::stats::SampleReduction,
1777) -> f64 {
1778    use crate::stats::{SampleReduction, SampleSource};
1779    match source {
1780        SampleSource::WakeLatencyNs => match reduction {
1781            SampleReduction::P99 => cg.p99_wake_latency_us,
1782            SampleReduction::Median => cg.median_wake_latency_us,
1783            SampleReduction::Cv => cg.wake_latency_cv,
1784            SampleReduction::P999 | SampleReduction::Mean | SampleReduction::Worst => {
1785                debug_assert!(false, "no CgroupStats wake reduction for {reduction:?}");
1786                f64::NAN
1787            }
1788        },
1789        SampleSource::RunDelayNs => match reduction {
1790            SampleReduction::Mean => cg.mean_run_delay_us,
1791            SampleReduction::Worst => cg.worst_run_delay_us,
1792            SampleReduction::P99
1793            | SampleReduction::P999
1794            | SampleReduction::Median
1795            | SampleReduction::Cv => {
1796                debug_assert!(
1797                    false,
1798                    "no CgroupStats run-delay reduction for {reduction:?}"
1799                );
1800                f64::NAN
1801            }
1802        },
1803        SampleSource::TimerLatencyNs => match reduction {
1804            SampleReduction::Median => cg.median_timer_latency_us,
1805            SampleReduction::P99 => cg.p99_timer_latency_us,
1806            SampleReduction::P999 => cg.p999_timer_latency_us,
1807            SampleReduction::Worst => cg.worst_timer_latency_us,
1808            SampleReduction::Cv | SampleReduction::Mean => {
1809                debug_assert!(false, "no CgroupStats timer reduction for {reduction:?}");
1810                f64::NAN
1811            }
1812        },
1813    }
1814}
1815
1816/// Populate cross-RUN aggregate entries for every registered
1817/// `crate::stats::MetricDef` whose `read_sample` returns finite
1818/// values across the entire sample series. Writes into
1819/// `target` (typically `ScenarioStats::ext_metrics`) under the
1820/// metric's registry name — the same key the per-phase
1821/// [`PhaseBucket::metrics`] uses, so cross-RUN and per-phase
1822/// consumers reference the same name.
1823///
1824/// Existing keys are NOT overwritten — a typed GauntletRow field's
1825/// value (populated via the MetricDef accessor at sidecar-write
1826/// time) wins on the read path, and this fn fills the gap for
1827/// registered metrics that have a `read_sample` wire but no typed
1828/// GauntletRow field. Without this fill, `cargo ktstr perf-delta`
1829/// silently skips the metric (read returns None on both sides, so the
1830/// `(None, None)` arm drops the pair).
1831///
1832/// Per-phase reduction dispatch is described on [`PhaseBucket`];
1833/// the cross-RUN fold here uses `crate::stats::aggregate_samples_for_phase`
1834/// over the full sample series, with TYPED_FIELD_NAMES gating to
1835/// avoid duplicating typed-accessor sources.
1836pub fn populate_run_ext_metrics(
1837    samples: &crate::scenario::sample::SampleSeries,
1838    target: &mut std::collections::BTreeMap<String, f64>,
1839) {
1840    // Typed-backed keys are skipped via the module-level TYPED_FIELD_NAMES
1841    // (shared with populate_run_ext_metrics_from_phases) so only
1842    // ext-metrics-only registry entries are populated here.
1843    for metric_def in crate::stats::METRICS {
1844        if target.contains_key(metric_def.name) {
1845            continue;
1846        }
1847        if TYPED_FIELD_NAMES.contains(&metric_def.name) {
1848            continue;
1849        }
1850        let readings: Vec<f64> = samples
1851            .iter_samples()
1852            .filter_map(|s| metric_def.read_sample(&s))
1853            .collect();
1854        if readings.is_empty() {
1855            continue;
1856        }
1857        if let Some(reduced) = crate::stats::aggregate_samples_for_phase(metric_def, &readings) {
1858            target.insert(metric_def.name.to_string(), reduced);
1859        }
1860    }
1861    // Run-level delivered-guest-CPU denominator for IRQ activity/fraction
1862    // metrics. Use the same earliest/latest per_cpu_time-bearing freezes as the
1863    // whole-run counter deltas above. Excluding CPUTIME_STEAL makes the rates
1864    // immune to host preemption without adding another observation source.
1865    if target.contains_key("total_hardirqs") {
1866        let placed: Vec<_> = samples
1867            .iter_samples()
1868            .filter(|s| !s.snapshot.per_cpu_time().is_empty())
1869            .filter_map(|s| s.elapsed_ms.map(|elapsed_ms| (s, elapsed_ms)))
1870            .collect();
1871        if let (Some((first, first_ms)), Some((last, last_ms))) = (
1872            placed.iter().min_by_key(|(_, elapsed_ms)| *elapsed_ms),
1873            placed.iter().max_by_key(|(_, elapsed_ms)| *elapsed_ms),
1874        ) && last_ms > first_ms
1875            && let Some(cpu_ns) = crate::assert::phase_build::guest_cpu_time_delta_ns(
1876                first.snapshot.per_cpu_time(),
1877                last.snapshot.per_cpu_time(),
1878            )
1879        {
1880            target
1881                .entry("total_phase_guest_cpu_ns".to_string())
1882                .or_insert(cpu_ns as f64);
1883            target
1884                .entry("total_phase_guest_cpu_sec".to_string())
1885                .or_insert(cpu_ns as f64 / 1e9);
1886        }
1887    }
1888    // Re-derive Rate metrics from the read_sample components just folded
1889    // in. populate_run_ext_metrics is pub and called standalone (tests,
1890    // and not only ahead of populate_run_ext_metrics_from_phases), so it
1891    // derives its own rates to stay self-contained.
1892    crate::stats::derive_rate_metrics(target);
1893}
1894
1895/// Derive the per-phase scalar metrics ([`crate::stats::MetricKind::PerPhase`])
1896/// for every cgroup carrier in each phase. Two families:
1897///
1898/// 1. NON-schbench carrier scalars (via [`write_carrier_scalars`], for EVERY
1899///    cgroup regardless of work type): the wake/run-delay/off-cpu distributions
1900///    + the migration/iterations/locality ratios + the carrier counters,
1901///    written ONLY into each carrier's `PhaseCgroupStats::metrics` (per-cgroup,
1902///    read via `phase_cgroup_metric`) — NO pooled [`PhaseBucket::metrics`]
1903///    entry; their run-level aggregate is the `worst_*` ext-metrics key.
1904/// 2. schbench scalars (via [`write_schbench_scalars`], only for cgroups
1905///    carrying a `SchbenchPhaseStats`): written per-cgroup into
1906///    `PhaseCgroupStats::metrics` AND, pooled across the phase's schbench
1907///    carriers, into [`PhaseBucket::metrics`] (read via `phase_metric`).
1908///    Percentiles MUST come from the POOLED histogram (combine the latency
1909///    histograms + integer-add the run-delay raw pairs), never an average of
1910///    per-cgroup percentiles; the run-delay means are ABSENT when `pcount == 0`
1911///    so a never-scheduled class is not a false `0`.
1912///
1913/// MUST run POST-merge: the buckets are final and keyed by `step_index`, so a
1914/// per-phase A/B claim reads the value via `phase_metric` /
1915/// `phase_cgroup_metric`. It runs after the per-cgroup carriers are folded
1916/// ([`fold_guest_per_cgroup_into_host_buckets`]) and after any
1917/// [`merge_matched_phase_buckets`] — both SKIP is_derived keys (the `continue`
1918/// in the merge loop), so deriving earlier would DROP the keys. Idempotent
1919/// (overwrites the keys), so callers that rebuild buckets per call (e.g.
1920/// `VmResult::phase_buckets`) may re-run it freely.
1921// doc_lazy_continuation: pre-existing numbered-list wording surfaced by the clippy
1922// 1.94 bump; renders fine. Suppress rather than reflow the prose.
1923#[allow(clippy::doc_lazy_continuation)]
1924pub(crate) fn derive_phase_metrics(phases: &mut [PhaseBucket]) {
1925    use crate::workload::schbench::run::SchbenchPhaseStats;
1926
1927    for bucket in phases.iter_mut() {
1928        // Derive the schbench scalars BOTH per cgroup (into pc.metrics — the
1929        // per-cgroup set: N cgroups -> N queryable sets) AND pooled across the
1930        // phase's cgroups (into bucket.metrics — the aggregate). Both go through the
1931        // SAME reducer (write_schbench_scalars) from the SAME carriers, so the
1932        // pooled set is the cross-cgroup re-pool of the per-cgroup carriers (a
1933        // percentile re-derives from the pooled histogram via PlatStats::combine =
1934        // bucket-count add, never averaged), and pooled == cross-cgroup re-pool.
1935        let mut pooled: Option<SchbenchPhaseStats> = None;
1936        let mut pooled_taobench: Option<crate::workload::taobench::run::TaobenchPhaseStats> = None;
1937        for pc in bucket.per_cgroup.values_mut() {
1938            // Non-schbench carrier-derived metrics (wake/run-delay/off-cpu
1939            // distributions + the migration/iterations/locality ratios + the
1940            // carrier counters), emitted for EVERY cgroup regardless of work
1941            // type. Runs BEFORE the schbench block: `write_carrier_scalars`
1942            // takes `&mut pc` (its summaries read many `pc` fields, then it
1943            // writes `pc.metrics`), so it cannot share the iteration with the
1944            // disjoint-field borrow the schbench block relies on.
1945            write_carrier_scalars(pc);
1946            // Disjoint field borrows: `s` reads pc.schbench, the reducer writes
1947            // pc.metrics — different fields, so the immutable + mutable borrows of
1948            // `*pc` coexist.
1949            if let Some(s) = pc.schbench.as_ref() {
1950                write_schbench_scalars(s, &mut pc.metrics);
1951                // `take()` completes the borrow before the reassignment in both arms.
1952                match pooled.take() {
1953                    Some(mut acc) => {
1954                        acc.merge(s);
1955                        pooled = Some(acc);
1956                    }
1957                    None => pooled = Some(s.clone()),
1958                }
1959            }
1960            // taobench per-phase: same disjoint-field-borrow pattern (`t` reads
1961            // pc.taobench, the reducer writes pc.metrics) + pool across cgroups.
1962            if let Some(t) = pc.taobench.as_ref() {
1963                write_taobench_scalars(t, &mut pc.metrics);
1964                match pooled_taobench.take() {
1965                    Some(mut acc) => {
1966                        acc.merge(t);
1967                        pooled_taobench = Some(acc);
1968                    }
1969                    None => pooled_taobench = Some(t.clone()),
1970                }
1971            }
1972        }
1973        if let Some(p) = pooled {
1974            write_schbench_scalars(&p, &mut bucket.metrics);
1975        }
1976        if let Some(t) = pooled_taobench {
1977            write_taobench_scalars(&t, &mut bucket.metrics);
1978        }
1979
1980        // Cross-cgroup phase pool for the canonical iteration_rate. Exclude a
1981        // carrier with no measured CPU time from BOTH sums: crediting its work
1982        // against another cgroup's CPU time would overstate efficiency.
1983        let total_cpu_time_ns = bucket
1984            .per_cgroup
1985            .values()
1986            .filter(|pc| pc.total_cpu_time_ns > 0)
1987            .map(|pc| pc.total_cpu_time_ns)
1988            .fold(0u64, u64::saturating_add);
1989        if total_cpu_time_ns > 0 {
1990            let total_iterations = bucket
1991                .per_cgroup
1992                .values()
1993                .filter(|pc| pc.total_cpu_time_ns > 0)
1994                .map(|pc| pc.total_iterations)
1995                .fold(0u64, u64::saturating_add);
1996            bucket.metrics.insert(
1997                "total_iterations_pooled".to_string(),
1998                total_iterations as f64,
1999            );
2000            bucket.metrics.insert(
2001                "total_cpu_time_sec".to_string(),
2002                total_cpu_time_ns as f64 / 1e9,
2003            );
2004            crate::stats::derive_rate_metrics(&mut bucket.metrics);
2005        }
2006    }
2007}
2008
2009/// Write the per-phase per-cgroup NON-schbench DERIVED scalars from ONE
2010/// carrier into `pc.metrics`, keyed by registry [`crate::stats::MetricDef`]
2011/// name — the per-cgroup analog of the run-level reductions, single-sourced
2012/// through the shared ratio helpers in [`crate::assert::reductions`] (so the
2013/// per-cgroup value and the run-level `worst_*` fold agree by construction)
2014/// and the carrier summary methods. Each gate mirrors the carrier's ABSENT
2015/// discipline: an empty wake pool, a not-measured off-CPU state, a
2016/// zero-worker cgroup, or a no-NUMA-pages cgroup emits no key (reads as
2017/// missing, never a false 0); `migration_ratio` is ALWAYS present (a measured
2018/// 0.0 when no iterations ran).
2019///
2020/// Unlike the schbench families, these families have NO pooled
2021/// `PhaseBucket::metrics` entry at all: as PerPhase / is_derived metrics whose
2022/// `read_sample` is `None`, the per-phase fold never writes them to
2023/// `bucket.metrics`. Their run-level aggregate is the `worst_*` key
2024/// `populate_run_distribution_metrics` writes into the run-level
2025/// `ScenarioStats::ext_metrics` map; per-phase they are queryable ONLY per-cgroup
2026/// via `pc.metrics`. So this writes ONLY `pc.metrics`, never `bucket.metrics` —
2027/// the per-cgroup carrier value is authoritative for the per-cgroup question.
2028///
2029/// Takes `&mut PhaseCgroupStats` (not `(&pc, &mut pc.metrics)`): it reads many
2030/// `pc` fields AND writes `pc.metrics`, so the summaries/ratios are bound to
2031/// locals from `&*pc` first, then the `&mut pc.metrics` borrow is taken once.
2032fn write_carrier_scalars(pc: &mut PhaseCgroupStats) {
2033    use crate::assert::reductions::{
2034        cross_node_migration_ratio_of, iterations_per_cpu_sec_of, iterations_per_worker_of,
2035        migration_ratio_of, page_locality_of,
2036    };
2037    // Read everything from &*pc into locals BEFORE the &mut pc.metrics borrow.
2038    let wake = pc.wake_summary().zip(pc.wake_cv());
2039    let timer = pc.timer_summary();
2040    let run_delay = pc.run_delay_summary();
2041    let off_cpu = pc.off_cpu_summary();
2042    let migration_ratio = migration_ratio_of(pc.total_migrations, pc.total_iterations);
2043    let ipw = iterations_per_worker_of(pc.num_workers, pc.total_iterations);
2044    let ipcs = iterations_per_cpu_sec_of(pc.num_workers, pc.total_cpu_time_ns, pc.total_iterations);
2045    let numa = if pc.numa_pages_total > 0 {
2046        Some((
2047            page_locality_of(pc.numa_pages_local, pc.numa_pages_total),
2048            cross_node_migration_ratio_of(pc.cross_node_migrated, pc.numa_pages_total),
2049        ))
2050    } else {
2051        None
2052    };
2053
2054    let m = &mut pc.metrics;
2055    // WAKE — all three or none (wake_summary + wake_cv share the empty-pool gate).
2056    if let Some(((p99, median), cv)) = wake {
2057        m.insert("p99_wake_latency_us".to_string(), p99);
2058        m.insert("median_wake_latency_us".to_string(), median);
2059        m.insert("wake_latency_cv".to_string(), cv);
2060    }
2061    // TIMER — all three or none (timer_summary shares the empty-pool gate).
2062    if let Some((median, p99, p999)) = timer {
2063        m.insert("median_timer_latency_us".to_string(), median);
2064        m.insert("p99_timer_latency_us".to_string(), p99);
2065        m.insert("p999_timer_latency_us".to_string(), p999);
2066    }
2067    // RUN-DELAY — both or none.
2068    if let Some((mean, worst)) = run_delay {
2069        m.insert("mean_run_delay_us".to_string(), mean);
2070        m.insert("max_run_delay_us".to_string(), worst);
2071    }
2072    // OFF-CPU — four or none (None == not-measured).
2073    if let Some((avg, min, max, spread)) = off_cpu {
2074        m.insert("avg_off_cpu_pct".to_string(), avg);
2075        m.insert("min_off_cpu_pct".to_string(), min);
2076        m.insert("max_off_cpu_pct".to_string(), max);
2077        m.insert("off_cpu_spread_pct".to_string(), spread);
2078    }
2079    // RATIOS — migration_ratio ALWAYS (measured 0.0 when no iterations ran).
2080    m.insert("migration_ratio".to_string(), migration_ratio);
2081    if let Some(v) = ipw {
2082        m.insert("iterations_per_worker".to_string(), v);
2083    }
2084    if let Some(v) = ipcs {
2085        m.insert("iteration_rate".to_string(), v);
2086    }
2087    // NUMA ratios only when pages were observed (mirrors the carrier's
2088    // numa_pages_total>0 gate; absent is more honest than a 0.0 default).
2089    if let Some((locality, cross_node)) = numa {
2090        m.insert("page_locality".to_string(), locality);
2091        m.insert("cross_node_migration_ratio".to_string(), cross_node);
2092    }
2093}
2094
2095/// Write the per-phase schbench scalar metrics derived from ONE
2096/// `SchbenchPhaseStats` (a single cgroup's carrier, or the cross-cgroup pool)
2097/// into `out`, keyed by registry [`crate::stats::MetricDef`] name. The sole
2098/// producer of these keys for both the per-cgroup ([`PhaseCgroupStats::metrics`])
2099/// and pooled ([`PhaseBucket::metrics`]) maps — one derivation, no duplicated
2100/// percentile / min/max / rps / sched-delay math. Each gate mirrors the carrier's
2101/// ABSENT discipline: an empty histogram or a zero pcount emits no key (reads as
2102/// missing, never a false 0); loop_count is always present (0 = a real measured
2103/// no-cycles outcome).
2104fn write_schbench_scalars(
2105    p: &crate::workload::schbench::run::SchbenchPhaseStats,
2106    out: &mut std::collections::BTreeMap<String, f64>,
2107) {
2108    use crate::stats::{
2109        SCHBENCH_LOOP_COUNT, SCHBENCH_REQUEST_MAX_US, SCHBENCH_REQUEST_MIN_US,
2110        SCHBENCH_REQUEST_P50_US, SCHBENCH_REQUEST_P90_US, SCHBENCH_REQUEST_P99_US,
2111        SCHBENCH_REQUEST_P999_US, SCHBENCH_RPS_MAX, SCHBENCH_RPS_MIN, SCHBENCH_RPS_P20,
2112        SCHBENCH_RPS_P50, SCHBENCH_RPS_P90, SCHBENCH_SCHED_DELAY_MSG_US,
2113        SCHBENCH_SCHED_DELAY_WORKER_US, SCHBENCH_WAKEUP_MAX_US, SCHBENCH_WAKEUP_MIN_US,
2114        SCHBENCH_WAKEUP_P50_US, SCHBENCH_WAKEUP_P90_US, SCHBENCH_WAKEUP_P99_US,
2115        SCHBENCH_WAKEUP_P999_US,
2116    };
2117    use crate::workload::schbench::plat::Pct;
2118
2119    if p.wakeup.sample_count() > 0 {
2120        let q = p.wakeup.percentiles();
2121        out.insert(
2122            SCHBENCH_WAKEUP_P50_US.to_string(),
2123            q.value_at(Pct::P50) as f64,
2124        );
2125        out.insert(
2126            SCHBENCH_WAKEUP_P90_US.to_string(),
2127            q.value_at(Pct::P90) as f64,
2128        );
2129        out.insert(
2130            SCHBENCH_WAKEUP_P99_US.to_string(),
2131            q.value_at(Pct::P99) as f64,
2132        );
2133        out.insert(
2134            SCHBENCH_WAKEUP_P999_US.to_string(),
2135            q.value_at(Pct::P999) as f64,
2136        );
2137        out.insert(SCHBENCH_WAKEUP_MIN_US.to_string(), q.min as f64);
2138        out.insert(SCHBENCH_WAKEUP_MAX_US.to_string(), q.max as f64);
2139    }
2140    if p.request.sample_count() > 0 {
2141        let q = p.request.percentiles();
2142        out.insert(
2143            SCHBENCH_REQUEST_P50_US.to_string(),
2144            q.value_at(Pct::P50) as f64,
2145        );
2146        out.insert(
2147            SCHBENCH_REQUEST_P90_US.to_string(),
2148            q.value_at(Pct::P90) as f64,
2149        );
2150        out.insert(
2151            SCHBENCH_REQUEST_P99_US.to_string(),
2152            q.value_at(Pct::P99) as f64,
2153        );
2154        out.insert(
2155            SCHBENCH_REQUEST_P999_US.to_string(),
2156            q.value_at(Pct::P999) as f64,
2157        );
2158        out.insert(SCHBENCH_REQUEST_MIN_US.to_string(), q.min as f64);
2159        out.insert(SCHBENCH_REQUEST_MAX_US.to_string(), q.max as f64);
2160    }
2161    // Per-phase achieved-RPS distribution (the control thread's per-second samples
2162    // attributed to this epoch). Gated on sample_count()>0 so a phase shorter than
2163    // the ~1s control cadence reads ABSENT, never rps=0. schbench's RPS table is
2164    // PLIST_FOR_RPS = 20/50/90 (schbench.c:130) + min/max.
2165    if p.rps.sample_count() > 0 {
2166        let r = p.rps.percentiles();
2167        out.insert(SCHBENCH_RPS_P20.to_string(), r.value_at(Pct::P20) as f64);
2168        out.insert(SCHBENCH_RPS_P50.to_string(), r.value_at(Pct::P50) as f64);
2169        out.insert(SCHBENCH_RPS_P90.to_string(), r.value_at(Pct::P90) as f64);
2170        out.insert(SCHBENCH_RPS_MIN.to_string(), r.min as f64);
2171        out.insert(SCHBENCH_RPS_MAX.to_string(), r.max as f64);
2172    }
2173    // Sample-weighted run-delay mean (Σrun_delay / Σpcount), ns→µs; ABSENT when
2174    // pcount==0 (a never-scheduled class) so it reads as missing, not 0.
2175    if p.msg_pcount > 0 {
2176        let mean_us = p.msg_run_delay_ns as f64 / p.msg_pcount as f64 / 1000.0;
2177        out.insert(SCHBENCH_SCHED_DELAY_MSG_US.to_string(), mean_us);
2178    }
2179    if p.worker_pcount > 0 {
2180        let mean_us = p.worker_run_delay_ns as f64 / p.worker_pcount as f64 / 1000.0;
2181        out.insert(SCHBENCH_SCHED_DELAY_WORKER_US.to_string(), mean_us);
2182    }
2183    // loop_count is always present for a schbench carrier: 0 = no cycles ran (a real
2184    // measured value; HigherBetter → worst), distinct from a non-schbench carrier
2185    // which has no schbench data at all (the caller skips it).
2186    out.insert(SCHBENCH_LOOP_COUNT.to_string(), p.loop_count as f64);
2187    // CPU-second-denominated throughput: completed cycles per CPU-second the
2188    // phase's WORKER threads actually received (Σ CLOCK_THREAD_CPUTIME_ID
2189    // deltas — message-thread CPU excluded, mirroring schbench's Σ worker
2190    // runtimes). Dilation-safe by construction; the wall picture reconstructs
2191    // as `rate / D` via the sidecar's `host_dilation`. ABSENT when no worker
2192    // CPU was measured (pre-field carrier, or a phase in which no worker ran).
2193    // The `schbench_rps_*` per-second samples above deliberately stay WALL
2194    // (upstream `rps_stats` parity — see the `SchbenchPhaseStats::rps` doc).
2195    if p.worker_cpu_ns > 0 {
2196        out.insert(
2197            crate::stats::SCHBENCH_LOOPS_PER_CPU_SEC.to_string(),
2198            p.loop_count as f64 / (p.worker_cpu_ns as f64 / 1e9),
2199        );
2200    }
2201}
2202
2203/// Write the per-phase taobench scalar metrics derived from ONE
2204/// `TaobenchStats` (a single cgroup's carrier, or the cross-cgroup pool)
2205/// into `out`, keyed by registry [`crate::stats::MetricDef`] name. The sole
2206/// producer of these keys for both the per-cgroup ([`PhaseCgroupStats::metrics`])
2207/// and pooled ([`PhaseBucket::metrics`]) maps. ABSENT discipline: the qps keys
2208/// only when client CPU time was measured (`cpu_time_ns > 0`); hit_ratio only
2209/// when ops completed (`total > 0`); hit_rate only when lookups were issued
2210/// (`get_cmds > 0`) — a not-measured value reads as missing, never a false 0.
2211///
2212/// DENOMINATION: the `taobench_*_ops_per_cpu_sec` keys are CPU-SECOND denominated —
2213/// completed ops per CPU-second the phase's client threads actually received
2214/// (Σ `CLOCK_THREAD_CPUTIME_ID`), not per wall second. Dilation-safe by
2215/// construction; the wall picture reconstructs as `rate / D` via the
2216/// sidecar's `host_dilation` (or exactly from the carrier's ops + wall
2217/// window).
2218fn write_taobench_scalars(
2219    p: &crate::workload::taobench::run::TaobenchPhaseStats,
2220    out: &mut std::collections::BTreeMap<String, f64>,
2221) {
2222    use crate::stats::{
2223        TAOBENCH_FAST_OPS_PER_CPU_SEC, TAOBENCH_HIT_RATE, TAOBENCH_HIT_RATIO,
2224        TAOBENCH_SLOW_OPS_PER_CPU_SEC, TAOBENCH_TOTAL_OPS_PER_CPU_SEC,
2225    };
2226    let c = &p.counters;
2227    let total = c.total_ops();
2228    if c.cpu_time_ns > 0 {
2229        let cpu_secs = c.cpu_time_ns as f64 / 1e9;
2230        out.insert(
2231            TAOBENCH_TOTAL_OPS_PER_CPU_SEC.to_string(),
2232            total as f64 / cpu_secs,
2233        );
2234        out.insert(
2235            TAOBENCH_FAST_OPS_PER_CPU_SEC.to_string(),
2236            c.fast_ops as f64 / cpu_secs,
2237        );
2238        out.insert(
2239            TAOBENCH_SLOW_OPS_PER_CPU_SEC.to_string(),
2240            c.slow_ops as f64 / cpu_secs,
2241        );
2242    }
2243    if total > 0 {
2244        out.insert(
2245            TAOBENCH_HIT_RATIO.to_string(),
2246            c.fast_ops as f64 / total as f64,
2247        );
2248    }
2249    if c.get_cmds > 0 {
2250        out.insert(
2251            TAOBENCH_HIT_RATE.to_string(),
2252            1.0 - (c.get_misses as f64 / c.get_cmds as f64),
2253        );
2254    }
2255    // Per-phase serve-latency percentiles (open-loop only): the µs distribution
2256    // pooled across this phase's cgroups, re-derived over the union histogram.
2257    // Absent when no serve samples (closed loop, or a stream with no completions).
2258    write_taobench_serve_scalars(&p.serve_lat, out);
2259}
2260
2261/// Write the per-phase taobench serve-latency percentile scalars from a pooled
2262/// `PlatStats` into `out` (registry keys), gated on `sample_count() > 0` so a
2263/// closed-loop / no-sample carrier emits nothing (absent, never a false 0).
2264fn write_taobench_serve_scalars(
2265    serve: &crate::workload::schbench::plat::PlatStats,
2266    out: &mut std::collections::BTreeMap<String, f64>,
2267) {
2268    use crate::stats::{
2269        TAOBENCH_SERVE_MAX_US, TAOBENCH_SERVE_MIN_US, TAOBENCH_SERVE_P50_US, TAOBENCH_SERVE_P90_US,
2270        TAOBENCH_SERVE_P99_US, TAOBENCH_SERVE_P999_US,
2271    };
2272    use crate::workload::schbench::plat::Pct;
2273    if serve.sample_count() == 0 {
2274        return;
2275    }
2276    let q = serve.percentiles();
2277    out.insert(
2278        TAOBENCH_SERVE_P50_US.to_string(),
2279        q.value_at(Pct::P50) as f64,
2280    );
2281    out.insert(
2282        TAOBENCH_SERVE_P90_US.to_string(),
2283        q.value_at(Pct::P90) as f64,
2284    );
2285    out.insert(
2286        TAOBENCH_SERVE_P99_US.to_string(),
2287        q.value_at(Pct::P99) as f64,
2288    );
2289    out.insert(
2290        TAOBENCH_SERVE_P999_US.to_string(),
2291        q.value_at(Pct::P999) as f64,
2292    );
2293    out.insert(TAOBENCH_SERVE_MIN_US.to_string(), q.min as f64);
2294    out.insert(TAOBENCH_SERVE_MAX_US.to_string(), q.max as f64);
2295}