ktstr/assert/run_metrics.rs
1use super::*;
2
3/// Aggregated statistics across all cgroups in a scenario.
4#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize, crate::Claim)]
5pub struct ScenarioStats {
6 /// Per-cgroup stats, one entry per cgroup.
7 pub cgroups: Vec<CgroupStats>,
8 /// Sum of workers across all cgroups.
9 pub total_workers: usize,
10 /// Sum of per-cgroup distinct CPU counts (not deduplicated across cgroups).
11 pub total_cpus: usize,
12 /// Sum of migration counts across all cgroups.
13 pub total_migrations: u64,
14 /// Worst spread across any cgroup (highest).
15 pub worst_spread: f64,
16 /// Worst gap across any cgroup (highest, ms). Paired with
17 /// `worst_gap_cpu` — both come from the same cgroup.
18 pub worst_gap_ms: u64,
19 /// CPU where the worst gap occurred across all cgroups. Paired
20 /// with `worst_gap_ms` — both come from the same cgroup.
21 pub worst_gap_cpu: usize,
22 /// Worst migration ratio across any cgroup (highest).
23 pub worst_migration_ratio: f64,
24 /// Sum of iteration counts across all cgroups.
25 pub total_iterations: u64,
26 // worst_page_locality is NO LONGER a typed field: it is
27 // `crate::stats::MetricKind::WorstLowest` (NumaLocal/NumaTotal), re-pooled
28 // None-aware post-merge by `populate_run_distribution_metrics` from the
29 // per-phase NUMA carriers (the lowest per-cgroup page_locality, a measured
30 // 0.0 winning) — the `worst_page_locality` method shared with
31 // `run_metric`. The deleted typed field folded via `fold_lowest_nonzero`,
32 // which SKIPPED a measured 0.0 and reported a better-than-worst cross-run
33 // value; the ext re-pool fixes both the field and the read surfaces at once.
34 // worst_cross_node_migration_ratio is NO LONGER a typed field: it is
35 // `crate::stats::MetricKind::WorstCrossNodeRatio`, re-pooled post-merge by
36 // `populate_run_distribution_metrics` from the per-phase NUMA carriers (the
37 // MAX per-cgroup cross-node churn ratio over the latest residency total) via
38 // the `worst_cross_node_migration_ratio` method shared with `run_metric`. The
39 // deleted typed `Gauge(Last)` field/GauntletRow column was merge-max-folded
40 // within-run but cross-run averaged each run's value over `passes_observed`
41 // (folding a NUMA-less passing run's 0.0 sentinel into the mean), AND
42 // diverged from `run_metric` (which already re-derived from the per-phase
43 // carriers) — so the sidecar and the in-test read gave different values on
44 // multi-phase runs. The ext re-pool writes no key for a never-measured run,
45 // so the MEAN divides only by runs that measured NUMA. A CHURN ratio
46 // (cumulative migration EVENTS / final-snapshot resident pages), so it can
47 // exceed 1.0; NOT a bounded `[0,1]` fraction.
48 // worst_wake_latency_tail_ratio is NO LONGER a typed field: it is
49 // `crate::stats::MetricKind::WakeLatencyTailRatio`, re-selected into
50 // `ext_metrics` post-merge by `populate_run_distribution_metrics` (max
51 // over the per-cgroup `CgroupStats::wake_latency_tail_ratio` values,
52 // floor-gated below WAKE_LATENCY_TAIL_RATIO_MIN_ITERATIONS); `MetricDef::read`
53 // surfaces it via the ext fallback.
54 /// Extensible metrics for the generic comparison pipeline.
55 /// Populated from per-cgroup ext_metrics (worst value across cgroups).
56 pub ext_metrics: BTreeMap<String, f64>,
57 /// Per-phase metric buckets in step-index order. A scenario
58 /// with N Steps populates `N + 1` entries: phase 0 is the
59 /// BASELINE settle window before Step 0 fires, phases
60 /// 1..=N align with Step 0..Step N-1 in scenario order
61 /// (1-indexed Steps so the BASELINE encoding doesn't collide
62 /// with first-Step's index).
63 ///
64 /// Empty when the scenario produced no periodic captures
65 /// (Default::default() yields `vec![]`). The existing
66 /// flat-bucket scalars on this struct are independent of the
67 /// per-phase view — they remain the "all phases merged"
68 /// reading, unchanged in semantics by the introduction of
69 /// `phases`.
70 ///
71 /// **Auto-populated by the framework**: scenarios that fire
72 /// periodic captures (via
73 /// [`crate::test_support::KtstrTestEntry::num_snapshots`] or
74 /// [`crate::scenario::ops::Op::CaptureSnapshot`]) have this
75 /// field populated automatically inside
76 /// `crate::test_support::eval`'s `evaluate_vm_result` —
77 /// test code never needs to call
78 /// [`crate::assert::build_phase_buckets`] manually. The auto-
79 /// populate path drains the snapshot bridge from the
80 /// [`crate::vmm::VmResult`] returned by the framework and folds
81 /// the per-sample readings through
82 /// `crate::stats::aggregate_samples_for_phase` per metric.
83 /// Single-phase scenarios that fire no captures leave this
84 /// `vec![]`; the flat-bucket scalars on this struct cover the
85 /// single-phase case.
86 ///
87 /// See [`PhaseBucket`] for the per-phase shape.
88 #[serde(default)]
89 pub phases: Vec<PhaseBucket>,
90}
91
92impl ScenarioStats {
93 /// Look up the phase bucket for a [`Phase`] — `Phase::BASELINE` for the
94 /// pre-first-Step settle window, `Phase::step(k)` for the test author's
95 /// 0-indexed Step k. The typed `Phase` keeps the 1-indexed encoding (Step 0
96 /// lives at the underlying `step_index = 1`) invisible at the call site.
97 ///
98 /// Returns `None` when no bucket with that phase exists (single-phase
99 /// scenario, the scenario didn't reach that Step, or a phase past the last).
100 pub fn phase(&self, phase: Phase) -> Option<&PhaseBucket> {
101 self.phases.iter().find(|p| p.step_index == phase.as_u16())
102 }
103
104 /// Shortcut: look up a single metric value in a specific
105 /// phase by phase-index. Returns `None` when:
106 /// (a) the phase is absent (no bucket with `step_index` in
107 /// [`Self::phases`]),
108 /// (b) the phase exists but had no finite samples for that
109 /// metric, OR
110 /// (c) `metric` is a dynamic [`crate::stats::MetricId`] key (a
111 /// scheduler-runtime / payload string) not present in this phase's
112 /// stores. A built-in [`crate::stats::BuiltinMetric`] cannot be a
113 /// typo — that is a compile error — so this case needs a dynamic key.
114 ///
115 /// Two stores are checked: [`PhaseBucket::metrics`] (via
116 /// [`PhaseBucket::get`]) and, failing that, the cross-cgroup phase
117 /// SUM of a per-cgroup Counter ([`PhaseBucket::cgroup_counter_total`])
118 /// for `"total_migrations"` / `"total_iterations"` / `"total_cpu_time_ns"` —
119 /// registered `Counter`s with no per-sample source, so they live ONLY in the
120 /// per-cgroup carriers; without this fallback the pooled lookup
121 /// returned a silent `None` while the per-cgroup
122 /// [`Self::phase_cgroup_metric`] surfaced the value. Symmetric with
123 /// [`crate::vmm::VmResult::phase_metric`].
124 ///
125 /// Sentinel-free: `Some(0.0)` means the reducer produced a
126 /// real zero from finite samples, NOT "missing data". See
127 /// [`PhaseBucket::metrics`] for the registry source. When
128 /// debugging an unexpected `None` on a dynamic key, check
129 /// [`crate::stats::MetricId::def`]`().is_some()` to tell an unregistered
130 /// key from absent data (built-in ids always resolve).
131 ///
132 /// Pass `Phase::BASELINE` for the settle window or `Phase::step(k)` for the
133 /// test author's 0-indexed Step k — the typed `Phase` hides the 1-indexed
134 /// encoding (see [`Self::phase`]).
135 pub fn phase_metric(
136 &self,
137 phase: Phase,
138 metric: impl Into<crate::stats::MetricId>,
139 ) -> Option<f64> {
140 let metric = metric.into();
141 self.phase(phase).and_then(|p| {
142 p.get(metric.as_str())
143 .or_else(|| p.cgroup_counter_total(metric.as_str()))
144 })
145 }
146
147 /// Cross-cgroup balance: the ratio of the busiest cell's per-worker
148 /// throughput to the quietest's — `max / min` over each cgroup's
149 /// [`CgroupStats::iterations_per_worker`]. The bread-and-butter
150 /// scheduler-fairness assertion (every balance test hand-rolls this
151 /// `max/min` over `self.cgroups` today).
152 ///
153 /// No-worker cgroups (`iterations_per_worker() == None`) are SKIPPED: a
154 /// 0-worker cell is a config condition, not a balance signal. Returns
155 /// `None` when fewer than two cgroups have workers (a ratio needs two);
156 /// check the cgroup count separately if every declared cell must have
157 /// workers. A cell that ran workers but completed zero iterations
158 /// (measured `Some(0.0)`) drives the ratio to `f64::INFINITY` so
159 /// starvation SURFACES rather than vanishing — matching the
160 /// `None`-vs-`Some(0.0)` discipline of
161 /// [`CgroupStats::iterations_per_worker`]. For an explicit starvation
162 /// gate, check `min > 0` over the same cgroups separately.
163 ///
164 /// Whole-run aggregate: this reads `self.cgroups`, which sums over all
165 /// phases. For a single phase's per-cgroup balance in a multi-phase
166 /// scenario, read each cgroup's per-phase throughput (`total_iterations`
167 /// / `num_workers`) from [`crate::vmm::VmResult::phase_cgroup`] — the
168 /// per-phase per-cgroup carriers now landed in
169 /// [`PhaseBucket::per_cgroup`].
170 pub fn cgroup_balance_ratio(&self) -> Option<f64> {
171 let mut min = f64::INFINITY;
172 let mut max = 0.0_f64;
173 let mut n = 0usize;
174 for cg in &self.cgroups {
175 if let Some(rate) = cg.iterations_per_worker() {
176 min = min.min(rate);
177 max = max.max(rate);
178 n += 1;
179 }
180 }
181 if n < 2 {
182 return None;
183 }
184 if min == 0.0 {
185 // A with-worker cell did zero work: starvation. Surface it as an
186 // infinite ratio rather than a NaN (0/0) or a hidden None.
187 return Some(f64::INFINITY);
188 }
189 Some(max / min)
190 }
191
192 /// Per-cgroup analog of [`Self::phase_metric`]: look up `metric` for a named
193 /// `cgroup` in a [`Phase`] (`Phase::BASELINE` / `Phase::step(k)`), via that
194 /// cgroup's per-phase carrier ([`PhaseCgroupStats::get`]), falling back to
195 /// [`PhaseCgroupStats::cgroup_counter`] for the per-cgroup Counters
196 /// `total_migrations`/`total_iterations`/`total_cpu_time_ns`. `None` when the
197 /// phase has no bucket, no carrier for `cgroup`, the carrier carried no finite
198 /// value for the metric, OR the metric is an unregistered dynamic key (a
199 /// built-in id is typo-proof; an unregistered [`crate::stats::MetricId`] has no
200 /// [`crate::stats::MetricId::def`], same as [`Self::phase_metric`]). The
201 /// N-cgroups-to-N-queryable-sets surface on the AssertResult-holding path (the
202 /// in-VM `post_vm` path uses [`crate::vmm::VmResult::phase_cgroup_metric`]).
203 pub fn phase_cgroup_metric(
204 &self,
205 phase: Phase,
206 cgroup: &str,
207 metric: impl Into<crate::stats::MetricId>,
208 ) -> Option<f64> {
209 let metric = metric.into();
210 self.phase(phase)
211 .and_then(|p| p.per_cgroup.get(cgroup))
212 .and_then(|pc| {
213 pc.get(metric.as_str())
214 .or_else(|| pc.cgroup_counter(metric.as_str()))
215 })
216 }
217
218 /// True iff the scenario produced at least one Step-phase
219 /// bucket (any phase with `step_index >= 1`). False when
220 /// `phases` is empty OR contains only `BASELINE` (the
221 /// pre-first-Step settle window).
222 ///
223 /// Use this to fail a phase-aware assertion BEFORE calling
224 /// [`Self::phase`] with a `Phase::step(k)` on a scenario that
225 /// silently never advanced past BASELINE: a test that declared
226 /// no `Step`s, OR a scenario that bailed in setup before any
227 /// `Step` ran, would otherwise see [`Self::phase`] return
228 /// `None` for every Step and the test would either panic on
229 /// `.expect(...)` or pass vacuously.
230 ///
231 /// ```ignore
232 /// anyhow::ensure!(
233 /// r.stats.has_steps(),
234 /// "scenario produced no Step-phase buckets — \
235 /// declare a Step or read Phase::BASELINE",
236 /// );
237 /// let throughput = r.stats.phase_metric(Phase::step(0), "throughput");
238 /// ```
239 pub fn has_steps(&self) -> bool {
240 self.phases.iter().any(|p| p.step_index >= 1)
241 }
242
243 /// Run-level value for a metric by registry name, for the
244 /// ext-sourced metric family that carries no typed
245 /// `ScenarioStats` field.
246 ///
247 /// Resolves [`Self::ext_metrics`] — the run-level map the
248 /// framework fills post-merge with every metric whose value has no
249 /// typed struct field: the pooled wake-latency / run-delay
250 /// distributions and worst-cgroup iteration efficiencies
251 /// (the `MetricKind::Distribution` / `MetricKind::WorstLowest`
252 /// registry kinds — `worst_p99_wake_latency_us`, `worst_run_delay_us`,
253 /// `worst_iterations_per_cpu_sec`, …), the canonical pooled
254 /// `iteration_rate`,
255 /// the per-thread-group `system_time_ns` / `user_time_ns`, and
256 /// `avg_imbalance_ratio` / `avg_dsq_depth`. This is the
257 /// run-level analogue of [`Self::phase_metric`] for that family:
258 /// code holding the run's [`AssertResult`] reads
259 /// `r.stats.run_metric("worst_run_delay_us")` instead of reaching
260 /// into the raw `ext_metrics` map by string key (`ScenarioStats` is
261 /// the [`AssertResult::stats`] field — the value a test body, or a
262 /// callback that builds an `AssertResult` via `collect_all` /
263 /// `execute_scenario`, holds). A `post_vm` callback instead receives
264 /// a `VmResult`, which has NO `stats` field and no run-level
265 /// Distribution surface — compare those cross-run via `cargo ktstr
266 /// perf-delta`.
267 ///
268 /// The ext family is populated only by the `#[ktstr_test]` eval
269 /// flow's post-merge producer
270 /// ([`populate_run_distribution_metrics`]). An `AssertResult` built
271 /// by a DIRECT host assertion (`assert_not_stuck` /
272 /// `AssertPlan::assert_cgroup`, which never run that producer)
273 /// carries the per-cgroup values on [`Self::cgroups`] but none of
274 /// these run-level roll-ups, so `run_metric` returns `None` for them
275 /// on that path — read the per-cgroup `CgroupStats` field directly
276 /// (e.g. `r.stats.cgroups[i].p99_wake_latency_us`) there.
277 ///
278 /// Sentinel-free, matching [`Self::phase_metric`]: `None` means
279 /// the metric is absent from this run (no contributing cgroup or
280 /// carrier, or a name not present in the map); `Some(0.0)` is a
281 /// real measured zero. Check [`crate::stats::MetricId::def`] on a dynamic
282 /// key to tell an unregistered key from genuinely-absent data (built-in
283 /// ids always resolve). (The map also carries any
284 /// user-defined extensible-metric keys, plus the framework-internal
285 /// Rate-component Counters — `total_iterations_pooled` /
286 /// `total_cpu_time_sec`, the numerator/denominator plumbing behind
287 /// `iteration_rate` — both of which resolve
288 /// here too; prefer the derived rate over its raw components.)
289 ///
290 /// RESOLVED here None-aware, ahead of the ext lookup, via a typed dispatch
291 /// (`typed_sentinel_metric`): the cross-cgroup metrics `worst_spread`,
292 /// `worst_migration_ratio`, `worst_gap_ms`, `total_migrations`,
293 /// `total_iterations`, `worst_page_locality`,
294 /// `worst_cross_node_migration_ratio`. The dispatch RE-DERIVES each from the
295 /// per-cgroup (`self.cgroups`) / per-phase (`self.phases[].per_cgroup`, the
296 /// NUMA metrics) carriers — `None` when no carrier measured it, `Some(0.0)`
297 /// for a measured zero — preserving the sentinel-free contract. The 5
298 /// non-NUMA metrics are 0.0-sentinel typed struct fields (a not-measured
299 /// carrier coerces to 0.0, indistinguishable from a measured zero, so the
300 /// re-derivation recovers the distinction); the two NUMA roll-ups
301 /// (`worst_page_locality`, `worst_cross_node_migration_ratio`) have no struct
302 /// field and re-pool purely from the per-phase NUMA carriers.
303 /// (`worst_wake_latency_tail_ratio` resolves via the ext lookup as the
304 /// `WakeLatencyTailRatio` key.)
305 ///
306 /// NOT resolved here (not in `ext_metrics`, no typed dispatch):
307 /// - the monitor-sourced run-level metrics (`max_imbalance_ratio`,
308 /// `max_dsq_depth`, `stuck_count`, `total_fallback`,
309 /// `total_keep_last`), which `ScenarioStats` does not hold
310 /// run-level — read those per-phase via [`Self::phase_metric`].
311 ///
312 /// So this resolves the ext-sourced family AND the typed cross-cgroup fields
313 /// (via the dispatch); only the monitor-sourced run-level metrics remain
314 /// unresolved here (`ScenarioStats` holds them only per-phase).
315 pub fn run_metric(&self, metric: impl Into<crate::stats::MetricId>) -> Option<f64> {
316 let id = metric.into();
317 // The sentinel-prone cross-cgroup run-level metrics re-derive None-aware
318 // from the per-cgroup / per-phase carriers, recovering the never-measured
319 // vs measured-zero distinction the 0.0-coerced struct fields (or, for
320 // worst_page_locality, the absent field) cannot carry. A non-sentinel id
321 // returns `None` from the dispatch and falls through to the ext-metrics map.
322 if let crate::stats::MetricId::Builtin(b) = &id
323 && let Some(resolved) = self.typed_sentinel_metric(*b)
324 {
325 return resolved;
326 }
327 self.ext_metrics.get(id.as_str()).copied()
328 }
329
330 /// Re-derive a typed run-level metric None-aware from the per-cgroup /
331 /// per-phase carriers — recovering the never-measured (`None`) vs
332 /// measured-zero (`Some(0.0)`) distinction a bare 0.0-sentinel `f64` cannot
333 /// carry. The 0.0-sentinel typed struct fields (`worst_spread`,
334 /// `worst_migration_ratio`, ...) coerce a not-measured carrier to 0.0 in
335 /// `scenario_stats_for_cgroup` (indistinguishable from a measured zero);
336 /// `worst_page_locality` is no longer a struct field at all — it re-pools
337 /// purely from the per-phase NUMA carriers (see below).
338 ///
339 /// Returns `None` when `m` is NOT one of the typed sentinel ids — the caller
340 /// then falls through to the ext-metrics lookup. `Some(None)` when `m` IS a
341 /// sentinel id but no contributing carrier exists (loud-absent). `Some(Some(v))`
342 /// for the re-derived value (a measured zero stays `Some(0.0)`).
343 ///
344 /// The 5 non-NUMA metrics source from `self.cgroups`, whose `Option` / counter
345 /// carriers preserve the measured-vs-unmeasured state. The 2 NUMA roll-ups
346 /// (`worst_page_locality`, `worst_cross_node_migration_ratio` — neither a
347 /// struct field) source from `self.phases[].per_cgroup`, for different
348 /// reasons: `page_locality` is structurally 0.0 on the `cgroup_stats` path
349 /// (it needs an expected-node set the reports-only builder lacks), so
350 /// `self.cgroups` cannot source it; `cross_node_migration_ratio` IS populated
351 /// on `cgroup_stats`, but only as a single pre-folded scalar that cannot carry
352 /// the cross-phase fold (SUM the per-phase migration-counter deltas over the
353 /// LATEST residency total), so it too re-pools from the per-phase
354 /// `PhaseCgroupStats` NUMA counters — which also single-sources it with the
355 /// ext/sidecar value. On a phase-less direct-assertion `AssertResult` (no NUMA
356 /// capture) both NUMA roll-ups read `None` (correct loud-absent).
357 fn typed_sentinel_metric(&self, m: crate::stats::BuiltinMetric) -> Option<Option<f64>> {
358 use crate::stats::BuiltinMetric as B;
359 Some(match m {
360 // worst_spread: highest spread over cgroups that measured it
361 // (CgroupStats::spread is already Option — None when no worker had
362 // measurable wall time). None iff no cgroup measured spread.
363 B::WorstSpread => self
364 .cgroups
365 .iter()
366 .filter_map(|c| c.spread)
367 .reduce(f64::max),
368 // worst_migration_ratio: highest migration ratio over cgroups that
369 // ran iterations (a 0.0 with total_iterations>0 is measured; with
370 // ==0 it is the rate-over-zero sentinel — never-measured).
371 B::WorstMigrationRatio => self
372 .cgroups
373 .iter()
374 .filter(|c| c.total_iterations > 0)
375 .map(|c| c.migration_ratio)
376 .reduce(f64::max),
377 // total_migrations / total_iterations: cross-cgroup SUMs. 0 is a real
378 // measured sum (cgroups ran, zero events); never-measured = no cgroups.
379 B::TotalMigrations => (!self.cgroups.is_empty()).then(|| {
380 self.cgroups
381 .iter()
382 .map(|c| c.total_migrations)
383 .fold(0u64, u64::saturating_add) as f64
384 }),
385 B::TotalIterations => (!self.cgroups.is_empty()).then(|| {
386 self.cgroups
387 .iter()
388 .map(|c| c.total_iterations)
389 .fold(0u64, u64::saturating_add) as f64
390 }),
391 // worst_gap_ms: longest gap over cgroups with workers (gap 0 with
392 // workers = measured "no gap observed"; never-measured = no workers).
393 B::WorstGapMs => self
394 .cgroups
395 .iter()
396 .filter(|c| c.num_workers > 0)
397 .map(|c| c.max_gap_ms as f64)
398 .reduce(f64::max),
399 // worst_page_locality: LOWEST per-cgroup locality (the worst cell) over
400 // cgroups that measured NUMA residency — a measured 0.0 (all pages
401 // off-node) is the worst and WINS the lowest fold. Shared with the
402 // ext/sidecar re-pool via worst_page_locality() so the typed read and
403 // the cross-run comparison value agree (the same fix replaces the
404 // fold_lowest_nonzero sentinel-skip on BOTH surfaces).
405 B::WorstPageLocality => self.worst_page_locality(),
406 // worst_cross_node_migration_ratio: highest cross-node churn ratio over
407 // cgroups that measured NUMA. Shared with the ext/sidecar re-pool via
408 // worst_cross_node_migration_ratio() so the typed read and the cross-run
409 // comparison value agree (the divergence the typed field had).
410 B::WorstCrossNodeMigrationRatio => self.worst_cross_node_migration_ratio(),
411 _ => return None,
412 })
413 }
414
415 /// Aggregate each cgroup's NUMA carriers across `self.phases` into
416 /// `(latest_local, latest_total, summed_migrated)`. The cross-phase fold is
417 /// ASYMMETRIC by counter class: `numa_pages_local` / `numa_pages_total` are
418 /// current-residency GAUGE snapshots (the live `/proc/.../numa_maps`
419 /// residency, recomputed each read), so the LATEST MEASURED phase's value is the run-end placement
420 /// — SUMMING across phases would multiply residency by the phase count (a
421 /// silent over-count); `cross_node_migrated` is a per-phase migration-counter
422 /// DELTA over disjoint intervals, so it SUMS to the run total. The shared
423 /// `latest_total` denominator serves both ratios (page_locality and
424 /// cross_node share one page total), matching `cgroup_stats`, which reads the
425 /// final-snapshot residency. One entry per cgroup that appeared in any phase.
426 ///
427 /// LATEST is None-aware: a later phase carrier that measured NO NUMA pages
428 /// (`numa_pages_total == 0` — an empty backdrop slice, a non-NUMA `WorkType`
429 /// phase, or a zero-worker carrier) does NOT overwrite an earlier MEASURED
430 /// residency. A bare overwrite would zero out the earlier snapshot, and the
431 /// `total > 0` filter in `worst_page_locality()` would then drop the cgroup
432 /// from the worst-pool — silently reporting a better-than-worst run-level
433 /// locality (the same measured-vs-unmeasured discipline the cross-cgroup fold
434 /// applies, extended to the cross-phase axis).
435 fn numa_agg_per_cgroup(&self) -> Vec<(u64, u64, u64)> {
436 let mut by_cg: std::collections::BTreeMap<&str, (u64, u64, u64)> =
437 std::collections::BTreeMap::new();
438 // self.phases is in step (chronological) order, so a later MEASURED
439 // phase's residency snapshot overwrites the earlier (LATEST wins); a
440 // not-measured (total == 0) phase leaves the prior snapshot intact. The
441 // migration deltas accumulate unconditionally (a 0 delta adds nothing).
442 for phase in &self.phases {
443 for (name, pc) in &phase.per_cgroup {
444 let e = by_cg.entry(name.as_str()).or_insert((0, 0, 0));
445 if pc.numa_pages_total > 0 {
446 e.0 = pc.numa_pages_local;
447 e.1 = pc.numa_pages_total;
448 }
449 // saturating: guest-runtime migration counter pooled across
450 // phases; never wrap the derived cross-node-migration ratio.
451 e.2 = e.2.saturating_add(pc.cross_node_migrated);
452 }
453 }
454 by_cg.into_values().collect()
455 }
456
457 /// The run-level WORST (lowest) per-cgroup page-locality fraction, re-pooled
458 /// None-aware from the per-phase NUMA carriers: the lowest
459 /// `page_locality_of(latest_local, latest_total)` over cgroups that measured
460 /// NUMA residency (`numa_pages_total > 0`) — a measured 0.0 (all pages
461 /// off-node) WINS the lowest; an all-unmeasured cohort yields `None`. Shared
462 /// by `run_metric`'s `WorstPageLocality` dispatch AND
463 /// `populate_run_distribution_metrics`'s ext/sidecar re-pool so the typed
464 /// read and the cross-run comparison value are byte-identical.
465 fn worst_page_locality(&self) -> Option<f64> {
466 self.numa_agg_per_cgroup()
467 .into_iter()
468 .filter(|&(_, total, _)| total > 0)
469 .map(|(local, total, _)| super::reductions::page_locality_of(local, total))
470 .reduce(f64::min)
471 }
472
473 /// The run-level WORST (highest) per-cgroup cross-node migration-churn ratio,
474 /// re-pooled from the per-phase NUMA carriers: the MAX
475 /// `cross_node_migration_ratio_of(summed_migrated, latest_total)` over cgroups
476 /// that measured NUMA residency (`numa_pages_total > 0`) — an all-unmeasured
477 /// cohort yields `None`. The polarity twin of `worst_page_locality` (LowerBetter
478 /// → highest-wins). Shared by `run_metric`'s `WorstCrossNodeMigrationRatio`
479 /// dispatch AND `populate_run_distribution_metrics`'s ext/sidecar re-pool so the
480 /// typed read and the cross-run comparison value are byte-identical (the typed
481 /// `Gauge(Last)` field diverged from this on multi-phase runs).
482 fn worst_cross_node_migration_ratio(&self) -> Option<f64> {
483 self.numa_agg_per_cgroup()
484 .into_iter()
485 .filter(|&(_, total, _)| total > 0)
486 .map(|(_, total, migrated)| {
487 super::reductions::cross_node_migration_ratio_of(migrated, total)
488 })
489 .reduce(f64::max)
490 }
491}
492
493/// Registry metric names that already have a typed `GauntletRow` field — the
494/// typed accessor populates them at `sidecar_to_row` time and
495/// `MetricDef::read` prefers the accessor over `ext_metrics`, so writing the
496/// same key into `ext_metrics` would create unread sidecar bloat AND
497/// double-source the run-level value. For `stuck_count` the typed whole-run
498/// count (`MonitorSummary::stuck_count`, windowed over the full sample
499/// stream) is authoritative; the per-phase fold sum shares the
500/// `is_cpu_stuck` predicate but is a lower-or-equal (`<=`),
501/// partition-dependent quantity (it drops cross-boundary + out-of-phase
502/// windows, so it falls strictly below once any of those is stuck), so
503/// injecting the ext copy would shadow the authoritative typed value with
504/// a redundant — and, once a dropped window is stuck, divergent —
505/// number. Both run-level ext-metrics populators consult this — the
506/// SampleSeries
507/// path ([`populate_run_ext_metrics`]) and the phase-fold path
508/// ([`populate_run_ext_metrics_from_phases`]) — so only ext-metrics-only
509/// registry entries are written and a typed-backed metric's run-level value
510/// always comes from its accessor. `max_imbalance_ratio` is included because
511/// its accessor reads the typed `GauntletRow.imbalance_ratio` (whole-run
512/// MonitorSummary); its per-phase monitor fold feeds rendering only.
513const TYPED_FIELD_NAMES: &[&str] = &[
514 "max_dsq_depth",
515 "max_imbalance_ratio",
516 "total_fallback",
517 "total_keep_last",
518 "stuck_count",
519 "total_iterations",
520 "total_migrations",
521];
522
523/// Sibling of [`populate_run_ext_metrics`] that mines per-phase
524/// metrics back into the run-level `ext_metrics` map. Closes the
525/// gap for registered metrics whose values live in
526/// `PhaseBucket.metrics` but never reach `ext_metrics` via the
527/// SampleSeries path (their `read_sample` returns `None`):
528/// `avg_imbalance_ratio` (sourced from MonitorSample windowing
529/// inside [`build_phase_buckets`]), `iteration_rate` (sourced from matching
530/// guest per-cgroup iteration and CPU-time carriers),
531/// and `system_time_ns` / `user_time_ns` (per-thread-group CPU-time
532/// deltas injected by `phase_group_cpu_delta` inside
533/// `buckets_from_grouped`). The fold is generic over every key
534/// present on any phase, so it carries any such phase-only metric (the
535/// ext-metrics-only set whose `read_sample` returns `None`). Keys with a
536/// typed `GauntletRow` field (`TYPED_FIELD_NAMES`) are SKIPPED: their
537/// run-level value comes from the typed accessor (which wins on read), so
538/// re-injecting them here would double-source the run aggregate — the
539/// hazard the const's doc describes. Their per-phase `PhaseBucket` value
540/// still feeds per-phase rendering.
541///
542/// Per-phase reduction dispatch is described on [`PhaseBucket`];
543/// the cross-phase fold here uses `sample_count` as the weight so
544/// Gauge(Avg) keys get the weighted mean (the correct cross-phase
545/// semantic for typical-load metrics) while other kinds fold per
546/// their natural reduction. Existing keys in `target` are not
547/// overwritten — `read_sample` path values win when both produced
548/// an entry.
549///
550/// Without this fill, `cargo ktstr perf-delta` silently misses
551/// these phase-only metrics (avg_imbalance_ratio, iteration_rate,
552/// system_time_ns, user_time_ns) in flat-row output because
553/// `MetricDef::read` falls back to ext_metrics and finds nothing.
554pub fn populate_run_ext_metrics_from_phases(
555 phases: &[PhaseBucket],
556 target: &mut std::collections::BTreeMap<String, f64>,
557) {
558 // No early-return on empty `phases`: the derive_rate_metrics post-pass
559 // below must still run over whatever components populate_run_ext_metrics
560 // already inserted into `target` (the empty-phases case), so a run-level
561 // Rate is re-derived rather than silently dropped. The loops below are
562 // no-ops when `phases` is empty.
563 // Collect every metric key that appears on any phase.
564 let mut keys: std::collections::BTreeSet<&String> = std::collections::BTreeSet::new();
565 for phase in phases {
566 for key in phase.metrics.keys() {
567 keys.insert(key);
568 }
569 }
570 for key in keys {
571 if target.contains_key(key) {
572 continue;
573 }
574 let Some(def) = crate::stats::metric_def(key) else {
575 continue;
576 };
577 // Derived metrics (every `is_derived()`: Rate / Distribution / WorstLowest /
578 // WakeLatencyTailRatio / WorstCrossNodeRatio / PerPhase / PerRunDistribution)
579 // are produced
580 // from their pooled components, not folded as per-phase values: skip
581 // here. A Rate re-derives after the loop (Σnum/Σdenom over the folded
582 // components); the distributional kinds (Distribution / WorstLowest /
583 // WakeLatencyTailRatio / WorstCrossNodeRatio) are re-pooled run-level by
584 // `populate_run_distribution_metrics` (and never appear in
585 // phase.metrics anyway); PerPhase is re-derived by `derive_phase_metrics`.
586 // Folding a ready-made derived value would lose
587 // the re-pool, and routing one into aggregate_samples_weighted within
588 // a run is not its producer path.
589 if def.kind.is_derived() {
590 continue;
591 }
592 // Typed-backed keys (those in TYPED_FIELD_NAMES — a typed GauntletRow
593 // accessor that wins on read) must NOT be re-injected into ext_metrics
594 // from the phase fold: the ext copy would be unread bloat and, for
595 // stuck_count (whose per-phase fold sum is `<=` the typed whole-run
596 // count, strictly below once a cross-boundary/out-of-phase window is
597 // stuck — they share the is_cpu_stuck predicate but the run-level
598 // count windows the full stream), a redundant-or-divergent value,
599 // not a guaranteed duplicate. Their per-phase
600 // PhaseBucket value still feeds rendering; the run-level value stays
601 // the typed path. Mirrors the sibling populate_run_ext_metrics.
602 // (Without this, folding max_imbalance_ratio + stuck_count onto
603 // captured buckets would leak both into ext_metrics on the common
604 // path.)
605 if TYPED_FIELD_NAMES.contains(&key.as_str()) {
606 continue;
607 }
608 // avg_nr_running is NOT typed-backed (no GauntletRow accessor), but its
609 // authoritative run-level value is MonitorSummary::avg_nr_running, folded
610 // by fold_run_level_ext. It is the one monitor-summary-fold key that ALSO
611 // appears in per-phase bucket.metrics (fold_monitor_into_bucket writes it
612 // for rendering). Skip it here so the per-phase re-pool never claims the
613 // run-level key: VmResult::run_metric runs this re-pool BEFORE
614 // fold_run_level_ext, and the fold's `or_insert` would then no-op,
615 // silently replacing the whole-run value with the per-phase weighted
616 // mean. Its per-phase PhaseBucket value still feeds rendering +
617 // change-detection. (TYPED_FIELD_NAMES is the typed-accessor analogue of
618 // this same skip rationale.)
619 if key == "avg_nr_running" {
620 continue;
621 }
622 // Per-phase (value, sample_count) for the kind-aware fold.
623 // A phase that doesn't carry the key contributes nothing.
624 // Lock-step shape enforced by the (f64, usize) pair type.
625 // `sample_count.max(1)` is load-bearing for Gauge(Avg) keys: a
626 // synthesized zero-capture phase carrying a
627 // capture-independent Gauge(Avg) value at sample_count==0 gets
628 // weight 1 (one phase observation) rather than being zero-weighted
629 // out of the run-level mean. The floor is a no-op for
630 // Counter/DeltaSum keys, which sum with weights ignored (see
631 // aggregate_finite): iteration_rate's guest-carrier components
632 // total_iterations_pooled / total_cpu_time_sec are such Counters, so
633 // a zero-capture step's work is INCLUDED in the re-pooled rate
634 // (iteration_rate itself is a Rate, skipped above and re-derived
635 // below). A regression dropping
636 // the floor would silently re-drop a zero-capture step's Gauge(Avg)
637 // value from the sidecar aggregate.
638 let pairs: Vec<(f64, usize)> = phases
639 .iter()
640 .filter_map(|phase| {
641 phase
642 .metrics
643 .get(key)
644 .copied()
645 .map(|v| (v, phase.sample_count.max(1)))
646 })
647 .collect();
648 if pairs.is_empty() {
649 continue;
650 }
651 // PerPhaseDeltaSum folds cross-PHASE by SUM: the run-level value is the
652 // sum of the disjoint per-phase OBSERVED CPU-time deltas (a lower bound
653 // on total run CPU time — excludes head / tail / inter-phase-gap
654 // windows; see the MetricKind::PerPhaseDeltaSum doc). It is NOT routed
655 // through aggregate_samples_weighted, whose PerPhaseDeltaSum arm
656 // implements the CROSS-RUN unweighted mean — mean-folding the phases
657 // here would double-count duration (the bug this kind fixes). Weights
658 // (phase sample_count) are irrelevant to a sum.
659 if def.kind == crate::stats::MetricKind::PerPhaseDeltaSum {
660 let sum: f64 = pairs.iter().map(|(v, _)| v).sum();
661 target.insert(key.clone(), sum);
662 continue;
663 }
664 if let Some(reduced) = crate::stats::aggregate_samples_weighted(&pairs, def.kind) {
665 target.insert(key.clone(), reduced);
666 }
667 }
668 // Re-derive Rate metrics from the now-folded components so the run
669 // rate is Σnumerator / Σdenominator (the components folded by their
670 // own kinds above — a Counter numerator summed across phases).
671 crate::stats::derive_rate_metrics(target);
672}
673
674/// Run the FULL run-level `ext_metrics` population sequence into `stats` — the
675/// single source of truth shared by the eval layer (`evaluate_vm_result`) and
676/// [`crate::vmm::VmResult::run_metric`], so the two produce byte-identical
677/// run-level ext maps for the same run. Reads `samples` + `stats.phases` +
678/// `stats.cgroups`, writes `stats.ext_metrics`, in the canonical order:
679/// 1. `derive_phase_metrics` — derive per-cgroup and pooled phase scalars,
680/// including the CPU-denominated `iteration_rate` components.
681/// 2. [`populate_run_ext_metrics`] — the `read_sample`-wired registry family
682/// (over every freeze in `samples`), then the whole-run wall + IRQ rates.
683/// 3. [`populate_run_ext_metrics_from_phases`] — the phase-only ext metrics
684/// whose `read_sample` is `None` (`avg_imbalance_ratio`, `iteration_rate`,
685/// `system_time_ns`, `user_time_ns`, the per-CPU IRQ spatial maxes, and the
686/// per-cgroup PSI-irq spatial maxes).
687/// 4. [`populate_run_pooled_iteration_rate`] — add any whole-run
688/// cgroup carriers to the canonical `iteration_rate` pool.
689/// 5. [`populate_run_pooled_taobench`] — the whole-run taobench throughput + hit Rates
690/// pooled cross-cgroup (from `stats.cgroups[].taobench_whole`).
691/// 6. [`populate_run_pooled_taobench_distribution`] — the taobench whole-run
692/// open-loop serve-latency `*_whole` percentiles (union of the per-phase
693/// per-cgroup serve `PlatStats` histograms, percentile re-derived over the union).
694/// 7. [`populate_run_pooled_schbench`] — the schbench whole-run loop Counter +
695/// role-separate run-delay gate-Rates (from
696/// `stats.phases[].per_cgroup[].schbench` raw pairs, summed over phases+cgroups).
697/// 8. [`populate_run_pooled_schbench_distribution`] — the schbench whole-run
698/// latency/rps `*_whole` percentiles (union of the per-phase per-cgroup
699/// `PlatStats` histograms, percentile re-derived over the union).
700/// 9. [`populate_run_distribution_metrics`] — the `Distribution` / `WorstLowest`
701/// / `WakeLatencyTailRatio` / `WorstCrossNodeRatio` re-pools (from
702/// `stats.phases[].per_cgroup` raw samples + `stats.cgroups`).
703///
704/// ORDER IS LOAD-BEARING: step 1 must precede step 3 so the canonical
705/// CPU-denominated phase rate exists before the run fold. Step 2 must also
706/// precede step 3 so whole-run IRQ-counter deltas land before step 3's
707/// `contains_key` skip; steps 4-9 fold the per-cgroup roll-up after the
708/// per-phase families.
709///
710/// `stats.phases` may be pre- or post-derived. Step 1 is idempotent and
711/// deliberately owns this derivation so callers cannot accidentally omit the
712/// guest CPU-time components required by `iteration_rate`.
713pub fn populate_run_ext_all(
714 stats: &mut ScenarioStats,
715 samples: &crate::scenario::sample::SampleSeries,
716) {
717 // CPU-denominated phase rates require the guest per-cgroup carriers. Derive
718 // them before the phase fold so both the per-phase and run-level views use
719 // the same delivered-CPU currency.
720 derive_phase_metrics(&mut stats.phases);
721 populate_run_ext_metrics(samples, &mut stats.ext_metrics);
722 populate_run_ext_metrics_from_phases(&stats.phases, &mut stats.ext_metrics);
723 populate_run_pooled_iteration_rate(stats);
724 populate_run_pooled_taobench(stats);
725 populate_run_pooled_taobench_distribution(stats);
726 populate_run_pooled_schbench(stats);
727 populate_run_pooled_schbench_distribution(stats);
728 populate_run_distribution_metrics(stats);
729}
730
731/// Inject the canonical run-level pooled `iteration_rate` Rate's two Counter
732/// components into `stats.ext_metrics`, summed across the cgroups that have
733/// measured on-CPU time — the cross-cgroup re-pool axis. Rather than routing
734/// the per-cgroup efficiency through `AssertResult::merge`'s worst-by-polarity
735/// `ext_metrics` fold (which picks the WORST cgroup's value, not Σ, and has
736/// no derive post-pass), this reads the already-merged `stats.cgroups` vec
737/// directly: `iteration_rate` = Σ`total_iterations` /
738/// Σ(`total_cpu_time_ns`/1e9) over cgroups with `total_cpu_time_ns > 0` — the
739/// per-cgroup [`CgroupStats::iterations_per_cpu_sec`] re-pooled, NOT a mean of
740/// per-cgroup ratios, NOT the worst single cgroup.
741///
742/// MUST run at the eval layer AFTER the cgroup-bearing merges (every merge that
743/// contributes a [`CgroupStats`], so `stats.cgroups` holds every per-cgroup
744/// entry) and BEFORE the sidecar write. The trailing monitor-verdict merge at
745/// the eval layer merges an `inconclusive()` carrying empty `stats` (no cgroups,
746/// no ext keys), so it is safe to run after this. If component injection ever
747/// moved BEFORE a cgroup-bearing merge, that worst-by-polarity fold would
748/// min/max these Counter keys into single-cgroup scalars, silently corrupting
749/// the pooled sum.
750///
751/// A cgroup with `total_cpu_time_ns == 0` (schedstat unavailable, or
752/// `num_workers == 0`) is EXCLUDED from BOTH sums — mirroring the per-cgroup
753/// [`CgroupStats::iterations_per_cpu_sec`] None-on-zero (`total_cpu_time_ns >
754/// 0` implies `num_workers > 0`, so the one predicate covers both). Crediting
755/// an unmeasured cgroup's iterations against the measured cgroups' CPU-seconds
756/// would overstate cohort efficiency — the silent-wrong-answer this gate
757/// prevents. Both components are inserted both-or-neither (the
758/// `derive_rate_metrics` co-location invariant), only when the summed MEASURED
759/// on-CPU time is > 0 (every cgroup unmeasured ⇒ no rate). The ns→s `/1e9` is
760/// applied ONCE here on the summed ns (not per-cgroup, to avoid repeated float
761/// rounding), since `derive_rate_metrics` is a bare num/den.
762/// `total_iterations_pooled` is a DISTINCT ext-only key, not the typed
763/// `total_iterations` (skipped from ext_metrics; it folds cross-RUN as a MEAN
764/// — a display average — while a Rate numerator must SUM-fold so Σnum/Σdenom
765/// re-pools, so one shared key cannot carry both folds). Because it sums only
766/// MEASURED cgroups, it is ≤ the merge-summed typed `total_iterations` (which
767/// includes any zero-cpu-time cgroups), and equals it unless an excluded
768/// zero-cpu-time cgroup carried iterations>0.
769pub fn populate_run_pooled_iteration_rate(stats: &mut ScenarioStats) {
770 // Exclude cgroups with no measured on-CPU time from BOTH sums (mirrors the
771 // per-cgroup None-on-zero): crediting an unmeasured cgroup's iterations
772 // against the measured cgroups' CPU-seconds would overstate efficiency.
773 // saturating fold: pool the guest-runtime cpu-time-ns / iteration counters
774 // across cgroups; a plain `.sum()` would debug-panic / release-wrap on a
775 // corrupt/hostile component, silently corrupting iteration_rate.
776 let summed_ns: u64 = stats
777 .cgroups
778 .iter()
779 .filter(|c| c.total_cpu_time_ns > 0)
780 .map(|c| c.total_cpu_time_ns)
781 .fold(0u64, u64::saturating_add);
782 if summed_ns == 0 {
783 return;
784 }
785 let summed_iters: u64 = stats
786 .cgroups
787 .iter()
788 .filter(|c| c.total_cpu_time_ns > 0)
789 .map(|c| c.total_iterations)
790 .fold(0u64, u64::saturating_add);
791 stats
792 .ext_metrics
793 .insert("total_iterations_pooled".to_string(), summed_iters as f64);
794 stats
795 .ext_metrics
796 .insert("total_cpu_time_sec".to_string(), summed_ns as f64 / 1e9);
797 crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
798}
799
800/// Inject the whole-run taobench engine's CPU-throughput + hit Rate components into
801/// `stats.ext_metrics`, pooled across the run's `WorkType::Taobench` cgroups.
802/// Each cgroup carries its workers' merged whole-run aggregate
803/// ([`crate::assert::CgroupStats::taobench_whole`]); this folds those across
804/// cgroups (Σ ops, MAX wall window, Σ client CPU — per `TaobenchStats::merge`)
805/// and writes the `total_taobench_*` Counter components (`ops`, `fast_ops`,
806/// `slow_ops`, `wall_sec`, `cpu_sec`, plus the command-time `get_cmds` /
807/// `get_hits`), from which `crate::stats::derive_rate_metrics` derives the
808/// CPU-SECOND-denominated `taobench_total_ops_per_cpu_sec_whole`,
809/// `taobench_fast_ops_per_cpu_sec_whole`,
810/// `taobench_slow_ops_per_cpu_sec_whole` (Σops / Σclient
811/// CPU-sec — dilation-safe; wall reconstructs as `rate / D` or exactly as
812/// `Σops / total_taobench_wall_sec`), the response-time
813/// `taobench_hit_fraction` (Σfast/Σcompleted), and the command-time
814/// `taobench_command_hit_rate` (Σhits/Σcmds). The whole-run Rate keys are
815/// registered METRICS, so — unlike the per-phase
816/// `taobench_*_ops_per_cpu_sec`
817/// (`MetricKind::PerPhase`, invisible to the whole-run cross-run fold) — they
818/// reach the perf-delta `--noise-adjust` spread analysis. The open-loop
819/// serve-latency distribution is a SEPARATE pool
820/// ([`populate_run_pooled_taobench_distribution`], the `*_us_whole` keys).
821///
822/// MUST run post-`merge` (after every cgroup-bearing merge has populated
823/// `stats.cgroups`), exactly like [`populate_run_pooled_iteration_rate`]:
824/// an earlier run would pool over an incomplete cgroup set. A run with no
825/// Taobench cgroup writes nothing (the pool is `None`) — the keys stay absent,
826/// keeping a non-taobench run distinct from a measured zero.
827///
828/// Numerators are retained whenever a Taobench carrier exists. Wall and CPU
829/// denominators are inserted independently only when positive, so a missing
830/// wall window cannot suppress a valid CPU-denominated rate (and vice versa).
831/// The three throughput Rates derive only with measured client CPU time;
832/// `taobench_hit_fraction` =
833/// `total_taobench_fast_ops` / `total_taobench_ops` derives iff ops completed
834/// (`total_taobench_ops > 0`) and `taobench_command_hit_rate` =
835/// `total_taobench_get_hits` / `total_taobench_get_cmds` iff lookups issued
836/// (`get_cmds > 0`); `derive_rate_metrics` skips a zero-denominator rate, so a
837/// window-but-no-ops run gets qps=0 keys but no false hit fraction / hit rate.
838/// The ns→s `/1e9` is applied ONCE here (not in `derive_rate_metrics`, a bare
839/// num/den), mirroring `total_cpu_time_sec`. Cross-RUN the components SUM-fold
840/// (Counter), so each Rate re-pools as Σnumerator / Σdenominator over the cohort
841/// — the aggregate throughput / hit rate, not a mean of per-run values.
842pub fn populate_run_pooled_taobench(stats: &mut ScenarioStats) {
843 use crate::stats::{
844 TOTAL_TAOBENCH_FAST_OPS, TOTAL_TAOBENCH_GET_CMDS, TOTAL_TAOBENCH_GET_HITS,
845 TOTAL_TAOBENCH_OPS, TOTAL_TAOBENCH_SLOW_OPS, TOTAL_TAOBENCH_WALL_SEC,
846 };
847 // Pool the per-cgroup whole-run aggregates across the run's Taobench cgroups:
848 // Σ ops, MAX wall window (shared by the concurrent cohorts). `None` when no
849 // cgroup ran a Taobench worker.
850 let pooled = stats
851 .cgroups
852 .iter()
853 .filter_map(|c| c.taobench_whole.as_ref())
854 .fold(
855 None,
856 |acc: Option<crate::workload::taobench::run::TaobenchStats>, t| {
857 Some(match acc {
858 Some(mut a) => {
859 a.merge(t);
860 a
861 }
862 None => *t,
863 })
864 },
865 );
866 let Some(w) = pooled else {
867 return;
868 };
869 let c = &w;
870 stats
871 .ext_metrics
872 .insert(TOTAL_TAOBENCH_OPS.to_string(), c.total_ops() as f64);
873 stats
874 .ext_metrics
875 .insert(TOTAL_TAOBENCH_FAST_OPS.to_string(), c.fast_ops as f64);
876 stats
877 .ext_metrics
878 .insert(TOTAL_TAOBENCH_SLOW_OPS.to_string(), c.slow_ops as f64);
879 if c.elapsed_ns > 0 {
880 stats.ext_metrics.insert(
881 TOTAL_TAOBENCH_WALL_SEC.to_string(),
882 c.elapsed_ns as f64 / 1e9,
883 );
884 }
885 // CPU-second window: Σ client-thread CLOCK_THREAD_CPUTIME_ID across the
886 // pooled cgroups — the `taobench_*_ops_per_cpu_sec_whole` Rate DENOMINATOR (the
887 // wall window above stays as a component: it is window evidence, not a
888 // throughput rate, and preserves the exact wall reconstruction
889 // `Σops / Σwall` alongside the `rate / D` approximation). Inserted only
890 // when measured (>0) so the CPU rates read absent — never a false rate —
891 // on a pre-field carrier.
892 if c.cpu_time_ns > 0 {
893 stats.ext_metrics.insert(
894 crate::stats::TOTAL_TAOBENCH_CPU_SEC.to_string(),
895 c.cpu_time_ns as f64 / 1e9,
896 );
897 }
898 // Command-time hit components: hits = cmds − misses (request-time). The Rate
899 // taobench_command_hit_rate = Σhits / Σcmds re-derives via derive_rate_metrics
900 // (skipped, hence absent, when no lookups issued). Diverges from the
901 // response-time taobench_hit_fraction under open-loop arrival.
902 stats
903 .ext_metrics
904 .insert(TOTAL_TAOBENCH_GET_CMDS.to_string(), c.get_cmds as f64);
905 stats.ext_metrics.insert(
906 TOTAL_TAOBENCH_GET_HITS.to_string(),
907 c.get_cmds.saturating_sub(c.get_misses) as f64,
908 );
909 crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
910}
911
912/// Inject the taobench WHOLE-RUN open-loop serve-latency percentiles into
913/// `stats.ext_metrics` as the `taobench_serve_*_us_whole` keys, re-pooled
914/// run-level by UNIONING the per-phase per-cgroup serve `PlatStats` histograms
915/// (`stats.phases[].per_cgroup[].taobench.serve_lat`) across every step-attributed
916/// phase and every cgroup, then re-deriving each percentile / min / max over the
917/// merged histogram (`PlatStats::combine` is an associative bucket-count add, so
918/// the merged histogram is the faithful pooled sample set and the re-derived
919/// percentile is the percentile OF the union — NOT a mean of per-source
920/// percentiles). The taobench analog of
921/// [`populate_run_pooled_schbench_distribution`], with the same source: the
922/// BASELINE (epoch 0) and inter-step-gap (`u32::MAX`) epochs are excluded (they
923/// are dropped from `stats.phases` by `expand_backdrop_phase_buckets`), so this is
924/// the steady-state serve distribution over the measured steps — a faithful
925/// PerRunDistribution, distinct from the standalone driver's full-run histogram.
926///
927/// Runs in [`populate_run_ext_all`] (post-merge); reads the per-phase carriers
928/// (disjoint from the taobench counter pool) and writes distinct `*_whole` keys,
929/// so it is order-independent. Keys are written only when the merged histogram
930/// has samples (`sample_count() > 0`) — a closed-loop run (no serve samples)
931/// reads ABSENT, never a false 0. The keys are `MetricKind::PerRunDistribution`:
932/// noise-compared per-run by `crate::stats::noise_findings`, NEVER cross-run
933/// folded (a percentile of a union is not a mean of per-run percentiles, and the
934/// per-phase histograms are dropped at the cross-run boundary).
935pub fn populate_run_pooled_taobench_distribution(stats: &mut ScenarioStats) {
936 use crate::stats::{
937 TAOBENCH_SERVE_MAX_US_WHOLE, TAOBENCH_SERVE_MIN_US_WHOLE, TAOBENCH_SERVE_P50_US_WHOLE,
938 TAOBENCH_SERVE_P90_US_WHOLE, TAOBENCH_SERVE_P99_US_WHOLE, TAOBENCH_SERVE_P999_US_WHOLE,
939 };
940 use crate::workload::schbench::plat::{Pct, PlatStats};
941
942 // Union the per-phase per-cgroup serve histograms across the whole run.
943 let mut serve = PlatStats::default();
944 for phase in &stats.phases {
945 for pc in phase.per_cgroup.values() {
946 if let Some(t) = pc.taobench.as_ref() {
947 serve.combine(&t.serve_lat);
948 }
949 }
950 }
951 if serve.sample_count() == 0 {
952 return;
953 }
954 let q = serve.percentiles();
955 stats.ext_metrics.insert(
956 TAOBENCH_SERVE_P50_US_WHOLE.to_string(),
957 q.value_at(Pct::P50) as f64,
958 );
959 stats.ext_metrics.insert(
960 TAOBENCH_SERVE_P90_US_WHOLE.to_string(),
961 q.value_at(Pct::P90) as f64,
962 );
963 stats.ext_metrics.insert(
964 TAOBENCH_SERVE_P99_US_WHOLE.to_string(),
965 q.value_at(Pct::P99) as f64,
966 );
967 stats.ext_metrics.insert(
968 TAOBENCH_SERVE_P999_US_WHOLE.to_string(),
969 q.value_at(Pct::P999) as f64,
970 );
971 stats
972 .ext_metrics
973 .insert(TAOBENCH_SERVE_MIN_US_WHOLE.to_string(), q.min as f64);
974 stats
975 .ext_metrics
976 .insert(TAOBENCH_SERVE_MAX_US_WHOLE.to_string(), q.max as f64);
977}
978
979/// Inject the schbench whole-run Class-3 metrics — the loop Counter and the
980/// role-separate run-delay gate-Rate components — into `stats.ext_metrics`,
981/// summed across EVERY phase and EVERY cgroup from the per-phase
982/// `SchbenchPhaseStats` raw pairs (`stats.phases[].per_cgroup[].schbench`). The
983/// raw `(run_delay_ns, pcount)` pairs and `loop_count` are integer and
984/// associative, so summing across phases+cgroups gives the run-level totals; the
985/// two `*_run_delay_ns_per_sched` Rates then re-derive Σrun_delay/Σpcount (the
986/// sample-weighted per-schedule mean — NOT a mean of per-run means). The MESSAGE
987/// and WORKER thread roles pool SEPARATELY (different per-schedule wait
988/// populations — never cross-pool).
989///
990/// Runs in [`populate_run_ext_all`] (post-merge, after
991/// [`populate_run_pooled_taobench`]); reads the per-phase carriers (a disjoint
992/// source from the iterations/taobench pools) and writes distinct
993/// `total_schbench_*` / `schbench_*_run_delay_ns_per_sched` keys, so it is
994/// order-independent. A run with no schbench carrier writes nothing (keys stay
995/// absent — a non-schbench run is distinct from a measured zero).
996///
997/// Both-or-neither PER ROLE: each role's two Counter components are inserted only
998/// when that role was scheduled (`pcount > 0`), so `derive_rate_metrics` yields
999/// the role's gate-Rate iff it ran (never a 0/0); the two roles are independent
1000/// (a worker-only run emits only the worker Rate). `total_schbench_loops` is
1001/// always written when any schbench carrier ran (0 is a measured zero). The
1002/// per-phase `sched_delay_msg/worker_us` is the SAME Σrun_delay_ns/Σpcount
1003/// per-schedule mean at phase scope (NOT schbench's native mean-of-per-thread-
1004/// means, a separate whole-run SchbenchResult stat) and stays PerPhase
1005/// display-only — only these Rates gate; no double-count. Cross-RUN the
1006/// components SUM-fold (Counter), so each Rate re-pools Σrun_delay/Σpcount.
1007pub fn populate_run_pooled_schbench(stats: &mut ScenarioStats) {
1008 use crate::stats::{
1009 TOTAL_SCHBENCH_LOOPS, TOTAL_SCHBENCH_MSG_PCOUNT, TOTAL_SCHBENCH_MSG_RUN_DELAY_NS,
1010 TOTAL_SCHBENCH_WORKER_PCOUNT, TOTAL_SCHBENCH_WORKER_RUN_DELAY_NS,
1011 };
1012 let mut msg_run_delay_ns: u64 = 0;
1013 let mut msg_pcount: u64 = 0;
1014 let mut worker_run_delay_ns: u64 = 0;
1015 let mut worker_pcount: u64 = 0;
1016 let mut loops: u64 = 0;
1017 let mut worker_cpu_ns: u64 = 0;
1018 let mut any = false;
1019 for phase in &stats.phases {
1020 for pc in phase.per_cgroup.values() {
1021 if let Some(s) = pc.schbench.as_ref() {
1022 any = true;
1023 // saturating: guest-runtime run-delay-ns / pcount / loop /
1024 // cpu counters pooled across phases+cgroups (matches the
1025 // already-saturating SchbenchPhaseStats::merge); never wrap a
1026 // gate-Rate.
1027 msg_run_delay_ns = msg_run_delay_ns.saturating_add(s.msg_run_delay_ns);
1028 msg_pcount = msg_pcount.saturating_add(s.msg_pcount);
1029 worker_run_delay_ns = worker_run_delay_ns.saturating_add(s.worker_run_delay_ns);
1030 worker_pcount = worker_pcount.saturating_add(s.worker_pcount);
1031 loops = loops.saturating_add(s.loop_count);
1032 worker_cpu_ns = worker_cpu_ns.saturating_add(s.worker_cpu_ns);
1033 }
1034 }
1035 }
1036 if !any {
1037 return;
1038 }
1039 stats
1040 .ext_metrics
1041 .insert(TOTAL_SCHBENCH_LOOPS.to_string(), loops as f64);
1042 // Whole-run worker CPU-second window — the CPU-SECOND-denominated
1043 // `schbench_total_loops_per_cpu_sec` Rate denominator (Σloops / Σworker
1044 // CPU-sec; message-thread CPU excluded, mirroring schbench's Σ worker
1045 // runtimes). Dilation-safe; wall reconstructs as `rate / D` via the
1046 // sidecar's `host_dilation`. Inserted only when measured (>0) so the
1047 // Rate reads absent on a pre-field carrier, never a false rate.
1048 if worker_cpu_ns > 0 {
1049 stats.ext_metrics.insert(
1050 crate::stats::TOTAL_SCHBENCH_WORKER_CPU_SEC.to_string(),
1051 worker_cpu_ns as f64 / 1e9,
1052 );
1053 }
1054 if msg_pcount > 0 {
1055 stats.ext_metrics.insert(
1056 TOTAL_SCHBENCH_MSG_RUN_DELAY_NS.to_string(),
1057 msg_run_delay_ns as f64,
1058 );
1059 stats
1060 .ext_metrics
1061 .insert(TOTAL_SCHBENCH_MSG_PCOUNT.to_string(), msg_pcount as f64);
1062 }
1063 if worker_pcount > 0 {
1064 stats.ext_metrics.insert(
1065 TOTAL_SCHBENCH_WORKER_RUN_DELAY_NS.to_string(),
1066 worker_run_delay_ns as f64,
1067 );
1068 stats.ext_metrics.insert(
1069 TOTAL_SCHBENCH_WORKER_PCOUNT.to_string(),
1070 worker_pcount as f64,
1071 );
1072 }
1073 crate::stats::derive_rate_metrics(&mut stats.ext_metrics);
1074}
1075
1076/// Inject the schbench whole-run DISTRIBUTIONAL metrics (the wakeup /
1077/// request latency percentiles + min/max and the achieved-rps percentiles) into
1078/// `stats.ext_metrics` as the `*_whole` keys, re-pooled run-level by UNIONING the
1079/// per-phase per-cgroup `PlatStats` histograms
1080/// (`stats.phases[].per_cgroup[].schbench.{wakeup,request,rps}`) across EVERY
1081/// phase and EVERY cgroup, then re-deriving each percentile / min / max over the
1082/// merged histogram. `PlatStats::combine` is an associative bucket-count add, so
1083/// the merged histogram is the FAITHFUL union and the re-derived percentile is
1084/// the percentile OF the pooled sample set — NOT a mean of per-phase / per-cgroup
1085/// percentiles (the percentile operator is non-linear). This is the schbench
1086/// histogram analog of [`populate_run_distribution_metrics`]'s raw-sample union.
1087///
1088/// Runs in [`populate_run_ext_all`] (post-merge, after
1089/// [`populate_run_pooled_schbench`]); reads the per-phase carriers (disjoint from
1090/// the iterations / taobench / schbench loop/run-delay pools) and writes distinct
1091/// `*_whole` keys, so it is order-independent. Each stream's keys are written
1092/// only when its merged histogram has samples (`sample_count() > 0`) — a stream
1093/// with no samples (e.g. a sub-1s run with no rps samples) reads ABSENT, never a
1094/// false 0 (mirrors the per-phase `write_schbench_scalars` gating and the
1095/// carrier-less graceful degradation of [`populate_run_distribution_metrics`]).
1096///
1097/// The `*_whole` keys are `crate::stats::MetricKind::PerRunDistribution`:
1098/// noise-compared per-run by `crate::stats::noise_findings` (each run's own p99),
1099/// NEVER cross-RUN folded (a percentile of a union is not a mean of per-run
1100/// percentiles, and the per-phase histograms are dropped at the cross-run
1101/// boundary), so they are gated out of the cross-RUN ext fold and the within-run
1102/// reducers (`is_derived`). Distinct names from the per-phase percentile keys
1103/// (one registry name = one kind), produced solely here.
1104pub fn populate_run_pooled_schbench_distribution(stats: &mut ScenarioStats) {
1105 use crate::stats::{
1106 SCHBENCH_REQUEST_MAX_US_WHOLE, SCHBENCH_REQUEST_MIN_US_WHOLE,
1107 SCHBENCH_REQUEST_P50_US_WHOLE, SCHBENCH_REQUEST_P90_US_WHOLE,
1108 SCHBENCH_REQUEST_P99_US_WHOLE, SCHBENCH_REQUEST_P999_US_WHOLE, SCHBENCH_RPS_MAX_WHOLE,
1109 SCHBENCH_RPS_MIN_WHOLE, SCHBENCH_RPS_P20_WHOLE, SCHBENCH_RPS_P50_WHOLE,
1110 SCHBENCH_RPS_P90_WHOLE, SCHBENCH_WAKEUP_MAX_US_WHOLE, SCHBENCH_WAKEUP_MIN_US_WHOLE,
1111 SCHBENCH_WAKEUP_P50_US_WHOLE, SCHBENCH_WAKEUP_P90_US_WHOLE, SCHBENCH_WAKEUP_P99_US_WHOLE,
1112 SCHBENCH_WAKEUP_P999_US_WHOLE,
1113 };
1114 use crate::workload::schbench::plat::{Pct, PlatStats};
1115
1116 // Union the per-stream histograms across ALL phases+cgroups (combine =
1117 // associative bucket-count add → the faithful pooled histogram).
1118 let mut wakeup = PlatStats::default();
1119 let mut request = PlatStats::default();
1120 let mut rps = PlatStats::default();
1121 for phase in &stats.phases {
1122 for pc in phase.per_cgroup.values() {
1123 if let Some(s) = pc.schbench.as_ref() {
1124 wakeup.combine(&s.wakeup);
1125 request.combine(&s.request);
1126 rps.combine(&s.rps);
1127 }
1128 }
1129 }
1130 // Latency streams: 4 percentiles + min/max, re-derived over the union, µs.
1131 if wakeup.sample_count() > 0 {
1132 let q = wakeup.percentiles();
1133 stats.ext_metrics.insert(
1134 SCHBENCH_WAKEUP_P50_US_WHOLE.to_string(),
1135 q.value_at(Pct::P50) as f64,
1136 );
1137 stats.ext_metrics.insert(
1138 SCHBENCH_WAKEUP_P90_US_WHOLE.to_string(),
1139 q.value_at(Pct::P90) as f64,
1140 );
1141 stats.ext_metrics.insert(
1142 SCHBENCH_WAKEUP_P99_US_WHOLE.to_string(),
1143 q.value_at(Pct::P99) as f64,
1144 );
1145 stats.ext_metrics.insert(
1146 SCHBENCH_WAKEUP_P999_US_WHOLE.to_string(),
1147 q.value_at(Pct::P999) as f64,
1148 );
1149 stats
1150 .ext_metrics
1151 .insert(SCHBENCH_WAKEUP_MIN_US_WHOLE.to_string(), q.min as f64);
1152 stats
1153 .ext_metrics
1154 .insert(SCHBENCH_WAKEUP_MAX_US_WHOLE.to_string(), q.max as f64);
1155 }
1156 if request.sample_count() > 0 {
1157 let q = request.percentiles();
1158 stats.ext_metrics.insert(
1159 SCHBENCH_REQUEST_P50_US_WHOLE.to_string(),
1160 q.value_at(Pct::P50) as f64,
1161 );
1162 stats.ext_metrics.insert(
1163 SCHBENCH_REQUEST_P90_US_WHOLE.to_string(),
1164 q.value_at(Pct::P90) as f64,
1165 );
1166 stats.ext_metrics.insert(
1167 SCHBENCH_REQUEST_P99_US_WHOLE.to_string(),
1168 q.value_at(Pct::P99) as f64,
1169 );
1170 stats.ext_metrics.insert(
1171 SCHBENCH_REQUEST_P999_US_WHOLE.to_string(),
1172 q.value_at(Pct::P999) as f64,
1173 );
1174 stats
1175 .ext_metrics
1176 .insert(SCHBENCH_REQUEST_MIN_US_WHOLE.to_string(), q.min as f64);
1177 stats
1178 .ext_metrics
1179 .insert(SCHBENCH_REQUEST_MAX_US_WHOLE.to_string(), q.max as f64);
1180 }
1181 // RPS stream: PLIST_FOR_RPS = 20/50/90 + min/max (the schbench rps table).
1182 if rps.sample_count() > 0 {
1183 let r = rps.percentiles();
1184 stats.ext_metrics.insert(
1185 SCHBENCH_RPS_P20_WHOLE.to_string(),
1186 r.value_at(Pct::P20) as f64,
1187 );
1188 stats.ext_metrics.insert(
1189 SCHBENCH_RPS_P50_WHOLE.to_string(),
1190 r.value_at(Pct::P50) as f64,
1191 );
1192 stats.ext_metrics.insert(
1193 SCHBENCH_RPS_P90_WHOLE.to_string(),
1194 r.value_at(Pct::P90) as f64,
1195 );
1196 stats
1197 .ext_metrics
1198 .insert(SCHBENCH_RPS_MIN_WHOLE.to_string(), r.min as f64);
1199 stats
1200 .ext_metrics
1201 .insert(SCHBENCH_RPS_MAX_WHOLE.to_string(), r.max as f64);
1202 }
1203}
1204
1205/// Populate run-level DERIVED distributional metrics into
1206/// `stats.ext_metrics`: every registered `MetricKind::Distribution`,
1207/// `MetricKind::WorstLowest`, `MetricKind::WakeLatencyTailRatio`, and
1208/// `MetricKind::WorstCrossNodeRatio`. This is the SOLE
1209/// within-run producer of those metrics' values — they carry no per-phase
1210/// sample slice and no cross-cgroup merge fold, and their registry accessors
1211/// are `|_| None`, so `MetricDef::read` reads the value
1212/// written here from `ext_metrics`.
1213///
1214/// DISTRIBUTION (the 5 wake / run-delay aggregates): pools the RAW sample
1215/// vectors held in `stats.phases[].per_cgroup` across EVERY phase and EVERY
1216/// cgroup into one combined set, then recomputes the percentile / CV / mean
1217/// / extreme over it — the statistic of the union, NOT a max or mean of
1218/// per-cgroup reductions (the percentile of a union is not the max of
1219/// per-source percentiles). The ns→µs scale is applied ONCE here (the
1220/// carriers store raw ns, per [`PhaseCgroupStats::run_delays_ns`]). The wake
1221/// pool is population-WEIGHTED: each phase carrier's samples carry weight
1222/// `wake_sample_total / wake_latencies_ns.len()`, so a phase whose reservoir
1223/// hit the cap contributes by true population, not capped length (the
1224/// cross-PHASE de-skew) — reduced via the weighted percentile / moments.
1225/// The run-delay pool is unweighted (per-worker, never reservoir-capped, so
1226/// length IS population). Below the wake cap every weight is 1.0, so the
1227/// weighted P99 / median / mean / worst are byte-identical to the unweighted
1228/// concat; the weighted CV matches only within ~1e-9 (it sums the mean in f64
1229/// where the unweighted path sums in u64 — a weighted variance cannot keep the
1230/// u64 sum).
1231///
1232/// CARRIER-LESS FOLD (graceful degradation): a cgroup whose raw samples are
1233/// NOT in the pool — a backdrop epoch that fell on BASELINE or the
1234/// inter-step gap (no paired host bucket, so no carrier) or a cgroup whose
1235/// carrier was stripped/empty (`strip_phase_cgroup_samples`) — is NOT
1236/// dropped. Its
1237/// surviving per-cgroup [`CgroupStats`] reduction folds worst-wins (max — every
1238/// Distribution metric is `LowerBetter`, registry-gated) into the pooled value.
1239/// The CgroupStats reductions are never stripped — `stats.cgroups[]` is the
1240/// already-reduced `cgroup_stats(reports)` output, a SEPARATE reduction path
1241/// from the per-phase carriers — so a carrier-less cgroup always has a source.
1242/// When EVERY carrier is empty (a fully-stripped run) the pool is empty and the
1243/// result degenerates to the max over every cgroup's reduction — the pre-Item-7
1244/// cross-cgroup max. NOTE the value CLASS of a folded cgroup differs from a
1245/// pooled one for the P99 / Median / Mean / CV reductions: a pooled cgroup
1246/// contributes to the percentile of the union; a carrier-less cgroup
1247/// contributes its per-cgroup reduction worst-wins (a worst-cgroup proxy, not
1248/// pooled). For the `SampleReduction::Worst` reduction the two COINCIDE
1249/// (max-of-union == max-of-per-cgroup-maxes), so the carrier-less fold is exact
1250/// there, not a proxy. A second asymmetry specific to CV (from the population
1251/// weighting): the POOLED CV divides variance/mean by Σ per-sample weights (the
1252/// reconstructed population), while a carrier-less cgroup's folded CV is
1253/// [`cgroup_stats`]'s UNWEIGHTED CV (`n = all_latencies.len()`). The two
1254/// coincide below the cap (all weights 1.0) and diverge above it; the mix is
1255/// sound — a carrier-less cgroup has no per-phase weight data to
1256/// population-weight (its carrier is absent by definition), and both feed the
1257/// same LowerBetter worst-wins max. Backdrop step-phase carriers now join
1258/// the pool directly (per-epoch expansion in `collect_handles`); only the
1259/// carrier-less cases above fold worst-wins.
1260///
1261/// WORSTLOWEST (the 2 iteration efficiencies): the lowest (worst) cgroup's
1262/// efficiency, computed per-cgroup from the `stats.cgroups[]` COUNTERS via
1263/// [`CgroupStats::iterations_per_worker`] / [`CgroupStats::iterations_per_cpu_sec`]
1264/// and the None-aware lowest-wins fold (a measured `Some(0.0)` — starvation
1265/// — wins; a no-data `None` is skipped; an all-`None` cohort writes no key,
1266/// preserving absence as a missing ext entry rather than a `0.0`). The
1267/// counters survive stripping, so WorstLowest needs no fallback branch.
1268///
1269/// Runs post-merge at the eval layer beside
1270/// [`populate_run_pooled_iteration_rate`], AFTER the per-cgroup
1271/// carriers are folded into `stats.phases` and BEFORE the sidecar write, so
1272/// `stats.phases[].per_cgroup` is fully merged and `stats.cgroups` is the
1273/// final per-cgroup roll-up.
1274pub fn populate_run_distribution_metrics(stats: &mut ScenarioStats) {
1275 // Pool the per-phase per-cgroup raw sample vectors across every phase and
1276 // cgroup ONCE for the Distribution PRIMARY path, then sort so the
1277 // percentile reductions can index directly. `wake_latencies_ns` is
1278 // per-WAKEUP (reservoir-capped at MAX_WAKE_SAMPLES on the carrier because
1279 // it can reach 100k); `run_delays_ns` is per-WORKER (one sample/worker, not
1280 // capped), so the run-delay pool is total-workers × phases — genuinely
1281 // small. The wake pool is NOT intrinsically small: it is the union of the
1282 // per-carrier wake vectors, num_carriers × MAX_WAKE_SAMPLES worst case, so
1283 // its size is bounded by the upstream 16 MiB bulk-frame cap on the arriving
1284 // carriers (strip_phase_cgroup_samples is the overflow lever) rather than by
1285 // being tiny — no OOM risk, no cap needed here. Both are transient: reduced
1286 // to scalars here, never re-serialized.
1287 // Wake samples carry a per-sample population WEIGHT (`wake_sample_total /
1288 // reservoir len`) so a >cap phase contributes in proportion to its true
1289 // population, not its guest-capped length (the cross-PHASE de-skew). Run-delay
1290 // samples are per-worker and never reservoir-capped (no `*_sample_total`), so
1291 // their length IS their population — pooled unweighted.
1292 let mut wake_pool: Vec<(u64, f64)> = Vec::new();
1293 // Distinct timer-latency pool, population-WEIGHTED like
1294 // wake_pool (reservoir-capped, so a >cap phase carries weight
1295 // timer_sample_total/len for the cross-phase de-skew).
1296 let mut timer_pool: Vec<(u64, f64)> = Vec::new();
1297 let mut run_delay_pool: Vec<u64> = Vec::new();
1298 // Names of cgroups that contributed NON-EMPTY samples to each pool. A
1299 // cgroup absent here — a backdrop epoch that fell on BASELINE / the
1300 // inter-step gap (no paired host bucket, so no carrier) or a
1301 // stripped/empty carrier — is NOT dropped from the run-level
1302 // Distribution: the re-pool folds its surviving per-cgroup CgroupStats
1303 // reduction worst-wins (see `populate_run_distribution_metrics_from`).
1304 // Backdrop step-phase carriers now join the pool directly (per-epoch
1305 // expansion in collect_handles), so a step-matched backdrop epoch pools
1306 // rather than worst-wins-folds.
1307 //
1308 // The fallback dedup keys on cgroup NAME (a `stats.cgroups` entry whose
1309 // name is in `*_carriers` is pooled, not reduction-folded), which assumes
1310 // carrier-bearing and carrier-less cgroup names are DISJOINT. That holds
1311 // WITHIN one step's collect (cgroupfs path uniqueness — two live cgroups
1312 // cannot share a name, mkdir would EEXIST — and a single collect_handles
1313 // call attaches carriers to all its handles or none). It does NOT hold
1314 // across STEPS: `AssertResult::merge` extends `stats.cgroups` per
1315 // (handle, step), so a name that carried samples at step k recurs at step
1316 // k+1, and the step-(k+1) entry is skipped by this dedup (its name is in
1317 // `*_carriers`). That only OMITS a contribution, never vanishes the metric
1318 // (the step-k pool still produces it). A skipped step-(k+1) entry whose
1319 // carrier is merely EMPTY (collected no samples) is harmless: its per-cgroup
1320 // reduction is the trivial zero a worst-wins f64::max ignores. The only
1321 // LOSSY case is a step-(k+1) entry STRIPPED of live samples while step k
1322 // survives, and that cannot arise today: `strip_phase_cgroup_samples` strips
1323 // RUN-WIDE (every phase at once), so a run is never partially stripped per
1324 // step. A backdrop name now enters `*_carriers` (pooled once via its
1325 // per-epoch expansion) so it is skipped from the reduction-fold — and a
1326 // backdrop and a step-local cgroup cannot share a live name (cgroupfs
1327 // mkdir EEXIST; a backdrop is live the whole scenario), so each
1328 // stats.cgroups entry still contributes via exactly one of {pool,
1329 // reduction-fold} — no double count.
1330 let mut wake_carriers: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
1331 let mut timer_carriers: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
1332 let mut run_delay_carriers: std::collections::BTreeSet<&str> =
1333 std::collections::BTreeSet::new();
1334 for phase in &stats.phases {
1335 for (cgname, pcg) in &phase.per_cgroup {
1336 if !pcg.wake_latencies_ns.is_empty() {
1337 // Per-sample weight = true population / surviving reservoir size.
1338 // A ≤cap carrier has len == wake_sample_total → weight 1.0, so the
1339 // pool is value-for-value with the unweighted concat; a >cap
1340 // carrier's capped samples each stand for `total/len > 1` true
1341 // wakes, restoring the cross-phase population proportion.
1342 //
1343 // INVARIANT: `reservoir_push` bumps wake_sample_total on EVERY
1344 // wakeup but pushes into the reservoir only up to MAX_WAKE_SAMPLES,
1345 // and both the carrier merge and `phase_cgroup_stats` SUM the two,
1346 // so wake_sample_total >= len always (== len below the cap). A
1347 // carrier violating that — samples present but a zeroed/under-count
1348 // total — would yield weight < 1 and silently UNDER-weight (at
1349 // weight 0, DROP) its samples. Clamp the numerator to len so a
1350 // malformed carrier degrades to unit weight (reservoir treated as
1351 // its own population) instead of dropping data; debug_assert the
1352 // invariant so a real counting bug surfaces in dev.
1353 let len = pcg.wake_latencies_ns.len() as u64;
1354 debug_assert!(
1355 pcg.wake_sample_total >= len,
1356 "wake_sample_total ({}) < reservoir len ({}): malformed carrier",
1357 pcg.wake_sample_total,
1358 len,
1359 );
1360 let w = pcg.wake_sample_total.max(len) as f64 / len as f64;
1361 wake_pool.extend(pcg.wake_latencies_ns.iter().map(|&v| (v, w)));
1362 wake_carriers.insert(cgname.as_str());
1363 }
1364 if !pcg.timer_latencies_ns.is_empty() {
1365 // Population-weighted exactly like the wake pool: a >cap phase's
1366 // capped samples each stand for timer_sample_total/len true
1367 // wakes, restoring the cross-phase population proportion.
1368 let len = pcg.timer_latencies_ns.len() as u64;
1369 debug_assert!(
1370 pcg.timer_sample_total >= len,
1371 "timer_sample_total ({}) < reservoir len ({}): malformed carrier",
1372 pcg.timer_sample_total,
1373 len,
1374 );
1375 let w = pcg.timer_sample_total.max(len) as f64 / len as f64;
1376 timer_pool.extend(pcg.timer_latencies_ns.iter().map(|&v| (v, w)));
1377 timer_carriers.insert(cgname.as_str());
1378 }
1379 if !pcg.run_delays_ns.is_empty() {
1380 run_delay_pool.extend_from_slice(&pcg.run_delays_ns);
1381 run_delay_carriers.insert(cgname.as_str());
1382 }
1383 }
1384 }
1385 wake_pool.sort_unstable_by_key(|&(v, _)| v);
1386 timer_pool.sort_unstable_by_key(|&(v, _)| v);
1387 run_delay_pool.sort_unstable();
1388 populate_run_distribution_metrics_from(
1389 &mut stats.ext_metrics,
1390 crate::stats::METRICS.iter().filter_map(|m| {
1391 matches!(
1392 m.kind,
1393 crate::stats::MetricKind::Distribution { .. }
1394 | crate::stats::MetricKind::WorstLowest { .. }
1395 | crate::stats::MetricKind::WakeLatencyTailRatio
1396 )
1397 .then_some((m.name, m.kind))
1398 }),
1399 &wake_pool,
1400 &wake_carriers,
1401 &timer_pool,
1402 &timer_carriers,
1403 &run_delay_pool,
1404 &run_delay_carriers,
1405 &stats.cgroups,
1406 stats.total_iterations,
1407 );
1408 // worst_page_locality (WorstLowest{NumaLocal,NumaTotal}) re-pools from the
1409 // per-phase NUMA carriers, which the _from helper above cannot reach (it
1410 // takes only stats.cgroups, and the reports-only CgroupStats hardcodes
1411 // page_locality 0.0). Single-sourced with run_metric's WorstPageLocality via
1412 // worst_page_locality(): the lowest per-cgroup page_locality over cgroups
1413 // that measured NUMA residency (a measured 0.0 — all off-node — winning); an
1414 // all-unmeasured cohort writes no key (absence preserved as a missing ext
1415 // entry, never a 0.0 sentinel). numa_agg_per_cgroup()'s borrow ends before
1416 // the insert.
1417 if let Some(v) = stats.worst_page_locality() {
1418 stats
1419 .ext_metrics
1420 .insert("worst_page_locality".to_string(), v);
1421 }
1422 // worst_cross_node_migration_ratio (WorstCrossNodeRatio) re-pools from the same
1423 // per-phase NUMA carriers — the MAX per-cgroup churn ratio over the latest
1424 // residency total — single-sourced with run_metric's WorstCrossNodeMigrationRatio
1425 // via worst_cross_node_migration_ratio(); an all-unmeasured cohort writes no key
1426 // (absence preserved). Not handled by the _from helper above (it has only
1427 // stats.cgroups, and the metric is excluded from that filter).
1428 if let Some(v) = stats.worst_cross_node_migration_ratio() {
1429 stats
1430 .ext_metrics
1431 .insert("worst_cross_node_migration_ratio".to_string(), v);
1432 }
1433}
1434
1435/// Inner of [`populate_run_distribution_metrics`] taking the metric specs
1436/// `(name, kind)` and the pre-pooled+SORTED sample sets explicitly, so the
1437/// re-pool math is unit-testable without registered metrics (the
1438/// `derive_rate_metrics_from` precedent). `wake_pool` / `run_delay_pool` are
1439/// the cross-phase+cross-cgroup raw-ns unions (ascending); `*_carriers` name
1440/// the cgroups that contributed samples to each pool; `cgroups` supplies the
1441/// WorstLowest counters and the per-cgroup reductions that carrier-less
1442/// cgroups (backdrop / stripped) fold into the Distribution result.
1443#[allow(clippy::too_many_arguments)]
1444pub(crate) fn populate_run_distribution_metrics_from<'a>(
1445 target: &mut std::collections::BTreeMap<String, f64>,
1446 metrics: impl Iterator<Item = (&'a str, crate::stats::MetricKind)>,
1447 wake_pool: &[(u64, f64)],
1448 wake_carriers: &std::collections::BTreeSet<&str>,
1449 timer_pool: &[(u64, f64)],
1450 timer_carriers: &std::collections::BTreeSet<&str>,
1451 run_delay_pool: &[u64],
1452 run_delay_carriers: &std::collections::BTreeSet<&str>,
1453 cgroups: &[CgroupStats],
1454 run_total_iterations: u64,
1455) {
1456 use crate::stats::{MetricKind, SampleSource, WorstLowestDenominator, WorstLowestNumerator};
1457 for (name, kind) in metrics {
1458 let value: Option<f64> = match kind {
1459 MetricKind::Distribution { source, reduction } => {
1460 // Pool the carried samples (the thesis: percentile of the
1461 // UNION), then fold worst-wins (max — Distribution is
1462 // LowerBetter, registry-gated) the surviving per-cgroup
1463 // reduction of every cgroup WITHOUT a carrier-with-samples for
1464 // this source (a backdrop, or a stripped/empty carrier), so no
1465 // cgroup is dropped from the run-level distribution. When EVERY
1466 // carrier is empty (fully stripped) the pool is empty and this
1467 // degenerates to the max over every cgroup — the pre-Item-7
1468 // cross-cgroup max.
1469 //
1470 // Pool reduction is per-source: WakeLatencyNs is population-WEIGHTED
1471 // (each phase's guest-capped samples carry weight
1472 // wake_sample_total/len, so a >cap phase contributes by true
1473 // population not capped length — the cross-PHASE de-skew, via
1474 // reduce_weighted_sorted_distribution); RunDelayNs is unweighted
1475 // (per-worker, never reservoir-capped, so length IS population, via
1476 // reduce_sorted_distribution).
1477 //
1478 // CONTRACT (now uniform with WorstLowest and WakeLatencyTailRatio
1479 // below): a cohort with NO measurement for this source anywhere
1480 // (empty carrier pool AND every non-carrier cgroup not-measured)
1481 // yields ABSENCE (None), not Some(0.0). A percentile / mean over
1482 // zero samples is undefined, not a measured zero — folding its
1483 // 0.0 sentinel into the cross-run mean would, for the LowerBetter
1484 // wake/timer/run-delay metrics, falsely drag the mean toward
1485 // "perfect". The non-carrier fold below is gated on
1486 // `cg.measured_for(source)` so an unmeasured cgroup contributes
1487 // nothing; a non-carrier cgroup that DID measure (a genuine
1488 // measured zero, e.g. workers that queued for no time) still
1489 // contributes its real value. This matches WorstLowest (None when
1490 // every iterations_per_*() is None) and WakeLatencyTailRatio (None
1491 // when no cgroup has a tail) — every distributional kind now emits
1492 // None for a no-measurement run.
1493 let (mut v, carriers): (Option<f64>, &std::collections::BTreeSet<&str>) =
1494 match source {
1495 SampleSource::WakeLatencyNs => (
1496 (!wake_pool.is_empty())
1497 .then(|| reduce_weighted_sorted_distribution(wake_pool, reduction)),
1498 wake_carriers,
1499 ),
1500 SampleSource::TimerLatencyNs => (
1501 (!timer_pool.is_empty()).then(|| {
1502 reduce_weighted_sorted_distribution(timer_pool, reduction)
1503 }),
1504 timer_carriers,
1505 ),
1506 SampleSource::RunDelayNs => (
1507 (!run_delay_pool.is_empty())
1508 .then(|| reduce_sorted_distribution(run_delay_pool, reduction)),
1509 run_delay_carriers,
1510 ),
1511 };
1512 for cg in cgroups {
1513 if !carriers.contains(cg.cgroup_name.as_str()) && cg.measured_for(source) {
1514 let r = distribution_cgroup_reduction(cg, source, reduction);
1515 v = Some(v.map_or(r, |acc| acc.max(r)));
1516 }
1517 }
1518 v
1519 }
1520 // The iterations-efficiency WorstLowest selectors (numerator
1521 // Iterations) re-pool from the `stats.cgroups` counters here; the
1522 // denominator picks the per-cgroup efficiency method. The
1523 // page-locality selector (numerator NumaLocal) re-pools instead from
1524 // the per-phase NUMA carriers in populate_run_distribution_metrics
1525 // (this fn has only stats.cgroups, not stats.phases), so it is
1526 // skipped here (the NumaLocal arm below returns None).
1527 //
1528 // In a MULTI-STEP scenario `AssertResult::merge` extends
1529 // `stats.cgroups` per (handle, step), so the same cgroup name
1530 // appears once per step; this selects the lowest single
1531 // (handle, step) entry, NOT a per-name whole-run efficiency. That
1532 // preserves the deleted `fold_lowest_some` granularity exactly and
1533 // mirrors `populate_run_pooled_iteration_rate`, which sums
1534 // over the same per-(handle, step) entries.
1535 MetricKind::WorstLowest {
1536 numerator: WorstLowestNumerator::Iterations,
1537 denominator,
1538 } => {
1539 let mut worst: Option<f64> = None;
1540 for cg in cgroups {
1541 let per_cg = match denominator {
1542 WorstLowestDenominator::NumWorkers => cg.iterations_per_worker(),
1543 WorstLowestDenominator::CpuTimeNs => cg.iterations_per_cpu_sec(),
1544 // NumaTotal pairs only with the NumaLocal numerator
1545 // (handled in the NumaLocal arm below) — never with
1546 // Iterations.
1547 WorstLowestDenominator::NumaTotal => None,
1548 };
1549 // Lowest-wins, None-aware (the semantic the deleted
1550 // `fold_lowest_some` carried in `AssertResult::merge`): a
1551 // measured `Some(0.0)` (starvation) wins the worst bucket;
1552 // a `None` is skipped.
1553 if let Some(v) = per_cg
1554 && worst.is_none_or(|w| v < w)
1555 {
1556 worst = Some(v);
1557 }
1558 }
1559 worst
1560 }
1561 // page-locality (numerator NumaLocal): re-pooled from the per-phase
1562 // NUMA carriers in populate_run_distribution_metrics, not here.
1563 MetricKind::WorstLowest {
1564 numerator: WorstLowestNumerator::NumaLocal,
1565 ..
1566 } => None,
1567 // Worst-cgroup wake-latency tail amplification: the MAX over each
1568 // cgroup's own p99/median ratio (`CgroupStats::wake_latency_tail_ratio`).
1569 // Emit NO key below the min-iterations noise floor (low-N ratios are
1570 // single-outlier noise, not a distributional signal — gated HERE at
1571 // the producer, NOT via a meaned-iteration accessor on the
1572 // aggregated row), and none when no cgroup carried a measurable tail
1573 // (every per-cgroup ratio 0.0, i.e. no median wake latency anywhere).
1574 // Absence then stays distinct from a measured value and no
1575 // sub-threshold run enters the cross-RUN mean. `wake_latency_tail_ratio`
1576 // returns 0.0 for a cgroup with no wake samples (median <= 0), which
1577 // a max-wins fold over the r > 0.0 reals correctly skips.
1578 MetricKind::WakeLatencyTailRatio => {
1579 if run_total_iterations < crate::stats::WAKE_LATENCY_TAIL_RATIO_MIN_ITERATIONS {
1580 None
1581 } else {
1582 let mut worst: Option<f64> = None;
1583 for cg in cgroups {
1584 let r = cg.wake_latency_tail_ratio();
1585 if r > 0.0 {
1586 worst = Some(worst.map_or(r, |w| w.max(r)));
1587 }
1588 }
1589 worst
1590 }
1591 }
1592 _ => None,
1593 };
1594 // Insert only a real, FINITE value: an absent key (all-None
1595 // WorstLowest cohort, or no cgroups at all) stays distinct from a
1596 // measured 0.0, matching the None-vs-Some(0.0) contract the typed
1597 // Option carried. The is_finite guard is a no-op for every
1598 // registry-valid metric (reduce_sorted_distribution reduces non-empty
1599 // pools with CV guarded to 0.0 on zero mean; WorstLowest reuses
1600 // iterations_per_worker()/iterations_per_cpu_sec() which return None on
1601 // a zero denominator), but it MATTERS for the registry-impossible
1602 // cross-source arm of distribution_cgroup_reduction: that arm returns
1603 // NaN, and when a Distribution has no pool (every carrier stripped) the
1604 // carrier-less fold can carry that NaN to `v`. An inserted NaN would
1605 // fail the ENTIRE serde_json sidecar write (serde_json rejects
1606 // non-finite), losing ALL run telemetry — so the guard degrades a
1607 // misauthored metric to ABSENCE here rather than risking that write
1608 // failure downstream.
1609 if let Some(v) = value.filter(|v| v.is_finite()) {
1610 target.insert(name.to_string(), v);
1611 }
1612 }
1613}
1614
1615/// Reduce a NON-EMPTY ascending-sorted raw-ns sample pool to one
1616/// [`crate::stats::SampleReduction`] value, ns→µs once. Mirrors the
1617/// per-cgroup reductions [`cgroup_stats`] computes (p99 / median via
1618/// [`percentile`], CV with `n = pool.len()`, mean, max) so the run-level
1619/// re-pool reproduces them over the COMBINED cross-cgroup set — to within
1620/// FP tolerance for CV / mean, not bit-exactly: this sums over the
1621/// ASCENDING-sorted pool while `cgroup_stats` sums over the unsorted
1622/// arrival order, so the float results differ by ~1e-15 (the parity test
1623/// `repool_distribution_value_for_value_with_cgroup_stats` uses a 1e-9
1624/// bound). Same "distribution-equivalent, not byte-identical" framing as
1625/// the `wake_latencies_ns` carrier doc.
1626pub(crate) fn reduce_sorted_distribution(
1627 sorted: &[u64],
1628 reduction: crate::stats::SampleReduction,
1629) -> f64 {
1630 use crate::stats::SampleReduction;
1631 match reduction {
1632 SampleReduction::P99 => percentile(sorted, 0.99) as f64 / 1000.0,
1633 SampleReduction::P999 => percentile(sorted, 0.999) as f64 / 1000.0,
1634 SampleReduction::Median => percentile(sorted, 0.5) as f64 / 1000.0,
1635 SampleReduction::Cv => {
1636 let n = sorted.len() as f64;
1637 let mean_ns = sorted.iter().sum::<u64>() as f64 / n;
1638 if mean_ns > 0.0 {
1639 let variance = sorted
1640 .iter()
1641 .map(|&v| (v as f64 - mean_ns).powi(2))
1642 .sum::<f64>()
1643 / n;
1644 variance.sqrt() / mean_ns
1645 } else {
1646 0.0
1647 }
1648 }
1649 // Divide ONCE on the summed/maxed ns (the carriers store raw ns):
1650 // mean(ns)/1000 == mean(ns/1000) and max(ns)/1000 == max(ns/1000).
1651 // Sum in f64 (not u64-then-cast) to match cgroup_stats's f64 run-delay
1652 // accumulation and PhaseCgroupStats::run_delay_summary — overflow-safe
1653 // (an f64 sum saturates toward +inf; a u64 sum would panic in debug /
1654 // silently wrap in release on a pathological pool), value identical
1655 // within the 1e-9 parity bound. (The Cv arm's mean_ns above keeps the u64 sum
1656 // because cgroup_stats's CV also u64-sums all_latencies — matching it is
1657 // exact-parity-preserving there.)
1658 SampleReduction::Mean => {
1659 sorted.iter().map(|&v| v as f64).sum::<f64>() / sorted.len() as f64 / 1000.0
1660 }
1661 // Sorted ascending, so the last element is the max.
1662 SampleReduction::Worst => *sorted.last().expect("non-empty by caller") as f64 / 1000.0,
1663 }
1664}
1665
1666/// Weighted nearest-rank percentile over a value-sorted `(value, weight)` pool —
1667/// the weighted sibling of [`percentile`]. Matches `percentile`'s convention
1668/// (the value at 1-indexed rank `ceil(W * p)`, `W` = total weight, floored at
1669/// rank 1) so with UNIT weights (every weight `1.0`) it returns byte-identically:
1670/// cumulative weight after `k` elements is `k`, `ceil(W*p) == ceil(n*p)`, and the
1671/// `.max(1.0)` floor mirrors `percentile`'s `saturating_sub(1)`, so the crossing
1672/// element is `percentile`'s `sorted[ceil(n*p)-1]` for p>0 and `sorted[0]` at
1673/// p=0. Used by the run-level wake re-pool to weight each phase's samples by
1674/// true population.
1675pub(crate) fn weighted_percentile(sorted: &[(u64, f64)], p: f64) -> u64 {
1676 if sorted.is_empty() {
1677 return 0;
1678 }
1679 debug_assert!(
1680 sorted.windows(2).all(|w| w[0].0 <= w[1].0),
1681 "weighted_percentile() requires value-sorted input",
1682 );
1683 let total: f64 = sorted.iter().map(|&(_, w)| w).sum();
1684 // Nearest-rank target, floored at 1 so `p == 0.0` maps to the first element
1685 // (mirrors percentile's saturating_sub(1) flooring rank 0 to index 0).
1686 let target = (total * p).ceil().max(1.0);
1687 let mut cum = 0.0;
1688 for &(v, w) in sorted {
1689 cum += w;
1690 if cum >= target {
1691 return v;
1692 }
1693 }
1694 sorted.last().map(|&(v, _)| v).unwrap_or(0)
1695}
1696
1697/// Weighted sibling of [`reduce_sorted_distribution`] for the wake-latency
1698/// re-pool: each `(value, weight)` carries a per-sample weight of
1699/// `wake_sample_total / reservoir_len`, so a >cap phase (reservoir-capped on the
1700/// guest) contributes in proportion to its TRUE population, not its capped
1701/// length — removing the cross-PHASE length-skew. With UNIT weights (every phase
1702/// ≤cap, so `len == wake_sample_total`) it reduces byte-identically to
1703/// [`reduce_sorted_distribution`] for P99 / Median / Mean / Worst; the Cv arm
1704/// differs only by the f64-vs-u64 mean sum. For the small fixed pool the parity
1705/// test uses, that gap is ~1e-15 (within its 1e-9 bound), but it grows ~n·ε with
1706/// pool size — a cross-phase pool can reach millions of samples (~1e-9–1e-8 on a
1707/// high-CV pool), so a LARGE-pool parity test must not assume a universal 1e-15.
1708/// A weighted variance cannot keep the u64 sum. Exhaustive over SampleReduction,
1709/// mirroring [`reduce_sorted_distribution`], so a new variant fails the build.
1710///
1711/// The Cv / Mean `total_w <= 0.0` guards and [`weighted_percentile`]'s
1712/// all-weight-zero fall-through are degenerate-input belts: the capture-path
1713/// caller [`populate_run_distribution_metrics`] clamps every per-sample weight to
1714/// a floor of 1.0, so `total_w >= len >= 1` there and those branches are
1715/// unreachable on the production path.
1716pub(crate) fn reduce_weighted_sorted_distribution(
1717 sorted: &[(u64, f64)],
1718 reduction: crate::stats::SampleReduction,
1719) -> f64 {
1720 use crate::stats::SampleReduction;
1721 match reduction {
1722 SampleReduction::P99 => weighted_percentile(sorted, 0.99) as f64 / 1000.0,
1723 SampleReduction::P999 => weighted_percentile(sorted, 0.999) as f64 / 1000.0,
1724 SampleReduction::Median => weighted_percentile(sorted, 0.5) as f64 / 1000.0,
1725 SampleReduction::Cv => {
1726 let total_w: f64 = sorted.iter().map(|&(_, w)| w).sum();
1727 if total_w <= 0.0 {
1728 return 0.0;
1729 }
1730 let mean_ns = sorted.iter().map(|&(v, w)| v as f64 * w).sum::<f64>() / total_w;
1731 if mean_ns > 0.0 {
1732 let variance = sorted
1733 .iter()
1734 .map(|&(v, w)| w * (v as f64 - mean_ns).powi(2))
1735 .sum::<f64>()
1736 / total_w;
1737 variance.sqrt() / mean_ns
1738 } else {
1739 0.0
1740 }
1741 }
1742 SampleReduction::Mean => {
1743 let total_w: f64 = sorted.iter().map(|&(_, w)| w).sum();
1744 if total_w <= 0.0 {
1745 return 0.0;
1746 }
1747 sorted.iter().map(|&(v, w)| v as f64 * w).sum::<f64>() / total_w / 1000.0
1748 }
1749 // Max value present, weight-invariant — last element of the value-sorted pool.
1750 SampleReduction::Worst => sorted.last().map(|&(v, _)| v).unwrap_or(0) as f64 / 1000.0,
1751 }
1752}
1753
1754/// One cgroup's surviving [`CgroupStats`] reduction for a
1755/// [`crate::stats::MetricKind::Distribution`] (source, reduction) pair — the
1756/// value folded worst-wins into the run-level distribution for a cgroup whose
1757/// raw samples are NOT in the pool (a backdrop, or a stripped/empty carrier).
1758/// Worst-wins is `f64::max` (every Distribution metric is `LowerBetter`,
1759/// enforced by `every_metric_has_kind_consistent_with_naming`).
1760///
1761/// Per-source match, EXHAUSTIVE over SampleReduction (no `_` catch-all,
1762/// mirroring reduce_sorted_distribution) so a new SampleSource or
1763/// SampleReduction variant fails the build until a reduction field is wired.
1764/// The cross-source reductions (a wake source asking for a run-delay reduction,
1765/// or vice versa) are registry-impossible (no CgroupStats field exists), so
1766/// they debug_assert in tests and, in release, return `f64::NAN` rather than
1767/// 0.0 — NaN is IGNORED by the caller's `f64::max` worst-wins fold, and if it
1768/// still reaches `populate_run_distribution_metrics`'s insert (a pool-less
1769/// Distribution whose every carrier-less cgroup hits this arm) the is_finite
1770/// insert guard drops it to absence. Either way a registry-authoring mistake
1771/// drops the bogus contribution instead of folding a 0.0 that a LowerBetter
1772/// metric would read as "perfect".
1773fn distribution_cgroup_reduction(
1774 cg: &CgroupStats,
1775 source: crate::stats::SampleSource,
1776 reduction: crate::stats::SampleReduction,
1777) -> f64 {
1778 use crate::stats::{SampleReduction, SampleSource};
1779 match source {
1780 SampleSource::WakeLatencyNs => match reduction {
1781 SampleReduction::P99 => cg.p99_wake_latency_us,
1782 SampleReduction::Median => cg.median_wake_latency_us,
1783 SampleReduction::Cv => cg.wake_latency_cv,
1784 SampleReduction::P999 | SampleReduction::Mean | SampleReduction::Worst => {
1785 debug_assert!(false, "no CgroupStats wake reduction for {reduction:?}");
1786 f64::NAN
1787 }
1788 },
1789 SampleSource::RunDelayNs => match reduction {
1790 SampleReduction::Mean => cg.mean_run_delay_us,
1791 SampleReduction::Worst => cg.worst_run_delay_us,
1792 SampleReduction::P99
1793 | SampleReduction::P999
1794 | SampleReduction::Median
1795 | SampleReduction::Cv => {
1796 debug_assert!(
1797 false,
1798 "no CgroupStats run-delay reduction for {reduction:?}"
1799 );
1800 f64::NAN
1801 }
1802 },
1803 SampleSource::TimerLatencyNs => match reduction {
1804 SampleReduction::Median => cg.median_timer_latency_us,
1805 SampleReduction::P99 => cg.p99_timer_latency_us,
1806 SampleReduction::P999 => cg.p999_timer_latency_us,
1807 SampleReduction::Worst => cg.worst_timer_latency_us,
1808 SampleReduction::Cv | SampleReduction::Mean => {
1809 debug_assert!(false, "no CgroupStats timer reduction for {reduction:?}");
1810 f64::NAN
1811 }
1812 },
1813 }
1814}
1815
1816/// Populate cross-RUN aggregate entries for every registered
1817/// `crate::stats::MetricDef` whose `read_sample` returns finite
1818/// values across the entire sample series. Writes into
1819/// `target` (typically `ScenarioStats::ext_metrics`) under the
1820/// metric's registry name — the same key the per-phase
1821/// [`PhaseBucket::metrics`] uses, so cross-RUN and per-phase
1822/// consumers reference the same name.
1823///
1824/// Existing keys are NOT overwritten — a typed GauntletRow field's
1825/// value (populated via the MetricDef accessor at sidecar-write
1826/// time) wins on the read path, and this fn fills the gap for
1827/// registered metrics that have a `read_sample` wire but no typed
1828/// GauntletRow field. Without this fill, `cargo ktstr perf-delta`
1829/// silently skips the metric (read returns None on both sides, so the
1830/// `(None, None)` arm drops the pair).
1831///
1832/// Per-phase reduction dispatch is described on [`PhaseBucket`];
1833/// the cross-RUN fold here uses `crate::stats::aggregate_samples_for_phase`
1834/// over the full sample series, with TYPED_FIELD_NAMES gating to
1835/// avoid duplicating typed-accessor sources.
1836pub fn populate_run_ext_metrics(
1837 samples: &crate::scenario::sample::SampleSeries,
1838 target: &mut std::collections::BTreeMap<String, f64>,
1839) {
1840 // Typed-backed keys are skipped via the module-level TYPED_FIELD_NAMES
1841 // (shared with populate_run_ext_metrics_from_phases) so only
1842 // ext-metrics-only registry entries are populated here.
1843 for metric_def in crate::stats::METRICS {
1844 if target.contains_key(metric_def.name) {
1845 continue;
1846 }
1847 if TYPED_FIELD_NAMES.contains(&metric_def.name) {
1848 continue;
1849 }
1850 let readings: Vec<f64> = samples
1851 .iter_samples()
1852 .filter_map(|s| metric_def.read_sample(&s))
1853 .collect();
1854 if readings.is_empty() {
1855 continue;
1856 }
1857 if let Some(reduced) = crate::stats::aggregate_samples_for_phase(metric_def, &readings) {
1858 target.insert(metric_def.name.to_string(), reduced);
1859 }
1860 }
1861 // Run-level delivered-guest-CPU denominator for IRQ activity/fraction
1862 // metrics. Use the same earliest/latest per_cpu_time-bearing freezes as the
1863 // whole-run counter deltas above. Excluding CPUTIME_STEAL makes the rates
1864 // immune to host preemption without adding another observation source.
1865 if target.contains_key("total_hardirqs") {
1866 let placed: Vec<_> = samples
1867 .iter_samples()
1868 .filter(|s| !s.snapshot.per_cpu_time().is_empty())
1869 .filter_map(|s| s.elapsed_ms.map(|elapsed_ms| (s, elapsed_ms)))
1870 .collect();
1871 if let (Some((first, first_ms)), Some((last, last_ms))) = (
1872 placed.iter().min_by_key(|(_, elapsed_ms)| *elapsed_ms),
1873 placed.iter().max_by_key(|(_, elapsed_ms)| *elapsed_ms),
1874 ) && last_ms > first_ms
1875 && let Some(cpu_ns) = crate::assert::phase_build::guest_cpu_time_delta_ns(
1876 first.snapshot.per_cpu_time(),
1877 last.snapshot.per_cpu_time(),
1878 )
1879 {
1880 target
1881 .entry("total_phase_guest_cpu_ns".to_string())
1882 .or_insert(cpu_ns as f64);
1883 target
1884 .entry("total_phase_guest_cpu_sec".to_string())
1885 .or_insert(cpu_ns as f64 / 1e9);
1886 }
1887 }
1888 // Re-derive Rate metrics from the read_sample components just folded
1889 // in. populate_run_ext_metrics is pub and called standalone (tests,
1890 // and not only ahead of populate_run_ext_metrics_from_phases), so it
1891 // derives its own rates to stay self-contained.
1892 crate::stats::derive_rate_metrics(target);
1893}
1894
1895/// Derive the per-phase scalar metrics ([`crate::stats::MetricKind::PerPhase`])
1896/// for every cgroup carrier in each phase. Two families:
1897///
1898/// 1. NON-schbench carrier scalars (via [`write_carrier_scalars`], for EVERY
1899/// cgroup regardless of work type): the wake/run-delay/off-cpu distributions
1900/// + the migration/iterations/locality ratios + the carrier counters,
1901/// written ONLY into each carrier's `PhaseCgroupStats::metrics` (per-cgroup,
1902/// read via `phase_cgroup_metric`) — NO pooled [`PhaseBucket::metrics`]
1903/// entry; their run-level aggregate is the `worst_*` ext-metrics key.
1904/// 2. schbench scalars (via [`write_schbench_scalars`], only for cgroups
1905/// carrying a `SchbenchPhaseStats`): written per-cgroup into
1906/// `PhaseCgroupStats::metrics` AND, pooled across the phase's schbench
1907/// carriers, into [`PhaseBucket::metrics`] (read via `phase_metric`).
1908/// Percentiles MUST come from the POOLED histogram (combine the latency
1909/// histograms + integer-add the run-delay raw pairs), never an average of
1910/// per-cgroup percentiles; the run-delay means are ABSENT when `pcount == 0`
1911/// so a never-scheduled class is not a false `0`.
1912///
1913/// MUST run POST-merge: the buckets are final and keyed by `step_index`, so a
1914/// per-phase A/B claim reads the value via `phase_metric` /
1915/// `phase_cgroup_metric`. It runs after the per-cgroup carriers are folded
1916/// ([`fold_guest_per_cgroup_into_host_buckets`]) and after any
1917/// [`merge_matched_phase_buckets`] — both SKIP is_derived keys (the `continue`
1918/// in the merge loop), so deriving earlier would DROP the keys. Idempotent
1919/// (overwrites the keys), so callers that rebuild buckets per call (e.g.
1920/// `VmResult::phase_buckets`) may re-run it freely.
1921// doc_lazy_continuation: pre-existing numbered-list wording surfaced by the clippy
1922// 1.94 bump; renders fine. Suppress rather than reflow the prose.
1923#[allow(clippy::doc_lazy_continuation)]
1924pub(crate) fn derive_phase_metrics(phases: &mut [PhaseBucket]) {
1925 use crate::workload::schbench::run::SchbenchPhaseStats;
1926
1927 for bucket in phases.iter_mut() {
1928 // Derive the schbench scalars BOTH per cgroup (into pc.metrics — the
1929 // per-cgroup set: N cgroups -> N queryable sets) AND pooled across the
1930 // phase's cgroups (into bucket.metrics — the aggregate). Both go through the
1931 // SAME reducer (write_schbench_scalars) from the SAME carriers, so the
1932 // pooled set is the cross-cgroup re-pool of the per-cgroup carriers (a
1933 // percentile re-derives from the pooled histogram via PlatStats::combine =
1934 // bucket-count add, never averaged), and pooled == cross-cgroup re-pool.
1935 let mut pooled: Option<SchbenchPhaseStats> = None;
1936 let mut pooled_taobench: Option<crate::workload::taobench::run::TaobenchPhaseStats> = None;
1937 for pc in bucket.per_cgroup.values_mut() {
1938 // Non-schbench carrier-derived metrics (wake/run-delay/off-cpu
1939 // distributions + the migration/iterations/locality ratios + the
1940 // carrier counters), emitted for EVERY cgroup regardless of work
1941 // type. Runs BEFORE the schbench block: `write_carrier_scalars`
1942 // takes `&mut pc` (its summaries read many `pc` fields, then it
1943 // writes `pc.metrics`), so it cannot share the iteration with the
1944 // disjoint-field borrow the schbench block relies on.
1945 write_carrier_scalars(pc);
1946 // Disjoint field borrows: `s` reads pc.schbench, the reducer writes
1947 // pc.metrics — different fields, so the immutable + mutable borrows of
1948 // `*pc` coexist.
1949 if let Some(s) = pc.schbench.as_ref() {
1950 write_schbench_scalars(s, &mut pc.metrics);
1951 // `take()` completes the borrow before the reassignment in both arms.
1952 match pooled.take() {
1953 Some(mut acc) => {
1954 acc.merge(s);
1955 pooled = Some(acc);
1956 }
1957 None => pooled = Some(s.clone()),
1958 }
1959 }
1960 // taobench per-phase: same disjoint-field-borrow pattern (`t` reads
1961 // pc.taobench, the reducer writes pc.metrics) + pool across cgroups.
1962 if let Some(t) = pc.taobench.as_ref() {
1963 write_taobench_scalars(t, &mut pc.metrics);
1964 match pooled_taobench.take() {
1965 Some(mut acc) => {
1966 acc.merge(t);
1967 pooled_taobench = Some(acc);
1968 }
1969 None => pooled_taobench = Some(t.clone()),
1970 }
1971 }
1972 }
1973 if let Some(p) = pooled {
1974 write_schbench_scalars(&p, &mut bucket.metrics);
1975 }
1976 if let Some(t) = pooled_taobench {
1977 write_taobench_scalars(&t, &mut bucket.metrics);
1978 }
1979
1980 // Cross-cgroup phase pool for the canonical iteration_rate. Exclude a
1981 // carrier with no measured CPU time from BOTH sums: crediting its work
1982 // against another cgroup's CPU time would overstate efficiency.
1983 let total_cpu_time_ns = bucket
1984 .per_cgroup
1985 .values()
1986 .filter(|pc| pc.total_cpu_time_ns > 0)
1987 .map(|pc| pc.total_cpu_time_ns)
1988 .fold(0u64, u64::saturating_add);
1989 if total_cpu_time_ns > 0 {
1990 let total_iterations = bucket
1991 .per_cgroup
1992 .values()
1993 .filter(|pc| pc.total_cpu_time_ns > 0)
1994 .map(|pc| pc.total_iterations)
1995 .fold(0u64, u64::saturating_add);
1996 bucket.metrics.insert(
1997 "total_iterations_pooled".to_string(),
1998 total_iterations as f64,
1999 );
2000 bucket.metrics.insert(
2001 "total_cpu_time_sec".to_string(),
2002 total_cpu_time_ns as f64 / 1e9,
2003 );
2004 crate::stats::derive_rate_metrics(&mut bucket.metrics);
2005 }
2006 }
2007}
2008
2009/// Write the per-phase per-cgroup NON-schbench DERIVED scalars from ONE
2010/// carrier into `pc.metrics`, keyed by registry [`crate::stats::MetricDef`]
2011/// name — the per-cgroup analog of the run-level reductions, single-sourced
2012/// through the shared ratio helpers in [`crate::assert::reductions`] (so the
2013/// per-cgroup value and the run-level `worst_*` fold agree by construction)
2014/// and the carrier summary methods. Each gate mirrors the carrier's ABSENT
2015/// discipline: an empty wake pool, a not-measured off-CPU state, a
2016/// zero-worker cgroup, or a no-NUMA-pages cgroup emits no key (reads as
2017/// missing, never a false 0); `migration_ratio` is ALWAYS present (a measured
2018/// 0.0 when no iterations ran).
2019///
2020/// Unlike the schbench families, these families have NO pooled
2021/// `PhaseBucket::metrics` entry at all: as PerPhase / is_derived metrics whose
2022/// `read_sample` is `None`, the per-phase fold never writes them to
2023/// `bucket.metrics`. Their run-level aggregate is the `worst_*` key
2024/// `populate_run_distribution_metrics` writes into the run-level
2025/// `ScenarioStats::ext_metrics` map; per-phase they are queryable ONLY per-cgroup
2026/// via `pc.metrics`. So this writes ONLY `pc.metrics`, never `bucket.metrics` —
2027/// the per-cgroup carrier value is authoritative for the per-cgroup question.
2028///
2029/// Takes `&mut PhaseCgroupStats` (not `(&pc, &mut pc.metrics)`): it reads many
2030/// `pc` fields AND writes `pc.metrics`, so the summaries/ratios are bound to
2031/// locals from `&*pc` first, then the `&mut pc.metrics` borrow is taken once.
2032fn write_carrier_scalars(pc: &mut PhaseCgroupStats) {
2033 use crate::assert::reductions::{
2034 cross_node_migration_ratio_of, iterations_per_cpu_sec_of, iterations_per_worker_of,
2035 migration_ratio_of, page_locality_of,
2036 };
2037 // Read everything from &*pc into locals BEFORE the &mut pc.metrics borrow.
2038 let wake = pc.wake_summary().zip(pc.wake_cv());
2039 let timer = pc.timer_summary();
2040 let run_delay = pc.run_delay_summary();
2041 let off_cpu = pc.off_cpu_summary();
2042 let migration_ratio = migration_ratio_of(pc.total_migrations, pc.total_iterations);
2043 let ipw = iterations_per_worker_of(pc.num_workers, pc.total_iterations);
2044 let ipcs = iterations_per_cpu_sec_of(pc.num_workers, pc.total_cpu_time_ns, pc.total_iterations);
2045 let numa = if pc.numa_pages_total > 0 {
2046 Some((
2047 page_locality_of(pc.numa_pages_local, pc.numa_pages_total),
2048 cross_node_migration_ratio_of(pc.cross_node_migrated, pc.numa_pages_total),
2049 ))
2050 } else {
2051 None
2052 };
2053
2054 let m = &mut pc.metrics;
2055 // WAKE — all three or none (wake_summary + wake_cv share the empty-pool gate).
2056 if let Some(((p99, median), cv)) = wake {
2057 m.insert("p99_wake_latency_us".to_string(), p99);
2058 m.insert("median_wake_latency_us".to_string(), median);
2059 m.insert("wake_latency_cv".to_string(), cv);
2060 }
2061 // TIMER — all three or none (timer_summary shares the empty-pool gate).
2062 if let Some((median, p99, p999)) = timer {
2063 m.insert("median_timer_latency_us".to_string(), median);
2064 m.insert("p99_timer_latency_us".to_string(), p99);
2065 m.insert("p999_timer_latency_us".to_string(), p999);
2066 }
2067 // RUN-DELAY — both or none.
2068 if let Some((mean, worst)) = run_delay {
2069 m.insert("mean_run_delay_us".to_string(), mean);
2070 m.insert("max_run_delay_us".to_string(), worst);
2071 }
2072 // OFF-CPU — four or none (None == not-measured).
2073 if let Some((avg, min, max, spread)) = off_cpu {
2074 m.insert("avg_off_cpu_pct".to_string(), avg);
2075 m.insert("min_off_cpu_pct".to_string(), min);
2076 m.insert("max_off_cpu_pct".to_string(), max);
2077 m.insert("off_cpu_spread_pct".to_string(), spread);
2078 }
2079 // RATIOS — migration_ratio ALWAYS (measured 0.0 when no iterations ran).
2080 m.insert("migration_ratio".to_string(), migration_ratio);
2081 if let Some(v) = ipw {
2082 m.insert("iterations_per_worker".to_string(), v);
2083 }
2084 if let Some(v) = ipcs {
2085 m.insert("iteration_rate".to_string(), v);
2086 }
2087 // NUMA ratios only when pages were observed (mirrors the carrier's
2088 // numa_pages_total>0 gate; absent is more honest than a 0.0 default).
2089 if let Some((locality, cross_node)) = numa {
2090 m.insert("page_locality".to_string(), locality);
2091 m.insert("cross_node_migration_ratio".to_string(), cross_node);
2092 }
2093}
2094
2095/// Write the per-phase schbench scalar metrics derived from ONE
2096/// `SchbenchPhaseStats` (a single cgroup's carrier, or the cross-cgroup pool)
2097/// into `out`, keyed by registry [`crate::stats::MetricDef`] name. The sole
2098/// producer of these keys for both the per-cgroup ([`PhaseCgroupStats::metrics`])
2099/// and pooled ([`PhaseBucket::metrics`]) maps — one derivation, no duplicated
2100/// percentile / min/max / rps / sched-delay math. Each gate mirrors the carrier's
2101/// ABSENT discipline: an empty histogram or a zero pcount emits no key (reads as
2102/// missing, never a false 0); loop_count is always present (0 = a real measured
2103/// no-cycles outcome).
2104fn write_schbench_scalars(
2105 p: &crate::workload::schbench::run::SchbenchPhaseStats,
2106 out: &mut std::collections::BTreeMap<String, f64>,
2107) {
2108 use crate::stats::{
2109 SCHBENCH_LOOP_COUNT, SCHBENCH_REQUEST_MAX_US, SCHBENCH_REQUEST_MIN_US,
2110 SCHBENCH_REQUEST_P50_US, SCHBENCH_REQUEST_P90_US, SCHBENCH_REQUEST_P99_US,
2111 SCHBENCH_REQUEST_P999_US, SCHBENCH_RPS_MAX, SCHBENCH_RPS_MIN, SCHBENCH_RPS_P20,
2112 SCHBENCH_RPS_P50, SCHBENCH_RPS_P90, SCHBENCH_SCHED_DELAY_MSG_US,
2113 SCHBENCH_SCHED_DELAY_WORKER_US, SCHBENCH_WAKEUP_MAX_US, SCHBENCH_WAKEUP_MIN_US,
2114 SCHBENCH_WAKEUP_P50_US, SCHBENCH_WAKEUP_P90_US, SCHBENCH_WAKEUP_P99_US,
2115 SCHBENCH_WAKEUP_P999_US,
2116 };
2117 use crate::workload::schbench::plat::Pct;
2118
2119 if p.wakeup.sample_count() > 0 {
2120 let q = p.wakeup.percentiles();
2121 out.insert(
2122 SCHBENCH_WAKEUP_P50_US.to_string(),
2123 q.value_at(Pct::P50) as f64,
2124 );
2125 out.insert(
2126 SCHBENCH_WAKEUP_P90_US.to_string(),
2127 q.value_at(Pct::P90) as f64,
2128 );
2129 out.insert(
2130 SCHBENCH_WAKEUP_P99_US.to_string(),
2131 q.value_at(Pct::P99) as f64,
2132 );
2133 out.insert(
2134 SCHBENCH_WAKEUP_P999_US.to_string(),
2135 q.value_at(Pct::P999) as f64,
2136 );
2137 out.insert(SCHBENCH_WAKEUP_MIN_US.to_string(), q.min as f64);
2138 out.insert(SCHBENCH_WAKEUP_MAX_US.to_string(), q.max as f64);
2139 }
2140 if p.request.sample_count() > 0 {
2141 let q = p.request.percentiles();
2142 out.insert(
2143 SCHBENCH_REQUEST_P50_US.to_string(),
2144 q.value_at(Pct::P50) as f64,
2145 );
2146 out.insert(
2147 SCHBENCH_REQUEST_P90_US.to_string(),
2148 q.value_at(Pct::P90) as f64,
2149 );
2150 out.insert(
2151 SCHBENCH_REQUEST_P99_US.to_string(),
2152 q.value_at(Pct::P99) as f64,
2153 );
2154 out.insert(
2155 SCHBENCH_REQUEST_P999_US.to_string(),
2156 q.value_at(Pct::P999) as f64,
2157 );
2158 out.insert(SCHBENCH_REQUEST_MIN_US.to_string(), q.min as f64);
2159 out.insert(SCHBENCH_REQUEST_MAX_US.to_string(), q.max as f64);
2160 }
2161 // Per-phase achieved-RPS distribution (the control thread's per-second samples
2162 // attributed to this epoch). Gated on sample_count()>0 so a phase shorter than
2163 // the ~1s control cadence reads ABSENT, never rps=0. schbench's RPS table is
2164 // PLIST_FOR_RPS = 20/50/90 (schbench.c:130) + min/max.
2165 if p.rps.sample_count() > 0 {
2166 let r = p.rps.percentiles();
2167 out.insert(SCHBENCH_RPS_P20.to_string(), r.value_at(Pct::P20) as f64);
2168 out.insert(SCHBENCH_RPS_P50.to_string(), r.value_at(Pct::P50) as f64);
2169 out.insert(SCHBENCH_RPS_P90.to_string(), r.value_at(Pct::P90) as f64);
2170 out.insert(SCHBENCH_RPS_MIN.to_string(), r.min as f64);
2171 out.insert(SCHBENCH_RPS_MAX.to_string(), r.max as f64);
2172 }
2173 // Sample-weighted run-delay mean (Σrun_delay / Σpcount), ns→µs; ABSENT when
2174 // pcount==0 (a never-scheduled class) so it reads as missing, not 0.
2175 if p.msg_pcount > 0 {
2176 let mean_us = p.msg_run_delay_ns as f64 / p.msg_pcount as f64 / 1000.0;
2177 out.insert(SCHBENCH_SCHED_DELAY_MSG_US.to_string(), mean_us);
2178 }
2179 if p.worker_pcount > 0 {
2180 let mean_us = p.worker_run_delay_ns as f64 / p.worker_pcount as f64 / 1000.0;
2181 out.insert(SCHBENCH_SCHED_DELAY_WORKER_US.to_string(), mean_us);
2182 }
2183 // loop_count is always present for a schbench carrier: 0 = no cycles ran (a real
2184 // measured value; HigherBetter → worst), distinct from a non-schbench carrier
2185 // which has no schbench data at all (the caller skips it).
2186 out.insert(SCHBENCH_LOOP_COUNT.to_string(), p.loop_count as f64);
2187 // CPU-second-denominated throughput: completed cycles per CPU-second the
2188 // phase's WORKER threads actually received (Σ CLOCK_THREAD_CPUTIME_ID
2189 // deltas — message-thread CPU excluded, mirroring schbench's Σ worker
2190 // runtimes). Dilation-safe by construction; the wall picture reconstructs
2191 // as `rate / D` via the sidecar's `host_dilation`. ABSENT when no worker
2192 // CPU was measured (pre-field carrier, or a phase in which no worker ran).
2193 // The `schbench_rps_*` per-second samples above deliberately stay WALL
2194 // (upstream `rps_stats` parity — see the `SchbenchPhaseStats::rps` doc).
2195 if p.worker_cpu_ns > 0 {
2196 out.insert(
2197 crate::stats::SCHBENCH_LOOPS_PER_CPU_SEC.to_string(),
2198 p.loop_count as f64 / (p.worker_cpu_ns as f64 / 1e9),
2199 );
2200 }
2201}
2202
2203/// Write the per-phase taobench scalar metrics derived from ONE
2204/// `TaobenchStats` (a single cgroup's carrier, or the cross-cgroup pool)
2205/// into `out`, keyed by registry [`crate::stats::MetricDef`] name. The sole
2206/// producer of these keys for both the per-cgroup ([`PhaseCgroupStats::metrics`])
2207/// and pooled ([`PhaseBucket::metrics`]) maps. ABSENT discipline: the qps keys
2208/// only when client CPU time was measured (`cpu_time_ns > 0`); hit_ratio only
2209/// when ops completed (`total > 0`); hit_rate only when lookups were issued
2210/// (`get_cmds > 0`) — a not-measured value reads as missing, never a false 0.
2211///
2212/// DENOMINATION: the `taobench_*_ops_per_cpu_sec` keys are CPU-SECOND denominated —
2213/// completed ops per CPU-second the phase's client threads actually received
2214/// (Σ `CLOCK_THREAD_CPUTIME_ID`), not per wall second. Dilation-safe by
2215/// construction; the wall picture reconstructs as `rate / D` via the
2216/// sidecar's `host_dilation` (or exactly from the carrier's ops + wall
2217/// window).
2218fn write_taobench_scalars(
2219 p: &crate::workload::taobench::run::TaobenchPhaseStats,
2220 out: &mut std::collections::BTreeMap<String, f64>,
2221) {
2222 use crate::stats::{
2223 TAOBENCH_FAST_OPS_PER_CPU_SEC, TAOBENCH_HIT_RATE, TAOBENCH_HIT_RATIO,
2224 TAOBENCH_SLOW_OPS_PER_CPU_SEC, TAOBENCH_TOTAL_OPS_PER_CPU_SEC,
2225 };
2226 let c = &p.counters;
2227 let total = c.total_ops();
2228 if c.cpu_time_ns > 0 {
2229 let cpu_secs = c.cpu_time_ns as f64 / 1e9;
2230 out.insert(
2231 TAOBENCH_TOTAL_OPS_PER_CPU_SEC.to_string(),
2232 total as f64 / cpu_secs,
2233 );
2234 out.insert(
2235 TAOBENCH_FAST_OPS_PER_CPU_SEC.to_string(),
2236 c.fast_ops as f64 / cpu_secs,
2237 );
2238 out.insert(
2239 TAOBENCH_SLOW_OPS_PER_CPU_SEC.to_string(),
2240 c.slow_ops as f64 / cpu_secs,
2241 );
2242 }
2243 if total > 0 {
2244 out.insert(
2245 TAOBENCH_HIT_RATIO.to_string(),
2246 c.fast_ops as f64 / total as f64,
2247 );
2248 }
2249 if c.get_cmds > 0 {
2250 out.insert(
2251 TAOBENCH_HIT_RATE.to_string(),
2252 1.0 - (c.get_misses as f64 / c.get_cmds as f64),
2253 );
2254 }
2255 // Per-phase serve-latency percentiles (open-loop only): the µs distribution
2256 // pooled across this phase's cgroups, re-derived over the union histogram.
2257 // Absent when no serve samples (closed loop, or a stream with no completions).
2258 write_taobench_serve_scalars(&p.serve_lat, out);
2259}
2260
2261/// Write the per-phase taobench serve-latency percentile scalars from a pooled
2262/// `PlatStats` into `out` (registry keys), gated on `sample_count() > 0` so a
2263/// closed-loop / no-sample carrier emits nothing (absent, never a false 0).
2264fn write_taobench_serve_scalars(
2265 serve: &crate::workload::schbench::plat::PlatStats,
2266 out: &mut std::collections::BTreeMap<String, f64>,
2267) {
2268 use crate::stats::{
2269 TAOBENCH_SERVE_MAX_US, TAOBENCH_SERVE_MIN_US, TAOBENCH_SERVE_P50_US, TAOBENCH_SERVE_P90_US,
2270 TAOBENCH_SERVE_P99_US, TAOBENCH_SERVE_P999_US,
2271 };
2272 use crate::workload::schbench::plat::Pct;
2273 if serve.sample_count() == 0 {
2274 return;
2275 }
2276 let q = serve.percentiles();
2277 out.insert(
2278 TAOBENCH_SERVE_P50_US.to_string(),
2279 q.value_at(Pct::P50) as f64,
2280 );
2281 out.insert(
2282 TAOBENCH_SERVE_P90_US.to_string(),
2283 q.value_at(Pct::P90) as f64,
2284 );
2285 out.insert(
2286 TAOBENCH_SERVE_P99_US.to_string(),
2287 q.value_at(Pct::P99) as f64,
2288 );
2289 out.insert(
2290 TAOBENCH_SERVE_P999_US.to_string(),
2291 q.value_at(Pct::P999) as f64,
2292 );
2293 out.insert(TAOBENCH_SERVE_MIN_US.to_string(), q.min as f64);
2294 out.insert(TAOBENCH_SERVE_MAX_US.to_string(), q.max as f64);
2295}