ktstr/assert/stats_types.rs
1use super::*;
2
3/// Verdict for a single test scenario.
4///
5/// # Reading the verdict
6///
7/// Inspect the terminal verdict via [`Self::outcome`] (returns the
8/// folded [`Outcome`] enum) or the convenience accessors
9/// [`Self::is_pass`] / [`Self::is_fail`] / [`Self::is_inconclusive`] /
10/// [`Self::is_skip`]. Iterate the per-variant payloads via
11/// [`Self::failure_details`] (all [`Outcome::Fail`] payloads),
12/// [`Self::inconclusive_details`] (all [`Outcome::Inconclusive`]
13/// payloads), and [`Self::skip_details`] (all [`Outcome::Skip`]
14/// payloads). All four bool accessors mirror
15/// [`Outcome::is_pass`] / [`Outcome::is_fail`] /
16/// [`Outcome::is_inconclusive`] / [`Outcome::is_skip`].
17///
18/// # Recording outcomes
19///
20/// Producers use the atomic mutators [`Self::record_fail`] /
21/// [`Self::record_skip`] / [`Self::record_inconclusive`] /
22/// [`Self::record_pass`] (each pushes a single [`Outcome`] variant
23/// onto [`Self::outcomes`]) and the escape hatch
24/// [`Self::record_outcome`] for pre-folded values. Constructors
25/// [`Self::pass`] / [`Self::skip`] / [`Self::fail`] seed the
26/// outcomes vec with the corresponding variant; [`Self::pass`] is
27/// zero-allocation (empty vec; the Pass identity element).
28///
29/// **Wire-format stability**: this struct is postcard-serialized as
30/// part of the in-VM `MSG_TYPE_TEST_RESULT` payload and as
31/// sidecar artifacts under `~/.cache/ktstr`. The wire format is
32/// **not stable across crate versions** — pre-1.0, fields can be
33/// added, removed, or reshaped at any time, and old sidecars must
34/// be regenerated after upgrades (re-running the affected tests
35/// produces a fresh sidecar). Per the project's pre-1.0 no-compat
36/// stance, no `#[serde(default)]` shims are added for old payloads.
37#[must_use = "test verdict is lost if not checked"]
38#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
39pub struct AssertResult {
40 /// Recorded terminal verdicts in emission order, one entry per
41 /// check that explicitly called [`Self::record_pass`],
42 /// [`Self::record_skip`], [`Self::record_inconclusive`], or
43 /// [`Self::record_fail`] (plus the single entry seeded by
44 /// [`Self::skip`] / [`Self::fail`] constructors).
45 ///
46 /// **Empty `outcomes` is the Pass identity** — [`Self::pass`]
47 /// constructs with `outcomes: vec![]`, and [`Self::outcome`]
48 /// resolves an empty `outcomes` to [`Outcome::Pass`] via the
49 /// ordered `Fail > Inconclusive > Pass > Skip` precedence (NOT a
50 /// fold via [`Outcome::merge`]; see [`Self::outcome`]), so a
51 /// never-touched accumulator naturally resolves to Pass without
52 /// any allocation. `record_pass()` is
53 /// for the rare case where a test explicitly records a passing
54 /// check (e.g. per-check helpers that document what passed);
55 /// `pass()` is the zero-state "nothing failed so far"
56 /// constructor.
57 ///
58 /// The folded terminal verdict is computed by [`Self::outcome`]
59 /// per the precedence `Fail > Inconclusive > Pass > Skip`. Use
60 /// [`Self::is_pass`] / [`Self::is_fail`] /
61 /// [`Self::is_inconclusive`] / [`Self::is_skip`] for bool
62 /// checks; use [`Self::failure_details`] /
63 /// [`Self::inconclusive_details`] / [`Self::skip_details`] to
64 /// iterate the per-variant [`AssertDetail`] payloads.
65 pub outcomes: Vec<Outcome>,
66 /// Structured records of every passing claim. Counterpart to
67 /// [`Self::outcomes`]: where `outcomes` carries terminal-verdict
68 /// records (Fail/Skip/Pass per-check), `passes` carries the
69 /// positive confirmations every comparator's pass arm emits via
70 /// [`Verdict`]'s `record_pass_unary` / `record_pass_binary`
71 /// helpers.
72 /// Empty in tests that don't exercise the structured-pass path
73 /// (the no-claim base case), populated whenever a [`Verdict`]
74 /// records claims. The auto-repro renderer iterates both vecs
75 /// to compose the bracketed phase-grouped output that surfaces
76 /// passing context alongside failing assertions.
77 ///
78 /// **Bounded by [`MAX_RECORDED_PASSES`]** — past that count,
79 /// further pushes drop on the floor and a single sentinel
80 /// record named [`PASSES_TRUNCATION_SENTINEL_NAME`] appears at
81 /// the tail. Use the sentinel-name check (not `len()`
82 /// arithmetic) to detect truncation.
83 ///
84 /// **Test-author convention**: do NOT pin `result.passes` shape
85 /// or contents in test assertions unless the test exists
86 /// specifically to verify the structured-pass surface (e.g.
87 /// the auto-repro renderer's own coverage tests). The field
88 /// exists for the renderer's consumption; pinning it
89 /// elsewhere makes the test surface viral — every new
90 /// comparator that fires under the test starts churning the
91 /// pin. Pin `outcome()`, `failure_details()`, and `measurements` for
92 /// scenario verification.
93 pub passes: Vec<PassDetail>,
94 /// Aggregated stats from all workers in this scenario.
95 pub stats: ScenarioStats,
96 /// Structured measurements attached via [`Self::note_value`] /
97 /// [`Verdict::note_value`]. Distinct from [`Self::outcomes`] —
98 /// outcomes carry typed verdict variants with `AssertDetail`
99 /// payloads for operator triage, `measurements` carries typed
100 /// `(key, NoteValue)` pairs for programmatic consumption (sidecar
101 /// parsers, `perf-delta`, regression dashboards).
102 pub measurements: std::collections::BTreeMap<String, NoteValue>,
103 /// Informational annotations attached via [`Self::note`] /
104 /// [`Verdict::note`]. Structurally separated from [`Self::outcomes`]
105 /// so the failure stream stays purely failure-shaped: sidecar
106 /// consumers iterating `details` count real failures without
107 /// the "forgot to filter notes" silent-miscount class of bug
108 /// that the prior `DetailKind::Note` variant on [`AssertDetail`]
109 /// invited. The auto-repro renderer surfaces these alongside the
110 /// failure summary so the operator still sees them on a failing
111 /// run.
112 pub info_notes: Vec<InfoNote>,
113}
114
115/// Per-cgroup statistics from worker telemetry.
116///
117/// # Percentile convention
118///
119/// `p99_wake_latency_us` and `median_wake_latency_us` are computed
120/// by `percentile` using the NEAREST-RANK (Type 1) definition:
121/// the value at `ceil(n * p) - 1` in sorted order. No interpolation
122/// between samples. This matches the percentile convention used
123/// throughout schbench and the BPF latency histograms the project
124/// cross-references, so a `ktstr` p99 reading aligns with a
125/// schbench `lat99` without adjustment. For small `n` (wake
126/// reservoirs cap at `MAX_WAKE_SAMPLES = 100_000` per worker —
127/// see `workload.rs`) nearest-rank is also numerically stable —
128/// interpolation between the two nearest ranks would be
129/// implementation-defined at sample-set boundaries.
130///
131/// # CV pooling scope
132///
133/// `wake_latency_cv` is POOLED across every sample from every
134/// worker in the cgroup, not a per-worker CV averaged back. That
135/// collapses per-worker dispersion into the cgroup-wide signal:
136/// two workers with uniformly low jitter but different means
137/// produce a high pooled CV (mean-shift between workers inflates
138/// stddev), while per-worker CV would show neither worker as
139/// bad. This is intentional for the fairness threshold
140/// (`max_wake_latency_cv`): a scheduler that gives worker A
141/// 10µs wakes and worker B 1ms wakes is failing fairness even if
142/// each worker on its own is tight. Tests comparing single-worker
143/// behavior should scope their assertions to per-worker data
144/// rather than this aggregate.
145///
146/// # Derived ratios
147///
148/// Two metrics are DERIVED rather than measured and live as
149/// `&self` methods, NOT as serde-serialized fields:
150/// [`Self::wake_latency_tail_ratio`] (= p99/median) and
151/// [`Self::iterations_per_worker`] (= total_iterations/num_workers).
152/// Pre-1.0 cleanup eliminated the prior stored-field shadow and
153/// `derive_ratios` stamper. Consumers always recompute on read,
154/// so a hand-constructed fixture or a deserialized sidecar from an
155/// older build cannot silently carry a stale ratio. The run-level
156/// worst-cgroup tail ratio (`crate::stats::MetricKind::WakeLatencyTailRatio`,
157/// an `ext_metrics` entry) and the iterations efficiencies
158/// (`worst_iterations_per_worker` / `worst_iterations_per_cpu_sec`) are all
159/// re-pooled POST-merge by [`populate_run_distribution_metrics`] — the tail
160/// ratio as the max over [`Self::wake_latency_tail_ratio`] across per-cgroup
161/// [`Self`] entries, the efficiencies lowest-wins from
162/// [`Self::iterations_per_worker`] / [`Self::iterations_per_cpu_sec`].
163#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize, crate::Claim)]
164pub struct CgroupStats {
165 /// Cgroup name (the workload-handle label this telemetry belongs to),
166 /// or empty for unlabeled call sites (`collect_all`, bare
167 /// `assert_cgroup`). Set post-hoc by `collect_handles` (in
168 /// `crate::scenario`) where the name is in scope; `cgroup_stats`
169 /// itself has only the reports and leaves it empty. Lets a PASSING-run
170 /// consumer say which cgroup's work landed on which CPUs.
171 pub cgroup_name: String,
172 /// Number of workers in this cgroup.
173 pub num_workers: usize,
174 /// Distinct CPUs the workers in this cgroup actually ran on (union of
175 /// each [`crate::workload::WorkerReport::cpus_used`]). `num_cpus` is
176 /// its length, kept for the existing rollups; this set surfaces WHICH
177 /// CPUs (not just how many) on every run, pass or fail.
178 pub cpus_used: BTreeSet<usize>,
179 /// Distinct CPUs used across all workers in this cgroup
180 /// (`cpus_used.len()`).
181 pub num_cpus: usize,
182 /// Mean off-CPU percentage across workers (off_cpu_ns /
183 /// wall_time_ns * 100). `None` when no worker reported a
184 /// positive `wall_time_ns` (off-CPU% is undefined without wall
185 /// time) — distinct from `Some(0.0)`, a measured "never off
186 /// CPU". The `Option` keeps a not-measured cgroup from reading
187 /// as a perfectly-on-CPU one in the telemetry consumers
188 /// (`ScenarioStats.cgroups`).
189 pub avg_off_cpu_pct: Option<f64>,
190 /// Minimum off-CPU percentage across workers. `None` under the
191 /// same no-measurable-wall-time condition as `avg_off_cpu_pct`.
192 pub min_off_cpu_pct: Option<f64>,
193 /// Maximum off-CPU percentage across workers. `None` under the
194 /// same no-measurable-wall-time condition as `avg_off_cpu_pct`.
195 pub max_off_cpu_pct: Option<f64>,
196 /// `max_off_cpu_pct - min_off_cpu_pct`. Measures scheduling
197 /// fairness within the cgroup. `None` when off-CPU% was not
198 /// measured (no worker with positive wall time) — a not-measured
199 /// cgroup is inconclusive for fairness, NOT "spread 0 = perfectly
200 /// fair". `Some(0.0)` means a real measured zero spread.
201 pub spread: Option<f64>,
202 /// Longest scheduling gap across all workers (ms).
203 pub max_gap_ms: u64,
204 /// CPU where the longest scheduling gap occurred.
205 pub max_gap_cpu: usize,
206 /// Sum of CPU migration counts across all workers.
207 pub total_migrations: u64,
208 /// Migrations per iteration (total_migrations / total_iterations).
209 pub migration_ratio: f64,
210 /// 99th percentile wake latency across all workers (microseconds).
211 pub p99_wake_latency_us: f64,
212 /// Median wake latency across all workers (microseconds).
213 pub median_wake_latency_us: f64,
214 /// Coefficient of variation (stddev / mean) of wake latencies.
215 ///
216 /// Computed over the POOLED latency samples from every worker in
217 /// the cgroup, not as a mean of per-worker CVs. Per-worker
218 /// dispersion is therefore masked: a cgroup with one tight
219 /// worker and one wildly variable worker can report a moderate
220 /// pooled CV that looks healthier than either constituent. Use
221 /// [`WorkerReport::wake_latencies_ns`] directly if per-worker
222 /// CV is needed.
223 pub wake_latency_cv: f64,
224 /// Whether any worker in this cgroup recorded a wake-latency sample.
225 /// `false` makes the wake reductions above (`p99_wake_latency_us`,
226 /// `median_wake_latency_us`, `wake_latency_cv`) a not-measured sentinel
227 /// `0.0` rather than a measured zero — a percentile over zero samples is
228 /// undefined, not "instant wakes". The run-level distributional re-pool
229 /// (`populate_run_distribution_metrics`) reads this to EXCLUDE a
230 /// no-wake-sample cgroup from the cross-run mean instead of folding its
231 /// `0.0` in (which, for the LowerBetter wake metrics, would falsely drag
232 /// the mean toward "perfect"). Same not-measured-vs-measured-zero
233 /// discipline the off-CPU% `Option` fields above carry.
234 pub wake_measured: bool,
235 /// Median timer-latency across all workers (microseconds) — the
236 /// [`crate::workload::WorkType::TimerLatency`] cyclictest probe's per-cgroup
237 /// pooled reduction over
238 /// [`crate::workload::WorkerReport::timer_latencies_ns`]. `0.0` when no
239 /// worker recorded timer samples.
240 pub median_timer_latency_us: f64,
241 /// 99th-percentile timer-latency across all workers (microseconds). See
242 /// [`Self::median_timer_latency_us`].
243 pub p99_timer_latency_us: f64,
244 /// 99.9th-percentile (deep-tail) timer-latency across all workers
245 /// (microseconds). See [`Self::median_timer_latency_us`].
246 pub p999_timer_latency_us: f64,
247 /// Worst (maximum) timer-latency across all workers (microseconds). See
248 /// [`Self::median_timer_latency_us`].
249 pub worst_timer_latency_us: f64,
250 /// Whether any worker in this cgroup recorded a timer-latency sample.
251 /// `false` makes the timer reductions above a not-measured sentinel `0.0`
252 /// (no [`crate::workload::WorkType::TimerLatency`] worker ran), distinct
253 /// from a measured zero. Read by the run-level re-pool to EXCLUDE a
254 /// no-timer-sample cgroup from the cross-run mean. Mirrors
255 /// [`Self::wake_measured`] for the timer carrier.
256 pub timer_measured: bool,
257 /// Sum of iteration counts across all workers.
258 pub total_iterations: u64,
259 /// Sum of per-worker on-CPU time (nanoseconds), from each worker's
260 /// schedstat run time ([`crate::workload::WorkerReport::schedstat_cpu_time_ns`]
261 /// — `task->se.sum_exec_runtime`, the FIRST `/proc/<pid>/schedstat` field
262 /// (`sched_info` supplies only the run_delay/pcount fields 2/3, not the
263 /// on-CPU time), the summable per-thread proxy for the cgroup's
264 /// `cpu.stat usage_usec`).
265 /// Denominator for [`Self::iterations_per_cpu_sec`], the
266 /// overcommit-invariant per-cell rate. `0` when no worker reported on-CPU
267 /// time (the accessor then returns `None`).
268 pub total_cpu_time_ns: u64,
269 /// Mean schedstat run delay across workers (microseconds).
270 pub mean_run_delay_us: f64,
271 /// Worst schedstat run delay across workers (microseconds).
272 pub worst_run_delay_us: f64,
273 /// Whether this cgroup had any worker to measure run-delay from
274 /// (`!run_delays.is_empty()`, i.e. `num_workers > 0`) — `false` only for a
275 /// worker-less cgroup, keeping a degenerate empty cohort from folding a
276 /// sentinel `0.0` into the cross-run run-delay mean. Unlike wake/timer
277 /// (per-sample streams a running worker may never emit), run-delay is one
278 /// `sched_info.run_delay` value per worker, always present once a worker
279 /// exists: a worker that never queued reads a real measured `0.0`, not a
280 /// no-measurement sentinel. `sched_info.run_delay` accumulates whenever
281 /// `CONFIG_SCHED_INFO` is built in (compile-time — forced on in ktstr,
282 /// `select`ed by both `CONFIG_SCHEDSTATS` and `CONFIG_TASK_DELAY_ACCT`),
283 /// with no gate on the runtime `kernel.sched_schedstats` key (that key
284 /// gates only the `schedstat_*` rq/se aggregates, never `run_delay`), so
285 /// run-delay is genuinely measured on every ktstr run and worker-presence
286 /// is the correct measured predicate. Mirrors [`Self::wake_measured`] for
287 /// the run-delay carrier.
288 pub run_delay_measured: bool,
289 /// Fraction of pages on the expected NUMA node(s) (0.0-1.0).
290 /// Derived from `/proc/self/numa_maps` and the worker's
291 /// [`MemPolicy`](crate::workload::MemPolicy).
292 pub page_locality: f64,
293 /// Cross-node page migration ratio from `/proc/vmstat`
294 /// `numa_pages_migrated` delta divided by total allocated pages.
295 pub cross_node_migration_ratio: f64,
296 /// Whole-run taobench engine COUNTER aggregate pooled across this cgroup's
297 /// [`crate::workload::WorkType::Taobench`] workers (Σ ops, MAX wall window —
298 /// the window is shared by concurrent workers, per
299 /// [`crate::workload::WorkerReport::taobench_whole`]). `None` for every
300 /// non-taobench cgroup. A RAW carrier, like [`Self::total_iterations`] /
301 /// [`Self::total_cpu_time_ns`] — not a reduced ratio: the run-level
302 /// cross-cgroup pool [`crate::assert::populate_run_pooled_taobench`] folds it
303 /// into the `total_taobench_*` Counter components and the derived
304 /// `taobench_*_ops_per_cpu_sec_whole` / `taobench_hit_fraction` /
305 /// `taobench_command_hit_rate`
306 /// Rates in [`Self::ext_metrics`] (whole-run keys visible to `--noise-adjust`
307 /// spread, unlike the per-phase `taobench_*_ops_per_cpu_sec` which are
308 /// `MetricKind::PerPhase`). Whole-run, NOT summable from the per-phase
309 /// `PhaseCgroupStats::taobench` carriers (per-phase `elapsed_ns` is
310 /// MAX-merged across concurrent threads, so summing phase windows is the
311 /// wrong throughput denominator), so the engine's authoritative whole-run aggregate
312 /// is shipped from the worker. Holds COUNTERS only
313 /// ([`TaobenchStats`](crate::workload::taobench::run::TaobenchStats)) — the
314 /// serve-latency histogram is per-phase data on `PhaseCgroupStats::taobench`,
315 /// and the whole-run serve distribution (`taobench_serve_*_us_whole`) is the
316 /// union of those per-phase histograms. `pub` (every `CgroupStats` field is
317 /// `pub` and the struct is preluded, so a test author can read the counters).
318 /// `#[claim(skip)]`: a raw aggregate carrier, not a test-author claim
319 /// surface — assertions run against the host-derived run-level `taobench_*`
320 /// Rate / serve-latency metrics, mirroring
321 /// [`crate::workload::WorkerReport::taobench_whole`].
322 #[claim(skip)]
323 pub taobench_whole: Option<crate::workload::taobench::run::TaobenchStats>,
324 /// Extensible metrics for the generic comparison pipeline.
325 pub ext_metrics: BTreeMap<String, f64>,
326}
327
328/// Per-phase per-cgroup raw telemetry components — the per-phase analogue of
329/// [`CgroupStats`]. Holds RAW components (sample vectors + counters), NOT the
330/// reduced ratios/percentiles [`CgroupStats`] computes, so whole-run and
331/// cross-run aggregates RE-POOL from the components at every level (the
332/// per-phase telemetry thesis: an aggregate is recomputed over the pooled
333/// components, never averaged from ready-made per-phase reductions — a
334/// percentile or weighted ratio cannot be recovered from per-phase scalars).
335/// Covers every TYPED [`CgroupStats`] reduction: avg/min/max off-CPU% and
336/// spread from `off_cpu_pcts`; p99/median/CV wake latency from
337/// `wake_latencies_ns`; mean/worst run-delay from `run_delays_ns`;
338/// migration_ratio, iterations_per_cpu_sec, iterations_per_worker,
339/// page_locality, cross_node_migration_ratio from their counter components;
340/// the COUPLED worst gap (ms + the CPU that owned it) from `max_gap_ms` /
341/// `max_gap_cpu`; cpus_used / num_cpus from `cpus_used`. EXCLUDES
342/// [`CgroupStats::ext_metrics`] (the generic extensible map — a per-phase
343/// per-cgroup custom metric is a future extension, not part of the typed
344/// carrier). Lives in [`PhaseBucket::per_cgroup`], keyed by cgroup name. The
345/// structural carrier is empty until a capture path populates it per phase.
346#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize, crate::Claim)]
347pub struct PhaseCgroupStats {
348 /// Worker count in this cgroup for the phase — the denominator for the
349 /// re-pooled per-worker iteration rate (`iterations_per_worker` =
350 /// `total_iterations` / this). This is a set CARDINALITY (`reports.len()`),
351 /// not a kernel counter, but it SUMs in `merge` because a single cgroup name
352 /// can emit MULTIPLE carriers in one step — `collect_handles` builds one per
353 /// `WorkloadHandle`, and a `CgroupDef` with several `WorkSpec` entries
354 /// (`.work(..).work(..)`) spawns one handle per `WorkSpec` under the same
355 /// name (`apply_setup`). Those carriers cover DISJOINT worker subsets, so the
356 /// cardinality of their union is the SUM (4 + 2 → 6), matching [`cgroup_stats`]
357 /// over the pooled reports (`reports.len()`); a MAX would understate the count
358 /// and inflate `iterations_per_worker`. (The disjointness is the real
359 /// justification — were carriers ever to overlap, the SUM would over-count.)
360 pub num_workers: usize,
361 /// Distinct CPUs the cgroup's workers ran on in the phase (union of each
362 /// worker's `cpus_used`). Re-pools [`CgroupStats::cpus_used`] / `num_cpus`
363 /// (= the set / its length) via a set UNION.
364 pub cpus_used: std::collections::BTreeSet<usize>,
365 /// Pooled per-wakeup latency samples (ns) across the cgroup's workers in
366 /// the phase, un-reduced so p99 / median / CV re-pool over the combined set.
367 /// The POOL is reservoir-capped at `MAX_WAKE_SAMPLES` (the per-worker bound,
368 /// re-applied when same-name carriers merge so the carrier payload stays
369 /// bounded on the size-limited guest bulk port — without it the pool would be
370 /// `workers × MAX_WAKE_SAMPLES`); `wake_sample_total` carries the true
371 /// pre-cap population. The CARRIER-level reductions divide by
372 /// `wake_latencies_ns.len()` (this capped pool size), NOT by
373 /// `wake_sample_total`: [`Self::wake_summary`] takes p99 / median over `len`,
374 /// and [`cgroup_stats`] computes `cv = stddev/mean` with
375 /// `n = all_latencies.len()`. The RUN-level cross-phase re-pool
376 /// ([`populate_run_distribution_metrics`]) instead population-WEIGHTS (see
377 /// the PARITY CONTRACT below): its CV / mean divide by Σ per-sample weights
378 /// (the reconstructed true population), which equals `len` only below the cap.
379 ///
380 /// PARITY CONTRACT (the one component whose parity is size-dependent): for
381 /// pools ≤ `MAX_WAKE_SAMPLES` the reservoir IS the full concatenation, so the
382 /// p99 / median / CV re-pool reproduces [`cgroup_stats`] VALUE-FOR-VALUE.
383 /// Above the cap the carrier holds a distribution-preserving reservoir
384 /// SUBSAMPLE while [`cgroup_stats`] reduces over the full per-worker concat,
385 /// so the re-pool is DISTRIBUTION-EQUIVALENT, not byte-identical (the bounded
386 /// bulk-port frame forbids carrying the full pool; staged reservoirs cannot be
387 /// byte-identical to a single full-pool reduction). This is BY DESIGN:
388 /// `cgroup_stats` stays the uncapped run-level authority (capping it to match
389 /// the carrier would discard most of a multi-worker cgroup's samples to chase
390 /// a sub-display-precision artifact), and the carrier's >cap merge is WEIGHTED
391 /// by `wake_sample_total` (`Self::weighted_merge_reservoirs`) so the subsample
392 /// is an UNBIASED sample of the combined population — no smaller-population
393 /// skew. Both layers de-skew the cap: the carrier MERGE weights by
394 /// `wake_sample_total` (`Self::weighted_merge_reservoirs`), and the
395 /// cross-PHASE run-level pool in `populate_run_distribution_metrics` weights
396 /// each phase carrier's samples by `wake_sample_total / wake_latencies_ns.len()`
397 /// (so a phase that exceeded the cap contributes by true population, not
398 /// capped length) and reduces with the weighted percentile / moments — the
399 /// prior length-weighted concat is gone. Below the cap every weight is 1.0,
400 /// so the weighted P99 / median / mean / worst are BYTE-identical to the
401 /// unweighted concat; the weighted CV matches only within ~1e-9 (it sums in
402 /// f64 where the unweighted path sums the mean in u64 — a weighted variance
403 /// cannot keep the u64 sum).
404 pub wake_latencies_ns: Vec<u64>,
405 /// True wakeup count before reservoir clamping (`wake_latencies_ns` is
406 /// capped), so the re-pool can report the real population size. An
407 /// intentional ADDITION over [`CgroupStats`] (which has no such field), NOT
408 /// a mirrored reduction — do not strip it in a strict-parity audit; it is
409 /// the only source of the true wakeup population once `wake_latencies_ns` is
410 /// reservoir-clamped, and it is for REPORTING, not the CV denominator.
411 pub wake_sample_total: u64,
412 /// Pooled per-timer-cycle latency samples (ns) across the cgroup's
413 /// [`crate::workload::WorkType::TimerLatency`] workers in the phase,
414 /// un-reduced so median / p99 / p999 / worst re-pool over the combined set.
415 /// Reservoir-capped at `MAX_WAKE_SAMPLES` on same-name-carrier merge exactly
416 /// like `wake_latencies_ns` (population-weighted >cap); `timer_sample_total`
417 /// carries the true pre-cap population. Distinct carrier from
418 /// `wake_latencies_ns`.
419 pub timer_latencies_ns: Vec<u64>,
420 /// True timer-cycle count before reservoir clamping (`timer_latencies_ns` is
421 /// capped), so the re-pool reports the real population. Mirrors
422 /// `wake_sample_total` for the timer carrier.
423 pub timer_sample_total: u64,
424 /// Pooled per-worker schedstat run-delay samples (RAW ns) for the phase,
425 /// un-reduced so mean / worst run-delay re-pool over the combined set; the
426 /// re-pool converts ns → µs to match [`CgroupStats`]'s run-delay-µs fields.
427 /// Stored as raw kernel ns (like `wake_latencies_ns`), not pre-converted,
428 /// per the raw-component thesis. GRANULARITY: unlike `wake_latencies_ns`
429 /// (one per WAKEUP), each entry here is ONE per-worker value — that
430 /// worker's `sched_info.run_delay` delta over the carrier's window: the
431 /// whole-run `schedstat_run_delay_ns` (end−start) for the step-local
432 /// carrier, or the per-phase delta for the backdrop slice carrier. So the
433 /// pool size is the worker
434 /// count, the mean is the average per-worker total queued-to-run delay, and
435 /// `worst_run_delay_us` selects the single worker with the largest total
436 /// queued-to-run delay (NOT the worst single dispatch).
437 pub run_delays_ns: Vec<u64>,
438 /// Per-worker off-CPU% samples for the phase, un-reduced. Carried for the
439 /// per-phase per-cgroup off-CPU% RENDER — the avg / min / max /
440 /// spread of the combined set. NOT consumed by the run-level
441 /// distributional re-pool: off-CPU% has no run-level Distribution metric
442 /// (off-CPU%/spread is intrinsically per-cgroup, so the run-level
443 /// `worst_spread` stays the cross-cgroup max of per-cgroup
444 /// [`CgroupStats::spread`] via the typed [`AssertResult::merge`] fold, not a
445 /// pooled distribution). An EMPTY vec is the not-measured state (no worker
446 /// with positive wall time), preserving the not-measured vs measured-zero
447 /// distinction [`CgroupStats`] keeps. Stored as raw samples, not pre-reduced
448 /// extremes, because the mean is unrecoverable from min/max alone for >2
449 /// workers. Each sample is `off_cpu_ns /
450 /// wall_time_ns * 100`, where `off_cpu_ns = wall_time_ns - cpu_time_ns` and
451 /// `cpu_time_ns` is the `CLOCK_THREAD_CPUTIME_ID` thread on-CPU time
452 /// (workload/worker `off_cpu_ns` at report build). `total_cpu_time_ns` is a
453 /// DISTINCT on-CPU measurement (`schedstat_cpu_time_ns`, the `/proc`
454 /// schedstat `se.sum_exec_runtime`): both ultimately track on-CPU runtime but
455 /// are sampled at different points (the `CLOCK_THREAD_CPUTIME_ID` read folds
456 /// the in-flight delta; the schedstat field reads the stored value), so the
457 /// two need not be byte-identical and must not be cross-wired in a re-pool.
458 pub off_cpu_pcts: Vec<f64>,
459 /// Sum of per-worker CPU-migration counts in the phase (Counter).
460 pub total_migrations: u64,
461 /// Sum of per-worker iteration counts in the phase (Counter).
462 pub total_iterations: u64,
463 /// Sum of per-worker on-CPU time (ns) in the phase — the
464 /// overcommit-invariant rate denominator (Counter). Sourced from
465 /// `schedstat_cpu_time_ns` (the `/proc` schedstat `se.sum_exec_runtime`,
466 /// rq-charged on-CPU ns) — a DISTINCT on-CPU-time sample from the
467 /// `CLOCK_THREAD_CPUTIME_ID` time behind `off_cpu_pcts` (different sample
468 /// point; not byte-identical), so do not cross-wire the two in a re-pool.
469 pub total_cpu_time_ns: u64,
470 /// Pages on the expected NUMA node(s) — page-locality numerator. A per-task
471 /// `/proc/self/numa_maps` residency GAUGE (current snapshot of the task's mm,
472 /// recomputed each read — the kernel zeroes and re-walks the page tables),
473 /// SPATIALLY summed across the cgroup's workers within a phase: disjoint-mm
474 /// under the `CloneMode::Fork` default (the true cgroup total), but
475 /// `CloneMode::Thread` siblings share one mm and the SUM over-counts shared
476 /// pages once per thread (caveat inherited from `WorkerReport::numa_pages`).
477 /// The CROSS-PHASE fold takes the LATEST measured snapshot (see
478 /// `numa_agg_per_cgroup`), never a sum (summing residency across phases
479 /// over-counts by the phase count).
480 pub numa_pages_local: u64,
481 /// Total allocated pages — the SHARED denominator for BOTH page_locality
482 /// (`numa_pages_local` / this) AND cross_node_migration_ratio
483 /// (`cross_node_migrated` / this). A per-task `/proc/self/numa_maps` residency
484 /// GAUGE (same class/folds as `numa_pages_local`: within-phase SUM across
485 /// workers — disjoint-mm under the `CloneMode::Fork` default, Thread-mode
486 /// over-count caveat inherited from `WorkerReport::numa_pages` — cross-phase
487 /// LATEST snapshot); the kernel computes both ratios over the identical page
488 /// total, so one field serves both — a separate cross_node_total would invite
489 /// a silent desync.
490 pub numa_pages_total: u64,
491 /// Cross-node migrated pages — cross_node_migration_ratio numerator
492 /// (denominator is `numa_pages_total`). A SYSTEM-WIDE
493 /// `/proc/vmstat numa_pages_migrated` monotonic-COUNTER delta each worker
494 /// observes redundantly, so the within-phase fold is MAX across
495 /// workers/sources (summing would inflate it by the worker count — mirrors
496 /// [`CgroupStats`]'s deliberate max-fold); the CROSS-PHASE fold SUMs the
497 /// per-phase deltas over disjoint intervals to the run total.
498 pub cross_node_migrated: u64,
499 /// Longest scheduling gap (ms) across the cgroup's workers in the phase,
500 /// coupled with `max_gap_cpu`. A Peak folded as an ARGMAX of the (ms, cpu)
501 /// pair so the worst gap and its CPU survive together — mirrors
502 /// [`CgroupStats`]'s `max_gap_ms` / `max_gap_cpu` coupling (a bare
503 /// independent max would desync the gap from its CPU).
504 pub max_gap_ms: u64,
505 /// CPU that owned the worst scheduling gap — `max_gap_ms`'s argmax
506 /// companion. Folded together with `max_gap_ms`, never independently.
507 pub max_gap_cpu: usize,
508 /// True when this carrier's raw sample vectors (`wake_latencies_ns` /
509 /// `timer_latencies_ns` / `run_delays_ns` / `off_cpu_pcts`, plus the
510 /// `schbench` histograms) were dropped by
511 /// `AssertResult::strip_phase_cgroup_samples` to fit the size-limited guest
512 /// bulk frame — distinct from a carrier that genuinely measured no samples.
513 /// The reduced counters survive; only the per-phase distribution render
514 /// loses its source, so the render shows "samples stripped" rather
515 /// than the not-measured "n/a". Defaults to `false` (not stripped) and is set
516 /// only on a carrier that actually HAD samples to drop; ORs across `merge` so
517 /// a merged carrier is stripped if either input was.
518 pub stripped: bool,
519 /// Per-cgroup DERIVED scalar metrics for this (phase, cgroup), keyed by
520 /// `crate::stats::MetricDef` name — the per-cgroup analog of
521 /// [`PhaseBucket::metrics`] (which is the pooled-across-cgroups set). Populated
522 /// post-fold by `derive_phase_metrics` (both the schbench per-phase family AND
523 /// the non-schbench families — wake p99/median/cv, mean/max run-delay,
524 /// avg/min/max/spread off-CPU%, and the migration / iterations / locality
525 /// ratios) from the SAME reducers that fill
526 /// the pooled map, so a
527 /// test can query "metric M of cgroup C in phase P" as readily as the phase
528 /// aggregate (N cgroups -> N queryable sets + the pooled aggregate). DERIVED, not a
529 /// raw component: `PhaseCgroupStats::merge` leaves it empty and it is (re)derived
530 /// POST-merge, exactly as the pooled map skips is_derived keys in
531 /// `merge_matched_phase_buckets`. ALWAYS serialized (no `skip_serializing_if`):
532 /// PhaseCgroupStats rides the postcard bulk-TLV port, a NON-self-describing
533 /// POSITIONAL format — a conditionally-omitted field desyncs the byte stream and
534 /// corrupts the fields after it (here `schbench`), so the field must always be
535 /// present. No `serde(default)` either: pre-1.0, old sidecar/cache data is
536 /// disposable and regenerates (no compat shim). Read via [`Self::get`]; the
537 /// `crate::Claim` derive skips a `BTreeMap` field (matching
538 /// [`PhaseBucket::metrics`], which has no Claim accessor either).
539 pub metrics: std::collections::BTreeMap<String, f64>,
540 /// Per-phase schbench engine metrics for a `WorkType::Schbench` backdrop
541 /// cgroup (`None` for every non-schbench carrier). Pooled across the
542 /// cgroup's workers by [`PhaseCgroupStats::merge`] (histogram
543 /// bucket-add + integer-add of the run-delay raw pairs). The schbench
544 /// per-phase derivation reads it, pools across cgroups, and derives the
545 /// per-phase percentile / run-delay-mean / loop-count scalars into
546 /// [`PhaseBucket::metrics`]. `pub(crate)`: the element type is `pub(crate)`
547 /// and a per-phase carrier is internal — test authors read `PhaseBucket`,
548 /// not this. Non-`pub` so the `crate::Claim` derive skips it (a percentile
549 /// histogram has no meaningful scalar claim accessor).
550 pub(crate) schbench: Option<crate::workload::schbench::run::SchbenchPhaseStats>,
551 /// Per-phase taobench engine metrics for a `WorkType::Taobench` backdrop
552 /// cgroup (`None` for every non-taobench carrier). Pooled across the cgroup's
553 /// workers by [`PhaseCgroupStats::merge`] (counter-add, wall-window MAX, serve
554 /// histograms unioned). The taobench per-phase derivation reads it, pools
555 /// across cgroups, and derives the per-phase qps / hit-ratio / serve-latency
556 /// scalars into [`PhaseBucket::metrics`]. `pub(crate)`: an internal carrier.
557 pub(crate) taobench: Option<crate::workload::taobench::run::TaobenchPhaseStats>,
558}
559
560impl PhaseCgroupStats {
561 /// Component-wise union of two per-phase per-cgroup data for the SAME
562 /// cgroup name (same `step_index`). Fold rule by component class:
563 /// - sample vectors (`wake_latencies_ns`, `run_delays_ns`, `off_cpu_pcts`)
564 /// CONCAT, so the re-pool sees the combined set, never a mean of
565 /// per-source reductions;
566 /// - the CPU set (`cpus_used`) UNIONs;
567 /// - genuine Counters (`num_workers`, `wake_sample_total`,
568 /// `total_migrations`, `total_iterations`, `total_cpu_time_ns`) SUM —
569 /// `num_workers` included, because a multi-`WorkSpec` cgroup emits one
570 /// carrier per handle covering DISJOINT worker subsets, so summing
571 /// reproduces the pooled count (see the `num_workers` field doc);
572 /// - the residency GAUGEs (`numa_pages_local`, `numa_pages_total`) also SUM
573 /// here — the WITHIN-PHASE spatial sum across the cgroup's workers
574 /// (disjoint-mm under the `CloneMode::Fork` default; Thread-mode shares one
575 /// mm and over-counts, the caveat inherited from `WorkerReport::numa_pages`),
576 /// NOT a cross-phase sum (the cross-phase fold takes the LATEST snapshot in
577 /// `numa_agg_per_cgroup`);
578 /// - `cross_node_migrated`, a system-wide vmstat monotonic-Counter delta each
579 /// worker observes redundantly, takes the MAX across workers (summing would
580 /// inflate it); its cross-phase fold SUMs the per-phase deltas;
581 /// - the COUPLED worst gap (`max_gap_ms`, `max_gap_cpu`) folds as an
582 /// ARGMAX — the pair from whichever side has the larger ms (b's on tie,
583 /// matching the builders' `max_by_key` last-wins) so the gap and its CPU
584 /// stay bound together.
585 ///
586 /// The counter SUMs use plain `+`: debug builds panic on overflow rather
587 /// than wrapping. The realistic magnitudes (iteration / ns counts far
588 /// below `u64::MAX` even pooled across a long run) keep overflow
589 /// unreachable; a loud debug panic is preferred over a silently wrong
590 /// re-pool denominator.
591 pub(crate) fn merge(a: PhaseCgroupStats, b: PhaseCgroupStats) -> PhaseCgroupStats {
592 // Merge the two capped wake-latency reservoirs. Same-name carriers (a
593 // multi-`WorkSpec` cgroup's per-handle carriers) merge ON THE GUEST before
594 // the AssertResult is serialized over the bulk port, so K carriers must
595 // not concat to K × MAX_WAKE_SAMPLES (it could overrun the 16 MiB frame,
596 // flipping a PASS to a truncated FAIL).
597 //
598 // ≤cap: the concatenation IS the true combined population, so it passes
599 // through unchanged — value-for-value parity with cgroup_stats for small
600 // pools (only >cap pools become a subsample; see the `wake_latencies_ns`
601 // field doc). >cap: a WEIGHTED reservoir merge weighted by each carrier's
602 // true pre-cap population (`wake_sample_total`), so the merged sample is an
603 // UNBIASED uniform sample of the combined population — NOT the
604 // smaller-population-skewed reservoir-of-reservoirs an unweighted
605 // concat-and-re-cap produced (which weighted by reservoir LENGTH ≈ 50/50,
606 // ignoring the true populations).
607 let cap = crate::workload::MAX_WAKE_SAMPLES;
608 let wake_latencies_ns = if a.wake_latencies_ns.len() + b.wake_latencies_ns.len() <= cap {
609 let mut v = a.wake_latencies_ns;
610 v.extend(b.wake_latencies_ns);
611 v
612 } else {
613 Self::weighted_merge_reservoirs(
614 &a.wake_latencies_ns,
615 a.wake_sample_total,
616 &b.wake_latencies_ns,
617 b.wake_sample_total,
618 cap,
619 )
620 };
621 // Same bounded merge for the distinct timer reservoir:
622 // <=cap concatenation is value-for-value; >cap a population-weighted
623 // reservoir merge keeps it an unbiased sample of the combined population.
624 let timer_latencies_ns = if a.timer_latencies_ns.len() + b.timer_latencies_ns.len() <= cap {
625 let mut v = a.timer_latencies_ns;
626 v.extend(b.timer_latencies_ns);
627 v
628 } else {
629 Self::weighted_merge_reservoirs(
630 &a.timer_latencies_ns,
631 a.timer_sample_total,
632 &b.timer_latencies_ns,
633 b.timer_sample_total,
634 cap,
635 )
636 };
637 let mut run_delays_ns = a.run_delays_ns;
638 run_delays_ns.extend(b.run_delays_ns);
639 let mut off_cpu_pcts = a.off_cpu_pcts;
640 off_cpu_pcts.extend(b.off_cpu_pcts);
641 let mut cpus_used = a.cpus_used;
642 cpus_used.extend(b.cpus_used);
643 // Coupled worst-gap ARGMAX: take the (ms, cpu) pair together from the
644 // side with the larger gap (b's on tie, matching the builders'
645 // max_by_key last-wins) so the CPU stays bound to the gap it owned — a
646 // bare independent max would desync them. The last-wins tie-break is
647 // parity-coupled to fold order: AssertResult::merge folds same-name
648 // carriers in the order reports are pooled (handle iteration order), so
649 // on an equal-gap tie this yields the same CPU as a single cgroup_stats
650 // over the concatenated reports. A reordered fold would break that parity.
651 let (max_gap_ms, max_gap_cpu) = if b.max_gap_ms >= a.max_gap_ms {
652 (b.max_gap_ms, b.max_gap_cpu)
653 } else {
654 (a.max_gap_ms, a.max_gap_cpu)
655 };
656 // Per-phase schbench: OR-with-combine. Both Some → merge the pooled
657 // histograms + integer-add the run-delay raw pairs / loop_count
658 // (SchbenchPhaseStats::merge, the SAME associative+commutative op the
659 // guest engine uses to pool across message threads); one Some → carry
660 // it; both None (non-schbench cgroup) → None.
661 let schbench = match (a.schbench, b.schbench) {
662 (Some(mut x), Some(y)) => {
663 x.merge(&y);
664 Some(x)
665 }
666 (Some(x), None) | (None, Some(x)) => Some(x),
667 (None, None) => None,
668 };
669 // Per-phase taobench: OR-with-merge — both Some → counter-add + wall-window
670 // MAX + serve-latency histogram union (TaobenchPhaseStats::merge, the SAME
671 // op the guest engine pools with); one Some → carry; both None
672 // (non-taobench cgroup) → None.
673 let taobench = match (a.taobench, b.taobench) {
674 (Some(mut x), Some(y)) => {
675 x.merge(&y);
676 Some(x)
677 }
678 (Some(x), None) | (None, Some(x)) => Some(x),
679 (None, None) => None,
680 };
681 // saturating_add: pool the guest-runtime monotonic counters/times (the
682 // sample totals, migrations, iterations, cpu-time-ns, numa pages) across
683 // cgroups. Overflow is physically unreachable for honest data (real
684 // counts << u64::MAX), but a corrupt/hostile guest value would otherwise
685 // debug-panic or release-wrap to a silently-wrong derived metric;
686 // saturating is exact for every in-range value. num_workers is a
687 // ktstr-configured TOPOLOGY count (bounded by the guest CPU count, not a
688 // runtime accumulator), so it cannot overflow and keeps plain `+`.
689 // cross_node_migrated pools as MAX (a system-wide vmstat snapshot shared
690 // by the concurrent cgroups), not a sum.
691 PhaseCgroupStats {
692 num_workers: a.num_workers + b.num_workers,
693 cpus_used,
694 wake_latencies_ns,
695 wake_sample_total: a.wake_sample_total.saturating_add(b.wake_sample_total),
696 timer_latencies_ns,
697 timer_sample_total: a.timer_sample_total.saturating_add(b.timer_sample_total),
698 run_delays_ns,
699 off_cpu_pcts,
700 total_migrations: a.total_migrations.saturating_add(b.total_migrations),
701 total_iterations: a.total_iterations.saturating_add(b.total_iterations),
702 total_cpu_time_ns: a.total_cpu_time_ns.saturating_add(b.total_cpu_time_ns),
703 numa_pages_local: a.numa_pages_local.saturating_add(b.numa_pages_local),
704 numa_pages_total: a.numa_pages_total.saturating_add(b.numa_pages_total),
705 cross_node_migrated: a.cross_node_migrated.max(b.cross_node_migrated),
706 max_gap_ms,
707 max_gap_cpu,
708 stripped: a.stripped || b.stripped,
709 // DERIVED, not a raw component: left EMPTY here and (re)derived post-merge
710 // by derive_phase_metrics (the fold runs derive AFTER merging, the
711 // same reason the pooled map skips is_derived keys in
712 // merge_matched_phase_buckets). Folding stale per-operand metrics would
713 // double-count; the post-merge re-derive is the sole producer.
714 metrics: std::collections::BTreeMap::new(),
715 schbench,
716 taobench,
717 }
718 }
719
720 /// Look up this cgroup's per-phase DERIVED value for `metric_name` — the
721 /// per-cgroup analog of [`PhaseBucket::get`] (see [`Self::metrics`]). `None`
722 /// when this cgroup carried no finite samples for the metric (the ABSENT
723 /// discipline), distinct from `Some(0.0)` (a real reducer zero).
724 pub fn get(&self, metric_name: &str) -> Option<f64> {
725 self.metrics.get(metric_name).copied()
726 }
727
728 /// Like [`Self::get`] but panics citing the metric keys actually present when
729 /// the metric is absent — use when the caller knows this cgroup MUST carry the
730 /// metric in the phase.
731 pub fn expect_metric(&self, metric_name: &str) -> f64 {
732 self.get(metric_name).unwrap_or_else(|| {
733 panic!(
734 "PhaseCgroupStats::expect_metric: metric '{}' absent from this \
735 per-cgroup carrier (num_workers={}, stripped={}); keys present: {:?}. \
736 Causes: (a) the cgroup produced no finite samples for it; (b) metric \
737 name typo; (c) a non-schbench carrier has no schbench keys.",
738 metric_name,
739 self.num_workers,
740 self.stripped,
741 self.metrics.keys().collect::<Vec<_>>(),
742 )
743 })
744 }
745
746 /// The per-cgroup analog of [`PhaseBucket::cgroup_counter_total`] for the
747 /// three per-cgroup Counters that live ONLY on the carrier (no derived
748 /// `metrics` entry): `total_migrations`, `total_iterations`, and
749 /// `total_cpu_time_ns`. Lets
750 /// [`crate::vmm::VmResult::phase_cgroup_metric`] expose them by metric name
751 /// symmetrically with the pooled [`crate::vmm::VmResult::phase_metric`]
752 /// `cgroup_counter_total` fallback (the pooled `_total` suffix marks the
753 /// cross-cgroup SUM; this per-cgroup form returns one cgroup's value). `None`
754 /// for any other name (derived metrics are read via [`Self::get`]).
755 pub fn cgroup_counter(&self, name: &str) -> Option<f64> {
756 match name {
757 "total_migrations" => Some(self.total_migrations as f64),
758 "total_iterations" => Some(self.total_iterations as f64),
759 "total_cpu_time_ns" => Some(self.total_cpu_time_ns as f64),
760 _ => None,
761 }
762 }
763
764 /// Merge two CAPPED uniform reservoirs into one of size ≤ `cap` that is a
765 /// uniform sample of the COMBINED population. `a` is a uniform reservoir of
766 /// `w_a` true samples, `b` of `w_b` (their `wake_sample_total` weights). Each
767 /// output slot is drawn from `a` with probability `w_a / (w_a + w_b)` and from
768 /// `b` otherwise; within a source the index is uniform. Composing the
769 /// source-level uniform reservoir with the within-source uniform draw makes
770 /// each output a uniform draw from the combined population, so the merged
771 /// A-fraction is the TRUE `w_a / (w_a + w_b)`. This removes the equal-slot
772 /// ("reservoir-of-reservoirs") skew an unweighted concat-and-re-cap imposes:
773 /// two already-capped inputs concat ≈ 50/50 by LENGTH regardless of their true
774 /// populations, over-counting the smaller-population carrier. Sampling WITH
775 /// replacement is the correct estimator once the inputs are capped (each
776 /// reservoir element stands for `w/len` population units; the pre-cap samples
777 /// are gone).
778 ///
779 /// DETERMINISTIC: the xorshift64 stream is seeded from the inputs (populations +
780 /// lengths) so the merge is a PURE function of its arguments — unlike
781 /// `crate::workload::reservoir_push`, whose stream is gettid-seeded
782 /// thread-local (a merge run twice would otherwise differ). The triple-shift
783 /// mirrors the codebase's inline xorshift64 (`reservoir_push` /
784 /// `io::xorshift64`).
785 ///
786 /// Assumes `w_a + w_b < 2^64` — a realistic wake population is far below it
787 /// (2^64 wakeups is physically unreachable), so the single-u64 `s % total`
788 /// draw spans `[0, total)`. Callers gate on `a.len() + b.len() > cap`, which
789 /// (each input ≤ cap) guarantees both sources non-empty; the per-slot guards
790 /// below stay safe for a degenerate hand-built input regardless.
791 pub(crate) fn weighted_merge_reservoirs(
792 a: &[u64],
793 w_a: u64,
794 b: &[u64],
795 w_b: u64,
796 cap: usize,
797 ) -> Vec<u64> {
798 if a.is_empty() && b.is_empty() {
799 return Vec::new();
800 }
801 // Weights are the true populations; fall back to reservoir lengths if a
802 // (hand-built) carrier reports zero population alongside non-empty samples,
803 // keeping the split well-defined instead of dividing by a zero total. The
804 // mixed case (one weight 0, the other > 0) is left as-is: a zero weight
805 // sends every draw to the other source, the only defensible split for a
806 // source claiming zero population. Production maintains wake_sample_total
807 // >= len (reservoir_push counts every push), so neither edge is reachable
808 // on the capture path.
809 let (wa, wb) = if w_a == 0 && w_b == 0 {
810 (a.len() as u128, b.len() as u128)
811 } else {
812 (w_a as u128, w_b as u128)
813 };
814 let total = wa + wb;
815 // Loud-panic on the documented `w_a + w_b < 2^64` assumption (a realistic
816 // wake population is far below it): if total exceeded u64::MAX the
817 // `s as u128 % total` draw — s spans [0, 2^64) — could not reach
818 // [2^64, total) and would silently bias the source split. Matches the merge
819 // SUM's debug-panic-on-overflow discipline (loud over silently wrong).
820 debug_assert!(
821 total <= u64::MAX as u128,
822 "weighted_merge_reservoirs: w_a + w_b overflows u64 ({total}); source draw would bias",
823 );
824 // Golden-ratio Weyl multiplier (the codebase's standard PRNG seed mixer);
825 // a non-zero, input-derived seed makes the merge deterministic. xorshift64
826 // has 0 as a fixed point, hence the fallback.
827 const GOLDEN: u64 = 0x9E37_79B9_7F4A_7C15;
828 let mut s =
829 (w_a ^ w_b.rotate_left(32) ^ (a.len() as u64).rotate_left(16) ^ (b.len() as u64))
830 .wrapping_mul(GOLDEN);
831 if s == 0 {
832 s = GOLDEN;
833 }
834 let step = |x: u64| {
835 let mut v = x;
836 v ^= v << 13;
837 v ^= v >> 7;
838 v ^= v << 17;
839 v
840 };
841 let mut out = Vec::with_capacity(cap);
842 for _ in 0..cap {
843 s = step(s);
844 // Defensive empty-source guards: caller gates ensure both non-empty,
845 // but a stripped / zero-population fixture must never index an empty
846 // slice.
847 let from_a = if a.is_empty() {
848 false
849 } else if b.is_empty() {
850 true
851 } else {
852 (s as u128 % total) < wa
853 };
854 s = step(s);
855 if from_a {
856 out.push(a[(s % a.len() as u64) as usize]);
857 } else {
858 out.push(b[(s % b.len() as u64) as usize]);
859 }
860 }
861 out
862 }
863
864 /// Off-CPU% reduction for the per-phase per-cgroup render:
865 /// `(avg, min, max, spread)` over [`Self::off_cpu_pcts`], or `None` when
866 /// the vec is empty — the NOT-measured state (no worker had positive wall
867 /// time). Reduces the SAME per-worker pcts [`cgroup_stats`] reduces
868 /// (off_cpu_ns / wall_time_ns × 100), so for a phase spanning the whole run
869 /// it reproduces that whole-run reduction; `spread = max − min`.
870 /// `Some((0.0, ..))` is a MEASURED zero (distinct from the `None`
871 /// not-measured state), preserving the discipline the empty-vec contract on
872 /// `off_cpu_pcts` keeps. Display-only: never written back into a re-pool.
873 pub fn off_cpu_summary(&self) -> Option<(f64, f64, f64, f64)> {
874 let pcts = &self.off_cpu_pcts;
875 if pcts.is_empty() {
876 return None;
877 }
878 let min = pcts.iter().cloned().reduce(f64::min).expect("non-empty");
879 let max = pcts.iter().cloned().reduce(f64::max).expect("non-empty");
880 let avg = pcts.iter().sum::<f64>() / pcts.len() as f64;
881 Some((avg, min, max, max - min))
882 }
883
884 /// Wake-latency reduction for the per-phase render:
885 /// `(p99_us, median_us)` over the pooled [`Self::wake_latencies_ns`], or
886 /// `None` when the pool is empty. Nearest-rank percentile via `percentile`
887 /// (ns→µs once), reproducing [`cgroup_stats`]'s p99/median value-for-value
888 /// for the ≤cap pool (and the run-level re-pool's `reduce_sorted_distribution`).
889 /// Above `MAX_WAKE_SAMPLES` the pool is a distribution-preserving reservoir
890 /// subsample (see [`Self::wake_latencies_ns`]), so p99/median is then
891 /// distribution-equivalent, NOT byte-identical, to the full-pool reduction —
892 /// the rendered tail stays accurate, only exact parity is size-bounded.
893 /// `None`-on-empty omits the wake segment from the render rather than
894 /// painting a misleading 0µs (the display analogue of `cgroup_stats`'s
895 /// 0.0-sentinel, which has no Option to carry not-measured).
896 pub fn wake_summary(&self) -> Option<(f64, f64)> {
897 if self.wake_latencies_ns.is_empty() {
898 return None;
899 }
900 let mut sorted = self.wake_latencies_ns.clone();
901 sorted.sort_unstable();
902 let p99 = percentile(&sorted, 0.99) as f64 / 1000.0;
903 let median = percentile(&sorted, 0.5) as f64 / 1000.0;
904 Some((p99, median))
905 }
906
907 /// Timer-latency reduction for the per-phase metric emission:
908 /// `(median_us, p99_us, p999_us)` over the pooled
909 /// [`Self::timer_latencies_ns`] (nearest-rank, ns→µs), or `None` when the
910 /// pool is empty (the not-measured state — `write_carrier_scalars` omits all
911 /// three timer keys together). Reproduces the per-cgroup
912 /// [`crate::assert::cgroup_stats`] timer reductions value-for-value below the
913 /// reservoir cap (distribution-equivalent above it), exactly like
914 /// [`Self::wake_summary`]. The run-level WORST (max) is NOT here — it comes
915 /// from the cross-phase re-pool (`populate_run_distribution_metrics`).
916 pub fn timer_summary(&self) -> Option<(f64, f64, f64)> {
917 if self.timer_latencies_ns.is_empty() {
918 return None;
919 }
920 let mut sorted = self.timer_latencies_ns.clone();
921 sorted.sort_unstable();
922 let median = percentile(&sorted, 0.5) as f64 / 1000.0;
923 let p99 = percentile(&sorted, 0.99) as f64 / 1000.0;
924 let p999 = percentile(&sorted, 0.999) as f64 / 1000.0;
925 Some((median, p99, p999))
926 }
927
928 /// Wake-latency coefficient of variation (stddev / mean) over the pooled
929 /// [`Self::wake_latencies_ns`] (`n = len`), or `None` when the pool is
930 /// empty (the not-measured state — so the `run_metrics` carrier writer
931 /// omits all three wake keys together). `Some(0.0)` when
932 /// the mean is zero is a MEASURED zero. Reproduces [`cgroup_stats`]'s
933 /// `wake_latency_cv` value-for-value for the ≤cap pool (`Σv` in `u64`,
934 /// same accumulation); above `MAX_WAKE_SAMPLES` the pool is a
935 /// distribution-preserving reservoir subsample, so the CV is then
936 /// distribution-equivalent, not byte-identical.
937 pub fn wake_cv(&self) -> Option<f64> {
938 if self.wake_latencies_ns.is_empty() {
939 return None;
940 }
941 let n = self.wake_latencies_ns.len() as f64;
942 let mean_ns = self.wake_latencies_ns.iter().sum::<u64>() as f64 / n;
943 if mean_ns > 0.0 {
944 let variance = self
945 .wake_latencies_ns
946 .iter()
947 .map(|&v| (v as f64 - mean_ns).powi(2))
948 .sum::<f64>()
949 / n;
950 Some(variance.sqrt() / mean_ns)
951 } else {
952 Some(0.0)
953 }
954 }
955
956 /// Run-delay reduction for the per-phase render:
957 /// `(mean_us, worst_us)` over the per-worker [`Self::run_delays_ns`] (raw
958 /// ns), or `None` when empty. Divides ns→µs ONCE on the summed / maxed ns.
959 /// `worst` reproduces [`cgroup_stats`]'s value-for-value (`max(ns)/1000 ==
960 /// max(ns/1000)`, division is monotone). `mean` reproduces it to f64 ULP,
961 /// not bit-exactly: this f64-sums then divides once (`Σns/n/1000`), while
962 /// `cgroup_stats` divides each worker's ns by 1000 first then sums
963 /// (`Σ(ns/1000)/n`) — the same value reassociated, differing only
964 /// sub-display-precision (a divergent-input parity test bounds it at 1e-9).
965 /// Each sample is
966 /// one worker's whole-phase cumulative `sched_info.run_delay` delta, so
967 /// `mean` is the average per-worker total queued-to-run delay and `worst`
968 /// the largest. `None`-on-empty omits the segment.
969 pub fn run_delay_summary(&self) -> Option<(f64, f64)> {
970 if self.run_delays_ns.is_empty() {
971 return None;
972 }
973 let n = self.run_delays_ns.len() as f64;
974 // Sum in f64, NOT u64-then-cast: matches cgroup_stats's f64 accumulation
975 // and cannot integer-overflow (an f64 sum saturates toward +inf; a u64
976 // sum would panic in debug / silently wrap in release on a pathological
977 // pool). Values are identical within the documented 1e-9 ULP bound.
978 let mean = self.run_delays_ns.iter().map(|&v| v as f64).sum::<f64>() / n / 1000.0;
979 let worst = *self.run_delays_ns.iter().max().expect("non-empty") as f64 / 1000.0;
980 Some((mean, worst))
981 }
982}
983
984impl CgroupStats {
985 /// Wake-latency tail amplification:
986 /// `p99_wake_latency_us / median_wake_latency_us`. Returns `0.0`
987 /// when `median_wake_latency_us <= 0.0` so the result never
988 /// propagates `NaN` / `Infinity` into downstream
989 /// `finite_or_zero` filters. Method-only access (no stored
990 /// shadow) — recomputed every call from the raw fields.
991 ///
992 /// Unitless; ≥1.0 by definition of order statistics (p99 cannot
993 /// undershoot the median on the same sample set). Values far
994 /// above 1.0 signal a long tail — the scheduler wakes most
995 /// workers promptly but occasionally stalls some, a regression
996 /// axis that neither `median_*` nor `p99_*` exposes in
997 /// isolation.
998 pub fn wake_latency_tail_ratio(&self) -> f64 {
999 if self.median_wake_latency_us > 0.0 {
1000 self.p99_wake_latency_us / self.median_wake_latency_us
1001 } else {
1002 0.0
1003 }
1004 }
1005
1006 /// Whether this cgroup measured the given distribution `source`. Gates the
1007 /// run-level carrier-less re-pool in [`populate_run_distribution_metrics`]
1008 /// so a cgroup that recorded no samples for `source` contributes ABSENCE
1009 /// (leaving the fold `None` when no cgroup measured it), not a sentinel
1010 /// `0.0`. See [`Self::wake_measured`] / [`Self::timer_measured`] /
1011 /// [`Self::run_delay_measured`].
1012 pub fn measured_for(&self, source: crate::stats::SampleSource) -> bool {
1013 match source {
1014 crate::stats::SampleSource::WakeLatencyNs => self.wake_measured,
1015 crate::stats::SampleSource::TimerLatencyNs => self.timer_measured,
1016 crate::stats::SampleSource::RunDelayNs => self.run_delay_measured,
1017 }
1018 }
1019
1020 /// Throughput per parallel degree:
1021 /// `total_iterations / num_workers`. `None` when
1022 /// `num_workers == 0` (no worker reported, so per-worker
1023 /// throughput is undefined — distinct from a measured zero);
1024 /// `Some(0.0)` when workers ran but completed zero iterations
1025 /// (a real throughput collapse). The `None` / `Some(0.0)` split
1026 /// is load-bearing: the run-level worst-cgroup re-pool in
1027 /// [`populate_run_distribution_metrics`] (the
1028 /// `MetricKind::WorstLowest` arm) must treat a measured zero as
1029 /// the worst reading (it wins the "lowest" bucket) while skipping
1030 /// a no-data cgroup — collapsing both to `0.0` would hide a
1031 /// starved cgroup behind the no-data sentinel. Method-only
1032 /// access (no stored shadow) — recomputed every call from the
1033 /// raw fields.
1034 ///
1035 /// Only meaningful across runs of the SAME variant (equal
1036 /// scenario duration): cross-variant comparison is misleading
1037 /// because this metric is NOT rate-normalized — a longer-
1038 /// running scenario racks up more iterations per worker even if
1039 /// the scheduler is identical. `perf-delta`-style
1040 /// comparisons hold scenario, topology, and work_type constant
1041 /// before reading this method.
1042 pub fn iterations_per_worker(&self) -> Option<f64> {
1043 crate::assert::reductions::iterations_per_worker_of(self.num_workers, self.total_iterations)
1044 }
1045
1046 /// Worker iterations per CPU-second of on-CPU time consumed by this
1047 /// cgroup's workers — `total_iterations / (total_cpu_time_ns / 1e9)`.
1048 ///
1049 /// Unlike [`Self::iterations_per_worker`] (raw work, which scales with
1050 /// the host-CPU budget delivered to the guest) and a wall-time rate
1051 /// (which also drops under host oversubscription), this is
1052 /// OVERCOMMIT-INVARIANT: under `cpu_budget < vcpus` a cell completes
1053 /// proportionally fewer iterations AND consumes proportionally less
1054 /// on-CPU time, so the ratio cancels the lost host-CPU-time factor. Use
1055 /// it to compare per-cgroup throughput across `cpu_budget` settings.
1056 ///
1057 /// `None` when `num_workers == 0` (no worker — undefined, distinct from a
1058 /// measured zero) or `total_cpu_time_ns == 0` (no on-CPU time captured;
1059 /// returns inconclusive rather than `Inf`). For a pure busy-spin
1060 /// workload this rate is ~constant by construction, so it measures
1061 /// CPU-time EFFICIENCY; for the cross-cell ALLOCATION balance use
1062 /// [`ScenarioStats::cgroup_balance_ratio`] over `iterations_per_worker`.
1063 pub fn iterations_per_cpu_sec(&self) -> Option<f64> {
1064 crate::assert::reductions::iterations_per_cpu_sec_of(
1065 self.num_workers,
1066 self.total_cpu_time_ns,
1067 self.total_iterations,
1068 )
1069 }
1070}
1071
1072/// Identifier for a scenario phase. Newtype over `u16` carrying
1073/// the same 1-indexed encoding documented on every other
1074/// phase-touching site: `Phase::BASELINE` is the pre-first-Step
1075/// settle window (`u16` 0); `Phase::step(k)` is scenario Step `k`
1076/// at 1-indexed `u16` `k + 1`. The newtype catches the bug class
1077/// where a raw `u16` flows between sites that disagree about
1078/// 0-indexed vs 1-indexed Step encoding, and gives operators
1079/// readable construction at consumer sites (`Phase::BASELINE` /
1080/// `Phase::step(2)` instead of magic `0u16` / `3u16`).
1081///
1082/// Wire-format identical to a `u16` via `#[serde(transparent)]` —
1083/// the on-disk sidecar shape is unchanged from the bare-`u16`
1084/// pipeline, and existing JSON / typeshare consumers see the same
1085/// scalar field. `.phase_raw()` exposes the inner `u16` for paths
1086/// that hand the value to a serializer or formatter that does not
1087/// understand the newtype.
1088#[derive(
1089 Debug,
1090 Clone,
1091 Copy,
1092 PartialEq,
1093 Eq,
1094 Hash,
1095 PartialOrd,
1096 Ord,
1097 Default,
1098 serde::Serialize,
1099 serde::Deserialize,
1100)]
1101#[serde(transparent)]
1102pub struct Phase(u16);
1103
1104impl Phase {
1105 /// Pre-first-Step settle window. The framework writes
1106 /// `Phase::BASELINE` to `Ctx::current_step` at scenario start
1107 /// (before any Step's `current_step.store` advance), so any
1108 /// capture taken before the first Step transition stamps with
1109 /// this value.
1110 pub const BASELINE: Self = Self(0);
1111
1112 /// Construct a `Phase` for the `zero_indexed`-th scenario Step.
1113 /// The 1-indexed encoding (Step 0 → `u16` 1, Step 1 → `u16` 2,
1114 /// ...) keeps `BASELINE` unambiguous at `u16` 0. Saturates at
1115 /// `u16::MAX` rather than overflowing — a scenario with > 65k
1116 /// Steps is pathological and the saturating value still
1117 /// distinguishes "well past any real Step" from BASELINE.
1118 pub const fn step(zero_indexed: u16) -> Self {
1119 Self(zero_indexed.saturating_add(1))
1120 }
1121
1122 /// True iff this is `Phase::BASELINE` (the pre-first-Step
1123 /// settle window).
1124 pub const fn is_baseline(&self) -> bool {
1125 self.0 == 0
1126 }
1127
1128 /// Inner `u16`. Use this when handing the value to a
1129 /// serializer / formatter / external consumer that does not
1130 /// understand the newtype. Production callers that build a
1131 /// `Phase` for downstream comparison should prefer
1132 /// `Phase::BASELINE` / `Phase::step(k)` over wrapping a raw
1133 /// `u16` themselves.
1134 pub const fn as_u16(self) -> u16 {
1135 self.0
1136 }
1137}
1138
1139impl std::fmt::Display for Phase {
1140 /// `"BASELINE"` for [`Phase::BASELINE`], `"Step[k]"` for
1141 /// [`Phase::step`] (decoded back via the 1-indexed
1142 /// encoding). Matches the labels [`PhaseBucket`] embeds in
1143 /// `label` so operators see consistent phase identifiers
1144 /// across structured-sidecar reads and ad-hoc `format!`
1145 /// output.
1146 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1147 if self.is_baseline() {
1148 write!(f, "BASELINE")
1149 } else {
1150 write!(f, "Step[{}]", self.0 - 1)
1151 }
1152 }
1153}
1154
1155impl From<u16> for Phase {
1156 /// Wrap a raw 1-indexed encoded value as a [`Phase`]. Production
1157 /// paths that already have the encoded value (e.g. drained from
1158 /// the host-side mirror of `current_step`, or read out of a
1159 /// deserialized sidecar) construct the typed wrapper via this
1160 /// conversion without re-deriving the encoding.
1161 fn from(value: u16) -> Self {
1162 Self(value)
1163 }
1164}
1165
1166impl From<Phase> for u16 {
1167 fn from(value: Phase) -> Self {
1168 value.0
1169 }
1170}
1171
1172/// Per-phase metric bucket — one entry per scenario phase in
1173/// [`ScenarioStats::phases`].
1174///
1175/// A scenario with N Steps yields `N + 1` phases: phase 0 is the
1176/// BASELINE (pre-first-Step settle window), and phases 1..=N
1177/// correspond to Step 0..Step N-1 in scenario order. The
1178/// 1-indexed Step encoding (instead of 0-indexed) lets BASELINE
1179/// own `step_index = 0` unambiguously — a `step_index = 0` sample
1180/// is always settle, not first-Step.
1181///
1182/// Each bucket carries the metric values reduced over the phase's
1183/// sample window. For `crate::stats::MetricKind::Counter`
1184/// metrics the reduction is `last - first` across the phase's
1185/// periodic samples (cumulative-counter delta); for `Gauge` /
1186/// `Peak` / `Timestamp` it dispatches per the kind via
1187/// `crate::stats::aggregate_samples`. Missing metric keys mean
1188/// the phase had no finite samples for that metric.
1189///
1190/// Metric keys match `crate::stats::MetricDef::name` — see
1191/// `crate::stats::METRICS` for the canonical list of registered
1192/// metric names a `get` / `phase_metric` lookup expects.
1193#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize, crate::Claim)]
1194pub struct PhaseBucket {
1195 /// Phase index. `0` = BASELINE (pre-first-Step settle window).
1196 /// `1..=N` align with Step ordinals (1-indexed): Step 0 of the
1197 /// scenario lives at `step_index = 1`, Step 1 at
1198 /// `step_index = 2`, etc. The encoding avoids the collision
1199 /// where a 0-indexed Step would share `step_index = 0` with
1200 /// the BASELINE settle window.
1201 pub step_index: u16,
1202 /// Human-readable label. `"BASELINE"` for `step_index = 0`,
1203 /// `"Step[0]"` / `"Step[1]"` / ... for `step_index = 1..=N`.
1204 /// Mirrors the formatting used by
1205 /// `crate::timeline::Timeline`'s phase rendering so operator
1206 /// inspection of the formatted diagnostic and the structured
1207 /// sidecar yield the same phase identifiers.
1208 pub label: String,
1209 /// Phase window start: the MINIMUM per-sample time anchor in the
1210 /// phase — each sample's `boundary_offset_ms`, falling back to its
1211 /// `elapsed_ms`. Samples with neither anchor (both `None` — a
1212 /// not-measured timestamp) are excluded from the min.
1213 pub start_ms: u64,
1214 /// Phase window end: the MAXIMUM per-sample time anchor in the
1215 /// phase (the same `boundary_offset_ms`-or-`elapsed_ms` key as
1216 /// `start_ms`). A phase whose every sample is unanchored yields the
1217 /// inverted window `(start_ms = u64::MAX, end_ms = 0)`, which folds
1218 /// no monitor samples. Downstream renderers should not assume the
1219 /// value is closed against a stimulus event.
1220 pub end_ms: u64,
1221 /// Number of periodic samples bucketed into this phase. Zero
1222 /// when the phase fired no captures: BASELINE when the settle window
1223 /// was shorter than the periodic interval, OR a synthesized
1224 /// capture-free interior step (the
1225 /// `build_phase_buckets_with_stimulus` seam — a `StepStart`-step
1226 /// whose window held no periodic boundary still gets a bucket so its
1227 /// capture-independent `iteration_rate` is not dropped).
1228 pub sample_count: usize,
1229 /// Per-metric phase-aggregated values. See the [`PhaseBucket`]
1230 /// struct doc for the registry key source and per-kind reduction
1231 /// dispatch; missing keys mean the phase carried no finite
1232 /// samples for that metric (sentinel-free: `None` from the
1233 /// reducer surfaces as "key absent" rather than "value 0.0").
1234 pub metrics: std::collections::BTreeMap<String, f64>,
1235 /// Per-cgroup raw telemetry components for this phase, keyed by cgroup
1236 /// name (see [`PhaseCgroupStats`]). Empty until a capture path populates
1237 /// it; the structural carrier for the per-phase per-cgroup distributional
1238 /// re-pool. Whole-run = aggregate of these per-phase per-cgroup components.
1239 ///
1240 /// An ORPHAN bucket — a guest carrier whose `step_index` has NO paired host
1241 /// bucket (a dropped/absent StepStart frame, or a stimulus-less host/fixture
1242 /// path; NOT merely a short step, since `build_phase_buckets_with_stimulus`
1243 /// synthesizes a bucket for every StepStart so a captured-but-short step
1244 /// takes the matched arm) — is carried by
1245 /// `fold_guest_per_cgroup_into_host_buckets` with the shape
1246 /// `(start_ms, end_ms) == (0, 0)` AND empty `metrics` AND non-empty
1247 /// `per_cgroup` (it carries only these components). On every non-zero-duration
1248 /// window that shape is the orphan arm's, so the timeline render keys on it
1249 /// to surface "window not measured" rather than a misleading `0ms` (see
1250 /// `crate::timeline::phase_from_bucket`): a captured bucket has metrics. A
1251 /// zero-duration step at scenario start (`StepStart==StepEnd==0`) can also
1252 /// produce it via the matched arm, but harmlessly — a zero-duration step has
1253 /// no window, so "not measured" reads the same as "0ms".
1254 pub per_cgroup: std::collections::BTreeMap<String, PhaseCgroupStats>,
1255}
1256
1257impl PhaseBucket {
1258 /// Look up the phase-aggregated value for `metric_name` (see
1259 /// [`PhaseBucket::metrics`] for the registry source). Returns
1260 /// `None` when the phase carried no finite samples for that
1261 /// metric — distinct from `Some(0.0)` which means the reducer
1262 /// produced a real zero from finite samples.
1263 pub fn get(&self, metric_name: &str) -> Option<f64> {
1264 self.metrics.get(metric_name).copied()
1265 }
1266
1267 /// Like [`Self::get`], but panics with a diagnostic message citing
1268 /// the bucket's `step_index` + `label` + `sample_count` + the set
1269 /// of metric keys actually present when the metric is absent. Use
1270 /// when the caller knows the metric MUST be in the bucket (the
1271 /// phase fired samples and the metric is registered — see
1272 /// [`PhaseBucket::metrics`]) — the panic message tells the operator whether the cause is
1273 /// "phase produced no samples" (sample_count of 0) or "metric key
1274 /// typo" (positive sample_count but the key isn't in `metrics`).
1275 ///
1276 /// ```ignore
1277 /// let bucket = r.stats.phase(Phase::step(0)).expect("Step[0] phase");
1278 /// let throughput = bucket.expect_metric("throughput");
1279 /// ```
1280 pub fn expect_metric(&self, metric_name: &str) -> f64 {
1281 self.get(metric_name).unwrap_or_else(|| {
1282 panic!(
1283 "PhaseBucket::expect_metric: metric '{}' absent from phase \
1284 step_index={} ('{}') with sample_count={}. \
1285 metric keys present in this bucket: {:?}. \
1286 Possible causes: (a) phase carried 0 samples for this \
1287 metric (sample_count==0 means no captures landed in the \
1288 phase at all; sample_count>0 means captures landed but \
1289 the metric extracted no finite values from them); \
1290 (b) metric name typo — built-ins are typed via \
1291 crate::stats::BuiltinMetric (a typo is a compile \
1292 error); a dynamic key resolves via crate::stats::MetricId::def.",
1293 metric_name,
1294 self.step_index,
1295 self.label,
1296 self.sample_count,
1297 self.metrics.keys().collect::<Vec<_>>(),
1298 )
1299 })
1300 }
1301
1302 /// Cross-cgroup phase total for a per-cgroup Counter metric that lives
1303 /// in [`Self::per_cgroup`] but never in [`Self::metrics`] — currently
1304 /// `"total_migrations"`, `"total_iterations"`, and `"total_cpu_time_ns"`.
1305 ///
1306 /// These are registered `MetricKind::Counter`s
1307 /// (`crate::stats::METRICS`) whose per-sample source is absent
1308 /// (`crate::stats::MetricDef::read_sample` returns `None` — they are
1309 /// per-task guest counters not captured per tick), so
1310 /// [`crate::assert::build_phase_buckets`] never folds them into
1311 /// `metrics`; only the per-cgroup carrier ([`PhaseCgroupStats`], built
1312 /// by `phase_cgroup_stats`) holds them. This sums them across the
1313 /// phase's `per_cgroup` carriers — the SAME cross-cgroup Counter sum
1314 /// [`ScenarioStats::total_migrations`] takes run-level and
1315 /// `merge_matched_phase_buckets` takes per key — so the value is the
1316 /// phase total, not a per-cgroup fragment.
1317 ///
1318 /// `None` when the phase has no `per_cgroup` carriers (NOT measured —
1319 /// distinct from a measured `Some(0.0)` when carriers exist but counted
1320 /// zero) or `name` is not one of the per-cgroup-sourced counters.
1321 /// Surfacing them here never double-sources the run-level `ext_metrics`,
1322 /// but by two different mechanisms: `total_migrations` / `total_iterations`
1323 /// are in `TYPED_FIELD_NAMES` (`populate_run_ext_metrics_from_phases` skips
1324 /// them; the typed `GauntletRow` accessor stays the single run-level
1325 /// authority), while `total_cpu_time_ns` is NOT in `TYPED_FIELD_NAMES` —
1326 /// it is safe because it is never written into any pooled `metrics` map
1327 /// (its `MetricDef` accessor and `read_sample` both return `None`, so no
1328 /// run-level fold ever picks it up; it lives only on the per-cgroup
1329 /// carrier).
1330 ///
1331 /// Counterpart to [`Self::get`] (which reads `metrics` only).
1332 /// [`crate::vmm::VmResult::phase_metric`] falls back to this so a
1333 /// `post_vm` callback reading `phase_metric(phase, "total_migrations")`
1334 /// gets the value instead of a silent `None`.
1335 pub fn cgroup_counter_total(&self, name: &str) -> Option<f64> {
1336 let field: fn(&PhaseCgroupStats) -> u64 = match name {
1337 "total_migrations" => |c| c.total_migrations,
1338 "total_iterations" => |c| c.total_iterations,
1339 "total_cpu_time_ns" => |c| c.total_cpu_time_ns,
1340 _ => return None,
1341 };
1342 if self.per_cgroup.is_empty() {
1343 return None;
1344 }
1345 // saturating: overflow-safe cross-cgroup pool of the guest counter for
1346 // the post_vm by-name read (a wrapped sum would read silently-small).
1347 Some(
1348 self.per_cgroup
1349 .values()
1350 .map(field)
1351 .fold(0u64, u64::saturating_add) as f64,
1352 )
1353 }
1354}
1355
1356/// Merge two [`PhaseBucket`]s sharing the same `step_index` per
1357/// the per-MetricKind dispatch in [`crate::stats::MergeKind`].
1358/// Called by [`AssertResult::merge`] for matched buckets;
1359/// unmatched buckets are appended verbatim by the caller.
1360///
1361/// Window-invariant merge:
1362/// - `step_index`: equal by precondition (caller pairs buckets by
1363/// `step_index`), kept from `a`.
1364/// - `label`: kept from `a`. By construction the label is derived
1365/// purely from `step_index` (`"BASELINE"` / `"Step[k]"`) so both
1366/// sides agree.
1367/// - `start_ms`: `min(a.start_ms, b.start_ms)` so the merged
1368/// window covers the earliest start of either side.
1369/// - `end_ms`: `max(a.end_ms, b.end_ms)` so the merged window
1370/// covers the latest end. Drives the [`crate::stats::MergeKind::NonCommutative`]
1371/// tiebreak on Gauge(Last) / Timestamp metrics — the value
1372/// from the bucket whose `end_ms` is later wins.
1373/// - `sample_count`: `a + b`. Used as the weighting denominator
1374/// for the `MetricKind::Gauge(GaugeAgg::Avg)` weighted mean.
1375///
1376/// Per-metric merge dispatches on the metric's `crate::stats::MetricKind`
1377/// from the registry via [`crate::stats::metric_def`]:
1378/// - `MetricKind::Counter` → `a + b` (the two reduced values are
1379/// per-phase deltas; the merge across cgroups sums per-cgroup
1380/// contributions to the phase delta, mirroring how
1381/// `ScenarioStats::total_migrations` adds across cgroups).
1382/// - `MetricKind::Peak` and `MetricKind::Gauge(GaugeAgg::Max)` →
1383/// `max(a, b)` (the worst-case "peak that fired" survives).
1384/// - `MetricKind::Gauge(GaugeAgg::Avg)` → weighted mean
1385/// `(a * a_w + b * b_w) / (a_w + b_w)` where `a_w = a_count.max(1)`
1386/// and `b_w = b_count.max(1)` — the unbiased combination of both
1387/// sides' per-phase means weighted by sample population, each weight
1388/// floored at 1. The `.max(1)` floor (mirroring
1389/// `populate_run_ext_metrics_from_phases`) keeps a synthesized
1390/// zero-capture bucket's capture-independent Gauge(Avg) value
1391/// contributing one phase-observation of weight rather than being
1392/// zero-weighted out of
1393/// the merge — the silent-drop the synthesize seam exists to prevent.
1394/// With both counts > 0 the floor is a no-op (the plain
1395/// sample-population weighting); both counts zero degenerates to
1396/// `(a + b) / 2.0`.
1397/// - `MetricKind::Gauge(GaugeAgg::Last)` and `MetricKind::Timestamp`
1398/// → value from the bucket with the larger `end_ms`; ties keep
1399/// `a`'s value. Captures the "latest-sample-wins" semantic per
1400/// the [`crate::stats::MergeKind::NonCommutative`] contract.
1401/// - `MetricKind::Rate { .. }` → SKIPPED in the per-key fold and
1402/// re-derived from the pooled components by
1403/// [`crate::stats::derive_rate_metrics`] as a post-pass, so the
1404/// merged rate is `Σnumerator / Σdenominator` (each component
1405/// folds by its own kind first) rather than a fold of two
1406/// ready-made per-phase ratios.
1407///
1408/// Unregistered metric names (not in `crate::stats::METRICS`)
1409/// fall back to a commutative arithmetic mean
1410/// `(a + b) / 2.0`. The mean is the safest default for an unknown
1411/// kind: sum would over-count Gauge / Timestamp values, max would
1412/// lose Counter / Avg signal, and "last" requires a tiebreak the
1413/// caller can't compute without the kind. Producers attaching
1414/// unregistered metrics to a `PhaseBucket` should add them to
1415/// `METRICS` to get the typed merge instead of the fallback.
1416pub(crate) fn merge_matched_phase_buckets(a: PhaseBucket, b: PhaseBucket) -> PhaseBucket {
1417 assert_eq!(
1418 a.step_index, b.step_index,
1419 "merge_matched_phase_buckets: caller must pair by step_index",
1420 );
1421 let mut metrics = std::collections::BTreeMap::new();
1422 // Collect every key present on either side; iterate once,
1423 // dispatching per the kind of the key (or the unregistered
1424 // mean fallback) so the merge is single-pass.
1425 let mut keys: std::collections::BTreeSet<&String> = a.metrics.keys().collect();
1426 keys.extend(b.metrics.keys());
1427 for key in keys {
1428 // Derived metrics (every `is_derived()`: Rate / Distribution / WorstLowest /
1429 // WakeLatencyTailRatio / WorstCrossNodeRatio / PerPhase) are NOT merged
1430 // here: a Rate re-derives from the merged components in the post-pass
1431 // below; the distributional kinds (Distribution / WorstLowest /
1432 // WakeLatencyTailRatio / WorstCrossNodeRatio) re-pool run-level by
1433 // `populate_run_distribution_metrics` (they never appear in
1434 // phase.metrics — `aggregate_samples_for_phase` returns None — so this
1435 // skip is also a structural guard); PerPhase is re-derived by
1436 // `derive_phase_metrics`. Folding a ready-made derived value
1437 // would lose the re-pool.
1438 if crate::stats::metric_def(key).is_some_and(|m| m.kind.is_derived()) {
1439 continue;
1440 }
1441 let av = a.metrics.get(key).copied();
1442 let bv = b.metrics.get(key).copied();
1443 let merged = match (av, bv) {
1444 (Some(av), Some(bv)) => {
1445 let kind = crate::stats::metric_def(key).map(|m| m.kind);
1446 merge_metric_values(
1447 kind,
1448 av,
1449 bv,
1450 a.sample_count,
1451 b.sample_count,
1452 a.end_ms,
1453 b.end_ms,
1454 )
1455 }
1456 (Some(v), None) | (None, Some(v)) => v,
1457 (None, None) => continue,
1458 };
1459 metrics.insert(key.clone(), merged);
1460 }
1461 // Re-derive Rate metrics from the now-pooled components: each
1462 // component merged by its own kind above (a Counter numerator
1463 // summed), so the rate becomes Σnumerator / Σdenominator — the
1464 // correct re-pool, not a mean of the two phases' ready-made ratios.
1465 crate::stats::derive_rate_metrics(&mut metrics);
1466 // Union per_cgroup by cgroup name: a cgroup present on both sides folds
1467 // its raw components per PhaseCgroupStats::merge (concat samples, sum
1468 // counters, combine extremes); a cgroup on only one side is carried
1469 // verbatim. Empty ∪ empty = empty, so this is a no-op until a capture
1470 // path populates per_cgroup (the structural-carrier invariant).
1471 let mut per_cgroup = a.per_cgroup;
1472 for (name, b_cg) in b.per_cgroup {
1473 match per_cgroup.remove(&name) {
1474 Some(a_cg) => {
1475 per_cgroup.insert(name, PhaseCgroupStats::merge(a_cg, b_cg));
1476 }
1477 None => {
1478 per_cgroup.insert(name, b_cg);
1479 }
1480 }
1481 }
1482 PhaseBucket {
1483 step_index: a.step_index,
1484 label: a.label,
1485 start_ms: a.start_ms.min(b.start_ms),
1486 end_ms: a.end_ms.max(b.end_ms),
1487 sample_count: a.sample_count.saturating_add(b.sample_count),
1488 metrics,
1489 per_cgroup,
1490 }
1491}
1492
1493/// Fold the guest-collected per-phase `per_cgroup` carriers into the
1494/// host-rebuilt phase buckets, keyed by `step_index`.
1495///
1496/// The host rebuilds phase buckets from the periodic-capture series
1497/// (window + metric folds), but those buckets carry an empty `per_cgroup`
1498/// by construction. The guest collects per-cgroup RAW components per step
1499/// ([`crate::scenario::collect_handles`] under `collect_step`) into carrier
1500/// buckets whose only payload is `per_cgroup` — a merge-neutral
1501/// `(u64::MAX, 0)` window and empty `metrics`. Guest and host `step_index`
1502/// are the SAME 1-indexed value: the step loop stamps
1503/// `phase_step_index = step_idx + 1` onto BOTH the `StepStart` frames the
1504/// host rebuilds buckets from AND the `collect_step` carrier, so pairing by
1505/// `step_index` is exact and cannot drift.
1506///
1507/// Each guest carrier whose `step_index` matches a host bucket folds its
1508/// `per_cgroup` in via [`merge_matched_phase_buckets`] — a no-op on the
1509/// host's window (`min`/`max` against `MAX`/`0`), metrics (the carrier has
1510/// none, so each host key is carried verbatim), and `sample_count` (`+ 0`),
1511/// contributing ONLY the unioned `per_cgroup`. A guest `step_index` with no
1512/// host bucket — a DEFENSIVE case: the carrier's `step_index` has no `StepStart`
1513/// frame in the host stimulus timeline (a dropped/absent stimulus frame, or a
1514/// stimulus-less host/fixture path), since `build_phase_buckets_with_stimulus`
1515/// SYNTHESIZES a capture-free bucket for every StepStart-step, so a
1516/// captured-but-short step takes the matched arm above, not this one — is carried
1517/// verbatim with its window normalized to `(0, 0)` so duration consumers
1518/// (`end_ms - start_ms`) never underflow the merge-neutral sentinel — no
1519/// `per_cgroup` datum is silently dropped. With no guest carriers (a run
1520/// with no step-local cgroups) the host buckets pass through unchanged. The
1521/// returned vec is sorted by `step_index`.
1522pub(crate) fn fold_guest_per_cgroup_into_host_buckets(
1523 host_buckets: Vec<PhaseBucket>,
1524 guest_buckets: Vec<PhaseBucket>,
1525) -> Vec<PhaseBucket> {
1526 let host_len = host_buckets.len();
1527 // No-silent-drops: host buckets have unique step_index
1528 // (build_phase_buckets_with_stimulus emits one bucket per step_index), but
1529 // fold same-step_index duplicates via merge rather than a last-wins collect so
1530 // a future producer that violated the invariant DEGRADES to a merge, never a
1531 // silent release-mode drop. The debug_assert still trips loudly in test/debug.
1532 let mut by_idx: std::collections::BTreeMap<u16, PhaseBucket> =
1533 std::collections::BTreeMap::new();
1534 for b in host_buckets {
1535 match by_idx.remove(&b.step_index) {
1536 Some(existing) => {
1537 by_idx.insert(b.step_index, merge_matched_phase_buckets(existing, b));
1538 }
1539 None => {
1540 by_idx.insert(b.step_index, b);
1541 }
1542 }
1543 }
1544 debug_assert_eq!(
1545 by_idx.len(),
1546 host_len,
1547 "host buckets must have unique step_index; a collision merged (not dropped)",
1548 );
1549 for gb in guest_buckets {
1550 // Every guest carrier MUST carry the merge-neutral (u64::MAX, 0) sentinel
1551 // window (the step_per_cgroup_bucket invariant). Validate it BEFORE the
1552 // match so BOTH arms are guarded: the matched arm relies on the window
1553 // being merge-neutral (min/max no-op against the host window), and the
1554 // orphan arm normalizes it to (0,0). A future caller handing a
1555 // real-window carrier (incl. a duplicate orphan via the matched arm)
1556 // trips loudly instead of silently corrupting the merged window.
1557 debug_assert!(
1558 gb.start_ms == u64::MAX && gb.end_ms == 0,
1559 "guest carrier must carry the merge-neutral (u64::MAX, 0) window; got ({}, {})",
1560 gb.start_ms,
1561 gb.end_ms,
1562 );
1563 match by_idx.remove(&gb.step_index) {
1564 Some(hb) => {
1565 by_idx.insert(gb.step_index, merge_matched_phase_buckets(hb, gb));
1566 }
1567 None => {
1568 // Orphan arm: a guest carrier whose step_index has no host bucket.
1569 //
1570 // Invariant: build_phase_buckets_with_stimulus synthesizes a host
1571 // bucket for every StepStart-step, so a carrier whose step has a
1572 // StepStart frame always takes the matched arm above. This arm is
1573 // reached only by a carrier whose step has NO StepStart frame —
1574 // defensive, not produced by normal capture.
1575 //
1576 // Normalize the merge-neutral sentinel window to (0,0) so duration
1577 // consumers don't underflow it. The resulting (0,0)-window +
1578 // empty-metrics + non-empty-per_cgroup shape is the orphan
1579 // signature the timeline render keys on to show "window not
1580 // measured" instead of a misleading 0ms — the (0,0) means "no host
1581 // window known", NOT a measured zero-duration step.
1582 //
1583 // A zero-duration step at scenario start (StepStart==StepEnd==0)
1584 // produces the same shape via the matched arm, but harmlessly: a
1585 // zero-duration step has no window, so the render's "not measured"
1586 // reads the same as "0ms". See `crate::timeline::phase_from_bucket`.
1587 let mut orphan = gb;
1588 orphan.start_ms = 0;
1589 orphan.end_ms = 0;
1590 by_idx.insert(orphan.step_index, orphan);
1591 }
1592 }
1593 }
1594 by_idx.into_values().collect()
1595}
1596
1597/// Per-metric merge inner helper used by
1598/// [`merge_matched_phase_buckets`]. Dispatches on the metric's
1599/// `crate::stats::MetricKind` (or the unregistered fallback)
1600/// to combine two reduced values into one.
1601///
1602/// `a_count` / `b_count` are the source buckets' `sample_count`
1603/// fields, used as weights for `Gauge(Avg)`. `a_end_ms` /
1604/// `b_end_ms` are the source buckets' window-end timestamps,
1605/// used to pick the later sample for `Gauge(Last)` / `Timestamp`.
1606fn merge_metric_values(
1607 kind: Option<crate::stats::MetricKind>,
1608 a: f64,
1609 b: f64,
1610 a_count: usize,
1611 b_count: usize,
1612 a_end_ms: u64,
1613 b_end_ms: u64,
1614) -> f64 {
1615 use crate::stats::{GaugeAgg, MetricKind};
1616 match kind {
1617 // Counter (cumulative) and DeltaSum (sum of per-read deltas) merge
1618 // across AssertResults by summing the reduced values (commutative — see
1619 // MetricKind::merge_kind). PerPhaseDeltaSum shares the `a + b` arm, but
1620 // it is a run-wide POOLED per-phase scalar (injected once into the host
1621 // bucket, never per-cgroup), so this same-step-index merge is the
1622 // defensive path only — two real PerPhaseDeltaSum values are never
1623 // summed here.
1624 Some(MetricKind::Counter)
1625 | Some(MetricKind::DeltaSum)
1626 | Some(MetricKind::PerPhaseDeltaSum) => a + b,
1627 Some(MetricKind::Peak) | Some(MetricKind::Gauge(GaugeAgg::Max)) => a.max(b),
1628 Some(MetricKind::Gauge(GaugeAgg::Avg)) => {
1629 // Weight by sample_count, floored at 1: a sample_count==0
1630 // bucket carrying a capture-independent Gauge(Avg) value must
1631 // contribute one phase-observation of weight, not be
1632 // zero-weighted out of the merge. Mirrors the .max(1) floor in
1633 // populate_run_ext_metrics_from_phases. With both counts > 0
1634 // the floor is a no-op (the prior sample_count weighting);
1635 // with both 0 each still floors to weight 1, giving the
1636 // (a+b)/2 equal-weight mean (the aggregate_samples_weighted
1637 // zero-total-weight fallback is unreachable from here).
1638 // (iteration_rate — the original synthesized zero-capture case —
1639 // is now a MetricKind::Rate: merge_matched_phase_buckets skips it
1640 // via the Rate `continue` in its key loop (above this fn) and
1641 // re-pools it from its summed Counter components, so a Rate value
1642 // never reaches this Gauge(Avg) fold.)
1643 let a_w = a_count.max(1) as f64;
1644 let b_w = b_count.max(1) as f64;
1645 (a * a_w + b * b_w) / (a_w + b_w)
1646 }
1647 Some(MetricKind::Gauge(GaugeAgg::Last)) | Some(MetricKind::Timestamp) => {
1648 if b_end_ms > a_end_ms { b } else { a }
1649 }
1650 // Derived kinds (every `is_derived()`: Rate / Distribution / WorstLowest /
1651 // WakeLatencyTailRatio / WorstCrossNodeRatio / PerPhase /
1652 // PerRunDistribution) are skipped in the merge loop (see
1653 // `merge_matched_phase_buckets`'s `is_derived` continue) and produced
1654 // post-merge (`derive_rate_metrics` / `populate_run_distribution_metrics`
1655 // / `populate_run_pooled_schbench_distribution`), so a derived value never
1656 // reaches this per-value merge — folding a ready-made derived value
1657 // would lose the re-pool. (PerRunDistribution is additionally run-level
1658 // only: it never appears in a PhaseBucket, so it cannot reach this
1659 // per-phase-bucket merge at all.)
1660 Some(MetricKind::Rate { .. })
1661 | Some(MetricKind::Distribution { .. })
1662 | Some(MetricKind::WorstLowest { .. })
1663 | Some(MetricKind::WakeLatencyTailRatio)
1664 | Some(MetricKind::WorstCrossNodeRatio)
1665 | Some(MetricKind::PerPhase)
1666 | Some(MetricKind::PerRunDistribution) => unreachable!(
1667 "derived metrics (Rate/Distribution/WorstLowest/WakeLatencyTailRatio/WorstCrossNodeRatio/PerPhase/PerRunDistribution) are produced post-merge, not merged as values"
1668 ),
1669 // Unregistered metric: commutative mean fallback. Sum
1670 // would over-count Gauge values; max would lose Counter
1671 // signal; "last" needs a tiebreak the caller can't
1672 // compute without the kind. Mean is the safest commutative
1673 // default.
1674 None => (a + b) / 2.0,
1675 }
1676}