ktstr/assert/reductions.rs
1use super::*;
2
3/// Check that workers only ran on CPUs in `expected`.
4///
5/// Any worker that used a CPU outside the expected set produces a
6/// failure with the unexpected CPU IDs listed.
7///
8/// ```
9/// # use ktstr::assert::assert_isolation;
10/// # use ktstr::workload::WorkerReport;
11/// # use std::collections::BTreeSet;
12/// # let report = WorkerReport {
13/// # tid: 1, cpus_used: [0, 1].into_iter().collect(),
14/// # work_units: 100, cpu_time_ns: 1_000_000, wall_time_ns: 2_000_000,
15/// # off_cpu_ns: 1_000_000, migration_count: 0, migrations: vec![],
16/// # max_gap_ms: 0, max_gap_cpu: 0, max_gap_at_ms: 0,
17/// # wake_latencies_ns: vec![], wake_sample_total: 0,
18/// # iteration_costs_ns: vec![], iteration_cost_sample_total: 0,
19/// # iterations: 0,
20/// # schedstat_run_delay_ns: 0, schedstat_run_count: 0,
21/// # schedstat_cpu_time_ns: 0,
22/// # completed: true,
23/// # numa_pages: std::collections::BTreeMap::new(),
24/// # vmstat_numa_pages_migrated: 0,
25/// # exit_info: None,
26/// # is_messenger: false,
27/// # ..Default::default()
28/// # };
29/// let expected: BTreeSet<usize> = [0, 1, 2].into_iter().collect();
30/// assert!(assert_isolation(&[report], &expected).is_pass());
31/// ```
32pub fn assert_isolation(reports: &[WorkerReport], expected: &BTreeSet<usize>) -> AssertResult {
33 let mut r = AssertResult::pass();
34 for w in reports {
35 let bad: BTreeSet<usize> = w.cpus_used.difference(expected).copied().collect();
36 if !bad.is_empty() {
37 r.record_fail(AssertDetail::new(
38 DetailKind::Isolation,
39 format!("tid {} ran on unexpected CPUs {:?}", w.tid, bad),
40 ));
41 }
42 }
43 r
44}
45
46/// Nearest-rank percentile of a sorted slice (`p` in `[0.0, 1.0]`).
47///
48/// Returns the value at index `ceil(n * p) - 1`, clamped into
49/// `[0, n-1]`. For `n = 100` and `p = 0.99` this is `sorted[98]` (the
50/// 99th element in 1-indexed order), not `sorted[99]` (the max). The
51/// previous formulation, `ceil(n * 0.99)` without the `-1`, was
52/// off-by-one and returned the max for `n = 100`.
53///
54/// # Preconditions
55///
56/// `sorted` must be non-decreasing. The function indexes by rank
57/// without checking order, so an unsorted input silently returns
58/// the value at the computed index — a meaningless number. A
59/// `debug_assert!` enforces this in debug builds; release builds
60/// skip the check (the production callers sort immediately upstream
61/// — `cgroup_stats` and `assert_benchmarks` both
62/// `sorted.sort_unstable()` before this call — so the runtime
63/// guard is unnecessary in production paths).
64///
65/// An empty slice yields `0` (the caller should short-circuit
66/// before invoking).
67pub(crate) fn percentile(sorted: &[u64], p: f64) -> u64 {
68 if sorted.is_empty() {
69 return 0;
70 }
71 debug_assert!(
72 sorted.windows(2).all(|w| w[0] <= w[1]),
73 "percentile() requires sorted input; got slice with out-of-order pair",
74 );
75 let n = sorted.len();
76 let idx = ((n as f64 * p).ceil() as usize)
77 .saturating_sub(1)
78 .min(n - 1);
79 sorted[idx]
80}
81
82/// Build per-cgroup telemetry (pure measurement, no assertions) from
83/// worker reports. This is the SINGLE telemetry builder on the assertion
84/// path: `AssertPlan::assert_cgroup` calls it unconditionally and
85/// [`assert_not_stuck`] wraps it with the default fairness checks, so
86/// per-cgroup [`CgroupStats`] is never gated behind whether a worker-check
87/// assertion was configured. Empty `reports` yield a `num_workers == 0`
88/// `CgroupStats` (the reduces below collapse to 0.0/0), so a declared
89/// cgroup that collected no reports surfaces as a zero-worker entry rather
90/// than silently vanishing from [`ScenarioStats::cgroups`].
91pub fn cgroup_stats(reports: &[WorkerReport]) -> CgroupStats {
92 let cpus: BTreeSet<usize> = reports
93 .iter()
94 .flat_map(|w| w.cpus_used.iter().copied())
95 .collect();
96 let pcts: Vec<f64> = reports
97 .iter()
98 .filter(|w| w.wall_time_ns > 0)
99 .map(|w| w.off_cpu_ns as f64 / w.wall_time_ns as f64 * 100.0)
100 .collect();
101
102 // None when no worker had measurable wall time (pcts empty):
103 // off-CPU% is undefined, and a not-measured cgroup must not read
104 // as a measured 0% / spread-0 (perfectly fair) one. Some(_)
105 // otherwise, including a real measured zero.
106 let min = pcts.iter().cloned().reduce(f64::min);
107 let max = pcts.iter().cloned().reduce(f64::max);
108 let avg = if pcts.is_empty() {
109 None
110 } else {
111 Some(pcts.iter().sum::<f64>() / pcts.len() as f64)
112 };
113 let spread = match (min, max) {
114 (Some(lo), Some(hi)) => Some(hi - lo),
115 _ => None,
116 };
117
118 let worst_gap = reports.iter().max_by_key(|w| w.max_gap_ms);
119 let (gap_ms, gap_cpu) = worst_gap
120 .map(|w| (w.max_gap_ms, w.max_gap_cpu))
121 .unwrap_or((0, 0));
122
123 // Compute benchmarking stats from worker reports.
124 let all_latencies: Vec<u64> = reports
125 .iter()
126 .flat_map(|w| w.wake_latencies_ns.iter().copied())
127 .collect();
128 let (p99_us, median_us, lat_cv) = if all_latencies.is_empty() {
129 (0.0, 0.0, 0.0)
130 } else {
131 let mut sorted = all_latencies.clone();
132 sorted.sort_unstable();
133 let p99 = percentile(&sorted, 0.99) as f64 / 1000.0;
134 // Median routes through `percentile(sorted, 0.5)` so the
135 // nearest-rank algorithm matches every other percentile in
136 // the project (p99, schbench's `lat99`, the BPF latency
137 // histograms). A bare `sorted[n/2]` would pick the upper of
138 // the two middle samples for even `n`, while `percentile`
139 // returns the value at `ceil(n * 0.5) - 1` — the lower of
140 // the two middles — and that lower-bound convention is what
141 // the docs on [`CgroupStats::median_wake_latency_us`] and
142 // the schbench cross-reference promise.
143 let median = percentile(&sorted, 0.5) as f64 / 1000.0;
144 let n = all_latencies.len() as f64;
145 let mean_ns = all_latencies.iter().sum::<u64>() as f64 / n;
146 let cv = if mean_ns > 0.0 {
147 let variance = all_latencies
148 .iter()
149 .map(|&v| (v as f64 - mean_ns).powi(2))
150 .sum::<f64>()
151 / n;
152 variance.sqrt() / mean_ns
153 } else {
154 0.0
155 };
156 (p99, median, cv)
157 };
158
159 // Timer-latency reductions (WorkType::TimerLatency), pooled across the
160 // cgroup's workers exactly like the wake-latency block above but over the
161 // distinct `timer_latencies_ns` reservoir. p999 is the deep RT tail; worst
162 // is the single maximum. All `0.0` when no worker recorded timer samples.
163 let all_timer: Vec<u64> = reports
164 .iter()
165 .flat_map(|w| w.timer_latencies_ns.iter().copied())
166 .collect();
167 let (median_timer, p99_timer, p999_timer, worst_timer) = if all_timer.is_empty() {
168 (0.0, 0.0, 0.0, 0.0)
169 } else {
170 let mut sorted = all_timer.clone();
171 sorted.sort_unstable();
172 (
173 percentile(&sorted, 0.5) as f64 / 1000.0,
174 percentile(&sorted, 0.99) as f64 / 1000.0,
175 percentile(&sorted, 0.999) as f64 / 1000.0,
176 *sorted.last().expect("non-empty: checked above") as f64 / 1000.0,
177 )
178 };
179
180 // saturating folds throughout this builder: pool the per-worker guest-runtime
181 // counters (iterations, migrations, cpu-time-ns, sample totals, numa pages)
182 // overflow-safely. This worker->cgroup fold is the FIRST cross-source sum and
183 // the one most exposed to a corrupt/hostile WorkerReport; a u64::MAX component
184 // clamps to u64::MAX instead of debug-panicking / release-wrapping a per-cgroup
185 // total (and the derived migration-ratio / page-locality) to a silently-wrong
186 // value. saturating_add is exact for every in-range value.
187 let total_iters: u64 = reports
188 .iter()
189 .map(|w| w.iterations)
190 .fold(0u64, u64::saturating_add);
191 let run_delays: Vec<f64> = reports
192 .iter()
193 .map(|w| w.schedstat_run_delay_ns as f64 / 1000.0)
194 .collect();
195 let mean_run_delay = if run_delays.is_empty() {
196 0.0
197 } else {
198 run_delays.iter().sum::<f64>() / run_delays.len() as f64
199 };
200 let worst_run_delay = run_delays.iter().cloned().reduce(f64::max).unwrap_or(0.0);
201
202 let total_mig: u64 = reports
203 .iter()
204 .map(|w| w.migration_count)
205 .fold(0u64, u64::saturating_add);
206 let mig_ratio = migration_ratio_of(total_mig, total_iters);
207
208 // Cross-node page-migration ratio: pages migrated cross-node over the
209 // cgroup's total allocated pages. `vmstat_numa_pages_migrated` is a
210 // system-wide delta each worker captured over its own loop; concurrent
211 // workers observe overlapping deltas, so take the MAX across the cgroup
212 // (summing would inflate by the worker count) over the cgroup-wide
213 // total of allocated pages. Pure measurement — populated whenever NUMA
214 // pages were seen, 0.0 otherwise. (The `max_cross_node_migration_ratio`
215 // CHECK in `AssertPlan::assert_cgroup` recomputes the same raw counts
216 // for its diagnostic; this is the always-on telemetry.)
217 let total_numa_pages: u64 = reports
218 .iter()
219 .map(|w| {
220 w.numa_pages
221 .values()
222 .copied()
223 .fold(0u64, u64::saturating_add)
224 })
225 .fold(0u64, u64::saturating_add);
226 let migrated_pages: u64 = reports
227 .iter()
228 .map(|w| w.vmstat_numa_pages_migrated)
229 .max()
230 .unwrap_or(0);
231 let cross_node_ratio = cross_node_migration_ratio_of(migrated_pages, total_numa_pages);
232
233 // Whole-run taobench engine COUNTER aggregate, pooled across this cgroup's
234 // `WorkType::Taobench` workers: Σ ops, MAX wall window (shared across the
235 // concurrent workers), per `TaobenchStats::merge`. `None` for a cgroup with no
236 // Taobench worker. The run-level cross-cgroup pool + the qps/hit Rate
237 // derivation happen in `populate_run_pooled_taobench`; this is the per-cgroup
238 // raw carrier, the taobench analogue of `total_iterations`. Counters only —
239 // serve latency is per-phase data (the `taobench_serve_*_us_whole` keys union
240 // the `PhaseCgroupStats::taobench` histograms).
241 let taobench_whole = reports
242 .iter()
243 .filter_map(|w| w.taobench_whole.as_ref())
244 .fold(
245 None,
246 |acc: Option<crate::workload::taobench::run::TaobenchStats>, t| {
247 Some(match acc {
248 Some(mut a) => {
249 a.merge(t);
250 a
251 }
252 None => *t,
253 })
254 },
255 );
256
257 CgroupStats {
258 // Empty here; collect_handles labels the entry post-hoc (it has
259 // the cgroup name in scope, this reports-only builder does not).
260 cgroup_name: String::new(),
261 num_workers: reports.len(),
262 num_cpus: cpus.len(),
263 cpus_used: cpus,
264 avg_off_cpu_pct: avg,
265 min_off_cpu_pct: min,
266 max_off_cpu_pct: max,
267 spread,
268 max_gap_ms: gap_ms,
269 max_gap_cpu: gap_cpu,
270 total_migrations: total_mig,
271 migration_ratio: mig_ratio,
272 p99_wake_latency_us: p99_us,
273 median_wake_latency_us: median_us,
274 wake_latency_cv: lat_cv,
275 // `0.0` above is a not-measured sentinel iff no worker recorded a
276 // wake/timer sample; the flag distinguishes that from a measured zero
277 // so the run-level re-pool excludes (not folds in) a no-sample cgroup.
278 wake_measured: !all_latencies.is_empty(),
279 median_timer_latency_us: median_timer,
280 p99_timer_latency_us: p99_timer,
281 p999_timer_latency_us: p999_timer,
282 worst_timer_latency_us: worst_timer,
283 timer_measured: !all_timer.is_empty(),
284 total_iterations: total_iters,
285 total_cpu_time_ns: reports
286 .iter()
287 .map(|w| w.schedstat_cpu_time_ns)
288 .fold(0u64, u64::saturating_add),
289 mean_run_delay_us: mean_run_delay,
290 worst_run_delay_us: worst_run_delay,
291 // Run-delay is measured whenever a worker exists (a worker with `0.0`
292 // delay is a real measured zero); only a worker-less cohort is
293 // not-measured.
294 run_delay_measured: !run_delays.is_empty(),
295 // page_locality requires the expected NUMA node set (the cpuset's
296 // nodes), which this reports-only builder does not have. It is
297 // populated by `AssertPlan::assert_cgroup` when `numa_nodes` is
298 // supplied; left 0.0 here (no NUMA context).
299 page_locality: 0.0,
300 cross_node_migration_ratio: cross_node_ratio,
301 taobench_whole,
302 ext_metrics: BTreeMap::new(),
303 }
304}
305
306/// Migrations per iteration; `0.0` when no iterations ran (a measured-zero,
307/// not a rate over zero). Single-sourced so the per-phase carrier
308/// (`write_carrier_scalars`) and [`cgroup_stats`] (whose `migration_ratio`
309/// feeds the run-level `worst_migration_ratio` fold) compute it identically.
310pub(crate) fn migration_ratio_of(total_migrations: u64, total_iterations: u64) -> f64 {
311 if total_iterations > 0 {
312 total_migrations as f64 / total_iterations as f64
313 } else {
314 0.0
315 }
316}
317
318/// Cross-node migrated pages over the cgroup-wide total allocated pages; `0.0`
319/// when no NUMA pages were seen. Single-sourced with [`cgroup_stats`]. A CHURN
320/// ratio, NOT a bounded `[0,1]` fraction: the numerator is cumulative migration
321/// EVENTS (`/proc/vmstat numa_pages_migrated`, which counts each migration, so a
322/// page can be counted more than once) and the denominator is a residency
323/// SNAPSHOT — so the result can legitimately exceed 1.0 under heavy re-migration.
324pub(crate) fn cross_node_migration_ratio_of(migrated_pages: u64, total_numa_pages: u64) -> f64 {
325 if total_numa_pages > 0 {
326 migrated_pages as f64 / total_numa_pages as f64
327 } else {
328 0.0
329 }
330}
331
332/// Fraction of allocated pages on the expected NUMA nodes; `0.0` when no NUMA
333/// pages were seen. Single-sourced with the per-phase carrier.
334pub(crate) fn page_locality_of(numa_pages_local: u64, numa_pages_total: u64) -> f64 {
335 if numa_pages_total > 0 {
336 numa_pages_local as f64 / numa_pages_total as f64
337 } else {
338 0.0
339 }
340}
341
342/// Iterations per worker; `None` when there are no workers (undefined,
343/// distinct from a measured zero). Single-sourced with
344/// [`CgroupStats::iterations_per_worker`](crate::assert::CgroupStats::iterations_per_worker)
345/// so the run-level WorstLowest fold stays bit-identical.
346pub(crate) fn iterations_per_worker_of(num_workers: usize, total_iterations: u64) -> Option<f64> {
347 if num_workers > 0 {
348 Some(total_iterations as f64 / num_workers as f64)
349 } else {
350 None
351 }
352}
353
354/// Worker iterations per CPU-second (`total_iterations / (total_cpu_time_ns /
355/// 1e9)`); `None` when there are no workers or no on-CPU time captured.
356/// Single-sourced with
357/// [`CgroupStats::iterations_per_cpu_sec`](crate::assert::CgroupStats::iterations_per_cpu_sec).
358pub(crate) fn iterations_per_cpu_sec_of(
359 num_workers: usize,
360 total_cpu_time_ns: u64,
361 total_iterations: u64,
362) -> Option<f64> {
363 if num_workers == 0 || total_cpu_time_ns == 0 {
364 return None;
365 }
366 Some(total_iterations as f64 / (total_cpu_time_ns as f64 / 1e9))
367}
368
369/// Per-phase per-cgroup RAW-component builder — the sibling of [`cgroup_stats`]
370/// that emits [`PhaseCgroupStats`]'s un-reduced components instead of the
371/// reduced ratios/percentiles, so the distributional re-pool recomputes each
372/// aggregate from the pooled components at every level. Every [`CgroupStats`]
373/// reduction re-pools from these fields: avg/min/max/spread off-CPU% from
374/// `off_cpu_pcts`; p99/median/CV from `wake_latencies_ns`; mean/worst run-delay
375/// from `run_delays_ns` (RAW ns, the re-pool divides by 1000); migration_ratio
376/// / iterations_per_cpu_sec / iterations_per_worker from the counters;
377/// page_locality / cross_node_migration_ratio from the numa counters; the
378/// coupled worst gap from the argmax pair; cpus_used / num_cpus from `cpus_used`.
379///
380/// `expected_nodes` is this cgroup's cpuset NUMA-node set (from
381/// [`crate::topology::TestTopology::numa_nodes_for_cpuset`]); `numa_pages_local`
382/// is the page count on those nodes (0 when `None`, mirroring [`cgroup_stats`]
383/// leaving `page_locality` 0.0 without NUMA context — the partition lives with
384/// the caller that has the node set, as [`AssertPlan::assert_cgroup`] does).
385/// The whole-run [`cgroup_stats`] reductions stay the run-level authority; this
386/// feeds the per-phase [`PhaseBucket::per_cgroup`] carrier.
387///
388/// RE-POOL GUARD CONTRACT: the div-by-zero / not-measured guards live in
389/// [`cgroup_stats`], NOT in these raw components — a future re-pool over them
390/// MUST mirror them exactly or ship a NaN/Inf or a not-measured-vs-zero
391/// collapse: `migration_ratio` only when `total_iterations > 0`;
392/// `cross_node_migration_ratio` / `page_locality` only when `numa_pages_total >
393/// 0`; mean/worst run-delay only when `run_delays_ns` is non-empty; and
394/// avg/min/max/spread off-CPU% return None (not 0.0) when `off_cpu_pcts` is
395/// empty (the not-measured state).
396pub(crate) fn phase_cgroup_stats(
397 reports: &[WorkerReport],
398 expected_nodes: Option<&BTreeSet<usize>>,
399) -> PhaseCgroupStats {
400 let cpus_used: BTreeSet<usize> = reports
401 .iter()
402 .flat_map(|w| w.cpus_used.iter().copied())
403 .collect();
404 // Per-worker off-CPU% (only workers with measurable wall time), un-reduced.
405 // EMPTY = not measured: the re-pool then yields None for avg/min/max/spread,
406 // preserving the not-measured-vs-measured-zero distinction cgroup_stats keeps.
407 let off_cpu_pcts: Vec<f64> = reports
408 .iter()
409 .filter(|w| w.wall_time_ns > 0)
410 .map(|w| w.off_cpu_ns as f64 / w.wall_time_ns as f64 * 100.0)
411 .collect();
412 // Pool every worker's already per-worker-capped wake-latency vec, RE-CAPPING
413 // the concatenation at MAX_WAKE_SAMPLES via the same Algorithm-R reservoir the
414 // per-worker path uses. This carrier is the FIRST to serialize raw samples
415 // over the size-limited guest bulk port (the AssertResult); without the
416 // re-cap the pool would be workers × MAX_WAKE_SAMPLES and could overrun the
417 // 16 MiB frame on a many-core host, flipping a PASS to a truncated FAIL. The
418 // reservoir is distribution-preserving, so p99 / median / CV re-pool over it
419 // as cgroup_stats does over the per-worker pool; `wake_sample_total` keeps the
420 // TRUE pre-cap population for the re-pool. PARITY: for pools ≤
421 // MAX_WAKE_SAMPLES the reservoir is the full concatenation, so the re-pool is
422 // VALUE-FOR-VALUE with cgroup_stats; above the cap it is a distribution-
423 // preserving SUBSAMPLE (cgroup_stats keeps the full concat), so the re-pool is
424 // distribution-equivalent, not byte-identical — see the wake_latencies_ns
425 // field doc for the full contract. PhaseCgroupStats::merge keeps same-name
426 // carriers bounded too: ≤cap it concatenates (value-for-value), >cap it uses a
427 // population-WEIGHTED reservoir merge (weighted_merge_reservoirs) so the merged
428 // subsample is unbiased rather than length-skewed toward the smaller carrier.
429 let mut wake_latencies_ns: Vec<u64> = Vec::new();
430 let mut pooled_wake_count: u64 = 0;
431 for w in reports {
432 for &sample in &w.wake_latencies_ns {
433 crate::workload::reservoir_push(
434 &mut wake_latencies_ns,
435 &mut pooled_wake_count,
436 sample,
437 crate::workload::MAX_WAKE_SAMPLES,
438 );
439 }
440 }
441 // saturating folds (per cgroup_stats's rationale): overflow-safe pool of the
442 // per-worker guest-runtime counters into this per-phase carrier.
443 let wake_sample_total: u64 = reports
444 .iter()
445 .map(|w| w.wake_sample_total)
446 .fold(0u64, u64::saturating_add);
447 // Distinct timer reservoir, pooled across workers exactly like
448 // wake_latencies_ns; timer_sample_total keeps the true pre-cap population.
449 let mut timer_latencies_ns: Vec<u64> = Vec::new();
450 let mut pooled_timer_count: u64 = 0;
451 for w in reports {
452 for &sample in &w.timer_latencies_ns {
453 crate::workload::reservoir_push(
454 &mut timer_latencies_ns,
455 &mut pooled_timer_count,
456 sample,
457 crate::workload::MAX_WAKE_SAMPLES,
458 );
459 }
460 }
461 let timer_sample_total: u64 = reports
462 .iter()
463 .map(|w| w.timer_sample_total)
464 .fold(0u64, u64::saturating_add);
465 // RAW ns, one per worker — NOT divided by 1000. cgroup_stats divides at
466 // reduction time; the re-pool over the concatenated samples divides once,
467 // so pre-dividing here would double-divide (a 1000x error).
468 let run_delays_ns: Vec<u64> = reports.iter().map(|w| w.schedstat_run_delay_ns).collect();
469 // Coupled worst gap: take (ms, cpu) TOGETHER from the worst worker (argmax),
470 // never two independent maxes — keeps the gap bound to its CPU.
471 let (max_gap_ms, max_gap_cpu) = reports
472 .iter()
473 .max_by_key(|w| w.max_gap_ms)
474 .map(|w| (w.max_gap_ms, w.max_gap_cpu))
475 .unwrap_or((0, 0));
476 let total_migrations: u64 = reports
477 .iter()
478 .map(|w| w.migration_count)
479 .fold(0u64, u64::saturating_add);
480 let total_iterations: u64 = reports
481 .iter()
482 .map(|w| w.iterations)
483 .fold(0u64, u64::saturating_add);
484 // schedstat_cpu_time_ns (task->se.sum_exec_runtime), NOT cpu_time_ns
485 // (CLOCK_THREAD_CPUTIME_ID) — matches cgroup_stats's total_cpu_time_ns.
486 let total_cpu_time_ns: u64 = reports
487 .iter()
488 .map(|w| w.schedstat_cpu_time_ns)
489 .fold(0u64, u64::saturating_add);
490 let numa_pages_total: u64 = reports
491 .iter()
492 .map(|w| {
493 w.numa_pages
494 .values()
495 .copied()
496 .fold(0u64, u64::saturating_add)
497 })
498 .fold(0u64, u64::saturating_add);
499 // System-wide /proc/vmstat numa_pages_migrated delta each worker observes
500 // redundantly -> MAX, not SUM (summing inflates by the worker count).
501 let cross_node_migrated: u64 = reports
502 .iter()
503 .map(|w| w.vmstat_numa_pages_migrated)
504 .max()
505 .unwrap_or(0);
506 // Pages on the cgroup's expected NUMA nodes (page_locality numerator),
507 // partitioned exactly as AssertPlan::assert_cgroup does; 0 without a node
508 // set (mirrors cgroup_stats leaving page_locality 0.0 absent NUMA context).
509 let numa_pages_local: u64 = expected_nodes
510 .map(|nodes| {
511 let mut local = 0u64;
512 for w in reports {
513 for (&node, &count) in &w.numa_pages {
514 if nodes.contains(&node) {
515 local = local.saturating_add(count);
516 }
517 }
518 }
519 local
520 })
521 .unwrap_or(0);
522 PhaseCgroupStats {
523 num_workers: reports.len(),
524 cpus_used,
525 wake_latencies_ns,
526 wake_sample_total,
527 timer_latencies_ns,
528 timer_sample_total,
529 run_delays_ns,
530 off_cpu_pcts,
531 total_migrations,
532 total_iterations,
533 total_cpu_time_ns,
534 numa_pages_local,
535 numa_pages_total,
536 cross_node_migrated,
537 max_gap_ms,
538 max_gap_cpu,
539 // Fresh carrier built from worker reports — never stripped.
540 stripped: false,
541 // Derived scalars are filled post-build by derive_phase_metrics.
542 metrics: std::collections::BTreeMap::new(),
543 // schbench rides PhaseSlice (the backdrop per-phase carrier), not the
544 // whole-run WorkerReports this fn pools, so it is always None here.
545 schbench: None,
546 taobench: None,
547 }
548}
549
550/// Build a per-cgroup carrier from ONE already-per-phase backdrop
551/// [`crate::workload::PhaseSlice`] (no whole-run differencing — the
552/// slice's counter fields are already per-phase deltas). The single-worker
553/// analog of [`phase_cgroup_stats`]: `num_workers` is 1 and each list field
554/// carries this one worker's value, so [`PhaseCgroupStats::merge`] pools
555/// slices across backdrop workers into a per-epoch carrier identically to
556/// how [`phase_cgroup_stats`] pools whole-run reports.
557pub(crate) fn phase_slice_to_cgroup_stats(
558 slice: &crate::workload::PhaseSlice,
559 expected_nodes: Option<&BTreeSet<usize>>,
560) -> PhaseCgroupStats {
561 // Per-phase off-CPU%, one value, only when wall time was measured
562 // (wall_ns == 0 => the worker never ran this phase => EMPTY = not
563 // measured, matching phase_cgroup_stats's not-measured contract).
564 let off_cpu_pcts: Vec<f64> = if slice.wall_ns > 0 {
565 vec![slice.off_cpu_ns as f64 / slice.wall_ns as f64 * 100.0]
566 } else {
567 Vec::new()
568 };
569 // saturating folds: overflow-safe pool of this slice's per-node page counts.
570 let numa_pages_total: u64 = slice
571 .numa_pages
572 .values()
573 .copied()
574 .fold(0u64, u64::saturating_add);
575 let numa_pages_local: u64 = expected_nodes
576 .map(|nodes| {
577 slice
578 .numa_pages
579 .iter()
580 .filter(|(node, _)| nodes.contains(node))
581 .map(|(_, &count)| count)
582 .fold(0u64, u64::saturating_add)
583 })
584 .unwrap_or(0);
585 PhaseCgroupStats {
586 num_workers: 1,
587 cpus_used: slice.cpus_used.clone(),
588 wake_latencies_ns: slice.wake_latencies_ns.clone(),
589 wake_sample_total: slice.wake_sample_total,
590 timer_latencies_ns: slice.timer_latencies_ns.clone(),
591 timer_sample_total: slice.timer_sample_total,
592 // RAW ns, one per worker (NOT divided) — same contract as
593 // phase_cgroup_stats::run_delays_ns.
594 run_delays_ns: vec![slice.run_delay_ns],
595 off_cpu_pcts,
596 total_migrations: slice.migration_count,
597 total_iterations: slice.iterations,
598 total_cpu_time_ns: slice.schedstat_cpu_time_ns,
599 numa_pages_local,
600 numa_pages_total,
601 cross_node_migrated: slice.vmstat_numa_pages_migrated,
602 max_gap_ms: slice.max_gap_ms,
603 max_gap_cpu: slice.max_gap_cpu,
604 stripped: false,
605 // Derived scalars are filled post-build by derive_phase_metrics.
606 metrics: std::collections::BTreeMap::new(),
607 // Carry the per-phase schbench engine metrics through (None for every
608 // non-schbench backdrop slice); PhaseCgroupStats::merge pools them.
609 schbench: slice.schbench.clone(),
610 // Carry the per-phase taobench engine metrics through (None for every
611 // non-taobench backdrop slice); PhaseCgroupStats::merge pools them.
612 taobench: slice.taobench.clone(),
613 }
614}
615
616/// Pool a set of backdrop [`crate::workload::PhaseSlice`]s (all for
617/// the SAME epoch, one per worker) into a single per-cgroup carrier via
618/// [`PhaseCgroupStats::merge`] — the per-phase analog of
619/// [`phase_cgroup_stats`] pooling whole-run reports. An empty input yields
620/// a zero-worker carrier (all fields empty/0, `stripped: false`) so a phase
621/// no backdrop worker observed renders as not-measured rather than
622/// panicking.
623pub(crate) fn pool_phase_slice_stats(
624 slices: &[&crate::workload::PhaseSlice],
625 expected_nodes: Option<&BTreeSet<usize>>,
626) -> PhaseCgroupStats {
627 let mut iter = slices
628 .iter()
629 .map(|s| phase_slice_to_cgroup_stats(s, expected_nodes));
630 match iter.next() {
631 Some(first) => iter.fold(first, PhaseCgroupStats::merge),
632 None => PhaseCgroupStats {
633 num_workers: 0,
634 cpus_used: BTreeSet::new(),
635 wake_latencies_ns: Vec::new(),
636 wake_sample_total: 0,
637 timer_latencies_ns: Vec::new(),
638 timer_sample_total: 0,
639 run_delays_ns: Vec::new(),
640 off_cpu_pcts: Vec::new(),
641 total_migrations: 0,
642 total_iterations: 0,
643 total_cpu_time_ns: 0,
644 numa_pages_local: 0,
645 numa_pages_total: 0,
646 cross_node_migrated: 0,
647 max_gap_ms: 0,
648 max_gap_cpu: 0,
649 stripped: false,
650 metrics: std::collections::BTreeMap::new(),
651 schbench: None,
652 taobench: None,
653 },
654 }
655}
656
657/// Expand a backdrop worker set's per-phase
658/// [`crate::workload::PhaseSlice`]s into one [`PhaseBucket`] per epoch,
659/// keyed by the epoch as `step_index`, each pooling that epoch's slices
660/// across workers via [`pool_phase_slice_stats`]. BASELINE (epoch 0) and
661/// inter-step-gap (`u32::MAX`) epochs are skipped — they have no paired
662/// host bucket and the host fold discards them. Called by the backdrop
663/// (None-`step_index`) arm of [`crate::scenario::collect_handles`]; the
664/// host's [`fold_guest_per_cgroup_into_host_buckets`] then unions these
665/// into the host-rebuilt buckets (matched epochs) or surfaces them as
666/// orphan not-measured windows. Extracted (rather than inlined in
667/// collect_handles, which calls `stop_and_collect`) so the grouping +
668/// per-epoch pooling is unit-testable directly.
669pub(crate) fn expand_backdrop_phase_buckets(
670 name: &str,
671 reports: &[WorkerReport],
672 expected_nodes: Option<&BTreeSet<usize>>,
673) -> Vec<PhaseBucket> {
674 let mut by_epoch: std::collections::BTreeMap<u32, Vec<&crate::workload::PhaseSlice>> =
675 std::collections::BTreeMap::new();
676 for report in reports {
677 for slice in &report.phase_slices {
678 if slice.phase_epoch == 0 || slice.phase_epoch == u32::MAX {
679 continue;
680 }
681 by_epoch.entry(slice.phase_epoch).or_default().push(slice);
682 }
683 }
684 by_epoch
685 .into_iter()
686 .map(|(epoch, slices)| {
687 let mut per_cgroup = std::collections::BTreeMap::new();
688 per_cgroup.insert(
689 name.to_string(),
690 pool_phase_slice_stats(&slices, expected_nodes),
691 );
692 // Lossless: a real epoch == u32::from(phase_step_index: u16),
693 // and 0 / u32::MAX are filtered above.
694 let step_index = epoch as u16;
695 PhaseBucket {
696 step_index,
697 label: Phase::from(step_index).to_string(),
698 start_ms: u64::MAX,
699 end_ms: 0,
700 sample_count: 0,
701 metrics: std::collections::BTreeMap::new(),
702 per_cgroup,
703 }
704 })
705 .collect()
706}
707
708/// Build the single-bucket guest-side per-phase carrier for one step-local
709/// cgroup: a [`PhaseBucket`] at `step_index` whose only payload is the
710/// `per_cgroup` entry `name -> phase_cgroup_stats(reports, expected_nodes)`.
711///
712/// The guest emits one of these per step-local cgroup at `collect_step`
713/// teardown ([`crate::scenario::collect_handles`]). The window is the
714/// merge-neutral `(u64::MAX, 0)` sentinel and `metrics` is empty: the carrier
715/// contributes ONLY `per_cgroup`. When folded into the host-rebuilt bucket of
716/// the same `step_index` ([`fold_guest_per_cgroup_into_host_buckets`] via
717/// [`merge_matched_phase_buckets`]) the `MAX`/`0` window is a no-op against the
718/// host's real window (`min`/`max`), so the host's window and metrics win and
719/// only `per_cgroup` is carried. The `label` uses [`Phase`]'s `Display` so an
720/// orphan carrier (no host bucket) still reads `BASELINE`/`Step[k]`.
721pub(crate) fn step_per_cgroup_bucket(
722 name: &str,
723 reports: &[WorkerReport],
724 expected_nodes: Option<&BTreeSet<usize>>,
725 step_index: u16,
726) -> PhaseBucket {
727 let mut per_cgroup = std::collections::BTreeMap::new();
728 per_cgroup.insert(
729 name.to_string(),
730 phase_cgroup_stats(reports, expected_nodes),
731 );
732 PhaseBucket {
733 step_index,
734 label: Phase::from(step_index).to_string(),
735 start_ms: u64::MAX,
736 end_ms: 0,
737 sample_count: 0,
738 metrics: std::collections::BTreeMap::new(),
739 per_cgroup,
740 }
741}
742
743/// Roll a single cgroup's [`CgroupStats`] up into a one-cgroup
744/// [`ScenarioStats`]. The KEPT typed `worst_*` fields carry this cgroup's
745/// values and fold across cgroups in [`AssertResult::merge`] by max (all are
746/// higher-is-worse): `worst_spread`, `worst_migration_ratio`, and the coupled
747/// `worst_gap_ms` / `worst_gap_cpu`. The two NUMA roll-ups
748/// (`worst_page_locality`, `worst_cross_node_migration_ratio`), the wake-latency
749/// / run-delay distributions, the iteration efficiencies, and the wake-latency
750/// tail ratio are NOT carried here — they have no typed field and re-pool
751/// run-level POST-merge in [`populate_run_distribution_metrics`]:
752/// `worst_page_locality` (WorstLowest) re-pools None-aware from
753/// `stats.phases[].per_cgroup` NUMA carriers (lowest per-cgroup locality, a
754/// measured 0.0 winning the lowest rather than being skipped as a sentinel),
755/// `worst_cross_node_migration_ratio` (WorstCrossNodeRatio) re-pools the MAX
756/// per-cgroup churn ratio from the same carriers, the tail ratio is the max over
757/// the per-cgroup `CgroupStats::wake_latency_tail_ratio`, and the distributions /
758/// efficiencies pool from `stats.phases[].per_cgroup` / `stats.cgroups`.
759/// `cgroups` carries exactly this one entry so merge appends one per
760/// handle without double-counting.
761pub(crate) fn scenario_stats_for_cgroup(cg: &CgroupStats) -> ScenarioStats {
762 ScenarioStats {
763 total_workers: cg.num_workers,
764 total_cpus: cg.num_cpus,
765 total_migrations: cg.total_migrations,
766 // worst_spread is higher-is-worse (merge takes max). A
767 // not-measured cgroup (`spread == None`) maps to 0.0 — the
768 // neutral element for max, and the gauntlet layer's
769 // documented no-data convention. A measured zero stays 0.0.
770 worst_spread: cg.spread.unwrap_or(0.0),
771 worst_gap_ms: cg.max_gap_ms,
772 worst_gap_cpu: cg.max_gap_cpu,
773 worst_migration_ratio: cg.migration_ratio,
774 total_iterations: cg.total_iterations,
775 // worst_page_locality and worst_cross_node_migration_ratio are no longer
776 // typed fields — both re-pool from the per-phase NUMA carriers in
777 // populate_run_distribution_metrics. cg.page_locality is structurally 0.0
778 // here (it needs an expected-node set this reports-only builder lacks);
779 // cg.cross_node_migration_ratio IS populated but is a single pre-folded
780 // scalar that cannot carry the cross-phase SUM-migrated / LATEST-total
781 // fold, so the per-phase carriers own both.
782 ext_metrics: cg.ext_metrics.clone(),
783 cgroups: vec![cg.clone()],
784 phases: Vec::new(),
785 }
786}
787
788/// Record the DEFAULT fairness outcomes (NoProgress / Unfair / Stuck) for one
789/// cgroup against the framework default thresholds
790/// ([`spread_threshold_pct`] / [`gap_threshold_ms`]). Telemetry is built
791/// separately by [`cgroup_stats`]; this only appends fail outcomes, so it
792/// is shared by [`assert_not_stuck`] and the `not_stuck` arm of
793/// [`AssertPlan::assert_cgroup`] without rebuilding stats.
794pub(crate) fn record_default_fairness(
795 r: &mut AssertResult,
796 cg: &CgroupStats,
797 reports: &[WorkerReport],
798) {
799 for w in reports {
800 if w.work_units == 0 {
801 r.record_fail(AssertDetail::new(
802 DetailKind::NoProgress,
803 format!("tid {} made no progress (0 work units)", w.tid),
804 ));
805 }
806 }
807 // Off-cpu spread above the default threshold, gated on >=2 workers
808 // with measurable wall time (the historical `pcts.len() >= 2`).
809 // `cg.spread` is None when off-CPU% was not measured — inconclusive,
810 // never flagged unfair.
811 let measurable = reports.iter().filter(|w| w.wall_time_ns > 0).count();
812 let spread_limit = spread_threshold_pct();
813 if let Some(spread) = cg.spread
814 && spread > spread_limit
815 && measurable >= 2
816 {
817 // NO `with_latency_gate`: off-CPU spread is a dimensionless
818 // PERCENTAGE difference across workers, not a ns interval, so the
819 // seam's ns-denominated `W(measured)` bound cannot score it. Like
820 // CV it stays annotate-only via `dilation_annotation`; only genuine
821 // ns-latency exemplars feed the tri-state (see [`LatencyGate`]).
822 r.record_fail(AssertDetail::new(
823 DetailKind::Unfair,
824 format!(
825 "unfair cgroup: spread={:.0}% ({:.0}-{:.0}%) {} workers on {} cpus (threshold {:.0}%)",
826 spread,
827 cg.min_off_cpu_pct.unwrap_or(0.0),
828 cg.max_off_cpu_pct.unwrap_or(0.0),
829 cg.num_workers,
830 cg.num_cpus,
831 spread_limit,
832 ),
833 ));
834 }
835 let gap_limit = gap_threshold_ms();
836 for w in reports {
837 // `max_gap_ms` is CPU-time between progress checkpoints
838 // (compute-without-progress; dilation-safe). The wall gap rides
839 // along as evidence — under starvation it diverges upward from
840 // the CPU gap (and the host appends the measured dilation D).
841 if w.max_gap_ms > gap_limit {
842 r.record_fail(AssertDetail::new(
843 DetailKind::Stuck,
844 format!(
845 "tid {} stuck {}ms cpu-gap on cpu{} at +{}ms (threshold {}ms, \
846 wall gap {}ms)",
847 w.tid,
848 w.max_gap_ms,
849 w.max_gap_cpu,
850 w.max_gap_at_ms,
851 gap_limit,
852 w.max_gap_wall_ms,
853 ),
854 ));
855 }
856 }
857}
858
859/// Default fairness check for one cgroup's worker reports: builds the
860/// per-cgroup telemetry ([`cgroup_stats`]) and records NoProgress / Unfair /
861/// Stuck against the framework default thresholds. Telemetry is ALWAYS
862/// populated — including a `num_workers == 0` entry for empty reports — so
863/// `r.stats.cgroups` is never empty for a declared cgroup, independent of
864/// whether any fail outcome fired.
865pub fn assert_not_stuck(reports: &[WorkerReport]) -> AssertResult {
866 let cg = cgroup_stats(reports);
867 let mut r = AssertResult::pass();
868 record_default_fairness(&mut r, &cg, reports);
869 r.stats = scenario_stats_for_cgroup(&cg);
870 r
871}
872
873/// Check throughput parity across workers: coefficient of variation and
874/// minimum work rate.
875///
876/// `max_cv`: maximum allowed coefficient of variation (stddev/mean) for
877/// work_units / cpu_time_ns across workers. `None` skips the CV check.
878///
879/// `min_rate`: minimum work_units per CPU-second. `None` skips the floor check.
880///
881/// When every worker recorded `cpu_time_ns == 0`, both gates record
882/// their OWN Inconclusive outcome (the CV gate emits a "CV cannot be
883/// computed" detail; the min_rate gate emits a "rates cannot be
884/// computed" detail). Each gate carries its own diagnostic so a
885/// caller that supplies only one of the two threshold parameters
886/// sees the matching Inconclusive message and an operator reading
887/// [`AssertResult::inconclusive_details`] can identify which gate(s)
888/// misfired without re-deriving the inputs.
889///
890/// ```
891/// # use ktstr::assert::assert_throughput_parity;
892/// # use ktstr::workload::WorkerReport;
893/// # let mk = |units, cpu_ns| WorkerReport {
894/// # tid: 1, cpus_used: [0].into_iter().collect(),
895/// # work_units: units, cpu_time_ns: cpu_ns, wall_time_ns: cpu_ns,
896/// # off_cpu_ns: cpu_ns, migration_count: 0, migrations: vec![],
897/// # max_gap_ms: 0, max_gap_cpu: 0, max_gap_at_ms: 0,
898/// # wake_latencies_ns: vec![], wake_sample_total: 0,
899/// # iteration_costs_ns: vec![], iteration_cost_sample_total: 0,
900/// # iterations: 0,
901/// # schedstat_run_delay_ns: 0, schedstat_run_count: 0,
902/// # schedstat_cpu_time_ns: 0,
903/// # completed: true,
904/// # numa_pages: std::collections::BTreeMap::new(),
905/// # vmstat_numa_pages_migrated: 0,
906/// # exit_info: None,
907/// # is_messenger: false,
908/// # ..Default::default()
909/// # };
910/// // Equal throughput -> low CV -> passes.
911/// let reports = [mk(1000, 1_000_000_000), mk(1000, 1_000_000_000)];
912/// assert!(assert_throughput_parity(&reports, Some(0.5), None).is_pass());
913/// ```
914pub fn assert_throughput_parity(
915 reports: &[WorkerReport],
916 max_cv: Option<f64>,
917 min_rate: Option<f64>,
918) -> AssertResult {
919 let mut r = AssertResult::pass();
920 if reports.is_empty() {
921 return r;
922 }
923
924 // Compute per-worker throughput: work_units / cpu_seconds
925 let rates: Vec<f64> = reports
926 .iter()
927 .map(|w| {
928 if w.cpu_time_ns == 0 {
929 0.0
930 } else {
931 w.work_units as f64 / (w.cpu_time_ns as f64 / 1e9)
932 }
933 })
934 .collect();
935
936 // CV is computed over MEASURED workers only (cpu_time_ns > 0). A zero-cpu
937 // worker's rate is unknowable — it is forced to 0.0 above so the min_rate
938 // gate below can index `reports[i]`, but that 0.0 is NOT a measured rate.
939 // Folding it into the mean/variance inflates the spread and FAILs a uniform
940 // workload that merely had one worker record no CPU time — the same exclusion
941 // the min_rate gate applies (it skips zero-cpu workers as "rate unknowable,
942 // not failing").
943 let measured: Vec<f64> = reports
944 .iter()
945 .zip(&rates)
946 .filter(|(w, _)| w.cpu_time_ns > 0)
947 .map(|(_, &rate)| rate)
948 .collect();
949 let mean = if measured.is_empty() {
950 0.0
951 } else {
952 measured.iter().sum::<f64>() / measured.len() as f64
953 };
954
955 // Detect the all-zero-cpu condition once so a call with both
956 // `max_cv` and `min_rate` set surfaces a single Inconclusive
957 // listing every threshold that couldn't evaluate, rather than
958 // emitting one record per gate (which produced duplicate
959 // "denominator is zero" diagnostics for the same root cause).
960 let all_zero_cpu = reports.iter().all(|w| w.cpu_time_ns == 0);
961
962 if all_zero_cpu && (max_cv.is_some() || min_rate.is_some()) {
963 let mut limits: Vec<String> = Vec::with_capacity(2);
964 if let Some(cv_limit) = max_cv {
965 limits.push(format!("max_cv {cv_limit:.3}"));
966 }
967 if let Some(floor) = min_rate {
968 limits.push(format!("min_rate {floor:.0}"));
969 }
970 r.record_inconclusive(AssertDetail::new(
971 DetailKind::Benchmark,
972 format!(
973 "throughput parity inconclusive: all {} workers recorded zero cpu_time_ns — \
974 denominator is zero, rates cannot be computed; {} neither pass nor fail \
975 (was the workload able to run?)",
976 reports.len(),
977 limits.join(" + "),
978 ),
979 ));
980 return r;
981 }
982
983 if let Some(cv_limit) = max_cv
984 && mean > 0.0
985 && measured.len() >= 2
986 {
987 let n_measured = measured.len() as f64;
988 let variance = measured.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / n_measured;
989 let stddev = variance.sqrt();
990 let cv = stddev / mean;
991 if cv > cv_limit {
992 r.record_fail(AssertDetail::new(
993 DetailKind::Benchmark,
994 format!(
995 "throughput CV {cv:.3} exceeds limit {cv_limit:.3} (mean={mean:.0} work/cpu_s)"
996 ),
997 ));
998 }
999 }
1000
1001 if let Some(floor) = min_rate {
1002 // Skip per-worker zero-cpu cases: their rate is forced to
1003 // 0.0 above, and comparing that to `floor` would synthesize
1004 // a guaranteed Fail with a misleading "below floor" message
1005 // when the real story is "this worker recorded no CPU time
1006 // — the rate is unknowable, not failing". The all-zero-cpu
1007 // case is already handled at the top of the function as a
1008 // single combined Inconclusive.
1009 for (i, &rate) in rates.iter().enumerate() {
1010 if reports[i].cpu_time_ns == 0 {
1011 continue;
1012 }
1013 if rate < floor {
1014 r.record_fail(AssertDetail::new(
1015 DetailKind::Benchmark,
1016 format!(
1017 "worker {} throughput {rate:.0} work/cpu_s below floor {floor:.0}",
1018 reports[i].tid
1019 ),
1020 ));
1021 }
1022 }
1023 }
1024
1025 r
1026}
1027
1028/// Check benchmarking metrics: p99 wake latency, wake latency CV,
1029/// and minimum iteration rate.
1030///
1031/// `min_iter_rate` is a floor on each worker's CPU-denominated
1032/// iteration rate — `iterations / cpu_time_sec`, the iterations per
1033/// CPU-second the worker actually received. This is intrinsically
1034/// dilation-safe: host preemption inflates wall time but not CPU
1035/// time, so the verdict measures the workload, not the host. In
1036/// performance mode (wall ≈ CPU) it reproduces the old wall-rate
1037/// number. A worker with `cpu_time_ns == 0` has a rate of 0 and
1038/// fails any positive floor (no CPU time spent ⇒ no work proven).
1039///
1040/// ```
1041/// # use ktstr::assert::assert_benchmarks;
1042/// # use ktstr::workload::WorkerReport;
1043/// # let report = WorkerReport {
1044/// # tid: 1, cpus_used: [0].into_iter().collect(),
1045/// # work_units: 1000, cpu_time_ns: 2_500_000_000,
1046/// # wall_time_ns: 5_000_000_000, off_cpu_ns: 2_500_000_000,
1047/// # migration_count: 0, migrations: vec![],
1048/// # max_gap_ms: 50, max_gap_cpu: 0, max_gap_at_ms: 1000,
1049/// # wake_latencies_ns: vec![100, 200, 300, 400, 500],
1050/// # wake_sample_total: 5,
1051/// # iteration_costs_ns: vec![], iteration_cost_sample_total: 0,
1052/// # iterations: 1000,
1053/// # schedstat_run_delay_ns: 0, schedstat_run_count: 0,
1054/// # schedstat_cpu_time_ns: 0,
1055/// # completed: true,
1056/// # numa_pages: std::collections::BTreeMap::new(),
1057/// # vmstat_numa_pages_migrated: 0,
1058/// # exit_info: None,
1059/// # is_messenger: false,
1060/// # ..Default::default()
1061/// # };
1062/// // p99 = 500ns, well under 10000ns limit.
1063/// assert!(assert_benchmarks(&[report], Some(10000), None, None).is_pass());
1064/// ```
1065pub fn assert_benchmarks(
1066 reports: &[WorkerReport],
1067 max_p99_ns: Option<u64>,
1068 max_cv: Option<f64>,
1069 min_iter_rate: Option<f64>,
1070) -> AssertResult {
1071 let mut r = AssertResult::pass();
1072 if reports.is_empty() {
1073 // No worker reports means nothing to measure — any benchmark
1074 // threshold the caller supplied cannot be evaluated. A silent
1075 // pass would let thresholds look "green" on a broken run that
1076 // never produced signal; surface it as skip so the operator
1077 // knows the benchmark was not actually exercised.
1078 return AssertResult::skip("no worker reports — benchmark skipped");
1079 }
1080
1081 // Collect all wake latencies across workers.
1082 let all_latencies: Vec<u64> = reports
1083 .iter()
1084 .flat_map(|w| w.wake_latencies_ns.iter().copied())
1085 .collect();
1086
1087 if let Some(p99_limit) = max_p99_ns
1088 && all_latencies.is_empty()
1089 {
1090 // Fail closed like the migration-ratio zero-denominator arm: a
1091 // p99 threshold over zero samples can neither pass nor fail. A
1092 // silent pass here let a wake-latency gate look "green" on a
1093 // workload that never blocks — the gate asserted nothing.
1094 r.record_inconclusive(AssertDetail::new(
1095 DetailKind::Benchmark,
1096 format!(
1097 "p99 wake latency inconclusive: 0 wake samples across {} workers — \
1098 the workload recorded no wakeups; threshold {p99_limit}ns neither \
1099 pass nor fail (does this workload block and wake at all?)",
1100 reports.len(),
1101 ),
1102 ));
1103 }
1104 if let Some(p99_limit) = max_p99_ns
1105 && !all_latencies.is_empty()
1106 {
1107 let mut sorted = all_latencies.clone();
1108 sorted.sort_unstable();
1109 let p99 = percentile(&sorted, 0.99);
1110 if p99 > p99_limit {
1111 // IN-SCOPE for the host contention seam: p99 wake latency is a
1112 // ns-denominated GUEST-WALL exemplar (one request's wakeup
1113 // wait), so host preemption during the Body phase inflates it
1114 // directly. Stamp the (measured, threshold) evidence so the
1115 // seam can re-run the tri-state verdict against its Body-phase
1116 // witness (`peak_window_delay_ns` bounds a single p99-long
1117 // interval's contamination). The gate fires in-guest where no
1118 // witness exists; the evidence is what lets the host judge.
1119 r.record_fail(
1120 AssertDetail::new(
1121 DetailKind::Benchmark,
1122 format!(
1123 "p99 wake latency {p99}ns exceeds limit {p99_limit}ns ({} samples)",
1124 sorted.len()
1125 ),
1126 )
1127 .with_latency_gate(p99, p99_limit),
1128 );
1129 }
1130 }
1131
1132 if let Some(cv_limit) = max_cv
1133 && all_latencies.len() >= 2
1134 {
1135 let n = all_latencies.len() as f64;
1136 let mean = all_latencies.iter().sum::<u64>() as f64 / n;
1137 if mean > 0.0 {
1138 let variance = all_latencies
1139 .iter()
1140 .map(|&v| (v as f64 - mean).powi(2))
1141 .sum::<f64>()
1142 / n;
1143 let cv = variance.sqrt() / mean;
1144 if cv > cv_limit {
1145 // NO `with_latency_gate`: CV is a dimensionless ratio
1146 // (stddev/mean), not a ns interval, so the seam's
1147 // ns-denominated `W(measured)` contamination bound cannot
1148 // score it — converting it to ns is not sound (a burst of
1149 // host run-delay can widen OR tighten a CV depending on
1150 // where in the sample set it lands). It stays annotate-only:
1151 // `dilation_annotation` already flags CV under measured
1152 // dilation. Only genuine ns-latency exemplars get the
1153 // tri-state (see the p99 gate above / [`LatencyGate`]).
1154 r.record_fail(AssertDetail::new(
1155 DetailKind::Benchmark,
1156 format!(
1157 "wake latency CV {cv:.3} exceeds limit {cv_limit:.3} (mean={mean:.0}ns)"
1158 ),
1159 ));
1160 }
1161 } else {
1162 // CV is dispersion / mean. With mean == 0 every captured
1163 // wake-latency sample was zero, so the denominator is
1164 // zero and CV is undefined — neither pass nor fail is
1165 // truthful. The same workload that fails to record
1166 // measurable wake latency at all (typically: nothing
1167 // actually woke, or every wake landed at <1ns and
1168 // truncated to zero in the ns counter) previously slid
1169 // past the gate as a silent pass; surface it as
1170 // Inconclusive so a broken benchmarking run does not
1171 // masquerade as a CV-compliant one.
1172 r.record_inconclusive(AssertDetail::new(
1173 DetailKind::Benchmark,
1174 format!(
1175 "wake latency CV inconclusive: all {} sample(s) had zero mean wake \
1176 latency — denominator is zero, CV cannot be computed; limit \
1177 {cv_limit:.3} neither pass nor fail (did any wake event capture a \
1178 non-zero latency?)",
1179 all_latencies.len(),
1180 ),
1181 ));
1182 }
1183 }
1184
1185 if let Some(rate_floor) = min_iter_rate {
1186 // CPU-DENOMINATED rate: iterations per CPU-second the worker
1187 // actually received (`cpu_time_ns`, a CLOCK_THREAD_CPUTIME_ID
1188 // reading), single-sourced through `iterations_per_cpu_sec_of`
1189 // so the denomination is bit-identical to the run-level fold.
1190 //
1191 // This makes the floor INTRINSICALLY dilation-safe: under host
1192 // preemption wall time inflates but CPU time does not, so a busy
1193 // worker's rate is unchanged (perf mode, where wall ≈ CPU, gives
1194 // numerically the same figure the old wall-rate gate produced).
1195 // The floor now means "iterations per CPU-second the worker was
1196 // GIVEN" — a semantic sharpening: a think-time / sleepy
1197 // workload's idle no longer sits in the denominator, so its rate
1198 // reflects work-per-CPU, not work-per-wall.
1199 //
1200 // A never-ran worker (`cpu_time_ns == 0`) has NO proven
1201 // throughput, so its rate is 0 and it fails any positive floor
1202 // — the correct verdict (no CPU time was spent, so no work was
1203 // demonstrated). There is deliberately no Inconclusive arm: 0
1204 // CPU time is a real, actionable failure, not an unknowable one.
1205 for w in reports {
1206 let cpu_rate = iterations_per_cpu_sec_of(1, w.cpu_time_ns, w.iterations).unwrap_or(0.0);
1207 if cpu_rate < rate_floor {
1208 r.record_fail(AssertDetail::new(
1209 DetailKind::Benchmark,
1210 format!(
1211 "worker {} iteration rate {cpu_rate:.1}/cpu-s below floor \
1212 {rate_floor:.1}/cpu-s",
1213 w.tid
1214 ),
1215 ));
1216 }
1217 }
1218 }
1219
1220 r
1221}
1222
1223/// Failure kinds for which the measured host dilation D is load-bearing
1224/// context in the failure message:
1225///
1226/// - [`DetailKind::Benchmark`] (p99 / CV wake latency) and
1227/// [`DetailKind::Unfair`] (off-CPU spread): these ceilings remain
1228/// WALL-clock denominated, so a trip can be host-preemption noise —
1229/// honest only in performance mode (D≈1.0).
1230/// - [`DetailKind::Stuck`] (the `max_gap_ms` gate): the VERDICT is
1231/// CPU-denominated (compute-without-progress — real at any D), but its
1232/// message carries the wall-gap EVIDENCE, and D is what relates the two
1233/// (wall ≈ cpu × D under pure host preemption; a wall gap far beyond
1234/// cpu × D points at guest-side starvation instead).
1235///
1236/// The iteration-rate gate also lands under `Benchmark` but is
1237/// CPU-denominated and dilation-safe; annotating it is harmless (the note
1238/// leads with the measured D, legitimate context for any failure under
1239/// load).
1240fn is_dilation_confounded_kind(kind: DetailKind) -> bool {
1241 matches!(
1242 kind,
1243 DetailKind::Benchmark | DetailKind::Unfair | DetailKind::Stuck
1244 )
1245}
1246
1247/// Host-dilation annotation appended to a FAILING verdict's rendered
1248/// message. Evidence only — no pass/fail behavior changes.
1249///
1250/// Returns empty unless BOTH a meaningful host dilation was measured
1251/// (`D > 1.05`; at or below that the run was effectively undilated —
1252/// performance mode — and there is nothing to explain) AND the verdict
1253/// carries at least one failure for which D is load-bearing context
1254/// ([`is_dilation_confounded_kind`]).
1255///
1256/// `dilation` is the host vCPU-schedstat `D` (`1 + Σrun_delay/Σon_cpu`),
1257/// a HOST-side measurement of the vCPU threads' schedstat — unavailable
1258/// in-guest where the gate actually fired, so this is applied at render
1259/// time in the test harness. `None` (a host without `CONFIG_SCHEDSTATS`)
1260/// is treated as `D = 1.0`: no annotation, no normalization.
1261pub(crate) fn dilation_annotation(dilation: Option<f64>, result: &AssertResult) -> String {
1262 let Some(d) = dilation else {
1263 return String::new();
1264 };
1265 if d <= 1.05
1266 || !result
1267 .failure_details()
1268 .any(|det| is_dilation_confounded_kind(det.kind))
1269 {
1270 return String::new();
1271 }
1272 // vCPUs received 1/D of the CPU they were runnable for (D = 1 +
1273 // run_delay/on_cpu ⇒ on_cpu/(on_cpu+run_delay) = 1/D).
1274 let pct = (100.0 / d).round() as u64;
1275 format!(
1276 "\n\n--- host dilation ---\n \
1277 measured host dilation D={d:.2}x this run (vCPUs received {pct}% of the CPU they \
1278 were runnable for).\n \
1279 wall-clock latency ceilings (max_p99_wake_latency_ns / max_wake_latency_cv / \
1280 max_spread_pct) include host preemption — these ceilings are honest in performance \
1281 mode, where D≈1.0. CPU-denominated gates (min_iteration_rate, max_gap_ms) are \
1282 dilation-safe; a Stuck failure's wall-gap evidence scales as ≈ cpu-gap × D under \
1283 host preemption alone — a wall gap far beyond that points at guest-side starvation."
1284 )
1285}
1286
1287/// Assert that every SCX event counter in `events` is at or below
1288/// `max_count`. `events` is a slice of `(name, count)` pairs sourced
1289/// from the kernel's per-task `scx_event_stats` (see `kernel/sched/ext.c`,
1290/// `SCX_EV_*` macros) — typically aggregated and surfaced via
1291/// `monitor::ScxEventDeltas` or sidecar `GauntletRow.fallback_count` /
1292/// `keep_last_count` fields. Pass `None` for `max_count` to require zero
1293/// (the strict default — error-class events should not fire under a
1294/// healthy scheduler).
1295///
1296/// The assertion is decoupled from the `monitor` module on purpose:
1297/// callers harvest the counters they care about (via the live monitor
1298/// path or by reading sidecar JSON post-hoc) and feed name/count
1299/// pairs in. This keeps the assert API surface decoupled from the
1300/// kernel-side counter inventory, which evolves across kernel
1301/// versions — adding a new `SCX_EV_*` does not force an API change
1302/// here.
1303///
1304/// Returns a passing result if every counter is within bound; failures
1305/// concatenate one [`AssertDetail`] per offending counter under
1306/// [`DetailKind::SchedulerEvent`] so an operator can identify which
1307/// events fired without scanning the full counter set.
1308///
1309/// ```
1310/// # use ktstr::assert::assert_scx_events_clean;
1311/// // Strict default — every counter must be zero.
1312/// let r = assert_scx_events_clean(&[("enq_skip_exiting", 0), ("dispatch_local_dsq_offline", 0)], None);
1313/// assert!(r.is_pass());
1314///
1315/// // A non-zero error-class counter fails.
1316/// let r = assert_scx_events_clean(&[("enq_skip_exiting", 7)], None);
1317/// assert!(r.is_fail());
1318///
1319/// // Caller-supplied bound tolerates small counts.
1320/// let r = assert_scx_events_clean(&[("dispatch_keep_last", 3)], Some(10));
1321/// assert!(r.is_pass());
1322/// ```
1323pub fn assert_scx_events_clean(events: &[(&str, i64)], max_count: Option<i64>) -> AssertResult {
1324 let mut r = AssertResult::pass();
1325 for (name, count) in events {
1326 // Kernel `scx_event_stats` counters are monotonic u64 — a
1327 // negative i64 here means the source data is corrupted
1328 // (counter reset, wraparound on a signed conversion, or
1329 // sidecar JSON bit-loss). Treat negatives as failures rather
1330 // than letting them silently pass `*count > bound` for any
1331 // non-negative bound.
1332 let failed = match max_count {
1333 // Strict default: every counter must be exactly zero.
1334 // `*count > 0` would let -5 slip through.
1335 None => *count != 0,
1336 // Bounded: reject negatives explicitly, then enforce
1337 // the upper bound.
1338 Some(bound) => *count < 0 || *count > bound,
1339 };
1340 if failed {
1341 let bound_desc = match max_count {
1342 None => "0".to_string(),
1343 Some(b) => b.to_string(),
1344 };
1345 r.record_fail(AssertDetail::new(
1346 DetailKind::SchedulerEvent,
1347 format!("scx event `{name}` count {count} exceeds bound {bound_desc}",),
1348 ));
1349 }
1350 }
1351 r
1352}
1353
1354/// Threshold-preset bundle for [`assert_thresholds`]. Captures the
1355/// guarantees a scheduler-under-test should meet on a healthy run:
1356/// wake latency stays within bound, per-iteration compute cost stays
1357/// within bound, CPU migrations stay within bound, and every worker
1358/// makes some forward progress.
1359///
1360/// Each `Option` field is independent — `None` skips that check. A
1361/// `AbsoluteThresholds` with every field `None` is a no-op (the
1362/// returned [`AssertResult`] always passes), useful as a starting
1363/// point for builder-style composition. Construct the all-`None`
1364/// thresholds via `AbsoluteThresholds::default()` and chain the
1365/// `max_*` / `min_*` setters (e.g. `AbsoluteThresholds::default().max_migrations(5)`)
1366/// or spread into a struct literal (`AbsoluteThresholds { max_migrations: Some(5), ..Default::default() }`).
1367/// Use [`Self::strict`] for the "every check enabled with sane defaults" preset.
1368///
1369/// Distinct from [`Assert`]: `Assert` is the merge-tree threshold
1370/// config consumed by the worker-side `AssertPlan`; `AbsoluteThresholds`
1371/// is a flat preset designed for direct invocation in test bodies
1372/// where the test author wants a one-call multi-field check without
1373/// engaging the merge chain. The two surfaces compose — a test can
1374/// run `assert_thresholds` against a worker-report slice AND merge the
1375/// `Assert`-derived result into the same accumulator via
1376/// [`AssertResult::merge`].
1377#[must_use = "AbsoluteThresholds only takes effect when passed to assert_thresholds"]
1378#[derive(Debug, Clone, Copy, Default)]
1379pub struct AbsoluteThresholds {
1380 /// Maximum acceptable p99 wake latency (nanoseconds). Compared
1381 /// against the pooled p99 across every worker's
1382 /// [`WorkerReport::wake_latencies_ns`]. `None` skips the check.
1383 /// Same units / semantics as [`Assert::max_p99_wake_latency_ns`].
1384 pub max_p99_wake_latency_ns: Option<u64>,
1385 /// Maximum acceptable p99 per-iteration compute cost (nanoseconds).
1386 /// Compared against the pooled p99 across every worker's
1387 /// [`WorkerReport::iteration_costs_ns`]. `None` skips the check.
1388 /// Only meaningful for compute work types that populate the
1389 /// reservoir (`AluHot`, `SmtSiblingSpin`, `IpcVariance`); blocking
1390 /// variants report empty `iteration_costs_ns` and the check is a
1391 /// no-op for those.
1392 pub max_iteration_cost_p99_ns: Option<u64>,
1393 /// Maximum acceptable total CPU migrations across every worker.
1394 /// Compared against the sum of [`WorkerReport::migration_count`].
1395 /// `None` skips the check. Distinct from
1396 /// [`Assert::max_migration_ratio`] (migrations per iteration) —
1397 /// this is an absolute count, useful when the test pins a known
1398 /// workload size and migrations should stay below a fixed ceiling
1399 /// regardless of how many iterations completed.
1400 pub max_migrations: Option<u64>,
1401 /// Minimum acceptable per-worker work_units. Every worker must
1402 /// have completed at least this many work units; one below-floor
1403 /// worker fails the check. `None` skips. Distinct from
1404 /// [`assert_not_stuck`]'s zero-work-units check, which gates
1405 /// only against literal zero — this gate accepts a non-zero
1406 /// floor so a test can reject "barely made progress" runs that
1407 /// pass the strict zero-work-units gate.
1408 pub min_work_units: Option<u64>,
1409}
1410
1411impl AbsoluteThresholds {
1412 /// Sane-default preset: p99 wake latency under 10ms, p99
1413 /// iteration cost under 1ms, total migrations under 1000, every
1414 /// worker completes ≥1 work unit. The defaults are deliberately
1415 /// loose — a threshold set tight enough to catch egregious
1416 /// regressions without flagging every routine scheduler
1417 /// perturbation. Tests
1418 /// that need tighter bounds should set the fields explicitly via
1419 /// the bare-verb builder methods rather than tuning these constants.
1420 pub const fn strict() -> Self {
1421 Self {
1422 max_p99_wake_latency_ns: Some(10_000_000),
1423 max_iteration_cost_p99_ns: Some(1_000_000),
1424 max_migrations: Some(1000),
1425 min_work_units: Some(1),
1426 }
1427 }
1428
1429 /// Builder setter for [`Self::max_p99_wake_latency_ns`].
1430 pub const fn max_p99_wake_latency_ns(mut self, v: u64) -> Self {
1431 self.max_p99_wake_latency_ns = Some(v);
1432 self
1433 }
1434
1435 /// Builder setter for [`Self::max_iteration_cost_p99_ns`].
1436 pub const fn max_iteration_cost_p99_ns(mut self, v: u64) -> Self {
1437 self.max_iteration_cost_p99_ns = Some(v);
1438 self
1439 }
1440
1441 /// Builder setter for [`Self::max_migrations`].
1442 pub const fn max_migrations(mut self, v: u64) -> Self {
1443 self.max_migrations = Some(v);
1444 self
1445 }
1446
1447 /// Builder setter for [`Self::min_work_units`].
1448 pub const fn min_work_units(mut self, v: u64) -> Self {
1449 self.min_work_units = Some(v);
1450 self
1451 }
1452}
1453
1454/// Run every check in `thresholds` against `reports`, merging results
1455/// into a single [`AssertResult`]. A `None` field on the thresholds
1456/// skips that check.
1457///
1458/// An empty `reports` slice short-circuits to a skip (`"no worker
1459/// reports to evaluate"`) regardless of thresholds content — silently
1460/// passing thresholds against zero samples would let them look
1461/// "green" on a run that produced no measurement.
1462///
1463/// Field-to-check mapping:
1464/// - `max_p99_wake_latency_ns` -> pooled p99 across every worker's
1465/// `wake_latencies_ns`; tagged [`DetailKind::Benchmark`].
1466/// - `max_iteration_cost_p99_ns` -> pooled p99 across every worker's
1467/// `iteration_costs_ns`; tagged [`DetailKind::Benchmark`].
1468/// - `max_migrations` -> sum of `migration_count` across workers;
1469/// tagged [`DetailKind::Migration`].
1470/// - `min_work_units` -> per-worker `work_units >= floor`; tagged
1471/// [`DetailKind::NoProgress`] when a worker is below the floor.
1472///
1473/// The wake-latency check delegates to [`assert_benchmarks`] for the
1474/// percentile path so the same nearest-rank algorithm applies; the
1475/// iteration-cost check uses an inline percentile call against the
1476/// pooled `iteration_costs_ns` reservoir.
1477///
1478/// ```
1479/// # use ktstr::assert::{AbsoluteThresholds, assert_thresholds};
1480/// # use ktstr::workload::WorkerReport;
1481/// # let report = WorkerReport {
1482/// # tid: 1, cpus_used: [0].into_iter().collect(),
1483/// # work_units: 1000, cpu_time_ns: 2_500_000_000,
1484/// # wall_time_ns: 5_000_000_000, off_cpu_ns: 2_500_000_000,
1485/// # migration_count: 5, migrations: vec![],
1486/// # max_gap_ms: 50, max_gap_cpu: 0, max_gap_at_ms: 1000,
1487/// # wake_latencies_ns: vec![100, 200, 300, 400, 500],
1488/// # wake_sample_total: 5,
1489/// # iteration_costs_ns: vec![1000, 2000, 3000, 4000, 5000],
1490/// # iteration_cost_sample_total: 5,
1491/// # iterations: 1000,
1492/// # schedstat_run_delay_ns: 0, schedstat_run_count: 0,
1493/// # schedstat_cpu_time_ns: 0,
1494/// # completed: true,
1495/// # numa_pages: std::collections::BTreeMap::new(),
1496/// # vmstat_numa_pages_migrated: 0,
1497/// # exit_info: None,
1498/// # affinity_error: None,
1499/// # is_messenger: false,
1500/// # group_idx: 0,
1501/// # phase_slices: vec![],
1502/// # ..Default::default()
1503/// # };
1504/// // Strict preset on a healthy run — passes.
1505/// let r = assert_thresholds(&[report], &AbsoluteThresholds::strict());
1506/// assert!(r.is_pass());
1507/// ```
1508pub fn assert_thresholds(
1509 reports: &[WorkerReport],
1510 thresholds: &AbsoluteThresholds,
1511) -> AssertResult {
1512 // Empty `reports` means nothing was measured. Returning a fresh
1513 // `pass()` here would silently green-light a broken run that
1514 // produced no signal. `assert_benchmarks` also skips on empty
1515 // reports, and merging that skip into a `pass()` would preserve it
1516 // (`AssertResult::merge` extends `outcomes`, and an all-Skip
1517 // non-empty `outcomes` folds to Skip) — but its message is generic.
1518 // Surface the skip directly so the operator sees the
1519 // thresholds-specific reason: no worker reports to evaluate.
1520 if reports.is_empty() {
1521 return AssertResult::skip("no worker reports to evaluate");
1522 }
1523
1524 let mut r = AssertResult::pass();
1525
1526 // Wake-latency p99: reuse the existing `assert_benchmarks` path
1527 // so the percentile algorithm stays unified. With `reports`
1528 // non-empty here, `assert_benchmarks` cannot return a skip —
1529 // the merge sees only pass/fail, preserving thresholds semantics.
1530 if thresholds.max_p99_wake_latency_ns.is_some() {
1531 r.merge(assert_benchmarks(
1532 reports,
1533 thresholds.max_p99_wake_latency_ns,
1534 None,
1535 None,
1536 ));
1537 }
1538
1539 // Iteration-cost p99: pooled across every worker's reservoir.
1540 // Skipped when no samples are present — compute work types that
1541 // populate `iteration_costs_ns` are sparse, so an empty pooled
1542 // set is the common case for blocking variants and not a failure.
1543 if let Some(cost_limit) = thresholds.max_iteration_cost_p99_ns {
1544 let all_costs: Vec<u64> = reports
1545 .iter()
1546 .flat_map(|w| w.iteration_costs_ns.iter().copied())
1547 .collect();
1548 if !all_costs.is_empty() {
1549 let mut sorted = all_costs.clone();
1550 sorted.sort_unstable();
1551 let p99 = percentile(&sorted, 0.99);
1552 if p99 > cost_limit {
1553 r.record_fail(AssertDetail::new(
1554 DetailKind::Benchmark,
1555 format!(
1556 "p99 iteration cost {p99}ns exceeds limit {cost_limit}ns ({} samples)",
1557 sorted.len(),
1558 ),
1559 ));
1560 }
1561 }
1562 }
1563
1564 // Total migrations across all workers: absolute-count gate
1565 // (distinct from migration_ratio which is a per-iteration rate).
1566 if let Some(max_mig) = thresholds.max_migrations {
1567 // saturating: a corrupt/hostile per-worker migration_count must clamp,
1568 // not wrap, this absolute-count gate — a wrapped small total would
1569 // silently PASS a max_migrations limit it should fail.
1570 let total_mig: u64 = reports
1571 .iter()
1572 .map(|w| w.migration_count)
1573 .fold(0u64, u64::saturating_add);
1574 if total_mig > max_mig {
1575 r.record_fail(AssertDetail::new(
1576 DetailKind::Migration,
1577 format!(
1578 "total migrations {total_mig} exceeds limit {max_mig} ({} workers)",
1579 reports.len(),
1580 ),
1581 ));
1582 }
1583 }
1584
1585 // Per-worker work_units floor: every worker must have completed
1586 // at least `min` work units. One no-progress worker fails the check.
1587 if let Some(min_units) = thresholds.min_work_units {
1588 for w in reports {
1589 if w.work_units < min_units {
1590 r.record_fail(AssertDetail::new(
1591 DetailKind::NoProgress,
1592 format!(
1593 "tid {} work_units {} below floor {min_units}",
1594 w.tid, w.work_units,
1595 ),
1596 ));
1597 }
1598 }
1599 }
1600
1601 r
1602}
1603
1604// (The legacy `Expect` / `Checks` / `CheckBuilder` types previously
1605// living here were replaced by the [`Verdict`]-based claim API
1606// (defined further up in this file). The new flow is
1607// `Assert::default_checks().verdict().claim_<field>(stats).at_most(N)` for
1608// stats-struct-derived accessors, or `claim!(verdict, expr)` for
1609// expression-labeled claims. Both produce
1610// [`ClaimBuilder`]/[`SetClaim`]/[`SeqClaim`] under the hood and
1611// record outcomes onto the same [`AssertResult`] envelope that
1612// `assert_not_stuck` / `assert_isolation` produce, so the two
1613// paths compose via [`Verdict::merge`].)