ktstr/assert/
phase_build.rs

1use super::*;
2
3/// Build host-observed phase buckets attributed against the guest stimulus
4/// timeline.
5///
6/// Unlike the plain [`build_phase_buckets`] (which groups by the
7/// bridge-stamped step_index), this re-groups each periodic capture by
8/// the guest step whose stimulus window contains the capture's
9/// workload-relative boundary offset (`Sample::boundary_offset_ms`).
10/// That offset is derived from the boundary schedule rather than the
11/// fire time, so it is immune to the deferred-fire burst that makes
12/// every capture stamp the same late CURRENT_STEP (the
13/// `phases.len() == 1` collapse). Captures with no offset (on-demand /
14/// fixture) fall back to their stamped step_index. Because the bucket
15/// windows are then workload-relative, the run-relative monitor samples
16/// are shifted by the stimulus/monitor clock skew before windowing.
17///
18/// Additionally synthesizes a capture-free `PhaseBucket`
19/// (`sample_count == 0`) for any stimulus `StepStart`-step that
20/// captured no periodic samples — the uniform whole-workload boundary
21/// placement (`compute_periodic_boundaries_ns`) is step-agnostic, so a
22/// short interior step can land zero captures and otherwise leave no
23/// bucket. The synthesized bucket carries the step's full stimulus window so
24/// in-window monitor metrics are still recovered. The returned vec holds one
25/// bucket per
26/// (captured phase ∪ `StepStart`-step), sorted by `step_index` — NOT
27/// one-per-captured-phase, so `len()` is no longer "number of captured
28/// phases".
29///
30/// This function deliberately does not derive `iteration_rate` from
31/// `StimulusEvent` wall-clock deltas. After the guest per-cgroup phase carriers
32/// are joined to these buckets, `derive_phase_metrics` derives
33/// the canonical rate as iterations per delivered guest CPU-second. A bucket
34/// with no carrier therefore has no `iteration_rate`, rather than a second
35/// wall-throughput metric under the same name.
36///
37/// Live caller: `evaluate_vm_result` at `src/test_support/eval/mod.rs`
38/// — has both the SampleSeries and the stimulus_events vec in scope.
39pub fn build_phase_buckets_with_stimulus(
40    samples: &crate::scenario::sample::SampleSeries,
41    stimulus_events: &[crate::timeline::StimulusEvent],
42) -> Vec<PhaseBucket> {
43    let monitor_samples: &[crate::monitor::MonitorSample] =
44        samples.monitor().map(|m| m.samples()).unwrap_or(&[]);
45    // vCPU-preemption exemption window for the per-phase stuck predicate,
46    // threaded into fold_monitor_into_bucket -> compute_metrics so the
47    // per-phase stuck_count uses the SAME predicate as the run-level
48    // MonitorSummary::stuck_count. 0 (no monitor) derives from CONFIG_HZ
49    // inside compute_metrics, matching from_samples_with_threshold.
50    let preemption_threshold_ns: u64 = samples
51        .monitor()
52        .map(|m| m.preemption_threshold_ns())
53        .unwrap_or(0);
54    // Re-group periodic captures by the guest step whose stimulus
55    // window contains each capture's workload-relative boundary offset,
56    // NOT the step_index stamped at (deferred) fire time — see
57    // [`crate::scenario::sample::SampleSeries::by_stimulus_phase`] for
58    // why the scheduled offset is the timing-independent truth (it
59    // survives the deferred-fire burst that collapses the stamped
60    // step_index to one phase).
61    let by_phase = samples.by_stimulus_phase(stimulus_events);
62    // Bucket windows are workload-relative (boundary-offset) here, so the
63    // monitor samples (run-relative) are shifted by the stimulus/monitor
64    // clock skew before windowing.
65    let monitor_to_window_offset_ms = monitor_clock_offset(stimulus_events, monitor_samples);
66    let mut buckets = buckets_from_grouped(
67        by_phase,
68        monitor_samples,
69        monitor_to_window_offset_ms,
70        preemption_threshold_ns,
71    );
72    let step_starts = crate::scenario::sample::step_starts_from_stimulus(stimulus_events);
73    synthesize_missing_step_buckets(
74        &mut buckets,
75        &step_starts,
76        stimulus_events,
77        monitor_samples,
78        monitor_to_window_offset_ms,
79        preemption_threshold_ns,
80    );
81    // Keep buckets in step_index order: buckets_from_grouped emits them
82    // sorted (BTreeMap key order), but the synthesize loop appends out of
83    // order. Downstream lookups resolve by step_index, but a sorted vec
84    // matches the captured-bucket invariant and keeps rendered output
85    // stable.
86    buckets.sort_by_key(|b| b.step_index);
87    buckets
88}
89
90/// Synthesize a `PhaseBucket` for any scenario step that has a stimulus
91/// StepStart but produced no capture bucket. Periodic boundaries are
92/// placed uniformly over the whole workload (step-agnostic — see
93/// `compute_periodic_boundaries_ns`), so a short interior step can
94/// capture zero samples and leave no bucket. The synthesized bucket
95/// carries the step's true stimulus window so `fold_monitor_into_bucket`
96/// still recovers its monitor-derived imbalance; `sample_count == 0` marks
97/// it capture-free for downstream consumers. A guest carrier can still attach
98/// a CPU-denominated `iteration_rate` during the later host/guest phase join.
99/// `Timeline::from_phase_buckets` gates monitor-derived metrics
100/// (imbalance/dsq/fallback/keep_last) behind both sides having samples — see
101/// `detect_boundary_changes`. BASELINE
102/// (step 0) is synthesized only if a StepStart carries
103/// `step_index == 0` — it is not special-cased.
104fn synthesize_missing_step_buckets(
105    buckets: &mut Vec<PhaseBucket>,
106    step_starts: &[(u64, u16)],
107    stimulus_events: &[crate::timeline::StimulusEvent],
108    monitor_samples: &[crate::monitor::MonitorSample],
109    monitor_to_window_offset_ms: i64,
110    preemption_threshold_ns: u64,
111) {
112    for &(start_ms, k) in step_starts {
113        if buckets.iter().any(|b| b.step_index == k) {
114            continue;
115        }
116        // Right edge: the step's own StepEnd, CLAMPED to the next
117        // step-start so a non-monotonic (corrupt-wire) StepEnd[k] >=
118        // StepStart[k+1] can never extend this synthesized window past the
119        // next step's start (which would fold the same MonitorSample into
120        // two adjacent synthesized buckets). Falls back to the next
121        // step-start alone, else open-ended (the last step on data with no
122        // StepEnd — e.g. sched-died — which the rate loop leaves rateless).
123        // Earliest StepEnd elapsed for k (min, symmetric with start_ms's
124        // earliest step-start), so a duplicate/corrupt StepEnd later in the
125        // vec can't pick a wrong right edge.
126        let step_end = stimulus_events
127            .iter()
128            .filter(|e| e.is_step_end && e.step_index == Some(k))
129            .map(|e| e.elapsed_ms)
130            .min();
131        // `ms > start_ms` (strict): two DISTINCT step_index sharing the
132        // same elapsed is corrupt-wire only (build_stimulus emits StepStarts
133        // at distinct monotone guest-clock times), so the equal-elapsed
134        // window-overlap edge is out of scope — its worst case is a
135        // redundant double-fold of one monitor sample, never a drop or panic.
136        let next_start = step_starts
137            .iter()
138            .filter(|&&(ms, _)| ms > start_ms)
139            .map(|&(ms, _)| ms)
140            .min();
141        // Last-resort right edge for a last step with no StepEnd and no
142        // successor: the scenario-end terminal event, bounding the window
143        // at run-end instead of folding every trailing teardown monitor
144        // sample into the bucket. NOTE the production sched-died path emits
145        // NEITHER a StepEnd NOR a ScenarioEnd (its early return skips both),
146        // so it carries no terminal here and falls to u64::MAX — matching
147        // Timeline::build, which extends the last phase to end-of-monitor.
148        // This terminal clamp therefore only binds for legacy/synthetic
149        // data carrying a ScenarioEnd frame without StepEnd frames.
150        let terminal = stimulus_events
151            .iter()
152            .find(|e| e.is_terminal)
153            .map(|e| e.elapsed_ms);
154        let end_ms = match (step_end, next_start) {
155            (Some(se), Some(ns)) => se.min(ns),
156            (Some(se), None) => se,
157            (None, Some(ns)) => ns,
158            (None, None) => terminal.unwrap_or(u64::MAX),
159        };
160        // BASELINE (step 0) is synthesized only if a StepStart carries
161        // step_index == 0; build_stimulus 1-indexes scenario Steps (Step 0
162        // -> step_index 1), so it never emits step_index 0 and BASELINE is
163        // intentionally never synthesized (its pre-first-Step settle window
164        // carries no meaningful per-step rate). The branch is kept for the
165        // convention, not for a production path.
166        let label = if k == 0 {
167            "BASELINE".to_string()
168        } else {
169            format!("Step[{}]", k.saturating_sub(1))
170        };
171        let mut bucket = PhaseBucket {
172            step_index: k,
173            label,
174            start_ms,
175            end_ms,
176            sample_count: 0,
177            metrics: std::collections::BTreeMap::new(),
178            per_cgroup: std::collections::BTreeMap::new(),
179        };
180        fold_monitor_into_bucket(
181            &mut bucket,
182            monitor_samples,
183            monitor_to_window_offset_ms,
184            preemption_threshold_ns,
185        );
186        buckets.push(bucket);
187    }
188}
189
190/// Build per-phase metric buckets from a sample series.
191///
192/// Walks [`crate::scenario::sample::SampleSeries::by_stamped_phase`]
193/// to group every stamped sample under its bridge-stamped
194/// `step_index` (NOT re-derived from elapsed-ms windows; the
195/// bridge stamp is authoritative because the capture path knows
196/// the phase it fired from while the time window cannot recover
197/// the phase when stimulus events arrive late or out of order).
198///
199/// For each phase observed (BASELINE under `step_index = 0`,
200/// scenario Steps under `step_index = 1..=N` per the 1-indexed
201/// phase convention) emits one [`PhaseBucket`] with `step_index`
202/// as the key, `label` derived per the BASELINE/Step\[k\]
203/// convention, `start_ms` / `end_ms` from the first / last
204/// sample's `elapsed_ms`, `sample_count` from the bucketed
205/// samples, and `metrics` from the per-kind reduction described
206/// on [`PhaseBucket`]. Metrics whose per-sample reading returns
207/// `None` for every sample in the bucket are omitted entirely
208/// (absent → "no data") rather than collapsed to `Some(0.0)`
209/// (real zero), preserving the sentinel-free contract.
210///
211/// Returns an empty `Vec` when the input series is empty (no
212/// samples captured), distinct from returning a single empty
213/// BASELINE bucket — the former means the periodic-capture path
214/// never fired, the latter means it fired but no metric reading
215/// came back.
216///
217/// No live production caller: `evaluate_vm_result` in
218/// `src/test_support/eval/mod.rs` builds its phase buckets through the
219/// stimulus-aware sibling `build_phase_buckets_with_stimulus` (via
220/// `precompute_early_series`), and `AssertResult.stats.phases` is
221/// populated from those stimulus buckets. This plain variant is
222/// reachable only by full path, for the rare no-stimulus-timeline
223/// case. Exposed `pub` (not `pub(crate)`)
224/// so out-of-tree consumers — payload authors writing custom
225/// eval paths against the publicly-drainable
226/// `result.snapshot_bridge` — can produce the same per-phase
227/// aggregate shape without re-implementing the bucketing logic.
228pub fn build_phase_buckets(samples: &crate::scenario::sample::SampleSeries) -> Vec<PhaseBucket> {
229    // Borrowed per-tick monitor samples (None when no MonitorReport
230    // was attached, e.g. host-only fixture tests).
231    let monitor_samples: &[crate::monitor::MonitorSample] =
232        samples.monitor().map(|m| m.samples()).unwrap_or(&[]);
233    // vCPU-preemption exemption window for the per-phase stuck predicate
234    // (see build_phase_buckets_with_stimulus). 0 (no monitor) derives from
235    // CONFIG_HZ inside compute_metrics.
236    let preemption_threshold_ns: u64 = samples
237        .monitor()
238        .map(|m| m.preemption_threshold_ns())
239        .unwrap_or(0);
240    // Group by the bridge-stamped step_index. Without a stimulus
241    // timeline to remap against, the stamped index is the only phase
242    // signal available — see `build_phase_buckets_with_stimulus` (and
243    // `SampleSeries::by_stimulus_phase`) for the offset-remap that
244    // corrects a deferred-fire burst (every capture stamping the same
245    // late CURRENT_STEP). The bucket window and the monitor folding
246    // share the run-relative frame here, so the clock offset is `0`;
247    // synthetic / legacy fixture samples carry `boundary_offset_ms =
248    // None` and the window falls back to `elapsed_ms`.
249    buckets_from_grouped(
250        samples.by_stamped_phase(),
251        monitor_samples,
252        0,
253        preemption_threshold_ns,
254    )
255}
256
257/// Per-phase CPU-time delta (ns) for one field family
258/// (`stime`/`signal_stime` or `utime`/`signal_utime`), folded host-side
259/// from the frozen `task_struct` enrichments captured at the phase's
260/// freeze boundaries. Backs the injected `system_time_ns` /
261/// `user_time_ns` per-phase metrics.
262///
263/// The unit is the THREAD GROUP, not the individual task: the kernel's
264/// `thread_group_cputime` is `signal_struct.{u,s}time` (the accumulator
265/// a dying thread's time is folded into at exit) plus the live threads'
266/// `task_struct.{u,s}time`. `signal_struct` is shared across a thread
267/// group, so its value is counted once per `tgid`; the live counters are
268/// summed across the group's threads. For each `tgid` the group total is
269/// taken at the FIRST and LAST sample in which the group had a READABLE
270/// total (ordered by capture time) and `last - first` is summed across
271/// groups.
272///
273/// Per-group first-seen/last-seen, NOT a per-sample cross-task SUM then a
274/// Counter `last - first`: the captured set changes between freezes
275/// (system tasks churn in and out). A task carrying a large cumulative
276/// counter that appears only in a LATER sample would dump its entire
277/// pre-phase history into a summed delta, inflating the phase value
278/// many-fold. Subtracting each group's OWN first-seen total cancels its
279/// pre-phase history, so a late-joining group contributes only what it
280/// accrued while observed — bounding the result by wall-clock × cores. A
281/// group's thread exiting mid-phase does not dip the total: its time
282/// moves from a `task_struct` counter into `signal_struct`, both of which
283/// the group total includes.
284///
285/// A `None` `signal_field` is a `signal_struct` translate miss, NOT a
286/// real zero (which reads `Some(0)`): such a sample is omitted for that
287/// group so every endpoint is a full live+signal total — mixing a
288/// live-only endpoint with a live+signal one would otherwise leak the
289/// cumulative accumulator as a phantom positive. A numeric `tgid` reused
290/// within the phase (process exit + PID realloc) can read lower at last
291/// than first; `saturating_sub` clamps that to 0 rather than wrapping.
292///
293/// Returns `None` when no group was observed with a readable total across
294/// at least two samples — no delta is measurable — keeping an absent
295/// per-phase bucket key distinct from a real `0` (a qualifying group
296/// whose counters did not advance yields `Some(0.0)`). Accumulates in
297/// `u128` to stay exact before the final `f64` (a phase can total many
298/// task-seconds of ns).
299fn phase_group_cpu_delta(
300    samples: &[crate::scenario::sample::Sample<'_>],
301    task_field: impl Fn(&crate::monitor::task_enrichment::TaskEnrichment) -> u64,
302    signal_field: impl Fn(&crate::monitor::task_enrichment::TaskEnrichment) -> Option<u64>,
303) -> Option<f64> {
304    // Per-tgid `thread_group_cputime` total at one sample: the shared
305    // signal_struct accumulator (once per group) plus the sum of the
306    // group's live threads' task_struct counter. A group is INCLUDED for
307    // a sample only when its signal accumulator was readable there (some
308    // thread of the tgid carried `Some`): a `None` is a signal_struct
309    // translate miss (see `task_enrichment.rs`), NOT a real zero — a real
310    // zero reads `Some(0)`. Omitting unreadable-signal groups keeps every
311    // endpoint a FULL group total (live + signal), so the first/last delta
312    // can never mix a live-only endpoint with a live+signal endpoint and
313    // leak the cumulative accumulator as a phantom positive.
314    let group_totals = |s: &crate::scenario::sample::Sample<'_>| {
315        let mut live: std::collections::HashMap<i32, u128> = std::collections::HashMap::new();
316        let mut signal: std::collections::HashMap<i32, Option<u128>> =
317            std::collections::HashMap::new();
318        for t in s.snapshot.task_enrichments() {
319            *live.entry(t.tgid).or_insert(0) += u128::from(task_field(t));
320            match signal_field(t) {
321                // Shared across the group — any readable thread fixes it.
322                Some(v) => {
323                    signal.insert(t.tgid, Some(u128::from(v)));
324                }
325                None => {
326                    signal.entry(t.tgid).or_insert(None);
327                }
328            }
329        }
330        live.into_iter()
331            .filter_map(|(tgid, l)| {
332                signal
333                    .get(&tgid)
334                    .copied()
335                    .flatten()
336                    .map(|sig| (tgid, l + sig))
337            })
338            .collect::<std::collections::HashMap<i32, u128>>()
339    };
340
341    // Order by capture time so first/last are the earliest/latest
342    // boundary — the grouped vec is not guaranteed sorted (the
343    // offset-remap in the stimulus path can reorder samples).
344    let mut ordered: Vec<&crate::scenario::sample::Sample<'_>> = samples.iter().collect();
345    // `elapsed_ms` is now Option: a sample with neither a
346    // boundary offset nor a measured elapsed has no time anchor — sort
347    // it LAST (u64::MAX), never first, so an untimestamped sample can't
348    // become the spurious earliest first_seen endpoint.
349    ordered.sort_by_key(|s| s.boundary_offset_ms.or(s.elapsed_ms).unwrap_or(u64::MAX));
350
351    // Per tgid: its full group total at the first and last sample in which
352    // it had a readable total, and how many such samples it had.
353    let mut first_seen: std::collections::HashMap<i32, u128> = std::collections::HashMap::new();
354    let mut last_seen: std::collections::HashMap<i32, u128> = std::collections::HashMap::new();
355    let mut readable_count: std::collections::HashMap<i32, u32> = std::collections::HashMap::new();
356    for s in ordered {
357        for (tgid, total) in group_totals(s) {
358            first_seen.entry(tgid).or_insert(total);
359            last_seen.insert(tgid, total);
360            *readable_count.entry(tgid).or_insert(0) += 1;
361        }
362    }
363
364    let mut sum: u128 = 0;
365    let mut measured = false;
366    for (tgid, last) in &last_seen {
367        // A delta needs the group observed with a readable total across
368        // TWO boundaries; one readable sample gives first == last and no
369        // measurable in-phase growth (and a tgid that appears in only one
370        // sample, or whose signal_struct never translated, never reaches
371        // 2). saturating_sub clamps a last < first read — a numeric tgid
372        // reused within the phase (process exit + PID realloc to a fresh
373        // group starting near zero) reads lower at last; clamp to 0 rather
374        // than wrap.
375        if readable_count.get(tgid).copied().unwrap_or(0) < 2 {
376            continue;
377        }
378        measured = true;
379        // first_seen always holds tgid (populated in the same pass).
380        let first = first_seen.get(tgid).copied().unwrap_or(*last);
381        sum += last.saturating_sub(first);
382    }
383    // None when no group was observed with a readable total across two
384    // samples — unmeasurable, so the bucket key stays ABSENT (distinct
385    // from a real 0: a qualifying group whose counters did not advance
386    // yields Some(0.0)).
387    measured.then_some(sum as f64)
388}
389
390/// Per-phase per-CPU spatial-max fold for a monotonic per-CPU counter: the
391/// BUSIEST CPU's delta over the phase (`max_key`) + the concentration ratio
392/// `max / mean` over the reporting CPUs (`concentration_key`). A custom
393/// per-CPU-delta fold — NOT a read_sample arm (read_sample yields the cross-CPU
394/// SUM as one f64, with no per-CPU vector). `field` selects the per-CPU counter
395/// (`|c| c.irqs_sum` for hardirqs, `|c| c.softirqs[NET_RX]` for NET_RX softirqs).
396///
397/// Endpoints are the earliest + latest freeze BY `win` (the SAME key the bucket
398/// window uses; `samples_in_phase` is NOT positionally ordered — the stimulus
399/// offset-remap can reorder it). The per-CPU delta is correlated by the
400/// per_cpu_time `cpu` FIELD (not vec position) over the CPUs present in BOTH
401/// endpoints; a CPU absent from either is skipped (hotplug defense — a no-op on
402/// ktstr's boot-fixed vCPU set, correct-by-construction if a hot-add Op lands).
403/// `saturating_sub` clamps a per-CPU counter regress (the kernel counters are
404/// monotonic; a regress would signal a reset). The per-CPU delta is taken
405/// FIRST, the spatial max SECOND — so a busiest CPU that SHIFTS across the phase
406/// is captured as max(per-CPU deltas), not a delta of spatial maxes.
407///
408/// Loud-absent: `max_key` needs at least 2 freezes spanning a positive `win`
409/// window and at least 1 defined-delta CPU; `concentration_key` ADDITIONALLY
410/// needs at least 2 defined-delta CPUs (a 1-CPU intersection makes max/mean == 1
411/// a structural artifact) and a positive mean (a NaN would fail the sidecar's
412/// is_finite insert-guard).
413fn fold_per_cpu_spatial_max(
414    metrics: &mut std::collections::BTreeMap<String, f64>,
415    samples_in_phase: &[crate::scenario::sample::Sample<'_>],
416    win: impl Fn(&crate::scenario::sample::Sample<'_>) -> Option<u64>,
417    field: impl Fn(&crate::monitor::dump::PerCpuTimeStats) -> u64,
418    max_key: &str,
419    concentration_key: &str,
420) {
421    let placed: Vec<(&crate::scenario::sample::Sample<'_>, u64)> = samples_in_phase
422        .iter()
423        .filter(|s| !s.snapshot.per_cpu_time().is_empty())
424        .filter_map(|s| win(s).map(|w| (s, w)))
425        .collect();
426    if let (Some(&(first_s, fw)), Some(&(last_s, lw))) = (
427        placed.iter().min_by_key(|(_, w)| *w),
428        placed.iter().max_by_key(|(_, w)| *w),
429    ) && fw < lw
430    {
431        // Per-CPU delta over the cpu-field intersection of the two endpoint
432        // freezes; clamp a regress to 0.
433        let deltas: Vec<f64> = first_s
434            .snapshot
435            .per_cpu_time()
436            .iter()
437            .filter_map(|c0| {
438                last_s
439                    .snapshot
440                    .per_cpu_time_at(c0.cpu)
441                    .map(|c1| field(c1).saturating_sub(field(c0)) as f64)
442            })
443            .collect();
444        if let Some(max) = deltas.iter().copied().reduce(f64::max) {
445            metrics.entry(max_key.to_string()).or_insert(max);
446            if deltas.len() >= 2 {
447                let mean = deltas.iter().sum::<f64>() / deltas.len() as f64;
448                if mean > 0.0 {
449                    metrics
450                        .entry(concentration_key.to_string())
451                        .or_insert(max / mean);
452                }
453            }
454        }
455    }
456}
457
458/// Delivered guest CPU time between two per-CPU cpustat snapshots, summed
459/// across CPUs. This is the guest's own CPU-time currency: user, nice, system,
460/// irq, softirq, idle, and iowait advance only while a vCPU is executing;
461/// CPUTIME_STEAL is deliberately excluded. Require the same complete, unique
462/// CPU-id set at both endpoints: the IRQ numerator is a whole-VM sum, so an
463/// intersection-only denominator could fabricate a rate from a partial read.
464/// Every component delta and sum saturates so a reset or hostile snapshot
465/// cannot wrap into a huge denominator.
466pub(crate) fn guest_cpu_time_delta_ns(
467    first: &[crate::monitor::dump::PerCpuTimeStats],
468    last: &[crate::monitor::dump::PerCpuTimeStats],
469) -> Option<u64> {
470    if first.is_empty() || first.len() != last.len() {
471        return None;
472    }
473    let first_ids: std::collections::BTreeSet<u32> = first.iter().map(|c| c.cpu).collect();
474    let last_ids: std::collections::BTreeSet<u32> = last.iter().map(|c| c.cpu).collect();
475    if first_ids.len() != first.len() || last_ids.len() != last.len() || first_ids != last_ids {
476        return None;
477    }
478    let total_without_steal = |c: &crate::monitor::dump::PerCpuTimeStats| {
479        [
480            c.cpustat_user_ns,
481            c.cpustat_nice_ns,
482            c.cpustat_system_ns,
483            c.cpustat_softirq_ns,
484            c.cpustat_irq_ns,
485            c.cpustat_idle_ns,
486            c.cpustat_iowait_ns,
487        ]
488    };
489    let mut total = 0u64;
490    for before in first {
491        let after = last
492            .iter()
493            .find(|after| after.cpu == before.cpu)
494            .expect("equal unique CPU-id sets checked above");
495        let cpu_delta = total_without_steal(after)
496            .into_iter()
497            .zip(total_without_steal(before))
498            .map(|(after, before)| after.saturating_sub(before))
499            .fold(0u64, u64::saturating_add);
500        total = total.saturating_add(cpu_delta);
501    }
502    (total > 0).then_some(total)
503}
504
505/// Per-CPU scx_layered util-compensation SCALE over a freeze span: the factor by
506/// which a CPU's useful-work (non-overhead) capacity is scaled up to compensate
507/// for IRQ / softirq / stolen time. Over the `first`->`last` cpustat delta,
508/// `scale = delta_total / available` where `delta_total` is the sum of ALL 8
509/// `kernel_cpustat[]` ns slots (user+nice+system+idle+iowait+irq+softirq+steal)
510/// and `available = delta_total - (irq + softirq + steal)`. Clamped to
511/// `[1.0, 20.0]`; `available == 0` (overhead consumed the whole window, or no
512/// forward progress) returns the floor `1.0` (no compensation) — scx_layered's
513/// `available > 0` guard, expressed over `u64` saturating deltas where `> 0`
514/// is `!= 0`.
515///
516/// Byte-faithful to scx_layered's `util_compensation` per-CPU compute. The
517/// ns-vs-µs unit is immaterial: the ratio cancels it (scx_layered reads /proc
518/// microseconds; here the host reads `kernel_cpustat` ns, the same slots
519/// `/proc/stat` formats from). Hostile-guest hardening: every per-slot delta is
520/// `saturating_sub` (a cpustat counter that regresses across the span — a
521/// CPU-hotplug reset, or a corrupt read — clamps to 0, never wraps to
522/// ~`u64::MAX`), exactly as scx_layered does. The one divergence from
523/// scx_layered: the 8-slot `delta_total` and 3-slot `overhead` sums use
524/// `saturating_add` (a hostile per-slot `u64::MAX` clamps the total, mirroring
525/// the per-CPU IRQ spatial sums in [`crate::stats::MetricDef::read_sample`]),
526/// where scx_layered combines its per-slot saturating-subs with plain `+` — its
527/// /proc inputs are trusted, ours are guest-controlled. With `overhead >= 0` the
528/// ratio is always `>= 1.0`, so the clamp's floor is belt-and-suspenders and the
529/// cap at 20.0 is the only active bound.
530pub(crate) fn cpu_util_comp_scale(
531    first: &crate::monitor::dump::PerCpuTimeStats,
532    last: &crate::monitor::dump::PerCpuTimeStats,
533) -> f64 {
534    let user = last.cpustat_user_ns.saturating_sub(first.cpustat_user_ns);
535    let nice = last.cpustat_nice_ns.saturating_sub(first.cpustat_nice_ns);
536    let system = last
537        .cpustat_system_ns
538        .saturating_sub(first.cpustat_system_ns);
539    let idle = last.cpustat_idle_ns.saturating_sub(first.cpustat_idle_ns);
540    let iowait = last
541        .cpustat_iowait_ns
542        .saturating_sub(first.cpustat_iowait_ns);
543    let irq = last.cpustat_irq_ns.saturating_sub(first.cpustat_irq_ns);
544    let softirq = last
545        .cpustat_softirq_ns
546        .saturating_sub(first.cpustat_softirq_ns);
547    let steal = last.cpustat_steal_ns.saturating_sub(first.cpustat_steal_ns);
548    let delta_total = [user, nice, system, idle, iowait, irq, softirq, steal]
549        .into_iter()
550        .fold(0u64, u64::saturating_add);
551    let overhead = irq.saturating_add(softirq).saturating_add(steal);
552    let available = delta_total.saturating_sub(overhead);
553    if available == 0 {
554        return 1.0;
555    }
556    (delta_total as f64 / available as f64).clamp(1.0, 20.0)
557}
558
559/// Per-phase mean ACROSS CPUs of the [`cpu_util_comp_scale`] over the
560/// `per_cpu_time` freeze span, inserted as `avg_cpu_util_comp_scale`. Endpoint
561/// selection (first/last by `win`), the cpu-field intersection (per-CPU paired
562/// by the `cpu` field; a CPU absent from either endpoint is skipped — hotplug
563/// churn), the saturating deltas, and the loud-absent discipline (no key when
564/// no CPU pairs or the window is degenerate, `fw == lw`) mirror
565/// [`fold_per_cpu_spatial_max`]. `Gauge(Avg)`, so it auto-folds to run-level
566/// (weighted mean across phases) and cross-run with no extra wiring.
567///
568/// Cross-boot caveat: `samples_in_phase` are the periodic freezes, whose
569/// window starts at `max(scenario-start, prereqs-ready)` — on a cold boot
570/// (accessor/attach lag) it starts later, so this key reflects a later
571/// workload sub-window. Across a cold/warm boot mix its cross-run magnitude
572/// shifts; use `--noise-adjust` (spread analysis absorbs the boot-timing
573/// jitter) for cross-run compares — a plain single-pair (cached) compare
574/// of this key is advisory.
575fn fold_util_comp_scale(
576    metrics: &mut std::collections::BTreeMap<String, f64>,
577    samples_in_phase: &[crate::scenario::sample::Sample<'_>],
578    win: impl Fn(&crate::scenario::sample::Sample<'_>) -> Option<u64>,
579) {
580    let placed: Vec<(&crate::scenario::sample::Sample<'_>, u64)> = samples_in_phase
581        .iter()
582        .filter(|s| !s.snapshot.per_cpu_time().is_empty())
583        .filter_map(|s| win(s).map(|w| (s, w)))
584        .collect();
585    if let (Some(&(first_s, fw)), Some(&(last_s, lw))) = (
586        placed.iter().min_by_key(|(_, w)| *w),
587        placed.iter().max_by_key(|(_, w)| *w),
588    ) && fw < lw
589    {
590        // Per-CPU clamped scale over the cpu-field intersection of the two
591        // endpoint freezes, then the arithmetic mean across the reporting CPUs.
592        let scales: Vec<f64> = first_s
593            .snapshot
594            .per_cpu_time()
595            .iter()
596            .filter_map(|c0| {
597                last_s
598                    .snapshot
599                    .per_cpu_time_at(c0.cpu)
600                    .map(|c1| cpu_util_comp_scale(c0, c1))
601            })
602            .collect();
603        if !scales.is_empty() {
604            let mean = scales.iter().sum::<f64>() / scales.len() as f64;
605            metrics
606                .entry("avg_cpu_util_comp_scale".to_string())
607                .or_insert(mean);
608        }
609    }
610}
611
612/// Per-phase per-task latency-criticality aggregate from the scheduler's
613/// `sdt_alloc` arena (scx_lavd's `task_ctx.normalized_lat_cri`, `[0, 1024]`).
614/// Over every freeze's `sdt_allocations` walk, pulls each live task_ctx's
615/// `normalized_lat_cri` — a host-rendered arena field (BPF_MAP_TYPE_ARENA), NOT
616/// a kernel counter and NOT a BPF `.bss` field — and folds across (freeze,
617/// task): mean -> `avg_task_lat_cri`, max -> `max_task_lat_cri`.
618///
619/// A GAUGE: `normalized_lat_cri` is an instantaneous per-task value lavd
620/// recomputes each schedule (scx_lavd `lat_cri.bpf.c`), so the fold is over ALL
621/// samples (mean / max over every (freeze, task) observation), NOT a first/last
622/// delta like the cpustat folds. `normalized` (not raw `lat_cri`) for cross-run
623/// comparability — raw `lat_cri` is squared + waker/wakee-propagated +
624/// load-dependent, so its magnitude is not comparable across runs.
625///
626/// Loud-absent + scheduler-agnostic by construction: a non-lavd payload has no
627/// `normalized_lat_cri` member, so [`crate::monitor::btf_render::RenderedValue::get`]
628/// returns `None` and the entry is skipped; if no entry yields the field, no key
629/// is inserted (the "no key when absent" discipline the sibling folds use). The
630/// member name is the only gate (lavd-unique) — no scheduler-name hardcoding.
631///
632/// Cross-boot caveat: like `fold_util_comp_scale`, these are the periodic
633/// freezes, whose window starts later on a cold boot (the prereq-ready
634/// anchor), so `avg_task_lat_cri`'s cross-run magnitude shifts across a
635/// cold/warm boot mix; use `--noise-adjust` for cross-run compares —
636/// a single-pair (cached) compare is advisory.
637fn fold_lat_cri(
638    metrics: &mut std::collections::BTreeMap<String, f64>,
639    samples_in_phase: &[crate::scenario::sample::Sample<'_>],
640) {
641    let mut sum = 0.0f64;
642    let mut count = 0usize;
643    let mut max = f64::MIN;
644    for s in samples_in_phase {
645        for alloc in &s.snapshot.report().sdt_allocations {
646            for entry in &alloc.entries {
647                if let Some(v) = entry
648                    .payload
649                    .get("normalized_lat_cri")
650                    .and_then(|r| r.as_u64())
651                {
652                    let v = v as f64;
653                    sum += v;
654                    count += 1;
655                    if v > max {
656                        max = v;
657                    }
658                }
659            }
660        }
661    }
662    if count > 0 {
663        metrics
664            .entry("avg_task_lat_cri".to_string())
665            .or_insert(sum / count as f64);
666        metrics.entry("max_task_lat_cri".to_string()).or_insert(max);
667    }
668}
669
670/// Per-phase per-CGROUP spatial axis over the workload-leaf PSI-irq captured at
671/// each freeze (`Snapshot::cgroup_psi`). The per-cgroup analog of
672/// [`fold_per_cpu_spatial_max`], producing three Peak metrics:
673///
674/// - `max_cgroup_psi_irq_avg10`: the worst leaf's IRQ-full pressure GAUGE
675///   (decoded avg10 percent) — per-freeze max across leaves, then max across the
676///   phase's freezes. A gauge, so no delta/baseline (a spatial-max of an
677///   instantaneous reading, the `max_avg_irq_util` shape on the cgroup axis).
678/// - `max_cgroup_irq_pressure`: the busiest leaf's IRQ-full stall DELTA over the
679///   phase (decoded µs) — per-leaf (last - first) `total_ns`, correlated by
680///   `cgroup_kva`, then the spatial max. A monotonic Counter, so delta-FIRST
681///   then spatial-max (the `max_cpu_hardirqs` shape). Informational
682///   (workload-confounded magnitude).
683/// - `max_cgroup_irq_pressure_concentration`: `max_cgroup_irq_pressure` / the
684///   mean per-leaf delta over the SAME leaf set — the busiest cell's share (the
685///   isolation/steering signal). max/MEAN, LowerBetter (the
686///   `max_cpu_hardirq_concentration` shape). Absent when < 2 reporting leaves or
687///   mean == 0.
688///
689/// Endpoints + the leaf intersection + saturating + loud-absent rules mirror
690/// [`fold_per_cpu_spatial_max`] (the `(cgroup_kva, serial_nr)` pair is the
691/// per-CPU `cpu`-field analog — the serial_nr disambiguates a freed slab KVA
692/// reused by a NEW cgroup, which `cgroup_kva` alone cannot; a leaf absent from
693/// either endpoint is skipped — leaf churn / a cgroup created mid-phase). All
694/// three are Peak → they auto-fold max-across-phases to run-level and cross-run
695/// MAX, no extra wiring.
696fn fold_per_cgroup_psi(
697    metrics: &mut std::collections::BTreeMap<String, f64>,
698    samples_in_phase: &[crate::scenario::sample::Sample<'_>],
699    win: impl Fn(&crate::scenario::sample::Sample<'_>) -> Option<u64>,
700) {
701    use crate::monitor::btf_offsets::{decode_avg10_percent, decode_total_us};
702    // avg10 gauge: per-freeze max across leaves, then max across the freezes.
703    let avg10_peak = samples_in_phase
704        .iter()
705        .filter_map(|s| {
706            s.snapshot
707                .cgroup_psi()
708                .iter()
709                .map(|c| decode_avg10_percent(c.avg10_raw))
710                .reduce(f64::max)
711        })
712        .reduce(f64::max);
713    if let Some(v) = avg10_peak {
714        metrics
715            .entry("max_cgroup_psi_irq_avg10".to_string())
716            .or_insert(v);
717    }
718
719    // total-delta spatial-max + concentration. Earliest + latest freeze by `win`
720    // (the bucket-window key; samples_in_phase is not positionally ordered). The
721    // per-leaf delta is correlated by `(cgroup_kva, serial_nr)` over the leaves
722    // present in BOTH endpoints; a leaf absent from either is skipped. The
723    // serial_nr guards KVA REUSE: a leaf rmdir'd mid-phase whose freed slab KVA a
724    // NEW cgroup then reused would alias by KVA alone and yield a bogus
725    // cross-cgroup delta — a different serial_nr at the same KVA is treated as
726    // absent (dropped), not differenced. `saturating_sub` then clamps only a
727    // (rare) genuine same-cgroup counter regress. delta FIRST, spatial max
728    // SECOND — a busiest cell that shifts across the phase is captured as
729    // max(per-leaf deltas), not a delta of spatial maxes.
730    let placed: Vec<(&crate::scenario::sample::Sample<'_>, u64)> = samples_in_phase
731        .iter()
732        .filter(|s| !s.snapshot.cgroup_psi().is_empty())
733        .filter_map(|s| win(s).map(|w| (s, w)))
734        .collect();
735    if let (Some(&(first_s, fw)), Some(&(last_s, lw))) = (
736        placed.iter().min_by_key(|(_, w)| *w),
737        placed.iter().max_by_key(|(_, w)| *w),
738    ) && fw < lw
739    {
740        let deltas: Vec<f64> = first_s
741            .snapshot
742            .cgroup_psi()
743            .iter()
744            .filter_map(|c0| {
745                last_s
746                    .snapshot
747                    .cgroup_psi()
748                    .iter()
749                    .find(|c1| c1.cgroup_kva == c0.cgroup_kva && c1.serial_nr == c0.serial_nr)
750                    .map(|c1| decode_total_us(c1.total_ns.saturating_sub(c0.total_ns)))
751            })
752            .collect();
753        if let Some(max) = deltas.iter().copied().reduce(f64::max) {
754            metrics
755                .entry("max_cgroup_irq_pressure".to_string())
756                .or_insert(max);
757            if deltas.len() >= 2 {
758                let mean = deltas.iter().sum::<f64>() / deltas.len() as f64;
759                if mean > 0.0 {
760                    metrics
761                        .entry("max_cgroup_irq_pressure_concentration".to_string())
762                        .or_insert(max / mean);
763                }
764            }
765        }
766    }
767}
768
769/// Assemble [`PhaseBucket`]s from a pre-grouped phase map. Shared by
770/// [`build_phase_buckets`] (grouping by the bridge-stamped step_index)
771/// and [`build_phase_buckets_with_stimulus`] (grouping by the
772/// offset-remapped step).
773///
774/// `monitor_to_window_offset_ms` is subtracted from each
775/// [`crate::monitor::MonitorSample`] `elapsed_ms` before the window
776/// test, bringing the monitor sample's run-relative timestamp into the
777/// bucket-window frame: `0` when both share the run-relative frame, the
778/// stimulus/monitor clock skew (see [`monitor_clock_offset`]) when the
779/// window is workload-relative (boundary-offset) but the monitor samples
780/// remain run-relative.
781///
782/// Each bucket folds the monitor samples whose (shifted) elapsed_ms
783/// lands in the bucket window — supplying metrics like
784/// `avg_imbalance_ratio` that need per-CPU full-class `rq.nr_running`,
785/// which the bridge-captured Snapshot does not expose (Snapshot carries
786/// scx_rq.nr_running only).
787///
788/// `preemption_threshold_ns` is forwarded to `fold_monitor_into_bucket`
789/// (and thence `compute_metrics`) for the per-phase stall predicate; see
790/// `fold_monitor_into_bucket`.
791fn buckets_from_grouped(
792    by_phase: std::collections::BTreeMap<u16, Vec<crate::scenario::sample::Sample<'_>>>,
793    monitor_samples: &[crate::monitor::MonitorSample],
794    monitor_to_window_offset_ms: i64,
795    preemption_threshold_ns: u64,
796) -> Vec<PhaseBucket> {
797    let mut out: Vec<PhaseBucket> = Vec::with_capacity(by_phase.len());
798    for (step_index, samples_in_phase) in by_phase {
799        let label = if step_index == 0 {
800            "BASELINE".to_string()
801        } else {
802            // Scenario-Step ordinal lives at `step_index - 1`
803            // because phase 0 is BASELINE under the 1-indexed
804            // encoding; saturate at 0 if the underflow guard
805            // ever fires (unreachable for the current encoding
806            // — step_index here came from the bucket key so the
807            // `> 0` branch is satisfied — but keep the guard so
808            // a future caller that hands in a synthetic
809            // `step_index = 0` does not panic).
810            format!("Step[{}]", step_index.saturating_sub(1))
811        };
812        let sample_count = samples_in_phase.len();
813        // Bucket window: prefer each capture's workload-relative
814        // scheduled boundary offset (stable across a deferred-fire
815        // burst) over the run-relative fire time; fall back to
816        // `elapsed_ms` per-sample when no offset was stamped (synthetic
817        // / on-demand). min/max rather than first/last because the
818        // offset-remap in the stimulus path can reorder samples
819        // relative to drain order.
820        let win = |s: &crate::scenario::sample::Sample<'_>| s.boundary_offset_ms.or(s.elapsed_ms);
821        let (start_ms, end_ms) = if samples_in_phase.is_empty() {
822            (0, u64::MAX)
823        } else {
824            let mut lo = u64::MAX;
825            let mut hi = 0u64;
826            for s in &samples_in_phase {
827                // Skip a sample with no time anchor (no boundary offset
828                // AND no measured elapsed): coercing it to 0
829                // would pull start_ms to 0 and over-fold monitor samples.
830                // If EVERY sample in the phase is unanchored, lo/hi stay
831                // (u64::MAX, 0) — an inverted window that folds nothing,
832                // the correct "no placeable samples" outcome.
833                if let Some(w) = win(s) {
834                    lo = lo.min(w);
835                    hi = hi.max(w);
836                }
837            }
838            (lo, hi)
839        };
840        let mut metrics: std::collections::BTreeMap<String, f64> =
841            std::collections::BTreeMap::new();
842        for metric_def in crate::stats::METRICS {
843            let per_sample_readings: Vec<f64> = samples_in_phase
844                .iter()
845                .filter_map(|s| metric_def.read_sample(s))
846                .collect();
847            if per_sample_readings.is_empty() {
848                // No per-sample reading for any sample in this
849                // bucket -- the metric is host-side-only
850                // (cross-cgroup fold) or its dispatch arm has
851                // not landed yet. Omit the key rather than
852                // collapsing to `Some(0.0)` so the renderer
853                // paints "absent" vs "real zero" distinctly.
854                continue;
855            }
856            if let Some(reduced) =
857                crate::stats::aggregate_samples_for_phase(metric_def, &per_sample_readings)
858            {
859                metrics.insert(metric_def.name.to_string(), reduced);
860            }
861        }
862        // Per-phase system / user CPU time (ns), injected post-hoc as a
863        // per-thread-GROUP delta over the phase's freeze samples (NOT a
864        // read_sample metric — a per-sample cross-task sum then a Counter
865        // delta inflates when the captured task set churns; see
866        // `phase_group_cpu_delta`). Observer-free: reads the frozen
867        // task_struct.{s,u}time + thread-group signal_struct accumulator
868        // already captured in each sample's enrichments.
869        let system_time_ns =
870            phase_group_cpu_delta(&samples_in_phase, |t| t.stime, |t| t.signal_stime);
871        let user_time_ns =
872            phase_group_cpu_delta(&samples_in_phase, |t| t.utime, |t| t.signal_utime);
873        if let Some(v) = system_time_ns {
874            metrics.entry("system_time_ns".to_string()).or_insert(v);
875        }
876        if let Some(v) = user_time_ns {
877            metrics.entry("user_time_ns".to_string()).or_insert(v);
878        }
879        // Same task-group population and endpoints as the two components above.
880        // This denominator makes system_cpu_fraction immune to how much CPU the
881        // host happened to deliver during the fixed guest-wall phase.
882        if let (Some(system), Some(user)) = (system_time_ns, user_time_ns) {
883            metrics
884                .entry("observed_task_cpu_time_ns".to_string())
885                .or_insert(system + user);
886        }
887        // Per-CPU spatial axes (busiest-CPU counter delta + concentration) over
888        // the per_cpu_time freezes: hardirqs (kstat.irqs_sum) and NET_RX softirqs
889        // (kstat.softirqs[NET_RX]). Both are monotonic per-CPU counters; the fold
890        // takes the per-CPU delta FIRST then the spatial max — see
891        // `fold_per_cpu_spatial_max` for the shared endpoint / cpu-field
892        // intersection / saturating / loud-absent rules.
893        fold_per_cpu_spatial_max(
894            &mut metrics,
895            &samples_in_phase,
896            win,
897            |c| c.irqs_sum,
898            "max_cpu_hardirqs",
899            "max_cpu_hardirq_concentration",
900        );
901        fold_per_cpu_spatial_max(
902            &mut metrics,
903            &samples_in_phase,
904            win,
905            |c| c.softirqs[crate::monitor::btf_offsets::SOFTIRQ_NET_RX],
906            "max_cpu_softirq_net_rx",
907            "max_cpu_softirq_net_rx_concentration",
908        );
909        // scx_layered util-compensation scale: per-CPU first->last cpustat delta
910        // -> clamped scale -> mean across CPUs (avg_cpu_util_comp_scale). Over
911        // the same per_cpu_time freezes as the spatial axes above, but a Gauge
912        // (per-CPU clamp-then-mean), not a spatial-max counter delta.
913        fold_util_comp_scale(&mut metrics, &samples_in_phase, win);
914        // scx_lavd per-task latency-criticality: mean/max of each live task_ctx's
915        // normalized_lat_cri ([0,1024]) over the sdt_alloc arena walk at every
916        // freeze (avg_task_lat_cri / max_task_lat_cri). A gauge folded over all
917        // (freeze, task) observations; loud-absent for non-lavd schedulers.
918        fold_lat_cri(&mut metrics, &samples_in_phase);
919        // Per-cgroup PSI-irq spatial axis: the busiest workload-leaf's IRQ-full
920        // stall delta + avg10 gauge + the cross-leaf concentration, over the
921        // cgroup_psi freezes (the host cgroup-walk capture). All Peak; auto-fold
922        // run-level.
923        fold_per_cgroup_psi(&mut metrics, &samples_in_phase, win);
924        let mut bucket = PhaseBucket {
925            step_index,
926            label,
927            start_ms,
928            end_ms,
929            sample_count,
930            metrics,
931            per_cgroup: std::collections::BTreeMap::new(),
932        };
933        // Per-phase MonitorSample windowing for monitor-derived metrics
934        // (avg_imbalance_ratio). Factored into `fold_monitor_into_bucket`
935        // so a synthesized zero-capture bucket (see
936        // `build_phase_buckets_with_stimulus`) recovers its in-window
937        // imbalance too — same formula and frame, but over the
938        // synthesized bucket's FULL stimulus window vs this captured
939        // bucket's narrower interior capture-offset span.
940        fold_monitor_into_bucket(
941            &mut bucket,
942            monitor_samples,
943            monitor_to_window_offset_ms,
944            preemption_threshold_ns,
945        );
946        // Guest-CPU denominators for IRQ activity/fraction rates. Select the
947        // exact per_cpu_time endpoints by the same phase-window key used by the
948        // IRQ counters, then sum non-steal cpustat deltas across their CPU-id
949        // intersection. Host preemption therefore cannot dilute these metrics.
950        if bucket.metrics.contains_key("total_hardirqs") {
951            let placed: Vec<(&crate::scenario::sample::Sample<'_>, u64)> = samples_in_phase
952                .iter()
953                .filter(|s| !s.snapshot.per_cpu_time().is_empty())
954                .filter_map(|s| win(s).map(|w| (s, w)))
955                .collect();
956            if let (Some(&(first, first_win)), Some(&(last, last_win))) = (
957                placed.iter().min_by_key(|(_, w)| *w),
958                placed.iter().max_by_key(|(_, w)| *w),
959            ) && first_win < last_win
960                && let Some(cpu_ns) = guest_cpu_time_delta_ns(
961                    first.snapshot.per_cpu_time(),
962                    last.snapshot.per_cpu_time(),
963                )
964            {
965                bucket
966                    .metrics
967                    .entry("total_phase_guest_cpu_ns".to_string())
968                    .or_insert(cpu_ns as f64);
969                bucket
970                    .metrics
971                    .entry("total_phase_guest_cpu_sec".to_string())
972                    .or_insert(cpu_ns as f64 / 1e9);
973            }
974        }
975        // Derive Rate metrics AFTER every component source is folded in:
976        // the METRICS reductions + system/user_time_ns above AND the
977        // monitor-injected components (e.g. avg_imbalance_ratio, whose ONLY
978        // source is fold_monitor_into_bucket). Deriving before the monitor
979        // fold would silently drop a Rate over a monitor-only component. A
980        // Rate has no samples of its own (see MetricKind::Rate); this is its
981        // per-phase producer. Inert until a Rate registers. The stimulus
982        // sibling build_phase_buckets_with_stimulus re-derives again after
983        // its own later injections (idempotent).
984        crate::stats::derive_rate_metrics(&mut bucket.metrics);
985        out.push(bucket);
986    }
987    out
988}
989
990/// Fold the per-CPU full-class imbalance from the monitor samples whose
991/// run-relative timestamp falls in `bucket`'s `[start_ms, end_ms)`
992/// window into `bucket.metrics["avg_imbalance_ratio"]`.
993///
994/// The monitor sample's run-relative `elapsed_ms` is shifted into the
995/// bucket-window frame (subtract `monitor_to_window_offset_ms`) before
996/// the half-open `[start_ms, end_ms)` test so a MonitorSample whose
997/// timestamp equals the boundary lands in exactly one bucket (not both
998/// adjacent buckets — the closed-on-right form double-counted boundary
999/// samples). Single-sample phases (`start_ms == end_ms`) use explicit
1000/// equality so the window is not empty.
1001///
1002/// Filters via [`crate::monitor::sample_looks_valid`] (implausible-DSQ
1003/// samples) before the fold; `compute_metrics` additionally drops
1004/// empty-cpus samples (which would default `imbalance_ratio` to 1.0 and
1005/// pull the mean toward "perfect balance", masking a real regression) —
1006/// matching the legacy `Timeline::build` path's filter discipline.
1007///
1008/// Folds the FULL monitor-derived metric set the legacy `Timeline::build`
1009/// reducer (`crate::timeline::compute_metrics`) produces —
1010/// `avg_imbalance_ratio`, `max_imbalance_ratio`, `avg_dsq_depth`,
1011/// `max_dsq_depth`, `stuck_count`, and the `total_fallback` /
1012/// `total_keep_last` counter deltas — over the bucket's window.
1013/// `avg_imbalance_ratio`, `max_imbalance_ratio`, and `stuck_count` are
1014/// folded for EVERY bucket with in-window monitor samples: none of the three
1015/// has a `read_sample` dispatch arm (`crate::stats` `read_sample` has arms
1016/// only for the dsq / fallback keys; these three fall to `_ => None`), so the
1017/// per-sample capture path never produces them and monitor is their only
1018/// per-bucket source on captured AND synthesized buckets alike — a captured
1019/// (common-case) phase must report its per-phase imbalance peak and stall
1020/// count, not drop them. (`avg_imbalance_ratio` is genuinely ext-metrics-only;
1021/// `max_imbalance_ratio` and `stuck_count` ALSO carry a typed `GauntletRow`
1022/// accessor sourced from the whole-run MonitorSummary, so their per-phase
1023/// fold here feeds per-phase RENDERING only — the run-level value stays the
1024/// typed accessor, and both `populate_run_ext_metrics*` skip them via
1025/// `TYPED_FIELD_NAMES` to avoid a double-source.) The dsq / fallback set
1026/// (`avg_dsq_depth`,
1027/// `max_dsq_depth`, `total_fallback`, `total_keep_last`) is folded ONLY for
1028/// a synthesized (`sample_count == 0`) bucket: a captured bucket sources
1029/// those from its read_sample captures and keeps its pre-synthesize
1030/// behavior, while a synthesized bucket has no captures, so monitor is its
1031/// only source — restoring the rendered timeline to PARITY with the old
1032/// `Timeline::build` fallback (the path a zero-capture-with-monitor run
1033/// took before the synthesize seam flipped it onto from_phase_buckets;
1034/// `format_phases` renders these folded metrics for a `sample_count == 0`
1035/// bucket via its `has_monitor_metrics` gate). Each key is `or_insert` so
1036/// it never overwrites a value already present. Parity with
1037/// `Timeline::build` is exact for the production case; for legacy
1038/// ScenarioEnd-but-no-StepEnd data the synthesized last-step window clamps
1039/// to the terminal rather than extending to end-of-monitor, and a
1040/// synthesized bucket's dsq metrics come from the monitor
1041/// `CpuSnapshot.local_dsq_depth` axis (vs a captured bucket's DSQ-walker
1042/// axis) — same metric, different sampling axis. Event rates are folded for
1043/// every monitor-bearing bucket from endpoint-matched event counters and vCPU
1044/// CPU clocks; they never fall back to bucket wall duration.
1045///
1046/// `bucket`'s `[start_ms, end_ms)` IS the window basis and differs by
1047/// bucket kind: a captured bucket's is the min/max of its samples'
1048/// interior capture offsets; a synthesized bucket's is its full
1049/// `[StepStart, StepEnd)` stimulus window. The monitor sample's
1050/// run-relative `elapsed_ms` is shifted into that frame (subtract
1051/// `monitor_to_window_offset_ms`) before the half-open test so a sample
1052/// on the boundary lands in exactly one bucket. Event-rate components use the
1053/// same `counter_delta` clamp as the run-level monitor summary.
1054fn fold_monitor_into_bucket(
1055    bucket: &mut PhaseBucket,
1056    monitor_samples: &[crate::monitor::MonitorSample],
1057    monitor_to_window_offset_ms: i64,
1058    preemption_threshold_ns: u64,
1059) {
1060    let start_ms = bucket.start_ms;
1061    let end_ms = bucket.end_ms;
1062    let in_window = |monitor_ms: u64| -> bool {
1063        let shifted = monitor_ms as i64 - monitor_to_window_offset_ms;
1064        if shifted < 0 {
1065            return false;
1066        }
1067        let m = shifted as u64;
1068        if start_ms == end_ms {
1069            m == start_ms
1070        } else {
1071            m >= start_ms && m < end_ms
1072        }
1073    };
1074    // Filter via sample_looks_valid (matches Timeline::build) so an invalid
1075    // sample (empty cpus -> imbalance_ratio 1.0 default) doesn't pull the
1076    // mean toward "perfect balance" and mask a real regression.
1077    let phase_monitor_samples: Vec<&crate::monitor::MonitorSample> = monitor_samples
1078        .iter()
1079        .filter(|s| in_window(s.elapsed_ms))
1080        .filter(|s| crate::monitor::sample_looks_valid(s))
1081        .collect();
1082    if phase_monitor_samples.is_empty() {
1083        return;
1084    }
1085    let pm = crate::timeline::compute_metrics(&phase_monitor_samples, preemption_threshold_ns);
1086    // sample_count == 0 IFF the bucket is synthesized: buckets_from_grouped
1087    // only emits >=1-sample buckets, the synthesize loop only emits 0-sample
1088    // ones. A CAPTURED bucket folds the source-less monitor signals (avg/max
1089    // imbalance + stuck_count — none have a read_sample arm) but NOT the
1090    // dsq / fallback set, which it sources from its read_sample captures.
1091    // Restricting only that dsq / fallback set to synthesized buckets
1092    // preserves captured-bucket behavior; a synthesized bucket has no
1093    // captures, so monitor is its only source and it takes the full set
1094    // (Timeline::build render parity).
1095    let synthesized = bucket.sample_count == 0;
1096    {
1097        let mut put = |key: &str, v: f64| {
1098            if v.is_finite() {
1099                bucket.metrics.entry(key.to_string()).or_insert(v);
1100            }
1101        };
1102        if let Some(v) = pm.avg_imbalance {
1103            put("avg_imbalance_ratio", v);
1104        }
1105        // avg_nr_running (Gauge(Avg)) is a monitor-axis signal (full-class
1106        // rq.nr_running, no read_sample dispatch arm), so like avg_imbalance_ratio
1107        // it is folded for EVERY monitor-bearing bucket (not gated on synthesized)
1108        // — captured buckets have no other source for it. It feeds per-phase
1109        // rendering + boundary change-detection only; the run-level value stays
1110        // MonitorSummary::avg_nr_running (fold_run_level_ext), so the run-level ext
1111        // re-pool (populate_run_ext_metrics_from_phases) SKIPS this key to avoid a
1112        // double-source.
1113        if let Some(v) = pm.avg_nr_running {
1114            put("avg_nr_running", v);
1115        }
1116        // max_imbalance_ratio (Peak) and stuck_count (Counter) have NO read_sample
1117        // dispatch arm (both fall to `_ => None` in crate::stats read_sample), so
1118        // the per-sample capture path never produces them and a CAPTURED bucket
1119        // would otherwise never carry them — they would surface only on synthesized
1120        // (zero-capture) buckets. Fold them for EVERY monitor-bearing bucket — like
1121        // avg_imbalance_ratio above — so a captured (common-case) phase reports its
1122        // per-phase imbalance peak and stuck count instead of dropping them. (Both
1123        // are monitor-axis signals: imbalance from full-class rq.nr_running, stuck
1124        // from non-advancing rq.clock across consecutive samples — neither is in
1125        // the guest Snapshot read_sample observes. Both ALSO carry a typed
1126        // run-level GauntletRow accessor, so this per-phase fold feeds per-phase
1127        // RENDERING only; TYPED_FIELD_NAMES keeps them out of the run-level
1128        // ext_metrics so the typed accessor stays the single run-level source.)
1129        // `or_insert` still guards against overwriting a value already present.
1130        if let Some(v) = pm.max_imbalance {
1131            put("max_imbalance_ratio", v);
1132        }
1133        if pm.stuck_count > 0 {
1134            put("stuck_count", pm.stuck_count as f64);
1135        }
1136        if synthesized {
1137            if let Some(v) = pm.avg_dsq_depth {
1138                put("avg_dsq_depth", v);
1139            }
1140            put("max_dsq_depth", pm.max_dsq_depth as f64);
1141        }
1142        // Canonical event activity rates for both captured and synthesized
1143        // buckets. Numerators and denominator share the same monitor endpoints.
1144        let has_events = |s: &&crate::monitor::MonitorSample| s.has_complete_event_counters();
1145        let first_ev = phase_monitor_samples.iter().copied().find(has_events);
1146        let last_ev = phase_monitor_samples.iter().copied().rev().find(has_events);
1147        if let (Some(first), Some(last)) = (first_ev, last_ev)
1148            && let Some(vcpu_sec) =
1149                crate::monitor::MonitorSummary::mean_vcpu_cpu_delta_secs(first, last)
1150        {
1151            let fb = crate::monitor::counter_delta(
1152                last.sum_event_field(|e| e.select_cpu_fallback).unwrap_or(0),
1153                first
1154                    .sum_event_field(|e| e.select_cpu_fallback)
1155                    .unwrap_or(0),
1156            );
1157            let kl = crate::monitor::counter_delta(
1158                last.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1159                first.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1160            );
1161            put("total_fallback_pooled", fb as f64);
1162            put("total_keep_last_pooled", kl as f64);
1163            put("total_event_vcpu_sec", vcpu_sec);
1164        }
1165    }
1166    crate::stats::derive_rate_metrics(&mut bucket.metrics);
1167}
1168
1169/// Clock skew (ms) between the host monitor's run-relative timeline and
1170/// the guest's scenario-relative stimulus timeline, computed the same
1171/// way as [`crate::timeline::Timeline::build`]: the first significant
1172/// monitor sample (elapsed > 500 ms, non-empty cpus) and the earliest
1173/// stimulus event roughly coincide at scenario start. Returns
1174/// `first_monitor_ms - first_stimulus_ms`; subtract from a monitor
1175/// sample's elapsed_ms to reach the scenario-relative (boundary-offset)
1176/// window frame. `0` when either timeline is empty (nothing to align,
1177/// so the run-relative frames are used as-is).
1178fn monitor_clock_offset(
1179    stimulus_events: &[crate::timeline::StimulusEvent],
1180    monitor_samples: &[crate::monitor::MonitorSample],
1181) -> i64 {
1182    if stimulus_events.is_empty() || monitor_samples.is_empty() {
1183        return 0;
1184    }
1185    let first_stimulus_ms = stimulus_events
1186        .iter()
1187        .map(|e| e.elapsed_ms)
1188        .min()
1189        .unwrap_or(0);
1190    let first_monitor_ms = monitor_samples
1191        .iter()
1192        .find(|s| s.elapsed_ms > 500 && !s.cpus.is_empty())
1193        .map(|s| s.elapsed_ms)
1194        .unwrap_or_else(|| monitor_samples.first().map(|s| s.elapsed_ms).unwrap_or(0));
1195    first_monitor_ms as i64 - first_stimulus_ms as i64
1196}