ktstr/
timeline.rs

1//! Stimulus/phase correlation for scenario execution.
2//!
3//! Correlates [`StimulusEvent`]s (cgroup operations, cpuset changes)
4//! with `MonitorSample` windows to
5//! measure per-phase scheduler behavior degradation. Produces
6//! [`Timeline`] entries consumed by the stats and reporting pipeline.
7
8use std::fmt;
9
10use crate::monitor::{MonitorSample, sample_looks_valid};
11
12// ---------------------------------------------------------------------------
13// TimelineContext — system context rendered as a header
14// ---------------------------------------------------------------------------
15
16/// System context for a timeline, rendered as a header block.
17#[derive(Debug, Clone, Default)]
18pub struct TimelineContext {
19    /// Kernel version string (e.g. "6.14.0-rc3+").
20    pub kernel: Option<String>,
21    /// Topology description (e.g. "2n4l4c2t (16 cpus)").
22    pub topology: Option<String>,
23    /// Scheduler name (e.g. "scx_mitosis").
24    pub scheduler: Option<String>,
25    /// Scenario name.
26    pub scenario: Option<String>,
27    /// Total run duration in seconds.
28    pub duration_s: Option<f64>,
29}
30
31// ---------------------------------------------------------------------------
32// StimulusEvent — what happened and when
33// ---------------------------------------------------------------------------
34
35/// A discrete event during scenario execution that may cause observable
36/// changes in scheduler behavior. Generated by step executors on the guest
37/// side and carried in the VM output alongside monitor samples.
38#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
39pub struct StimulusEvent {
40    /// Milliseconds since scenario start (guest monotonic clock).
41    pub elapsed_ms: u64,
42    /// Human-readable label. Produced as `"StepStart[k]"` by
43    /// [`Self::from_wire`] (the 0-indexed scenario Step ordinal),
44    /// `"ScenarioEnd"` by [`Self::terminal`], and the
45    /// `"BASELINE"`/`"Step[k]"` bucket label by the
46    /// `phase_from_bucket` placeholder. Test fixtures may carry any
47    /// label.
48    pub label: String,
49    /// What kind of operation triggered this event.
50    pub op_kind: Option<String>,
51    /// Additional context (e.g. "4 cpus", "cgroup=cg_0").
52    pub detail: Option<String>,
53    /// Cumulative worker iterations at this event. `Some(_)` for every
54    /// event built from the wire (the wire counter is always present —
55    /// see [`Self::from_wire`]); `Some(0)` is a legitimate "no iterations
56    /// accumulated yet" value. `None` is reserved for synthetic or
57    /// placeholder events.
58    ///
59    /// This raw boundary telemetry is not divided by `elapsed_ms`.
60    /// `iteration_rate` has one definition throughout ktstr:
61    /// iterations per delivered guest CPU-second from matching per-cgroup
62    /// phase carriers. Deriving another rate here would reintroduce a
63    /// host-contention-sensitive wall-throughput metric.
64    pub total_iterations: Option<u64>,
65    /// 1-indexed scenario step this event belongs to (the same
66    /// encoding the bridge stamps: `1..=N` for Step ordinals), or
67    /// `None` for non-step events (including the terminal scenario-end
68    /// boundary; see `is_terminal`). Carried explicitly from the wire
69    /// `StimulusPayload.step_index` so the periodic-capture phase
70    /// attribution can map a capture's workload-relative boundary
71    /// offset onto the guest's own step timeline without parsing the
72    /// human-readable `label`.
73    pub step_index: Option<u16>,
74    /// True only for the synthetic scenario-end boundary the eval
75    /// walker appends from the `ScenarioEnd` wire frame's final
76    /// `total_iterations`. It is retained for raw timeline completeness and
77    /// legacy consumers, but is not a denominator for `iteration_rate`. It is
78    /// not a step start:
79    /// `step_index` is `None` so it seeds no [`crate::assert::PhaseBucket`]
80    /// (excluded from the step-start timeline), and [`Timeline::build`]
81    /// skips it when laying out phase boundaries so it never renders a
82    /// phantom trailing phase.
83    pub is_terminal: bool,
84    /// True for a per-step END event (decoded from a
85    /// `crate::vmm::wire::MsgType::StepEnd` frame via
86    /// [`Self::from_step_end`]). It carries the SAME 1-indexed
87    /// `step_index` as its StepStart and its step's end-of-hold
88    /// `total_iterations`. It is not a step start, so [`Timeline::build`]
89    /// filters it out of its step-start list to avoid a phantom phase.
90    pub is_step_end: bool,
91}
92
93impl StimulusEvent {
94    /// Build a timeline event from a deserialized wire stimulus event.
95    /// Centralizes the wire→timeline mapping so the production eval path
96    /// (`evaluate_vm_result`) and out-of-tree consumers — post_vm
97    /// callbacks folding `VmResult::stimulus_timeline()` (which calls
98    /// this internally) through
99    /// [`crate::assert::build_phase_buckets_with_stimulus`] — produce
100    /// identical events. The wire `step_index` is the bridge 1-indexed
101    /// convention (`Step[k]` -> `k + 1`, BASELINE owns 0); the human
102    /// `label` renders the 0-indexed Scenario-Step ordinal
103    /// (`step_index - 1`) to match the `PhaseBucket` `Step[k]` labels,
104    /// while the `step_index` field keeps the 1-indexed wire value for
105    /// phase-bucket remap. `total_iterations` is carried verbatim as
106    /// `Some(_)`: the wire field is a cumulative counter that is always
107    /// populated (the guest sums live worker iterations at every step
108    /// boundary), so `0` is a legitimate baseline reading — the first
109    /// step's frame fires right after its workers spawn and may genuinely
110    /// read zero.
111    pub fn from_wire(ev: &crate::vmm::wire::StimulusEvent) -> Self {
112        Self {
113            elapsed_ms: ev.elapsed_ms as u64,
114            label: format!("StepStart[{}]", ev.step_index.saturating_sub(1)),
115            op_kind: Some(format!("ops={}", ev.op_count)),
116            detail: Some(format!(
117                "{} cgroups, {} workers",
118                ev.cgroup_count, ev.worker_count,
119            )),
120            total_iterations: Some(ev.total_iterations),
121            step_index: Some(ev.step_index),
122            is_terminal: false,
123            is_step_end: false,
124        }
125    }
126
127    /// Build a per-step END event from a `crate::vmm::wire::MsgType::StepEnd`
128    /// frame (reuses the `crate::vmm::wire::StimulusEvent` wire body).
129    /// Carries the SAME 1-indexed `step_index` as the step's StepStart
130    /// and the step's end-of-hold `total_iterations`, with `is_step_end`
131    /// set. `is_terminal` is false (it is a real per-step boundary, not the
132    /// scenario-end terminal).
133    pub fn from_step_end(ev: &crate::vmm::wire::StimulusEvent) -> Self {
134        Self {
135            elapsed_ms: ev.elapsed_ms as u64,
136            label: format!("StepEnd[{}]", ev.step_index.saturating_sub(1)),
137            op_kind: Some(format!("ops={}", ev.op_count)),
138            detail: Some(format!(
139                "{} cgroups, {} workers",
140                ev.cgroup_count, ev.worker_count,
141            )),
142            total_iterations: Some(ev.total_iterations),
143            step_index: Some(ev.step_index),
144            is_terminal: false,
145            is_step_end: true,
146        }
147    }
148
149    /// Build the synthetic terminal boundary event from the
150    /// `ScenarioEnd` wire frame's final cumulative `total_iterations`
151    /// and scenario-relative `elapsed_ms`. Appended once, after every
152    /// per-step [`Self::from_wire`] event.
153    /// `step_index` is `None` (it is not a step start — it seeds no
154    /// [`crate::assert::PhaseBucket`]) and `is_terminal` is set so
155    /// [`Timeline::build`] treats it as a right boundary only, never a
156    /// phase. `elapsed_ms` is in the same guest-monotonic frame as the
157    /// step events (both come from `scenario_start.elapsed()`), so the
158    /// last-step duration is well-formed.
159    pub fn terminal(elapsed_ms: u64, total_iterations: u64) -> Self {
160        Self {
161            elapsed_ms,
162            label: "ScenarioEnd".to_string(),
163            op_kind: None,
164            detail: None,
165            total_iterations: Some(total_iterations),
166            step_index: None,
167            is_terminal: true,
168            is_step_end: false,
169        }
170    }
171
172    /// The scenario [`Phase`](crate::assert::Phase) this event belongs to,
173    /// or `None` for the terminal scenario-end boundary (which seeds no
174    /// phase). Use THIS — not the raw [`Self::step_index`] field — to key
175    /// per-phase lookups. `step_index` carries the bridge 1-indexed wire
176    /// convention (`Step k` -> `Some(k + 1)`) while `label` renders the
177    /// 0-indexed `k`, so reading the field directly invites the 0-vs-1
178    /// off-by-one this method removes: it maps the wire value onto the same
179    /// [`Phase`](crate::assert::Phase) newtype the
180    /// [`ScenarioStats`](crate::assert::ScenarioStats) /
181    /// [`PhaseBucket`](crate::assert::PhaseBucket) accessors are keyed by
182    /// (`Phase::step(k)`). Step events carry `step_index >= 1`, so the
183    /// `saturating_sub(1)` is exact.
184    pub fn phase(&self) -> Option<crate::assert::Phase> {
185        self.step_index
186            .map(|si| crate::assert::Phase::step(si.saturating_sub(1)))
187    }
188}
189
190// ---------------------------------------------------------------------------
191// Phase — a time window between consecutive stimulus events
192// ---------------------------------------------------------------------------
193
194/// Metrics aggregated from monitor samples within a phase.
195#[derive(Debug, Clone, Default)]
196pub struct PhaseMetrics {
197    pub sample_count: usize,
198    /// Mean CPU-imbalance ratio over the phase's valid samples. `None`
199    /// when the phase had no valid samples (monitor-only `Timeline::build`)
200    /// or its source bucket carried no `avg_imbalance_ratio` metric
201    /// (snapshot `from_phase_buckets`) — distinct from a real `Some(0.0)`
202    /// (perfectly balanced). The change detector compares it only when
203    /// both sides are `Some`, so an absent phase never reads as a false
204    /// zero-imbalance.
205    pub avg_imbalance: Option<f64>,
206    /// Peak CPU-imbalance ratio over the phase's valid samples. `None` on
207    /// the same no-data conditions as [`Self::avg_imbalance`].
208    pub max_imbalance: Option<f64>,
209    /// Mean local-DSQ depth over the phase's valid samples. `None` on the
210    /// same no-data conditions as [`Self::avg_imbalance`].
211    pub avg_dsq_depth: Option<f64>,
212    /// Mean runqueue occupancy (full-class `rq.nr_running`) over the phase's
213    /// valid samples — per-sample mean of the per-CPU `nr_running`, averaged
214    /// over samples (the `avg_dsq_depth` shape). `None` on the same no-data
215    /// conditions as [`Self::avg_imbalance`]. The run-level value stays
216    /// `MonitorSummary::avg_nr_running` (folded by `fold_run_level_ext`); this
217    /// per-phase field feeds rendering + boundary change-detection only and is
218    /// kept out of the run-level ext re-pool (see `populate_run_ext_metrics_from_phases`).
219    pub avg_nr_running: Option<f64>,
220    pub max_dsq_depth: u32,
221    pub stuck_count: usize,
222    /// select_cpu_fallback events per mean delivered vCPU CPU-second.
223    /// `None` when event counters or complete endpoint CPU clocks are unavailable.
224    pub fallback_rate: Option<f64>,
225    /// dispatch_keep_last events per mean delivered vCPU CPU-second.
226    /// `None` under the same conditions as [`Self::fallback_rate`].
227    pub keep_last_rate: Option<f64>,
228    /// Worker iterations per delivered guest CPU-second during this phase.
229    /// Derived from matching guest per-cgroup iteration and CPU-time carriers.
230    pub iteration_rate: Option<f64>,
231}
232
233/// Direction of change at a phase boundary.
234#[derive(Debug, Clone, Copy, PartialEq, Eq)]
235pub enum ChangeDirection {
236    Improved,
237    Degraded,
238}
239
240impl fmt::Display for ChangeDirection {
241    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
242        match self {
243            ChangeDirection::Improved => write!(f, "IMPROVEMENT"),
244            ChangeDirection::Degraded => write!(f, "DEGRADATION"),
245        }
246    }
247}
248
249/// Detected change at a stimulus boundary.
250#[derive(Debug, Clone)]
251pub struct PhaseChange {
252    pub direction: ChangeDirection,
253    pub metric: String,
254    pub before: f64,
255    pub after: f64,
256}
257
258/// A time window between two consecutive stimulus events.
259#[derive(Debug, Clone)]
260pub struct Phase {
261    pub index: usize,
262    pub start_ms: u64,
263    pub end_ms: u64,
264    /// The stimulus event that starts this phase (None for the initial phase).
265    pub stimulus: Option<StimulusEvent>,
266    pub metrics: PhaseMetrics,
267    /// Changes detected at this phase's stimulus boundary.
268    pub changes: Vec<PhaseChange>,
269    /// Per-cgroup raw telemetry for this phase, keyed by cgroup name. Carried
270    /// from [`crate::assert::PhaseBucket::per_cgroup`] on the
271    /// [`Self`]-via-`from_phase_buckets` path; empty on the monitor-only
272    /// [`Timeline::build`] path (which has no carriers). Rendered as a
273    /// per-cgroup sub-block by display-time reduction
274    /// ([`crate::assert::PhaseCgroupStats::off_cpu_summary`] etc.); never a
275    /// change-detection input (those are [`PhaseMetrics`] scalars only).
276    pub per_cgroup: std::collections::BTreeMap<String, crate::assert::PhaseCgroupStats>,
277    /// True when `(start_ms, end_ms)` is the normalized `(0, 0)` of an ORPHAN
278    /// carrier (no measured host window) rather than a real window. Set in
279    /// `phase_from_bucket` by the orphan shape signature `(0,0)` + empty
280    /// `metrics` + non-empty `per_cgroup` (unique to
281    /// `crate::assert::fold_guest_per_cgroup_into_host_buckets`'s orphan arm).
282    /// The render shows "window not measured" instead of a misleading `0ms`.
283    /// Always `false` on the [`Timeline::build`] path.
284    pub not_measured_window: bool,
285}
286
287// ---------------------------------------------------------------------------
288// Timeline
289// ---------------------------------------------------------------------------
290
291/// Correlated timeline of stimulus events and monitor observations.
292#[derive(Debug, Clone)]
293pub struct Timeline {
294    pub phases: Vec<Phase>,
295}
296
297/// Minimum delta in imbalance ratio to flag a change (avoids noise).
298const IMBALANCE_THRESHOLD: f64 = 0.5;
299/// Minimum delta in DSQ depth to flag a change.
300const DSQ_THRESHOLD: f64 = 3.0;
301/// Minimum delta in mean runqueue depth (avg_nr_running) to flag a change:
302/// one additional runnable task per CPU on average between phases.
303const NR_RUNNING_THRESHOLD: f64 = 1.0;
304/// Minimum delta in fallback rate (events/vCPU-second) to flag a change.
305const FALLBACK_RATE_THRESHOLD: f64 = 10.0;
306/// Minimum delta in keep_last rate (events/vCPU-second) to flag a change.
307const KEEP_LAST_RATE_THRESHOLD: f64 = 10.0;
308/// Minimum relative change in iteration rate to flag a throughput change.
309/// 0.3 = 30% drop or increase.
310const ITERATION_RATE_REL_THRESHOLD: f64 = 0.3;
311
312/// Create a PhaseChange if the delta between `before` and `after` exceeds
313/// `threshold`. `higher_is_worse` determines degradation direction: when
314/// true, a positive delta means Degraded; when false, a negative delta
315/// means Degraded.
316fn detect_change(
317    before: f64,
318    after: f64,
319    threshold: f64,
320    metric: &str,
321    higher_is_worse: bool,
322) -> Option<PhaseChange> {
323    let delta = after - before;
324    if delta.abs() <= threshold {
325        return None;
326    }
327    let degraded = if higher_is_worse {
328        delta > 0.0
329    } else {
330        delta < 0.0
331    };
332    Some(PhaseChange {
333        direction: if degraded {
334            ChangeDirection::Degraded
335        } else {
336            ChangeDirection::Improved
337        },
338        metric: metric.to_string(),
339        before,
340        after,
341    })
342}
343
344/// Per-boundary change set between two adjacent phases, shared by both
345/// [`Timeline::build`] and [`Timeline::from_phase_buckets`].
346///
347/// Iteration efficiency (`iteration_rate`) is compared whenever BOTH phases
348/// carry a guest-carrier-derived CPU rate and the earlier one is positive —
349/// INCLUDING a synthesized zero-capture step. It survives a capture gap because
350/// its iteration and delivered-CPU components do not depend on periodic
351/// captures; gating it on `sample_count` would silently drop a real phase
352/// transition.
353///
354/// Asymmetry from the `bi > 0.0` div-by-zero guard: a COLLAPSE into a
355/// zero-rate (incl. a synthesized measured-zero) step is flagged
356/// (before > 0), but a RECOVERY out of one (before == 0 -> positive) is
357/// not — there is no relative baseline to divide by. This is a
358/// deliberate tradeoff, not an oversight: an unchanged zero is genuinely
359/// not a degradation, and the collapse direction (the one this task
360/// targets) is caught.
361///
362/// The monitor-derived metrics (imbalance, dsq depth, fallback rate,
363/// keep_last rate) ARE gated on both phases having real samples. The
364/// values themselves can be real even on a synthesized phase (monitor
365/// windows fold into the bucket), but they come from a DIFFERENT
366/// sampling basis (folded monitor window vs captured periodic samples)
367/// than a captured neighbor's, so cross-comparing them is
368/// apples-to-oranges — suppressed. Iteration efficiency is exempt because it
369/// uses the same guest-carrier currency on both sides.
370fn detect_boundary_changes(before: &PhaseMetrics, after: &PhaseMetrics) -> Vec<PhaseChange> {
371    let mut changes = Vec::new();
372
373    // Monitor-derived metrics (absolute-unit gauges/rates with
374    // fixed-magnitude thresholds) — gated on both phases having real
375    // samples. Pushed FIRST to preserve the historical render order
376    // (these before throughput in a multi-change boundary; format_phases
377    // renders changes in vec order).
378    if before.sample_count > 0 && after.sample_count > 0 {
379        if let (Some(bi), Some(ai)) = (before.avg_imbalance, after.avg_imbalance) {
380            changes.extend(detect_change(
381                bi,
382                ai,
383                IMBALANCE_THRESHOLD,
384                "imbalance",
385                true,
386            ));
387        }
388        if let (Some(bd), Some(ad)) = (before.avg_dsq_depth, after.avg_dsq_depth) {
389            changes.extend(detect_change(bd, ad, DSQ_THRESHOLD, "dsq_depth", true));
390        }
391        if let (Some(bn), Some(an)) = (before.avg_nr_running, after.avg_nr_running) {
392            changes.extend(detect_change(
393                bn,
394                an,
395                NR_RUNNING_THRESHOLD,
396                "nr_running",
397                true,
398            ));
399        }
400        if let (Some(bf), Some(af)) = (before.fallback_rate, after.fallback_rate) {
401            changes.extend(detect_change(
402                bf,
403                af,
404                FALLBACK_RATE_THRESHOLD,
405                "fallback",
406                true,
407            ));
408        }
409        if let (Some(bk), Some(ak)) = (before.keep_last_rate, after.keep_last_rate) {
410            changes.extend(detect_change(
411                bk,
412                ak,
413                KEEP_LAST_RATE_THRESHOLD,
414                "keep_last",
415                true,
416            ));
417        }
418    }
419
420    // Throughput is the SOLE rate-class field (counter-delta / elapsed), so
421    // it uses a RELATIVE threshold (rel = (ai-bi)/bi vs
422    // ITERATION_RATE_REL_THRESHOLD) and cannot route through detect_change's
423    // absolute-delta gate above — fixed-unit metrics get absolute
424    // thresholds, this gets a relative one (a semantic-class decision, not
425    // arbitrary). Strict `>`: an exactly-at-threshold relative change is not
426    // flagged. Pushed last so it renders after the monitor metrics.
427    if let (Some(bi), Some(ai)) = (before.iteration_rate, after.iteration_rate)
428        && bi > 0.0
429    {
430        let rel = (ai - bi) / bi;
431        if rel.abs() > ITERATION_RATE_REL_THRESHOLD {
432            changes.push(PhaseChange {
433                direction: if rel < 0.0 {
434                    ChangeDirection::Degraded
435                } else {
436                    ChangeDirection::Improved
437                },
438                metric: "throughput".to_string(),
439                before: bi,
440                after: ai,
441            });
442        }
443    }
444
445    changes
446}
447
448impl Timeline {
449    /// Build a timeline from stimulus events and monitor samples.
450    ///
451    /// Clock alignment: stimulus events use guest monotonic time (ms since
452    /// scenario start). Monitor samples use host monotonic time (ms since
453    /// VM boot). The first stimulus event's timestamp and the first
454    /// non-trivial monitor sample (after 500ms warmup) approximately
455    /// coincide. We compute an offset to align them.
456    ///
457    /// Returns an empty timeline if either input is empty.
458    /// Build a Timeline from stimulus events + raw monitor
459    /// samples via the per-window `compute_metrics` reduction.
460    /// The production success path uses [`Self::from_phase_buckets`]
461    /// (which folds pre-bucketed PhaseBuckets); `build` is the fallback
462    /// evaluate_vm_result takes only for a run with an EMPTY PhaseBuckets
463    /// vec but monitor samples present — i.e. no periodic captures AND no
464    /// stimulus Steps. A monitor-only run that DID run Steps now
465    /// synthesizes a capture-free bucket per StepStart (see
466    /// [`crate::assert::build_phase_buckets_with_stimulus`]), so its vec is
467    /// non-empty and it takes the from_phase_buckets path (whose
468    /// fold_monitor_into_bucket recovers the same monitor-derived metric
469    /// set this path computes). Both entry points produce the same
470    /// Timeline field shape; from_phase_buckets is preferred when buckets
471    /// are available because it avoids the per-MonitorSample reduction.
472    ///
473    /// `preemption_threshold_ns` threads the vCPU-preemption exemption
474    /// window into the per-phase stall predicate (see `compute_metrics`);
475    /// the production caller passes the run's
476    /// `MonitorReport::preemption_threshold_ns`. `0` derives it from the
477    /// guest kernel `CONFIG_HZ`.
478    pub fn build(
479        stimulus_events: &[StimulusEvent],
480        monitor_samples: &[MonitorSample],
481        preemption_threshold_ns: u64,
482    ) -> Self {
483        if stimulus_events.is_empty() || monitor_samples.is_empty() {
484            return Self { phases: Vec::new() };
485        }
486
487        let mut events = stimulus_events.to_vec();
488        // Total-order on an elapsed_ms tie: StepEnd before StepStart
489        // (`!is_step_end` is false=0 for StepEnd) so a zero-length
490        // inter-step gap at the guest's coarse-ms clock attributes the
491        // step-local StepStart[k]->StepEnd[k] rate to bucket k, never the
492        // cross-step StepStart[k]->StepStart[k+1] delta. Mirrors the same
493        // sort in build_phase_buckets_with_stimulus so the two rate
494        // consumers stay identical.
495        events.sort_by_key(|e| (e.elapsed_ms, !e.is_step_end));
496
497        // Clock alignment: find the offset between guest stimulus time
498        // and host monitor time. The first stimulus event (ScenarioStart)
499        // and the first monitor sample with plausible data roughly coincide.
500        let first_stimulus_ms = events[0].elapsed_ms;
501        let first_monitor_ms = monitor_samples
502            .iter()
503            .find(|s| s.elapsed_ms > 500 && !s.cpus.is_empty())
504            .map(|s| s.elapsed_ms)
505            .unwrap_or_else(|| monitor_samples.first().map(|s| s.elapsed_ms).unwrap_or(0));
506
507        // offset: add this to a stimulus timestamp to get monitor time
508        let offset = first_monitor_ms as i64 - first_stimulus_ms as i64;
509
510        // Define phase boundaries from consecutive stimulus events.
511        // Each pair (events[i], events[i+1]) bounds a phase.
512        // The last event to end-of-data is also a phase.
513        let last_monitor_ms = monitor_samples.last().map(|s| s.elapsed_ms).unwrap_or(0);
514
515        // StepStart events only — the PHASE-LAYOUT set. Per-step StepEnd
516        // events are excluded here because a StepEnd seeds no new phase
517        // (it is an end-of-hold marker, not a step boundary); including
518        // them would produce a phantom extra phase and misalign the dense
519        // phase index. StepEnd events are NOT discarded, though: the
520        // step-local iteration_rate loop below pairs each StepStart[k]
521        // with its own StepEnd[k] (looked up by step_index in the full
522        // `events` vec), matching build_phase_buckets_with_stimulus. The
523        // dense-index cross-step pairing is kept only as a fallback for
524        // steps that have no StepEnd (a sched-died step, or legacy data
525        // predating the StepEnd frame).
526        let step_events: Vec<&StimulusEvent> = events
527            .iter()
528            .filter(|e| !e.is_terminal && !e.is_step_end)
529            .collect();
530
531        let mut boundaries: Vec<(u64, u64, Option<StimulusEvent>)> = Vec::new();
532        for i in 0..step_events.len() {
533            let start = (step_events[i].elapsed_ms as i64 + offset).max(0) as u64;
534            // The LAST step phase extends to end-of-monitor-data, NOT to
535            // the terminal event: the terminal is a rate boundary only,
536            // and clamping the last phase's metric window to it would
537            // drop trailing monitor samples (the host keeps sampling
538            // through teardown). Preserves the pre-terminal window.
539            let end = if i + 1 < step_events.len() {
540                (step_events[i + 1].elapsed_ms as i64 + offset).max(0) as u64
541            } else {
542                last_monitor_ms.saturating_add(1)
543            };
544            let stimulus = if i == 0 {
545                None
546            } else {
547                Some(step_events[i].clone())
548            };
549            boundaries.push((start, end, stimulus));
550        }
551
552        // Assign monitor samples to phases and compute metrics.
553        let mut phases: Vec<Phase> = Vec::with_capacity(boundaries.len());
554        for (idx, (start, end, stimulus)) in boundaries.into_iter().enumerate() {
555            let phase_samples: Vec<&MonitorSample> = monitor_samples
556                .iter()
557                .filter(|s| s.elapsed_ms >= start && s.elapsed_ms < end && sample_looks_valid(s))
558                .collect();
559
560            let metrics = compute_metrics(&phase_samples, preemption_threshold_ns);
561
562            phases.push(Phase {
563                // Enumerate position over the phase-bearing step_events,
564                // NOT the bucket step_index `phase_from_bucket` uses. This
565                // path's input is a monitor-only / legacy / test stream with
566                // no step_index-bearing Stimulus frames (the settle is
567                // step_events[0]; later events follow), so enumerate IS the
568                // step identity and `index == 0 => BASELINE` holds for this
569                // model. It diverges from the step_index model ONLY for a
570                // step_index-bearing stream with no leading settle (the
571                // production stimulus shape), which never reaches `build`:
572                // build is the monitor-only fallback, taken only when there
573                // are no PhaseBuckets, i.e. no StepStarts
574                // (build_phase_buckets_with_stimulus synthesizes a bucket
575                // per StepStart) — so a step_index-bearing production stream
576                // always takes from_phase_buckets, never this path.
577                index: idx,
578                start_ms: start,
579                end_ms: end,
580                stimulus,
581                metrics,
582                changes: Vec::new(),
583                // Monitor-only path: no per-cgroup carriers, real window.
584                per_cgroup: std::collections::BTreeMap::new(),
585                not_measured_window: false,
586            });
587        }
588
589        // Detect changes at each phase boundary. Throughput is compared
590        // even across synthesized zero-capture steps; monitor-derived
591        // metrics stay gated on both phases having real samples (see
592        // [`detect_boundary_changes`]).
593        for i in 1..phases.len() {
594            let changes = detect_boundary_changes(&phases[i - 1].metrics, &phases[i].metrics);
595            phases[i].changes = changes;
596        }
597
598        Self { phases }
599    }
600
601    /// Format the timeline with a system context header.
602    ///
603    /// Tests without a real context pass `&TimelineContext::default()`;
604    /// the header lines (`kernel:`, `topology:`, etc.) are omitted but
605    /// the `--- timeline ---` prefix is preserved.
606    // No parameterless format() sibling: output with default context
607    // is byte-identical, but the only non-test caller
608    // (crate::test_support::eval) always has real context, so format()
609    // would be dead code.
610    pub fn format_with_context(&self, ctx: &TimelineContext) -> String {
611        if self.phases.is_empty() {
612            return String::new();
613        }
614
615        let mut out = String::from("--- timeline ---\n");
616
617        // Render context header.
618        let mut header_parts = Vec::new();
619        if let Some(ref k) = ctx.kernel {
620            header_parts.push(format!("kernel: {k}"));
621        }
622        if let Some(ref t) = ctx.topology {
623            header_parts.push(format!("topology: {t}"));
624        }
625        if let Some(ref s) = ctx.scheduler {
626            header_parts.push(format!("scheduler: {s}"));
627        }
628        if let Some(ref s) = ctx.scenario {
629            header_parts.push(format!("scenario: {s}"));
630        }
631        if let Some(d) = ctx.duration_s {
632            header_parts.push(format!("duration: {d:.1}s"));
633        }
634        if !header_parts.is_empty() {
635            for part in &header_parts {
636                out.push_str(part);
637                out.push_str("  ");
638            }
639            // Trim trailing "  " appended by the last iteration.
640            // Explicit length guard so a future edit that stops
641            // appending the separator here can't underflow.
642            if out.len() >= 2 {
643                out.truncate(out.len() - 2);
644            }
645            out.push('\n');
646        }
647
648        self.format_phases(&mut out);
649        out
650    }
651
652    /// Render phase details into the output buffer.
653    fn format_phases(&self, out: &mut String) {
654        for phase in &self.phases {
655            let duration_ms = phase.end_ms.saturating_sub(phase.start_ms);
656            // An orphan carrier carries a normalized (0,0) window that is NOT a
657            // measured zero-duration step; surface it as not-measured rather
658            // than a misleading 0ms (the None-vs-Some discipline the per-metric
659            // renders use, applied to the window).
660            let window = if phase.not_measured_window {
661                "window not measured".to_string()
662            } else {
663                format!("{duration_ms}ms")
664            };
665
666            if phase.index == 0 {
667                // Phase 0 is the settle window before any stimulus.
668                out.push_str(&format!(
669                    "\nBASELINE (settle, {}, {} samples):\n",
670                    window, phase.metrics.sample_count,
671                ));
672            } else {
673                let label_start = phase
674                    .stimulus
675                    .as_ref()
676                    .map(|s| {
677                        let mut l = s.label.clone();
678                        if let Some(op) = &s.op_kind {
679                            l.push(' ');
680                            l.push_str(op);
681                        }
682                        l
683                    })
684                    .unwrap_or_else(|| "?".to_string());
685
686                out.push_str(&format!(
687                    "\nPhase {}: {} ({}, {} samples):\n",
688                    phase.index, label_start, window, phase.metrics.sample_count,
689                ));
690            }
691
692            let m = &phase.metrics;
693            // Render the metric block whenever the phase carries
694            // monitor-derived metrics, not only when it captured periodic
695            // samples: a SYNTHESIZED zero-capture bucket
696            // (build_phase_buckets_with_stimulus) has sample_count 0 but
697            // fold_monitor_into_bucket fills its imbalance / dsq / fallback
698            // / stuck from in-window monitor samples — render them for
699            // parity with the legacy Timeline::build path (which a
700            // zero-capture-with-monitor run took before the synthesize
701            // seam flipped it onto from_phase_buckets).
702            let has_monitor_metrics = m.avg_imbalance.is_some()
703                || m.max_imbalance.is_some()
704                || m.avg_dsq_depth.is_some()
705                || m.avg_nr_running.is_some()
706                || m.max_dsq_depth > 0
707                || m.fallback_rate.is_some()
708                || m.keep_last_rate.is_some()
709                || m.stuck_count > 0;
710            if m.sample_count > 0 || has_monitor_metrics {
711                out.push_str(&format!(
712                    "  imbalance: avg={} max={} | dsq: avg={} max={} | nr_run: avg={}",
713                    m.avg_imbalance
714                        .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
715                    m.max_imbalance
716                        .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
717                    m.avg_dsq_depth
718                        .map_or_else(|| "n/a".to_string(), |v| format!("{v:.0}")),
719                    m.max_dsq_depth,
720                    m.avg_nr_running
721                        .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
722                ));
723                if let Some(fb) = m.fallback_rate {
724                    out.push_str(&format!(" | fallback: {:.0}/vcpu-s", fb));
725                }
726                if let Some(kl) = m.keep_last_rate {
727                    out.push_str(&format!(" | keep_last: {:.0}/vcpu-s", kl));
728                }
729                if let Some(ir) = m.iteration_rate {
730                    out.push_str(&format!(" | iteration rate: {ir:.0} iter/cpu-s"));
731                }
732                out.push('\n');
733                if m.stuck_count > 0 {
734                    out.push_str(&format!("  stuck: {}\n", m.stuck_count));
735                }
736            } else if let Some(ir) = m.iteration_rate {
737                out.push_str(&format!(
738                    "  [no samples] | iteration rate: {ir:.0} iter/cpu-s\n"
739                ));
740            } else {
741                out.push_str("  [no samples]\n");
742            }
743
744            format_phase_cgroups(out, &phase.per_cgroup);
745
746            if let Some(ref stim) = phase.stimulus {
747                let detail = stim.detail.as_deref().unwrap_or("");
748                let op = stim.op_kind.as_deref().unwrap_or("?");
749                out.push_str(&format!("  >>> {}: {op}", stim.label));
750                if !detail.is_empty() {
751                    out.push_str(&format!(" ({detail})"));
752                }
753                out.push('\n');
754            }
755
756            for change in &phase.changes {
757                let delta = change.after - change.before;
758                let sign = if delta > 0.0 { "+" } else { "" };
759                out.push_str(&format!(
760                    "  >>> {}: {} {sign}{:.1}\n",
761                    change.direction, change.metric, delta,
762                ));
763            }
764        }
765    }
766
767    /// Build a [`Timeline`] from pre-bucketed
768    /// [`crate::assert::PhaseBucket`]s emitted by the metric pipeline.
769    /// Preferred over [`Self::build`] when the caller already has
770    /// `PhaseBucket`s in hand — avoids re-deriving phase boundaries
771    /// from stimulus events + monitor samples by walking the buckets
772    /// directly.
773    ///
774    /// One [`Phase`] is emitted per bucket, in `step_index` order.
775    /// `PhaseMetrics` fields are populated from the bucket's
776    /// `metrics` map via a name-keyed mapping:
777    ///
778    /// | PhaseBucket metric key  | PhaseMetrics field      |
779    /// |-------------------------|-------------------------|
780    /// | `max_imbalance_ratio`   | `max_imbalance`         |
781    /// | `avg_imbalance_ratio`   | `avg_imbalance`         |
782    /// | `max_dsq_depth`         | `max_dsq_depth`         |
783    /// | `avg_dsq_depth`         | `avg_dsq_depth`         |
784    /// | `avg_nr_running`        | `avg_nr_running`        |
785    /// | `stuck_count`           | `stuck_count`           |
786    /// | `total_fallback`        | `fallback_rate` (rate)  |
787    /// | `total_keep_last`       | `keep_last_rate` (rate) |
788    /// | `iteration_rate`        | `iteration_rate`        |
789    ///
790    /// Rate fields (`fallback_rate`, `keep_last_rate`) are divided by mean
791    /// delivered vCPU CPU-seconds. They stay `None` when complete vCPU CPU
792    /// clock coverage is unavailable.
793    ///
794    /// `iteration_rate` is present only when matching guest per-cgroup phase
795    /// carriers provide both iterations and positive CPU time. It defaults to
796    /// `None` when `PhaseBucket.metrics` has no such rate.
797    ///
798    /// `changes` (boundary degradation detection) IS computed
799    /// here by diffing adjacent `PhaseMetrics` fields — same
800    /// detection logic [`Self::build`] uses, applied after the
801    /// per-bucket conversion. avg_imbalance + avg_dsq_depth are
802    /// supplied by PhaseBucket so the detection runs on the same
803    /// fields as the legacy path.
804    pub fn from_phase_buckets(
805        phase_buckets: &[crate::assert::PhaseBucket],
806        stimulus_events: &[StimulusEvent],
807        _ctx: &TimelineContext,
808    ) -> Self {
809        let mut sorted: Vec<&crate::assert::PhaseBucket> = phase_buckets.iter().collect();
810        sorted.sort_by_key(|b| b.step_index);
811        // Sort stimulus events by elapsed_ms so correlation finds
812        // the closest event for each bucket window deterministically.
813        // The terminal scenario-end event is excluded: it carries no
814        // step ops/detail to render and its elapsed_ms lands past
815        // every bucket window, so it would never correlate — filtering
816        // it keeps the correlation set to real step starts only.
817        // Per-step StepEnd events are likewise excluded so each bucket's
818        // rendered op/detail label correlates to the step's defining
819        // StepStart, not its end-of-hold marker.
820        let mut sorted_events: Vec<&StimulusEvent> = stimulus_events
821            .iter()
822            .filter(|e| !e.is_terminal && !e.is_step_end)
823            .collect();
824        sorted_events.sort_by_key(|e| e.elapsed_ms);
825        let mut phases: Vec<Phase> = sorted
826            .into_iter()
827            .map(|b| phase_from_bucket(b, &sorted_events))
828            .collect();
829        // Boundary-change detection — shares [`detect_boundary_changes`]
830        // with [`Self::build`]. Walks each adjacent (prev, curr) pair and
831        // records significant deltas on the LATER phase's `changes` vec so
832        // the operator sees "what changed when entering this phase".
833        // Iteration rate is compared even when a side is a synthesized
834        // zero-capture bucket: a guest carrier is independent of periodic
835        // captures, so an efficiency collapse entering or leaving it must
836        // surface. The monitor-derived
837        // metrics (imbalance / dsq depth / fallback / keep_last) stay gated
838        // inside the helper on both phases having real samples, so a
839        // partial-metric phase never paints a phantom non-throughput change.
840        // An orphan phase (a not-measured (0,0) window with all-None
841        // PhaseMetrics) sits between its neighbors here, so detect_boundary_changes
842        // compares each real neighbor against the orphan's None metrics — which
843        // the helper gates away (both sides must be Some) — rather than across
844        // the unmeasured window. INTENTIONAL: there is no data for the orphan's
845        // step, so flagging a phase-k-1 -> phase-k+1 change as if they were
846        // adjacent would assert a transition over an unmeasured interval. This is
847        // render-only (Phase.changes has no verdict/sidecar/A-B consumer).
848        for i in 1..phases.len() {
849            let changes = detect_boundary_changes(&phases[i - 1].metrics, &phases[i].metrics);
850            phases[i].changes = changes;
851        }
852        Self { phases }
853    }
854
855    /// Test helper — collect all degradation changes across phases.
856    /// Retained after the gauntlet analyzer was removed; the scenarios
857    /// pipeline consumes `Timeline` via `format_with_context` and does
858    /// not read degradations directly.
859    #[cfg(test)]
860    pub fn degradations(&self) -> Vec<(&Phase, &PhaseChange)> {
861        let mut out = Vec::new();
862        for phase in &self.phases {
863            for change in &phase.changes {
864                if change.direction == ChangeDirection::Degraded {
865                    out.push((phase, change));
866                }
867            }
868        }
869        out
870    }
871}
872
873// ---------------------------------------------------------------------------
874// PhaseBucket → Phase conversion
875// ---------------------------------------------------------------------------
876
877/// Build a [`Phase`] from a [`crate::assert::PhaseBucket`]. The phase
878/// index is the bucket's `step_index` (BASELINE = 0, scenario Step k =
879/// k + 1), NOT the enumerate position in the vec — so `format_phases`
880/// keys its BASELINE-vs-Step label on the true phase identity, and a run
881/// whose first bucket is a Step (no BASELINE bucket, e.g. under
882/// `--cell-parent-cgroup` where BASELINE captured nothing) renders that
883/// Step correctly rather than mislabeling it as BASELINE. The metric map
884/// is projected onto the named `PhaseMetrics` fields per the table in
885/// [`Timeline::from_phase_buckets`]. BASELINE (`step_index` 0) emits
886/// `stimulus = None`; later phases synthesize a [`StimulusEvent`] whose
887/// label / op_kind come from the bucket label so the failure-message
888/// renderer prints a recognizable phase header.
889fn phase_from_bucket(b: &crate::assert::PhaseBucket, sorted_events: &[&StimulusEvent]) -> Phase {
890    let metrics = PhaseMetrics {
891        sample_count: b.sample_count,
892        avg_imbalance: b.metrics.get("avg_imbalance_ratio").copied(),
893        max_imbalance: b.metrics.get("max_imbalance_ratio").copied(),
894        avg_dsq_depth: b.metrics.get("avg_dsq_depth").copied(),
895        avg_nr_running: b.metrics.get("avg_nr_running").copied(),
896        max_dsq_depth: b
897            .metrics
898            .get("max_dsq_depth")
899            .map(|v| v.round() as u32)
900            .unwrap_or(0),
901        stuck_count: b
902            .metrics
903            .get("stuck_count")
904            .map(|v| v.round() as usize)
905            .unwrap_or(0),
906        fallback_rate: b.metrics.get("fallback_per_vcpu_sec").copied(),
907        keep_last_rate: b.metrics.get("keep_last_per_vcpu_sec").copied(),
908        // Derived rates are already endpoint-matched and pooled in the bucket.
909        iteration_rate: b.metrics.get("iteration_rate").copied(),
910    };
911    let stimulus = if b.step_index == 0 {
912        None
913    } else {
914        // Correlate with the closest StimulusEvent whose
915        // elapsed_ms falls in [start_ms, end_ms]. Carrying the
916        // real event preserves op_kind + detail in the failure-
917        // message timeline render — `phase_from_bucket`'s prior
918        // synthesis of a placeholder StimulusEvent with op_kind
919        // = None / detail = None produced "Step[N]: ?" headers
920        // that lost the operator-facing per-phase context the
921        // legacy Timeline::build path carried.
922        let correlated = sorted_events.iter().find(|e| {
923            if b.start_ms == b.end_ms {
924                e.elapsed_ms == b.start_ms
925            } else {
926                e.elapsed_ms >= b.start_ms && e.elapsed_ms < b.end_ms
927            }
928        });
929        match correlated {
930            Some(ev) => Some((*ev).clone()),
931            None => Some(StimulusEvent {
932                elapsed_ms: b.start_ms,
933                label: b.label.clone(),
934                op_kind: None,
935                detail: None,
936                total_iterations: None,
937                // Synthetic placeholder for a bucket with no
938                // correlated stimulus event; no authoritative step
939                // ordinal to carry.
940                step_index: None,
941                is_terminal: false,
942                is_step_end: false,
943            }),
944        }
945    };
946    // Orphan signature: fold_guest_per_cgroup_into_host_buckets normalizes a
947    // guest carrier with no paired host bucket to a (0,0) window carrying ONLY
948    // per_cgroup (empty metrics). A captured bucket has metrics, so the
949    // (0,0)+empty-metrics+non-empty-per_cgroup shape is the orphan arm's on
950    // every NON-zero-duration window. The one other producer is a ZERO-duration
951    // step at scenario start (StepStart[k] == StepEnd[k] == 0 -> a synthesized
952    // host (0,0) window with empty metrics, since duration 0 yields no rate and
953    // no in-window monitor sample, then MATCHED with a same-step guest carrier),
954    // but that collision is HARMLESS: a zero-duration step has no window to
955    // measure, so "window not measured" reads the same as the "0ms" it would
956    // otherwise show. Display-only, with no verdict/sidecar consumer,
957    // so the marker needs no serialized PhaseBucket flag.
958    let not_measured_window =
959        b.start_ms == 0 && b.end_ms == 0 && b.metrics.is_empty() && !b.per_cgroup.is_empty();
960    Phase {
961        index: b.step_index as usize,
962        start_ms: b.start_ms,
963        end_ms: b.end_ms,
964        stimulus,
965        metrics,
966        changes: Vec::new(),
967        per_cgroup: b.per_cgroup.clone(),
968        not_measured_window,
969    }
970}
971
972/// Cap on per-cgroup lines rendered per phase, bounding the failure-message
973/// size for a many-cgroup scenario (the sched_log render caps similarly).
974/// Truncation is by BTreeMap NAME order (deterministic, no ranking math); a
975/// "+J more" note records the drop so the cap never reads as "all cgroups".
976const MAX_RENDERED_CGROUPS: usize = 16;
977
978/// Render the per-cgroup sub-block for one phase: one line per cgroup in
979/// BTreeMap (name) order, reduced at display time via the
980/// [`crate::assert::PhaseCgroupStats`] summaries. Empty `per_cgroup` (the
981/// monitor-only path, or a phase that carried no per-cgroup components)
982/// renders nothing. The None-vs-Some(0.0)
983/// discipline carries through: an absent off-CPU reduction renders `n/a` (NOT
984/// `0.0%`), and an empty wake / run-delay pool OMITS that segment rather than
985/// painting a misleading `0µs`.
986fn format_phase_cgroups(
987    out: &mut String,
988    per_cgroup: &std::collections::BTreeMap<String, crate::assert::PhaseCgroupStats>,
989) {
990    if per_cgroup.is_empty() {
991        return;
992    }
993    out.push_str("  per-cgroup:\n");
994    for (name, pcg) in per_cgroup.iter().take(MAX_RENDERED_CGROUPS) {
995        out.push_str(&format!("    {name}: "));
996        // A stripped carrier had its raw sample vectors dropped to fit the bulk
997        // frame, so off-cpu / wake / run-delay summaries are all absent — but
998        // that is NOT "not measured". Surface the size-limit drop explicitly so
999        // the operator does not read it as a quiet cgroup.
1000        if pcg.stripped {
1001            out.push_str("samples stripped (size limit)");
1002        } else {
1003            match pcg.off_cpu_summary() {
1004                Some((avg, min, max, spread)) => out.push_str(&format!(
1005                    "off-cpu avg={avg:.1}% min={min:.1}% max={max:.1}% spread={spread:.1}%"
1006                )),
1007                None => out.push_str("off-cpu n/a"),
1008            }
1009            if let Some((p99, median)) = pcg.wake_summary() {
1010                out.push_str(&format!(
1011                    " | wake p99={p99:.0}\u{00b5}s median={median:.0}\u{00b5}s"
1012                ));
1013            }
1014            if let Some((mean, worst)) = pcg.run_delay_summary() {
1015                out.push_str(&format!(
1016                    " | run-delay mean={mean:.0}\u{00b5}s worst={worst:.0}\u{00b5}s"
1017                ));
1018            }
1019        }
1020        out.push_str(&format!(
1021            " | iters={} migrations={}",
1022            pcg.total_iterations, pcg.total_migrations
1023        ));
1024        // Gap is a Peak with no Option: 0 means "no notable gap", so omit it
1025        // rather than print a noisy gap=0ms on every quiet cgroup.
1026        if pcg.max_gap_ms > 0 {
1027            out.push_str(&format!(
1028                " | gap={}ms@cpu{}",
1029                pcg.max_gap_ms, pcg.max_gap_cpu
1030            ));
1031        }
1032        out.push('\n');
1033    }
1034    let total = per_cgroup.len();
1035    if total > MAX_RENDERED_CGROUPS {
1036        let dropped = total - MAX_RENDERED_CGROUPS;
1037        let noun = if dropped == 1 { "cgroup" } else { "cgroups" };
1038        out.push_str(&format!("    (+{dropped} more {noun})\n"));
1039    }
1040}
1041
1042// ---------------------------------------------------------------------------
1043// Metric computation
1044// ---------------------------------------------------------------------------
1045
1046/// Reduce a phase's monitor samples to [`PhaseMetrics`].
1047///
1048/// `preemption_threshold_ns` is the vCPU-preemption exemption window for
1049/// stuck detection: a non-advancing `rq_clock` on a CPU whose
1050/// `vcpu_cpu_time_ns` advanced by less than this is a host-preemption
1051/// artifact, not a genuine stuck condition, and is exempt. Pass `0` to derive it
1052/// from the guest kernel's `CONFIG_HZ` via
1053/// `crate::monitor::vcpu_preemption_threshold_ns` — the same resolution
1054/// [`MonitorSummary::from_samples_with_threshold`](crate::monitor::MonitorSummary::from_samples_with_threshold)
1055/// applies, so the per-phase `stuck_count` applies the SAME per-(CPU,
1056/// window) `is_cpu_stuck` predicate as the run-level
1057/// `MonitorSummary::stuck_count` (run-level `>=` Σ per-phase: it also
1058/// windows across phase boundaries).
1059pub(crate) fn compute_metrics(
1060    samples: &[&MonitorSample],
1061    preemption_threshold_ns: u64,
1062) -> PhaseMetrics {
1063    if samples.is_empty() {
1064        return PhaseMetrics::default();
1065    }
1066
1067    // Filter out samples with implausible data (e.g. garbage DSQ depths
1068    // from uninitialized guest memory) before computing metrics.
1069    let valid: Vec<&MonitorSample> = samples
1070        .iter()
1071        .copied()
1072        .filter(|s| !s.cpus.is_empty() && sample_looks_valid(s))
1073        .collect();
1074
1075    if valid.is_empty() {
1076        return PhaseMetrics {
1077            sample_count: 0,
1078            ..PhaseMetrics::default()
1079        };
1080    }
1081
1082    let mut total_imbalance = 0.0f64;
1083    let mut max_imbalance = 0.0f64;
1084    let mut total_dsq = 0.0f64;
1085    let mut total_nr_running = 0.0f64;
1086    let mut max_dsq = 0u32;
1087    let mut stuck_count = 0usize;
1088
1089    for sample in &valid {
1090        for cpu in &sample.cpus {
1091            max_dsq = max_dsq.max(cpu.local_dsq_depth);
1092        }
1093        let ratio = sample.imbalance_ratio();
1094        total_imbalance += ratio;
1095        if ratio > max_imbalance {
1096            max_imbalance = ratio;
1097        }
1098
1099        let avg_dsq_this: f64 = sample
1100            .cpus
1101            .iter()
1102            .map(|c| c.local_dsq_depth as f64)
1103            .sum::<f64>()
1104            / sample.cpus.len() as f64;
1105        total_dsq += avg_dsq_this;
1106
1107        // Per-sample mean of per-CPU nr_running (full-class runqueue depth),
1108        // averaged over samples below — the avg_dsq_depth shape. `valid`
1109        // guarantees `!cpus.is_empty()`, so the divisor is nonzero.
1110        let avg_nr_this: f64 =
1111            sample.cpus.iter().map(|c| c.nr_running as f64).sum::<f64>() / sample.cpus.len() as f64;
1112        total_nr_running += avg_nr_this;
1113    }
1114
1115    // Stuck detection between consecutive valid samples in this phase.
1116    // Route through `is_cpu_stuck` (the shared predicate the run-level
1117    // `MonitorSummary` path also uses) so the per-phase stuck count and the
1118    // run-level stuck count apply the identical NOHZ-idle and
1119    // vCPU-preemption exemptions — the per-phase count uses the SAME
1120    // predicate as the run-level one (run-level `>=` Σ per-phase: it also
1121    // counts the boundary-straddling window pair and out-of-phase samples).
1122    let threshold = if preemption_threshold_ns > 0 {
1123        preemption_threshold_ns
1124    } else {
1125        crate::monitor::vcpu_preemption_threshold_ns(None)
1126    };
1127    for w in valid.windows(2) {
1128        let prev = w[0];
1129        let curr = w[1];
1130        let cpu_count = prev.cpus.len().min(curr.cpus.len());
1131        for cpu in 0..cpu_count {
1132            if crate::monitor::reader::is_cpu_stuck(&prev.cpus[cpu], &curr.cpus[cpu], threshold) {
1133                stuck_count += 1;
1134            }
1135        }
1136    }
1137
1138    // Event counter rates use the same first/last event samples for the
1139    // numerator and mean-vCPU-CPU-time denominator. Host-preempted wall time
1140    // contributes neither, so the rate is dilation-resistant.
1141    let has_events = |s: &&MonitorSample| s.has_complete_event_counters();
1142    let first_ev = valid.iter().copied().find(|s| has_events(s));
1143    let last_ev = valid.iter().copied().rev().find(|s| has_events(s));
1144
1145    let (fallback_rate, keep_last_rate) = match (first_ev, last_ev) {
1146        (Some(first), Some(last)) => {
1147            // Event counters can reset mid-run (scheduler restart) and
1148            // produce a negative raw delta. Shared helper clamps to
1149            // >= 0 so the computed rate never goes negative; same
1150            // semantics as MonitorSummary::compute_event_deltas.
1151            let fb_delta = crate::monitor::counter_delta(
1152                last.sum_event_field(|e| e.select_cpu_fallback).unwrap_or(0),
1153                first
1154                    .sum_event_field(|e| e.select_cpu_fallback)
1155                    .unwrap_or(0),
1156            );
1157            let kl_delta = crate::monitor::counter_delta(
1158                last.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1159                first.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1160            );
1161            match crate::monitor::MonitorSummary::mean_vcpu_cpu_delta_secs(first, last) {
1162                Some(vcpu_sec) => (
1163                    Some(fb_delta as f64 / vcpu_sec),
1164                    Some(kl_delta as f64 / vcpu_sec),
1165                ),
1166                None => (None, None),
1167            }
1168        }
1169        _ => (None, None),
1170    };
1171
1172    let valid_count = valid.len();
1173    let n = valid_count as f64;
1174    // None when no valid samples — avoids a 0.0/0.0 NaN and keeps "no
1175    // data" distinct from a real zero (the detector skips None sides).
1176    PhaseMetrics {
1177        sample_count: valid_count,
1178        avg_imbalance: (valid_count > 0).then(|| total_imbalance / n),
1179        max_imbalance: (valid_count > 0).then_some(max_imbalance),
1180        avg_dsq_depth: (valid_count > 0).then(|| total_dsq / n),
1181        avg_nr_running: (valid_count > 0).then(|| total_nr_running / n),
1182        max_dsq_depth: max_dsq,
1183        stuck_count,
1184        fallback_rate,
1185        keep_last_rate,
1186        iteration_rate: None,
1187    }
1188}
1189
1190// ---------------------------------------------------------------------------
1191// Tests
1192// ---------------------------------------------------------------------------
1193
1194#[cfg(test)]
1195mod tests {
1196    use super::*;
1197    use crate::monitor::{CpuSnapshot, MonitorSample};
1198
1199    /// `StimulusEvent::phase()` maps the 1-indexed wire `step_index` onto the
1200    /// canonical [`crate::assert::Phase`] (StepStart and StepEnd of step `k`
1201    /// both -> `Phase::step(k)`); the scenario-end terminal seeds no phase.
1202    #[test]
1203    fn stimulus_event_phase_maps_wire_step_index_to_phase() {
1204        use crate::assert::Phase;
1205        // Wire step_index 1 (Step 0) -> Phase::step(0); 2 (Step 1) -> step(1).
1206        assert_eq!(
1207            StimulusEvent::from_wire(&wire_event(0, 1, 0)).phase(),
1208            Some(Phase::step(0)),
1209        );
1210        assert_eq!(
1211            StimulusEvent::from_wire(&wire_event(100, 2, 50)).phase(),
1212            Some(Phase::step(1)),
1213        );
1214        // StepEnd carries the same step_index -> same Phase as its StepStart.
1215        assert_eq!(
1216            StimulusEvent::from_step_end(&wire_event(200, 2, 90)).phase(),
1217            Some(Phase::step(1)),
1218        );
1219        // The terminal boundary is not a step -> no Phase.
1220        assert_eq!(StimulusEvent::terminal(300, 100).phase(), None);
1221    }
1222
1223    fn sample(elapsed_ms: u64, cpus: Vec<(u32, u32, u64)>) -> MonitorSample {
1224        MonitorSample {
1225            bpf_map_fields: Vec::new(),
1226            prog_stats: None,
1227            psi_irq: None,
1228            elapsed_ms,
1229            cpus: cpus
1230                .into_iter()
1231                .map(|(nr_running, dsq, rq_clock)| CpuSnapshot {
1232                    nr_running,
1233                    scx_nr_running: 0,
1234                    local_dsq_depth: dsq,
1235                    rq_clock,
1236                    scx_flags: 0,
1237                    event_counters: None,
1238                    schedstat: None,
1239                    vcpu_cpu_time_ns: None,
1240                    vcpu_perf: None,
1241                    avg_irq_util: None,
1242                    sched_domains: None,
1243                })
1244                .collect(),
1245        }
1246    }
1247
1248    fn stimulus(elapsed_ms: u64, label: &str) -> StimulusEvent {
1249        StimulusEvent {
1250            elapsed_ms,
1251            label: label.to_string(),
1252            op_kind: None,
1253            detail: None,
1254            total_iterations: None,
1255            step_index: None,
1256            is_terminal: false,
1257            is_step_end: false,
1258        }
1259    }
1260
1261    #[test]
1262    fn empty_inputs_empty_timeline() {
1263        let t = Timeline::build(&[], &[], 0);
1264        assert!(t.phases.is_empty());
1265    }
1266
1267    #[test]
1268    fn no_stimulus_empty_timeline() {
1269        let samples = vec![sample(1000, vec![(2, 1, 100)])];
1270        let t = Timeline::build(&[], &samples, 0);
1271        assert!(t.phases.is_empty());
1272    }
1273
1274    #[test]
1275    fn no_monitor_empty_timeline() {
1276        let events = vec![stimulus(0, "ScenarioStart")];
1277        let t = Timeline::build(&events, &[], 0);
1278        assert!(t.phases.is_empty());
1279    }
1280
1281    #[test]
1282    fn single_event_single_phase() {
1283        let events = vec![stimulus(0, "ScenarioStart")];
1284        let samples = vec![
1285            sample(600, vec![(2, 1, 100), (2, 1, 200)]),
1286            sample(700, vec![(2, 1, 300), (2, 1, 400)]),
1287        ];
1288        let t = Timeline::build(&events, &samples, 0);
1289        assert_eq!(t.phases.len(), 1);
1290        // Both samples — including the one AT last_monitor_ms (700) —
1291        // must fall inside the single phase's [start, last_monitor_ms+1)
1292        // window. A > 0 check passes even if the last-sample-inclusion
1293        // off-by-one (end = last_monitor_ms+1) regressed to +0, dropping
1294        // the 700 sample. Pin the exact count.
1295        assert_eq!(t.phases[0].metrics.sample_count, 2);
1296    }
1297
1298    #[test]
1299    fn two_events_two_phases() {
1300        let events = vec![stimulus(0, "ScenarioStart"), stimulus(3000, "StepStart[0]")];
1301        let samples: Vec<MonitorSample> = (5..65)
1302            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
1303            .collect();
1304        let t = Timeline::build(&events, &samples, 0);
1305        assert_eq!(t.phases.len(), 2);
1306        // Pin WHERE the boundary fell, not just non-emptiness: 60 samples
1307        // at i*100 (i in 5..65 → 500..6400); the >500 warmup drops the
1308        // 500 sample (i=5), leaving 59. The StepStart[0]@3000 boundary
1309        // (offset-adjusted) splits them 30/29. A > 0 check passes even if
1310        // the offset/boundary math shifted the split point while leaving
1311        // samples on both sides.
1312        assert_eq!(t.phases[0].metrics.sample_count, 30);
1313        assert_eq!(t.phases[1].metrics.sample_count, 29);
1314        assert_eq!(
1315            t.phases[0].metrics.sample_count + t.phases[1].metrics.sample_count,
1316            59,
1317            "59 = 60 samples minus the 500ms sample dropped by the >500 warmup",
1318        );
1319    }
1320
1321    #[test]
1322    fn improvement_detected() {
1323        // Phase 0: imbalanced
1324        // Phase 1: balanced
1325        let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
1326        let mut samples = Vec::new();
1327        for i in 5..15 {
1328            samples.push(sample(
1329                i * 100,
1330                vec![(1, 1, i * 1000), (5, 1, i * 1000 + 100)],
1331            ));
1332        }
1333        for i in 15..25 {
1334            samples.push(sample(
1335                i * 100,
1336                vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
1337            ));
1338        }
1339        let t = Timeline::build(&events, &samples, 0);
1340        let improvements: Vec<_> = t
1341            .phases
1342            .iter()
1343            .flat_map(|p| p.changes.iter())
1344            .filter(|c| c.direction == ChangeDirection::Improved)
1345            .collect();
1346        assert!(!improvements.is_empty());
1347    }
1348
1349    #[test]
1350    fn format_non_empty() {
1351        let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
1352        let samples: Vec<MonitorSample> = (5..25)
1353            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
1354            .collect();
1355        let t = Timeline::build(&events, &samples, 0);
1356        let formatted = t.format_with_context(&TimelineContext::default());
1357        assert!(formatted.contains("BASELINE"));
1358        assert!(formatted.contains("Phase 1"));
1359        assert!(formatted.contains("imbalance"));
1360    }
1361
1362    /// A synthesized zero-capture step (sample_count==0) still renders a
1363    /// carrier-derived throughput metric in the formatted timeline, not only
1364    /// "[no samples]". Pins the synthesized-step visibility in format_phases. The
1365    /// BASELINE bucket holds step_index 0 (its phase index, the settle
1366    /// render) so the synthesized step lands at a Phase index that takes
1367    /// the metric path.
1368    #[test]
1369    fn format_renders_synthesized_step_throughput() {
1370        let buckets = vec![
1371            crate::assert::PhaseBucket {
1372                per_cgroup: Default::default(),
1373                step_index: 0,
1374                label: "BASELINE".to_string(),
1375                start_ms: 0,
1376                end_ms: 1000,
1377                sample_count: 2,
1378                metrics: std::collections::BTreeMap::new(),
1379            },
1380            crate::assert::PhaseBucket {
1381                per_cgroup: Default::default(),
1382                step_index: 1,
1383                label: "Step[0]".to_string(),
1384                start_ms: 1000,
1385                end_ms: 2000,
1386                sample_count: 0, // synthesized zero-capture step
1387                metrics: std::collections::BTreeMap::from([("iteration_rate".to_string(), 1500.0)]),
1388            },
1389        ];
1390        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1391        let formatted = t.format_with_context(&TimelineContext::default());
1392        assert!(
1393            formatted.contains("iteration rate: 1500 iter/cpu-s"),
1394            "a synthesized step must render its guest-carrier iteration rate, \
1395             not only '[no samples]'; got:\n{formatted}",
1396        );
1397    }
1398
1399    // -- orphan not-measured marker + per-cgroup sub-block render --
1400
1401    /// An orphan bucket — the unique (0,0)-window + empty-metrics +
1402    /// non-empty-per_cgroup shape from the fold's orphan arm — renders "window
1403    /// not measured", NOT a misleading "0ms".
1404    #[test]
1405    fn format_renders_orphan_window_as_not_measured() {
1406        let mut per_cgroup = std::collections::BTreeMap::new();
1407        per_cgroup.insert(
1408            "cg".to_string(),
1409            crate::assert::PhaseCgroupStats {
1410                off_cpu_pcts: vec![80.0],
1411                total_iterations: 900_000,
1412                ..Default::default()
1413            },
1414        );
1415        let buckets = vec![crate::assert::PhaseBucket {
1416            per_cgroup,
1417            step_index: 1,
1418            label: "Step[0]".to_string(),
1419            start_ms: 0,
1420            end_ms: 0,
1421            sample_count: 0,
1422            metrics: std::collections::BTreeMap::new(),
1423        }];
1424        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1425        let formatted = t.format_with_context(&TimelineContext::default());
1426        assert!(
1427            formatted.contains("window not measured"),
1428            "orphan window must render not-measured; got:\n{formatted}",
1429        );
1430        assert!(
1431            !formatted.contains("(0ms,"),
1432            "orphan must NOT render as 0ms; got:\n{formatted}",
1433        );
1434        // The whole point of routing orphan carriers through the render (vs the
1435        // pre-fold path that dropped them) is that their per-cgroup telemetry —
1436        // an orphan's ONLY payload — SURFACES alongside the not-measured marker.
1437        assert!(
1438            formatted.contains("per-cgroup:"),
1439            "orphan's per-cgroup sub-block must render; got:\n{formatted}",
1440        );
1441        assert!(
1442            formatted.contains("cg: off-cpu avg=80.0%"),
1443            "orphan carrier's off-cpu reduction must render; got:\n{formatted}",
1444        );
1445        assert!(
1446            formatted.contains("iters=900000"),
1447            "orphan carrier's counters must render; got:\n{formatted}",
1448        );
1449        // The cg carrier has off-cpu but NO wake/run-delay pools — pin the
1450        // per-line omit when off-cpu is the sole reduction present (the most
1451        // likely real orphan-carrier shape).
1452        let cg_line = formatted
1453            .lines()
1454            .find(|l| l.contains("cg: off-cpu"))
1455            .expect("cg line");
1456        assert!(!cg_line.contains("wake p99="), "got:\n{formatted}");
1457        assert!(!cg_line.contains("run-delay mean="), "got:\n{formatted}");
1458    }
1459
1460    /// Orphan-adjacency suppression: an orphan phase (all-None
1461    /// PhaseMetrics) BETWEEN two real phases makes detect_boundary_changes
1462    /// compare each real neighbor against the orphan's gated-away None metrics,
1463    /// NOT across the unmeasured window — so a step1->step3 throughput collapse
1464    /// is NOT flagged on step3 (no data for the intervening orphan step).
1465    /// INTENTIONAL + render-only; pins the documented from_phase_buckets behavior.
1466    #[test]
1467    fn format_orphan_phase_suppresses_cross_orphan_change_detection() {
1468        let real = |step: u16, rate: f64| crate::assert::PhaseBucket {
1469            per_cgroup: Default::default(),
1470            step_index: step,
1471            label: format!("Step[{}]", step - 1),
1472            start_ms: step as u64 * 1000,
1473            end_ms: step as u64 * 1000 + 500,
1474            sample_count: 5,
1475            metrics: std::collections::BTreeMap::from([("iteration_rate".to_string(), rate)]),
1476        };
1477        let mut orphan_pc = std::collections::BTreeMap::new();
1478        orphan_pc.insert(
1479            "cg".to_string(),
1480            crate::assert::PhaseCgroupStats {
1481                off_cpu_pcts: vec![50.0],
1482                ..Default::default()
1483            },
1484        );
1485        let orphan = crate::assert::PhaseBucket {
1486            per_cgroup: orphan_pc,
1487            step_index: 2,
1488            label: "Step[1]".to_string(),
1489            start_ms: 0,
1490            end_ms: 0,
1491            sample_count: 0,
1492            metrics: std::collections::BTreeMap::new(),
1493        };
1494        // CONTROL: step1 (rate 10000) adjacent to step3 (rate 1000) — a 90%
1495        // collapse (> the 30% rel threshold) IS flagged as a change on step3.
1496        let ctrl = Timeline::from_phase_buckets(
1497            &[real(1, 10000.0), real(3, 1000.0)],
1498            &[],
1499            &TimelineContext::default(),
1500        );
1501        let ctrl_step3 = ctrl.phases.iter().find(|p| p.index == 3).expect("step3");
1502        assert!(
1503            !ctrl_step3.changes.is_empty(),
1504            "control: adjacent step1->step3 throughput collapse IS flagged; got: {:?}",
1505            ctrl_step3.changes,
1506        );
1507        // With the orphan between, step3 is compared against the orphan's None
1508        // metrics -> the change is suppressed (no data for the gap).
1509        let t = Timeline::from_phase_buckets(
1510            &[real(1, 10000.0), orphan, real(3, 1000.0)],
1511            &[],
1512            &TimelineContext::default(),
1513        );
1514        let step3 = t.phases.iter().find(|p| p.index == 3).expect("step3");
1515        assert!(
1516            step3.changes.is_empty(),
1517            "orphan between suppresses the cross-orphan change; got: {:?}",
1518            step3.changes,
1519        );
1520        assert!(
1521            t.format_with_context(&TimelineContext::default())
1522                .contains("window not measured"),
1523            "the orphan still renders not-measured",
1524        );
1525    }
1526
1527    /// The orphan signature is GUARDED on empty metrics: a (0,0)-window bucket
1528    /// that carries metrics is a captured bucket (or a measured zero-duration
1529    /// step), NOT an orphan — it renders 0ms, never "window not measured".
1530    #[test]
1531    fn format_does_not_mark_not_measured_when_metrics_present() {
1532        let mut per_cgroup = std::collections::BTreeMap::new();
1533        per_cgroup.insert(
1534            "cg".to_string(),
1535            crate::assert::PhaseCgroupStats {
1536                off_cpu_pcts: vec![50.0],
1537                ..Default::default()
1538            },
1539        );
1540        let buckets = vec![crate::assert::PhaseBucket {
1541            per_cgroup,
1542            step_index: 1,
1543            label: "Step[0]".to_string(),
1544            start_ms: 0,
1545            end_ms: 0,
1546            sample_count: 1,
1547            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1548        }];
1549        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1550        let formatted = t.format_with_context(&TimelineContext::default());
1551        assert!(
1552            !formatted.contains("window not measured"),
1553            "a (0,0) window WITH metrics is captured, not an orphan; got:\n{formatted}",
1554        );
1555        assert!(
1556            formatted.contains("(0ms,"),
1557            "a measured zero-duration window renders 0ms; got:\n{formatted}",
1558        );
1559    }
1560
1561    /// The per-cgroup sub-block renders one line per cgroup (name order), with
1562    /// the None-vs-Some discipline: a not-measured off-CPU reduction renders
1563    /// `n/a` (not `0.0%`), and an empty wake/run-delay pool omits that segment.
1564    #[test]
1565    fn format_renders_per_cgroup_subblock_none_aware() {
1566        let mut per_cgroup = std::collections::BTreeMap::new();
1567        per_cgroup.insert(
1568            "cg_a".to_string(),
1569            crate::assert::PhaseCgroupStats {
1570                off_cpu_pcts: vec![80.0, 84.0],
1571                wake_latencies_ns: vec![100_000, 120_000],
1572                run_delays_ns: vec![45_000],
1573                total_iterations: 900_000,
1574                total_migrations: 12,
1575                max_gap_ms: 8,
1576                max_gap_cpu: 3,
1577                ..Default::default()
1578            },
1579        );
1580        per_cgroup.insert(
1581            "cg_b".to_string(),
1582            crate::assert::PhaseCgroupStats {
1583                off_cpu_pcts: vec![],
1584                total_iterations: 80_000,
1585                ..Default::default()
1586            },
1587        );
1588        let buckets = vec![crate::assert::PhaseBucket {
1589            per_cgroup,
1590            step_index: 1,
1591            label: "Step[0]".to_string(),
1592            start_ms: 1000,
1593            end_ms: 6000,
1594            sample_count: 10,
1595            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.5)]),
1596        }];
1597        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1598        let formatted = t.format_with_context(&TimelineContext::default());
1599        assert!(
1600            formatted.contains("per-cgroup:"),
1601            "sub-block header; got:\n{formatted}"
1602        );
1603        assert!(
1604            formatted.contains("cg_a: off-cpu avg=82.0% min=80.0% max=84.0% spread=4.0%"),
1605            "got:\n{formatted}",
1606        );
1607        assert!(
1608            formatted.contains("wake p99="),
1609            "wake segment present; got:\n{formatted}"
1610        );
1611        assert!(
1612            formatted.contains("run-delay mean="),
1613            "run-delay segment present; got:\n{formatted}",
1614        );
1615        assert!(
1616            formatted.contains("iters=900000 migrations=12"),
1617            "counters present; got:\n{formatted}",
1618        );
1619        assert!(
1620            formatted.contains("cg_b: off-cpu n/a"),
1621            "not-measured off-cpu is n/a, NOT 0.0%; got:\n{formatted}",
1622        );
1623        // cg_a has a coupled gap (8ms @ cpu 3) -> rendered with its cpu.
1624        assert!(
1625            formatted.contains("gap=8ms@cpu3"),
1626            "coupled gap renders ms@cpu; got:\n{formatted}",
1627        );
1628        // cg_b has no wake/run-delay pools AND max_gap_ms==0 -> those segments
1629        // omitted on its line (gap is omitted at 0, not printed as gap=0ms).
1630        let cg_b_line = formatted
1631            .lines()
1632            .find(|l| l.contains("cg_b:"))
1633            .expect("cg_b line");
1634        assert!(!cg_b_line.contains("wake p99="), "got:\n{formatted}");
1635        assert!(
1636            !cg_b_line.contains("gap="),
1637            "gap omitted at 0; got:\n{formatted}"
1638        );
1639        // BTreeMap name order: cg_a before cg_b.
1640        assert!(
1641            formatted.find("cg_a:").unwrap() < formatted.find("cg_b:").unwrap(),
1642            "cgroups render in name order; got:\n{formatted}",
1643        );
1644    }
1645
1646    /// A stripped carrier (raw sample vectors dropped to fit the size-limited
1647    /// bulk frame) renders "samples stripped (size limit)" — distinct from a
1648    /// not-measured carrier's "off-cpu n/a" — so an operator does not read a
1649    /// size-limit drop as a quiet cgroup. The surviving counters still render.
1650    #[test]
1651    fn format_renders_stripped_carrier_distinctly() {
1652        let mut per_cgroup = std::collections::BTreeMap::new();
1653        per_cgroup.insert(
1654            "cg_stripped".to_string(),
1655            crate::assert::PhaseCgroupStats {
1656                stripped: true,
1657                total_iterations: 500_000,
1658                total_migrations: 9,
1659                ..Default::default()
1660            },
1661        );
1662        per_cgroup.insert(
1663            "cg_quiet".to_string(),
1664            // Genuinely measured nothing (NOT stripped).
1665            crate::assert::PhaseCgroupStats {
1666                total_iterations: 1_000,
1667                ..Default::default()
1668            },
1669        );
1670        let buckets = vec![crate::assert::PhaseBucket {
1671            per_cgroup,
1672            step_index: 1,
1673            label: "Step[0]".to_string(),
1674            start_ms: 1000,
1675            end_ms: 6000,
1676            sample_count: 10,
1677            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.5)]),
1678        }];
1679        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1680        let formatted = t.format_with_context(&TimelineContext::default());
1681        assert!(
1682            formatted.contains("cg_stripped: samples stripped (size limit)"),
1683            "stripped carrier shows the size-limit marker; got:\n{formatted}",
1684        );
1685        assert!(
1686            formatted.contains("iters=500000 migrations=9"),
1687            "stripped carrier still renders its surviving counters; got:\n{formatted}",
1688        );
1689        assert!(
1690            formatted.contains("cg_quiet: off-cpu n/a"),
1691            "a not-stripped, not-measured carrier stays n/a; got:\n{formatted}",
1692        );
1693        assert!(
1694            !formatted.contains("cg_quiet: samples stripped"),
1695            "a not-stripped carrier must NOT show the stripped marker; got:\n{formatted}",
1696        );
1697    }
1698
1699    /// Measured-zero off-CPU renders `0.0%` (a real zero), distinct from the
1700    /// `n/a` not-measured state — the kind-specific None-vs-Some(0.0) boundary.
1701    #[test]
1702    fn format_renders_measured_zero_off_cpu_as_zero_not_na() {
1703        let mut per_cgroup = std::collections::BTreeMap::new();
1704        per_cgroup.insert(
1705            "cg".to_string(),
1706            crate::assert::PhaseCgroupStats {
1707                off_cpu_pcts: vec![0.0, 0.0],
1708                ..Default::default()
1709            },
1710        );
1711        let buckets = vec![crate::assert::PhaseBucket {
1712            per_cgroup,
1713            step_index: 1,
1714            label: "Step[0]".to_string(),
1715            start_ms: 1000,
1716            end_ms: 2000,
1717            sample_count: 1,
1718            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1719        }];
1720        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1721        let formatted = t.format_with_context(&TimelineContext::default());
1722        assert!(
1723            formatted.contains("off-cpu avg=0.0%"),
1724            "measured zero off-cpu is 0.0%, not n/a; got:\n{formatted}",
1725        );
1726        assert!(!formatted.contains("off-cpu n/a"), "got:\n{formatted}");
1727    }
1728
1729    /// Symmetric with the off-cpu measured-zero test for wake + run-delay: a
1730    /// NON-empty pool of zeros is a measured zero (Some) and renders `0µs`, NOT
1731    /// omitted (which only an EMPTY pool -> None does). Guards against a refactor
1732    /// that special-cased a zero reduction to None (collapsing measured-zero
1733    /// into not-measured), silently omitting a real zero-latency reading.
1734    #[test]
1735    fn format_renders_measured_zero_wake_and_run_delay_not_omitted() {
1736        let mut per_cgroup = std::collections::BTreeMap::new();
1737        per_cgroup.insert(
1738            "cg".to_string(),
1739            crate::assert::PhaseCgroupStats {
1740                off_cpu_pcts: vec![5.0],
1741                wake_latencies_ns: vec![0],
1742                run_delays_ns: vec![0],
1743                ..Default::default()
1744            },
1745        );
1746        let buckets = vec![crate::assert::PhaseBucket {
1747            per_cgroup,
1748            step_index: 1,
1749            label: "Step[0]".to_string(),
1750            start_ms: 1000,
1751            end_ms: 2000,
1752            sample_count: 1,
1753            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1754        }];
1755        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1756        let formatted = t.format_with_context(&TimelineContext::default());
1757        assert!(
1758            formatted.contains("wake p99=0\u{00b5}s"),
1759            "measured-zero wake renders 0µs, not omitted; got:\n{formatted}",
1760        );
1761        assert!(
1762            formatted.contains("run-delay mean=0\u{00b5}s"),
1763            "measured-zero run-delay renders 0µs, not omitted; got:\n{formatted}",
1764        );
1765    }
1766
1767    /// The per-cgroup sub-block caps at MAX_RENDERED_CGROUPS (16) by name
1768    /// order, with a "+J more" note — bounding failure-message size.
1769    #[test]
1770    fn format_caps_per_cgroup_subblock() {
1771        let mut per_cgroup = std::collections::BTreeMap::new();
1772        for i in 0..20 {
1773            per_cgroup.insert(
1774                format!("cg{i:02}"),
1775                crate::assert::PhaseCgroupStats {
1776                    off_cpu_pcts: vec![10.0],
1777                    ..Default::default()
1778                },
1779            );
1780        }
1781        let buckets = vec![crate::assert::PhaseBucket {
1782            per_cgroup,
1783            step_index: 1,
1784            label: "Step[0]".to_string(),
1785            start_ms: 1000,
1786            end_ms: 2000,
1787            sample_count: 1,
1788            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1789        }];
1790        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1791        let formatted = t.format_with_context(&TimelineContext::default());
1792        assert!(
1793            formatted.contains("(+4 more cgroups)"),
1794            "20 cgroups capped at 16 -> +4 more; got:\n{formatted}",
1795        );
1796        assert!(
1797            formatted.contains("cg15:"),
1798            "first 16 rendered; got:\n{formatted}"
1799        );
1800        assert!(
1801            !formatted.contains("cg16:"),
1802            "cg16 is beyond the cap; got:\n{formatted}",
1803        );
1804    }
1805
1806    /// No per-cgroup sub-block when the phase carries no carriers (the
1807    /// monitor-only path, or a phase with empty per_cgroup).
1808    #[test]
1809    fn format_omits_per_cgroup_subblock_when_empty() {
1810        let buckets = vec![crate::assert::PhaseBucket {
1811            per_cgroup: Default::default(),
1812            step_index: 1,
1813            label: "Step[0]".to_string(),
1814            start_ms: 1000,
1815            end_ms: 2000,
1816            sample_count: 1,
1817            metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1818        }];
1819        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1820        let formatted = t.format_with_context(&TimelineContext::default());
1821        assert!(
1822            !formatted.contains("per-cgroup:"),
1823            "no sub-block when carriers absent; got:\n{formatted}",
1824        );
1825    }
1826
1827    /// Cap off-by-one boundary: EXACTLY 16 cgroups render all 16 with NO
1828    /// "more cgroups" note (total > MAX is false); 17 render 16 + "(+1 more
1829    /// cgroups)". Pins the `>` vs `>=` / `take(N)` edge against a silent drop
1830    /// (one cgroup gone, no note) or a "(+0 more)" lie.
1831    #[test]
1832    fn format_per_cgroup_cap_boundary_16_and_17() {
1833        let mk = |n: usize| {
1834            let mut per_cgroup = std::collections::BTreeMap::new();
1835            for i in 0..n {
1836                per_cgroup.insert(
1837                    format!("cg{i:02}"),
1838                    crate::assert::PhaseCgroupStats {
1839                        off_cpu_pcts: vec![10.0],
1840                        ..Default::default()
1841                    },
1842                );
1843            }
1844            let buckets = vec![crate::assert::PhaseBucket {
1845                per_cgroup,
1846                step_index: 1,
1847                label: "Step[0]".to_string(),
1848                start_ms: 1000,
1849                end_ms: 2000,
1850                sample_count: 1,
1851                metrics: std::collections::BTreeMap::from([(
1852                    "avg_imbalance_ratio".to_string(),
1853                    1.0,
1854                )]),
1855            }];
1856            Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default())
1857                .format_with_context(&TimelineContext::default())
1858        };
1859        let at16 = mk(16);
1860        assert!(at16.contains("cg15:"), "16th cgroup renders; got:\n{at16}");
1861        assert!(
1862            !at16.contains("more cgroups"),
1863            "exactly 16 has NO truncation note; got:\n{at16}",
1864        );
1865        let at17 = mk(17);
1866        assert!(at17.contains("cg15:"), "got:\n{at17}");
1867        assert!(
1868            !at17.contains("cg16:"),
1869            "17th is past the cap; got:\n{at17}"
1870        );
1871        assert!(
1872            at17.contains("(+1 more cgroup)") && !at17.contains("(+1 more cgroups)"),
1873            "17 cgroups -> exactly +1 more (singular 'cgroup'); got:\n{at17}",
1874        );
1875    }
1876
1877    /// A synthesized (sample_count==0) bucket carrying monitor-derived
1878    /// metrics renders the imbalance/dsq block, not just "[no samples]" —
1879    /// the render-layer half of the monitor-parity handling. format_phases
1880    /// gates that block on `has_monitor_metrics`, not `sample_count > 0`.
1881    #[test]
1882    fn format_renders_synthesized_step_monitor_metrics() {
1883        let buckets = vec![
1884            crate::assert::PhaseBucket {
1885                per_cgroup: Default::default(),
1886                step_index: 0,
1887                label: "BASELINE".to_string(),
1888                start_ms: 0,
1889                end_ms: 1000,
1890                sample_count: 2,
1891                metrics: std::collections::BTreeMap::new(),
1892            },
1893            crate::assert::PhaseBucket {
1894                per_cgroup: Default::default(),
1895                step_index: 1,
1896                label: "Step[0]".to_string(),
1897                start_ms: 1000,
1898                end_ms: 2000,
1899                sample_count: 0, // synthesized zero-capture step
1900                metrics: std::collections::BTreeMap::from([
1901                    ("avg_imbalance_ratio".to_string(), 2.0),
1902                    ("max_imbalance_ratio".to_string(), 3.0),
1903                    ("avg_dsq_depth".to_string(), 5.0),
1904                    ("max_dsq_depth".to_string(), 7.0),
1905                ]),
1906            },
1907        ];
1908        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1909        let formatted = t.format_with_context(&TimelineContext::default());
1910        assert!(
1911            formatted.contains("imbalance: avg=2.0 max=3.0"),
1912            "synthesized bucket's folded imbalance must render, not \
1913             '[no samples]'; got:\n{formatted}",
1914        );
1915        assert!(
1916            formatted.contains("dsq: avg=5 max=7"),
1917            "synthesized bucket's folded dsq must render; got:\n{formatted}",
1918        );
1919    }
1920
1921    /// A from_phase_buckets render whose first bucket is a Step (no
1922    /// BASELINE bucket — e.g. under --cell-parent-cgroup where BASELINE
1923    /// captured nothing) must NOT mislabel that Step as "BASELINE".
1924    /// phase.index is the bucket's step_index, so format_phases' index==0
1925    /// BASELINE check fires only for a real BASELINE bucket.
1926    #[test]
1927    fn format_no_baseline_bucket_does_not_mislabel_first_step() {
1928        let buckets = vec![
1929            crate::assert::PhaseBucket {
1930                per_cgroup: Default::default(),
1931                step_index: 1, // scenario Step 0; NO BASELINE bucket present
1932                label: "Step[0]".to_string(),
1933                start_ms: 1000,
1934                end_ms: 2000,
1935                sample_count: 3,
1936                metrics: std::collections::BTreeMap::from([(
1937                    "avg_imbalance_ratio".to_string(),
1938                    1.0,
1939                )]),
1940            },
1941            crate::assert::PhaseBucket {
1942                per_cgroup: Default::default(),
1943                step_index: 2,
1944                label: "Step[1]".to_string(),
1945                start_ms: 2000,
1946                end_ms: 3000,
1947                sample_count: 3,
1948                metrics: std::collections::BTreeMap::from([(
1949                    "avg_imbalance_ratio".to_string(),
1950                    1.0,
1951                )]),
1952            },
1953        ];
1954        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1955        let formatted = t.format_with_context(&TimelineContext::default());
1956        assert!(
1957            !formatted.contains("BASELINE"),
1958            "no BASELINE bucket -> no BASELINE label; the first Step must not \
1959             be mislabeled as BASELINE; got:\n{formatted}",
1960        );
1961        assert!(
1962            formatted.contains("Phase 1"),
1963            "the first Step renders as 'Phase 1' (its step_index), not \
1964             'Phase 0'/BASELINE; got:\n{formatted}",
1965        );
1966    }
1967
1968    /// Sparse / non-contiguous step_index renders the TRUE step number:
1969    /// BASELINE(0) + a Step at step_index 3 renders "Phase 3", not the
1970    /// enumerate-position "Phase 1". Pins index == step_index for a
1971    /// non-zero, non-contiguous label — the case that distinguishes
1972    /// step_index from the old enumerate index (a revert to enumerate
1973    /// would pass the contiguous tests but fail this one).
1974    #[test]
1975    fn format_sparse_step_index_renders_true_step_number() {
1976        let buckets = vec![
1977            crate::assert::PhaseBucket {
1978                per_cgroup: Default::default(),
1979                step_index: 0,
1980                label: "BASELINE".to_string(),
1981                start_ms: 0,
1982                end_ms: 100,
1983                sample_count: 2,
1984                metrics: std::collections::BTreeMap::new(),
1985            },
1986            crate::assert::PhaseBucket {
1987                per_cgroup: Default::default(),
1988                step_index: 3, // sparse: Steps 0/1 absent, only step_index 3
1989                label: "Step[2]".to_string(),
1990                start_ms: 200,
1991                end_ms: 300,
1992                sample_count: 2,
1993                metrics: std::collections::BTreeMap::from([(
1994                    "avg_imbalance_ratio".to_string(),
1995                    1.0,
1996                )]),
1997            },
1998        ];
1999        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
2000        let formatted = t.format_with_context(&TimelineContext::default());
2001        assert!(
2002            formatted.contains("Phase 3"),
2003            "a sparse Step at step_index 3 must render 'Phase 3' (its \
2004             step_index), not the enumerate-position 'Phase 1'; got:\n{formatted}",
2005        );
2006        assert!(
2007            !formatted.contains("Phase 1"),
2008            "the enumerate-position 'Phase 1' must NOT appear for a \
2009             step_index-3 bucket; got:\n{formatted}",
2010        );
2011    }
2012
2013    #[test]
2014    fn unsorted_events_sorted() {
2015        let events = vec![stimulus(3000, "StepStart[0]"), stimulus(0, "ScenarioStart")];
2016        let samples: Vec<MonitorSample> = (5..35)
2017            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2018            .collect();
2019        let t = Timeline::build(&events, &samples, 0);
2020        assert_eq!(t.phases.len(), 2);
2021        // First phase should be from ScenarioStart (earliest).
2022        assert!(t.phases[0].stimulus.is_none());
2023    }
2024
2025    #[test]
2026    fn stuck_detected_in_phase() {
2027        let events = vec![stimulus(0, "ScenarioStart")];
2028        let samples = vec![
2029            sample(600, vec![(1, 0, 5000), (1, 0, 6000)]),
2030            sample(700, vec![(1, 0, 5000), (1, 0, 7000)]), // cpu0 stalled
2031        ];
2032        let t = Timeline::build(&events, &samples, 0);
2033        assert_eq!(t.phases[0].metrics.stuck_count, 1);
2034    }
2035
2036    #[test]
2037    fn compute_metrics_stuck_count_accumulates_across_windows() {
2038        // cpu0 frozen across TWO consecutive windows -> stuck_count == 2.
2039        // Pins that the per-phase path counts every stuck (CPU, window),
2040        // mirroring the run-level accumulation (both breaks removed).
2041        let s1 = sample(100, vec![(1, 0, 5000), (1, 0, 6000)]);
2042        let s2 = sample(200, vec![(1, 0, 5000), (1, 0, 6100)]);
2043        let s3 = sample(300, vec![(1, 0, 5000), (1, 0, 6200)]);
2044        let refs: Vec<&MonitorSample> = vec![&s1, &s2, &s3];
2045        let m = compute_metrics(&refs, 0);
2046        assert_eq!(m.stuck_count, 2);
2047    }
2048
2049    #[test]
2050    fn run_level_stuck_count_ge_sum_of_per_phase() {
2051        // Same is_cpu_stuck predicate, different windowing domains: the
2052        // run-level path windows(2) over the FULL stream and counts the
2053        // pair straddling a phase split; partitioning the stream into
2054        // per-phase subsets drops that boundary pair. So run-level >= Σ
2055        // per-phase (strict here) — pins the documented inequality.
2056        let s1 = sample(100, vec![(1, 0, 5000), (1, 0, 6000)]);
2057        let s2 = sample(200, vec![(1, 0, 5000), (1, 0, 6100)]);
2058        let s3 = sample(300, vec![(1, 0, 5000), (1, 0, 6200)]);
2059        let samples = vec![s1, s2, s3];
2060        // Run-level: windows(2) over all 3 -> cpu0 frozen in both windows.
2061        let run_level = crate::monitor::MonitorSummary::from_samples(&samples).stuck_count;
2062        assert_eq!(run_level, 2);
2063        // Partition after s2 (phase A = [s1,s2], phase B = [s3]): the
2064        // (s2,s3) boundary pair is in neither phase's windows(2).
2065        let phase_a: Vec<&MonitorSample> = vec![&samples[0], &samples[1]];
2066        let phase_b: Vec<&MonitorSample> = vec![&samples[2]];
2067        let sum_per_phase =
2068            compute_metrics(&phase_a, 0).stuck_count + compute_metrics(&phase_b, 0).stuck_count;
2069        assert_eq!(
2070            sum_per_phase, 1,
2071            "the (s2,s3) boundary pair is in neither phase"
2072        );
2073        assert!(
2074            run_level > sum_per_phase,
2075            "run-level counts the boundary-straddling pair the per-phase partition drops"
2076        );
2077    }
2078
2079    #[test]
2080    fn compute_metrics_empty() {
2081        let m = compute_metrics(&[], 0);
2082        assert_eq!(m.sample_count, 0);
2083        // No samples -> no measurement, not a false 0.0 (the sentinel fix).
2084        assert_eq!(m.avg_imbalance, None);
2085        assert_eq!(m.max_imbalance, None);
2086        assert_eq!(m.avg_dsq_depth, None);
2087        assert_eq!(m.avg_nr_running, None);
2088        assert_eq!(m.max_dsq_depth, 0);
2089    }
2090
2091    #[test]
2092    fn stimulus_event_with_detail() {
2093        let e = StimulusEvent {
2094            elapsed_ms: 100,
2095            label: "StepStart[0]".to_string(),
2096            op_kind: Some("SetCpuset".to_string()),
2097            detail: Some("4 cpus".to_string()),
2098            total_iterations: None,
2099            step_index: None,
2100            is_terminal: false,
2101            is_step_end: false,
2102        };
2103        let events = vec![stimulus(0, "ScenarioStart"), e];
2104        let samples: Vec<MonitorSample> = (5..25)
2105            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2106            .collect();
2107        let t = Timeline::build(&events, &samples, 0);
2108        let formatted = t.format_with_context(&TimelineContext::default());
2109        assert!(formatted.contains("SetCpuset"));
2110        assert!(formatted.contains("4 cpus"));
2111    }
2112
2113    #[test]
2114    fn many_phases() {
2115        let events: Vec<StimulusEvent> = (0..10)
2116            .map(|i| stimulus(i * 500, &format!("Step[{i}]")))
2117            .collect();
2118        let samples: Vec<MonitorSample> = (5..55)
2119            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
2120            .collect();
2121        let t = Timeline::build(&events, &samples, 0);
2122        assert_eq!(t.phases.len(), 10);
2123    }
2124
2125    #[test]
2126    fn phase_metrics_accuracy() {
2127        let s1 = sample(600, vec![(1, 3, 100), (4, 5, 200)]); // ratio=4, avg_dsq=4
2128        let s2 = sample(700, vec![(2, 1, 300), (2, 7, 400)]); // ratio=1, avg_dsq=4
2129        let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2130        let m = compute_metrics(&refs, 0);
2131        assert_eq!(m.sample_count, 2);
2132        assert!((m.avg_imbalance.unwrap() - 2.5).abs() < 0.01); // (4+1)/2
2133        // nr_running per CPU: s1=(1,4)->per-sample mean 2.5, s2=(2,2)->mean 2.0;
2134        // averaged over the 2 samples = (2.5+2.0)/2 = 2.25.
2135        assert!((m.avg_nr_running.unwrap() - 2.25).abs() < 0.01);
2136        assert!((m.max_imbalance.unwrap() - 4.0).abs() < 0.01);
2137        assert_eq!(m.max_dsq_depth, 7);
2138    }
2139
2140    // -- ChangeDirection Display tests --
2141
2142    #[test]
2143    fn change_direction_display() {
2144        assert_eq!(format!("{}", ChangeDirection::Improved), "IMPROVEMENT");
2145        assert_eq!(format!("{}", ChangeDirection::Degraded), "DEGRADATION");
2146    }
2147
2148    // -- compute_metrics with event counters --
2149
2150    #[test]
2151    fn compute_metrics_with_event_counters() {
2152        use crate::monitor::ScxEventCounters;
2153
2154        let s1 = MonitorSample {
2155            bpf_map_fields: Vec::new(),
2156            prog_stats: None,
2157            psi_irq: None,
2158            elapsed_ms: 600,
2159            cpus: vec![CpuSnapshot {
2160                nr_running: 2,
2161                local_dsq_depth: 1,
2162                rq_clock: 100,
2163                scx_nr_running: 0,
2164                scx_flags: 0,
2165                event_counters: Some(ScxEventCounters {
2166                    select_cpu_fallback: 10,
2167                    dispatch_keep_last: 5,
2168                    ..Default::default()
2169                }),
2170                schedstat: None,
2171                vcpu_cpu_time_ns: Some(1_000_000_000),
2172                vcpu_perf: None,
2173                avg_irq_util: None,
2174                sched_domains: None,
2175            }],
2176        };
2177        let s2 = MonitorSample {
2178            bpf_map_fields: Vec::new(),
2179            prog_stats: None,
2180            psi_irq: None,
2181            elapsed_ms: 1600,
2182            cpus: vec![CpuSnapshot {
2183                nr_running: 2,
2184                local_dsq_depth: 1,
2185                rq_clock: 200,
2186                scx_nr_running: 0,
2187                scx_flags: 0,
2188                event_counters: Some(ScxEventCounters {
2189                    select_cpu_fallback: 110,
2190                    dispatch_keep_last: 55,
2191                    ..Default::default()
2192                }),
2193                schedstat: None,
2194                vcpu_cpu_time_ns: Some(2_000_000_000),
2195                vcpu_perf: None,
2196                avg_irq_util: None,
2197                sched_domains: None,
2198            }],
2199        };
2200        let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2201        let m = compute_metrics(&refs, 0);
2202        // fallback delta: 100 over 1.0 delivered vCPU CPU-second.
2203        assert!((m.fallback_rate.unwrap() - 100.0).abs() < 0.01);
2204        // keep_last delta: 50 over 1.0 delivered vCPU CPU-second.
2205        assert!((m.keep_last_rate.unwrap() - 50.0).abs() < 0.01);
2206
2207        let s2_dilated = MonitorSample {
2208            elapsed_ms: 100_600,
2209            ..s2.clone()
2210        };
2211        let dilated_refs = vec![&s1, &s2_dilated];
2212        let dilated = compute_metrics(&dilated_refs, 0);
2213        assert_eq!(
2214            dilated.fallback_rate, m.fallback_rate,
2215            "100x host wall dilation must not change the vCPU-denominated rate",
2216        );
2217        assert_eq!(dilated.keep_last_rate, m.keep_last_rate);
2218    }
2219
2220    #[test]
2221    fn compute_metrics_no_event_counters() {
2222        let s1 = sample(600, vec![(2, 1, 100)]);
2223        let s2 = sample(700, vec![(2, 1, 200)]);
2224        let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2225        let m = compute_metrics(&refs, 0);
2226        assert!(m.fallback_rate.is_none());
2227        assert!(m.keep_last_rate.is_none());
2228    }
2229
2230    #[test]
2231    fn compute_metrics_counter_reset_clamps_rates_to_non_negative() {
2232        // A scheduler restart between samples resets event counters
2233        // to smaller (or zero) values. Raw `last - first` then
2234        // produces a negative delta, which would flow into
2235        // `fallback_rate = delta / vCPU CPU time` and report a negative
2236        // rate. The shared counter_delta helper clamps to 0.
2237        use crate::monitor::ScxEventCounters;
2238
2239        let s1 = MonitorSample {
2240            bpf_map_fields: Vec::new(),
2241            prog_stats: None,
2242            psi_irq: None,
2243            elapsed_ms: 0,
2244            cpus: vec![CpuSnapshot {
2245                nr_running: 2,
2246                local_dsq_depth: 1,
2247                rq_clock: 100,
2248                scx_nr_running: 0,
2249                scx_flags: 0,
2250                event_counters: Some(ScxEventCounters {
2251                    select_cpu_fallback: 1000,
2252                    dispatch_keep_last: 500,
2253                    ..Default::default()
2254                }),
2255                schedstat: None,
2256                vcpu_cpu_time_ns: Some(1_000_000_000),
2257                vcpu_perf: None,
2258                avg_irq_util: None,
2259                sched_domains: None,
2260            }],
2261        };
2262        let s2 = MonitorSample {
2263            bpf_map_fields: Vec::new(),
2264            prog_stats: None,
2265            psi_irq: None,
2266            elapsed_ms: 1000,
2267            cpus: vec![CpuSnapshot {
2268                nr_running: 2,
2269                local_dsq_depth: 1,
2270                rq_clock: 200,
2271                scx_nr_running: 0,
2272                scx_flags: 0,
2273                event_counters: Some(ScxEventCounters {
2274                    select_cpu_fallback: 5,
2275                    dispatch_keep_last: 2,
2276                    ..Default::default()
2277                }),
2278                schedstat: None,
2279                vcpu_cpu_time_ns: Some(2_000_000_000),
2280                vcpu_perf: None,
2281                avg_irq_util: None,
2282                sched_domains: None,
2283            }],
2284        };
2285        let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2286        let m = compute_metrics(&refs, 0);
2287        let fb = m.fallback_rate.expect("reset still produces Some rate");
2288        let kl = m.keep_last_rate.expect("reset still produces Some rate");
2289        assert!(
2290            fb >= 0.0,
2291            "reset must not produce negative fallback_rate, got {fb}"
2292        );
2293        assert!(
2294            kl >= 0.0,
2295            "reset must not produce negative keep_last_rate, got {kl}"
2296        );
2297    }
2298
2299    // -- format with stalls --
2300
2301    #[test]
2302    fn format_with_stalls_shown() {
2303        let events = vec![stimulus(0, "ScenarioStart")];
2304        let samples = vec![
2305            sample(600, vec![(1, 0, 5000), (1, 0, 6000)]),
2306            sample(700, vec![(1, 0, 5000), (1, 0, 7000)]), // cpu0 stalled
2307        ];
2308        let t = Timeline::build(&events, &samples, 0);
2309        let formatted = t.format_with_context(&TimelineContext::default());
2310        assert!(formatted.contains("stuck: 1"));
2311    }
2312
2313    // -- format with no samples in a phase --
2314
2315    #[test]
2316    fn format_phase_no_samples() {
2317        // Create a phase with no samples by making a phase boundary far
2318        // beyond the last monitor sample's time.
2319        let events = vec![
2320            stimulus(0, "ScenarioStart"),
2321            stimulus(100, "StepStart[0]"),
2322            stimulus(50000, "StepStart[1]"),
2323        ];
2324        // All samples are in the middle phase window.
2325        let samples: Vec<MonitorSample> = (5..15)
2326            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
2327            .collect();
2328        let t = Timeline::build(&events, &samples, 0);
2329        let formatted = t.format_with_context(&TimelineContext::default());
2330        // The last phase (50000+offset to end) should have no samples.
2331        assert!(formatted.contains("[no samples]"));
2332    }
2333
2334    // -- timeline with fallback rate change detection --
2335
2336    #[test]
2337    fn fallback_rate_degradation_detected() {
2338        use crate::monitor::ScxEventCounters;
2339
2340        let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
2341        let mut samples = Vec::new();
2342        // Phase 0: zero fallback rate (counter stays constant).
2343        for i in 5..15 {
2344            samples.push(MonitorSample {
2345                bpf_map_fields: Vec::new(),
2346                prog_stats: None,
2347                psi_irq: None,
2348                elapsed_ms: i * 100,
2349                cpus: vec![CpuSnapshot {
2350                    nr_running: 2,
2351                    local_dsq_depth: 1,
2352                    rq_clock: i * 1000,
2353                    scx_nr_running: 0,
2354                    scx_flags: 0,
2355                    event_counters: Some(ScxEventCounters {
2356                        select_cpu_fallback: 0,
2357                        dispatch_keep_last: 0,
2358                        ..Default::default()
2359                    }),
2360                    schedstat: None,
2361                    vcpu_cpu_time_ns: Some(i * 100_000_000),
2362                    vcpu_perf: None,
2363                    avg_irq_util: None,
2364                    sched_domains: None,
2365                }],
2366            });
2367        }
2368        // Phase 1: very high fallback activity per delivered vCPU CPU-second.
2369        for i in 15..25 {
2370            samples.push(MonitorSample {
2371                bpf_map_fields: Vec::new(),
2372                prog_stats: None,
2373                psi_irq: None,
2374                elapsed_ms: i * 100,
2375                cpus: vec![CpuSnapshot {
2376                    nr_running: 2,
2377                    local_dsq_depth: 1,
2378                    rq_clock: i * 1000,
2379                    scx_nr_running: 0,
2380                    scx_flags: 0,
2381                    event_counters: Some(ScxEventCounters {
2382                        select_cpu_fallback: (i as i64 - 15) * 50,
2383                        dispatch_keep_last: 0,
2384                        ..Default::default()
2385                    }),
2386                    schedstat: None,
2387                    vcpu_cpu_time_ns: Some(i * 100_000_000),
2388                    vcpu_perf: None,
2389                    avg_irq_util: None,
2390                    sched_domains: None,
2391                }],
2392            });
2393        }
2394        let t = Timeline::build(&events, &samples, 0);
2395        let degs: Vec<_> = t
2396            .degradations()
2397            .into_iter()
2398            .filter(|(_, c)| c.metric == "fallback")
2399            .collect();
2400        assert!(!degs.is_empty());
2401    }
2402
2403    // -- format_with_context tests --
2404
2405    #[test]
2406    fn format_with_context_includes_header() {
2407        let events = vec![stimulus(0, "ScenarioStart")];
2408        let samples = vec![
2409            sample(600, vec![(2, 1, 100), (2, 1, 200)]),
2410            sample(700, vec![(2, 1, 300), (2, 1, 400)]),
2411        ];
2412        let t = Timeline::build(&events, &samples, 0);
2413        let ctx = TimelineContext {
2414            kernel: Some("6.14.0-rc3+".to_string()),
2415            topology: Some("2n4l4c2t (16 cpus)".to_string()),
2416            scheduler: Some("scx_mitosis".to_string()),
2417            scenario: Some("proportional".to_string()),
2418            duration_s: Some(20.5),
2419        };
2420        let formatted = t.format_with_context(&ctx);
2421        assert!(formatted.contains("--- timeline ---"));
2422        assert!(formatted.contains("kernel: 6.14.0-rc3+"));
2423        assert!(formatted.contains("topology: 2n4l4c2t (16 cpus)"));
2424        assert!(formatted.contains("scheduler: scx_mitosis"));
2425        assert!(formatted.contains("scenario: proportional"));
2426        assert!(formatted.contains("duration: 20.5s"));
2427        assert!(formatted.contains("BASELINE"));
2428    }
2429
2430    #[test]
2431    fn format_with_context_partial_fields() {
2432        let events = vec![stimulus(0, "ScenarioStart")];
2433        let samples = vec![sample(600, vec![(2, 1, 100)])];
2434        let t = Timeline::build(&events, &samples, 0);
2435        let ctx = TimelineContext {
2436            kernel: None,
2437            topology: Some("1n1l1c1t (1 cpus)".to_string()),
2438            scheduler: None,
2439            scenario: Some("basic".to_string()),
2440            duration_s: None,
2441        };
2442        let formatted = t.format_with_context(&ctx);
2443        assert!(formatted.contains("topology: 1n1l1c1t"));
2444        assert!(formatted.contains("scenario: basic"));
2445        assert!(!formatted.contains("kernel:"));
2446        assert!(!formatted.contains("scheduler:"));
2447        assert!(!formatted.contains("duration:"));
2448    }
2449
2450    #[test]
2451    fn format_with_context_empty_timeline() {
2452        let t = Timeline { phases: vec![] };
2453        let ctx = TimelineContext {
2454            kernel: Some("6.14.0".to_string()),
2455            ..Default::default()
2456        };
2457        assert!(t.format_with_context(&ctx).is_empty());
2458    }
2459
2460    #[test]
2461    fn format_with_context_empty_context() {
2462        let events = vec![stimulus(0, "ScenarioStart")];
2463        let samples = vec![sample(600, vec![(2, 1, 100)])];
2464        let t = Timeline::build(&events, &samples, 0);
2465        let ctx = TimelineContext::default();
2466        let formatted = t.format_with_context(&ctx);
2467        // Should have the timeline header and phases but no context line.
2468        assert!(formatted.contains("--- timeline ---"));
2469        assert!(formatted.contains("BASELINE"));
2470        // The line after "--- timeline ---\n" should be "\nBASELINE" (no context line).
2471        let after_header = &formatted["--- timeline ---\n".len()..];
2472        assert!(after_header.starts_with('\n'));
2473    }
2474
2475    #[test]
2476    fn garbage_dsq_samples_filtered_from_metrics() {
2477        // Samples with DSQ depth above DSQ_PLAUSIBILITY_CEILING should be
2478        // excluded from phase metrics (the bug: garbage values like 1.5B
2479        // were flowing into timeline output).
2480        let events = vec![stimulus(0, "ScenarioStart")];
2481        let garbage_dsq = 1_550_435_906u32;
2482        let samples = vec![
2483            // Garbage sample (DSQ above ceiling).
2484            MonitorSample {
2485                bpf_map_fields: Vec::new(),
2486                prog_stats: None,
2487                psi_irq: None,
2488                elapsed_ms: 600,
2489                cpus: vec![CpuSnapshot {
2490                    nr_running: 1,
2491                    local_dsq_depth: garbage_dsq,
2492                    rq_clock: 1000,
2493                    ..Default::default()
2494                }],
2495            },
2496            // Valid sample.
2497            sample(700, vec![(2, 3, 2000)]),
2498        ];
2499        let t = Timeline::build(&events, &samples, 0);
2500        assert_eq!(t.phases.len(), 1);
2501        // Only the valid sample should be counted.
2502        assert_eq!(t.phases[0].metrics.sample_count, 1);
2503        assert_eq!(t.phases[0].metrics.max_dsq_depth, 3);
2504    }
2505
2506    #[test]
2507    fn all_garbage_samples_yield_no_metrics() {
2508        let events = vec![stimulus(0, "ScenarioStart")];
2509        let samples = vec![MonitorSample {
2510            bpf_map_fields: Vec::new(),
2511            prog_stats: None,
2512            psi_irq: None,
2513            elapsed_ms: 600,
2514            cpus: vec![CpuSnapshot {
2515                nr_running: 1,
2516                local_dsq_depth: 50_000,
2517                rq_clock: 1000,
2518                ..Default::default()
2519            }],
2520        }];
2521        let t = Timeline::build(&events, &samples, 0);
2522        assert_eq!(t.phases[0].metrics.sample_count, 0);
2523    }
2524
2525    // ---------------------------------------------------------------
2526    // Negative test: timeline detects degradation at phase transition
2527    // ---------------------------------------------------------------
2528
2529    #[test]
2530    fn neg_timeline_detects_imbalance_degradation() {
2531        let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2532        let mut samples = Vec::new();
2533        for i in 6..25 {
2534            samples.push(sample(
2535                i * 100,
2536                vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2537            ));
2538        }
2539        for i in 26..45 {
2540            samples.push(sample(
2541                i * 100,
2542                vec![(1, 1, i * 1000), (10, 1, i * 1000 + 100)],
2543            ));
2544        }
2545        let t = Timeline::build(&events, &samples, 0);
2546        assert_eq!(t.phases.len(), 2, "must have 2 phases");
2547        assert!(!t.degradations().is_empty());
2548
2549        // Phase 0 (baseline) must have samples and reasonable metrics.
2550        assert!(
2551            t.phases[0].metrics.sample_count > 0,
2552            "baseline must have samples"
2553        );
2554        assert!(
2555            (t.phases[0].metrics.avg_imbalance.unwrap() - 1.0).abs() < 0.5,
2556            "baseline imbalance should be ~1.0, got {:?}",
2557            t.phases[0].metrics.avg_imbalance,
2558        );
2559
2560        // Phase 1 must have the stimulus label and degradation.
2561        assert!(
2562            t.phases[1].metrics.sample_count > 0,
2563            "phase 1 must have samples"
2564        );
2565        assert!(
2566            t.phases[1]
2567                .stimulus
2568                .as_ref()
2569                .is_some_and(|s| s.label == "StepStart[0]"),
2570            "phase 1 stimulus must be StepStart[0]",
2571        );
2572
2573        let degs = t.degradations();
2574        assert!(!degs.is_empty());
2575        let (phase, change) = &degs[0];
2576        assert_eq!(phase.index, 1);
2577        assert_eq!(change.metric, "imbalance");
2578        assert_eq!(change.direction, ChangeDirection::Degraded);
2579        let delta = change.after - change.before;
2580        assert!(delta > 0.0, "delta must be positive for degradation");
2581        assert!(
2582            delta > IMBALANCE_THRESHOLD,
2583            "delta {:.1} must exceed threshold {:.1}",
2584            delta,
2585            IMBALANCE_THRESHOLD
2586        );
2587        assert!(
2588            change.before < 2.0,
2589            "before should be low: {:.1}",
2590            change.before
2591        );
2592        assert!(
2593            change.after > 5.0,
2594            "after should be high: {:.1}",
2595            change.after
2596        );
2597
2598        // Format output must be parseable.
2599        let formatted = t.format_with_context(&TimelineContext::default());
2600        assert!(
2601            formatted.contains("BASELINE"),
2602            "format must include BASELINE phase"
2603        );
2604        assert!(formatted.contains("Phase 1"), "format must include Phase 1");
2605        assert!(
2606            formatted.contains("DEGRADATION"),
2607            "format must include DEGRADATION label"
2608        );
2609        assert!(
2610            formatted.contains("imbalance"),
2611            "format must name the metric"
2612        );
2613    }
2614
2615    #[test]
2616    fn neg_timeline_detects_dsq_depth_degradation() {
2617        let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2618        let mut samples = Vec::new();
2619        for i in 6..25 {
2620            samples.push(sample(
2621                i * 100,
2622                vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2623            ));
2624        }
2625        for i in 26..45 {
2626            samples.push(sample(
2627                i * 100,
2628                vec![(2, 20, i * 1000), (2, 20, i * 1000 + 100)],
2629            ));
2630        }
2631        let t = Timeline::build(&events, &samples, 0);
2632        assert!(
2633            !t.degradations().is_empty(),
2634            "DSQ depth jump must be detected"
2635        );
2636        let degs = t.degradations();
2637        let dsq_deg = degs.iter().find(|(_, c)| c.metric == "dsq_depth");
2638        assert!(dsq_deg.is_some(), "must detect dsq_depth degradation");
2639        let (phase, change) = dsq_deg.unwrap();
2640        assert_eq!(phase.index, 1);
2641        assert_eq!(change.direction, ChangeDirection::Degraded);
2642        let delta = change.after - change.before;
2643        assert!(
2644            delta > DSQ_THRESHOLD,
2645            "dsq delta {:.1} must exceed threshold {:.1}",
2646            delta,
2647            DSQ_THRESHOLD
2648        );
2649        assert!(
2650            change.before < 5.0,
2651            "before dsq should be low: {:.1}",
2652            change.before
2653        );
2654        assert!(
2655            change.after > 15.0,
2656            "after dsq should be high: {:.1}",
2657            change.after
2658        );
2659
2660        let formatted = t.format_with_context(&TimelineContext::default());
2661        assert!(
2662            formatted.contains("dsq_depth"),
2663            "format must name dsq_depth"
2664        );
2665        assert!(
2666            formatted.contains("DEGRADATION"),
2667            "format must label degradation"
2668        );
2669    }
2670
2671    #[test]
2672    fn neg_timeline_no_degradation_when_stable() {
2673        let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2674        let mut samples = Vec::new();
2675        for i in 6..45 {
2676            samples.push(sample(
2677                i * 100,
2678                vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2679            ));
2680        }
2681        let t = Timeline::build(&events, &samples, 0);
2682        assert_eq!(t.phases.len(), 2, "must have 2 phases");
2683        assert!(t.phases[0].metrics.sample_count > 0);
2684        assert!(t.phases[1].metrics.sample_count > 0);
2685        assert!(
2686            t.degradations().is_empty(),
2687            "stable phases must not show degradation"
2688        );
2689        assert!(t.degradations().is_empty());
2690        // All phase changes should be empty.
2691        for phase in &t.phases {
2692            assert!(
2693                phase.changes.is_empty(),
2694                "phase {} should have no changes",
2695                phase.index
2696            );
2697        }
2698    }
2699
2700    // -- detect_change direct tests --
2701
2702    #[test]
2703    fn detect_change_higher_is_worse_positive_delta_degraded() {
2704        let c = detect_change(1.0, 5.0, 0.5, "imbalance", true).unwrap();
2705        assert_eq!(c.direction, ChangeDirection::Degraded);
2706        assert_eq!(c.metric, "imbalance");
2707        assert!((c.before - 1.0).abs() < f64::EPSILON);
2708        assert!((c.after - 5.0).abs() < f64::EPSILON);
2709    }
2710
2711    #[test]
2712    fn detect_change_higher_is_worse_negative_delta_improved() {
2713        let c = detect_change(5.0, 1.0, 0.5, "imbalance", true).unwrap();
2714        assert_eq!(c.direction, ChangeDirection::Improved);
2715    }
2716
2717    #[test]
2718    fn detect_change_lower_is_worse_negative_delta_degraded() {
2719        let c = detect_change(100.0, 50.0, 10.0, "throughput", false).unwrap();
2720        assert_eq!(c.direction, ChangeDirection::Degraded);
2721    }
2722
2723    #[test]
2724    fn detect_change_lower_is_worse_positive_delta_improved() {
2725        let c = detect_change(50.0, 100.0, 10.0, "throughput", false).unwrap();
2726        assert_eq!(c.direction, ChangeDirection::Improved);
2727    }
2728
2729    #[test]
2730    fn detect_change_below_threshold_returns_none() {
2731        assert!(detect_change(1.0, 1.3, 0.5, "imbalance", true).is_none());
2732    }
2733
2734    #[test]
2735    fn detect_change_exactly_at_threshold_returns_none() {
2736        assert!(detect_change(1.0, 1.5, 0.5, "imbalance", true).is_none());
2737    }
2738
2739    // -- detect_boundary_changes: throughput across synthesized steps --
2740
2741    /// Iteration efficiency is flagged across a synthesized zero-capture
2742    /// phase (sample_count 0) when a guest carrier supplies a real rate,
2743    /// while monitor-derived metrics stay gated on samples, so
2744    /// the synthesized side never paints a phantom imbalance change.
2745    /// This is the collapse-suppression invariant: a throughput collapse entering or
2746    /// leaving a capture-free step must not be silently dropped.
2747    #[test]
2748    fn detect_boundary_changes_flags_throughput_across_synthesized_phase() {
2749        let before = PhaseMetrics {
2750            sample_count: 30,
2751            iteration_rate: Some(1000.0),
2752            avg_imbalance: Some(1.0),
2753            ..Default::default()
2754        };
2755        let after = PhaseMetrics {
2756            sample_count: 0,             // synthesized zero-capture step
2757            iteration_rate: Some(300.0), // 70% collapse
2758            avg_imbalance: Some(99.0),   // partial/default — must be ignored
2759            ..Default::default()
2760        };
2761        let changes = detect_boundary_changes(&before, &after);
2762        let throughput: Vec<_> = changes
2763            .iter()
2764            .filter(|c| c.metric == "throughput")
2765            .collect();
2766        assert_eq!(
2767            throughput.len(),
2768            1,
2769            "throughput collapse across a synthesized step must be flagged: {changes:?}",
2770        );
2771        assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
2772        assert!(
2773            !changes.iter().any(|c| c.metric == "imbalance"),
2774            "monitor metrics must stay gated when a side has 0 samples: {changes:?}",
2775        );
2776    }
2777
2778    /// The monitor-metric gate only suppresses a ZERO-sample side: when
2779    /// both phases captured samples, monitor-derived changes still
2780    /// surface. Guards the collapse suppression from over-suppressing the normal
2781    /// captured-to-captured boundary.
2782    #[test]
2783    fn detect_boundary_changes_reports_monitor_metrics_when_both_sampled() {
2784        let before = PhaseMetrics {
2785            sample_count: 30,
2786            iteration_rate: Some(1000.0),
2787            avg_imbalance: Some(1.0),
2788            ..Default::default()
2789        };
2790        let after = PhaseMetrics {
2791            sample_count: 30,
2792            iteration_rate: Some(1000.0), // unchanged throughput
2793            avg_imbalance: Some(5.0),     // imbalance jump > IMBALANCE_THRESHOLD
2794            ..Default::default()
2795        };
2796        let changes = detect_boundary_changes(&before, &after);
2797        assert!(
2798            changes.iter().any(|c| c.metric == "imbalance"),
2799            "imbalance change must surface when both sides sampled: {changes:?}",
2800        );
2801        assert!(
2802            !changes.iter().any(|c| c.metric == "throughput"),
2803            "unchanged throughput must not be flagged: {changes:?}",
2804        );
2805    }
2806
2807    /// A per-phase avg_nr_running (mean runqueue depth) rise above
2808    /// NR_RUNNING_THRESHOLD between two sampled phases is flagged as a
2809    /// degradation (higher runqueue depth = more contention). Gated on both
2810    /// phases having real samples, like the other monitor-derived gauges.
2811    #[test]
2812    fn detect_boundary_changes_flags_nr_running_jump() {
2813        let before = PhaseMetrics {
2814            sample_count: 30,
2815            avg_nr_running: Some(1.0),
2816            ..Default::default()
2817        };
2818        let after = PhaseMetrics {
2819            sample_count: 30,
2820            avg_nr_running: Some(3.0), // +2.0 > NR_RUNNING_THRESHOLD (1.0)
2821            ..Default::default()
2822        };
2823        let changes = detect_boundary_changes(&before, &after);
2824        let nr: Vec<_> = changes
2825            .iter()
2826            .filter(|c| c.metric == "nr_running")
2827            .collect();
2828        assert_eq!(
2829            nr.len(),
2830            1,
2831            "avg_nr_running jump must be flagged: {changes:?}"
2832        );
2833        assert_eq!(
2834            nr[0].direction,
2835            ChangeDirection::Degraded,
2836            "rising runqueue depth is a degradation",
2837        );
2838        // A zero-sample side suppresses it (same gate as the sibling gauges).
2839        let after_synth = PhaseMetrics {
2840            sample_count: 0,
2841            avg_nr_running: Some(3.0),
2842            ..Default::default()
2843        };
2844        assert!(
2845            !detect_boundary_changes(&before, &after_synth)
2846                .iter()
2847                .any(|c| c.metric == "nr_running"),
2848            "nr_running must stay gated when a side has 0 samples",
2849        );
2850    }
2851
2852    /// The per-phase timeline narrative renders avg_nr_running (`nr_run: avg=`)
2853    /// alongside imbalance + dsq for a sampled phase.
2854    #[test]
2855    fn format_phases_renders_avg_nr_running() {
2856        // Two CPUs with nr_running 2 and 4 -> per-sample mean 3.0.
2857        let samples: Vec<MonitorSample> = (5..25)
2858            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (4, 1, i * 1000 + 100)]))
2859            .collect();
2860        let events = vec![stimulus(0, "ScenarioStart")];
2861        let t = Timeline::build(&events, &samples, 0);
2862        let formatted = t.format_with_context(&TimelineContext::default());
2863        assert!(
2864            formatted.contains("nr_run: avg="),
2865            "per-phase narrative must render avg_nr_running: {formatted}",
2866        );
2867    }
2868
2869    /// Throughput uses a STRICT relative threshold: a change of exactly
2870    /// ITERATION_RATE_REL_THRESHOLD (30%) is NOT flagged. Pins the `>`
2871    /// boundary so a future `>` -> `>=` flip is caught (the absolute
2872    /// detect_change gate is a different code path, tested separately).
2873    #[test]
2874    fn detect_boundary_changes_throughput_exactly_at_threshold_not_flagged() {
2875        let before = PhaseMetrics {
2876            sample_count: 30,
2877            iteration_rate: Some(1000.0),
2878            ..Default::default()
2879        };
2880        let after = PhaseMetrics {
2881            sample_count: 30,
2882            iteration_rate: Some(700.0), // rel = -0.3 exactly
2883            ..Default::default()
2884        };
2885        assert_eq!((700.0 - 1000.0) / 1000.0, -ITERATION_RATE_REL_THRESHOLD);
2886        assert!(
2887            !detect_boundary_changes(&before, &after)
2888                .iter()
2889                .any(|c| c.metric == "throughput"),
2890            "an exactly-30% relative change must not flag (strict >)",
2891        );
2892    }
2893
2894    /// synthesized -> synthesized boundary (BOTH sides sample_count 0,
2895    /// both carrying a real guest-carrier-derived rate): throughput is still
2896    /// compared (the gate is symmetric and sample_count-independent for
2897    /// throughput) and monitor metrics stay suppressed on both sides.
2898    /// Pins the zero->zero cell of the transition matrix against a future
2899    /// edit that re-couples throughput to a per-side sample_count check.
2900    #[test]
2901    fn detect_boundary_changes_synthesized_to_synthesized_flags_throughput() {
2902        let before = PhaseMetrics {
2903            sample_count: 0,
2904            iteration_rate: Some(1000.0),
2905            avg_imbalance: Some(1.0),
2906            ..Default::default()
2907        };
2908        let after = PhaseMetrics {
2909            sample_count: 0,
2910            iteration_rate: Some(300.0), // 70% collapse
2911            avg_imbalance: Some(99.0),   // wild — must stay gated
2912            ..Default::default()
2913        };
2914        let changes = detect_boundary_changes(&before, &after);
2915        let throughput: Vec<_> = changes
2916            .iter()
2917            .filter(|c| c.metric == "throughput")
2918            .collect();
2919        assert_eq!(
2920            throughput.len(),
2921            1,
2922            "zero->zero throughput collapse must flag: {changes:?}"
2923        );
2924        assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
2925        assert!(
2926            !changes.iter().any(|c| c.metric == "imbalance"),
2927            "monitor metrics gated on both zero-sample sides: {changes:?}",
2928        );
2929    }
2930
2931    // -- iteration_rate computation tests --
2932
2933    fn stimulus_with_iters(elapsed_ms: u64, label: &str, total_iterations: u64) -> StimulusEvent {
2934        StimulusEvent {
2935            elapsed_ms,
2936            label: label.to_string(),
2937            op_kind: None,
2938            detail: None,
2939            total_iterations: Some(total_iterations),
2940            step_index: None,
2941            is_terminal: false,
2942            is_step_end: false,
2943        }
2944    }
2945
2946    #[test]
2947    fn consecutive_wall_events_do_not_create_iteration_rate() {
2948        let events = vec![
2949            stimulus_with_iters(0, "ScenarioStart", 0),
2950            stimulus_with_iters(3000, "StepStart[0]", 3000),
2951        ];
2952        let samples: Vec<MonitorSample> = (5..35)
2953            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2954            .collect();
2955        let t = Timeline::build(&events, &samples, 0);
2956        assert_eq!(t.phases.len(), 2);
2957        let rate = t.phases[0].metrics.iteration_rate;
2958        assert!(
2959            rate.is_none(),
2960            "wall-only stimulus events must not produce iteration_rate"
2961        );
2962    }
2963
2964    #[test]
2965    fn wall_events_without_iterations_do_not_create_iteration_rate() {
2966        // Wall events never create the CPU-denominated iteration rate.
2967        let events = vec![stimulus(0, "ScenarioStart"), stimulus(3000, "StepStart[0]")];
2968        let samples: Vec<MonitorSample> = (5..35)
2969            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2970            .collect();
2971        let t = Timeline::build(&events, &samples, 0);
2972        assert!(t.phases[0].metrics.iteration_rate.is_none());
2973        assert!(t.phases[1].metrics.iteration_rate.is_none());
2974    }
2975
2976    /// Build a wire `StimulusEvent` so tests can drive the FULL
2977    /// `from_wire` path (the production conversion) rather than
2978    /// constructing the timeline event directly — the latter bypassed
2979    /// the `total_iterations == 0` sentinel.
2980    fn wire_event(
2981        elapsed_ms: u32,
2982        step_index: u16,
2983        total_iterations: u64,
2984    ) -> crate::vmm::wire::StimulusEvent {
2985        crate::vmm::wire::StimulusEvent {
2986            elapsed_ms,
2987            step_index,
2988            op_count: 0,
2989            op_kinds: 0,
2990            cgroup_count: 0,
2991            worker_count: 1,
2992            total_iterations,
2993        }
2994    }
2995
2996    #[test]
2997    fn from_wire_zero_iterations_is_some_baseline() {
2998        // total_iterations is a cumulative counter, so a
2999        // start-of-window 0 is a legitimate baseline, NOT a missing
3000        // sample. from_wire must carry Some(0), never collapse it to
3001        // None.
3002        let te = StimulusEvent::from_wire(&wire_event(0, 1, 0));
3003        assert_eq!(te.total_iterations, Some(0));
3004        assert_eq!(te.step_index, Some(1));
3005        assert!(!te.is_terminal);
3006        assert!(
3007            !te.is_step_end,
3008            "a StepStart-derived event is not a StepEnd"
3009        );
3010    }
3011
3012    #[test]
3013    fn from_step_end_carries_step_index_and_marks_step_end() {
3014        // A StepEnd frame reuses the StimulusEvent wire body.
3015        // from_step_end must carry the same 1-indexed step_index and the
3016        // step's end-of-hold total_iterations, flag is_step_end, and leave
3017        // is_terminal off (it is a real per-step boundary, not the
3018        // scenario terminal).
3019        let te = StimulusEvent::from_step_end(&wire_event(1_900, 1, 9_000));
3020        assert_eq!(te.step_index, Some(1));
3021        assert_eq!(te.total_iterations, Some(9_000));
3022        assert!(te.is_step_end, "StepEnd-derived event must set is_step_end");
3023        assert!(
3024            !te.is_terminal,
3025            "StepEnd is a per-step boundary, not the scenario terminal",
3026        );
3027    }
3028
3029    #[test]
3030    fn from_wire_first_step_zero_baseline_yields_rate() {
3031        // First-step zero-baseline regression, driven through the FULL from_wire path
3032        // (unit tests previously injected Some(0) directly, masking the
3033        // wire 0->None collapse). First step frame reads 0 cumulative
3034        // iterations, the second reads 3000; the first phase must get a
3035        // rate rather than a silent None.
3036        let events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(3000, 2, 3000)]
3037            .iter()
3038            .map(StimulusEvent::from_wire)
3039            .collect();
3040        let samples: Vec<MonitorSample> = (5..35)
3041            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3042            .collect();
3043        let t = Timeline::build(&events, &samples, 0);
3044        assert!(t.phases[0].metrics.iteration_rate.is_none());
3045    }
3046
3047    #[test]
3048    fn terminal_event_gives_last_step_rate_without_phantom_phase() {
3049        // The last step has no successor step event, so its
3050        // iteration_rate needs the terminal scenario-end boundary. The
3051        // terminal must supply that boundary WITHOUT adding a phantom
3052        // trailing phase.
3053        let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3054            .iter()
3055            .map(StimulusEvent::from_wire)
3056            .collect();
3057        events.push(StimulusEvent::terminal(4000, 10000));
3058        let samples: Vec<MonitorSample> = (5..45)
3059            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3060            .collect();
3061        let t = Timeline::build(&events, &samples, 0);
3062        assert_eq!(
3063            t.phases.len(),
3064            2,
3065            "two step events -> two phases; terminal seeds none",
3066        );
3067        assert!(t.phases[1].metrics.iteration_rate.is_none());
3068    }
3069
3070    #[test]
3071    fn build_filters_step_end_events_no_phantom_phase() {
3072        // A StepEnd must be filtered from the PHASE-LAYOUT set
3073        // (it is an end-of-hold marker, not a step boundary) so it neither
3074        // adds a phantom phase nor misaligns the dense phase index. Two
3075        // StepStart events with an interleaved StepEnd still yield exactly
3076        // two phases. (StepEnd is still consumed for the step-local RATE —
3077        // see build_pairs_step_local_when_step_end_events_present.)
3078        let events: Vec<StimulusEvent> = vec![
3079            StimulusEvent::from_wire(&wire_event(0, 1, 0)),
3080            StimulusEvent::from_step_end(&wire_event(1_900, 1, 9_000)),
3081            StimulusEvent::from_wire(&wire_event(2_000, 2, 9_000)),
3082        ];
3083        let samples: Vec<MonitorSample> = (5..35)
3084            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3085            .collect();
3086        let t = Timeline::build(&events, &samples, 0);
3087        assert_eq!(
3088            t.phases.len(),
3089            2,
3090            "two StepStart events -> two phases; the interleaved StepEnd seeds none",
3091        );
3092    }
3093
3094    #[test]
3095    fn build_pairs_step_local_when_step_end_events_present() {
3096        // The monitor-only Timeline::build fallback must ALSO
3097        // use step-local StepStart[k] -> StepEnd[k] pairing when StepEnd
3098        // events are present (they are emitted independent of snapshot
3099        // captures), NOT the cross-step StepStart[k] -> StepStart[k+1]
3100        // pairing that reads 0 -> 0 for respawned-per-step workers. Two
3101        // fresh-per-step steps (each StepStart reads ~0); without
3102        // step-local pairing phase 0 would be None (0 -> 0 cross-step).
3103        // With it, both phases get a positive rate.
3104        let events: Vec<StimulusEvent> = vec![
3105            StimulusEvent::from_wire(&wire_event(0, 1, 0)), // StepStart[0], iters 0
3106            StimulusEvent::from_step_end(&wire_event(1_000, 1, 5_000)), // StepEnd[0], iters 5000
3107            StimulusEvent::from_wire(&wire_event(1_100, 2, 0)), // StepStart[1] respawned, iters 0
3108            StimulusEvent::from_step_end(&wire_event(2_100, 2, 3_000)), // StepEnd[1], iters 3000
3109        ];
3110        let samples: Vec<MonitorSample> = (1..30)
3111            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3112            .collect();
3113        let t = Timeline::build(&events, &samples, 0);
3114        assert_eq!(
3115            t.phases.len(),
3116            2,
3117            "two StepStart events -> two phases (each StepEnd seeds none)",
3118        );
3119        assert!(t.phases[0].metrics.iteration_rate.is_none());
3120        assert!(t.phases[1].metrics.iteration_rate.is_none());
3121    }
3122
3123    #[test]
3124    fn build_stalled_step_with_step_end_reports_measured_zero_not_cross_step() {
3125        // Monitor-only path: a step that HAS a StepEnd but
3126        // stalled (StepEnd[k] == StepStart[k]) reports its MEASURED-ZERO
3127        // step-local rate (Some(0.0)) — its StepEnd lookup hits, so the
3128        // cross-step fallback must NOT run. Mirrors the snapshot path's
3129        // build_phase_buckets_with_stimulus_stalled_step_reports_measured_zero.
3130        // Step 0 stalls (0 -> 0); a persistent population reads 500 at
3131        // StepStart[1], so a cross-step StepStart[0] -> StepStart[1] leak
3132        // would be ~454/s. Step 1 advances 500 -> 5500 (5000/s).
3133        let events: Vec<StimulusEvent> = vec![
3134            StimulusEvent::from_wire(&wire_event(0, 1, 0)), // StepStart[0], iters 0
3135            StimulusEvent::from_step_end(&wire_event(1_000, 1, 0)), // StepEnd[0], STALLED 0
3136            StimulusEvent::from_wire(&wire_event(1_100, 2, 500)), // StepStart[1], persistent 500
3137            StimulusEvent::from_step_end(&wire_event(2_100, 2, 5_500)), // StepEnd[1], iters 5500
3138        ];
3139        let samples: Vec<MonitorSample> = (1..30)
3140            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3141            .collect();
3142        let t = Timeline::build(&events, &samples, 0);
3143        assert_eq!(t.phases.len(), 2);
3144        assert_eq!(
3145            t.phases[0].metrics.iteration_rate, None,
3146            "wall-only events do not produce the delivered-CPU rate",
3147        );
3148        assert!(t.phases[1].metrics.iteration_rate.is_none());
3149    }
3150
3151    #[test]
3152    fn terminal_event_single_step_rate() {
3153        // Boundary case: a one-step scenario (first == last). With the
3154        // 0 baseline and the terminal boundary supplying the right edge,
3155        // the single step still gets a rate, and the terminal adds no
3156        // phase.
3157        let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0)]
3158            .iter()
3159            .map(StimulusEvent::from_wire)
3160            .collect();
3161        events.push(StimulusEvent::terminal(3000, 9000));
3162        let samples: Vec<MonitorSample> = (5..35)
3163            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3164            .collect();
3165        let t = Timeline::build(&events, &samples, 0);
3166        assert_eq!(
3167            t.phases.len(),
3168            1,
3169            "single step -> one phase; terminal adds none"
3170        );
3171        assert!(t.phases[0].metrics.iteration_rate.is_none());
3172    }
3173
3174    #[test]
3175    fn terminal_event_stalled_last_step_does_not_fabricate_cpu_rate() {
3176        let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3177            .iter()
3178            .map(StimulusEvent::from_wire)
3179            .collect();
3180        events.push(StimulusEvent::terminal(4000, 4000)); // no advance
3181        let samples: Vec<MonitorSample> = (5..45)
3182            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3183            .collect();
3184        let t = Timeline::build(&events, &samples, 0);
3185        assert_eq!(t.phases.len(), 2);
3186        assert_eq!(
3187            t.phases[1].metrics.iteration_rate, None,
3188            "wall-only terminal events do not produce iteration_rate",
3189        );
3190    }
3191
3192    #[test]
3193    fn iteration_rate_counter_decrease_yields_no_rate() {
3194        // A counter DECREASE between consecutive step frames (e.g. a
3195        // step-local worker population reset) is unmeasurable and must NOT
3196        // produce a negative or conflated rate — the `e < s` guard drops
3197        // the pair, returning None (distinct from `e == s`, which is a
3198        // measured-zero Some(0.0)). Pin it so a future change that loosens
3199        // the guard to allow a negative delta fails here.
3200        let events: Vec<StimulusEvent> = [
3201            wire_event(0, 1, 0),
3202            wire_event(2000, 2, 5000),
3203            wire_event(3000, 3, 1000), // counter dropped 5000 -> 1000
3204        ]
3205        .iter()
3206        .map(StimulusEvent::from_wire)
3207        .collect();
3208        let samples: Vec<MonitorSample> = (5..35)
3209            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3210            .collect();
3211        let t = Timeline::build(&events, &samples, 0);
3212        // phase 1 is step 2 (frame iters 5000 -> next 1000): decrease.
3213        assert!(
3214            t.phases[1].metrics.iteration_rate.is_none(),
3215            "a counter decrease must not manufacture a (negative) rate",
3216        );
3217    }
3218
3219    #[test]
3220    fn iteration_rate_zero_duration_yields_no_rate() {
3221        // Two consecutive frames with identical elapsed_ms -> the rate
3222        // denominator is 0; the duration==0 guard must drop the pair
3223        // rather than divide and produce inf/NaN.
3224        let events: Vec<StimulusEvent> = [wire_event(1000, 1, 0), wire_event(1000, 2, 2000)]
3225            .iter()
3226            .map(StimulusEvent::from_wire)
3227            .collect();
3228        let samples: Vec<MonitorSample> = (5..35)
3229            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3230            .collect();
3231        let t = Timeline::build(&events, &samples, 0);
3232        assert!(
3233            t.phases[0].metrics.iteration_rate.is_none(),
3234            "zero-duration pair must not divide; rate stays None",
3235        );
3236    }
3237
3238    #[test]
3239    fn terminal_not_last_does_not_misalign_or_misattribute() {
3240        // Robustness: even if a corrupt/out-of-order elapsed_ms made the
3241        // terminal sort BEFORE a real step, the explicit is_terminal
3242        // extraction (not positional) must keep the step phases aligned
3243        // and attribute the early step's rate correctly. A corrupt
3244        // terminal contributes no spurious rate (its position can't
3245        // shift the dense phase index).
3246        let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3247            .iter()
3248            .map(StimulusEvent::from_wire)
3249            .collect();
3250        // Terminal with elapsed_ms BEFORE step 2 (simulated corruption).
3251        events.push(StimulusEvent::terminal(500, 9000));
3252        let samples: Vec<MonitorSample> = (5..45)
3253            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3254            .collect();
3255        let t = Timeline::build(&events, &samples, 0);
3256        // Two step events -> two phases regardless of terminal position.
3257        assert_eq!(
3258            t.phases.len(),
3259            2,
3260            "terminal position must not change phase count"
3261        );
3262        // Phase 0 (step 1) still gets its correct rate (0 -> 4000 over
3263        // 2s = 2000/s): the misordered terminal did not misalign it.
3264        assert_eq!(
3265            t.phases[0].metrics.iteration_rate, None,
3266            "misordered wall-only events must not produce iteration_rate",
3267        );
3268    }
3269
3270    #[test]
3271    fn throughput_degradation_detected() {
3272        // Phase 0: high throughput (0 -> 10000 iters over ~2s = ~5000/s)
3273        // Phase 1: low throughput (10000 -> 11000 iters over ~2s = ~500/s)
3274        // 90% drop exceeds ITERATION_RATE_REL_THRESHOLD (0.3).
3275        let events = vec![
3276            stimulus_with_iters(0, "ScenarioStart", 0),
3277            stimulus_with_iters(2000, "StepStart[0]", 10000),
3278            stimulus_with_iters(4000, "StepEnd[0]", 11000),
3279        ];
3280        let samples: Vec<MonitorSample> = (5..45)
3281            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3282            .collect();
3283        let t = Timeline::build(&events, &samples, 0);
3284        assert_eq!(t.phases.len(), 3);
3285        assert!(
3286            t.phases
3287                .iter()
3288                .all(|phase| phase.metrics.iteration_rate.is_none())
3289        );
3290        let degs: Vec<_> = t
3291            .degradations()
3292            .into_iter()
3293            .filter(|(_, c)| c.metric == "throughput")
3294            .collect();
3295        assert!(
3296            degs.is_empty(),
3297            "wall-only stimulus deltas must not create throughput findings"
3298        );
3299    }
3300
3301    #[test]
3302    fn wall_counter_collapse_does_not_create_throughput_finding() {
3303        let events = vec![
3304            stimulus_with_iters(0, "ScenarioStart", 0),
3305            stimulus_with_iters(2000, "StepStart[0]", 10000), // phase 0: ~5000/s
3306            stimulus_with_iters(4000, "StepStart[1]", 10000), // phase 1: 0/s (stalled)
3307        ];
3308        let samples: Vec<MonitorSample> = (5..45)
3309            .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3310            .collect();
3311        let t = Timeline::build(&events, &samples, 0);
3312        assert_eq!(
3313            t.phases[1].metrics.iteration_rate, None,
3314            "wall-only events do not create a delivered-CPU zero",
3315        );
3316        let degs: Vec<_> = t
3317            .degradations()
3318            .into_iter()
3319            .filter(|(p, c)| p.index == 1 && c.metric == "throughput")
3320            .collect();
3321        assert!(degs.is_empty());
3322    }
3323
3324    #[test]
3325    fn throughput_improvement_detected() {
3326        // Phase 0: low throughput (0 -> 500 iters over ~2s = ~250/s)
3327        // Phase 1: high throughput (500 -> 10500 iters over ~2s = ~5000/s)
3328        // >30% increase should be flagged as improvement.
3329        let events = vec![
3330            stimulus_with_iters(0, "ScenarioStart", 0),
3331            stimulus_with_iters(2000, "StepStart[0]", 500),
3332            stimulus_with_iters(4000, "StepEnd[0]", 10500),
3333        ];
3334        let samples: Vec<MonitorSample> = (5..45)
3335            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3336            .collect();
3337        let t = Timeline::build(&events, &samples, 0);
3338        let improvements: Vec<_> = t
3339            .phases
3340            .iter()
3341            .flat_map(|p| p.changes.iter())
3342            .filter(|c| c.metric == "throughput" && c.direction == ChangeDirection::Improved)
3343            .collect();
3344        assert!(
3345            improvements.is_empty(),
3346            "wall-only stimulus deltas must not create throughput findings"
3347        );
3348    }
3349
3350    #[test]
3351    fn throughput_stable_below_threshold() {
3352        // Wall-only event deltas never create the canonical rate.
3353        let events = vec![
3354            stimulus_with_iters(0, "ScenarioStart", 0),
3355            stimulus_with_iters(2000, "StepStart[0]", 2000),
3356            stimulus_with_iters(4000, "StepEnd[0]", 3800),
3357        ];
3358        let samples: Vec<MonitorSample> = (5..45)
3359            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3360            .collect();
3361        let t = Timeline::build(&events, &samples, 0);
3362        let throughput_changes: Vec<_> = t
3363            .phases
3364            .iter()
3365            .flat_map(|p| p.changes.iter())
3366            .filter(|c| c.metric == "throughput")
3367            .collect();
3368        assert!(
3369            throughput_changes.is_empty(),
3370            "10% change should not trigger throughput change detection"
3371        );
3372    }
3373
3374    #[test]
3375    fn from_phase_buckets_maps_known_metrics_and_renders_phase_block() {
3376        use crate::assert::PhaseBucket;
3377        use std::collections::BTreeMap;
3378        let mut s0_metrics = BTreeMap::new();
3379        s0_metrics.insert("max_dsq_depth".to_string(), 7.0);
3380        s0_metrics.insert("avg_dsq_depth".to_string(), 2.5);
3381        s0_metrics.insert("max_imbalance_ratio".to_string(), 3.5);
3382        s0_metrics.insert("avg_imbalance_ratio".to_string(), 1.8);
3383        s0_metrics.insert("avg_nr_running".to_string(), 3.0);
3384        s0_metrics.insert("total_fallback".to_string(), 200.0);
3385        s0_metrics.insert("fallback_per_vcpu_sec".to_string(), 40.0);
3386        let buckets = vec![
3387            PhaseBucket {
3388                per_cgroup: Default::default(),
3389                step_index: 0,
3390                label: "BASELINE".to_string(),
3391                start_ms: 0,
3392                end_ms: 1000,
3393                sample_count: 5,
3394                metrics: BTreeMap::new(),
3395            },
3396            PhaseBucket {
3397                per_cgroup: Default::default(),
3398                step_index: 1,
3399                label: "Step[0]".to_string(),
3400                start_ms: 1000,
3401                end_ms: 6000,
3402                sample_count: 20,
3403                metrics: s0_metrics,
3404            },
3405        ];
3406        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3407        assert_eq!(t.phases.len(), 2);
3408        // Phase 0 (BASELINE) — no stimulus, no metrics.
3409        assert!(t.phases[0].stimulus.is_none());
3410        assert_eq!(t.phases[0].metrics.sample_count, 5);
3411        assert_eq!(t.phases[0].metrics.max_dsq_depth, 0);
3412        // Phase 1 (Step[0]) — stimulus set, metrics projected from
3413        // the bucket map.
3414        assert!(t.phases[1].stimulus.is_some());
3415        assert_eq!(t.phases[1].stimulus.as_ref().unwrap().label, "Step[0]");
3416        assert_eq!(t.phases[1].metrics.sample_count, 20);
3417        assert_eq!(t.phases[1].metrics.max_dsq_depth, 7);
3418        assert!((t.phases[1].metrics.avg_dsq_depth.unwrap() - 2.5).abs() < f64::EPSILON);
3419        assert!((t.phases[1].metrics.max_imbalance.unwrap() - 3.5).abs() < f64::EPSILON);
3420        assert!((t.phases[1].metrics.avg_imbalance.unwrap() - 1.8).abs() < f64::EPSILON);
3421        // Snapshot-path read (phase_from_bucket): bucket.metrics["avg_nr_running"]
3422        // projects to PhaseMetrics.avg_nr_running; phase 0's empty map -> None.
3423        assert!((t.phases[1].metrics.avg_nr_running.unwrap() - 3.0).abs() < f64::EPSILON);
3424        assert_eq!(t.phases[0].metrics.avg_nr_running, None);
3425        // The canonical CPU-denominated rate is read verbatim; the raw total
3426        // is not divided by this bucket's wall window.
3427        assert_eq!(t.phases[1].metrics.fallback_rate, Some(40.0));
3428        // keep_last_rate absent because no canonical rate is present.
3429        assert_eq!(t.phases[1].metrics.keep_last_rate, None);
3430        // No guest iteration/CPU carrier was provided.
3431        assert_eq!(t.phases[1].metrics.iteration_rate, None);
3432        // Render produces a non-empty timeline block.
3433        let formatted = t.format_with_context(&TimelineContext::default());
3434        assert!(formatted.contains("--- timeline ---"));
3435        assert!(formatted.contains("BASELINE"));
3436        assert!(formatted.contains("Step[0]"));
3437    }
3438
3439    /// Collapse-suppression production path: a throughput collapse INTO a synthesized
3440    /// zero-capture step surfaces through from_phase_buckets (the path
3441    /// `evaluate_vm_result` prefers), not only via the helper unit test.
3442    /// BASELINE captures samples + an iteration_rate; Step[0] is
3443    /// synthesized (sample_count 0) with a collapsed iteration_rate AND a
3444    /// divergent imbalance that must stay gated. Re-adding the old
3445    /// `if sample_count==0 { continue }` gate at this call site makes
3446    /// phases[1].changes empty, so this test fails — it is the
3447    /// regression pin for the removed gate. The producer half — that
3448    /// build_phase_buckets_with_stimulus actually populates a synthesized
3449    /// bucket's iteration_rate from a guest carrier — is pinned by the phase
3450    /// carrier derivation tests; together they pin the producer-to-consumer
3451    /// chain.
3452    #[test]
3453    fn from_phase_buckets_flags_throughput_into_synthesized_step() {
3454        use crate::assert::PhaseBucket;
3455        use std::collections::BTreeMap;
3456        let mut baseline_metrics = BTreeMap::new();
3457        baseline_metrics.insert("iteration_rate".to_string(), 1000.0);
3458        baseline_metrics.insert("avg_imbalance_ratio".to_string(), 1.0);
3459        let mut step_metrics = BTreeMap::new();
3460        step_metrics.insert("iteration_rate".to_string(), 300.0); // 70% collapse
3461        step_metrics.insert("avg_imbalance_ratio".to_string(), 99.0); // must stay gated
3462        let buckets = vec![
3463            PhaseBucket {
3464                per_cgroup: Default::default(),
3465                step_index: 0,
3466                label: "BASELINE".to_string(),
3467                start_ms: 0,
3468                end_ms: 1000,
3469                sample_count: 5,
3470                metrics: baseline_metrics,
3471            },
3472            PhaseBucket {
3473                per_cgroup: Default::default(),
3474                step_index: 1,
3475                label: "Step[0]".to_string(),
3476                start_ms: 1000,
3477                end_ms: 6000,
3478                sample_count: 0, // synthesized zero-capture step
3479                metrics: step_metrics,
3480            },
3481        ];
3482        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3483        assert_eq!(t.phases[1].metrics.sample_count, 0);
3484        assert_eq!(t.phases[1].metrics.iteration_rate, Some(300.0));
3485        let throughput: Vec<_> = t.phases[1]
3486            .changes
3487            .iter()
3488            .filter(|c| c.metric == "throughput")
3489            .collect();
3490        assert_eq!(
3491            throughput.len(),
3492            1,
3493            "throughput collapse into a synthesized step must surface via \
3494             from_phase_buckets: {:?}",
3495            t.phases[1].changes,
3496        );
3497        assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
3498        assert!(
3499            !t.phases[1].changes.iter().any(|c| c.metric == "imbalance"),
3500            "monitor metrics must stay gated for a zero-sample side: {:?}",
3501            t.phases[1].changes,
3502        );
3503    }
3504
3505    /// Boundary change-detection on the from_phase_buckets path — the
3506    /// PRODUCTION success path (`evaluate_vm_result` prefers
3507    /// from_phase_buckets over `build`). Two adjacent metric-bearing
3508    /// buckets whose avg_imbalance / avg_dsq_depth cross the thresholds
3509    /// in the worsening direction must record Degraded changes on the
3510    /// ENTERED phase (phases[1]), and the BASELINE phase records none.
3511    /// Without this, the from_phase_buckets boundary-change loop (the
3512    /// `for i in 1..phases.len()` call to detect_boundary_changes) ships
3513    /// unverified (a wrong threshold, inverted direction, wrong-phase
3514    /// recording, or wrong metric field would all slip past the other
3515    /// from_phase_buckets tests, which never trigger the loop).
3516    #[test]
3517    fn from_phase_buckets_detects_boundary_degradation() {
3518        use crate::assert::PhaseBucket;
3519        use std::collections::BTreeMap;
3520        let mut base = BTreeMap::new();
3521        base.insert("avg_imbalance_ratio".to_string(), 1.0);
3522        base.insert("avg_dsq_depth".to_string(), 1.0);
3523        let mut step = BTreeMap::new();
3524        step.insert("avg_imbalance_ratio".to_string(), 2.0); // +1.0 > 0.5 threshold
3525        step.insert("avg_dsq_depth".to_string(), 6.0); // +5.0 > 3.0 threshold
3526        let buckets = vec![
3527            PhaseBucket {
3528                per_cgroup: Default::default(),
3529                step_index: 0,
3530                label: "BASELINE".to_string(),
3531                start_ms: 0,
3532                end_ms: 1000,
3533                sample_count: 5,
3534                metrics: base,
3535            },
3536            PhaseBucket {
3537                per_cgroup: Default::default(),
3538                step_index: 1,
3539                label: "Step[0]".to_string(),
3540                start_ms: 1000,
3541                end_ms: 6000,
3542                sample_count: 20,
3543                metrics: step,
3544            },
3545        ];
3546        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3547        // Change recorded on the ENTERED phase, never the prior one.
3548        assert!(
3549            t.phases[0].changes.is_empty(),
3550            "BASELINE has no prior phase to diff; changes belong to the entered phase",
3551        );
3552        let changes = &t.phases[1].changes;
3553        let imb = changes
3554            .iter()
3555            .find(|c| c.metric == "imbalance")
3556            .expect("imbalance change must fire (1.0 -> 2.0 crosses 0.5)");
3557        assert_eq!(imb.direction, ChangeDirection::Degraded);
3558        assert!((imb.before - 1.0).abs() < f64::EPSILON);
3559        assert!((imb.after - 2.0).abs() < f64::EPSILON);
3560        let dsq = changes
3561            .iter()
3562            .find(|c| c.metric == "dsq_depth")
3563            .expect("dsq_depth change must fire (1.0 -> 6.0 crosses 3.0)");
3564        assert_eq!(dsq.direction, ChangeDirection::Degraded);
3565        assert!((dsq.before - 1.0).abs() < f64::EPSILON);
3566        assert!((dsq.after - 6.0).abs() < f64::EPSILON);
3567    }
3568
3569    /// Sub-threshold deltas record NO change — guards a dropped/zeroed
3570    /// threshold that would fabricate spurious boundary changes.
3571    #[test]
3572    fn from_phase_buckets_subthreshold_records_no_change() {
3573        use crate::assert::PhaseBucket;
3574        use std::collections::BTreeMap;
3575        let mut base = BTreeMap::new();
3576        base.insert("avg_imbalance_ratio".to_string(), 1.0);
3577        base.insert("avg_dsq_depth".to_string(), 1.0);
3578        let mut step = BTreeMap::new();
3579        step.insert("avg_imbalance_ratio".to_string(), 1.2); // +0.2 < 0.5
3580        step.insert("avg_dsq_depth".to_string(), 2.0); // +1.0 < 3.0
3581        let buckets = vec![
3582            PhaseBucket {
3583                per_cgroup: Default::default(),
3584                step_index: 0,
3585                label: "BASELINE".to_string(),
3586                start_ms: 0,
3587                end_ms: 1000,
3588                sample_count: 5,
3589                metrics: base,
3590            },
3591            PhaseBucket {
3592                per_cgroup: Default::default(),
3593                step_index: 1,
3594                label: "Step[0]".to_string(),
3595                start_ms: 1000,
3596                end_ms: 6000,
3597                sample_count: 20,
3598                metrics: step,
3599            },
3600        ];
3601        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3602        assert!(
3603            t.phases[1].changes.is_empty(),
3604            "sub-threshold deltas must not record a boundary change",
3605        );
3606    }
3607
3608    /// Decreasing imbalance across the boundary records an IMPROVEMENT —
3609    /// locks the higher_is_worse direction so an inverted flag cannot
3610    /// report a regression as an improvement (or vice versa).
3611    #[test]
3612    fn from_phase_buckets_detects_boundary_improvement() {
3613        use crate::assert::PhaseBucket;
3614        use std::collections::BTreeMap;
3615        let mut base = BTreeMap::new();
3616        base.insert("avg_imbalance_ratio".to_string(), 2.0);
3617        let mut step = BTreeMap::new();
3618        step.insert("avg_imbalance_ratio".to_string(), 1.0); // -1.0, |delta|>0.5, after<before
3619        let buckets = vec![
3620            PhaseBucket {
3621                per_cgroup: Default::default(),
3622                step_index: 0,
3623                label: "BASELINE".to_string(),
3624                start_ms: 0,
3625                end_ms: 1000,
3626                sample_count: 5,
3627                metrics: base,
3628            },
3629            PhaseBucket {
3630                per_cgroup: Default::default(),
3631                step_index: 1,
3632                label: "Step[0]".to_string(),
3633                start_ms: 1000,
3634                end_ms: 6000,
3635                sample_count: 20,
3636                metrics: step,
3637            },
3638        ];
3639        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3640        let imb = t.phases[1]
3641            .changes
3642            .iter()
3643            .find(|c| c.metric == "imbalance")
3644            .expect("imbalance change must fire (2.0 -> 1.0 crosses 0.5)");
3645        assert_eq!(
3646            imb.direction,
3647            ChangeDirection::Improved,
3648            "a decreasing imbalance is an improvement, not a degradation",
3649        );
3650    }
3651
3652    /// from_phase_buckets must CORRELATE a real stimulus event into the
3653    /// phase header, carrying its op_kind + detail. Every other
3654    /// from_phase_buckets test passes `&[]`, so only the synthetic
3655    /// None-placeholder arm ran and the `Some(ev) => (*ev).clone()`
3656    /// correlation arm (added to stop headers degrading to "Step[N]: ?")
3657    /// was untested. A wrong interval bound or cloning the wrong event
3658    /// would drop the operator-facing op/detail with no failure.
3659    #[test]
3660    fn from_phase_buckets_correlates_real_stimulus_op_and_detail() {
3661        use crate::assert::PhaseBucket;
3662        use std::collections::BTreeMap;
3663        let event = StimulusEvent {
3664            elapsed_ms: 1000,
3665            label: "Step[0]".to_string(),
3666            op_kind: Some("SetCpuset".to_string()),
3667            detail: Some("4 cpus".to_string()),
3668            total_iterations: None,
3669            step_index: Some(1),
3670            is_terminal: false,
3671            is_step_end: false,
3672        };
3673        let buckets = vec![
3674            PhaseBucket {
3675                per_cgroup: Default::default(),
3676                step_index: 0,
3677                label: "BASELINE".to_string(),
3678                start_ms: 0,
3679                end_ms: 1000,
3680                sample_count: 5,
3681                metrics: BTreeMap::new(),
3682            },
3683            PhaseBucket {
3684                per_cgroup: Default::default(),
3685                step_index: 1,
3686                label: "Step[0]".to_string(),
3687                start_ms: 1000,
3688                end_ms: 6000,
3689                sample_count: 20,
3690                metrics: BTreeMap::new(),
3691            },
3692        ];
3693        let t = Timeline::from_phase_buckets(&buckets, &[event], &TimelineContext::default());
3694        let stim = t.phases[1]
3695            .stimulus
3696            .as_ref()
3697            .expect("Step[0] phase carries a stimulus");
3698        assert_eq!(
3699            stim.op_kind.as_deref(),
3700            Some("SetCpuset"),
3701            "the correlated event's op_kind must be carried, not the None placeholder",
3702        );
3703        assert_eq!(stim.detail.as_deref(), Some("4 cpus"));
3704    }
3705
3706    #[test]
3707    fn from_phase_buckets_zero_duration_window_emits_no_rate() {
3708        use crate::assert::PhaseBucket;
3709        use std::collections::BTreeMap;
3710        let mut metrics = BTreeMap::new();
3711        metrics.insert("total_fallback".to_string(), 100.0);
3712        let bucket = PhaseBucket {
3713            per_cgroup: Default::default(),
3714            step_index: 1,
3715            label: "Step[0]".to_string(),
3716            start_ms: 500,
3717            end_ms: 500,
3718            sample_count: 1,
3719            metrics,
3720        };
3721        let t = Timeline::from_phase_buckets(&[bucket], &[], &TimelineContext::default());
3722        // Degenerate window (start == end) yields duration_s == 0,
3723        // so rate divisions stay None rather than producing
3724        // spurious infinities.
3725        assert_eq!(t.phases[0].metrics.fallback_rate, None);
3726    }
3727
3728    #[test]
3729    fn from_phase_buckets_absent_imbalance_metric_is_none_not_zero() {
3730        // A bucket carrying no avg_imbalance_ratio / avg_dsq_depth
3731        // metric must yield None (no data), NOT Some(0.0) — so the change
3732        // detector skips it instead of comparing a false zero-imbalance.
3733        use crate::assert::PhaseBucket;
3734        use std::collections::BTreeMap;
3735        let bucket = PhaseBucket {
3736            per_cgroup: Default::default(),
3737            step_index: 1,
3738            label: "Step[0]".to_string(),
3739            start_ms: 100,
3740            end_ms: 600,
3741            sample_count: 3,
3742            metrics: BTreeMap::new(),
3743        };
3744        let t = Timeline::from_phase_buckets(&[bucket], &[], &TimelineContext::default());
3745        assert_eq!(t.phases[0].metrics.avg_imbalance, None);
3746        assert_eq!(t.phases[0].metrics.max_imbalance, None);
3747        assert_eq!(t.phases[0].metrics.avg_dsq_depth, None);
3748    }
3749
3750    #[test]
3751    fn from_phase_buckets_sorts_by_step_index() {
3752        use crate::assert::PhaseBucket;
3753        use std::collections::BTreeMap;
3754        // Out-of-order input; from_phase_buckets must sort by
3755        // step_index so the rendered phase block walks BASELINE
3756        // → Step[0] → Step[1] in time order regardless of
3757        // how the caller arranged the input vec.
3758        let buckets = vec![
3759            PhaseBucket {
3760                per_cgroup: Default::default(),
3761                step_index: 2,
3762                label: "Step[1]".to_string(),
3763                start_ms: 2000,
3764                end_ms: 3000,
3765                sample_count: 5,
3766                metrics: BTreeMap::new(),
3767            },
3768            PhaseBucket {
3769                per_cgroup: Default::default(),
3770                step_index: 0,
3771                label: "BASELINE".to_string(),
3772                start_ms: 0,
3773                end_ms: 500,
3774                sample_count: 2,
3775                metrics: BTreeMap::new(),
3776            },
3777            PhaseBucket {
3778                per_cgroup: Default::default(),
3779                step_index: 1,
3780                label: "Step[0]".to_string(),
3781                start_ms: 500,
3782                end_ms: 2000,
3783                sample_count: 5,
3784                metrics: BTreeMap::new(),
3785            },
3786        ];
3787        let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3788        assert_eq!(t.phases.len(), 3);
3789        assert_eq!(t.phases[0].start_ms, 0);
3790        assert_eq!(t.phases[1].start_ms, 500);
3791        assert_eq!(t.phases[2].start_ms, 2000);
3792    }
3793
3794    #[test]
3795    fn iteration_rate_in_formatted_output() {
3796        let events = vec![
3797            stimulus_with_iters(0, "ScenarioStart", 0),
3798            stimulus_with_iters(2000, "StepStart[0]", 5000),
3799        ];
3800        let samples: Vec<MonitorSample> = (5..25)
3801            .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3802            .collect();
3803        let t = Timeline::build(&events, &samples, 0);
3804        let formatted = t.format_with_context(&TimelineContext::default());
3805        assert!(!formatted.contains("iteration rate:"));
3806        assert!(!formatted.contains("iter/cpu-s"));
3807    }
3808}