1use std::fmt;
9
10use crate::monitor::{MonitorSample, sample_looks_valid};
11
12#[derive(Debug, Clone, Default)]
18pub struct TimelineContext {
19 pub kernel: Option<String>,
21 pub topology: Option<String>,
23 pub scheduler: Option<String>,
25 pub scenario: Option<String>,
27 pub duration_s: Option<f64>,
29}
30
31#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
39pub struct StimulusEvent {
40 pub elapsed_ms: u64,
42 pub label: String,
49 pub op_kind: Option<String>,
51 pub detail: Option<String>,
53 pub total_iterations: Option<u64>,
65 pub step_index: Option<u16>,
74 pub is_terminal: bool,
84 pub is_step_end: bool,
91}
92
93impl StimulusEvent {
94 pub fn from_wire(ev: &crate::vmm::wire::StimulusEvent) -> Self {
112 Self {
113 elapsed_ms: ev.elapsed_ms as u64,
114 label: format!("StepStart[{}]", ev.step_index.saturating_sub(1)),
115 op_kind: Some(format!("ops={}", ev.op_count)),
116 detail: Some(format!(
117 "{} cgroups, {} workers",
118 ev.cgroup_count, ev.worker_count,
119 )),
120 total_iterations: Some(ev.total_iterations),
121 step_index: Some(ev.step_index),
122 is_terminal: false,
123 is_step_end: false,
124 }
125 }
126
127 pub fn from_step_end(ev: &crate::vmm::wire::StimulusEvent) -> Self {
134 Self {
135 elapsed_ms: ev.elapsed_ms as u64,
136 label: format!("StepEnd[{}]", ev.step_index.saturating_sub(1)),
137 op_kind: Some(format!("ops={}", ev.op_count)),
138 detail: Some(format!(
139 "{} cgroups, {} workers",
140 ev.cgroup_count, ev.worker_count,
141 )),
142 total_iterations: Some(ev.total_iterations),
143 step_index: Some(ev.step_index),
144 is_terminal: false,
145 is_step_end: true,
146 }
147 }
148
149 pub fn terminal(elapsed_ms: u64, total_iterations: u64) -> Self {
160 Self {
161 elapsed_ms,
162 label: "ScenarioEnd".to_string(),
163 op_kind: None,
164 detail: None,
165 total_iterations: Some(total_iterations),
166 step_index: None,
167 is_terminal: true,
168 is_step_end: false,
169 }
170 }
171
172 pub fn phase(&self) -> Option<crate::assert::Phase> {
185 self.step_index
186 .map(|si| crate::assert::Phase::step(si.saturating_sub(1)))
187 }
188}
189
190#[derive(Debug, Clone, Default)]
196pub struct PhaseMetrics {
197 pub sample_count: usize,
198 pub avg_imbalance: Option<f64>,
206 pub max_imbalance: Option<f64>,
209 pub avg_dsq_depth: Option<f64>,
212 pub avg_nr_running: Option<f64>,
220 pub max_dsq_depth: u32,
221 pub stuck_count: usize,
222 pub fallback_rate: Option<f64>,
225 pub keep_last_rate: Option<f64>,
228 pub iteration_rate: Option<f64>,
231}
232
233#[derive(Debug, Clone, Copy, PartialEq, Eq)]
235pub enum ChangeDirection {
236 Improved,
237 Degraded,
238}
239
240impl fmt::Display for ChangeDirection {
241 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
242 match self {
243 ChangeDirection::Improved => write!(f, "IMPROVEMENT"),
244 ChangeDirection::Degraded => write!(f, "DEGRADATION"),
245 }
246 }
247}
248
249#[derive(Debug, Clone)]
251pub struct PhaseChange {
252 pub direction: ChangeDirection,
253 pub metric: String,
254 pub before: f64,
255 pub after: f64,
256}
257
258#[derive(Debug, Clone)]
260pub struct Phase {
261 pub index: usize,
262 pub start_ms: u64,
263 pub end_ms: u64,
264 pub stimulus: Option<StimulusEvent>,
266 pub metrics: PhaseMetrics,
267 pub changes: Vec<PhaseChange>,
269 pub per_cgroup: std::collections::BTreeMap<String, crate::assert::PhaseCgroupStats>,
277 pub not_measured_window: bool,
285}
286
287#[derive(Debug, Clone)]
293pub struct Timeline {
294 pub phases: Vec<Phase>,
295}
296
297const IMBALANCE_THRESHOLD: f64 = 0.5;
299const DSQ_THRESHOLD: f64 = 3.0;
301const NR_RUNNING_THRESHOLD: f64 = 1.0;
304const FALLBACK_RATE_THRESHOLD: f64 = 10.0;
306const KEEP_LAST_RATE_THRESHOLD: f64 = 10.0;
308const ITERATION_RATE_REL_THRESHOLD: f64 = 0.3;
311
312fn detect_change(
317 before: f64,
318 after: f64,
319 threshold: f64,
320 metric: &str,
321 higher_is_worse: bool,
322) -> Option<PhaseChange> {
323 let delta = after - before;
324 if delta.abs() <= threshold {
325 return None;
326 }
327 let degraded = if higher_is_worse {
328 delta > 0.0
329 } else {
330 delta < 0.0
331 };
332 Some(PhaseChange {
333 direction: if degraded {
334 ChangeDirection::Degraded
335 } else {
336 ChangeDirection::Improved
337 },
338 metric: metric.to_string(),
339 before,
340 after,
341 })
342}
343
344fn detect_boundary_changes(before: &PhaseMetrics, after: &PhaseMetrics) -> Vec<PhaseChange> {
371 let mut changes = Vec::new();
372
373 if before.sample_count > 0 && after.sample_count > 0 {
379 if let (Some(bi), Some(ai)) = (before.avg_imbalance, after.avg_imbalance) {
380 changes.extend(detect_change(
381 bi,
382 ai,
383 IMBALANCE_THRESHOLD,
384 "imbalance",
385 true,
386 ));
387 }
388 if let (Some(bd), Some(ad)) = (before.avg_dsq_depth, after.avg_dsq_depth) {
389 changes.extend(detect_change(bd, ad, DSQ_THRESHOLD, "dsq_depth", true));
390 }
391 if let (Some(bn), Some(an)) = (before.avg_nr_running, after.avg_nr_running) {
392 changes.extend(detect_change(
393 bn,
394 an,
395 NR_RUNNING_THRESHOLD,
396 "nr_running",
397 true,
398 ));
399 }
400 if let (Some(bf), Some(af)) = (before.fallback_rate, after.fallback_rate) {
401 changes.extend(detect_change(
402 bf,
403 af,
404 FALLBACK_RATE_THRESHOLD,
405 "fallback",
406 true,
407 ));
408 }
409 if let (Some(bk), Some(ak)) = (before.keep_last_rate, after.keep_last_rate) {
410 changes.extend(detect_change(
411 bk,
412 ak,
413 KEEP_LAST_RATE_THRESHOLD,
414 "keep_last",
415 true,
416 ));
417 }
418 }
419
420 if let (Some(bi), Some(ai)) = (before.iteration_rate, after.iteration_rate)
428 && bi > 0.0
429 {
430 let rel = (ai - bi) / bi;
431 if rel.abs() > ITERATION_RATE_REL_THRESHOLD {
432 changes.push(PhaseChange {
433 direction: if rel < 0.0 {
434 ChangeDirection::Degraded
435 } else {
436 ChangeDirection::Improved
437 },
438 metric: "throughput".to_string(),
439 before: bi,
440 after: ai,
441 });
442 }
443 }
444
445 changes
446}
447
448impl Timeline {
449 pub fn build(
479 stimulus_events: &[StimulusEvent],
480 monitor_samples: &[MonitorSample],
481 preemption_threshold_ns: u64,
482 ) -> Self {
483 if stimulus_events.is_empty() || monitor_samples.is_empty() {
484 return Self { phases: Vec::new() };
485 }
486
487 let mut events = stimulus_events.to_vec();
488 events.sort_by_key(|e| (e.elapsed_ms, !e.is_step_end));
496
497 let first_stimulus_ms = events[0].elapsed_ms;
501 let first_monitor_ms = monitor_samples
502 .iter()
503 .find(|s| s.elapsed_ms > 500 && !s.cpus.is_empty())
504 .map(|s| s.elapsed_ms)
505 .unwrap_or_else(|| monitor_samples.first().map(|s| s.elapsed_ms).unwrap_or(0));
506
507 let offset = first_monitor_ms as i64 - first_stimulus_ms as i64;
509
510 let last_monitor_ms = monitor_samples.last().map(|s| s.elapsed_ms).unwrap_or(0);
514
515 let step_events: Vec<&StimulusEvent> = events
527 .iter()
528 .filter(|e| !e.is_terminal && !e.is_step_end)
529 .collect();
530
531 let mut boundaries: Vec<(u64, u64, Option<StimulusEvent>)> = Vec::new();
532 for i in 0..step_events.len() {
533 let start = (step_events[i].elapsed_ms as i64 + offset).max(0) as u64;
534 let end = if i + 1 < step_events.len() {
540 (step_events[i + 1].elapsed_ms as i64 + offset).max(0) as u64
541 } else {
542 last_monitor_ms.saturating_add(1)
543 };
544 let stimulus = if i == 0 {
545 None
546 } else {
547 Some(step_events[i].clone())
548 };
549 boundaries.push((start, end, stimulus));
550 }
551
552 let mut phases: Vec<Phase> = Vec::with_capacity(boundaries.len());
554 for (idx, (start, end, stimulus)) in boundaries.into_iter().enumerate() {
555 let phase_samples: Vec<&MonitorSample> = monitor_samples
556 .iter()
557 .filter(|s| s.elapsed_ms >= start && s.elapsed_ms < end && sample_looks_valid(s))
558 .collect();
559
560 let metrics = compute_metrics(&phase_samples, preemption_threshold_ns);
561
562 phases.push(Phase {
563 index: idx,
578 start_ms: start,
579 end_ms: end,
580 stimulus,
581 metrics,
582 changes: Vec::new(),
583 per_cgroup: std::collections::BTreeMap::new(),
585 not_measured_window: false,
586 });
587 }
588
589 for i in 1..phases.len() {
594 let changes = detect_boundary_changes(&phases[i - 1].metrics, &phases[i].metrics);
595 phases[i].changes = changes;
596 }
597
598 Self { phases }
599 }
600
601 pub fn format_with_context(&self, ctx: &TimelineContext) -> String {
611 if self.phases.is_empty() {
612 return String::new();
613 }
614
615 let mut out = String::from("--- timeline ---\n");
616
617 let mut header_parts = Vec::new();
619 if let Some(ref k) = ctx.kernel {
620 header_parts.push(format!("kernel: {k}"));
621 }
622 if let Some(ref t) = ctx.topology {
623 header_parts.push(format!("topology: {t}"));
624 }
625 if let Some(ref s) = ctx.scheduler {
626 header_parts.push(format!("scheduler: {s}"));
627 }
628 if let Some(ref s) = ctx.scenario {
629 header_parts.push(format!("scenario: {s}"));
630 }
631 if let Some(d) = ctx.duration_s {
632 header_parts.push(format!("duration: {d:.1}s"));
633 }
634 if !header_parts.is_empty() {
635 for part in &header_parts {
636 out.push_str(part);
637 out.push_str(" ");
638 }
639 if out.len() >= 2 {
643 out.truncate(out.len() - 2);
644 }
645 out.push('\n');
646 }
647
648 self.format_phases(&mut out);
649 out
650 }
651
652 fn format_phases(&self, out: &mut String) {
654 for phase in &self.phases {
655 let duration_ms = phase.end_ms.saturating_sub(phase.start_ms);
656 let window = if phase.not_measured_window {
661 "window not measured".to_string()
662 } else {
663 format!("{duration_ms}ms")
664 };
665
666 if phase.index == 0 {
667 out.push_str(&format!(
669 "\nBASELINE (settle, {}, {} samples):\n",
670 window, phase.metrics.sample_count,
671 ));
672 } else {
673 let label_start = phase
674 .stimulus
675 .as_ref()
676 .map(|s| {
677 let mut l = s.label.clone();
678 if let Some(op) = &s.op_kind {
679 l.push(' ');
680 l.push_str(op);
681 }
682 l
683 })
684 .unwrap_or_else(|| "?".to_string());
685
686 out.push_str(&format!(
687 "\nPhase {}: {} ({}, {} samples):\n",
688 phase.index, label_start, window, phase.metrics.sample_count,
689 ));
690 }
691
692 let m = &phase.metrics;
693 let has_monitor_metrics = m.avg_imbalance.is_some()
703 || m.max_imbalance.is_some()
704 || m.avg_dsq_depth.is_some()
705 || m.avg_nr_running.is_some()
706 || m.max_dsq_depth > 0
707 || m.fallback_rate.is_some()
708 || m.keep_last_rate.is_some()
709 || m.stuck_count > 0;
710 if m.sample_count > 0 || has_monitor_metrics {
711 out.push_str(&format!(
712 " imbalance: avg={} max={} | dsq: avg={} max={} | nr_run: avg={}",
713 m.avg_imbalance
714 .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
715 m.max_imbalance
716 .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
717 m.avg_dsq_depth
718 .map_or_else(|| "n/a".to_string(), |v| format!("{v:.0}")),
719 m.max_dsq_depth,
720 m.avg_nr_running
721 .map_or_else(|| "n/a".to_string(), |v| format!("{v:.1}")),
722 ));
723 if let Some(fb) = m.fallback_rate {
724 out.push_str(&format!(" | fallback: {:.0}/vcpu-s", fb));
725 }
726 if let Some(kl) = m.keep_last_rate {
727 out.push_str(&format!(" | keep_last: {:.0}/vcpu-s", kl));
728 }
729 if let Some(ir) = m.iteration_rate {
730 out.push_str(&format!(" | iteration rate: {ir:.0} iter/cpu-s"));
731 }
732 out.push('\n');
733 if m.stuck_count > 0 {
734 out.push_str(&format!(" stuck: {}\n", m.stuck_count));
735 }
736 } else if let Some(ir) = m.iteration_rate {
737 out.push_str(&format!(
738 " [no samples] | iteration rate: {ir:.0} iter/cpu-s\n"
739 ));
740 } else {
741 out.push_str(" [no samples]\n");
742 }
743
744 format_phase_cgroups(out, &phase.per_cgroup);
745
746 if let Some(ref stim) = phase.stimulus {
747 let detail = stim.detail.as_deref().unwrap_or("");
748 let op = stim.op_kind.as_deref().unwrap_or("?");
749 out.push_str(&format!(" >>> {}: {op}", stim.label));
750 if !detail.is_empty() {
751 out.push_str(&format!(" ({detail})"));
752 }
753 out.push('\n');
754 }
755
756 for change in &phase.changes {
757 let delta = change.after - change.before;
758 let sign = if delta > 0.0 { "+" } else { "" };
759 out.push_str(&format!(
760 " >>> {}: {} {sign}{:.1}\n",
761 change.direction, change.metric, delta,
762 ));
763 }
764 }
765 }
766
767 pub fn from_phase_buckets(
805 phase_buckets: &[crate::assert::PhaseBucket],
806 stimulus_events: &[StimulusEvent],
807 _ctx: &TimelineContext,
808 ) -> Self {
809 let mut sorted: Vec<&crate::assert::PhaseBucket> = phase_buckets.iter().collect();
810 sorted.sort_by_key(|b| b.step_index);
811 let mut sorted_events: Vec<&StimulusEvent> = stimulus_events
821 .iter()
822 .filter(|e| !e.is_terminal && !e.is_step_end)
823 .collect();
824 sorted_events.sort_by_key(|e| e.elapsed_ms);
825 let mut phases: Vec<Phase> = sorted
826 .into_iter()
827 .map(|b| phase_from_bucket(b, &sorted_events))
828 .collect();
829 for i in 1..phases.len() {
849 let changes = detect_boundary_changes(&phases[i - 1].metrics, &phases[i].metrics);
850 phases[i].changes = changes;
851 }
852 Self { phases }
853 }
854
855 #[cfg(test)]
860 pub fn degradations(&self) -> Vec<(&Phase, &PhaseChange)> {
861 let mut out = Vec::new();
862 for phase in &self.phases {
863 for change in &phase.changes {
864 if change.direction == ChangeDirection::Degraded {
865 out.push((phase, change));
866 }
867 }
868 }
869 out
870 }
871}
872
873fn phase_from_bucket(b: &crate::assert::PhaseBucket, sorted_events: &[&StimulusEvent]) -> Phase {
890 let metrics = PhaseMetrics {
891 sample_count: b.sample_count,
892 avg_imbalance: b.metrics.get("avg_imbalance_ratio").copied(),
893 max_imbalance: b.metrics.get("max_imbalance_ratio").copied(),
894 avg_dsq_depth: b.metrics.get("avg_dsq_depth").copied(),
895 avg_nr_running: b.metrics.get("avg_nr_running").copied(),
896 max_dsq_depth: b
897 .metrics
898 .get("max_dsq_depth")
899 .map(|v| v.round() as u32)
900 .unwrap_or(0),
901 stuck_count: b
902 .metrics
903 .get("stuck_count")
904 .map(|v| v.round() as usize)
905 .unwrap_or(0),
906 fallback_rate: b.metrics.get("fallback_per_vcpu_sec").copied(),
907 keep_last_rate: b.metrics.get("keep_last_per_vcpu_sec").copied(),
908 iteration_rate: b.metrics.get("iteration_rate").copied(),
910 };
911 let stimulus = if b.step_index == 0 {
912 None
913 } else {
914 let correlated = sorted_events.iter().find(|e| {
923 if b.start_ms == b.end_ms {
924 e.elapsed_ms == b.start_ms
925 } else {
926 e.elapsed_ms >= b.start_ms && e.elapsed_ms < b.end_ms
927 }
928 });
929 match correlated {
930 Some(ev) => Some((*ev).clone()),
931 None => Some(StimulusEvent {
932 elapsed_ms: b.start_ms,
933 label: b.label.clone(),
934 op_kind: None,
935 detail: None,
936 total_iterations: None,
937 step_index: None,
941 is_terminal: false,
942 is_step_end: false,
943 }),
944 }
945 };
946 let not_measured_window =
959 b.start_ms == 0 && b.end_ms == 0 && b.metrics.is_empty() && !b.per_cgroup.is_empty();
960 Phase {
961 index: b.step_index as usize,
962 start_ms: b.start_ms,
963 end_ms: b.end_ms,
964 stimulus,
965 metrics,
966 changes: Vec::new(),
967 per_cgroup: b.per_cgroup.clone(),
968 not_measured_window,
969 }
970}
971
972const MAX_RENDERED_CGROUPS: usize = 16;
977
978fn format_phase_cgroups(
987 out: &mut String,
988 per_cgroup: &std::collections::BTreeMap<String, crate::assert::PhaseCgroupStats>,
989) {
990 if per_cgroup.is_empty() {
991 return;
992 }
993 out.push_str(" per-cgroup:\n");
994 for (name, pcg) in per_cgroup.iter().take(MAX_RENDERED_CGROUPS) {
995 out.push_str(&format!(" {name}: "));
996 if pcg.stripped {
1001 out.push_str("samples stripped (size limit)");
1002 } else {
1003 match pcg.off_cpu_summary() {
1004 Some((avg, min, max, spread)) => out.push_str(&format!(
1005 "off-cpu avg={avg:.1}% min={min:.1}% max={max:.1}% spread={spread:.1}%"
1006 )),
1007 None => out.push_str("off-cpu n/a"),
1008 }
1009 if let Some((p99, median)) = pcg.wake_summary() {
1010 out.push_str(&format!(
1011 " | wake p99={p99:.0}\u{00b5}s median={median:.0}\u{00b5}s"
1012 ));
1013 }
1014 if let Some((mean, worst)) = pcg.run_delay_summary() {
1015 out.push_str(&format!(
1016 " | run-delay mean={mean:.0}\u{00b5}s worst={worst:.0}\u{00b5}s"
1017 ));
1018 }
1019 }
1020 out.push_str(&format!(
1021 " | iters={} migrations={}",
1022 pcg.total_iterations, pcg.total_migrations
1023 ));
1024 if pcg.max_gap_ms > 0 {
1027 out.push_str(&format!(
1028 " | gap={}ms@cpu{}",
1029 pcg.max_gap_ms, pcg.max_gap_cpu
1030 ));
1031 }
1032 out.push('\n');
1033 }
1034 let total = per_cgroup.len();
1035 if total > MAX_RENDERED_CGROUPS {
1036 let dropped = total - MAX_RENDERED_CGROUPS;
1037 let noun = if dropped == 1 { "cgroup" } else { "cgroups" };
1038 out.push_str(&format!(" (+{dropped} more {noun})\n"));
1039 }
1040}
1041
1042pub(crate) fn compute_metrics(
1060 samples: &[&MonitorSample],
1061 preemption_threshold_ns: u64,
1062) -> PhaseMetrics {
1063 if samples.is_empty() {
1064 return PhaseMetrics::default();
1065 }
1066
1067 let valid: Vec<&MonitorSample> = samples
1070 .iter()
1071 .copied()
1072 .filter(|s| !s.cpus.is_empty() && sample_looks_valid(s))
1073 .collect();
1074
1075 if valid.is_empty() {
1076 return PhaseMetrics {
1077 sample_count: 0,
1078 ..PhaseMetrics::default()
1079 };
1080 }
1081
1082 let mut total_imbalance = 0.0f64;
1083 let mut max_imbalance = 0.0f64;
1084 let mut total_dsq = 0.0f64;
1085 let mut total_nr_running = 0.0f64;
1086 let mut max_dsq = 0u32;
1087 let mut stuck_count = 0usize;
1088
1089 for sample in &valid {
1090 for cpu in &sample.cpus {
1091 max_dsq = max_dsq.max(cpu.local_dsq_depth);
1092 }
1093 let ratio = sample.imbalance_ratio();
1094 total_imbalance += ratio;
1095 if ratio > max_imbalance {
1096 max_imbalance = ratio;
1097 }
1098
1099 let avg_dsq_this: f64 = sample
1100 .cpus
1101 .iter()
1102 .map(|c| c.local_dsq_depth as f64)
1103 .sum::<f64>()
1104 / sample.cpus.len() as f64;
1105 total_dsq += avg_dsq_this;
1106
1107 let avg_nr_this: f64 =
1111 sample.cpus.iter().map(|c| c.nr_running as f64).sum::<f64>() / sample.cpus.len() as f64;
1112 total_nr_running += avg_nr_this;
1113 }
1114
1115 let threshold = if preemption_threshold_ns > 0 {
1123 preemption_threshold_ns
1124 } else {
1125 crate::monitor::vcpu_preemption_threshold_ns(None)
1126 };
1127 for w in valid.windows(2) {
1128 let prev = w[0];
1129 let curr = w[1];
1130 let cpu_count = prev.cpus.len().min(curr.cpus.len());
1131 for cpu in 0..cpu_count {
1132 if crate::monitor::reader::is_cpu_stuck(&prev.cpus[cpu], &curr.cpus[cpu], threshold) {
1133 stuck_count += 1;
1134 }
1135 }
1136 }
1137
1138 let has_events = |s: &&MonitorSample| s.has_complete_event_counters();
1142 let first_ev = valid.iter().copied().find(|s| has_events(s));
1143 let last_ev = valid.iter().copied().rev().find(|s| has_events(s));
1144
1145 let (fallback_rate, keep_last_rate) = match (first_ev, last_ev) {
1146 (Some(first), Some(last)) => {
1147 let fb_delta = crate::monitor::counter_delta(
1152 last.sum_event_field(|e| e.select_cpu_fallback).unwrap_or(0),
1153 first
1154 .sum_event_field(|e| e.select_cpu_fallback)
1155 .unwrap_or(0),
1156 );
1157 let kl_delta = crate::monitor::counter_delta(
1158 last.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1159 first.sum_event_field(|e| e.dispatch_keep_last).unwrap_or(0),
1160 );
1161 match crate::monitor::MonitorSummary::mean_vcpu_cpu_delta_secs(first, last) {
1162 Some(vcpu_sec) => (
1163 Some(fb_delta as f64 / vcpu_sec),
1164 Some(kl_delta as f64 / vcpu_sec),
1165 ),
1166 None => (None, None),
1167 }
1168 }
1169 _ => (None, None),
1170 };
1171
1172 let valid_count = valid.len();
1173 let n = valid_count as f64;
1174 PhaseMetrics {
1177 sample_count: valid_count,
1178 avg_imbalance: (valid_count > 0).then(|| total_imbalance / n),
1179 max_imbalance: (valid_count > 0).then_some(max_imbalance),
1180 avg_dsq_depth: (valid_count > 0).then(|| total_dsq / n),
1181 avg_nr_running: (valid_count > 0).then(|| total_nr_running / n),
1182 max_dsq_depth: max_dsq,
1183 stuck_count,
1184 fallback_rate,
1185 keep_last_rate,
1186 iteration_rate: None,
1187 }
1188}
1189
1190#[cfg(test)]
1195mod tests {
1196 use super::*;
1197 use crate::monitor::{CpuSnapshot, MonitorSample};
1198
1199 #[test]
1203 fn stimulus_event_phase_maps_wire_step_index_to_phase() {
1204 use crate::assert::Phase;
1205 assert_eq!(
1207 StimulusEvent::from_wire(&wire_event(0, 1, 0)).phase(),
1208 Some(Phase::step(0)),
1209 );
1210 assert_eq!(
1211 StimulusEvent::from_wire(&wire_event(100, 2, 50)).phase(),
1212 Some(Phase::step(1)),
1213 );
1214 assert_eq!(
1216 StimulusEvent::from_step_end(&wire_event(200, 2, 90)).phase(),
1217 Some(Phase::step(1)),
1218 );
1219 assert_eq!(StimulusEvent::terminal(300, 100).phase(), None);
1221 }
1222
1223 fn sample(elapsed_ms: u64, cpus: Vec<(u32, u32, u64)>) -> MonitorSample {
1224 MonitorSample {
1225 bpf_map_fields: Vec::new(),
1226 prog_stats: None,
1227 psi_irq: None,
1228 elapsed_ms,
1229 cpus: cpus
1230 .into_iter()
1231 .map(|(nr_running, dsq, rq_clock)| CpuSnapshot {
1232 nr_running,
1233 scx_nr_running: 0,
1234 local_dsq_depth: dsq,
1235 rq_clock,
1236 scx_flags: 0,
1237 event_counters: None,
1238 schedstat: None,
1239 vcpu_cpu_time_ns: None,
1240 vcpu_perf: None,
1241 avg_irq_util: None,
1242 sched_domains: None,
1243 })
1244 .collect(),
1245 }
1246 }
1247
1248 fn stimulus(elapsed_ms: u64, label: &str) -> StimulusEvent {
1249 StimulusEvent {
1250 elapsed_ms,
1251 label: label.to_string(),
1252 op_kind: None,
1253 detail: None,
1254 total_iterations: None,
1255 step_index: None,
1256 is_terminal: false,
1257 is_step_end: false,
1258 }
1259 }
1260
1261 #[test]
1262 fn empty_inputs_empty_timeline() {
1263 let t = Timeline::build(&[], &[], 0);
1264 assert!(t.phases.is_empty());
1265 }
1266
1267 #[test]
1268 fn no_stimulus_empty_timeline() {
1269 let samples = vec![sample(1000, vec![(2, 1, 100)])];
1270 let t = Timeline::build(&[], &samples, 0);
1271 assert!(t.phases.is_empty());
1272 }
1273
1274 #[test]
1275 fn no_monitor_empty_timeline() {
1276 let events = vec![stimulus(0, "ScenarioStart")];
1277 let t = Timeline::build(&events, &[], 0);
1278 assert!(t.phases.is_empty());
1279 }
1280
1281 #[test]
1282 fn single_event_single_phase() {
1283 let events = vec![stimulus(0, "ScenarioStart")];
1284 let samples = vec![
1285 sample(600, vec![(2, 1, 100), (2, 1, 200)]),
1286 sample(700, vec![(2, 1, 300), (2, 1, 400)]),
1287 ];
1288 let t = Timeline::build(&events, &samples, 0);
1289 assert_eq!(t.phases.len(), 1);
1290 assert_eq!(t.phases[0].metrics.sample_count, 2);
1296 }
1297
1298 #[test]
1299 fn two_events_two_phases() {
1300 let events = vec![stimulus(0, "ScenarioStart"), stimulus(3000, "StepStart[0]")];
1301 let samples: Vec<MonitorSample> = (5..65)
1302 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
1303 .collect();
1304 let t = Timeline::build(&events, &samples, 0);
1305 assert_eq!(t.phases.len(), 2);
1306 assert_eq!(t.phases[0].metrics.sample_count, 30);
1313 assert_eq!(t.phases[1].metrics.sample_count, 29);
1314 assert_eq!(
1315 t.phases[0].metrics.sample_count + t.phases[1].metrics.sample_count,
1316 59,
1317 "59 = 60 samples minus the 500ms sample dropped by the >500 warmup",
1318 );
1319 }
1320
1321 #[test]
1322 fn improvement_detected() {
1323 let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
1326 let mut samples = Vec::new();
1327 for i in 5..15 {
1328 samples.push(sample(
1329 i * 100,
1330 vec![(1, 1, i * 1000), (5, 1, i * 1000 + 100)],
1331 ));
1332 }
1333 for i in 15..25 {
1334 samples.push(sample(
1335 i * 100,
1336 vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
1337 ));
1338 }
1339 let t = Timeline::build(&events, &samples, 0);
1340 let improvements: Vec<_> = t
1341 .phases
1342 .iter()
1343 .flat_map(|p| p.changes.iter())
1344 .filter(|c| c.direction == ChangeDirection::Improved)
1345 .collect();
1346 assert!(!improvements.is_empty());
1347 }
1348
1349 #[test]
1350 fn format_non_empty() {
1351 let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
1352 let samples: Vec<MonitorSample> = (5..25)
1353 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
1354 .collect();
1355 let t = Timeline::build(&events, &samples, 0);
1356 let formatted = t.format_with_context(&TimelineContext::default());
1357 assert!(formatted.contains("BASELINE"));
1358 assert!(formatted.contains("Phase 1"));
1359 assert!(formatted.contains("imbalance"));
1360 }
1361
1362 #[test]
1369 fn format_renders_synthesized_step_throughput() {
1370 let buckets = vec![
1371 crate::assert::PhaseBucket {
1372 per_cgroup: Default::default(),
1373 step_index: 0,
1374 label: "BASELINE".to_string(),
1375 start_ms: 0,
1376 end_ms: 1000,
1377 sample_count: 2,
1378 metrics: std::collections::BTreeMap::new(),
1379 },
1380 crate::assert::PhaseBucket {
1381 per_cgroup: Default::default(),
1382 step_index: 1,
1383 label: "Step[0]".to_string(),
1384 start_ms: 1000,
1385 end_ms: 2000,
1386 sample_count: 0, metrics: std::collections::BTreeMap::from([("iteration_rate".to_string(), 1500.0)]),
1388 },
1389 ];
1390 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1391 let formatted = t.format_with_context(&TimelineContext::default());
1392 assert!(
1393 formatted.contains("iteration rate: 1500 iter/cpu-s"),
1394 "a synthesized step must render its guest-carrier iteration rate, \
1395 not only '[no samples]'; got:\n{formatted}",
1396 );
1397 }
1398
1399 #[test]
1405 fn format_renders_orphan_window_as_not_measured() {
1406 let mut per_cgroup = std::collections::BTreeMap::new();
1407 per_cgroup.insert(
1408 "cg".to_string(),
1409 crate::assert::PhaseCgroupStats {
1410 off_cpu_pcts: vec![80.0],
1411 total_iterations: 900_000,
1412 ..Default::default()
1413 },
1414 );
1415 let buckets = vec![crate::assert::PhaseBucket {
1416 per_cgroup,
1417 step_index: 1,
1418 label: "Step[0]".to_string(),
1419 start_ms: 0,
1420 end_ms: 0,
1421 sample_count: 0,
1422 metrics: std::collections::BTreeMap::new(),
1423 }];
1424 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1425 let formatted = t.format_with_context(&TimelineContext::default());
1426 assert!(
1427 formatted.contains("window not measured"),
1428 "orphan window must render not-measured; got:\n{formatted}",
1429 );
1430 assert!(
1431 !formatted.contains("(0ms,"),
1432 "orphan must NOT render as 0ms; got:\n{formatted}",
1433 );
1434 assert!(
1438 formatted.contains("per-cgroup:"),
1439 "orphan's per-cgroup sub-block must render; got:\n{formatted}",
1440 );
1441 assert!(
1442 formatted.contains("cg: off-cpu avg=80.0%"),
1443 "orphan carrier's off-cpu reduction must render; got:\n{formatted}",
1444 );
1445 assert!(
1446 formatted.contains("iters=900000"),
1447 "orphan carrier's counters must render; got:\n{formatted}",
1448 );
1449 let cg_line = formatted
1453 .lines()
1454 .find(|l| l.contains("cg: off-cpu"))
1455 .expect("cg line");
1456 assert!(!cg_line.contains("wake p99="), "got:\n{formatted}");
1457 assert!(!cg_line.contains("run-delay mean="), "got:\n{formatted}");
1458 }
1459
1460 #[test]
1467 fn format_orphan_phase_suppresses_cross_orphan_change_detection() {
1468 let real = |step: u16, rate: f64| crate::assert::PhaseBucket {
1469 per_cgroup: Default::default(),
1470 step_index: step,
1471 label: format!("Step[{}]", step - 1),
1472 start_ms: step as u64 * 1000,
1473 end_ms: step as u64 * 1000 + 500,
1474 sample_count: 5,
1475 metrics: std::collections::BTreeMap::from([("iteration_rate".to_string(), rate)]),
1476 };
1477 let mut orphan_pc = std::collections::BTreeMap::new();
1478 orphan_pc.insert(
1479 "cg".to_string(),
1480 crate::assert::PhaseCgroupStats {
1481 off_cpu_pcts: vec![50.0],
1482 ..Default::default()
1483 },
1484 );
1485 let orphan = crate::assert::PhaseBucket {
1486 per_cgroup: orphan_pc,
1487 step_index: 2,
1488 label: "Step[1]".to_string(),
1489 start_ms: 0,
1490 end_ms: 0,
1491 sample_count: 0,
1492 metrics: std::collections::BTreeMap::new(),
1493 };
1494 let ctrl = Timeline::from_phase_buckets(
1497 &[real(1, 10000.0), real(3, 1000.0)],
1498 &[],
1499 &TimelineContext::default(),
1500 );
1501 let ctrl_step3 = ctrl.phases.iter().find(|p| p.index == 3).expect("step3");
1502 assert!(
1503 !ctrl_step3.changes.is_empty(),
1504 "control: adjacent step1->step3 throughput collapse IS flagged; got: {:?}",
1505 ctrl_step3.changes,
1506 );
1507 let t = Timeline::from_phase_buckets(
1510 &[real(1, 10000.0), orphan, real(3, 1000.0)],
1511 &[],
1512 &TimelineContext::default(),
1513 );
1514 let step3 = t.phases.iter().find(|p| p.index == 3).expect("step3");
1515 assert!(
1516 step3.changes.is_empty(),
1517 "orphan between suppresses the cross-orphan change; got: {:?}",
1518 step3.changes,
1519 );
1520 assert!(
1521 t.format_with_context(&TimelineContext::default())
1522 .contains("window not measured"),
1523 "the orphan still renders not-measured",
1524 );
1525 }
1526
1527 #[test]
1531 fn format_does_not_mark_not_measured_when_metrics_present() {
1532 let mut per_cgroup = std::collections::BTreeMap::new();
1533 per_cgroup.insert(
1534 "cg".to_string(),
1535 crate::assert::PhaseCgroupStats {
1536 off_cpu_pcts: vec![50.0],
1537 ..Default::default()
1538 },
1539 );
1540 let buckets = vec![crate::assert::PhaseBucket {
1541 per_cgroup,
1542 step_index: 1,
1543 label: "Step[0]".to_string(),
1544 start_ms: 0,
1545 end_ms: 0,
1546 sample_count: 1,
1547 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1548 }];
1549 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1550 let formatted = t.format_with_context(&TimelineContext::default());
1551 assert!(
1552 !formatted.contains("window not measured"),
1553 "a (0,0) window WITH metrics is captured, not an orphan; got:\n{formatted}",
1554 );
1555 assert!(
1556 formatted.contains("(0ms,"),
1557 "a measured zero-duration window renders 0ms; got:\n{formatted}",
1558 );
1559 }
1560
1561 #[test]
1565 fn format_renders_per_cgroup_subblock_none_aware() {
1566 let mut per_cgroup = std::collections::BTreeMap::new();
1567 per_cgroup.insert(
1568 "cg_a".to_string(),
1569 crate::assert::PhaseCgroupStats {
1570 off_cpu_pcts: vec![80.0, 84.0],
1571 wake_latencies_ns: vec![100_000, 120_000],
1572 run_delays_ns: vec![45_000],
1573 total_iterations: 900_000,
1574 total_migrations: 12,
1575 max_gap_ms: 8,
1576 max_gap_cpu: 3,
1577 ..Default::default()
1578 },
1579 );
1580 per_cgroup.insert(
1581 "cg_b".to_string(),
1582 crate::assert::PhaseCgroupStats {
1583 off_cpu_pcts: vec![],
1584 total_iterations: 80_000,
1585 ..Default::default()
1586 },
1587 );
1588 let buckets = vec![crate::assert::PhaseBucket {
1589 per_cgroup,
1590 step_index: 1,
1591 label: "Step[0]".to_string(),
1592 start_ms: 1000,
1593 end_ms: 6000,
1594 sample_count: 10,
1595 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.5)]),
1596 }];
1597 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1598 let formatted = t.format_with_context(&TimelineContext::default());
1599 assert!(
1600 formatted.contains("per-cgroup:"),
1601 "sub-block header; got:\n{formatted}"
1602 );
1603 assert!(
1604 formatted.contains("cg_a: off-cpu avg=82.0% min=80.0% max=84.0% spread=4.0%"),
1605 "got:\n{formatted}",
1606 );
1607 assert!(
1608 formatted.contains("wake p99="),
1609 "wake segment present; got:\n{formatted}"
1610 );
1611 assert!(
1612 formatted.contains("run-delay mean="),
1613 "run-delay segment present; got:\n{formatted}",
1614 );
1615 assert!(
1616 formatted.contains("iters=900000 migrations=12"),
1617 "counters present; got:\n{formatted}",
1618 );
1619 assert!(
1620 formatted.contains("cg_b: off-cpu n/a"),
1621 "not-measured off-cpu is n/a, NOT 0.0%; got:\n{formatted}",
1622 );
1623 assert!(
1625 formatted.contains("gap=8ms@cpu3"),
1626 "coupled gap renders ms@cpu; got:\n{formatted}",
1627 );
1628 let cg_b_line = formatted
1631 .lines()
1632 .find(|l| l.contains("cg_b:"))
1633 .expect("cg_b line");
1634 assert!(!cg_b_line.contains("wake p99="), "got:\n{formatted}");
1635 assert!(
1636 !cg_b_line.contains("gap="),
1637 "gap omitted at 0; got:\n{formatted}"
1638 );
1639 assert!(
1641 formatted.find("cg_a:").unwrap() < formatted.find("cg_b:").unwrap(),
1642 "cgroups render in name order; got:\n{formatted}",
1643 );
1644 }
1645
1646 #[test]
1651 fn format_renders_stripped_carrier_distinctly() {
1652 let mut per_cgroup = std::collections::BTreeMap::new();
1653 per_cgroup.insert(
1654 "cg_stripped".to_string(),
1655 crate::assert::PhaseCgroupStats {
1656 stripped: true,
1657 total_iterations: 500_000,
1658 total_migrations: 9,
1659 ..Default::default()
1660 },
1661 );
1662 per_cgroup.insert(
1663 "cg_quiet".to_string(),
1664 crate::assert::PhaseCgroupStats {
1666 total_iterations: 1_000,
1667 ..Default::default()
1668 },
1669 );
1670 let buckets = vec![crate::assert::PhaseBucket {
1671 per_cgroup,
1672 step_index: 1,
1673 label: "Step[0]".to_string(),
1674 start_ms: 1000,
1675 end_ms: 6000,
1676 sample_count: 10,
1677 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.5)]),
1678 }];
1679 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1680 let formatted = t.format_with_context(&TimelineContext::default());
1681 assert!(
1682 formatted.contains("cg_stripped: samples stripped (size limit)"),
1683 "stripped carrier shows the size-limit marker; got:\n{formatted}",
1684 );
1685 assert!(
1686 formatted.contains("iters=500000 migrations=9"),
1687 "stripped carrier still renders its surviving counters; got:\n{formatted}",
1688 );
1689 assert!(
1690 formatted.contains("cg_quiet: off-cpu n/a"),
1691 "a not-stripped, not-measured carrier stays n/a; got:\n{formatted}",
1692 );
1693 assert!(
1694 !formatted.contains("cg_quiet: samples stripped"),
1695 "a not-stripped carrier must NOT show the stripped marker; got:\n{formatted}",
1696 );
1697 }
1698
1699 #[test]
1702 fn format_renders_measured_zero_off_cpu_as_zero_not_na() {
1703 let mut per_cgroup = std::collections::BTreeMap::new();
1704 per_cgroup.insert(
1705 "cg".to_string(),
1706 crate::assert::PhaseCgroupStats {
1707 off_cpu_pcts: vec![0.0, 0.0],
1708 ..Default::default()
1709 },
1710 );
1711 let buckets = vec![crate::assert::PhaseBucket {
1712 per_cgroup,
1713 step_index: 1,
1714 label: "Step[0]".to_string(),
1715 start_ms: 1000,
1716 end_ms: 2000,
1717 sample_count: 1,
1718 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1719 }];
1720 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1721 let formatted = t.format_with_context(&TimelineContext::default());
1722 assert!(
1723 formatted.contains("off-cpu avg=0.0%"),
1724 "measured zero off-cpu is 0.0%, not n/a; got:\n{formatted}",
1725 );
1726 assert!(!formatted.contains("off-cpu n/a"), "got:\n{formatted}");
1727 }
1728
1729 #[test]
1735 fn format_renders_measured_zero_wake_and_run_delay_not_omitted() {
1736 let mut per_cgroup = std::collections::BTreeMap::new();
1737 per_cgroup.insert(
1738 "cg".to_string(),
1739 crate::assert::PhaseCgroupStats {
1740 off_cpu_pcts: vec![5.0],
1741 wake_latencies_ns: vec![0],
1742 run_delays_ns: vec![0],
1743 ..Default::default()
1744 },
1745 );
1746 let buckets = vec![crate::assert::PhaseBucket {
1747 per_cgroup,
1748 step_index: 1,
1749 label: "Step[0]".to_string(),
1750 start_ms: 1000,
1751 end_ms: 2000,
1752 sample_count: 1,
1753 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1754 }];
1755 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1756 let formatted = t.format_with_context(&TimelineContext::default());
1757 assert!(
1758 formatted.contains("wake p99=0\u{00b5}s"),
1759 "measured-zero wake renders 0µs, not omitted; got:\n{formatted}",
1760 );
1761 assert!(
1762 formatted.contains("run-delay mean=0\u{00b5}s"),
1763 "measured-zero run-delay renders 0µs, not omitted; got:\n{formatted}",
1764 );
1765 }
1766
1767 #[test]
1770 fn format_caps_per_cgroup_subblock() {
1771 let mut per_cgroup = std::collections::BTreeMap::new();
1772 for i in 0..20 {
1773 per_cgroup.insert(
1774 format!("cg{i:02}"),
1775 crate::assert::PhaseCgroupStats {
1776 off_cpu_pcts: vec![10.0],
1777 ..Default::default()
1778 },
1779 );
1780 }
1781 let buckets = vec![crate::assert::PhaseBucket {
1782 per_cgroup,
1783 step_index: 1,
1784 label: "Step[0]".to_string(),
1785 start_ms: 1000,
1786 end_ms: 2000,
1787 sample_count: 1,
1788 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1789 }];
1790 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1791 let formatted = t.format_with_context(&TimelineContext::default());
1792 assert!(
1793 formatted.contains("(+4 more cgroups)"),
1794 "20 cgroups capped at 16 -> +4 more; got:\n{formatted}",
1795 );
1796 assert!(
1797 formatted.contains("cg15:"),
1798 "first 16 rendered; got:\n{formatted}"
1799 );
1800 assert!(
1801 !formatted.contains("cg16:"),
1802 "cg16 is beyond the cap; got:\n{formatted}",
1803 );
1804 }
1805
1806 #[test]
1809 fn format_omits_per_cgroup_subblock_when_empty() {
1810 let buckets = vec![crate::assert::PhaseBucket {
1811 per_cgroup: Default::default(),
1812 step_index: 1,
1813 label: "Step[0]".to_string(),
1814 start_ms: 1000,
1815 end_ms: 2000,
1816 sample_count: 1,
1817 metrics: std::collections::BTreeMap::from([("avg_imbalance_ratio".to_string(), 1.0)]),
1818 }];
1819 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1820 let formatted = t.format_with_context(&TimelineContext::default());
1821 assert!(
1822 !formatted.contains("per-cgroup:"),
1823 "no sub-block when carriers absent; got:\n{formatted}",
1824 );
1825 }
1826
1827 #[test]
1832 fn format_per_cgroup_cap_boundary_16_and_17() {
1833 let mk = |n: usize| {
1834 let mut per_cgroup = std::collections::BTreeMap::new();
1835 for i in 0..n {
1836 per_cgroup.insert(
1837 format!("cg{i:02}"),
1838 crate::assert::PhaseCgroupStats {
1839 off_cpu_pcts: vec![10.0],
1840 ..Default::default()
1841 },
1842 );
1843 }
1844 let buckets = vec![crate::assert::PhaseBucket {
1845 per_cgroup,
1846 step_index: 1,
1847 label: "Step[0]".to_string(),
1848 start_ms: 1000,
1849 end_ms: 2000,
1850 sample_count: 1,
1851 metrics: std::collections::BTreeMap::from([(
1852 "avg_imbalance_ratio".to_string(),
1853 1.0,
1854 )]),
1855 }];
1856 Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default())
1857 .format_with_context(&TimelineContext::default())
1858 };
1859 let at16 = mk(16);
1860 assert!(at16.contains("cg15:"), "16th cgroup renders; got:\n{at16}");
1861 assert!(
1862 !at16.contains("more cgroups"),
1863 "exactly 16 has NO truncation note; got:\n{at16}",
1864 );
1865 let at17 = mk(17);
1866 assert!(at17.contains("cg15:"), "got:\n{at17}");
1867 assert!(
1868 !at17.contains("cg16:"),
1869 "17th is past the cap; got:\n{at17}"
1870 );
1871 assert!(
1872 at17.contains("(+1 more cgroup)") && !at17.contains("(+1 more cgroups)"),
1873 "17 cgroups -> exactly +1 more (singular 'cgroup'); got:\n{at17}",
1874 );
1875 }
1876
1877 #[test]
1882 fn format_renders_synthesized_step_monitor_metrics() {
1883 let buckets = vec![
1884 crate::assert::PhaseBucket {
1885 per_cgroup: Default::default(),
1886 step_index: 0,
1887 label: "BASELINE".to_string(),
1888 start_ms: 0,
1889 end_ms: 1000,
1890 sample_count: 2,
1891 metrics: std::collections::BTreeMap::new(),
1892 },
1893 crate::assert::PhaseBucket {
1894 per_cgroup: Default::default(),
1895 step_index: 1,
1896 label: "Step[0]".to_string(),
1897 start_ms: 1000,
1898 end_ms: 2000,
1899 sample_count: 0, metrics: std::collections::BTreeMap::from([
1901 ("avg_imbalance_ratio".to_string(), 2.0),
1902 ("max_imbalance_ratio".to_string(), 3.0),
1903 ("avg_dsq_depth".to_string(), 5.0),
1904 ("max_dsq_depth".to_string(), 7.0),
1905 ]),
1906 },
1907 ];
1908 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1909 let formatted = t.format_with_context(&TimelineContext::default());
1910 assert!(
1911 formatted.contains("imbalance: avg=2.0 max=3.0"),
1912 "synthesized bucket's folded imbalance must render, not \
1913 '[no samples]'; got:\n{formatted}",
1914 );
1915 assert!(
1916 formatted.contains("dsq: avg=5 max=7"),
1917 "synthesized bucket's folded dsq must render; got:\n{formatted}",
1918 );
1919 }
1920
1921 #[test]
1927 fn format_no_baseline_bucket_does_not_mislabel_first_step() {
1928 let buckets = vec![
1929 crate::assert::PhaseBucket {
1930 per_cgroup: Default::default(),
1931 step_index: 1, label: "Step[0]".to_string(),
1933 start_ms: 1000,
1934 end_ms: 2000,
1935 sample_count: 3,
1936 metrics: std::collections::BTreeMap::from([(
1937 "avg_imbalance_ratio".to_string(),
1938 1.0,
1939 )]),
1940 },
1941 crate::assert::PhaseBucket {
1942 per_cgroup: Default::default(),
1943 step_index: 2,
1944 label: "Step[1]".to_string(),
1945 start_ms: 2000,
1946 end_ms: 3000,
1947 sample_count: 3,
1948 metrics: std::collections::BTreeMap::from([(
1949 "avg_imbalance_ratio".to_string(),
1950 1.0,
1951 )]),
1952 },
1953 ];
1954 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
1955 let formatted = t.format_with_context(&TimelineContext::default());
1956 assert!(
1957 !formatted.contains("BASELINE"),
1958 "no BASELINE bucket -> no BASELINE label; the first Step must not \
1959 be mislabeled as BASELINE; got:\n{formatted}",
1960 );
1961 assert!(
1962 formatted.contains("Phase 1"),
1963 "the first Step renders as 'Phase 1' (its step_index), not \
1964 'Phase 0'/BASELINE; got:\n{formatted}",
1965 );
1966 }
1967
1968 #[test]
1975 fn format_sparse_step_index_renders_true_step_number() {
1976 let buckets = vec![
1977 crate::assert::PhaseBucket {
1978 per_cgroup: Default::default(),
1979 step_index: 0,
1980 label: "BASELINE".to_string(),
1981 start_ms: 0,
1982 end_ms: 100,
1983 sample_count: 2,
1984 metrics: std::collections::BTreeMap::new(),
1985 },
1986 crate::assert::PhaseBucket {
1987 per_cgroup: Default::default(),
1988 step_index: 3, label: "Step[2]".to_string(),
1990 start_ms: 200,
1991 end_ms: 300,
1992 sample_count: 2,
1993 metrics: std::collections::BTreeMap::from([(
1994 "avg_imbalance_ratio".to_string(),
1995 1.0,
1996 )]),
1997 },
1998 ];
1999 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
2000 let formatted = t.format_with_context(&TimelineContext::default());
2001 assert!(
2002 formatted.contains("Phase 3"),
2003 "a sparse Step at step_index 3 must render 'Phase 3' (its \
2004 step_index), not the enumerate-position 'Phase 1'; got:\n{formatted}",
2005 );
2006 assert!(
2007 !formatted.contains("Phase 1"),
2008 "the enumerate-position 'Phase 1' must NOT appear for a \
2009 step_index-3 bucket; got:\n{formatted}",
2010 );
2011 }
2012
2013 #[test]
2014 fn unsorted_events_sorted() {
2015 let events = vec![stimulus(3000, "StepStart[0]"), stimulus(0, "ScenarioStart")];
2016 let samples: Vec<MonitorSample> = (5..35)
2017 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2018 .collect();
2019 let t = Timeline::build(&events, &samples, 0);
2020 assert_eq!(t.phases.len(), 2);
2021 assert!(t.phases[0].stimulus.is_none());
2023 }
2024
2025 #[test]
2026 fn stuck_detected_in_phase() {
2027 let events = vec![stimulus(0, "ScenarioStart")];
2028 let samples = vec![
2029 sample(600, vec![(1, 0, 5000), (1, 0, 6000)]),
2030 sample(700, vec![(1, 0, 5000), (1, 0, 7000)]), ];
2032 let t = Timeline::build(&events, &samples, 0);
2033 assert_eq!(t.phases[0].metrics.stuck_count, 1);
2034 }
2035
2036 #[test]
2037 fn compute_metrics_stuck_count_accumulates_across_windows() {
2038 let s1 = sample(100, vec![(1, 0, 5000), (1, 0, 6000)]);
2042 let s2 = sample(200, vec![(1, 0, 5000), (1, 0, 6100)]);
2043 let s3 = sample(300, vec![(1, 0, 5000), (1, 0, 6200)]);
2044 let refs: Vec<&MonitorSample> = vec![&s1, &s2, &s3];
2045 let m = compute_metrics(&refs, 0);
2046 assert_eq!(m.stuck_count, 2);
2047 }
2048
2049 #[test]
2050 fn run_level_stuck_count_ge_sum_of_per_phase() {
2051 let s1 = sample(100, vec![(1, 0, 5000), (1, 0, 6000)]);
2057 let s2 = sample(200, vec![(1, 0, 5000), (1, 0, 6100)]);
2058 let s3 = sample(300, vec![(1, 0, 5000), (1, 0, 6200)]);
2059 let samples = vec![s1, s2, s3];
2060 let run_level = crate::monitor::MonitorSummary::from_samples(&samples).stuck_count;
2062 assert_eq!(run_level, 2);
2063 let phase_a: Vec<&MonitorSample> = vec![&samples[0], &samples[1]];
2066 let phase_b: Vec<&MonitorSample> = vec![&samples[2]];
2067 let sum_per_phase =
2068 compute_metrics(&phase_a, 0).stuck_count + compute_metrics(&phase_b, 0).stuck_count;
2069 assert_eq!(
2070 sum_per_phase, 1,
2071 "the (s2,s3) boundary pair is in neither phase"
2072 );
2073 assert!(
2074 run_level > sum_per_phase,
2075 "run-level counts the boundary-straddling pair the per-phase partition drops"
2076 );
2077 }
2078
2079 #[test]
2080 fn compute_metrics_empty() {
2081 let m = compute_metrics(&[], 0);
2082 assert_eq!(m.sample_count, 0);
2083 assert_eq!(m.avg_imbalance, None);
2085 assert_eq!(m.max_imbalance, None);
2086 assert_eq!(m.avg_dsq_depth, None);
2087 assert_eq!(m.avg_nr_running, None);
2088 assert_eq!(m.max_dsq_depth, 0);
2089 }
2090
2091 #[test]
2092 fn stimulus_event_with_detail() {
2093 let e = StimulusEvent {
2094 elapsed_ms: 100,
2095 label: "StepStart[0]".to_string(),
2096 op_kind: Some("SetCpuset".to_string()),
2097 detail: Some("4 cpus".to_string()),
2098 total_iterations: None,
2099 step_index: None,
2100 is_terminal: false,
2101 is_step_end: false,
2102 };
2103 let events = vec![stimulus(0, "ScenarioStart"), e];
2104 let samples: Vec<MonitorSample> = (5..25)
2105 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2106 .collect();
2107 let t = Timeline::build(&events, &samples, 0);
2108 let formatted = t.format_with_context(&TimelineContext::default());
2109 assert!(formatted.contains("SetCpuset"));
2110 assert!(formatted.contains("4 cpus"));
2111 }
2112
2113 #[test]
2114 fn many_phases() {
2115 let events: Vec<StimulusEvent> = (0..10)
2116 .map(|i| stimulus(i * 500, &format!("Step[{i}]")))
2117 .collect();
2118 let samples: Vec<MonitorSample> = (5..55)
2119 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
2120 .collect();
2121 let t = Timeline::build(&events, &samples, 0);
2122 assert_eq!(t.phases.len(), 10);
2123 }
2124
2125 #[test]
2126 fn phase_metrics_accuracy() {
2127 let s1 = sample(600, vec![(1, 3, 100), (4, 5, 200)]); let s2 = sample(700, vec![(2, 1, 300), (2, 7, 400)]); let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2130 let m = compute_metrics(&refs, 0);
2131 assert_eq!(m.sample_count, 2);
2132 assert!((m.avg_imbalance.unwrap() - 2.5).abs() < 0.01); assert!((m.avg_nr_running.unwrap() - 2.25).abs() < 0.01);
2136 assert!((m.max_imbalance.unwrap() - 4.0).abs() < 0.01);
2137 assert_eq!(m.max_dsq_depth, 7);
2138 }
2139
2140 #[test]
2143 fn change_direction_display() {
2144 assert_eq!(format!("{}", ChangeDirection::Improved), "IMPROVEMENT");
2145 assert_eq!(format!("{}", ChangeDirection::Degraded), "DEGRADATION");
2146 }
2147
2148 #[test]
2151 fn compute_metrics_with_event_counters() {
2152 use crate::monitor::ScxEventCounters;
2153
2154 let s1 = MonitorSample {
2155 bpf_map_fields: Vec::new(),
2156 prog_stats: None,
2157 psi_irq: None,
2158 elapsed_ms: 600,
2159 cpus: vec![CpuSnapshot {
2160 nr_running: 2,
2161 local_dsq_depth: 1,
2162 rq_clock: 100,
2163 scx_nr_running: 0,
2164 scx_flags: 0,
2165 event_counters: Some(ScxEventCounters {
2166 select_cpu_fallback: 10,
2167 dispatch_keep_last: 5,
2168 ..Default::default()
2169 }),
2170 schedstat: None,
2171 vcpu_cpu_time_ns: Some(1_000_000_000),
2172 vcpu_perf: None,
2173 avg_irq_util: None,
2174 sched_domains: None,
2175 }],
2176 };
2177 let s2 = MonitorSample {
2178 bpf_map_fields: Vec::new(),
2179 prog_stats: None,
2180 psi_irq: None,
2181 elapsed_ms: 1600,
2182 cpus: vec![CpuSnapshot {
2183 nr_running: 2,
2184 local_dsq_depth: 1,
2185 rq_clock: 200,
2186 scx_nr_running: 0,
2187 scx_flags: 0,
2188 event_counters: Some(ScxEventCounters {
2189 select_cpu_fallback: 110,
2190 dispatch_keep_last: 55,
2191 ..Default::default()
2192 }),
2193 schedstat: None,
2194 vcpu_cpu_time_ns: Some(2_000_000_000),
2195 vcpu_perf: None,
2196 avg_irq_util: None,
2197 sched_domains: None,
2198 }],
2199 };
2200 let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2201 let m = compute_metrics(&refs, 0);
2202 assert!((m.fallback_rate.unwrap() - 100.0).abs() < 0.01);
2204 assert!((m.keep_last_rate.unwrap() - 50.0).abs() < 0.01);
2206
2207 let s2_dilated = MonitorSample {
2208 elapsed_ms: 100_600,
2209 ..s2.clone()
2210 };
2211 let dilated_refs = vec![&s1, &s2_dilated];
2212 let dilated = compute_metrics(&dilated_refs, 0);
2213 assert_eq!(
2214 dilated.fallback_rate, m.fallback_rate,
2215 "100x host wall dilation must not change the vCPU-denominated rate",
2216 );
2217 assert_eq!(dilated.keep_last_rate, m.keep_last_rate);
2218 }
2219
2220 #[test]
2221 fn compute_metrics_no_event_counters() {
2222 let s1 = sample(600, vec![(2, 1, 100)]);
2223 let s2 = sample(700, vec![(2, 1, 200)]);
2224 let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2225 let m = compute_metrics(&refs, 0);
2226 assert!(m.fallback_rate.is_none());
2227 assert!(m.keep_last_rate.is_none());
2228 }
2229
2230 #[test]
2231 fn compute_metrics_counter_reset_clamps_rates_to_non_negative() {
2232 use crate::monitor::ScxEventCounters;
2238
2239 let s1 = MonitorSample {
2240 bpf_map_fields: Vec::new(),
2241 prog_stats: None,
2242 psi_irq: None,
2243 elapsed_ms: 0,
2244 cpus: vec![CpuSnapshot {
2245 nr_running: 2,
2246 local_dsq_depth: 1,
2247 rq_clock: 100,
2248 scx_nr_running: 0,
2249 scx_flags: 0,
2250 event_counters: Some(ScxEventCounters {
2251 select_cpu_fallback: 1000,
2252 dispatch_keep_last: 500,
2253 ..Default::default()
2254 }),
2255 schedstat: None,
2256 vcpu_cpu_time_ns: Some(1_000_000_000),
2257 vcpu_perf: None,
2258 avg_irq_util: None,
2259 sched_domains: None,
2260 }],
2261 };
2262 let s2 = MonitorSample {
2263 bpf_map_fields: Vec::new(),
2264 prog_stats: None,
2265 psi_irq: None,
2266 elapsed_ms: 1000,
2267 cpus: vec![CpuSnapshot {
2268 nr_running: 2,
2269 local_dsq_depth: 1,
2270 rq_clock: 200,
2271 scx_nr_running: 0,
2272 scx_flags: 0,
2273 event_counters: Some(ScxEventCounters {
2274 select_cpu_fallback: 5,
2275 dispatch_keep_last: 2,
2276 ..Default::default()
2277 }),
2278 schedstat: None,
2279 vcpu_cpu_time_ns: Some(2_000_000_000),
2280 vcpu_perf: None,
2281 avg_irq_util: None,
2282 sched_domains: None,
2283 }],
2284 };
2285 let refs: Vec<&MonitorSample> = vec![&s1, &s2];
2286 let m = compute_metrics(&refs, 0);
2287 let fb = m.fallback_rate.expect("reset still produces Some rate");
2288 let kl = m.keep_last_rate.expect("reset still produces Some rate");
2289 assert!(
2290 fb >= 0.0,
2291 "reset must not produce negative fallback_rate, got {fb}"
2292 );
2293 assert!(
2294 kl >= 0.0,
2295 "reset must not produce negative keep_last_rate, got {kl}"
2296 );
2297 }
2298
2299 #[test]
2302 fn format_with_stalls_shown() {
2303 let events = vec![stimulus(0, "ScenarioStart")];
2304 let samples = vec![
2305 sample(600, vec![(1, 0, 5000), (1, 0, 6000)]),
2306 sample(700, vec![(1, 0, 5000), (1, 0, 7000)]), ];
2308 let t = Timeline::build(&events, &samples, 0);
2309 let formatted = t.format_with_context(&TimelineContext::default());
2310 assert!(formatted.contains("stuck: 1"));
2311 }
2312
2313 #[test]
2316 fn format_phase_no_samples() {
2317 let events = vec![
2320 stimulus(0, "ScenarioStart"),
2321 stimulus(100, "StepStart[0]"),
2322 stimulus(50000, "StepStart[1]"),
2323 ];
2324 let samples: Vec<MonitorSample> = (5..15)
2326 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
2327 .collect();
2328 let t = Timeline::build(&events, &samples, 0);
2329 let formatted = t.format_with_context(&TimelineContext::default());
2330 assert!(formatted.contains("[no samples]"));
2332 }
2333
2334 #[test]
2337 fn fallback_rate_degradation_detected() {
2338 use crate::monitor::ScxEventCounters;
2339
2340 let events = vec![stimulus(0, "ScenarioStart"), stimulus(1000, "StepStart[0]")];
2341 let mut samples = Vec::new();
2342 for i in 5..15 {
2344 samples.push(MonitorSample {
2345 bpf_map_fields: Vec::new(),
2346 prog_stats: None,
2347 psi_irq: None,
2348 elapsed_ms: i * 100,
2349 cpus: vec![CpuSnapshot {
2350 nr_running: 2,
2351 local_dsq_depth: 1,
2352 rq_clock: i * 1000,
2353 scx_nr_running: 0,
2354 scx_flags: 0,
2355 event_counters: Some(ScxEventCounters {
2356 select_cpu_fallback: 0,
2357 dispatch_keep_last: 0,
2358 ..Default::default()
2359 }),
2360 schedstat: None,
2361 vcpu_cpu_time_ns: Some(i * 100_000_000),
2362 vcpu_perf: None,
2363 avg_irq_util: None,
2364 sched_domains: None,
2365 }],
2366 });
2367 }
2368 for i in 15..25 {
2370 samples.push(MonitorSample {
2371 bpf_map_fields: Vec::new(),
2372 prog_stats: None,
2373 psi_irq: None,
2374 elapsed_ms: i * 100,
2375 cpus: vec![CpuSnapshot {
2376 nr_running: 2,
2377 local_dsq_depth: 1,
2378 rq_clock: i * 1000,
2379 scx_nr_running: 0,
2380 scx_flags: 0,
2381 event_counters: Some(ScxEventCounters {
2382 select_cpu_fallback: (i as i64 - 15) * 50,
2383 dispatch_keep_last: 0,
2384 ..Default::default()
2385 }),
2386 schedstat: None,
2387 vcpu_cpu_time_ns: Some(i * 100_000_000),
2388 vcpu_perf: None,
2389 avg_irq_util: None,
2390 sched_domains: None,
2391 }],
2392 });
2393 }
2394 let t = Timeline::build(&events, &samples, 0);
2395 let degs: Vec<_> = t
2396 .degradations()
2397 .into_iter()
2398 .filter(|(_, c)| c.metric == "fallback")
2399 .collect();
2400 assert!(!degs.is_empty());
2401 }
2402
2403 #[test]
2406 fn format_with_context_includes_header() {
2407 let events = vec![stimulus(0, "ScenarioStart")];
2408 let samples = vec![
2409 sample(600, vec![(2, 1, 100), (2, 1, 200)]),
2410 sample(700, vec![(2, 1, 300), (2, 1, 400)]),
2411 ];
2412 let t = Timeline::build(&events, &samples, 0);
2413 let ctx = TimelineContext {
2414 kernel: Some("6.14.0-rc3+".to_string()),
2415 topology: Some("2n4l4c2t (16 cpus)".to_string()),
2416 scheduler: Some("scx_mitosis".to_string()),
2417 scenario: Some("proportional".to_string()),
2418 duration_s: Some(20.5),
2419 };
2420 let formatted = t.format_with_context(&ctx);
2421 assert!(formatted.contains("--- timeline ---"));
2422 assert!(formatted.contains("kernel: 6.14.0-rc3+"));
2423 assert!(formatted.contains("topology: 2n4l4c2t (16 cpus)"));
2424 assert!(formatted.contains("scheduler: scx_mitosis"));
2425 assert!(formatted.contains("scenario: proportional"));
2426 assert!(formatted.contains("duration: 20.5s"));
2427 assert!(formatted.contains("BASELINE"));
2428 }
2429
2430 #[test]
2431 fn format_with_context_partial_fields() {
2432 let events = vec![stimulus(0, "ScenarioStart")];
2433 let samples = vec![sample(600, vec![(2, 1, 100)])];
2434 let t = Timeline::build(&events, &samples, 0);
2435 let ctx = TimelineContext {
2436 kernel: None,
2437 topology: Some("1n1l1c1t (1 cpus)".to_string()),
2438 scheduler: None,
2439 scenario: Some("basic".to_string()),
2440 duration_s: None,
2441 };
2442 let formatted = t.format_with_context(&ctx);
2443 assert!(formatted.contains("topology: 1n1l1c1t"));
2444 assert!(formatted.contains("scenario: basic"));
2445 assert!(!formatted.contains("kernel:"));
2446 assert!(!formatted.contains("scheduler:"));
2447 assert!(!formatted.contains("duration:"));
2448 }
2449
2450 #[test]
2451 fn format_with_context_empty_timeline() {
2452 let t = Timeline { phases: vec![] };
2453 let ctx = TimelineContext {
2454 kernel: Some("6.14.0".to_string()),
2455 ..Default::default()
2456 };
2457 assert!(t.format_with_context(&ctx).is_empty());
2458 }
2459
2460 #[test]
2461 fn format_with_context_empty_context() {
2462 let events = vec![stimulus(0, "ScenarioStart")];
2463 let samples = vec![sample(600, vec![(2, 1, 100)])];
2464 let t = Timeline::build(&events, &samples, 0);
2465 let ctx = TimelineContext::default();
2466 let formatted = t.format_with_context(&ctx);
2467 assert!(formatted.contains("--- timeline ---"));
2469 assert!(formatted.contains("BASELINE"));
2470 let after_header = &formatted["--- timeline ---\n".len()..];
2472 assert!(after_header.starts_with('\n'));
2473 }
2474
2475 #[test]
2476 fn garbage_dsq_samples_filtered_from_metrics() {
2477 let events = vec![stimulus(0, "ScenarioStart")];
2481 let garbage_dsq = 1_550_435_906u32;
2482 let samples = vec![
2483 MonitorSample {
2485 bpf_map_fields: Vec::new(),
2486 prog_stats: None,
2487 psi_irq: None,
2488 elapsed_ms: 600,
2489 cpus: vec![CpuSnapshot {
2490 nr_running: 1,
2491 local_dsq_depth: garbage_dsq,
2492 rq_clock: 1000,
2493 ..Default::default()
2494 }],
2495 },
2496 sample(700, vec![(2, 3, 2000)]),
2498 ];
2499 let t = Timeline::build(&events, &samples, 0);
2500 assert_eq!(t.phases.len(), 1);
2501 assert_eq!(t.phases[0].metrics.sample_count, 1);
2503 assert_eq!(t.phases[0].metrics.max_dsq_depth, 3);
2504 }
2505
2506 #[test]
2507 fn all_garbage_samples_yield_no_metrics() {
2508 let events = vec![stimulus(0, "ScenarioStart")];
2509 let samples = vec![MonitorSample {
2510 bpf_map_fields: Vec::new(),
2511 prog_stats: None,
2512 psi_irq: None,
2513 elapsed_ms: 600,
2514 cpus: vec![CpuSnapshot {
2515 nr_running: 1,
2516 local_dsq_depth: 50_000,
2517 rq_clock: 1000,
2518 ..Default::default()
2519 }],
2520 }];
2521 let t = Timeline::build(&events, &samples, 0);
2522 assert_eq!(t.phases[0].metrics.sample_count, 0);
2523 }
2524
2525 #[test]
2530 fn neg_timeline_detects_imbalance_degradation() {
2531 let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2532 let mut samples = Vec::new();
2533 for i in 6..25 {
2534 samples.push(sample(
2535 i * 100,
2536 vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2537 ));
2538 }
2539 for i in 26..45 {
2540 samples.push(sample(
2541 i * 100,
2542 vec![(1, 1, i * 1000), (10, 1, i * 1000 + 100)],
2543 ));
2544 }
2545 let t = Timeline::build(&events, &samples, 0);
2546 assert_eq!(t.phases.len(), 2, "must have 2 phases");
2547 assert!(!t.degradations().is_empty());
2548
2549 assert!(
2551 t.phases[0].metrics.sample_count > 0,
2552 "baseline must have samples"
2553 );
2554 assert!(
2555 (t.phases[0].metrics.avg_imbalance.unwrap() - 1.0).abs() < 0.5,
2556 "baseline imbalance should be ~1.0, got {:?}",
2557 t.phases[0].metrics.avg_imbalance,
2558 );
2559
2560 assert!(
2562 t.phases[1].metrics.sample_count > 0,
2563 "phase 1 must have samples"
2564 );
2565 assert!(
2566 t.phases[1]
2567 .stimulus
2568 .as_ref()
2569 .is_some_and(|s| s.label == "StepStart[0]"),
2570 "phase 1 stimulus must be StepStart[0]",
2571 );
2572
2573 let degs = t.degradations();
2574 assert!(!degs.is_empty());
2575 let (phase, change) = °s[0];
2576 assert_eq!(phase.index, 1);
2577 assert_eq!(change.metric, "imbalance");
2578 assert_eq!(change.direction, ChangeDirection::Degraded);
2579 let delta = change.after - change.before;
2580 assert!(delta > 0.0, "delta must be positive for degradation");
2581 assert!(
2582 delta > IMBALANCE_THRESHOLD,
2583 "delta {:.1} must exceed threshold {:.1}",
2584 delta,
2585 IMBALANCE_THRESHOLD
2586 );
2587 assert!(
2588 change.before < 2.0,
2589 "before should be low: {:.1}",
2590 change.before
2591 );
2592 assert!(
2593 change.after > 5.0,
2594 "after should be high: {:.1}",
2595 change.after
2596 );
2597
2598 let formatted = t.format_with_context(&TimelineContext::default());
2600 assert!(
2601 formatted.contains("BASELINE"),
2602 "format must include BASELINE phase"
2603 );
2604 assert!(formatted.contains("Phase 1"), "format must include Phase 1");
2605 assert!(
2606 formatted.contains("DEGRADATION"),
2607 "format must include DEGRADATION label"
2608 );
2609 assert!(
2610 formatted.contains("imbalance"),
2611 "format must name the metric"
2612 );
2613 }
2614
2615 #[test]
2616 fn neg_timeline_detects_dsq_depth_degradation() {
2617 let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2618 let mut samples = Vec::new();
2619 for i in 6..25 {
2620 samples.push(sample(
2621 i * 100,
2622 vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2623 ));
2624 }
2625 for i in 26..45 {
2626 samples.push(sample(
2627 i * 100,
2628 vec![(2, 20, i * 1000), (2, 20, i * 1000 + 100)],
2629 ));
2630 }
2631 let t = Timeline::build(&events, &samples, 0);
2632 assert!(
2633 !t.degradations().is_empty(),
2634 "DSQ depth jump must be detected"
2635 );
2636 let degs = t.degradations();
2637 let dsq_deg = degs.iter().find(|(_, c)| c.metric == "dsq_depth");
2638 assert!(dsq_deg.is_some(), "must detect dsq_depth degradation");
2639 let (phase, change) = dsq_deg.unwrap();
2640 assert_eq!(phase.index, 1);
2641 assert_eq!(change.direction, ChangeDirection::Degraded);
2642 let delta = change.after - change.before;
2643 assert!(
2644 delta > DSQ_THRESHOLD,
2645 "dsq delta {:.1} must exceed threshold {:.1}",
2646 delta,
2647 DSQ_THRESHOLD
2648 );
2649 assert!(
2650 change.before < 5.0,
2651 "before dsq should be low: {:.1}",
2652 change.before
2653 );
2654 assert!(
2655 change.after > 15.0,
2656 "after dsq should be high: {:.1}",
2657 change.after
2658 );
2659
2660 let formatted = t.format_with_context(&TimelineContext::default());
2661 assert!(
2662 formatted.contains("dsq_depth"),
2663 "format must name dsq_depth"
2664 );
2665 assert!(
2666 formatted.contains("DEGRADATION"),
2667 "format must label degradation"
2668 );
2669 }
2670
2671 #[test]
2672 fn neg_timeline_no_degradation_when_stable() {
2673 let events = vec![stimulus(0, "ScenarioStart"), stimulus(2000, "StepStart[0]")];
2674 let mut samples = Vec::new();
2675 for i in 6..45 {
2676 samples.push(sample(
2677 i * 100,
2678 vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)],
2679 ));
2680 }
2681 let t = Timeline::build(&events, &samples, 0);
2682 assert_eq!(t.phases.len(), 2, "must have 2 phases");
2683 assert!(t.phases[0].metrics.sample_count > 0);
2684 assert!(t.phases[1].metrics.sample_count > 0);
2685 assert!(
2686 t.degradations().is_empty(),
2687 "stable phases must not show degradation"
2688 );
2689 assert!(t.degradations().is_empty());
2690 for phase in &t.phases {
2692 assert!(
2693 phase.changes.is_empty(),
2694 "phase {} should have no changes",
2695 phase.index
2696 );
2697 }
2698 }
2699
2700 #[test]
2703 fn detect_change_higher_is_worse_positive_delta_degraded() {
2704 let c = detect_change(1.0, 5.0, 0.5, "imbalance", true).unwrap();
2705 assert_eq!(c.direction, ChangeDirection::Degraded);
2706 assert_eq!(c.metric, "imbalance");
2707 assert!((c.before - 1.0).abs() < f64::EPSILON);
2708 assert!((c.after - 5.0).abs() < f64::EPSILON);
2709 }
2710
2711 #[test]
2712 fn detect_change_higher_is_worse_negative_delta_improved() {
2713 let c = detect_change(5.0, 1.0, 0.5, "imbalance", true).unwrap();
2714 assert_eq!(c.direction, ChangeDirection::Improved);
2715 }
2716
2717 #[test]
2718 fn detect_change_lower_is_worse_negative_delta_degraded() {
2719 let c = detect_change(100.0, 50.0, 10.0, "throughput", false).unwrap();
2720 assert_eq!(c.direction, ChangeDirection::Degraded);
2721 }
2722
2723 #[test]
2724 fn detect_change_lower_is_worse_positive_delta_improved() {
2725 let c = detect_change(50.0, 100.0, 10.0, "throughput", false).unwrap();
2726 assert_eq!(c.direction, ChangeDirection::Improved);
2727 }
2728
2729 #[test]
2730 fn detect_change_below_threshold_returns_none() {
2731 assert!(detect_change(1.0, 1.3, 0.5, "imbalance", true).is_none());
2732 }
2733
2734 #[test]
2735 fn detect_change_exactly_at_threshold_returns_none() {
2736 assert!(detect_change(1.0, 1.5, 0.5, "imbalance", true).is_none());
2737 }
2738
2739 #[test]
2748 fn detect_boundary_changes_flags_throughput_across_synthesized_phase() {
2749 let before = PhaseMetrics {
2750 sample_count: 30,
2751 iteration_rate: Some(1000.0),
2752 avg_imbalance: Some(1.0),
2753 ..Default::default()
2754 };
2755 let after = PhaseMetrics {
2756 sample_count: 0, iteration_rate: Some(300.0), avg_imbalance: Some(99.0), ..Default::default()
2760 };
2761 let changes = detect_boundary_changes(&before, &after);
2762 let throughput: Vec<_> = changes
2763 .iter()
2764 .filter(|c| c.metric == "throughput")
2765 .collect();
2766 assert_eq!(
2767 throughput.len(),
2768 1,
2769 "throughput collapse across a synthesized step must be flagged: {changes:?}",
2770 );
2771 assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
2772 assert!(
2773 !changes.iter().any(|c| c.metric == "imbalance"),
2774 "monitor metrics must stay gated when a side has 0 samples: {changes:?}",
2775 );
2776 }
2777
2778 #[test]
2783 fn detect_boundary_changes_reports_monitor_metrics_when_both_sampled() {
2784 let before = PhaseMetrics {
2785 sample_count: 30,
2786 iteration_rate: Some(1000.0),
2787 avg_imbalance: Some(1.0),
2788 ..Default::default()
2789 };
2790 let after = PhaseMetrics {
2791 sample_count: 30,
2792 iteration_rate: Some(1000.0), avg_imbalance: Some(5.0), ..Default::default()
2795 };
2796 let changes = detect_boundary_changes(&before, &after);
2797 assert!(
2798 changes.iter().any(|c| c.metric == "imbalance"),
2799 "imbalance change must surface when both sides sampled: {changes:?}",
2800 );
2801 assert!(
2802 !changes.iter().any(|c| c.metric == "throughput"),
2803 "unchanged throughput must not be flagged: {changes:?}",
2804 );
2805 }
2806
2807 #[test]
2812 fn detect_boundary_changes_flags_nr_running_jump() {
2813 let before = PhaseMetrics {
2814 sample_count: 30,
2815 avg_nr_running: Some(1.0),
2816 ..Default::default()
2817 };
2818 let after = PhaseMetrics {
2819 sample_count: 30,
2820 avg_nr_running: Some(3.0), ..Default::default()
2822 };
2823 let changes = detect_boundary_changes(&before, &after);
2824 let nr: Vec<_> = changes
2825 .iter()
2826 .filter(|c| c.metric == "nr_running")
2827 .collect();
2828 assert_eq!(
2829 nr.len(),
2830 1,
2831 "avg_nr_running jump must be flagged: {changes:?}"
2832 );
2833 assert_eq!(
2834 nr[0].direction,
2835 ChangeDirection::Degraded,
2836 "rising runqueue depth is a degradation",
2837 );
2838 let after_synth = PhaseMetrics {
2840 sample_count: 0,
2841 avg_nr_running: Some(3.0),
2842 ..Default::default()
2843 };
2844 assert!(
2845 !detect_boundary_changes(&before, &after_synth)
2846 .iter()
2847 .any(|c| c.metric == "nr_running"),
2848 "nr_running must stay gated when a side has 0 samples",
2849 );
2850 }
2851
2852 #[test]
2855 fn format_phases_renders_avg_nr_running() {
2856 let samples: Vec<MonitorSample> = (5..25)
2858 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (4, 1, i * 1000 + 100)]))
2859 .collect();
2860 let events = vec![stimulus(0, "ScenarioStart")];
2861 let t = Timeline::build(&events, &samples, 0);
2862 let formatted = t.format_with_context(&TimelineContext::default());
2863 assert!(
2864 formatted.contains("nr_run: avg="),
2865 "per-phase narrative must render avg_nr_running: {formatted}",
2866 );
2867 }
2868
2869 #[test]
2874 fn detect_boundary_changes_throughput_exactly_at_threshold_not_flagged() {
2875 let before = PhaseMetrics {
2876 sample_count: 30,
2877 iteration_rate: Some(1000.0),
2878 ..Default::default()
2879 };
2880 let after = PhaseMetrics {
2881 sample_count: 30,
2882 iteration_rate: Some(700.0), ..Default::default()
2884 };
2885 assert_eq!((700.0 - 1000.0) / 1000.0, -ITERATION_RATE_REL_THRESHOLD);
2886 assert!(
2887 !detect_boundary_changes(&before, &after)
2888 .iter()
2889 .any(|c| c.metric == "throughput"),
2890 "an exactly-30% relative change must not flag (strict >)",
2891 );
2892 }
2893
2894 #[test]
2901 fn detect_boundary_changes_synthesized_to_synthesized_flags_throughput() {
2902 let before = PhaseMetrics {
2903 sample_count: 0,
2904 iteration_rate: Some(1000.0),
2905 avg_imbalance: Some(1.0),
2906 ..Default::default()
2907 };
2908 let after = PhaseMetrics {
2909 sample_count: 0,
2910 iteration_rate: Some(300.0), avg_imbalance: Some(99.0), ..Default::default()
2913 };
2914 let changes = detect_boundary_changes(&before, &after);
2915 let throughput: Vec<_> = changes
2916 .iter()
2917 .filter(|c| c.metric == "throughput")
2918 .collect();
2919 assert_eq!(
2920 throughput.len(),
2921 1,
2922 "zero->zero throughput collapse must flag: {changes:?}"
2923 );
2924 assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
2925 assert!(
2926 !changes.iter().any(|c| c.metric == "imbalance"),
2927 "monitor metrics gated on both zero-sample sides: {changes:?}",
2928 );
2929 }
2930
2931 fn stimulus_with_iters(elapsed_ms: u64, label: &str, total_iterations: u64) -> StimulusEvent {
2934 StimulusEvent {
2935 elapsed_ms,
2936 label: label.to_string(),
2937 op_kind: None,
2938 detail: None,
2939 total_iterations: Some(total_iterations),
2940 step_index: None,
2941 is_terminal: false,
2942 is_step_end: false,
2943 }
2944 }
2945
2946 #[test]
2947 fn consecutive_wall_events_do_not_create_iteration_rate() {
2948 let events = vec![
2949 stimulus_with_iters(0, "ScenarioStart", 0),
2950 stimulus_with_iters(3000, "StepStart[0]", 3000),
2951 ];
2952 let samples: Vec<MonitorSample> = (5..35)
2953 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2954 .collect();
2955 let t = Timeline::build(&events, &samples, 0);
2956 assert_eq!(t.phases.len(), 2);
2957 let rate = t.phases[0].metrics.iteration_rate;
2958 assert!(
2959 rate.is_none(),
2960 "wall-only stimulus events must not produce iteration_rate"
2961 );
2962 }
2963
2964 #[test]
2965 fn wall_events_without_iterations_do_not_create_iteration_rate() {
2966 let events = vec![stimulus(0, "ScenarioStart"), stimulus(3000, "StepStart[0]")];
2968 let samples: Vec<MonitorSample> = (5..35)
2969 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
2970 .collect();
2971 let t = Timeline::build(&events, &samples, 0);
2972 assert!(t.phases[0].metrics.iteration_rate.is_none());
2973 assert!(t.phases[1].metrics.iteration_rate.is_none());
2974 }
2975
2976 fn wire_event(
2981 elapsed_ms: u32,
2982 step_index: u16,
2983 total_iterations: u64,
2984 ) -> crate::vmm::wire::StimulusEvent {
2985 crate::vmm::wire::StimulusEvent {
2986 elapsed_ms,
2987 step_index,
2988 op_count: 0,
2989 op_kinds: 0,
2990 cgroup_count: 0,
2991 worker_count: 1,
2992 total_iterations,
2993 }
2994 }
2995
2996 #[test]
2997 fn from_wire_zero_iterations_is_some_baseline() {
2998 let te = StimulusEvent::from_wire(&wire_event(0, 1, 0));
3003 assert_eq!(te.total_iterations, Some(0));
3004 assert_eq!(te.step_index, Some(1));
3005 assert!(!te.is_terminal);
3006 assert!(
3007 !te.is_step_end,
3008 "a StepStart-derived event is not a StepEnd"
3009 );
3010 }
3011
3012 #[test]
3013 fn from_step_end_carries_step_index_and_marks_step_end() {
3014 let te = StimulusEvent::from_step_end(&wire_event(1_900, 1, 9_000));
3020 assert_eq!(te.step_index, Some(1));
3021 assert_eq!(te.total_iterations, Some(9_000));
3022 assert!(te.is_step_end, "StepEnd-derived event must set is_step_end");
3023 assert!(
3024 !te.is_terminal,
3025 "StepEnd is a per-step boundary, not the scenario terminal",
3026 );
3027 }
3028
3029 #[test]
3030 fn from_wire_first_step_zero_baseline_yields_rate() {
3031 let events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(3000, 2, 3000)]
3037 .iter()
3038 .map(StimulusEvent::from_wire)
3039 .collect();
3040 let samples: Vec<MonitorSample> = (5..35)
3041 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3042 .collect();
3043 let t = Timeline::build(&events, &samples, 0);
3044 assert!(t.phases[0].metrics.iteration_rate.is_none());
3045 }
3046
3047 #[test]
3048 fn terminal_event_gives_last_step_rate_without_phantom_phase() {
3049 let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3054 .iter()
3055 .map(StimulusEvent::from_wire)
3056 .collect();
3057 events.push(StimulusEvent::terminal(4000, 10000));
3058 let samples: Vec<MonitorSample> = (5..45)
3059 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3060 .collect();
3061 let t = Timeline::build(&events, &samples, 0);
3062 assert_eq!(
3063 t.phases.len(),
3064 2,
3065 "two step events -> two phases; terminal seeds none",
3066 );
3067 assert!(t.phases[1].metrics.iteration_rate.is_none());
3068 }
3069
3070 #[test]
3071 fn build_filters_step_end_events_no_phantom_phase() {
3072 let events: Vec<StimulusEvent> = vec![
3079 StimulusEvent::from_wire(&wire_event(0, 1, 0)),
3080 StimulusEvent::from_step_end(&wire_event(1_900, 1, 9_000)),
3081 StimulusEvent::from_wire(&wire_event(2_000, 2, 9_000)),
3082 ];
3083 let samples: Vec<MonitorSample> = (5..35)
3084 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3085 .collect();
3086 let t = Timeline::build(&events, &samples, 0);
3087 assert_eq!(
3088 t.phases.len(),
3089 2,
3090 "two StepStart events -> two phases; the interleaved StepEnd seeds none",
3091 );
3092 }
3093
3094 #[test]
3095 fn build_pairs_step_local_when_step_end_events_present() {
3096 let events: Vec<StimulusEvent> = vec![
3105 StimulusEvent::from_wire(&wire_event(0, 1, 0)), StimulusEvent::from_step_end(&wire_event(1_000, 1, 5_000)), StimulusEvent::from_wire(&wire_event(1_100, 2, 0)), StimulusEvent::from_step_end(&wire_event(2_100, 2, 3_000)), ];
3110 let samples: Vec<MonitorSample> = (1..30)
3111 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3112 .collect();
3113 let t = Timeline::build(&events, &samples, 0);
3114 assert_eq!(
3115 t.phases.len(),
3116 2,
3117 "two StepStart events -> two phases (each StepEnd seeds none)",
3118 );
3119 assert!(t.phases[0].metrics.iteration_rate.is_none());
3120 assert!(t.phases[1].metrics.iteration_rate.is_none());
3121 }
3122
3123 #[test]
3124 fn build_stalled_step_with_step_end_reports_measured_zero_not_cross_step() {
3125 let events: Vec<StimulusEvent> = vec![
3134 StimulusEvent::from_wire(&wire_event(0, 1, 0)), StimulusEvent::from_step_end(&wire_event(1_000, 1, 0)), StimulusEvent::from_wire(&wire_event(1_100, 2, 500)), StimulusEvent::from_step_end(&wire_event(2_100, 2, 5_500)), ];
3139 let samples: Vec<MonitorSample> = (1..30)
3140 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3141 .collect();
3142 let t = Timeline::build(&events, &samples, 0);
3143 assert_eq!(t.phases.len(), 2);
3144 assert_eq!(
3145 t.phases[0].metrics.iteration_rate, None,
3146 "wall-only events do not produce the delivered-CPU rate",
3147 );
3148 assert!(t.phases[1].metrics.iteration_rate.is_none());
3149 }
3150
3151 #[test]
3152 fn terminal_event_single_step_rate() {
3153 let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0)]
3158 .iter()
3159 .map(StimulusEvent::from_wire)
3160 .collect();
3161 events.push(StimulusEvent::terminal(3000, 9000));
3162 let samples: Vec<MonitorSample> = (5..35)
3163 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3164 .collect();
3165 let t = Timeline::build(&events, &samples, 0);
3166 assert_eq!(
3167 t.phases.len(),
3168 1,
3169 "single step -> one phase; terminal adds none"
3170 );
3171 assert!(t.phases[0].metrics.iteration_rate.is_none());
3172 }
3173
3174 #[test]
3175 fn terminal_event_stalled_last_step_does_not_fabricate_cpu_rate() {
3176 let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3177 .iter()
3178 .map(StimulusEvent::from_wire)
3179 .collect();
3180 events.push(StimulusEvent::terminal(4000, 4000)); let samples: Vec<MonitorSample> = (5..45)
3182 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3183 .collect();
3184 let t = Timeline::build(&events, &samples, 0);
3185 assert_eq!(t.phases.len(), 2);
3186 assert_eq!(
3187 t.phases[1].metrics.iteration_rate, None,
3188 "wall-only terminal events do not produce iteration_rate",
3189 );
3190 }
3191
3192 #[test]
3193 fn iteration_rate_counter_decrease_yields_no_rate() {
3194 let events: Vec<StimulusEvent> = [
3201 wire_event(0, 1, 0),
3202 wire_event(2000, 2, 5000),
3203 wire_event(3000, 3, 1000), ]
3205 .iter()
3206 .map(StimulusEvent::from_wire)
3207 .collect();
3208 let samples: Vec<MonitorSample> = (5..35)
3209 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3210 .collect();
3211 let t = Timeline::build(&events, &samples, 0);
3212 assert!(
3214 t.phases[1].metrics.iteration_rate.is_none(),
3215 "a counter decrease must not manufacture a (negative) rate",
3216 );
3217 }
3218
3219 #[test]
3220 fn iteration_rate_zero_duration_yields_no_rate() {
3221 let events: Vec<StimulusEvent> = [wire_event(1000, 1, 0), wire_event(1000, 2, 2000)]
3225 .iter()
3226 .map(StimulusEvent::from_wire)
3227 .collect();
3228 let samples: Vec<MonitorSample> = (5..35)
3229 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3230 .collect();
3231 let t = Timeline::build(&events, &samples, 0);
3232 assert!(
3233 t.phases[0].metrics.iteration_rate.is_none(),
3234 "zero-duration pair must not divide; rate stays None",
3235 );
3236 }
3237
3238 #[test]
3239 fn terminal_not_last_does_not_misalign_or_misattribute() {
3240 let mut events: Vec<StimulusEvent> = [wire_event(0, 1, 0), wire_event(2000, 2, 4000)]
3247 .iter()
3248 .map(StimulusEvent::from_wire)
3249 .collect();
3250 events.push(StimulusEvent::terminal(500, 9000));
3252 let samples: Vec<MonitorSample> = (5..45)
3253 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3254 .collect();
3255 let t = Timeline::build(&events, &samples, 0);
3256 assert_eq!(
3258 t.phases.len(),
3259 2,
3260 "terminal position must not change phase count"
3261 );
3262 assert_eq!(
3265 t.phases[0].metrics.iteration_rate, None,
3266 "misordered wall-only events must not produce iteration_rate",
3267 );
3268 }
3269
3270 #[test]
3271 fn throughput_degradation_detected() {
3272 let events = vec![
3276 stimulus_with_iters(0, "ScenarioStart", 0),
3277 stimulus_with_iters(2000, "StepStart[0]", 10000),
3278 stimulus_with_iters(4000, "StepEnd[0]", 11000),
3279 ];
3280 let samples: Vec<MonitorSample> = (5..45)
3281 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3282 .collect();
3283 let t = Timeline::build(&events, &samples, 0);
3284 assert_eq!(t.phases.len(), 3);
3285 assert!(
3286 t.phases
3287 .iter()
3288 .all(|phase| phase.metrics.iteration_rate.is_none())
3289 );
3290 let degs: Vec<_> = t
3291 .degradations()
3292 .into_iter()
3293 .filter(|(_, c)| c.metric == "throughput")
3294 .collect();
3295 assert!(
3296 degs.is_empty(),
3297 "wall-only stimulus deltas must not create throughput findings"
3298 );
3299 }
3300
3301 #[test]
3302 fn wall_counter_collapse_does_not_create_throughput_finding() {
3303 let events = vec![
3304 stimulus_with_iters(0, "ScenarioStart", 0),
3305 stimulus_with_iters(2000, "StepStart[0]", 10000), stimulus_with_iters(4000, "StepStart[1]", 10000), ];
3308 let samples: Vec<MonitorSample> = (5..45)
3309 .map(|i| sample(i * 100, vec![(2, 1, i * 1000)]))
3310 .collect();
3311 let t = Timeline::build(&events, &samples, 0);
3312 assert_eq!(
3313 t.phases[1].metrics.iteration_rate, None,
3314 "wall-only events do not create a delivered-CPU zero",
3315 );
3316 let degs: Vec<_> = t
3317 .degradations()
3318 .into_iter()
3319 .filter(|(p, c)| p.index == 1 && c.metric == "throughput")
3320 .collect();
3321 assert!(degs.is_empty());
3322 }
3323
3324 #[test]
3325 fn throughput_improvement_detected() {
3326 let events = vec![
3330 stimulus_with_iters(0, "ScenarioStart", 0),
3331 stimulus_with_iters(2000, "StepStart[0]", 500),
3332 stimulus_with_iters(4000, "StepEnd[0]", 10500),
3333 ];
3334 let samples: Vec<MonitorSample> = (5..45)
3335 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3336 .collect();
3337 let t = Timeline::build(&events, &samples, 0);
3338 let improvements: Vec<_> = t
3339 .phases
3340 .iter()
3341 .flat_map(|p| p.changes.iter())
3342 .filter(|c| c.metric == "throughput" && c.direction == ChangeDirection::Improved)
3343 .collect();
3344 assert!(
3345 improvements.is_empty(),
3346 "wall-only stimulus deltas must not create throughput findings"
3347 );
3348 }
3349
3350 #[test]
3351 fn throughput_stable_below_threshold() {
3352 let events = vec![
3354 stimulus_with_iters(0, "ScenarioStart", 0),
3355 stimulus_with_iters(2000, "StepStart[0]", 2000),
3356 stimulus_with_iters(4000, "StepEnd[0]", 3800),
3357 ];
3358 let samples: Vec<MonitorSample> = (5..45)
3359 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3360 .collect();
3361 let t = Timeline::build(&events, &samples, 0);
3362 let throughput_changes: Vec<_> = t
3363 .phases
3364 .iter()
3365 .flat_map(|p| p.changes.iter())
3366 .filter(|c| c.metric == "throughput")
3367 .collect();
3368 assert!(
3369 throughput_changes.is_empty(),
3370 "10% change should not trigger throughput change detection"
3371 );
3372 }
3373
3374 #[test]
3375 fn from_phase_buckets_maps_known_metrics_and_renders_phase_block() {
3376 use crate::assert::PhaseBucket;
3377 use std::collections::BTreeMap;
3378 let mut s0_metrics = BTreeMap::new();
3379 s0_metrics.insert("max_dsq_depth".to_string(), 7.0);
3380 s0_metrics.insert("avg_dsq_depth".to_string(), 2.5);
3381 s0_metrics.insert("max_imbalance_ratio".to_string(), 3.5);
3382 s0_metrics.insert("avg_imbalance_ratio".to_string(), 1.8);
3383 s0_metrics.insert("avg_nr_running".to_string(), 3.0);
3384 s0_metrics.insert("total_fallback".to_string(), 200.0);
3385 s0_metrics.insert("fallback_per_vcpu_sec".to_string(), 40.0);
3386 let buckets = vec![
3387 PhaseBucket {
3388 per_cgroup: Default::default(),
3389 step_index: 0,
3390 label: "BASELINE".to_string(),
3391 start_ms: 0,
3392 end_ms: 1000,
3393 sample_count: 5,
3394 metrics: BTreeMap::new(),
3395 },
3396 PhaseBucket {
3397 per_cgroup: Default::default(),
3398 step_index: 1,
3399 label: "Step[0]".to_string(),
3400 start_ms: 1000,
3401 end_ms: 6000,
3402 sample_count: 20,
3403 metrics: s0_metrics,
3404 },
3405 ];
3406 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3407 assert_eq!(t.phases.len(), 2);
3408 assert!(t.phases[0].stimulus.is_none());
3410 assert_eq!(t.phases[0].metrics.sample_count, 5);
3411 assert_eq!(t.phases[0].metrics.max_dsq_depth, 0);
3412 assert!(t.phases[1].stimulus.is_some());
3415 assert_eq!(t.phases[1].stimulus.as_ref().unwrap().label, "Step[0]");
3416 assert_eq!(t.phases[1].metrics.sample_count, 20);
3417 assert_eq!(t.phases[1].metrics.max_dsq_depth, 7);
3418 assert!((t.phases[1].metrics.avg_dsq_depth.unwrap() - 2.5).abs() < f64::EPSILON);
3419 assert!((t.phases[1].metrics.max_imbalance.unwrap() - 3.5).abs() < f64::EPSILON);
3420 assert!((t.phases[1].metrics.avg_imbalance.unwrap() - 1.8).abs() < f64::EPSILON);
3421 assert!((t.phases[1].metrics.avg_nr_running.unwrap() - 3.0).abs() < f64::EPSILON);
3424 assert_eq!(t.phases[0].metrics.avg_nr_running, None);
3425 assert_eq!(t.phases[1].metrics.fallback_rate, Some(40.0));
3428 assert_eq!(t.phases[1].metrics.keep_last_rate, None);
3430 assert_eq!(t.phases[1].metrics.iteration_rate, None);
3432 let formatted = t.format_with_context(&TimelineContext::default());
3434 assert!(formatted.contains("--- timeline ---"));
3435 assert!(formatted.contains("BASELINE"));
3436 assert!(formatted.contains("Step[0]"));
3437 }
3438
3439 #[test]
3453 fn from_phase_buckets_flags_throughput_into_synthesized_step() {
3454 use crate::assert::PhaseBucket;
3455 use std::collections::BTreeMap;
3456 let mut baseline_metrics = BTreeMap::new();
3457 baseline_metrics.insert("iteration_rate".to_string(), 1000.0);
3458 baseline_metrics.insert("avg_imbalance_ratio".to_string(), 1.0);
3459 let mut step_metrics = BTreeMap::new();
3460 step_metrics.insert("iteration_rate".to_string(), 300.0); step_metrics.insert("avg_imbalance_ratio".to_string(), 99.0); let buckets = vec![
3463 PhaseBucket {
3464 per_cgroup: Default::default(),
3465 step_index: 0,
3466 label: "BASELINE".to_string(),
3467 start_ms: 0,
3468 end_ms: 1000,
3469 sample_count: 5,
3470 metrics: baseline_metrics,
3471 },
3472 PhaseBucket {
3473 per_cgroup: Default::default(),
3474 step_index: 1,
3475 label: "Step[0]".to_string(),
3476 start_ms: 1000,
3477 end_ms: 6000,
3478 sample_count: 0, metrics: step_metrics,
3480 },
3481 ];
3482 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3483 assert_eq!(t.phases[1].metrics.sample_count, 0);
3484 assert_eq!(t.phases[1].metrics.iteration_rate, Some(300.0));
3485 let throughput: Vec<_> = t.phases[1]
3486 .changes
3487 .iter()
3488 .filter(|c| c.metric == "throughput")
3489 .collect();
3490 assert_eq!(
3491 throughput.len(),
3492 1,
3493 "throughput collapse into a synthesized step must surface via \
3494 from_phase_buckets: {:?}",
3495 t.phases[1].changes,
3496 );
3497 assert_eq!(throughput[0].direction, ChangeDirection::Degraded);
3498 assert!(
3499 !t.phases[1].changes.iter().any(|c| c.metric == "imbalance"),
3500 "monitor metrics must stay gated for a zero-sample side: {:?}",
3501 t.phases[1].changes,
3502 );
3503 }
3504
3505 #[test]
3517 fn from_phase_buckets_detects_boundary_degradation() {
3518 use crate::assert::PhaseBucket;
3519 use std::collections::BTreeMap;
3520 let mut base = BTreeMap::new();
3521 base.insert("avg_imbalance_ratio".to_string(), 1.0);
3522 base.insert("avg_dsq_depth".to_string(), 1.0);
3523 let mut step = BTreeMap::new();
3524 step.insert("avg_imbalance_ratio".to_string(), 2.0); step.insert("avg_dsq_depth".to_string(), 6.0); let buckets = vec![
3527 PhaseBucket {
3528 per_cgroup: Default::default(),
3529 step_index: 0,
3530 label: "BASELINE".to_string(),
3531 start_ms: 0,
3532 end_ms: 1000,
3533 sample_count: 5,
3534 metrics: base,
3535 },
3536 PhaseBucket {
3537 per_cgroup: Default::default(),
3538 step_index: 1,
3539 label: "Step[0]".to_string(),
3540 start_ms: 1000,
3541 end_ms: 6000,
3542 sample_count: 20,
3543 metrics: step,
3544 },
3545 ];
3546 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3547 assert!(
3549 t.phases[0].changes.is_empty(),
3550 "BASELINE has no prior phase to diff; changes belong to the entered phase",
3551 );
3552 let changes = &t.phases[1].changes;
3553 let imb = changes
3554 .iter()
3555 .find(|c| c.metric == "imbalance")
3556 .expect("imbalance change must fire (1.0 -> 2.0 crosses 0.5)");
3557 assert_eq!(imb.direction, ChangeDirection::Degraded);
3558 assert!((imb.before - 1.0).abs() < f64::EPSILON);
3559 assert!((imb.after - 2.0).abs() < f64::EPSILON);
3560 let dsq = changes
3561 .iter()
3562 .find(|c| c.metric == "dsq_depth")
3563 .expect("dsq_depth change must fire (1.0 -> 6.0 crosses 3.0)");
3564 assert_eq!(dsq.direction, ChangeDirection::Degraded);
3565 assert!((dsq.before - 1.0).abs() < f64::EPSILON);
3566 assert!((dsq.after - 6.0).abs() < f64::EPSILON);
3567 }
3568
3569 #[test]
3572 fn from_phase_buckets_subthreshold_records_no_change() {
3573 use crate::assert::PhaseBucket;
3574 use std::collections::BTreeMap;
3575 let mut base = BTreeMap::new();
3576 base.insert("avg_imbalance_ratio".to_string(), 1.0);
3577 base.insert("avg_dsq_depth".to_string(), 1.0);
3578 let mut step = BTreeMap::new();
3579 step.insert("avg_imbalance_ratio".to_string(), 1.2); step.insert("avg_dsq_depth".to_string(), 2.0); let buckets = vec![
3582 PhaseBucket {
3583 per_cgroup: Default::default(),
3584 step_index: 0,
3585 label: "BASELINE".to_string(),
3586 start_ms: 0,
3587 end_ms: 1000,
3588 sample_count: 5,
3589 metrics: base,
3590 },
3591 PhaseBucket {
3592 per_cgroup: Default::default(),
3593 step_index: 1,
3594 label: "Step[0]".to_string(),
3595 start_ms: 1000,
3596 end_ms: 6000,
3597 sample_count: 20,
3598 metrics: step,
3599 },
3600 ];
3601 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3602 assert!(
3603 t.phases[1].changes.is_empty(),
3604 "sub-threshold deltas must not record a boundary change",
3605 );
3606 }
3607
3608 #[test]
3612 fn from_phase_buckets_detects_boundary_improvement() {
3613 use crate::assert::PhaseBucket;
3614 use std::collections::BTreeMap;
3615 let mut base = BTreeMap::new();
3616 base.insert("avg_imbalance_ratio".to_string(), 2.0);
3617 let mut step = BTreeMap::new();
3618 step.insert("avg_imbalance_ratio".to_string(), 1.0); let buckets = vec![
3620 PhaseBucket {
3621 per_cgroup: Default::default(),
3622 step_index: 0,
3623 label: "BASELINE".to_string(),
3624 start_ms: 0,
3625 end_ms: 1000,
3626 sample_count: 5,
3627 metrics: base,
3628 },
3629 PhaseBucket {
3630 per_cgroup: Default::default(),
3631 step_index: 1,
3632 label: "Step[0]".to_string(),
3633 start_ms: 1000,
3634 end_ms: 6000,
3635 sample_count: 20,
3636 metrics: step,
3637 },
3638 ];
3639 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3640 let imb = t.phases[1]
3641 .changes
3642 .iter()
3643 .find(|c| c.metric == "imbalance")
3644 .expect("imbalance change must fire (2.0 -> 1.0 crosses 0.5)");
3645 assert_eq!(
3646 imb.direction,
3647 ChangeDirection::Improved,
3648 "a decreasing imbalance is an improvement, not a degradation",
3649 );
3650 }
3651
3652 #[test]
3660 fn from_phase_buckets_correlates_real_stimulus_op_and_detail() {
3661 use crate::assert::PhaseBucket;
3662 use std::collections::BTreeMap;
3663 let event = StimulusEvent {
3664 elapsed_ms: 1000,
3665 label: "Step[0]".to_string(),
3666 op_kind: Some("SetCpuset".to_string()),
3667 detail: Some("4 cpus".to_string()),
3668 total_iterations: None,
3669 step_index: Some(1),
3670 is_terminal: false,
3671 is_step_end: false,
3672 };
3673 let buckets = vec![
3674 PhaseBucket {
3675 per_cgroup: Default::default(),
3676 step_index: 0,
3677 label: "BASELINE".to_string(),
3678 start_ms: 0,
3679 end_ms: 1000,
3680 sample_count: 5,
3681 metrics: BTreeMap::new(),
3682 },
3683 PhaseBucket {
3684 per_cgroup: Default::default(),
3685 step_index: 1,
3686 label: "Step[0]".to_string(),
3687 start_ms: 1000,
3688 end_ms: 6000,
3689 sample_count: 20,
3690 metrics: BTreeMap::new(),
3691 },
3692 ];
3693 let t = Timeline::from_phase_buckets(&buckets, &[event], &TimelineContext::default());
3694 let stim = t.phases[1]
3695 .stimulus
3696 .as_ref()
3697 .expect("Step[0] phase carries a stimulus");
3698 assert_eq!(
3699 stim.op_kind.as_deref(),
3700 Some("SetCpuset"),
3701 "the correlated event's op_kind must be carried, not the None placeholder",
3702 );
3703 assert_eq!(stim.detail.as_deref(), Some("4 cpus"));
3704 }
3705
3706 #[test]
3707 fn from_phase_buckets_zero_duration_window_emits_no_rate() {
3708 use crate::assert::PhaseBucket;
3709 use std::collections::BTreeMap;
3710 let mut metrics = BTreeMap::new();
3711 metrics.insert("total_fallback".to_string(), 100.0);
3712 let bucket = PhaseBucket {
3713 per_cgroup: Default::default(),
3714 step_index: 1,
3715 label: "Step[0]".to_string(),
3716 start_ms: 500,
3717 end_ms: 500,
3718 sample_count: 1,
3719 metrics,
3720 };
3721 let t = Timeline::from_phase_buckets(&[bucket], &[], &TimelineContext::default());
3722 assert_eq!(t.phases[0].metrics.fallback_rate, None);
3726 }
3727
3728 #[test]
3729 fn from_phase_buckets_absent_imbalance_metric_is_none_not_zero() {
3730 use crate::assert::PhaseBucket;
3734 use std::collections::BTreeMap;
3735 let bucket = PhaseBucket {
3736 per_cgroup: Default::default(),
3737 step_index: 1,
3738 label: "Step[0]".to_string(),
3739 start_ms: 100,
3740 end_ms: 600,
3741 sample_count: 3,
3742 metrics: BTreeMap::new(),
3743 };
3744 let t = Timeline::from_phase_buckets(&[bucket], &[], &TimelineContext::default());
3745 assert_eq!(t.phases[0].metrics.avg_imbalance, None);
3746 assert_eq!(t.phases[0].metrics.max_imbalance, None);
3747 assert_eq!(t.phases[0].metrics.avg_dsq_depth, None);
3748 }
3749
3750 #[test]
3751 fn from_phase_buckets_sorts_by_step_index() {
3752 use crate::assert::PhaseBucket;
3753 use std::collections::BTreeMap;
3754 let buckets = vec![
3759 PhaseBucket {
3760 per_cgroup: Default::default(),
3761 step_index: 2,
3762 label: "Step[1]".to_string(),
3763 start_ms: 2000,
3764 end_ms: 3000,
3765 sample_count: 5,
3766 metrics: BTreeMap::new(),
3767 },
3768 PhaseBucket {
3769 per_cgroup: Default::default(),
3770 step_index: 0,
3771 label: "BASELINE".to_string(),
3772 start_ms: 0,
3773 end_ms: 500,
3774 sample_count: 2,
3775 metrics: BTreeMap::new(),
3776 },
3777 PhaseBucket {
3778 per_cgroup: Default::default(),
3779 step_index: 1,
3780 label: "Step[0]".to_string(),
3781 start_ms: 500,
3782 end_ms: 2000,
3783 sample_count: 5,
3784 metrics: BTreeMap::new(),
3785 },
3786 ];
3787 let t = Timeline::from_phase_buckets(&buckets, &[], &TimelineContext::default());
3788 assert_eq!(t.phases.len(), 3);
3789 assert_eq!(t.phases[0].start_ms, 0);
3790 assert_eq!(t.phases[1].start_ms, 500);
3791 assert_eq!(t.phases[2].start_ms, 2000);
3792 }
3793
3794 #[test]
3795 fn iteration_rate_in_formatted_output() {
3796 let events = vec![
3797 stimulus_with_iters(0, "ScenarioStart", 0),
3798 stimulus_with_iters(2000, "StepStart[0]", 5000),
3799 ];
3800 let samples: Vec<MonitorSample> = (5..25)
3801 .map(|i| sample(i * 100, vec![(2, 1, i * 1000), (2, 1, i * 1000 + 100)]))
3802 .collect();
3803 let t = Timeline::build(&events, &samples, 0);
3804 let formatted = t.format_with_context(&TimelineContext::default());
3805 assert!(!formatted.contains("iteration rate:"));
3806 assert!(!formatted.contains("iter/cpu-s"));
3807 }
3808}