ktstr/test_support/eval/post_vm.rs
1//! Host-side post_vm plumbing: the post_vm error marker types
2//! (ScxBpfErrorMatcherMismatch,
3//! PostVmAssertionFailure, HostSkipRequest, ExpectAutoReproSatisfied),
4//! the conditional/unconditional callback combiner + dispatch, the
5//! post_vm_skip helper, and skip-sidecar recording. Split out of
6//! eval/mod.rs to keep the module under the size ceiling.
7
8use super::*;
9
10/// Marker error type attached as `anyhow::Context` to the failure
11/// `Err` produced when an scx_bpf_error matcher
12/// ([`crate::assert::Assert::expect_scx_bpf_error_contains`] or
13/// [`crate::assert::Assert::expect_scx_bpf_error_matches`]) mismatched
14/// the captured scheduler log / sched_ext dump corpus.
15///
16/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
17/// downcasts the error chain for this marker in the `expect_err = true`
18/// branch and refuses to invert the verdict to a pass — a reproducer
19/// that fired the WRONG bug must fail loudly, not silently invert to
20/// "test passed" via `expect_err`. Without the marker, the matcher's
21/// diagnostic surfaces in stderr but the exit code follows the normal
22/// expect_err inversion path.
23#[derive(Debug, Clone, Copy)]
24pub(crate) struct ScxBpfErrorMatcherMismatch;
25
26impl std::fmt::Display for ScxBpfErrorMatcherMismatch {
27 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
28 write!(
29 f,
30 "scx_bpf_error matcher mismatch — the reproducer matcher rejected \
31 this failure mode; expect_err inversion bypassed"
32 )
33 }
34}
35
36impl std::error::Error for ScxBpfErrorMatcherMismatch {}
37
38/// Marker error type attached as `anyhow::Context` to the `Err`
39/// `resolve_scheduler` returns when the `Discover` `cargo build -p
40/// <sched>` FAILED. A scheduler that cannot be built is a failed test —
41/// the resolver never falls back to a possibly-stale pre-built binary.
42///
43/// Dispatch (`crate::test_support::dispatch`) downcasts the error chain
44/// for this marker and forces a hard FAIL EVEN under `expect_err = true`.
45/// The semantic boundary mirrors [`PostVmAssertionFailure`]: `expect_err`
46/// inverts a GUEST-side expected failure, but a build-infra failure is a
47/// HOST-side fault that must never masquerade as the expected guest
48/// failure — without the marker an `expect_err` test whose scheduler
49/// build broke would silently invert to PASS. Same
50/// `anyhow::Context` attachment + `downcast_ref` chain-walk as the
51/// sibling markers; the dispatch guard sits with the other host-side
52/// hard-fail markers, before the `expect_err` inversion.
53#[derive(Debug, Clone, Copy)]
54pub(crate) struct SchedulerBuildRefused;
55
56impl std::fmt::Display for SchedulerBuildRefused {
57 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58 write!(
59 f,
60 "scheduler build refused — the workspace build failed; a \
61 scheduler that cannot be built is a failed test \
62 (expect_err inversion bypassed)"
63 )
64 }
65}
66
67impl std::error::Error for SchedulerBuildRefused {}
68
69/// Marker attached to host-framework failures which occur after a VM run has
70/// started (for example, immutable scheduler CAS coordination or cast-analysis
71/// mapping failure).
72///
73/// These are neither guest behavior nor host-capacity outcomes. Dispatch must
74/// hard-fail them before resource skip classification or `expect_err`
75/// inversion can reinterpret the result.
76#[derive(Debug, Clone, Copy)]
77pub(crate) struct FrameworkInfrastructureFailure;
78
79impl std::fmt::Display for FrameworkInfrastructureFailure {
80 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81 write!(
82 f,
83 "ktstr framework infrastructure failed \
84 (resource classification and expect_err inversion bypassed)"
85 )
86 }
87}
88
89impl std::error::Error for FrameworkInfrastructureFailure {}
90
91/// Marker error type attached as `anyhow::Context` to the failure
92/// `Err` produced by `run_ktstr_test_inner_impl` when a host-side
93/// `post_vm` / `post_vm_unconditional` callback returned `Err`
94/// (which `evaluate_vm_result` has already folded into the verdict —
95/// as an `Other` detail in the parse-success arm, as a message prefix
96/// in the parse-fail arms).
97///
98/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
99/// downcasts the error chain for this marker and refuses to invert the
100/// verdict to a pass — even under `expect_err = true`. The semantic
101/// boundary: `expect_err` inverts a GUEST-side expected failure (the
102/// scheduler stalled, the workload bailed), but a HOST-side `post_vm`
103/// assertion is always honored. A failure-dump render test that
104/// triggers an expected stall to PRODUCE the dump, then asserts the
105/// dump's contents in `post_vm`, must fail loudly when the dump renders
106/// wrong — not silently invert to "passed" because the stall it relied
107/// on was "expected". Without the marker, the post_vm diagnostic
108/// surfaces in stderr but the exit code follows the normal expect_err
109/// inversion path (a false PASS).
110///
111/// Mirrors [`ScxBpfErrorMatcherMismatch`]: same `anyhow::Context`
112/// attachment, same `downcast_ref` chain-walk at the dispatch arm. The
113/// dispatch arm is positioned AFTER the resource-contention / topology
114/// skip arms (a skip means the test never ran) but BEFORE the
115/// [`ExpectAutoReproSatisfied`] and `expect_err` inversion arms, so a
116/// real host-side regression wins over any inversion.
117#[derive(Debug, Clone, Copy)]
118pub(crate) struct PostVmAssertionFailure;
119
120impl std::fmt::Display for PostVmAssertionFailure {
121 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
122 write!(
123 f,
124 "host-side post_vm assertion failed — expect_err inversion bypassed \
125 (a host-side check is honored even when the accompanying guest-side \
126 failure is expected)"
127 )
128 }
129}
130
131impl std::error::Error for PostVmAssertionFailure {}
132
133/// Marker error type attached as `anyhow::Context` to a `post_vm` /
134/// `post_vm_unconditional` `Err` to request a test SKIP (not a
135/// failure): the host-side callback determined the run is
136/// INCONCLUSIVE — the VM could not produce the artifact the assertion
137/// needs (e.g. a load-starved VM whose BPF probe never attached, so
138/// the failure dump is a placeholder), as opposed to a real
139/// regression. The eval fn detects this marker (context-aware
140/// `downcast_ref`, at the `HostSkipRequest` gate) and returns
141/// [`crate::assert::AssertResult::skip`] instead of folding the `Err`
142/// into the verdict.
143///
144/// A real [`PostVmAssertionFailure`] in a sibling callback DOMINATES:
145/// [`combine_post_vm_errs`] preserves the skip marker only when BOTH
146/// callbacks request skip (or only one callback ran); a genuine
147/// failure alongside a skip request collapses to a failure, so a skip
148/// request can never mask a regression.
149#[derive(Debug, Clone, Copy)]
150pub(crate) struct HostSkipRequest;
151
152impl std::fmt::Display for HostSkipRequest {
153 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
154 write!(
155 f,
156 "host-side post_vm requested skip — the run is inconclusive \
157 (the VM could not produce the artifact the assertion needs)"
158 )
159 }
160}
161
162impl std::error::Error for HostSkipRequest {}
163
164/// Marker error type attached as `anyhow::Context` to the failure
165/// `Err` produced by `evaluate_vm_result` when
166/// [`apply_expect_auto_repro_inversion`] has set
167/// `result.expect_auto_repro_satisfied = true`: the primary VM
168/// produced a Fail AND a shape-valid `.repro.wprof.pb` artifact
169/// landed on disk from the auto-repro VM.
170///
171/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
172/// downcasts the error chain for this marker and routes the verdict
173/// to `EXIT_PASS`. The underlying `AssertResult` is NOT mutated —
174/// the original failure detail still surfaces in stderr/dump
175/// rendering so an operator chasing why `expect_auto_repro` fired
176/// sees the original failure trail alongside the inversion notice.
177#[derive(Debug, Clone, Copy)]
178pub(crate) struct ExpectAutoReproSatisfied;
179
180impl std::fmt::Display for ExpectAutoReproSatisfied {
181 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182 write!(
183 f,
184 "expect_auto_repro satisfied — the primary test failed and the \
185 auto-repro VM produced a shape-valid .repro.wprof.pb artifact; \
186 verdict inverted to PASS"
187 )
188 }
189}
190
191impl std::error::Error for ExpectAutoReproSatisfied {}
192
193/// Marker error type attached as `anyhow::Context` to the failure `Err`
194/// [`render_failure_verdict_message`]
195/// builds when `entry.survives_storm` is set AND the failing
196/// [`AssertResult`] carries a scheduler-death
197/// `DetailKind` (`SchedulerCrashed` / `SchedulerExitedCleanly` /
198/// `SchedulerDiedUnknownReason`). `err_to_exit_code` downcasts it and forces
199/// `EXIT_FAIL` with a survival-specific explainer, positioned BEFORE the
200/// [`ExpectAutoReproSatisfied`] / `expect_err` inversion arms so a survival
201/// violation can never be inverted to PASS (the validate-time
202/// `survives_storm`/`expect_err` mutex already forbids that combination;
203/// the ordering is defense-in-depth). Mirrors [`ScxBpfErrorMatcherMismatch`].
204#[derive(Debug, Clone, Copy)]
205pub(crate) struct SurvivesStormViolated;
206
207impl std::fmt::Display for SurvivesStormViolated {
208 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
209 write!(
210 f,
211 "survives_storm asserted but the scx scheduler did not survive \
212 the run — it died or was ejected during a hold"
213 )
214 }
215}
216
217impl std::error::Error for SurvivesStormViolated {}
218
219/// Combine the conditional and unconditional `post_vm` failure
220/// signals. When both callbacks fail in the same run, surface
221/// BOTH errors in a single chained message so a debugging
222/// operator sees both regressions on the first pass — a `.or()`
223/// would silently drop the unconditional signal whenever the
224/// conditional also fired, defeating the whole point of the
225/// unconditional callback.
226pub(crate) fn combine_post_vm_errs(
227 conditional: Option<anyhow::Error>,
228 unconditional: Option<anyhow::Error>,
229) -> Option<anyhow::Error> {
230 match (conditional, unconditional) {
231 (Some(c), Some(u)) => {
232 // A genuine failure dominates a skip request: collapse to a
233 // skip only when BOTH callbacks requested skip (both
234 // inconclusive). Otherwise a real PostVmAssertionFailure
235 // must surface, so the chained message wins and the
236 // HostSkipRequest marker is intentionally dropped.
237 let both_skip = c.downcast_ref::<HostSkipRequest>().is_some()
238 && u.downcast_ref::<HostSkipRequest>().is_some();
239 let combined = anyhow::anyhow!("post_vm: {c:#}; post_vm_unconditional: {u:#}");
240 Some(if both_skip {
241 combined.context(HostSkipRequest)
242 } else {
243 combined
244 })
245 }
246 (Some(c), None) => Some(c),
247 (None, Some(u)) => Some(u),
248 (None, None) => None,
249 }
250}
251
252/// Request a test SKIP from a `post_vm` / `post_vm_unconditional`
253/// callback: `return Err(post_vm_skip(reason))` when the run is
254/// INCONCLUSIVE — the VM could not produce the artifact the assertion
255/// needs (e.g. a load-starved VM whose BPF probe never attached,
256/// leaving a placeholder failure dump), as distinct from a real
257/// regression. The framework detects the attached `HostSkipRequest`
258/// marker and converts the run to
259/// [`crate::assert::AssertResult::skip`] instead of a failure.
260///
261/// A genuine `Err` from a sibling callback dominates (see
262/// `combine_post_vm_errs`): a skip request never masks a regression.
263pub fn post_vm_skip(reason: impl Into<String>) -> anyhow::Error {
264 anyhow::anyhow!("{}", reason.into()).context(HostSkipRequest)
265}
266
267/// Witnessed-starvation evidence for capture-bearing assertions: the
268/// host dilation `D` that PROVES the run was contended enough to
269/// explain missing/degraded periodic captures, or `None` on a quiet
270/// (or unwitnessed) host.
271///
272/// The periodic capture chain is readiness-gated (KASLR publish +
273/// accessor-init worker) and its boundaries are wall-anchored inside
274/// the workload window; under heavy host saturation the cold-cache
275/// accessor build alone can outlast a short workload, so every
276/// boundary defers past run end and `periodic_fired == 0` with zero
277/// real captures — an ENVIRONMENTAL outcome, not a capture-pipeline
278/// regression (observed across the periodic/freeze test family on
279/// saturated 96-CPU CI runners; the same cells pass idle). Assertions
280/// that require captures consult this first and convert their failure
281/// into a host-side SKIP (`post_vm_skip`) when starvation is proven,
282/// mirroring the tri-state latency gates: a quiet-host miss still
283/// FAILS (the regression the assertion exists to catch).
284///
285/// Witness preference: Body-phase dilation (the capture boundaries
286/// live inside the workload window), falling back to the whole-run
287/// schedstat `D`. The 1.5 bar follows `PERF_ISOLATION_D_MAX`'s
288/// derivation (worst measured-quiet `D` ≈ 1.21; > 1.5 is unambiguous
289/// external contention). `None` (no witness sampled) is NOT proof —
290/// callers must fail in that case; absence of evidence cannot launder
291/// a regression.
292pub fn capture_starvation_witness(result: &crate::vmm::VmResult) -> Option<f64> {
293 const CAPTURE_STARVATION_D_MAX: f64 = 1.5;
294 let d = result
295 .contention_witness
296 .as_ref()
297 .and_then(|w| w.body_dilation())
298 .or_else(|| {
299 result
300 .host_vcpu_schedstat
301 .as_ref()
302 .and_then(|s| s.dilation())
303 })?;
304 (d > CAPTURE_STARVATION_D_MAX).then_some(d)
305}
306
307/// Environmental skip for a zero-scheduler-activity reading (e.g.
308/// `nr_dispatched` / `nr_yielded` read 0 across every periodic sample)
309/// that the guest's OWN sched_ext runnable-stall watchdog explains: when
310/// the console carries a watchdog `ScxExitKind::Stall` exit AND
311/// [`capture_starvation_witness`] proves the host was contended, the BPF
312/// scheduler was ejected because a descheduling host dilated the 5s
313/// GUEST-time stall window past the timeout — not because the scheduler
314/// failed to dispatch. Returns `Err(post_vm_skip(..))` in that case; `Ok`
315/// otherwise, so a Stall on a quiet host (no witness) or any non-Stall
316/// zero still reaches the caller's own regression `ensure!`.
317///
318/// Callers gate their zero-activity failure on it:
319/// ```ignore
320/// if !any_progress {
321/// stall_ejection_skip(result)?;
322/// }
323/// anyhow::ensure!(any_progress, "... never dispatched ...");
324/// ```
325pub fn stall_ejection_skip(result: &crate::vmm::VmResult) -> anyhow::Result<()> {
326 let stalled = crate::monitor::dmesg_scx::parse_kmsg_window(&result.stderr)
327 .iter()
328 .any(|e| e.kind == crate::monitor::dmesg_scx::ScxExitKind::Stall);
329 if stalled && let Some(d) = capture_starvation_witness(result) {
330 return Err(post_vm_skip(format!(
331 "scx scheduler ejected by the guest sched_ext runnable-stall \
332 watchdog under witnessed host contention (D={d:.2}): the 5s stall \
333 timer measures GUEST time, which a descheduling host dilates into \
334 a false stall — the zero-activity reading is that ejection, not a \
335 scheduler regression; environmental non-verdict",
336 )));
337 }
338 Ok(())
339}
340
341/// Generic sub-minimum starvation skip: `Err(post_vm_skip(..))` when the
342/// run produced FEWER data-bearing units than the caller's assertion
343/// requires (`have < need`) AND [`capture_starvation_witness`] proves the
344/// host was contended enough to explain the shortfall; `Ok(())`
345/// otherwise. The caller passes the SAME count its own assertion checks
346/// (real captures, PSI-carrying captures, series length, …) so the gate
347/// and the assertion cannot disagree about the boundary — an earlier
348/// zero-only gate let `1 of 6` runs through to fail on a `>= 2` minimum
349/// under the identical environmental condition. Quiet-host shortfalls
350/// (zero OR sub-minimum) still fall through and fail with the caller's
351/// specific diagnosis.
352pub fn starved_below_minimum_skip(
353 result: &crate::vmm::VmResult,
354 have: usize,
355 need: usize,
356 what: &str,
357) -> anyhow::Result<()> {
358 if have >= need {
359 return Ok(());
360 }
361 // PRIMARY arm — readiness vs window: when the periodic prereqs
362 // (KASLR publish + accessors) became ready at/after the capture
363 // window's end, or never became ready at all, every boundary was
364 // structurally unreachable — zero/short captures were inevitable at
365 // ANY host dilation. This is the honest signal the D-threshold arm
366 // missed in the field: a ~7 s cold accessor build outruns a 4-5 s
367 // workload window at D ≈ 1.2, well under any defensible contention
368 // bar. Skip carries both timestamps.
369 let ready = result.periodic_prereqs_ready;
370 let window_end = result.periodic_window_end;
371 let structurally_impossible = match (ready, window_end) {
372 // Never ready: nothing could ever fire.
373 (None, _) => true,
374 // Ready, but at/after the window closed.
375 (Some(r), Some(w)) => r >= w,
376 // Ready but the window never resolved: the boundaries were
377 // never computed, so no capture could fire.
378 (Some(_), None) => true,
379 };
380 if result.periodic_target > 0 && structurally_impossible {
381 return Err(post_vm_skip(format!(
382 "only {have} of the required {need} {what}: the capture prereqs \
383 (KASLR publish + accessor init) {} the capture window \
384 (prereqs_ready={:?}, window_end={:?}) — captures were \
385 structurally impossible this run at any host dilation; \
386 environmental non-verdict",
387 if ready.is_none() {
388 "never became ready within"
389 } else {
390 "became ready only after"
391 },
392 ready,
393 window_end,
394 )));
395 }
396 // SECONDARY arm — witnessed contention: readiness landed in time
397 // but the captures still fell short (e.g. rendezvous degradation,
398 // data-bearing rows missing) under a provably saturated host.
399 if let Some(d) = capture_starvation_witness(result) {
400 return Err(post_vm_skip(format!(
401 "only {have} of the required {need} {what} under witnessed host \
402 contention (D={d:.2}, prereqs_ready={ready:?}, \
403 window_end={window_end:?}): the capture chain was starved below \
404 the assertion's minimum — environmental non-verdict; the \
405 dependent assertions cannot be evaluated",
406 )));
407 }
408 Ok(())
409}
410
411/// Periodic-capture starvation gate for capture-requiring post_vm
412/// assertions: [`starved_below_minimum_skip`] keyed on the bridge's
413/// REAL (non-placeholder) capture count vs the test's stated minimum.
414/// Call at the TOP of a post_vm callback with the same minimum its
415/// assertions enforce: `periodic_starvation_gate(result, 2)?;`. Tests
416/// whose assertions key on a narrower data-bearing count (e.g. "N
417/// captures carrying per-cgroup PSI") should ALSO call
418/// [`starved_below_minimum_skip`] with that count at the assertion
419/// site. No-op when the entry configured no periodic captures.
420pub fn periodic_starvation_gate(
421 result: &crate::vmm::VmResult,
422 min_real: usize,
423) -> anyhow::Result<()> {
424 if result.periodic_target == 0 {
425 return Ok(());
426 }
427 starved_below_minimum_skip(
428 result,
429 result.snapshot_bridge.periodic_real_count() as usize,
430 min_real,
431 "real (non-placeholder) periodic captures",
432 )
433}
434
435/// Dispatch the entry's `post_vm` + `post_vm_unconditional`
436/// callbacks and combine their failure signals.
437///
438/// - `post_vm` runs only when the guest reported a non-Fail
439/// `AssertResult` (Skip / Inconclusive / Pass) — the
440/// `guest_already_failed` parameter folds the
441/// `parse_assert_result_from_drain` lookup the call site does.
442/// The skip mirrors the suppression contract documented on
443/// `KtstrTestEntry::post_vm`.
444///
445/// - `post_vm_unconditional` ALWAYS runs — bypasses the
446/// guest-fail suppression that gates `post_vm`. The callback
447/// owns its own skip-on-crash logic (or doesn't, when the
448/// intent is "assert on host-side artifact regardless of
449/// guest-side outcome").
450///
451/// Both callbacks route through [`invoke_post_vm_callback`] so a
452/// panic in either body becomes an `anyhow::Error` rather than
453/// unwinding past the call site (which would leak VM resources;
454/// see the helper doc).
455///
456/// Returns the combined `Option<anyhow::Error>` via
457/// [`combine_post_vm_errs`]: when both callbacks fail, the
458/// chained message names both errors so the operator sees both
459/// regressions on the first pass instead of a two-pass debug
460/// cycle. `.or()` would silently drop the unconditional fail
461/// when the conditional also fired.
462pub(crate) fn run_post_vm_callbacks(
463 entry: &KtstrTestEntry,
464 result: &crate::vmm::VmResult,
465 guest_already_failed: bool,
466) -> Option<anyhow::Error> {
467 let conditional = if guest_already_failed {
468 None
469 } else {
470 entry
471 .post_vm
472 .and_then(|cb| invoke_post_vm_callback(cb, result, "post_vm"))
473 };
474 let unconditional = entry
475 .post_vm_unconditional
476 .and_then(|cb| invoke_post_vm_callback(cb, result, "post_vm_unconditional"));
477 combine_post_vm_errs(conditional, unconditional)
478}
479
480/// Invoke a `post_vm` / `post_vm_unconditional` callback with panic
481/// catch. Converts a panic to `anyhow::Error` so the panic message
482/// surfaces in the test failure output AND the rest of the
483/// post-VM teardown (`write_placeholder_failure_dump_if_missing`,
484/// `drop(vm)` releasing CPU/LLC flocks + guest memory + kernel-cache
485/// reader flock) still runs.
486///
487/// Without the catch, a panicking callback would unwind past the
488/// placeholder-dump emission and past `drop(vm)`, leaking VM
489/// resources (flocks, guest memory) until process exit or the next
490/// test's drop reclaims them. Same hazard for `Ok` returns from
491/// callbacks that subsequently panic in their inner state — both
492/// paths fold into this single guard.
493///
494/// `label` is woven into the error message so the operator sees
495/// which callback panicked (`post_vm` vs `post_vm_unconditional`)
496/// when both are wired and both fire.
497///
498/// Returns `Some(err)` when the callback returns `Err` OR panics;
499/// returns `None` when the callback returns `Ok(())`. Mirrors the
500/// shape `.err()` produces from `Result` so the caller's
501/// `.and_then(|cb| ...)` flows unchanged.
502///
503/// Under `panic = "abort"` (release builds — see `Cargo.toml
504/// [profile.release]`), `catch_unwind` is a no-op: a panic aborts
505/// the process before this function returns. The wrap is still
506/// safe — `catch_unwind` is always defined, just inert — and the
507/// debug builds get the leak protection that exposes regressions
508/// before they ship.
509pub(crate) fn invoke_post_vm_callback(
510 cb: super::super::PostVmCallback,
511 result: &crate::vmm::VmResult,
512 label: &'static str,
513) -> Option<anyhow::Error> {
514 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cb(result))) {
515 Ok(Ok(())) => None,
516 Ok(Err(e)) => Some(e),
517 Err(payload) => {
518 let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {
519 (*s).to_string()
520 } else if let Some(s) = payload.downcast_ref::<String>() {
521 s.clone()
522 } else {
523 "<non-string panic payload>".to_string()
524 };
525 Some(anyhow::anyhow!("{label} callback panicked: {msg}"))
526 }
527 }
528}
529
530/// Write a skip sidecar for `entry`, logging to stderr on failure
531/// without propagating the error. Called wherever a run is skipped
532/// before producing a real result: the skip-class catch-all and the
533/// VM build / VM run arms in [`run_ktstr_test_inner`] (each fires on a
534/// host-incapacity class — `TopologyInsufficient` or
535/// `PerfModeUnavailable`; transient `ResourceContention` is a
536/// retryable failure and records no skip), and the
537/// performance-mode / coverage gates at the plain-run entry points in
538/// the crate `dispatch` module. All must record the skip for stats
539/// tooling but cannot meaningfully handle a sidecar-write failure
540/// beyond logging it — the skip itself is still valid; only post-run
541/// stats tooling loses visibility.
542pub(crate) fn record_skip_sidecar(
543 entry: &KtstrTestEntry,
544 topo: Option<&crate::test_support::topo::TopoOverride>,
545) -> crate::vmm::topology::Topology {
546 // Resolve the topology the run of this (entry, override) WOULD boot,
547 // via the same resolve_vm_topology the run path uses, so a preset's
548 // skip and run record the identical topology -> identical
549 // variant_hash -> the retry overwrites instead of coexisting. For a
550 // plain test (topo = None) this is entry.topology. Returned so a
551 // host-class caller can write its `.host-skip.json` marker under the
552 // same variant hash without re-resolving.
553 let (resolved_topology, _memory_mib) =
554 crate::test_support::runtime::resolve_vm_topology(entry, topo);
555 if let Err(e) = write_skip_sidecar(entry, &resolved_topology) {
556 // Dual-emit at warn level: an unwritten skip sidecar costs
557 // the run no correctness — the test still skipped — but
558 // silently drops post-run stats tooling's visibility into
559 // the skip, so operators debugging a missing row in a
560 // gauntlet report need a loud-enough log to notice. The
561 // eprintln surfaces under direct nextest / cargo-ktstr
562 // invocations where no tracing subscriber is installed;
563 // the tracing::warn lands in every structured-log consumer
564 // (cargo-ktstr, downstream pipelines) at warn level rather
565 // than the previous implicit debug visibility.
566 let entry_name = entry.name;
567 let rendered = format!("{e:#}");
568 eprintln!("ktstr_test: warn: skip-sidecar write failed for {entry_name}: {rendered}");
569 tracing::warn!(
570 test = %entry_name,
571 err = %rendered,
572 "skip-sidecar write failed — stats tooling will not see this skip",
573 );
574 }
575 resolved_topology
576}
577
578#[cfg(test)]
579mod post_vm_skip_tests {
580 //! Locks in the post_vm→skip mechanism. `post_vm_skip` attaches the
581 //! [`HostSkipRequest`] marker (found by the context-aware
582 //! `downcast_ref` the eval gate uses); `combine_post_vm_errs`
583 //! preserves a lone skip request but lets a genuine sibling failure
584 //! DOMINATE — a skip request must never mask a real regression. A
585 //! revert of either the marker attach or the both-skip gate flips a
586 //! cell here.
587 use super::{HostSkipRequest, PostVmAssertionFailure, combine_post_vm_errs, post_vm_skip};
588
589 fn real_fail() -> anyhow::Error {
590 anyhow::anyhow!("real host-side regression").context(PostVmAssertionFailure)
591 }
592
593 #[test]
594 fn post_vm_skip_carries_marker() {
595 assert!(
596 post_vm_skip("inconclusive: placeholder dump")
597 .downcast_ref::<HostSkipRequest>()
598 .is_some()
599 );
600 }
601
602 #[test]
603 fn combine_lone_unconditional_skip_preserved() {
604 let c = combine_post_vm_errs(None, Some(post_vm_skip("ph"))).unwrap();
605 assert!(c.downcast_ref::<HostSkipRequest>().is_some());
606 }
607
608 #[test]
609 fn combine_lone_conditional_skip_preserved() {
610 let c = combine_post_vm_errs(Some(post_vm_skip("ph")), None).unwrap();
611 assert!(c.downcast_ref::<HostSkipRequest>().is_some());
612 }
613
614 #[test]
615 fn combine_both_skip_yields_skip() {
616 let c = combine_post_vm_errs(Some(post_vm_skip("a")), Some(post_vm_skip("b"))).unwrap();
617 assert!(c.downcast_ref::<HostSkipRequest>().is_some());
618 }
619
620 #[test]
621 fn combine_skip_plus_real_fail_does_not_skip() {
622 // A genuine failure alongside a skip request collapses to a
623 // failure: the combined Err must NOT carry HostSkipRequest, so the
624 // eval gate folds it as a failure (re-attaching PostVmAssertionFailure)
625 // rather than skipping — a regression is never masked.
626 let c = combine_post_vm_errs(Some(post_vm_skip("ph")), Some(real_fail())).unwrap();
627 assert!(c.downcast_ref::<HostSkipRequest>().is_none());
628 }
629
630 #[test]
631 fn combine_real_fail_plus_skip_does_not_skip() {
632 let c = combine_post_vm_errs(Some(real_fail()), Some(post_vm_skip("ph"))).unwrap();
633 assert!(c.downcast_ref::<HostSkipRequest>().is_none());
634 }
635}
636
637#[cfg(test)]
638mod starvation_witness_tests {
639 use super::*;
640 use crate::vmm::{HostVcpuSchedstat, VmResult};
641
642 fn with_dilation(on_cpu_ns: u64, run_delay_ns: u64) -> VmResult {
643 VmResult {
644 host_vcpu_schedstat: Some(HostVcpuSchedstat {
645 total_on_cpu_ns: on_cpu_ns,
646 total_run_delay_ns: run_delay_ns,
647 sampled_vcpus: 1,
648 }),
649 ..VmResult::test_fixture()
650 }
651 }
652
653 /// No witness sampled → None (absence of evidence cannot launder a
654 /// regression); quiet host (D ≈ 1.1) → None; contended (D = 3) →
655 /// Some(3.0). Whole-run fallback path (fixture has no per-phase
656 /// witness).
657 #[test]
658 fn capture_starvation_witness_thresholds() {
659 assert_eq!(capture_starvation_witness(&VmResult::test_fixture()), None);
660 assert_eq!(capture_starvation_witness(&with_dilation(10, 1)), None);
661 let d = capture_starvation_witness(&with_dilation(10, 20)).expect("D=3 is contended");
662 assert!((d - 3.0).abs() < 1e-9);
663 }
664
665 /// Fixture with the readiness-vs-window fields set: prereqs ready
666 /// at `ready_s`, window ending at `end_s` (None = never/unresolved).
667 fn with_window(
668 base: VmResult,
669 target: u32,
670 ready_s: Option<u64>,
671 end_s: Option<u64>,
672 ) -> VmResult {
673 VmResult {
674 periodic_target: target,
675 periodic_prereqs_ready: ready_s.map(std::time::Duration::from_secs),
676 periodic_window_end: end_s.map(std::time::Duration::from_secs),
677 ..base
678 }
679 }
680
681 /// Gate arms, primary (readiness-vs-window) first:
682 /// - readiness never / after the window end → structural skip at
683 /// ANY dilation (the arm the D-threshold missed in the field:
684 /// a ~7 s accessor build vs a 4-5 s window at D ≈ 1.2);
685 /// - readiness in time + contended → witness skip;
686 /// - readiness in time + quiet → Ok (the caller's own assertion
687 /// fails — the real-regression case stays catchable);
688 /// - no periodic target → Ok regardless.
689 #[test]
690 fn periodic_starvation_gate_arms() {
691 // No periodic target: Ok even when contended.
692 let r = with_dilation(10, 20);
693 assert!(periodic_starvation_gate(&r, 1).is_ok());
694 // Readiness AFTER the window end, QUIET host: structural skip.
695 let r = with_window(with_dilation(10, 1), 2, Some(9), Some(5));
696 let err = periodic_starvation_gate(&r, 1).expect_err("late readiness must gate");
697 assert!(err.downcast_ref::<HostSkipRequest>().is_some());
698 // Readiness NEVER, quiet host: structural skip.
699 let r = with_window(with_dilation(10, 1), 2, None, Some(5));
700 assert!(periodic_starvation_gate(&r, 1).is_err());
701 // Window never resolved (no boundary could fire): structural skip.
702 let r = with_window(with_dilation(10, 1), 2, Some(1), None);
703 assert!(periodic_starvation_gate(&r, 1).is_err());
704 // Readiness IN TIME + contended: witness skip.
705 let r = with_window(with_dilation(10, 20), 2, Some(1), Some(5));
706 let err = periodic_starvation_gate(&r, 1).expect_err("contended run must gate");
707 assert!(err.downcast_ref::<HostSkipRequest>().is_some());
708 // Readiness IN TIME + quiet + zero captures: Ok — the caller
709 // fails with its own diagnosis (a capture-pipeline regression).
710 let r = with_window(with_dilation(10, 1), 2, Some(1), Some(5));
711 assert!(periodic_starvation_gate(&r, 1).is_ok());
712 }
713
714 /// stall_ejection_skip three-way: a watchdog Stall exit on a
715 /// CONTENDED host (D=3) is an environmental skip; the same Stall on a
716 /// QUIET host stays the caller's failure (Ok, so its `ensure!` fires);
717 /// a contended host with NO stall line is likewise the caller's to
718 /// fail — absence of the watchdog signature cannot launder a real
719 /// zero-activity regression.
720 #[test]
721 fn stall_ejection_skip_arms() {
722 const STALL: &str = "sched_ext: BPF scheduler \"scx_test\" disabled (runnable task stall)";
723
724 let contended_stall = VmResult {
725 stderr: STALL.into(),
726 ..with_dilation(10, 20)
727 };
728 let err = stall_ejection_skip(&contended_stall).expect_err("stall + contention must skip");
729 assert!(err.downcast_ref::<HostSkipRequest>().is_some());
730
731 let quiet_stall = VmResult {
732 stderr: STALL.into(),
733 ..with_dilation(10, 1)
734 };
735 assert!(
736 stall_ejection_skip(&quiet_stall).is_ok(),
737 "a stall on a quiet host is a real fail, not an environmental skip",
738 );
739
740 let contended_no_stall = with_dilation(10, 20);
741 assert!(
742 stall_ejection_skip(&contended_no_stall).is_ok(),
743 "contention without a watchdog stall line leaves the caller to fail",
744 );
745 }
746
747 /// The generic sub-minimum gate: `have < need` + (structural OR
748 /// witnessed) = skip; `have >= need` never gates; readiness-in-time
749 /// quiet host never gates.
750 #[test]
751 fn starved_below_minimum_arms() {
752 // Sub-minimum + contended (readiness in time): witness skip —
753 // the 1-of-6 shape.
754 let contended = with_window(with_dilation(10, 20), 6, Some(1), Some(5));
755 let err = starved_below_minimum_skip(&contended, 1, 3, "captures")
756 .expect_err("sub-minimum under contention must gate");
757 assert!(err.downcast_ref::<HostSkipRequest>().is_some());
758 // At/above minimum: never gates, even contended + late-ready.
759 let late = with_window(with_dilation(10, 20), 6, Some(9), Some(5));
760 assert!(starved_below_minimum_skip(&late, 3, 3, "captures").is_ok());
761 // Sub-minimum, quiet, readiness in time: falls through (caller
762 // fails with its own diagnosis).
763 let quiet = with_window(with_dilation(10, 1), 6, Some(1), Some(5));
764 assert!(starved_below_minimum_skip(&quiet, 1, 3, "captures").is_ok());
765 // Sub-minimum, quiet, readiness LATE: structural skip.
766 let quiet_late = with_window(with_dilation(10, 1), 6, Some(9), Some(5));
767 assert!(starved_below_minimum_skip(&quiet_late, 1, 3, "captures").is_err());
768 }
769}