ktstr/test_support/eval/
post_vm.rs

1//! Host-side post_vm plumbing: the post_vm error marker types
2//! (ScxBpfErrorMatcherMismatch,
3//! PostVmAssertionFailure, HostSkipRequest, ExpectAutoReproSatisfied),
4//! the conditional/unconditional callback combiner + dispatch, the
5//! post_vm_skip helper, and skip-sidecar recording. Split out of
6//! eval/mod.rs to keep the module under the size ceiling.
7
8use super::*;
9
10/// Marker error type attached as `anyhow::Context` to the failure
11/// `Err` produced when an scx_bpf_error matcher
12/// ([`crate::assert::Assert::expect_scx_bpf_error_contains`] or
13/// [`crate::assert::Assert::expect_scx_bpf_error_matches`]) mismatched
14/// the captured scheduler log / sched_ext dump corpus.
15///
16/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
17/// downcasts the error chain for this marker in the `expect_err = true`
18/// branch and refuses to invert the verdict to a pass — a reproducer
19/// that fired the WRONG bug must fail loudly, not silently invert to
20/// "test passed" via `expect_err`. Without the marker, the matcher's
21/// diagnostic surfaces in stderr but the exit code follows the normal
22/// expect_err inversion path.
23#[derive(Debug, Clone, Copy)]
24pub(crate) struct ScxBpfErrorMatcherMismatch;
25
26impl std::fmt::Display for ScxBpfErrorMatcherMismatch {
27    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
28        write!(
29            f,
30            "scx_bpf_error matcher mismatch — the reproducer matcher rejected \
31             this failure mode; expect_err inversion bypassed"
32        )
33    }
34}
35
36impl std::error::Error for ScxBpfErrorMatcherMismatch {}
37
38/// Marker error type attached as `anyhow::Context` to the `Err`
39/// `resolve_scheduler` returns when the `Discover` `cargo build -p
40/// <sched>` FAILED. A scheduler that cannot be built is a failed test —
41/// the resolver never falls back to a possibly-stale pre-built binary.
42///
43/// Dispatch (`crate::test_support::dispatch`) downcasts the error chain
44/// for this marker and forces a hard FAIL EVEN under `expect_err = true`.
45/// The semantic boundary mirrors [`PostVmAssertionFailure`]: `expect_err`
46/// inverts a GUEST-side expected failure, but a build-infra failure is a
47/// HOST-side fault that must never masquerade as the expected guest
48/// failure — without the marker an `expect_err` test whose scheduler
49/// build broke would silently invert to PASS. Same
50/// `anyhow::Context` attachment + `downcast_ref` chain-walk as the
51/// sibling markers; the dispatch guard sits with the other host-side
52/// hard-fail markers, before the `expect_err` inversion.
53#[derive(Debug, Clone, Copy)]
54pub(crate) struct SchedulerBuildRefused;
55
56impl std::fmt::Display for SchedulerBuildRefused {
57    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58        write!(
59            f,
60            "scheduler build refused — the workspace build failed; a \
61             scheduler that cannot be built is a failed test \
62             (expect_err inversion bypassed)"
63        )
64    }
65}
66
67impl std::error::Error for SchedulerBuildRefused {}
68
69/// Marker attached to host-framework failures which occur after a VM run has
70/// started (for example, immutable scheduler CAS coordination or cast-analysis
71/// mapping failure).
72///
73/// These are neither guest behavior nor host-capacity outcomes. Dispatch must
74/// hard-fail them before resource skip classification or `expect_err`
75/// inversion can reinterpret the result.
76#[derive(Debug, Clone, Copy)]
77pub(crate) struct FrameworkInfrastructureFailure;
78
79impl std::fmt::Display for FrameworkInfrastructureFailure {
80    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81        write!(
82            f,
83            "ktstr framework infrastructure failed \
84             (resource classification and expect_err inversion bypassed)"
85        )
86    }
87}
88
89impl std::error::Error for FrameworkInfrastructureFailure {}
90
91/// Marker error type attached as `anyhow::Context` to the failure
92/// `Err` produced by `run_ktstr_test_inner_impl` when a host-side
93/// `post_vm` / `post_vm_unconditional` callback returned `Err`
94/// (which `evaluate_vm_result` has already folded into the verdict —
95/// as an `Other` detail in the parse-success arm, as a message prefix
96/// in the parse-fail arms).
97///
98/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
99/// downcasts the error chain for this marker and refuses to invert the
100/// verdict to a pass — even under `expect_err = true`. The semantic
101/// boundary: `expect_err` inverts a GUEST-side expected failure (the
102/// scheduler stalled, the workload bailed), but a HOST-side `post_vm`
103/// assertion is always honored. A failure-dump render test that
104/// triggers an expected stall to PRODUCE the dump, then asserts the
105/// dump's contents in `post_vm`, must fail loudly when the dump renders
106/// wrong — not silently invert to "passed" because the stall it relied
107/// on was "expected". Without the marker, the post_vm diagnostic
108/// surfaces in stderr but the exit code follows the normal expect_err
109/// inversion path (a false PASS).
110///
111/// Mirrors [`ScxBpfErrorMatcherMismatch`]: same `anyhow::Context`
112/// attachment, same `downcast_ref` chain-walk at the dispatch arm. The
113/// dispatch arm is positioned AFTER the resource-contention / topology
114/// skip arms (a skip means the test never ran) but BEFORE the
115/// [`ExpectAutoReproSatisfied`] and `expect_err` inversion arms, so a
116/// real host-side regression wins over any inversion.
117#[derive(Debug, Clone, Copy)]
118pub(crate) struct PostVmAssertionFailure;
119
120impl std::fmt::Display for PostVmAssertionFailure {
121    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
122        write!(
123            f,
124            "host-side post_vm assertion failed — expect_err inversion bypassed \
125             (a host-side check is honored even when the accompanying guest-side \
126             failure is expected)"
127        )
128    }
129}
130
131impl std::error::Error for PostVmAssertionFailure {}
132
133/// Marker error type attached as `anyhow::Context` to a `post_vm` /
134/// `post_vm_unconditional` `Err` to request a test SKIP (not a
135/// failure): the host-side callback determined the run is
136/// INCONCLUSIVE — the VM could not produce the artifact the assertion
137/// needs (e.g. a load-starved VM whose BPF probe never attached, so
138/// the failure dump is a placeholder), as opposed to a real
139/// regression. The eval fn detects this marker (context-aware
140/// `downcast_ref`, at the `HostSkipRequest` gate) and returns
141/// [`crate::assert::AssertResult::skip`] instead of folding the `Err`
142/// into the verdict.
143///
144/// A real [`PostVmAssertionFailure`] in a sibling callback DOMINATES:
145/// [`combine_post_vm_errs`] preserves the skip marker only when BOTH
146/// callbacks request skip (or only one callback ran); a genuine
147/// failure alongside a skip request collapses to a failure, so a skip
148/// request can never mask a regression.
149#[derive(Debug, Clone, Copy)]
150pub(crate) struct HostSkipRequest;
151
152impl std::fmt::Display for HostSkipRequest {
153    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
154        write!(
155            f,
156            "host-side post_vm requested skip — the run is inconclusive \
157             (the VM could not produce the artifact the assertion needs)"
158        )
159    }
160}
161
162impl std::error::Error for HostSkipRequest {}
163
164/// Marker error type attached as `anyhow::Context` to the failure
165/// `Err` produced by `evaluate_vm_result` when
166/// [`apply_expect_auto_repro_inversion`] has set
167/// `result.expect_auto_repro_satisfied = true`: the primary VM
168/// produced a Fail AND a shape-valid `.repro.wprof.pb` artifact
169/// landed on disk from the auto-repro VM.
170///
171/// Dispatch (`crate::test_support::dispatch::result_to_exit_code`)
172/// downcasts the error chain for this marker and routes the verdict
173/// to `EXIT_PASS`. The underlying `AssertResult` is NOT mutated —
174/// the original failure detail still surfaces in stderr/dump
175/// rendering so an operator chasing why `expect_auto_repro` fired
176/// sees the original failure trail alongside the inversion notice.
177#[derive(Debug, Clone, Copy)]
178pub(crate) struct ExpectAutoReproSatisfied;
179
180impl std::fmt::Display for ExpectAutoReproSatisfied {
181    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182        write!(
183            f,
184            "expect_auto_repro satisfied — the primary test failed and the \
185             auto-repro VM produced a shape-valid .repro.wprof.pb artifact; \
186             verdict inverted to PASS"
187        )
188    }
189}
190
191impl std::error::Error for ExpectAutoReproSatisfied {}
192
193/// Marker error type attached as `anyhow::Context` to the failure `Err`
194/// [`render_failure_verdict_message`]
195/// builds when `entry.survives_storm` is set AND the failing
196/// [`AssertResult`] carries a scheduler-death
197/// `DetailKind` (`SchedulerCrashed` / `SchedulerExitedCleanly` /
198/// `SchedulerDiedUnknownReason`). `err_to_exit_code` downcasts it and forces
199/// `EXIT_FAIL` with a survival-specific explainer, positioned BEFORE the
200/// [`ExpectAutoReproSatisfied`] / `expect_err` inversion arms so a survival
201/// violation can never be inverted to PASS (the validate-time
202/// `survives_storm`/`expect_err` mutex already forbids that combination;
203/// the ordering is defense-in-depth). Mirrors [`ScxBpfErrorMatcherMismatch`].
204#[derive(Debug, Clone, Copy)]
205pub(crate) struct SurvivesStormViolated;
206
207impl std::fmt::Display for SurvivesStormViolated {
208    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
209        write!(
210            f,
211            "survives_storm asserted but the scx scheduler did not survive \
212             the run — it died or was ejected during a hold"
213        )
214    }
215}
216
217impl std::error::Error for SurvivesStormViolated {}
218
219/// Combine the conditional and unconditional `post_vm` failure
220/// signals. When both callbacks fail in the same run, surface
221/// BOTH errors in a single chained message so a debugging
222/// operator sees both regressions on the first pass — a `.or()`
223/// would silently drop the unconditional signal whenever the
224/// conditional also fired, defeating the whole point of the
225/// unconditional callback.
226pub(crate) fn combine_post_vm_errs(
227    conditional: Option<anyhow::Error>,
228    unconditional: Option<anyhow::Error>,
229) -> Option<anyhow::Error> {
230    match (conditional, unconditional) {
231        (Some(c), Some(u)) => {
232            // A genuine failure dominates a skip request: collapse to a
233            // skip only when BOTH callbacks requested skip (both
234            // inconclusive). Otherwise a real PostVmAssertionFailure
235            // must surface, so the chained message wins and the
236            // HostSkipRequest marker is intentionally dropped.
237            let both_skip = c.downcast_ref::<HostSkipRequest>().is_some()
238                && u.downcast_ref::<HostSkipRequest>().is_some();
239            let combined = anyhow::anyhow!("post_vm: {c:#}; post_vm_unconditional: {u:#}");
240            Some(if both_skip {
241                combined.context(HostSkipRequest)
242            } else {
243                combined
244            })
245        }
246        (Some(c), None) => Some(c),
247        (None, Some(u)) => Some(u),
248        (None, None) => None,
249    }
250}
251
252/// Request a test SKIP from a `post_vm` / `post_vm_unconditional`
253/// callback: `return Err(post_vm_skip(reason))` when the run is
254/// INCONCLUSIVE — the VM could not produce the artifact the assertion
255/// needs (e.g. a load-starved VM whose BPF probe never attached,
256/// leaving a placeholder failure dump), as distinct from a real
257/// regression. The framework detects the attached `HostSkipRequest`
258/// marker and converts the run to
259/// [`crate::assert::AssertResult::skip`] instead of a failure.
260///
261/// A genuine `Err` from a sibling callback dominates (see
262/// `combine_post_vm_errs`): a skip request never masks a regression.
263pub fn post_vm_skip(reason: impl Into<String>) -> anyhow::Error {
264    anyhow::anyhow!("{}", reason.into()).context(HostSkipRequest)
265}
266
267/// Witnessed-starvation evidence for capture-bearing assertions: the
268/// host dilation `D` that PROVES the run was contended enough to
269/// explain missing/degraded periodic captures, or `None` on a quiet
270/// (or unwitnessed) host.
271///
272/// The periodic capture chain is readiness-gated (KASLR publish +
273/// accessor-init worker) and its boundaries are wall-anchored inside
274/// the workload window; under heavy host saturation the cold-cache
275/// accessor build alone can outlast a short workload, so every
276/// boundary defers past run end and `periodic_fired == 0` with zero
277/// real captures — an ENVIRONMENTAL outcome, not a capture-pipeline
278/// regression (observed across the periodic/freeze test family on
279/// saturated 96-CPU CI runners; the same cells pass idle). Assertions
280/// that require captures consult this first and convert their failure
281/// into a host-side SKIP (`post_vm_skip`) when starvation is proven,
282/// mirroring the tri-state latency gates: a quiet-host miss still
283/// FAILS (the regression the assertion exists to catch).
284///
285/// Witness preference: Body-phase dilation (the capture boundaries
286/// live inside the workload window), falling back to the whole-run
287/// schedstat `D`. The 1.5 bar follows `PERF_ISOLATION_D_MAX`'s
288/// derivation (worst measured-quiet `D` ≈ 1.21; > 1.5 is unambiguous
289/// external contention). `None` (no witness sampled) is NOT proof —
290/// callers must fail in that case; absence of evidence cannot launder
291/// a regression.
292pub fn capture_starvation_witness(result: &crate::vmm::VmResult) -> Option<f64> {
293    const CAPTURE_STARVATION_D_MAX: f64 = 1.5;
294    let d = result
295        .contention_witness
296        .as_ref()
297        .and_then(|w| w.body_dilation())
298        .or_else(|| {
299            result
300                .host_vcpu_schedstat
301                .as_ref()
302                .and_then(|s| s.dilation())
303        })?;
304    (d > CAPTURE_STARVATION_D_MAX).then_some(d)
305}
306
307/// Environmental skip for a zero-scheduler-activity reading (e.g.
308/// `nr_dispatched` / `nr_yielded` read 0 across every periodic sample)
309/// that the guest's OWN sched_ext runnable-stall watchdog explains: when
310/// the console carries a watchdog `ScxExitKind::Stall` exit AND
311/// [`capture_starvation_witness`] proves the host was contended, the BPF
312/// scheduler was ejected because a descheduling host dilated the 5s
313/// GUEST-time stall window past the timeout — not because the scheduler
314/// failed to dispatch. Returns `Err(post_vm_skip(..))` in that case; `Ok`
315/// otherwise, so a Stall on a quiet host (no witness) or any non-Stall
316/// zero still reaches the caller's own regression `ensure!`.
317///
318/// Callers gate their zero-activity failure on it:
319/// ```ignore
320/// if !any_progress {
321///     stall_ejection_skip(result)?;
322/// }
323/// anyhow::ensure!(any_progress, "... never dispatched ...");
324/// ```
325pub fn stall_ejection_skip(result: &crate::vmm::VmResult) -> anyhow::Result<()> {
326    let stalled = crate::monitor::dmesg_scx::parse_kmsg_window(&result.stderr)
327        .iter()
328        .any(|e| e.kind == crate::monitor::dmesg_scx::ScxExitKind::Stall);
329    if stalled && let Some(d) = capture_starvation_witness(result) {
330        return Err(post_vm_skip(format!(
331            "scx scheduler ejected by the guest sched_ext runnable-stall \
332             watchdog under witnessed host contention (D={d:.2}): the 5s stall \
333             timer measures GUEST time, which a descheduling host dilates into \
334             a false stall — the zero-activity reading is that ejection, not a \
335             scheduler regression; environmental non-verdict",
336        )));
337    }
338    Ok(())
339}
340
341/// Generic sub-minimum starvation skip: `Err(post_vm_skip(..))` when the
342/// run produced FEWER data-bearing units than the caller's assertion
343/// requires (`have < need`) AND [`capture_starvation_witness`] proves the
344/// host was contended enough to explain the shortfall; `Ok(())`
345/// otherwise. The caller passes the SAME count its own assertion checks
346/// (real captures, PSI-carrying captures, series length, …) so the gate
347/// and the assertion cannot disagree about the boundary — an earlier
348/// zero-only gate let `1 of 6` runs through to fail on a `>= 2` minimum
349/// under the identical environmental condition. Quiet-host shortfalls
350/// (zero OR sub-minimum) still fall through and fail with the caller's
351/// specific diagnosis.
352pub fn starved_below_minimum_skip(
353    result: &crate::vmm::VmResult,
354    have: usize,
355    need: usize,
356    what: &str,
357) -> anyhow::Result<()> {
358    if have >= need {
359        return Ok(());
360    }
361    // PRIMARY arm — readiness vs window: when the periodic prereqs
362    // (KASLR publish + accessors) became ready at/after the capture
363    // window's end, or never became ready at all, every boundary was
364    // structurally unreachable — zero/short captures were inevitable at
365    // ANY host dilation. This is the honest signal the D-threshold arm
366    // missed in the field: a ~7 s cold accessor build outruns a 4-5 s
367    // workload window at D ≈ 1.2, well under any defensible contention
368    // bar. Skip carries both timestamps.
369    let ready = result.periodic_prereqs_ready;
370    let window_end = result.periodic_window_end;
371    let structurally_impossible = match (ready, window_end) {
372        // Never ready: nothing could ever fire.
373        (None, _) => true,
374        // Ready, but at/after the window closed.
375        (Some(r), Some(w)) => r >= w,
376        // Ready but the window never resolved: the boundaries were
377        // never computed, so no capture could fire.
378        (Some(_), None) => true,
379    };
380    if result.periodic_target > 0 && structurally_impossible {
381        return Err(post_vm_skip(format!(
382            "only {have} of the required {need} {what}: the capture prereqs \
383             (KASLR publish + accessor init) {} the capture window \
384             (prereqs_ready={:?}, window_end={:?}) — captures were \
385             structurally impossible this run at any host dilation; \
386             environmental non-verdict",
387            if ready.is_none() {
388                "never became ready within"
389            } else {
390                "became ready only after"
391            },
392            ready,
393            window_end,
394        )));
395    }
396    // SECONDARY arm — witnessed contention: readiness landed in time
397    // but the captures still fell short (e.g. rendezvous degradation,
398    // data-bearing rows missing) under a provably saturated host.
399    if let Some(d) = capture_starvation_witness(result) {
400        return Err(post_vm_skip(format!(
401            "only {have} of the required {need} {what} under witnessed host \
402             contention (D={d:.2}, prereqs_ready={ready:?}, \
403             window_end={window_end:?}): the capture chain was starved below \
404             the assertion's minimum — environmental non-verdict; the \
405             dependent assertions cannot be evaluated",
406        )));
407    }
408    Ok(())
409}
410
411/// Periodic-capture starvation gate for capture-requiring post_vm
412/// assertions: [`starved_below_minimum_skip`] keyed on the bridge's
413/// REAL (non-placeholder) capture count vs the test's stated minimum.
414/// Call at the TOP of a post_vm callback with the same minimum its
415/// assertions enforce: `periodic_starvation_gate(result, 2)?;`. Tests
416/// whose assertions key on a narrower data-bearing count (e.g. "N
417/// captures carrying per-cgroup PSI") should ALSO call
418/// [`starved_below_minimum_skip`] with that count at the assertion
419/// site. No-op when the entry configured no periodic captures.
420pub fn periodic_starvation_gate(
421    result: &crate::vmm::VmResult,
422    min_real: usize,
423) -> anyhow::Result<()> {
424    if result.periodic_target == 0 {
425        return Ok(());
426    }
427    starved_below_minimum_skip(
428        result,
429        result.snapshot_bridge.periodic_real_count() as usize,
430        min_real,
431        "real (non-placeholder) periodic captures",
432    )
433}
434
435/// Dispatch the entry's `post_vm` + `post_vm_unconditional`
436/// callbacks and combine their failure signals.
437///
438/// - `post_vm` runs only when the guest reported a non-Fail
439///   `AssertResult` (Skip / Inconclusive / Pass) — the
440///   `guest_already_failed` parameter folds the
441///   `parse_assert_result_from_drain` lookup the call site does.
442///   The skip mirrors the suppression contract documented on
443///   `KtstrTestEntry::post_vm`.
444///
445/// - `post_vm_unconditional` ALWAYS runs — bypasses the
446///   guest-fail suppression that gates `post_vm`. The callback
447///   owns its own skip-on-crash logic (or doesn't, when the
448///   intent is "assert on host-side artifact regardless of
449///   guest-side outcome").
450///
451/// Both callbacks route through [`invoke_post_vm_callback`] so a
452/// panic in either body becomes an `anyhow::Error` rather than
453/// unwinding past the call site (which would leak VM resources;
454/// see the helper doc).
455///
456/// Returns the combined `Option<anyhow::Error>` via
457/// [`combine_post_vm_errs`]: when both callbacks fail, the
458/// chained message names both errors so the operator sees both
459/// regressions on the first pass instead of a two-pass debug
460/// cycle. `.or()` would silently drop the unconditional fail
461/// when the conditional also fired.
462pub(crate) fn run_post_vm_callbacks(
463    entry: &KtstrTestEntry,
464    result: &crate::vmm::VmResult,
465    guest_already_failed: bool,
466) -> Option<anyhow::Error> {
467    let conditional = if guest_already_failed {
468        None
469    } else {
470        entry
471            .post_vm
472            .and_then(|cb| invoke_post_vm_callback(cb, result, "post_vm"))
473    };
474    let unconditional = entry
475        .post_vm_unconditional
476        .and_then(|cb| invoke_post_vm_callback(cb, result, "post_vm_unconditional"));
477    combine_post_vm_errs(conditional, unconditional)
478}
479
480/// Invoke a `post_vm` / `post_vm_unconditional` callback with panic
481/// catch. Converts a panic to `anyhow::Error` so the panic message
482/// surfaces in the test failure output AND the rest of the
483/// post-VM teardown (`write_placeholder_failure_dump_if_missing`,
484/// `drop(vm)` releasing CPU/LLC flocks + guest memory + kernel-cache
485/// reader flock) still runs.
486///
487/// Without the catch, a panicking callback would unwind past the
488/// placeholder-dump emission and past `drop(vm)`, leaking VM
489/// resources (flocks, guest memory) until process exit or the next
490/// test's drop reclaims them. Same hazard for `Ok` returns from
491/// callbacks that subsequently panic in their inner state — both
492/// paths fold into this single guard.
493///
494/// `label` is woven into the error message so the operator sees
495/// which callback panicked (`post_vm` vs `post_vm_unconditional`)
496/// when both are wired and both fire.
497///
498/// Returns `Some(err)` when the callback returns `Err` OR panics;
499/// returns `None` when the callback returns `Ok(())`. Mirrors the
500/// shape `.err()` produces from `Result` so the caller's
501/// `.and_then(|cb| ...)` flows unchanged.
502///
503/// Under `panic = "abort"` (release builds — see `Cargo.toml
504/// [profile.release]`), `catch_unwind` is a no-op: a panic aborts
505/// the process before this function returns. The wrap is still
506/// safe — `catch_unwind` is always defined, just inert — and the
507/// debug builds get the leak protection that exposes regressions
508/// before they ship.
509pub(crate) fn invoke_post_vm_callback(
510    cb: super::super::PostVmCallback,
511    result: &crate::vmm::VmResult,
512    label: &'static str,
513) -> Option<anyhow::Error> {
514    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cb(result))) {
515        Ok(Ok(())) => None,
516        Ok(Err(e)) => Some(e),
517        Err(payload) => {
518            let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {
519                (*s).to_string()
520            } else if let Some(s) = payload.downcast_ref::<String>() {
521                s.clone()
522            } else {
523                "<non-string panic payload>".to_string()
524            };
525            Some(anyhow::anyhow!("{label} callback panicked: {msg}"))
526        }
527    }
528}
529
530/// Write a skip sidecar for `entry`, logging to stderr on failure
531/// without propagating the error. Called wherever a run is skipped
532/// before producing a real result: the skip-class catch-all and the
533/// VM build / VM run arms in [`run_ktstr_test_inner`] (each fires on a
534/// host-incapacity class — `TopologyInsufficient` or
535/// `PerfModeUnavailable`; transient `ResourceContention` is a
536/// retryable failure and records no skip), and the
537/// performance-mode / coverage gates at the plain-run entry points in
538/// the crate `dispatch` module. All must record the skip for stats
539/// tooling but cannot meaningfully handle a sidecar-write failure
540/// beyond logging it — the skip itself is still valid; only post-run
541/// stats tooling loses visibility.
542pub(crate) fn record_skip_sidecar(
543    entry: &KtstrTestEntry,
544    topo: Option<&crate::test_support::topo::TopoOverride>,
545) -> crate::vmm::topology::Topology {
546    // Resolve the topology the run of this (entry, override) WOULD boot,
547    // via the same resolve_vm_topology the run path uses, so a preset's
548    // skip and run record the identical topology -> identical
549    // variant_hash -> the retry overwrites instead of coexisting. For a
550    // plain test (topo = None) this is entry.topology. Returned so a
551    // host-class caller can write its `.host-skip.json` marker under the
552    // same variant hash without re-resolving.
553    let (resolved_topology, _memory_mib) =
554        crate::test_support::runtime::resolve_vm_topology(entry, topo);
555    if let Err(e) = write_skip_sidecar(entry, &resolved_topology) {
556        // Dual-emit at warn level: an unwritten skip sidecar costs
557        // the run no correctness — the test still skipped — but
558        // silently drops post-run stats tooling's visibility into
559        // the skip, so operators debugging a missing row in a
560        // gauntlet report need a loud-enough log to notice. The
561        // eprintln surfaces under direct nextest / cargo-ktstr
562        // invocations where no tracing subscriber is installed;
563        // the tracing::warn lands in every structured-log consumer
564        // (cargo-ktstr, downstream pipelines) at warn level rather
565        // than the previous implicit debug visibility.
566        let entry_name = entry.name;
567        let rendered = format!("{e:#}");
568        eprintln!("ktstr_test: warn: skip-sidecar write failed for {entry_name}: {rendered}");
569        tracing::warn!(
570            test = %entry_name,
571            err = %rendered,
572            "skip-sidecar write failed — stats tooling will not see this skip",
573        );
574    }
575    resolved_topology
576}
577
578#[cfg(test)]
579mod post_vm_skip_tests {
580    //! Locks in the post_vm→skip mechanism. `post_vm_skip` attaches the
581    //! [`HostSkipRequest`] marker (found by the context-aware
582    //! `downcast_ref` the eval gate uses); `combine_post_vm_errs`
583    //! preserves a lone skip request but lets a genuine sibling failure
584    //! DOMINATE — a skip request must never mask a real regression. A
585    //! revert of either the marker attach or the both-skip gate flips a
586    //! cell here.
587    use super::{HostSkipRequest, PostVmAssertionFailure, combine_post_vm_errs, post_vm_skip};
588
589    fn real_fail() -> anyhow::Error {
590        anyhow::anyhow!("real host-side regression").context(PostVmAssertionFailure)
591    }
592
593    #[test]
594    fn post_vm_skip_carries_marker() {
595        assert!(
596            post_vm_skip("inconclusive: placeholder dump")
597                .downcast_ref::<HostSkipRequest>()
598                .is_some()
599        );
600    }
601
602    #[test]
603    fn combine_lone_unconditional_skip_preserved() {
604        let c = combine_post_vm_errs(None, Some(post_vm_skip("ph"))).unwrap();
605        assert!(c.downcast_ref::<HostSkipRequest>().is_some());
606    }
607
608    #[test]
609    fn combine_lone_conditional_skip_preserved() {
610        let c = combine_post_vm_errs(Some(post_vm_skip("ph")), None).unwrap();
611        assert!(c.downcast_ref::<HostSkipRequest>().is_some());
612    }
613
614    #[test]
615    fn combine_both_skip_yields_skip() {
616        let c = combine_post_vm_errs(Some(post_vm_skip("a")), Some(post_vm_skip("b"))).unwrap();
617        assert!(c.downcast_ref::<HostSkipRequest>().is_some());
618    }
619
620    #[test]
621    fn combine_skip_plus_real_fail_does_not_skip() {
622        // A genuine failure alongside a skip request collapses to a
623        // failure: the combined Err must NOT carry HostSkipRequest, so the
624        // eval gate folds it as a failure (re-attaching PostVmAssertionFailure)
625        // rather than skipping — a regression is never masked.
626        let c = combine_post_vm_errs(Some(post_vm_skip("ph")), Some(real_fail())).unwrap();
627        assert!(c.downcast_ref::<HostSkipRequest>().is_none());
628    }
629
630    #[test]
631    fn combine_real_fail_plus_skip_does_not_skip() {
632        let c = combine_post_vm_errs(Some(real_fail()), Some(post_vm_skip("ph"))).unwrap();
633        assert!(c.downcast_ref::<HostSkipRequest>().is_none());
634    }
635}
636
637#[cfg(test)]
638mod starvation_witness_tests {
639    use super::*;
640    use crate::vmm::{HostVcpuSchedstat, VmResult};
641
642    fn with_dilation(on_cpu_ns: u64, run_delay_ns: u64) -> VmResult {
643        VmResult {
644            host_vcpu_schedstat: Some(HostVcpuSchedstat {
645                total_on_cpu_ns: on_cpu_ns,
646                total_run_delay_ns: run_delay_ns,
647                sampled_vcpus: 1,
648            }),
649            ..VmResult::test_fixture()
650        }
651    }
652
653    /// No witness sampled → None (absence of evidence cannot launder a
654    /// regression); quiet host (D ≈ 1.1) → None; contended (D = 3) →
655    /// Some(3.0). Whole-run fallback path (fixture has no per-phase
656    /// witness).
657    #[test]
658    fn capture_starvation_witness_thresholds() {
659        assert_eq!(capture_starvation_witness(&VmResult::test_fixture()), None);
660        assert_eq!(capture_starvation_witness(&with_dilation(10, 1)), None);
661        let d = capture_starvation_witness(&with_dilation(10, 20)).expect("D=3 is contended");
662        assert!((d - 3.0).abs() < 1e-9);
663    }
664
665    /// Fixture with the readiness-vs-window fields set: prereqs ready
666    /// at `ready_s`, window ending at `end_s` (None = never/unresolved).
667    fn with_window(
668        base: VmResult,
669        target: u32,
670        ready_s: Option<u64>,
671        end_s: Option<u64>,
672    ) -> VmResult {
673        VmResult {
674            periodic_target: target,
675            periodic_prereqs_ready: ready_s.map(std::time::Duration::from_secs),
676            periodic_window_end: end_s.map(std::time::Duration::from_secs),
677            ..base
678        }
679    }
680
681    /// Gate arms, primary (readiness-vs-window) first:
682    ///   - readiness never / after the window end → structural skip at
683    ///     ANY dilation (the arm the D-threshold missed in the field:
684    ///     a ~7 s accessor build vs a 4-5 s window at D ≈ 1.2);
685    ///   - readiness in time + contended → witness skip;
686    ///   - readiness in time + quiet → Ok (the caller's own assertion
687    ///     fails — the real-regression case stays catchable);
688    ///   - no periodic target → Ok regardless.
689    #[test]
690    fn periodic_starvation_gate_arms() {
691        // No periodic target: Ok even when contended.
692        let r = with_dilation(10, 20);
693        assert!(periodic_starvation_gate(&r, 1).is_ok());
694        // Readiness AFTER the window end, QUIET host: structural skip.
695        let r = with_window(with_dilation(10, 1), 2, Some(9), Some(5));
696        let err = periodic_starvation_gate(&r, 1).expect_err("late readiness must gate");
697        assert!(err.downcast_ref::<HostSkipRequest>().is_some());
698        // Readiness NEVER, quiet host: structural skip.
699        let r = with_window(with_dilation(10, 1), 2, None, Some(5));
700        assert!(periodic_starvation_gate(&r, 1).is_err());
701        // Window never resolved (no boundary could fire): structural skip.
702        let r = with_window(with_dilation(10, 1), 2, Some(1), None);
703        assert!(periodic_starvation_gate(&r, 1).is_err());
704        // Readiness IN TIME + contended: witness skip.
705        let r = with_window(with_dilation(10, 20), 2, Some(1), Some(5));
706        let err = periodic_starvation_gate(&r, 1).expect_err("contended run must gate");
707        assert!(err.downcast_ref::<HostSkipRequest>().is_some());
708        // Readiness IN TIME + quiet + zero captures: Ok — the caller
709        // fails with its own diagnosis (a capture-pipeline regression).
710        let r = with_window(with_dilation(10, 1), 2, Some(1), Some(5));
711        assert!(periodic_starvation_gate(&r, 1).is_ok());
712    }
713
714    /// stall_ejection_skip three-way: a watchdog Stall exit on a
715    /// CONTENDED host (D=3) is an environmental skip; the same Stall on a
716    /// QUIET host stays the caller's failure (Ok, so its `ensure!` fires);
717    /// a contended host with NO stall line is likewise the caller's to
718    /// fail — absence of the watchdog signature cannot launder a real
719    /// zero-activity regression.
720    #[test]
721    fn stall_ejection_skip_arms() {
722        const STALL: &str = "sched_ext: BPF scheduler \"scx_test\" disabled (runnable task stall)";
723
724        let contended_stall = VmResult {
725            stderr: STALL.into(),
726            ..with_dilation(10, 20)
727        };
728        let err = stall_ejection_skip(&contended_stall).expect_err("stall + contention must skip");
729        assert!(err.downcast_ref::<HostSkipRequest>().is_some());
730
731        let quiet_stall = VmResult {
732            stderr: STALL.into(),
733            ..with_dilation(10, 1)
734        };
735        assert!(
736            stall_ejection_skip(&quiet_stall).is_ok(),
737            "a stall on a quiet host is a real fail, not an environmental skip",
738        );
739
740        let contended_no_stall = with_dilation(10, 20);
741        assert!(
742            stall_ejection_skip(&contended_no_stall).is_ok(),
743            "contention without a watchdog stall line leaves the caller to fail",
744        );
745    }
746
747    /// The generic sub-minimum gate: `have < need` + (structural OR
748    /// witnessed) = skip; `have >= need` never gates; readiness-in-time
749    /// quiet host never gates.
750    #[test]
751    fn starved_below_minimum_arms() {
752        // Sub-minimum + contended (readiness in time): witness skip —
753        // the 1-of-6 shape.
754        let contended = with_window(with_dilation(10, 20), 6, Some(1), Some(5));
755        let err = starved_below_minimum_skip(&contended, 1, 3, "captures")
756            .expect_err("sub-minimum under contention must gate");
757        assert!(err.downcast_ref::<HostSkipRequest>().is_some());
758        // At/above minimum: never gates, even contended + late-ready.
759        let late = with_window(with_dilation(10, 20), 6, Some(9), Some(5));
760        assert!(starved_below_minimum_skip(&late, 3, 3, "captures").is_ok());
761        // Sub-minimum, quiet, readiness in time: falls through (caller
762        // fails with its own diagnosis).
763        let quiet = with_window(with_dilation(10, 1), 6, Some(1), Some(5));
764        assert!(starved_below_minimum_skip(&quiet, 1, 3, "captures").is_ok());
765        // Sub-minimum, quiet, readiness LATE: structural skip.
766        let quiet_late = with_window(with_dilation(10, 1), 6, Some(9), Some(5));
767        assert!(starved_below_minimum_skip(&quiet_late, 1, 3, "captures").is_err());
768    }
769}