ktstr/scenario/ops/
mod.rs

1//! Composable ops/steps system for dynamic cgroup topology changes.
2//!
3//! [`Op`] is an atomic cgroup operation. [`Step`] sequences ops with a
4//! hold period. [`CgroupDef`] bundles create + cpuset + spawn into a
5//! single declaration. [`execute_steps()`] runs a step sequence with
6//! scheduler liveness checks and stimulus event recording.
7//!
8//! See the [Ops and Steps](https://ktstr.dev/guide/concepts/ops.html)
9//! chapter for a guide.
10//!
11//! # Cgroup tooling at a glance
12//!
13//! ktstr exposes the cgroup v2 surface across two layers — declarative
14//! steady-state via [`CgroupDef`] (set at scenario-setup time, holds
15//! for the cgroup's lifetime) and imperative state-transitions via
16//! [`Op`] (applied mid-step, describe transitions over time):
17//!
18//! | Knob | Layer | API entry | Underlying file | When to use |
19//! |------|-------|-----------|-----------------|-------------|
20//! | CPU affinity | setup | [`CgroupDef::cpuset`] | `cpuset.cpus` | Bind workers to a CPU subset for the whole run. |
21//! | NUMA-mem affinity | setup | [`CgroupDef::cpuset_mems`] | `cpuset.mems` | Constrain allocations to specific NUMA nodes. |
22//! | CPU bandwidth | setup | [`CgroupDef::cpu_quota_pct`] / [`CgroupDef::cpu_quota`] / [`CgroupDef::cpu_unlimited`] | `cpu.max` | Cap CPU time per period (1 CPU at 50% / 2 CPU at 100% / etc). |
23//! | CPU share weight | setup | [`CgroupDef::cpu_weight`] | `cpu.weight` | Bias relative CPU share when siblings contend. |
24//! | Memory ceiling | setup | [`CgroupDef::memory_max`] / [`CgroupDef::memory_unlimited`] | `memory.max` | Hard ceiling — exceeding triggers cgroup OOM. |
25//! | Memory throttle | setup | [`CgroupDef::memory_high`] | `memory.high` | Soft throttle: triggers reclaim, not OOM. |
26//! | Memory protection | setup | [`CgroupDef::memory_low`] | `memory.low` | Soft protection: kernel reclaims from siblings first. |
27//! | Swap cap | setup | [`CgroupDef::memory_swap_max`] / [`CgroupDef::memory_swap_unlimited`] | `memory.swap.max` | Cap how much memory can spill to swap (CONFIG_SWAP=y). |
28//! | IO share | setup | [`CgroupDef::io_weight`] | `io.weight` | Bias relative IO share when siblings contend. |
29//! | Task ceiling | setup | [`CgroupDef::pids_max`] / [`CgroupDef::pids_unlimited`] | `pids.max` | Cap process+thread count — fork/clone returns EAGAIN at limit. |
30//! | Mid-run cpuset rebind | mid-step | [`Op::set_cpuset`] / [`Op::clear_cpuset`] / [`Op::swap_cpusets`] | `cpuset.cpus` | Move cpuset on a live cgroup mid-scenario. |
31//! | Mid-run task migration | mid-step | [`Op::move_all_tasks`] | `cgroup.procs` | Move workers from one cgroup to another. |
32//! | Pause/resume | mid-step | [`Op::freeze_cgroup`] / [`Op::unfreeze_cgroup`] | `cgroup.freeze` | Suspend every task in the cgroup; resume later. |
33//! | Add/remove cgroup | mid-step | [`Op::add_cgroup`] / [`Op::remove_cgroup`] / [`Op::stop_cgroup`] | (cgroupfs mkdir/rmdir) | Spawn / tear down a cgroup mid-scenario. |
34//!
35//! # Worked examples
36//!
37//! * **Static topology** (one cgroup, fixed cpuset, weight-biased
38//!   compute): [`CgroupDef`] type-level docs.
39//! * **Suspend/resume** (3-Step idiom — run, freeze, run again):
40//!   [`Op::FreezeCgroup`] doc.
41//! * **Memory-cap teardown** (rewind a base CgroupDef's swap cap):
42//!   [`CgroupDef::memory_swap_unlimited`] doc.
43//!
44//! # Implementation entry points
45//!
46//! Every knob ends in [`crate::cgroup::CgroupOps`] (production:
47//! [`crate::cgroup::CgroupManager`]; tests: a recording `MockCgroupOps`
48//! double). `apply_setup` runs the [`CgroupDef`] passes; `apply_ops`
49//! dispatches the [`Op`] variants. Both share `ctx.cgroups` so a test
50//! that uses both layers writes through the same RAII teardown
51//! (`crate::scenario::CgroupGroup::Drop`).
52//!
53//! # File layout
54//!
55//! `types` holds the data model: [`Op`], [`CgroupDef`], [`Step`],
56//! [`HoldSpec`], [`Setup`], [`CpusetSpec`], the per-controller limits
57//! structs, and every builder constructor. Re-exported from this module
58//! so external paths remain `crate::scenario::ops::Op` etc. The executor
59//! drives that model against [`crate::cgroup::CgroupOps`] via `apply_setup`
60//! (sibling `setup` module) and `apply_ops` (sibling `dispatch` module),
61//! and exposes the [`execute_steps`] / [`execute_scenario`] family of
62//! public entry points (this file).
63
64mod types;
65pub use types::*;
66
67mod setup;
68pub use setup::PLACEMENT_LOG_PATH;
69use setup::apply_setup;
70
71mod dispatch;
72#[cfg(test)]
73use dispatch::{
74    REPLACE_NOT_TRYING_DEADLINE_S, build_kernel_op_request, dispatch_kernel_op_request,
75    merge_adjacent_cold_writes, staged_scheduler_log_path, wait_for_accessor_publish_or_bail,
76    wait_for_worker_state_not_trying_or_bail, write_entries_from_writes,
77};
78use dispatch::{apply_ops, render_cgroup_key};
79// Re-export the scx-state reader + enum. The guest-init scheduler
80// attach-readiness predicate (rust_init::scheduler::scx_attach_ready) reads
81// scx_state() to confirm SCX_ENABLED before the workload dispatches, and the
82// survives_storm probe's host unit test (`test_support::probe_tests`) drives
83// scheduler liveness on the host (no VM) via the set_test_scx_state seam.
84// `dispatch` is otherwise private to `ops`.
85#[cfg(test)]
86pub(crate) use dispatch::set_test_scx_state;
87pub(crate) use dispatch::{ScxState, scx_down, scx_state};
88
89use std::collections::BTreeSet;
90use std::thread;
91use std::time::Duration;
92
93use anyhow::{Context, Result};
94
95use crate::assert::AssertResult;
96use crate::scenario::backdrop;
97#[cfg(test)]
98use crate::scenario::process_alive;
99use crate::scenario::{CgroupGroup, Ctx};
100use crate::vmm::guest_comms;
101use crate::vmm::wire::StimulusPayload;
102use crate::workload::{MemPolicy, WorkloadHandle};
103
104// ---------------------------------------------------------------------------
105// Step executor
106// ---------------------------------------------------------------------------
107
108/// Persistent scenario-wide state owned by
109/// [`execute_scenario_with`]. Lives for the entire step sequence;
110/// cgroups, workload handles, and payload handles declared by the
111/// [`Backdrop`](backdrop::Backdrop) go here and only tear
112/// down at scenario end (success or Err). See [`StepState`] for
113/// the step-local counterpart.
114struct BackdropState<'a> {
115    /// RAII cgroup guard for persistent cgroups — removes them on drop.
116    cgroups: CgroupGroup<'a>,
117    /// Active workload handles in persistent cgroups, keyed by cgroup name.
118    handles: Vec<(String, WorkloadHandle)>,
119    /// Resolved cpusets per persistent cgroup name.
120    cpusets: std::collections::HashMap<String, BTreeSet<usize>>,
121    /// Active payload-binary handles owned by the backdrop. Drained
122    /// via `.kill()` at scenario teardown so the metric-emission
123    /// pipeline still fires.
124    payload_handles: Vec<PayloadEntry>,
125    /// BPF map fds opened via [`crate::scenario::ops::types::Op::PinBpfMap`].
126    /// Keyed by the map name the caller requested; the
127    /// [`std::os::fd::OwnedFd`] holds an extra refcount on the
128    /// kernel-side `struct bpf_map` so the map survives any
129    /// scheduler-process teardown (including
130    /// [`crate::scenario::ops::types::Op::ReplaceScheduler`]) until
131    /// scenario end. Drops close the fds and release the refcount.
132    pinned_bpf_maps: std::collections::HashMap<String, std::os::fd::OwnedFd>,
133}
134
135impl<'a> BackdropState<'a> {
136    /// Empty backdrop state (no persistent entities), scoped to `ctx.cgroups`.
137    fn empty(ctx: &'a Ctx) -> Self {
138        Self {
139            cgroups: CgroupGroup::new(ctx.cgroups),
140            handles: Vec::new(),
141            cpusets: std::collections::HashMap::new(),
142            payload_handles: Vec::new(),
143            pinned_bpf_maps: std::collections::HashMap::new(),
144        }
145    }
146}
147
148/// Step-local execution state. Fresh per step, torn down at step
149/// boundary: cgroups removed (via RAII drop), workload handles
150/// collected, payload handles killed with metric emission. Any ops
151/// in the step that reference a cgroup name look here first before
152/// falling through to [`BackdropState`].
153struct StepState<'a> {
154    /// RAII cgroup guard — removes step-local cgroups on drop.
155    cgroups: CgroupGroup<'a>,
156    /// Active workload handles keyed by step-local cgroup name.
157    handles: Vec<(String, WorkloadHandle)>,
158    /// Resolved cpusets per step-local cgroup name, for isolation checks.
159    cpusets: std::collections::HashMap<String, BTreeSet<usize>>,
160    /// Active payload-binary handles keyed by cgroup name. Each entry
161    /// came from either a [`CgroupDef::workload`] spawn in
162    /// `apply_setup` or an explicit [`Op::RunPayload`] invocation;
163    /// `source` tags which path spawned it so the duplicate-name
164    /// dedup in `Op::RunPayload` can point at the original site. All
165    /// are killed during step-teardown / cgroup removal so cgroupfs
166    /// cleanup never trips EBUSY on a live process.
167    payload_handles: Vec<PayloadEntry>,
168    /// Host-mode worker stuck monitor, started lazily at the end of
169    /// the first successful [`apply_setup`] when running outside a
170    /// VM (no `is_guest`, no `cargo_test_mode`) and at least one
171    /// worker exists. The handle's [`Drop`] joins the polling
172    /// thread when [`StepState`] drops; [`collect_step`] drains
173    /// any accumulated reports before that drop so they reach the
174    /// scenario's [`AssertResult`]. `None` in every guest-side
175    /// scenario and in `cargo_test_mode` runs — the host-side
176    /// monitor is the only stuck signal available in host-mode,
177    /// where the freeze coordinator / KVM-side stall plumbing is
178    /// not running. See [`crate::scenario::host_stuck`] for the
179    /// signal definition and detection latency contract.
180    stuck_monitor: Option<crate::scenario::host_stuck::StuckMonitorHandle>,
181}
182
183impl<'a> StepState<'a> {
184    /// Empty step state scoped to `ctx.cgroups`.
185    fn empty(ctx: &'a Ctx) -> Self {
186        Self {
187            cgroups: CgroupGroup::new(ctx.cgroups),
188            handles: Vec::new(),
189            cpusets: std::collections::HashMap::new(),
190            payload_handles: Vec::new(),
191            stuck_monitor: None,
192        }
193    }
194}
195
196/// Combined mutable view over step-local and backdrop state.
197///
198/// Every function that touches execution state (apply_setup,
199/// apply_ops, the drain helpers) receives a
200/// `ScenarioState`; lookups prefer step-local, falling through to
201/// backdrop. New state created via ops/setup inside a step writes
202/// to step-local by default — that is the primary mechanism
203/// enforcing per-step bounded lifetime. Setup for the Backdrop
204/// itself (run once before the step loop) writes straight to the
205/// backdrop side via [`ScenarioState::with_target_backdrop`].
206struct ScenarioState<'a, 'b> {
207    step: &'b mut StepState<'a>,
208    backdrop: &'b mut BackdropState<'a>,
209    /// When true, all mutations route to [`Self::backdrop`] instead
210    /// of [`Self::step`]. Set by [`Self::with_target_backdrop`] when
211    /// running the Backdrop's initial `apply_setup` / `apply_ops`
212    /// before the first step.
213    target_backdrop: bool,
214}
215
216impl<'a, 'b> ScenarioState<'a, 'b> {
217    /// Build a combined scenario view. Starts with the step-local
218    /// slot as the mutation target — call [`Self::with_target_backdrop`]
219    /// to flip into backdrop-setup mode for Backdrop's own
220    /// apply_setup / apply_ops pass.
221    fn new(step: &'b mut StepState<'a>, backdrop: &'b mut BackdropState<'a>) -> Self {
222        Self {
223            step,
224            backdrop,
225            target_backdrop: false,
226        }
227    }
228
229    /// Run `f` with writes routed to the backdrop side.
230    fn with_target_backdrop<R>(&mut self, f: impl FnOnce(&mut Self) -> R) -> R {
231        let prev = self.target_backdrop;
232        self.target_backdrop = true;
233        let r = f(self);
234        self.target_backdrop = prev;
235        r
236    }
237
238    /// `cgroups` group that receives newly-created cgroups. Step-local
239    /// by default; backdrop when [`Self::with_target_backdrop`] is active.
240    fn target_cgroups(&mut self) -> &mut CgroupGroup<'a> {
241        if self.target_backdrop {
242            &mut self.backdrop.cgroups
243        } else {
244            &mut self.step.cgroups
245        }
246    }
247
248    /// `handles` vec that receives newly-spawned workload handles.
249    fn target_handles(&mut self) -> &mut Vec<(String, WorkloadHandle)> {
250        if self.target_backdrop {
251            &mut self.backdrop.handles
252        } else {
253            &mut self.step.handles
254        }
255    }
256
257    /// `cpusets` map that receives resolved cpusets for new cgroups.
258    fn target_cpusets(&mut self) -> &mut std::collections::HashMap<String, BTreeSet<usize>> {
259        if self.target_backdrop {
260            &mut self.backdrop.cpusets
261        } else {
262            &mut self.step.cpusets
263        }
264    }
265
266    /// `payload_handles` vec that receives newly-spawned payload handles.
267    fn target_payload_handles(&mut self) -> &mut Vec<PayloadEntry> {
268        if self.target_backdrop {
269            &mut self.backdrop.payload_handles
270        } else {
271            &mut self.step.payload_handles
272        }
273    }
274
275    /// Resolved cpuset for a cgroup name, looked up step-first then backdrop.
276    fn lookup_cpuset(&self, name: &str) -> Option<&BTreeSet<usize>> {
277        self.step
278            .cpusets
279            .get(name)
280            .or_else(|| self.backdrop.cpusets.get(name))
281    }
282
283    /// Returns the live payload handle matching the composite key
284    /// (`payload_name`, `cgroup_key`) from either step-local or
285    /// backdrop state, or `None` when no entry matches. Used for
286    /// the `Op::RunPayload` duplicate guard, which now treats
287    /// "same payload in a different cgroup" as legitimate rather
288    /// than a name collision.
289    fn find_live_payload_with_cgroup(
290        &self,
291        payload_name: &str,
292        cgroup_key: &str,
293    ) -> Option<&PayloadEntry> {
294        let matches =
295            |e: &&PayloadEntry| e.handle.payload_name() == payload_name && e.cgroup == cgroup_key;
296        self.step
297            .payload_handles
298            .iter()
299            .find(matches)
300            .or_else(|| self.backdrop.payload_handles.iter().find(matches))
301    }
302
303    /// Drop a payload handle by composite key (`name`, optional
304    /// `cgroup`). Checks step-local first, then backdrop.
305    ///
306    /// - `cgroup = Some(c)`: exact match on both name and cgroup.
307    /// - `cgroup = None`: if exactly one entry matches `name` across
308    ///   both slots, consume it (backward-compat for
309    ///   `Op::wait_payload(name)` / `Op::kill_payload(name)` when
310    ///   only one copy is live). If two or more match, returns
311    ///   `Err(ambiguous_cgroups)` where `ambiguous_cgroups` is the
312    ///   list of cgroup keys for the candidates so the caller can
313    ///   produce an actionable error.
314    ///
315    /// Returns `Ok(None)` when no entry matches.
316    fn take_payload_by_name(
317        &mut self,
318        name: &str,
319        cgroup: Option<&str>,
320    ) -> std::result::Result<Option<PayloadEntry>, Vec<String>> {
321        if let Some(c) = cgroup {
322            // Composite-key path: exact match on both.
323            if let Some(idx) = self
324                .step
325                .payload_handles
326                .iter()
327                .position(|e| e.handle.payload_name() == name && e.cgroup == c)
328            {
329                return Ok(Some(self.step.payload_handles.swap_remove(idx)));
330            }
331            if let Some(idx) = self
332                .backdrop
333                .payload_handles
334                .iter()
335                .position(|e| e.handle.payload_name() == name && e.cgroup == c)
336            {
337                return Ok(Some(self.backdrop.payload_handles.swap_remove(idx)));
338            }
339            return Ok(None);
340        }
341        // Name-only path: disambiguate across both slots before
342        // consuming, so a mid-test wait on an ambiguous name
343        // surfaces the caller's bug rather than silently waiting
344        // on the first match.
345        let mut step_idx: Option<usize> = None;
346        let mut backdrop_idx: Option<usize> = None;
347        let mut cgroups: Vec<String> = Vec::new();
348        for (i, e) in self.step.payload_handles.iter().enumerate() {
349            if e.handle.payload_name() == name {
350                if step_idx.is_none() {
351                    step_idx = Some(i);
352                }
353                cgroups.push(e.cgroup.clone());
354            }
355        }
356        for (i, e) in self.backdrop.payload_handles.iter().enumerate() {
357            if e.handle.payload_name() == name {
358                if backdrop_idx.is_none() && step_idx.is_none() {
359                    backdrop_idx = Some(i);
360                }
361                cgroups.push(e.cgroup.clone());
362            }
363        }
364        if cgroups.len() > 1 {
365            return Err(cgroups);
366        }
367        if let Some(i) = step_idx {
368            return Ok(Some(self.step.payload_handles.swap_remove(i)));
369        }
370        if let Some(i) = backdrop_idx {
371            return Ok(Some(self.backdrop.payload_handles.swap_remove(i)));
372        }
373        Ok(None)
374    }
375
376    /// Drain every live payload handle in step + backdrop state by
377    /// calling `.kill()` so the metric-emission pipeline fires. Used
378    /// on error paths in the step loop so mid-scenario failure still
379    /// leaves a usable sidecar.
380    fn drain_all_payloads(&mut self) {
381        drain_all_payload_handles(&mut self.step.payload_handles);
382        drain_all_payload_handles(&mut self.backdrop.payload_handles);
383    }
384
385    /// Kill every payload handle (step-first, then backdrop) whose
386    /// cgroup matches `cgroup`. Called before a cgroup removal so
387    /// cgroupfs cleanup does not trip EBUSY on a live process.
388    fn drain_payloads_for_cgroup(&mut self, cgroup: &str) {
389        drain_payload_handles_for_cgroup(&mut self.step.payload_handles, cgroup);
390        drain_payload_handles_for_cgroup(&mut self.backdrop.payload_handles, cgroup);
391    }
392
393    /// Remove every workload handle whose key matches `cgroup`. The
394    /// handles themselves drop (which SIGKILLs the workers) — this is
395    /// appropriate for `Op::StopCgroup` and `Op::RemoveCgroup`.
396    fn drop_handles_for_cgroup(&mut self, cgroup: &str) {
397        self.step.handles.retain(|(n, _)| n.as_str() != cgroup);
398        self.backdrop.handles.retain(|(n, _)| n.as_str() != cgroup);
399    }
400
401    /// Forget a tracked cpuset (step-first, then backdrop) for a cgroup.
402    fn forget_cpuset(&mut self, cgroup: &str) {
403        self.step.cpusets.remove(cgroup);
404        self.backdrop.cpusets.remove(cgroup);
405    }
406
407    /// Record / overwrite the resolved cpuset for a cgroup. If the
408    /// cgroup is known to step-local state, the step-local entry
409    /// updates; if it's known to backdrop, the backdrop entry
410    /// updates; otherwise the entry goes into the currently-active
411    /// target (step-local, or backdrop inside `with_target_backdrop`).
412    fn record_cpuset(&mut self, cgroup: &str, cpuset: BTreeSet<usize>) {
413        if self.step.cpusets.contains_key(cgroup) {
414            self.step.cpusets.insert(cgroup.to_string(), cpuset);
415        } else if self.backdrop.cpusets.contains_key(cgroup) {
416            self.backdrop.cpusets.insert(cgroup.to_string(), cpuset);
417        } else {
418            self.target_cpusets().insert(cgroup.to_string(), cpuset);
419        }
420    }
421
422    /// Re-key every workload handle from `from` to `to`. When `to`
423    /// names a Backdrop-owned cgroup, step-local handles are also
424    /// transferred into [`Self::backdrop`] so their lifetime extends
425    /// to scenario end instead of dying at step teardown. Backdrop
426    /// handles stay in the backdrop slot regardless of `to`.
427    ///
428    /// Called by `Op::MoveAllTasks` after the kernel-side
429    /// `cgroup.procs` writes succeed so subsequent ops that address
430    /// the moved workers by cgroup name find them under the new key
431    /// and in the correct state slot.
432    fn rename_handles(&mut self, from: &str, to: &str) {
433        let to_is_backdrop = self.cgroup_name_is_backdrop(to);
434        if to_is_backdrop {
435            // Move step-local handles keyed under `from` into the
436            // backdrop slot, re-keyed to `to`. Iterate in reverse so
437            // swap_remove indices stay stable.
438            let mut i = self.step.handles.len();
439            while i > 0 {
440                i -= 1;
441                if self.step.handles[i].0.as_str() == from {
442                    let (_, handle) = self.step.handles.swap_remove(i);
443                    self.backdrop.handles.push((to.to_string(), handle));
444                }
445            }
446        } else {
447            // Step-local destination: keep ownership, just rename.
448            for (name, _) in &mut self.step.handles {
449                if name.as_str() == from {
450                    *name = to.to_string();
451                }
452            }
453        }
454        // Backdrop handles are never demoted to step-local ownership
455        // regardless of destination — a backdrop worker is declared
456        // persistent and stays persistent for the scenario. Rename
457        // in place so subsequent ops still find it under the new key.
458        for (name, _) in &mut self.backdrop.handles {
459            if name.as_str() == from {
460                *name = to.to_string();
461            }
462        }
463    }
464
465    /// Iterate every live workload handle across step + backdrop.
466    /// Used by `Op::MoveAllTasks` / `Op::SetAffinity` which act on
467    /// whichever cgroup owns the handle without caring about which
468    /// state slot it's in.
469    fn all_handles(&self) -> impl Iterator<Item = &(String, WorkloadHandle)> {
470        self.step.handles.iter().chain(self.backdrop.handles.iter())
471    }
472
473    /// True iff a cgroup with the given name is already tracked by
474    /// either step-local or backdrop state. Used to reject duplicate
475    /// names at `apply_setup` time so a user can't accidentally
476    /// shadow a Backdrop cgroup with a step-local [`CgroupDef`].
477    fn cgroup_name_is_tracked(&self, name: &str) -> bool {
478        self.step.cgroups.names().iter().any(|n| n == name)
479            || self.backdrop.cgroups.names().iter().any(|n| n == name)
480    }
481
482    /// True iff a cgroup with the given name is tracked by backdrop
483    /// (persistent) state. Used by `Op::MoveAllTasks` to decide
484    /// handle-ownership transfer direction (step→backdrop transfers
485    /// the handle into the persistent slot; backdrop→step-local is
486    /// rejected because it would orphan workers at step teardown).
487    fn cgroup_name_is_backdrop(&self, name: &str) -> bool {
488        self.backdrop.cgroups.names().iter().any(|n| n == name)
489    }
490}
491
492/// Whether a live payload handle was spawned by an explicit
493/// [`Op::RunPayload`] inside the step or by a
494/// [`CgroupDef::workload`] attachment at `apply_setup`. Held by
495/// every [`PayloadEntry`] so the dedup path in `Op::RunPayload`
496/// can name the original source when rejecting a second spawn of
497/// the same name.
498#[derive(Debug, Clone, Copy, PartialEq, Eq)]
499enum PayloadSource {
500    /// Spawned by `CgroupDef::workload(&payload)` during `apply_setup`.
501    CgroupDefWorkload,
502    /// Spawned by `Op::RunPayload { payload, .. }` inside the step's ops.
503    OpRunPayload,
504}
505
506impl PayloadSource {
507    /// Human-readable tag for error output. Describes the API surface
508    /// that originated the spawn, not the internal dispatch site.
509    fn describe(self) -> &'static str {
510        match self {
511            PayloadSource::CgroupDefWorkload => "CgroupDef::workload",
512            PayloadSource::OpRunPayload => "Op::RunPayload",
513        }
514    }
515}
516
517/// One live payload handle plus the cgroup it runs inside and the
518/// API surface that spawned it. `cgroup` is empty iff
519/// `source == PayloadSource::OpRunPayload` was invoked without a
520/// `cgroup = Some(...)` argument — in which case the payload runs
521/// in whatever cgroup its parent process inherited (no explicit
522/// placement).
523struct PayloadEntry {
524    cgroup: String,
525    source: PayloadSource,
526    handle: crate::scenario::payload_run::PayloadHandle,
527}
528
529/// Map the BPF probe's current scheduler-exit classification onto
530/// the [`crate::assert::DetailKind`] variant the three liveness
531/// emission sites push. Reads [`crate::probe::process::sched_exit_kind`]
532/// which mirrors the probe's `ktstr_err_exit_detected` BSS latch
533/// across threads.
534///
535/// Returns:
536/// - `SchedulerCrashed` when the probe observed a non-clean kernel
537///   exit (any path that latched `ktstr_err_exit_detected`).
538/// - `SchedulerExitedCleanly` when the probe ran but never observed
539///   the latch (clean `SCX_EXIT_NONE` teardown, or the scheduler
540///   exited for a benign reason).
541/// - `SchedulerDiedUnknownReason` when the probe has not classified
542///   yet — typically the probe pipeline never wired for this run
543///   (host-only test, no scheduler attached) or the poll thread has
544///   not completed a first iteration since the prior reset.
545fn sched_died_detail_kind() -> crate::assert::DetailKind {
546    use crate::assert::DetailKind;
547    use crate::probe::process::{SchedExitKind, sched_exit_kind};
548    match sched_exit_kind() {
549        SchedExitKind::Crashed => DetailKind::SchedulerCrashed,
550        SchedExitKind::Clean => DetailKind::SchedulerExitedCleanly,
551        SchedExitKind::Unknown => DetailKind::SchedulerDiedUnknownReason,
552    }
553}
554
555#[cfg(not(test))]
556fn scheduler_process_died() -> Option<bool> {
557    match crate::vmm::rust_init::current_scheduler_liveness() {
558        Ok(Some(alive)) => Some(!alive),
559        Ok(None) => None,
560        // An unusable retained pidfd cannot prove scheduler liveness; fail
561        // closed exactly like the prior process probe.
562        Err(_) => Some(true),
563    }
564}
565
566#[cfg(test)]
567fn scheduler_process_died() -> Option<bool> {
568    crate::vmm::rust_init::sched_pid().map(|pid| !process_alive(pid))
569}
570
571/// Classify scheduler liveness for the guest-side `survives_storm` probe:
572/// `Some(kind)` when a scheduler is still expected to run but has died or gone
573/// down — its retained pidfd is terminal OR the scx state reads
574/// `disabling`/`disabled` ([`dispatch::scx_down`]) — and `None` otherwise.
575///
576/// Mirrors the inter-step / post-loop liveness gate `run_scenario` runs for
577/// `execute_*` scenarios, so a clean `Op::DetachScheduler` — which removes the
578/// owner before this would observe `disabled` — does not trip it. The kind
579/// comes from [`sched_died_detail_kind`]; in the primary test VM the BPF
580/// err-exit latch is unread, so the kind is
581/// [`crate::assert::DetailKind::SchedulerDiedUnknownReason`] (the scx state is
582/// kind-less — a crash and a clean unregister both read `disabling`/`disabled`).
583pub(crate) fn sched_liveness_failure_kind() -> Option<crate::assert::DetailKind> {
584    if scheduler_process_died().is_some_and(|died| died || dispatch::scx_down()) {
585        Some(sched_died_detail_kind())
586    } else {
587        None
588    }
589}
590
591/// Execute a single step with CgroupDefs that hold for the full duration.
592///
593/// Convenience wrapper around [`execute_steps`] for the common pattern
594/// of creating cgroups and running them for [`HoldSpec::FULL`].
595pub fn execute_defs(ctx: &Ctx, defs: Vec<CgroupDef>) -> Result<AssertResult> {
596    execute_steps(ctx, vec![Step::with_defs(defs, HoldSpec::FULL)])
597}
598
599/// Block until the host freeze coordinator has ADOPTED its
600/// kernel-symbol accessor — signalled via the
601/// `SIGNAL_ACCESSOR_READY` wake byte →
602/// `accessor_ready_latch` (set by `hvc0_poll_loop`). A failure dump
603/// captured by a stall AFTER this returns renders real BPF map values
604/// instead of placeholders, because the coordinator's `owned_accessor`
605/// is adopted before the stall fires. Call this in a dump-asserting
606/// scenario before triggering its stall (e.g. before [`execute_steps`]
607/// with a `--stall-after` scheduler).
608///
609/// Guest-only: a no-op on the host (unit tests), where the latch is
610/// never armed. Warn-and-proceed on a 60s timeout — a never-adopted
611/// accessor is a worker failure, and surfacing it as a placeholder dump
612/// is more useful than blocking the test forever (mirrors the
613/// `wait_for_map_write` gate's soft-timeout policy). On that timeout the
614/// "renders real values" guarantee above does NOT hold: the wait returns
615/// without adoption and a subsequent stall may dump placeholders, which
616/// the test's post-VM dump assertions then surface as a failure.
617///
618/// The latch is sticky (level-triggered) and sets once, on the FIRST
619/// adoption. A re-init publish after a scheduler swap
620/// (`Op::ReplaceScheduler`) reuses the same latch, so a later
621/// `await_accessor_ready` returns immediately and does NOT re-synchronise
622/// on the post-swap adoption — today's only caller gates the first stall,
623/// before any swap.
624pub fn await_accessor_ready() {
625    if guest_comms::is_guest() {
626        let latch = crate::vmm::rust_init::accessor_ready_latch();
627        if !latch.wait_timeout(Duration::from_secs(60)) {
628            tracing::warn!(
629                "await_accessor_ready timed out after 60s — host freeze \
630                 coordinator did not signal accessor adoption; a dump from a \
631                 stall after this point may render placeholder map values"
632            );
633        }
634    }
635}
636
637/// Execute a sequence of steps against the given context.
638///
639/// Convenience wrapper around [`execute_steps_with`] that passes
640/// `None` for checks, falling back to `ctx.assert`. Use
641/// [`execute_steps_with`] when you need to override `ctx.assert`.
642pub fn execute_steps(ctx: &Ctx, steps: Vec<Step>) -> Result<AssertResult> {
643    execute_steps_with(ctx, steps, None)
644}
645
646/// Execute a [`Backdrop`](backdrop::Backdrop) + Steps sequence
647/// against the given context.
648///
649/// The Backdrop declares persistent scenario-wide state
650/// (long-running payloads, cgroups referenced by many Steps) while
651/// Steps express bounded per-phase behavior. The runtime sets up
652/// the Backdrop before the first Step, runs the Step sequence
653/// with per-Step teardown (cgroups removed, workload handles
654/// collected, payload handles killed at step boundary), and tears
655/// the Backdrop down at the end.
656pub fn execute_scenario(
657    ctx: &Ctx,
658    backdrop: backdrop::Backdrop,
659    steps: Vec<Step>,
660) -> Result<AssertResult> {
661    execute_scenario_with(ctx, backdrop, steps, None)
662}
663
664/// [`execute_scenario`] with an explicit
665/// [`Assert`](crate::assert::Assert) override — the Backdrop
666/// equivalent of [`execute_steps_with`].
667pub fn execute_scenario_with(
668    ctx: &Ctx,
669    backdrop: backdrop::Backdrop,
670    steps: Vec<Step>,
671    checks: Option<&crate::assert::Assert>,
672) -> Result<AssertResult> {
673    run_scenario(ctx, backdrop, steps, checks)
674}
675
676/// Execute steps with an explicit [`Assert`](crate::assert::Assert) for
677/// worker checks. When `checks` is `Some`, it overrides `ctx.assert`.
678/// When `None`, uses `ctx.assert` (the merged three-layer config).
679///
680/// Thin wrapper around [`execute_scenario_with`] with an empty
681/// [`Backdrop`](backdrop::Backdrop) — every Step's effects
682/// (cgroups, workloads, payloads) tear down at the step boundary.
683pub fn execute_steps_with(
684    ctx: &Ctx,
685    steps: Vec<Step>,
686    checks: Option<&crate::assert::Assert>,
687) -> Result<AssertResult> {
688    execute_scenario_with(ctx, backdrop::Backdrop::new(), steps, checks)
689}
690
691/// Compute the union of cgroup v2 controllers required by a
692/// Backdrop and Step sequence. Walks every [`CgroupDef`] declaration
693/// and every [`Op`] variant, returning the smallest set of
694/// controllers that must be enabled in `cgroup.subtree_control` for
695/// the scenario's per-knob writes to land.
696///
697/// Mapping:
698/// - [`CgroupDef::cpuset`] / [`CgroupDef::cpuset_mems`] → `Controller::Cpuset`
699/// - [`CgroupDef::cpu`] → `Controller::Cpu`
700/// - [`CgroupDef::memory`] → `Controller::Memory`
701/// - [`CgroupDef::pids`] → `Controller::Pids`
702/// - [`CgroupDef::io`] → `Controller::Io`
703/// - [`Op::SetCpuset`] / [`Op::ClearCpuset`] / [`Op::SwapCpusets`] /
704///   [`Op::SetAffinity`] → `Controller::Cpuset`
705/// - Every other [`Op`] variant ([`Op::FreezeCgroup`],
706///   [`Op::AddCgroup`], [`Op::Spawn`], [`Op::MoveAllTasks`], etc.)
707///   touches cgroup-core knobs (`cgroup.freeze`, `cgroup.procs`,
708///   `mkdir`/`rmdir`) which are ungated by any controller and
709///   contribute nothing to this set.
710///
711/// Returning the SMALLEST set lets a test that intentionally
712/// requires the absence of a controller (e.g. testing behavior on
713/// a kernel without `+cpu`) get an empty subtree_control write.
714fn required_controllers(
715    ctx: &Ctx,
716    backdrop: &backdrop::Backdrop,
717    steps: &[Step],
718) -> BTreeSet<crate::cgroup::Controller> {
719    use crate::cgroup::Controller;
720    fn absorb_def(set: &mut BTreeSet<Controller>, def: &CgroupDef) {
721        if def.cpuset.is_some() || def.cpuset_mems.is_some() {
722            set.insert(Controller::Cpuset);
723        }
724        if def.cpu.is_some() {
725            set.insert(Controller::Cpu);
726        }
727        if def.memory.is_some() {
728            set.insert(Controller::Memory);
729        }
730        if def.io.is_some() {
731            set.insert(Controller::Io);
732        }
733        if def.pids.is_some() {
734            set.insert(Controller::Pids);
735        }
736    }
737    fn absorb_op(set: &mut BTreeSet<Controller>, op: &Op) {
738        if matches!(
739            op,
740            Op::SetCpuset { .. }
741                | Op::ClearCpuset { .. }
742                | Op::SwapCpusets { .. }
743                | Op::SetAffinity { .. }
744        ) {
745            set.insert(Controller::Cpuset);
746        }
747        // AddCgroupDef carries a full CgroupDef whose knobs may
748        // require any of the same controllers absorb_def covers. The
749        // op-applied def goes through apply_setup at op-execute time,
750        // which writes to those controller files; the parent's
751        // subtree_control must already have the controllers enabled
752        // by then, so absorb the def's needs into the pre-scenario
753        // controller setup the same way step-local CgroupDefs do.
754        if let Op::AddCgroupDef { def } = op {
755            absorb_def(set, def);
756        }
757    }
758    let mut set = BTreeSet::new();
759    for def in &backdrop.cgroups {
760        absorb_def(&mut set, def);
761    }
762    for op in &backdrop.ops {
763        absorb_op(&mut set, op);
764    }
765    for step in steps {
766        for def in step.setup.resolve(ctx) {
767            absorb_def(&mut set, &def);
768        }
769        for op in &step.ops {
770            absorb_op(&mut set, op);
771        }
772    }
773    set
774}
775
776/// Validate every step hold spec and reject scheduler-kind Backdrop
777/// payloads up front so failures surface at the declaration, before
778/// any runtime state is created.
779fn validate_scenario_inputs(backdrop: &backdrop::Backdrop, steps: &[Step]) -> Result<()> {
780    // Validate every step's hold spec up front so a typo doesn't
781    // reach `Duration::from_secs_f64(NaN)` / `thread::sleep(ZERO)` /
782    // a no-yield Loop busy-wait after ops have already been applied.
783    for (i, step) in steps.iter().enumerate() {
784        if let Err(reason) = step.hold.validate() {
785            anyhow::bail!("step {i} hold validation: {reason}");
786        }
787    }
788    // Validate Backdrop payloads before creating any runtime state.
789    // Only binary payloads can be spawned by Op::RunPayload, which
790    // is what the Backdrop setup uses under the hood. Reject
791    // scheduler-kind payloads here so the failure surface is the
792    // Backdrop declaration, not a mid-scenario spawn error after
793    // cgroups have already been created.
794    for p in &backdrop.payloads {
795        if p.is_scheduler() {
796            anyhow::bail!(
797                "Backdrop::push_payload received scheduler-kind Payload '{}' — \
798                 only PayloadKind::Binary payloads run in the Backdrop; \
799                 place scheduler-kind payloads on the #[ktstr_test(scheduler = ...)] \
800                 attribute instead",
801                p.name,
802            );
803        }
804    }
805    // Scheduler-kind payloads smuggled via Backdrop::push_op(Op::RunPayload { ... })
806    // would otherwise bypass the check above and only bail deep inside
807    // apply_ops. Reject them here with a Backdrop-specific error so
808    // the failure surface matches the declaration surface.
809    for op in &backdrop.ops {
810        if let Op::RunPayload { payload, .. } = op
811            && payload.is_scheduler()
812        {
813            anyhow::bail!(
814                "Backdrop::push_op(Op::RunPayload) received scheduler-kind Payload '{}' — \
815                 only PayloadKind::Binary payloads run in the Backdrop; \
816                 place scheduler-kind payloads on the #[ktstr_test(scheduler = ...)] \
817                 attribute instead",
818                payload.name,
819            );
820        }
821    }
822    Ok(())
823}
824
825/// Guest-side gate that blocks until the host's queued BPF map writes
826/// land, so the workload phase observes fresh map values.
827fn wait_for_host_map_write(ctx: &Ctx) {
828    // When a host-side BPF map write is configured the test framework
829    // sets `wait_for_map_write=true`; in that case block until the
830    // guest's `hvc0_poll_loop` observes
831    // [`crate::vmm::virtio_console::SIGNAL_BPF_WRITE_DONE`] (pushed by
832    // the host's `bpf-map-write` thread after every queued
833    // `bpf_map_write` lands) and fires the `bpf_map_write_done` latch.
834    // Without this gate the workload phase races against the host's
835    // map writes and may observe a stale BPF map value.
836    //
837    // Guest-only path. On the host (unit tests) the latch is never
838    // armed, so we skip the wait entirely. The 60 s timeout matches
839    // the bpf-map-write thread's combined phase 1 + phase 2 budget
840    // (30 s accessor init + 30 s map discovery in
841    // `freeze_coord::start_bpf_map_write`); a real timeout means the
842    // host failed to resolve a map. The scenario continues anyway
843    // (rather than `bail!`) because the legacy rendezvous also let
844    // the guest proceed under its own timeout, and a bail here would
845    // mask the underlying host-side resolution failure with a
846    // test-side `Err`.
847    if ctx.wait_for_map_write && guest_comms::is_guest() {
848        let latch = crate::vmm::rust_init::bpf_map_write_done_latch();
849        if !latch.wait_timeout(Duration::from_secs(60)) {
850            tracing::warn!(
851                "wait_for_map_write timed out after 60s — host bpf-map-write \
852                 thread may have failed to resolve a queued map; proceeding \
853                 with the workload regardless"
854            );
855        }
856    }
857}
858
859/// Run persistent Backdrop setup. `Ok(None)` on success; `Ok(Some(r))`
860/// carries the failure AssertResult when setup errors so the caller
861/// returns it directly.
862fn run_backdrop_setup<'a>(
863    ctx: &'a Ctx,
864    backdrop: &backdrop::Backdrop,
865    backdrop_state: &mut BackdropState<'a>,
866    effective_checks: &crate::assert::Assert,
867    result: &AssertResult,
868) -> Result<Option<AssertResult>> {
869    // --- Backdrop setup (persistent) ---
870    // Run before the first Step. Cgroups + payloads declared on
871    // `backdrop` land in `backdrop_state` so they survive every
872    // Step's teardown. On error, drain Backdrop payload handles
873    // (metric emission) and propagate.
874    if backdrop.is_empty() {
875        return Ok(None);
876    }
877    let mut step_staging = StepState::empty(ctx);
878    let mut scratch = ScenarioState::new(&mut step_staging, backdrop_state);
879    let setup_res = scratch.with_target_backdrop(|s| {
880        // Order: cgroups → ops → payloads. CgroupDefs go first so
881        // a later `Op::add_cgroup` / `Op::run_payload_in_cgroup`
882        // can target cgroups that `apply_setup` just created.
883        // Payloads spawn last so `run_payload` resolving a cgroup
884        // placement lands inside a cgroup that either apply pass
885        // already built.
886        if !backdrop.cgroups.is_empty() {
887            apply_setup(ctx, s, &backdrop.cgroups)?;
888        }
889        // Raw ops: typically `Op::AddCgroup` for empty move-target
890        // cgroups (can't be expressed via CgroupDef because
891        // apply_setup forces a worker spawn), or placement-aware
892        // `Op::RunPayload` targeting a just-created backdrop
893        // cgroup.
894        if !backdrop.ops.is_empty() {
895            apply_ops(ctx, s, &backdrop.ops, false)?;
896        }
897        // Shorthand payloads: one Op::RunPayload per entry,
898        // inherited cgroup placement.
899        if !backdrop.payloads.is_empty() {
900            let ops: Vec<Op> = backdrop
901                .payloads
902                .iter()
903                .map(|p| Op::run_payload(p, Vec::<String>::new()))
904                .collect();
905            apply_ops(ctx, s, &ops, false)?;
906        }
907        Ok::<(), anyhow::Error>(())
908    });
909    if let Err(err) = setup_res {
910        // Scheduler crashed during backdrop setup (e.g. a worker
911        // tripped the BPF error before the first Step): SIGKILL the
912        // spawned workers up front so the collect reaps below aren't
913        // scheduling-gated behind fallback workers (see
914        // `sigkill_handles`). Gated on `scx_down` (crash-only) so a
915        // non-crash setup error keeps the graceful collect path.
916        if sched_crashed_unexpectedly() {
917            sigkill_handles(&backdrop_state.handles);
918            sigkill_handles(&step_staging.handles);
919        }
920        // Collect any workers that DID spawn before the failure
921        // so their stats reach the final result instead of being
922        // discarded by `WorkloadHandle::drop` (which SIGKILLs
923        // without gathering scheduler-side data). `collect_*`
924        // drain `payload_handles` internally, so the backdrop-
925        // and step-side payloads still get `.kill()` (SHM metric
926        // emission) on the error path.
927        //
928        // `with_target_backdrop` routes every target writer to
929        // the backdrop slot, so `step_staging` normally holds
930        // nothing — but collect defensively so a partial-failure
931        // path that leaks a non-backdrop write surfaces here
932        // rather than disappearing into `StepState::drop`.
933        let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
934        let staging_result = collect_step(
935            &mut step_staging,
936            effective_checks,
937            ctx.topo,
938            ctx.cgroups,
939            // Defensive backdrop-staging collect: no step attribution.
940            None,
941        );
942        r.merge(staging_result);
943        r.merge(result.clone());
944        // step_staging's CgroupGroup RAII still drops here,
945        // removing any cgroups the failed Backdrop setup routed
946        // into step-local state.
947        r.record_fail(crate::assert::AssertDetail::new(
948            crate::assert::DetailKind::Other,
949            format!("Backdrop setup failed: {err:#}"),
950        ));
951        return Ok(Some(r));
952    }
953    // `step_staging` should not have accumulated anything
954    // because `with_target_backdrop` routed every target writer
955    // to the backdrop side. Collect any stray handles defensively
956    // before dropping so a future refactor that leaks a non-
957    // backdrop write here surfaces as a missed teardown rather
958    // than silently discarded state.
959    drain_all_payload_handles(&mut step_staging.payload_handles);
960    Ok(None)
961}
962
963/// 1-indexed phase number for scenario Step `step_idx` (BASELINE=0,
964/// Step k -> k+1), saturating at `u16::MAX` past the encoding range.
965fn phase_step_index(step_idx: usize) -> u16 {
966    u16::try_from(step_idx)
967        .ok()
968        .and_then(|i| i.checked_add(1))
969        .unwrap_or(u16::MAX)
970}
971
972/// Build the inter-step scheduler-death failure result: collect
973/// backdrop workers, merge the accumulated result, record the
974/// `format_sched_died_after_step` detail.
975fn build_sched_died_after_step(
976    backdrop_state: &mut BackdropState<'_>,
977    result: AssertResult,
978    ctx: &Ctx,
979    effective_checks: &crate::assert::Assert,
980    step_idx: usize,
981    steps_len: usize,
982    scenario_start: std::time::Instant,
983) -> AssertResult {
984    // Collect backdrop-owned workload handles into the
985    // result before reporting the crash so whatever the
986    // persistent workers produced is still assertable.
987    let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
988    r.merge(result);
989    r.record_fail(crate::assert::AssertDetail::new(
990        sched_died_detail_kind(),
991        crate::assert::format_sched_died_after_step(
992            step_idx,
993            steps_len,
994            scenario_start.elapsed().as_secs_f64(),
995        ),
996    ));
997    r
998}
999
1000/// Build the step-error failure result: collect backdrop workers,
1001/// merge the accumulated result, record the `step N failed` detail.
1002fn build_step_failed(
1003    backdrop_state: &mut BackdropState<'_>,
1004    result: AssertResult,
1005    ctx: &Ctx,
1006    effective_checks: &crate::assert::Assert,
1007    step_idx: usize,
1008    err: &anyhow::Error,
1009) -> AssertResult {
1010    // Collect Backdrop-owned workload handles into a fresh
1011    // result first, then merge the accumulated step result
1012    // on top. `collect_backdrop` drains
1013    // `backdrop_state.payload_handles` internally, so the
1014    // backdrop-side payloads still get `.kill()` (metric
1015    // emission) on the error path. Ordering mirrors the
1016    // scheduler-crash path above so detail order is
1017    // consistent across both Ok(failed) returns.
1018    let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1019    r.merge(result);
1020    r.record_fail(crate::assert::AssertDetail::new(
1021        crate::assert::DetailKind::Other,
1022        format!("step {step_idx} failed: {err:#}"),
1023    ));
1024    r
1025}
1026
1027/// Build the sched-died-during-hold failure result: collect backdrop
1028/// workers, merge the accumulated result, record the
1029/// `format_sched_died_during_workload` detail.
1030fn build_sched_died_during_hold(
1031    backdrop_state: &mut BackdropState<'_>,
1032    result: AssertResult,
1033    ctx: &Ctx,
1034    effective_checks: &crate::assert::Assert,
1035    scenario_start: std::time::Instant,
1036) -> AssertResult {
1037    let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1038    r.merge(result);
1039    r.record_fail(crate::assert::AssertDetail::new(
1040        sched_died_detail_kind(),
1041        crate::assert::format_sched_died_during_workload(scenario_start.elapsed().as_secs_f64()),
1042    ));
1043    r
1044}
1045
1046/// Emit the ScenarioEnd marker, run the final liveness check, tear the
1047/// Backdrop down, and fold in a sched-died detail when the scheduler
1048/// died after the last hold.
1049fn finish_scenario(
1050    ctx: &Ctx,
1051    backdrop_state: &mut BackdropState<'_>,
1052    effective_checks: &crate::assert::Assert,
1053    scenario_start: std::time::Instant,
1054    steps_len: usize,
1055    final_total_iterations: Option<(u64, u64)>,
1056    mut result: AssertResult,
1057) -> AssertResult {
1058    // ScenarioEnd marker. Routes through `send_scenario_end`
1059    // (virtio-console port-1 bulk channel).
1060    // Carries the last cleanly-completed step's COINCIDENT (elapsed_ms,
1061    // iterations) pair so the host can give the last step an
1062    // `iteration_rate` over a well-formed window. The elapsed
1063    // is the end-of-hold timestamp captured WITH the count inside
1064    // run_step — NOT recomputed here, which would include the
1065    // pause+collect teardown wall-time and under-report the rate. When
1066    // no step completed a hold (e.g. every step's scheduler died) the
1067    // pair is `None`: fall back to (now, 0), which the host skips
1068    // (0 baseline / e<=s).
1069    if guest_comms::is_guest() {
1070        let (end_elapsed_ms, end_iterations) = final_total_iterations
1071            .unwrap_or_else(|| (scenario_start.elapsed().as_millis() as u64, 0));
1072        crate::vmm::guest_comms::send_scenario_end(end_elapsed_ms, end_iterations);
1073    }
1074
1075    // Final liveness check. Live `crate::vmm::rust_init::sched_pid()`
1076    // read instead of `ctx.sched_pid` snapshot so a mid-scenario
1077    // Op::ReplaceScheduler swap reflects the new pid here too.
1078    // sched_pid() == None ⇒ no scheduler configured (kernel-default
1079    // path) OR Op::DetachScheduler cleared it; no liveness to
1080    // report on either case.
1081    let sched_dead = scheduler_process_died().is_some_and(|died| died || dispatch::scx_down());
1082
1083    // Scheduler died after the last hold: SIGKILL the backdrop
1084    // workers up front so the teardown reap below isn't
1085    // scheduling-gated behind still-spinning workers that fell back
1086    // to the builtin scheduler (see `sigkill_handles`).
1087    if sched_dead {
1088        sigkill_handles(&backdrop_state.handles);
1089    }
1090
1091    // --- Backdrop teardown ---
1092    let backdrop_result = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1093    result.merge(backdrop_result);
1094
1095    if sched_dead {
1096        result.record_fail(crate::assert::AssertDetail::new(
1097            sched_died_detail_kind(),
1098            crate::assert::format_sched_died_after_all_steps(
1099                steps_len,
1100                scenario_start.elapsed().as_secs_f64(),
1101            ),
1102        ));
1103    }
1104
1105    result
1106}
1107
1108/// Guest workload-progress publisher. It emits scenario-elapsed time from the
1109/// scenario driver's EXISTING scheduler-liveness wakeups (plus lifecycle
1110/// boundaries), so the host can clock periodic captures off guest progress
1111/// without adding a periodically-runnable guest thread of its own.
1112///
1113/// The driver's hold path already wakes every 100 ms to inspect the BPF
1114/// err-exit latch / sched_ext state. Publishing from those wakes preserves the
1115/// heartbeat's cadence while the capture-active workload is running at no
1116/// additional scheduler wakeup cost. Setup and paused teardown publish only at
1117/// their natural boundaries; there is no useful periodic-capture progress to
1118/// report while the driver is not in a workload hold.
1119#[derive(Clone, Copy)]
1120struct WorkloadHeartbeat {
1121    scenario_start: std::time::Instant,
1122}
1123
1124impl WorkloadHeartbeat {
1125    fn publish(self) {
1126        crate::vmm::guest_comms::send_workload_progress(
1127            self.scenario_start.elapsed().as_millis() as u64
1128        );
1129    }
1130}
1131
1132/// Internal driver: runs Backdrop setup, the Step loop with
1133/// per-Step teardown, and final Backdrop teardown.
1134fn run_scenario(
1135    ctx: &Ctx,
1136    backdrop: backdrop::Backdrop,
1137    steps: Vec<Step>,
1138    checks: Option<&crate::assert::Assert>,
1139) -> Result<AssertResult> {
1140    validate_scenario_inputs(&backdrop, &steps)?;
1141    let effective_checks = checks.unwrap_or(&ctx.assert);
1142
1143    // Enable the controllers this scenario actually needs in
1144    // `cgroup.subtree_control` BEFORE any cgroupfs writes land. The
1145    // union is computed from every CgroupDef and Op declared in the
1146    // backdrop+steps; tests that declare no controller-gated knobs
1147    // get an empty set (parent dir created, no subtree_control walk).
1148    let required = required_controllers(ctx, &backdrop, &steps);
1149    ctx.cgroups
1150        .setup(&required)
1151        .context("enable cgroup controllers in subtree_control")?;
1152
1153    let mut backdrop_state = BackdropState::empty(ctx);
1154    let mut result = AssertResult::pass();
1155
1156    // (scenario-relative elapsed_ms, cumulative worker iteration count)
1157    // at the END of the most-recently completed step, captured
1158    // coincidently inside `run_step` while that step's workers are still
1159    // alive. Carries the LAST step's pair out of the loop for terminal
1160    // boundary telemetry. `None` until the first step completes a hold
1161    // cleanly.
1162    let mut final_total_iterations: Option<(u64, u64)> = None;
1163
1164    let scenario_start = std::time::Instant::now();
1165
1166    // ScenarioStart marker. `is_guest` short-circuits in host
1167    // contexts (unit tests) where the bulk port and SHM ring are
1168    // both absent and `send_scenario_start` would log a no-op warning.
1169    if guest_comms::is_guest() {
1170        crate::vmm::guest_comms::send_scenario_start();
1171    }
1172
1173    // Workload-progress publisher: drives the host's periodic-capture clock
1174    // off guest progress. Only present when this run declares periodic
1175    // captures (`KTSTR_AWAIT_PERIODIC_READY`, set iff num_snapshots > 0).
1176    // Unlike the old detached heartbeat, this is passive state carried by the
1177    // scenario driver; `hold_or_sched_died` publishes from wakeups the driver
1178    // already performs for scheduler liveness.
1179    let workload_heartbeat = (guest_comms::is_guest()
1180        && crate::vmm::rust_init::cmdline_val("KTSTR_AWAIT_PERIODIC_READY").is_some())
1181    .then_some(WorkloadHeartbeat { scenario_start });
1182    if let Some(heartbeat) = workload_heartbeat {
1183        heartbeat.publish();
1184    }
1185
1186    wait_for_host_map_write(ctx);
1187
1188    if let Some(r) = run_backdrop_setup(
1189        ctx,
1190        &backdrop,
1191        &mut backdrop_state,
1192        effective_checks,
1193        &result,
1194    )? {
1195        return Ok(r);
1196    }
1197
1198    // --- Step loop with per-Step teardown ---
1199    for (step_idx, step) in steps.iter().enumerate() {
1200        // Check scheduler liveness between steps (skip before first).
1201        // Live `crate::vmm::rust_init::sched_pid()` read instead of
1202        // `ctx.sched_pid` snapshot so a mid-scenario
1203        // `Op::ReplaceScheduler` swap is reflected — the swap
1204        // dispatcher atomically publishes the replacement's coherent
1205        // process owner, and this check then observes the new pid's
1206        // liveness (not the dead boot pid). `None` means
1207        // either no scheduler was configured at boot or
1208        // `Op::DetachScheduler` cleared the pid; the liveness probe
1209        // cannot meaningfully report on a pid that doesn't exist.
1210        if step_idx > 0 && scheduler_process_died().is_some_and(|died| died || dispatch::scx_down())
1211        {
1212            // Scheduler died between steps: the kernel has disabled
1213            // sched_ext and the backdrop workers have fallen back to
1214            // the builtin scheduler. SIGKILL them up front so the
1215            // collect reap below isn't scheduling-gated behind
1216            // still-spinning workers (see `sigkill_handles`).
1217            sigkill_handles(&backdrop_state.handles);
1218            return Ok(build_sched_died_after_step(
1219                &mut backdrop_state,
1220                result,
1221                ctx,
1222                effective_checks,
1223                step_idx,
1224                steps.len(),
1225                scenario_start,
1226            ));
1227        }
1228
1229        let mut step_state = StepState::empty(ctx);
1230        let mut sched_died_during_hold = false;
1231        // Publish the 1-indexed phase number for this Step so the
1232        // freeze-coordinator periodic-capture path and the on-demand
1233        // Op::CaptureSnapshot / Op::WatchSnapshot apply arms all
1234        // stamp the captures they take with the correct scenario
1235        // phase. The 1-indexed encoding (scenario Step k -> phase
1236        // k + 1) reserves phase 0 for the pre-first-Step BASELINE
1237        // settle window. `Release` pairs with the consumers'
1238        // `Acquire` load so a sample stamped with this value
1239        // happens-after any state the Step has set up before
1240        // calling run_step.
1241        let phase_step_index = phase_step_index(step_idx);
1242        ctx.current_step
1243            .store(phase_step_index, std::sync::atomic::Ordering::Release);
1244        // Broadcast the phase to this handle's backdrop (persistent)
1245        // workers so each drains its per-phase PhaseSlice at this
1246        // boundary. Step-local pools are NOT bumped here — they
1247        // re-spawn per step and get per-phase attribution via
1248        // collect_step's step_index instead.
1249        for (_, h) in &backdrop_state.handles {
1250            h.set_phase_epoch(u32::from(phase_step_index));
1251        }
1252        // Install the assert-side phase guard for the scenario
1253        // driver's thread for the duration of this Step. Every
1254        // AssertDetail / PassDetail / InfoNote constructed under
1255        // the run_step call below auto-stamps its `phase` field
1256        // with "Step[<step_idx>]" via the thread-local snapshot
1257        // in `crate::assert::current_phase_label`. On Drop the
1258        // prior label is restored (BASELINE outside any Step), so
1259        // assertions evaluated post-loop (e.g. at scenario
1260        // teardown) stamp with the right outer scope.
1261        let _phase_guard = crate::assert::PhaseGuard::install_step(step_idx as u16);
1262        let step_res = run_step(
1263            ctx,
1264            step,
1265            step_idx,
1266            &mut step_state,
1267            &mut backdrop_state,
1268            scenario_start,
1269            effective_checks,
1270            &mut sched_died_during_hold,
1271            &mut final_total_iterations,
1272            workload_heartbeat,
1273        );
1274
1275        // Close this step's hold window for backdrop workers: write the
1276        // inter-step gap sentinel so work done between this StepEnd and
1277        // the next StepStart is NOT folded into step `step_idx`'s slice
1278        // (matching the host's hold-only window clamp, and step-local
1279        // workers, which do not exist during the inter-step gap).
1280        for (_, h) in &backdrop_state.handles {
1281            h.set_phase_epoch(u32::MAX);
1282        }
1283
1284        // Scheduler died during this step's hold: the kernel has
1285        // disabled sched_ext and the workers (step and backdrop) have
1286        // fallen back to the builtin scheduler. If CPU-bound they
1287        // would CFS-starve the per-worker reap in the collect calls
1288        // below, so SIGKILL them all up front — the reaps then find
1289        // already-exiting workers (see `sigkill_handles`). Gated on
1290        // the death flag so the normal teardown keeps its graceful
1291        // cooperative-stop + report-collection path.
1292        if sched_died_during_hold || sched_crashed_unexpectedly() {
1293            sigkill_handles(&step_state.handles);
1294            sigkill_handles(&backdrop_state.handles);
1295        }
1296
1297        if guest_comms::is_guest() {
1298            crate::vmm::guest_comms::send_scenario_pause();
1299        }
1300
1301        let step_result = collect_step(
1302            &mut step_state,
1303            effective_checks,
1304            ctx.topo,
1305            ctx.cgroups,
1306            // The SAME 1-indexed value stamped onto this step's StepStart
1307            // frames (build_stimulus), so the guest per_cgroup keys on the
1308            // identical step_index the host rebuilds buckets under.
1309            Some(phase_step_index),
1310        );
1311        result.merge(step_result);
1312
1313        // A step-level error is converted into a failure on the
1314        // accumulated result after teardown has run so every step
1315        // boundary leaves clean state behind even on failure. The
1316        // caller keeps the prior-steps' merged AssertResult plus
1317        // the error context as a detail, instead of an opaque Err
1318        // that discards everything.
1319        if let Err(err) = step_res {
1320            // Scheduler may have crashed between the pre-collect gate
1321            // above and here (e.g. a non-crash step error, then scx went
1322            // down during `collect_step`): SIGKILL the backdrop workers
1323            // up front so this collect isn't scheduling-gated behind the
1324            // fallback pool (see `sigkill_handles`). Crashed-only gate so
1325            // a plain step error with a live scheduler keeps the graceful
1326            // cooperative-stop path.
1327            if sched_crashed_unexpectedly() {
1328                sigkill_handles(&backdrop_state.handles);
1329            }
1330            return Ok(build_step_failed(
1331                &mut backdrop_state,
1332                result,
1333                ctx,
1334                effective_checks,
1335                step_idx,
1336                &err,
1337            ));
1338        }
1339
1340        // Scheduler exited during the step's hold-period sleep —
1341        // [`run_step`] cut the hold short and stamped
1342        // `sched_died_during_hold`. Emit the in-step
1343        // sched-died message before continuing to the next step
1344        // boundary; otherwise the post-loop probe would fire after
1345        // the full scenario duration and stamp a misleading elapsed
1346        // time. Same Backdrop-then-step merge order as the
1347        // inter-step path above so detail ordering stays consistent.
1348        if sched_died_during_hold {
1349            return Ok(build_sched_died_during_hold(
1350                &mut backdrop_state,
1351                result,
1352                ctx,
1353                effective_checks,
1354                scenario_start,
1355            ));
1356        }
1357    }
1358
1359    Ok(finish_scenario(
1360        ctx,
1361        &mut backdrop_state,
1362        effective_checks,
1363        scenario_start,
1364        steps.len(),
1365        final_total_iterations,
1366        result,
1367    ))
1368}
1369
1370/// Sleep up to `dur`, returning early if `sched_pid` exits.
1371///
1372/// Returns `true` the first time the scheduler is observed dead,
1373/// `false` if the full duration elapsed with no death observed.
1374/// When `sched_pid` is `None` (kernel-default scheduling, no
1375/// scheduler process to monitor), behaves exactly like
1376/// [`thread::sleep`] and always returns `false`.
1377///
1378/// Implementation duplicates the pidfd retained by the coherent scheduler
1379/// owner and waits with `epoll_wait`, so the waiter is kernel-blocked on the
1380/// exact process identity until either the scheduler exits
1381/// (pidfd becomes readable) or the per-step hold elapses. This
1382/// drops crash-detection latency from one poll-tick (the previous
1383/// 100 ms cadence) to ~0: the kernel wakes the epoll waiter as
1384/// soon as the task transitions to EXIT_ZOMBIE. Mirrors
1385/// [`crate::scenario::payload_run`]'s `wait_with_deadline` shape.
1386/// Minimum kernel: Linux 5.3.
1387///
1388/// Deadline honoring: the `epoll_wait` timeout is re-derived from
1389/// `saturating_duration_since` each iteration so `EINTR` restarts
1390/// narrow the remaining window rather than extending it.
1391///
1392/// Failure handling: if the owner disappears between the caller's snapshot and
1393/// the pidfd duplication, the exact scheduler is already terminal and the
1394/// function returns `true`. Descriptor duplication, epoll creation/registration,
1395/// timeout conversion, and wait failures panic with an operator-actionable
1396/// message. Polling fallbacks were removed per the project-wide "no polling
1397/// fallbacks for evented paths" rule: these failures indicate a catastrophic
1398/// environment defect rather than a recoverable transient. A loud panic
1399/// surfaces the defect immediately; the prior silent sleep+probe fallback
1400/// masked it as test flakiness. Numeric `pidfd_open` exists only in the
1401/// host-unit-test seam below, whose synthetic scheduler PID has no guest owner.
1402///
1403/// Scheduling jitter under load can leave the actual elapsed time
1404/// modestly above `dur`.
1405/// Loud panic on env- or code-defect failures inside [`hold_or_sched_died`].
1406/// Centralised so every site emits the same module-qualified prefix
1407/// `ktstr::scenario::hold_or_sched_died` — operator can grep that exact
1408/// string to land at the panic source. `op` names the failed primitive,
1409/// `pid` carries the in-scope pid for cross-reference with /proc, `err`
1410/// renders the underlying errno or nix error, `advice` is the one-line
1411/// remediation classified by failure class (env vs framework code defect).
1412#[cold]
1413#[track_caller]
1414fn panic_evented_hold_defect(
1415    op: &str,
1416    pid: libc::pid_t,
1417    err: impl std::fmt::Display,
1418    advice: &str,
1419) -> ! {
1420    panic!("ktstr::scenario::hold_or_sched_died: {op} failed (pid={pid}): {err} — {advice}");
1421}
1422
1423/// True when a scheduler is still expected running (`sched_pid` set) AND
1424/// sched_ext has gone down (`disabling`/`disabled`) — an unexpected
1425/// crash/unregister of the scheduler under test, detected via the
1426/// probe-independent [`dispatch::scx_down`] sysfs read. Unlike the BPF
1427/// err-exit latch (whose host mirror is only populated in the
1428/// auto-repro/dump VM, never the primary VM that runs the test), this is
1429/// live in the PRIMARY VM. The `sched_pid` guard keeps a clean
1430/// [`Op::DetachScheduler`] — which clears the pid before the state
1431/// settles to `disabled` — off the SIGKILL fast-path. Used by the
1432/// teardown gates that don't already hold the scheduler pid in scope.
1433fn sched_crashed_unexpectedly() -> bool {
1434    crate::vmm::rust_init::sched_pid().is_some() && dispatch::scx_down()
1435}
1436
1437fn hold_or_sched_died(
1438    dur: Duration,
1439    sched_pid: Option<libc::pid_t>,
1440    workload_heartbeat: Option<WorkloadHeartbeat>,
1441) -> bool {
1442    use crate::probe::process::{SchedExitKind, sched_exit_kind};
1443    use nix::sys::epoll::{Epoll, EpollCreateFlags, EpollEvent, EpollFlags, EpollTimeout};
1444    use std::os::fd::AsFd;
1445    #[cfg(test)]
1446    use std::os::fd::{FromRawFd, OwnedFd};
1447
1448    // The BPF err-exit latch (read via `sched_exit_kind`) flips at the
1449    // error-class sched_ext exit — i.e. at the crash, before the crashed
1450    // scheduler PROCESS exits (a scheduler like scx_lavd observes the
1451    // disable through its ~1s userspace poll loop, then flushes its dump
1452    // and exits, so its process exit trails the crash by ~that long). Poll
1453    // it ALONGSIDE the pidfd so a crash aborts the hold at ~crash time, not
1454    // at the process exit. The pidfd remains the backstop for
1455    // clean/non-error process exits. 100ms is negligible vs the crash + the
1456    // scheduler's exit latency.
1457    const ERR_EXIT_POLL: Duration = Duration::from_millis(100);
1458    let crashed = || matches!(sched_exit_kind(), SchedExitKind::Crashed);
1459
1460    // Clone the pidfd retained by the coherent scheduler owner while its lock
1461    // is held. Reopening `sched_pid` after the lock is released would recreate
1462    // the numeric-PID reuse race that the owner record exists to eliminate.
1463    let exact_scheduler = match crate::vmm::rust_init::clone_current_scheduler_pidfd() {
1464        Ok(identity) => identity,
1465        Err(error) => panic_evented_hold_defect(
1466            "dup(current scheduler pidfd)",
1467            sched_pid.unwrap_or(-1),
1468            error,
1469            "the owner retains a live pidfd; dup failure indicates fd exhaustion",
1470        ),
1471    };
1472    #[cfg(test)]
1473    let exact_scheduler = match (exact_scheduler, sched_pid) {
1474        (Some(identity), _) => Some(identity),
1475        (None, Some(pid)) => {
1476            // Host unit tests inject a synthetic pid without installing the
1477            // guest-only process owner. Keep that seam test-only.
1478            let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0u32) as libc::c_int };
1479            if raw < 0 {
1480                None
1481            } else {
1482                Some((pid, unsafe { OwnedFd::from_raw_fd(raw) }))
1483            }
1484        }
1485        (None, None) => None,
1486    };
1487    if sched_pid.is_some() && exact_scheduler.is_none() {
1488        // The caller observed an owner immediately before this clone and a
1489        // concurrent orderly transition removed it. Treat that exact owner as
1490        // terminal rather than opening whichever task now owns the old number.
1491        return true;
1492    }
1493
1494    if dur.is_zero() {
1495        if let Some(heartbeat) = workload_heartbeat {
1496            heartbeat.publish();
1497        }
1498        return crashed()
1499            || (exact_scheduler.is_some() && dispatch::scx_down())
1500            || exact_scheduler.as_ref().is_some_and(|(_, pidfd)| {
1501                !matches!(crate::vmm::rust_init::pidfd_is_alive(pidfd), Ok(true))
1502            });
1503    }
1504    let deadline = std::time::Instant::now() + dur;
1505    let Some((pid, pidfd)) = exact_scheduler else {
1506        // No scheduler pid (host-only run, or the pid was not recorded):
1507        // no pidfd to wait on, but the err-exit latch still fires on a
1508        // crash — poll it in short intervals instead of sleeping blind
1509        // for the whole window.
1510        loop {
1511            if let Some(heartbeat) = workload_heartbeat {
1512                heartbeat.publish();
1513            }
1514            let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1515            if remaining.is_zero() {
1516                return crashed();
1517            }
1518            if crashed() {
1519                return true;
1520            }
1521            thread::sleep(remaining.min(ERR_EXIT_POLL));
1522        }
1523    };
1524
1525    // epoll setup. EPOLL_CLOEXEC matches `wait_with_deadline` to
1526    // avoid leaking the epoll fd into any post-fork descendant.
1527    let epoll = match Epoll::new(EpollCreateFlags::EPOLL_CLOEXEC) {
1528        Ok(e) => e,
1529        Err(e) => {
1530            let fd = std::os::fd::AsRawFd::as_raw_fd(&pidfd);
1531            panic_evented_hold_defect(
1532                "epoll_create1(EPOLL_CLOEXEC)",
1533                pid,
1534                format_args!("{e} (pidfd={fd})"),
1535                "epoll has been universally available since 2.6; failure = \
1536                 env defect — check ulimit -n and CONFIG_EPOLL",
1537            );
1538        }
1539    };
1540    // `data` field is unused — we only ever watch one fd. The add()
1541    // syscall still needs an `EpollEvent` with populated events.
1542    let event = EpollEvent::new(EpollFlags::EPOLLIN, 0);
1543    if let Err(e) = epoll.add(pidfd.as_fd(), event) {
1544        panic_evented_hold_defect(
1545            "epoll_ctl(ADD)",
1546            pid,
1547            e,
1548            "epoll_ctl(ADD) on a freshly-opened pidfd should never fail \
1549             in a healthy kernel; documented errors (EBADF/EEXIST/ENOMEM) \
1550             are env defects — likely fd exhaustion or memory pressure",
1551        );
1552    }
1553
1554    let mut events = [EpollEvent::empty()];
1555    loop {
1556        if let Some(heartbeat) = workload_heartbeat {
1557            heartbeat.publish();
1558        }
1559        let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1560        if remaining.is_zero() {
1561            // Hold elapsed without a wakeup. Re-probe ALL signals to
1562            // catch a race where the pid exited, the err-exit latch
1563            // flipped, or sched_ext went down (sysfs) between the last
1564            // `epoll_wait` return and the deadline check (e.g. during
1565            // EINTR re-entry). `pid` is `Some` here, so the
1566            // scheduler-expected guard for `scx_down` is satisfied.
1567            return crashed()
1568                || dispatch::scx_down()
1569                || !matches!(crate::vmm::rust_init::pidfd_is_alive(&pidfd), Ok(true));
1570        }
1571
1572        // Abort at the crash, not at the lingering process exit. Two
1573        // probe-independent signals fire well before a crashed
1574        // scheduler's process exits: the BPF err-exit latch (live only
1575        // in the auto-repro/dump VM) and `/sys/kernel/sched_ext/state`
1576        // going `disabling`/`disabled` (live in the PRIMARY VM, where the
1577        // latch mirror is never populated — this is what makes a primary
1578        // VM crash abort the hold at crash time rather than ~1s later at
1579        // process exit). `pid` is `Some` here, so the
1580        // scheduler-expected guard for `scx_down` holds.
1581        if crashed() || dispatch::scx_down() {
1582            return true;
1583        }
1584
1585        // Cap the epoll timeout at the latch poll interval so the latch
1586        // is re-checked every `ERR_EXIT_POLL`, not only when the pidfd
1587        // becomes readable. `PollTimeout` (aliased as `EpollTimeout`)
1588        // stores the value as `i32`; single-pass clamp via
1589        // `u128 → i32::MAX` so a large remainder saturates at the max
1590        // accepted value instead of overflowing through the u32.
1591        let poll = remaining.min(ERR_EXIT_POLL);
1592        let ms_i32 = poll.as_millis().min(i32::MAX as u128) as i32;
1593        let timeout_param = match EpollTimeout::try_from(ms_i32) {
1594            Ok(t) => t,
1595            Err(e) => {
1596                // ms_i32 was clamped to the i32 range above so
1597                // EpollTimeout::try_from (which accepts i32) cannot
1598                // overflow. Reaching this arm means the EpollTimeout API
1599                // changed shape — code defect, not env transient.
1600                panic_evented_hold_defect(
1601                    "EpollTimeout::try_from",
1602                    pid,
1603                    format_args!("{e} (input={ms_i32})"),
1604                    "input was pre-clamped to fit i32; failure indicates an \
1605                     upstream nix EpollTimeout API change requiring code update",
1606                );
1607            }
1608        };
1609
1610        match epoll.wait(&mut events, timeout_param) {
1611            Ok(0) => {
1612                // Poll-interval timeout, no pidfd event. Loop back to
1613                // re-check the err-exit latch and the deadline at the
1614                // top of the loop.
1615            }
1616            Ok(_) => {
1617                // pidfd became readable — task transitioned to
1618                // EXIT_ZOMBIE. Scheduler is dead.
1619                return true;
1620            }
1621            Err(nix::errno::Errno::EINTR) => {
1622                // Signal interrupted the wait; loop and re-compute
1623                // the remaining window.
1624            }
1625            Err(e) => {
1626                panic_evented_hold_defect(
1627                    "epoll_wait",
1628                    pid,
1629                    e,
1630                    "epoll_wait on a freshly-created epoll with a single \
1631                     valid pidfd cannot legitimately fail outside EINTR; \
1632                     documented errors (EBADF/EFAULT/EINVAL) are framework- \
1633                     internal memory-safety defects — investigate concurrent \
1634                     fd mutation or stack-frame corruption",
1635                );
1636            }
1637        }
1638    }
1639}
1640
1641/// Run a single step's setup + ops + hold against step-local state.
1642///
1643/// On error, the caller is expected to invoke `collect_step` for
1644/// per-step teardown (which runs regardless) and then propagate.
1645///
1646/// `sched_died_during_hold` is set to `true` when the hold-period
1647/// liveness poll observes the scheduler process exiting; the caller
1648/// uses this to emit [`crate::assert::format_sched_died_during_workload`]
1649/// instead of waiting for the post-loop probe to fire (which would
1650/// stamp the message with the full scenario duration even though
1651/// the death happened mid-step).
1652#[allow(clippy::too_many_arguments)]
1653fn run_step<'a>(
1654    ctx: &Ctx,
1655    step: &Step,
1656    step_idx: usize,
1657    step_state: &mut StepState<'a>,
1658    backdrop_state: &mut BackdropState<'a>,
1659    scenario_start: std::time::Instant,
1660    _effective_checks: &crate::assert::Assert,
1661    sched_died_during_hold: &mut bool,
1662    // Set to (scenario-relative elapsed_ms, cumulative worker iteration
1663    // count) sampled coincidently at this step's clean completion
1664    // Left unchanged on the sched-died early-return paths so
1665    // a failed step does not overwrite a prior step's good value with a
1666    // misleading post-death count.
1667    final_total_iterations: &mut Option<(u64, u64)>,
1668    workload_heartbeat: Option<WorkloadHeartbeat>,
1669) -> Result<()> {
1670    let mut scenario = ScenarioState::new(step_state, backdrop_state);
1671
1672    // Any `?` out of apply_ops / apply_setup would bypass the
1673    // per-step teardown ordering; `drain_on_err!` kills payload
1674    // handles across step + backdrop (metric-emitting) before
1675    // propagating so a mid-scenario spawn failure still leaves a
1676    // usable sidecar.
1677    macro_rules! drain_on_err {
1678        ($scenario:expr, $e:expr) => {
1679            match $e {
1680                Ok(v) => v,
1681                Err(err) => {
1682                    $scenario.drain_all_payloads();
1683                    return Err(err);
1684                }
1685            }
1686        };
1687    }
1688
1689    match step.hold {
1690        HoldSpec::Loop { interval } => {
1691            // Setup runs once before the loop.
1692            if !step.setup.is_empty() {
1693                let defs = step.setup.resolve(ctx);
1694                drain_on_err!(scenario, apply_setup(ctx, &mut scenario, &defs));
1695            }
1696            // Emit the Loop step's start stimulus frame + resume, like
1697            // the non-Loop arm below. A Loop step is a phase like any
1698            // other: without its own start frame the synthesized
1699            // scenario-end terminal would pair against the
1700            // PRIOR step's frame and misattribute the entire Loop
1701            // window's throughput to the wrong step. The resume
1702            // re-enables periodic captures for the loop body and matches
1703            // the prior step's `run_steps` pause (same pause/resume
1704            // pairing the non-Loop arm uses; the first-step unmatched
1705            // resume is already the existing pattern).
1706            if guest_comms::is_guest() {
1707                let payload = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1708                crate::vmm::guest_comms::send_stimulus(zerocopy::IntoBytes::as_bytes(&payload));
1709                crate::vmm::guest_comms::send_scenario_resume();
1710            }
1711            // Loop mode: apply ops repeatedly at interval until
1712            // the remaining scenario time is exhausted, or the
1713            // scheduler process exits — whichever fires first.
1714            let deadline = scenario_start + ctx.duration;
1715            while std::time::Instant::now() < deadline {
1716                drain_on_err!(scenario, apply_ops(ctx, &mut scenario, &step.ops, true));
1717                let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1718                // Live `sched_pid()` read so a mid-loop
1719                // Op::ReplaceScheduler swap is watched at the NEW
1720                // pid, not the stale boot snapshot in ctx.
1721                if hold_or_sched_died(
1722                    remaining.min(interval),
1723                    crate::vmm::rust_init::sched_pid(),
1724                    workload_heartbeat,
1725                ) {
1726                    *sched_died_during_hold = true;
1727                    return Ok(());
1728                }
1729            }
1730        }
1731        _ => {
1732            // Ops first (e.g. parent cgroup creation), then
1733            // CgroupDef setup (children with workers).
1734            //
1735            // Footgun: a workload-producing `CgroupDef` in
1736            // `step.setup` is invisible to `step.ops` operating on
1737            // it, because step.ops runs BEFORE apply_setup creates
1738            // the cgroup AND registers the WorkloadHandle in
1739            // `state.all_handles()`. The failure mode is per-Op:
1740            // `Op::MoveAllTasks` against the absent source filters
1741            // `state.all_handles()` by name, finds zero matches,
1742            // iterates an empty `pid_batches`, and exits with no
1743            // work done — silent (no error surfaced to the
1744            // operator); `Op::CaptureCgroupProcs` reading the
1745            // missing `cgroup.procs` via `read_procs` returns the
1746            // ENOENT `with_context` and propagates `?`, bailing the
1747            // ops phase — loud (the test crashes with the cgroup
1748            // name in the error). Tests that need a freshly-spawned
1749            // worker pool to feed step ops must declare the
1750            // producing `CgroupDef` in the Backdrop (which runs
1751            // before any Step) when `step.setup` is `Setup::Defs(_)`.
1752            // When `step.setup` is `Setup::Factory(_)` (runtime-
1753            // generated defs that can't be hoisted at edit time),
1754            // the only viable remediation is restructuring into
1755            // multiple Steps — producer factory in Step N's setup,
1756            // consumer ops in Step N+1.
1757            drain_on_err!(scenario, apply_ops(ctx, &mut scenario, &step.ops, false));
1758            if !step.setup.is_empty() {
1759                let defs = step.setup.resolve(ctx);
1760                drain_on_err!(scenario, apply_setup(ctx, &mut scenario, &defs));
1761            }
1762
1763            // Write stimulus event after applying ops. Routes through
1764            // `crate::vmm::guest_comms::send_stimulus` (virtio-console
1765            // port-1 bulk channel). `is_guest` keeps the
1766            // `build_stimulus` walk off the host where the write would
1767            // no-op.
1768            if guest_comms::is_guest() {
1769                let payload = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1770                crate::vmm::guest_comms::send_stimulus(zerocopy::IntoBytes::as_bytes(&payload));
1771            }
1772
1773            if guest_comms::is_guest() {
1774                crate::vmm::guest_comms::send_scenario_resume();
1775            }
1776            let hold_dur = match step.hold {
1777                HoldSpec::Frac(f) => Duration::from_secs_f64(ctx.duration.as_secs_f64() * f),
1778                HoldSpec::Fixed(d) => d,
1779                HoldSpec::Loop { .. } => unreachable!(),
1780            };
1781            let remaining = (scenario_start + ctx.duration)
1782                .saturating_duration_since(std::time::Instant::now());
1783            let hold_dur = hold_dur.min(remaining);
1784            // Live `sched_pid()` read — matches the loop arm above
1785            // so the hold watches the post-Op::ReplaceScheduler
1786            // pid, not the stale boot snapshot.
1787            if hold_or_sched_died(
1788                hold_dur,
1789                crate::vmm::rust_init::sched_pid(),
1790                workload_heartbeat,
1791            ) {
1792                *sched_died_during_hold = true;
1793                return Ok(());
1794            }
1795        }
1796    }
1797
1798    // Capture the cumulative iteration count AND its scenario-relative
1799    // timestamp at this step's end, AT THE SAME INSTANT, while the
1800    // step's workers are still alive — `collect_step` tears them down
1801    // right after `run_step` returns, so this is the last moment a
1802    // step-inclusive sum is observable. `run_steps` forwards the LAST
1803    // step's (elapsed_ms, iterations) pair in the widened `ScenarioEnd`
1804    // frame. Sample both HERE so the terminal telemetry describes one
1805    // coincident endpoint rather than mixing a pre-teardown count with a
1806    // post-teardown clock. Only reached on a clean hold; gated on `is_guest`
1807    // so the host-side scenario walk does not sum a handle set that never ran.
1808    if guest_comms::is_guest() {
1809        let end_elapsed_ms = scenario_start.elapsed().as_millis() as u64;
1810        let end_iterations: u64 = scenario
1811            .all_handles()
1812            .flat_map(|(_, h)| h.snapshot_iterations())
1813            .sum();
1814        *final_total_iterations = Some((end_elapsed_ms, end_iterations));
1815
1816        // Emit a per-step StepEnd frame carrying this step's coincident
1817        // end-of-hold (elapsed_ms, total_iterations) and the SAME
1818        // 1-indexed step_index as its StepStart. This preserves raw step-local
1819        // boundary telemetry; the canonical iteration rate comes from the
1820        // CPU-time carrier. build_stimulus supplies the op/cgroup/worker fields
1821        // + the step index; override its recomputed elapsed/iterations with the
1822        // values captured above so the StepEnd pair stays coincident with
1823        // `final_total_iterations` (no pause/teardown wall-time creep).
1824        let mut step_end = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1825        step_end.elapsed_ms = u32::try_from(end_elapsed_ms).unwrap_or(u32::MAX);
1826        step_end.total_iterations = end_iterations;
1827        guest_comms::send_step_end(zerocopy::IntoBytes::as_bytes(&step_end));
1828    }
1829
1830    Ok(())
1831}
1832
1833/// Build a StimulusPayload from the current scenario state (step + backdrop).
1834///
1835/// # step_idx u16 saturation
1836///
1837/// `step_idx` is a `usize` on the caller side but the wire
1838/// `StimulusPayload.step_index` is a `u16` — the slot is sized for
1839/// realistic scenarios (≤ 65 536 distinct indices, `0..=u16::MAX`).
1840/// Any `step_idx` > `u16::MAX as usize` is clamped to `u16::MAX` by
1841/// `to_u16` below, with a `tracing::warn!` that names the overflow.
1842/// Downstream consumers of the StepStart wire frame therefore see
1843/// every step past index `u16::MAX` collapsed onto the same
1844/// `step_index` value (`u16::MAX`) — the ordering is preserved for
1845/// the first 65 536 steps (indices `0..=u16::MAX`), but labels
1846/// saturate and become ambiguous once the scenario crosses the
1847/// boundary. Scenarios that need to distinguish individual steps
1848/// past `u16::MAX` must widen the wire schema field; the
1849/// saturating-clip preserves visible wake ordering at the cost of
1850/// individuality in the deep tail.
1851fn build_stimulus(
1852    scenario_start: &std::time::Instant,
1853    step_idx: usize,
1854    ops: &[Op],
1855    state: &ScenarioState<'_, '_>,
1856) -> StimulusPayload {
1857    let mut op_kinds: u32 = 0;
1858    for op in ops {
1859        op_kinds |= 1 << op.discriminant();
1860    }
1861
1862    let total_iterations: u64 = state
1863        .all_handles()
1864        .flat_map(|(_, h)| h.snapshot_iterations())
1865        .sum();
1866
1867    let cgroup_count = state.step.cgroups.names().len() + state.backdrop.cgroups.names().len();
1868    let worker_count = state.step.handles.len() + state.backdrop.handles.len();
1869
1870    // Saturate narrowing conversions for the wire schema: the
1871    // StimulusPayload fields are sized for realistic scenarios
1872    // (u32 ms, u16 counts) but `as u32` / `as u16` silently
1873    // wrap on overflow, poisoning downstream consumers. Log the
1874    // overflow so the operator sees which field exceeded its
1875    // bound and substitute MAX — clipped-high is a safer wire
1876    // value than silently wrapping to a small number.
1877    let to_u32 = |field: &str, v: u128| -> u32 {
1878        u32::try_from(v).unwrap_or_else(|_| {
1879            tracing::warn!(
1880                field,
1881                value = %v,
1882                "StimulusPayload field overflowed u32; saturating to u32::MAX",
1883            );
1884            u32::MAX
1885        })
1886    };
1887    let to_u16 = |field: &str, v: usize| -> u16 {
1888        u16::try_from(v).unwrap_or_else(|_| {
1889            tracing::warn!(
1890                field,
1891                value = v,
1892                "StimulusPayload field overflowed u16; saturating to u16::MAX",
1893            );
1894            u16::MAX
1895        })
1896    };
1897
1898    // Encode the 1-indexed phase number per the framework's
1899    // phase convention -- the BASELINE (pre-first-Step) window owns
1900    // 0, scenario Step k publishes k + 1. Saturate at u16::MAX
1901    // (rather than wrap) so a pathological 65k-step scenario still
1902    // produces a clipped-high value the host parser can recognise
1903    // instead of silently rolling over.
1904    let phase_step_index: u16 = u16::try_from(step_idx)
1905        .ok()
1906        .and_then(|i| i.checked_add(1))
1907        .unwrap_or_else(|| {
1908            tracing::warn!(
1909                field = "step_index",
1910                value = step_idx,
1911                "StimulusPayload step_index overflowed u16 after 1-indexed encoding; saturating to u16::MAX",
1912            );
1913            u16::MAX
1914        });
1915    StimulusPayload {
1916        elapsed_ms: to_u32("elapsed_ms", scenario_start.elapsed().as_millis()),
1917        step_index: phase_step_index,
1918        op_count: to_u16("op_count", ops.len()),
1919        op_kinds,
1920        cgroup_count: to_u16("cgroup_count", cgroup_count),
1921        worker_count: to_u16("worker_count", worker_count),
1922        total_iterations,
1923    }
1924}
1925
1926/// Validate that a MemPolicy's node set is consistent with the
1927/// cgroup's scenario intent — the cpuset the cgroup runs in and
1928/// the host topology.
1929///
1930/// # Empty-nodemask early return
1931///
1932/// Policies with no nodemask — [`MemPolicy::Default`] and
1933/// [`MemPolicy::Local`] — carry no node IDs to validate against,
1934/// so this function returns `Ok(())` unconditionally for them
1935/// (after the unknown-bit and mutual-exclusion flag guards run).
1936/// Every other variant — any variant carrying a nodemask,
1937/// currently [`MemPolicy::Bind`], [`MemPolicy::Preferred`],
1938/// [`MemPolicy::PreferredMany`], [`MemPolicy::Interleave`], and
1939/// [`MemPolicy::WeightedInterleave`] — reaches the cpuset /
1940/// host-topology coverage logic below.
1941///
1942/// # Why this is a scenario-intent check, not a kernel guard
1943///
1944/// ktstr writes `cpuset.cpus` on each cgroup but never writes
1945/// `cpuset.mems`, so `cpuset.mems` keeps its inherited default —
1946/// the permissive "all nodes" set in every ktstr deployment
1947/// shape (PID 1 inside the guest VM, cgroup root on the host).
1948/// The kernel's `set_mempolicy(2)` path always runs the policy's
1949/// nodemask through `mpol_set_nodemask` in `mm/mempolicy.c`, which
1950/// intersects it with the caller's `mems_allowed` before it is
1951/// stored on the task; because ktstr never narrows `mems_allowed`,
1952/// that intersection is an identity operation under ktstr's
1953/// deployment — the stored nodemask equals the one the caller
1954/// supplied, and the kernel never rejects or silently trims the
1955/// policy the way it would if `mems_allowed` were disjoint from
1956/// the requested set. Rejection of a mismatched policy is
1957/// therefore validator-only: if this function does not bail, the
1958/// policy lands on the syscall unchanged and `run_steps` commits
1959/// to running the worker with a misconfigured allocation target.
1960///
1961/// What the validator catches is a **scenario-design mismatch**:
1962/// you pinned CPUs on NUMA node X (via `CpusetSpec::Numa(X)`) but
1963/// asked the mempolicy to bind/prefer/interleave a disjoint node Y,
1964/// meaning the worker's compute is local to node X while its
1965/// allocations live on node Y — producing cross-socket traffic
1966/// that the test author almost certainly did not intend. Surface
1967/// the mismatch here before `run_steps` commits to the policy.
1968///
1969/// `MpolFlags::STATIC_NODES` is the rebind-behavior flag. Two
1970/// kernel sites encode the semantics: `mpol_set_nodemask` in
1971/// `mm/mempolicy.c` consumes the flag during policy creation (it
1972/// determines whether the supplied nodemask is stored absolute or
1973/// remapped against the caller's cpuset at install time), and
1974/// `mpol_rebind_policy` (same file) branches on the flag when the
1975/// cpuset's `mems_allowed` changes after the policy was installed
1976/// — with `STATIC_NODES` set, the stored nodemask is unchanged;
1977/// without it, the kernel remaps the nodemask against the new
1978/// `mems_allowed`. Since ktstr never rebinds `cpuset.mems` mid-run,
1979/// only the install-time semantics applies, and the flag is
1980/// effectively a cross-node-intent declaration for the validator's
1981/// purposes — a sign the author knows the intent is "allocations on
1982/// a node outside the CPU-affinity cpuset" and has opted in to
1983/// that shape.
1984///
1985/// # Flag-specific handling (in order of evaluation)
1986///
1987/// - `STATIC_NODES | RELATIVE_NODES` both set → bail: the kernel
1988///   rejects this combination with `EINVAL`; surfacing it here
1989///   names the offender before the syscall.
1990/// - `STATIC_NODES` only → the caller has declared intentional
1991///   cross-node placement. Skip the cpuset-intent check, but each
1992///   referenced node must exist on the host topology or the
1993///   kernel will reject the policy. Verify existence; bail with
1994///   the missing nodes if any.
1995/// - `RELATIVE_NODES` only → the nodemask is an ordinal into the
1996///   cpuset's allowed-nodes set. Cpuset coverage does not apply in
1997///   absolute-id terms, so bypass.
1998/// - No relevant flag set → enforce cpuset-intent coverage:
1999///   every policy node must appear in the cpuset's covered NUMA
2000///   nodes. Bail naming the uncovered nodes AND both escape
2001///   hatches (STATIC_NODES opt-in; widening the cpuset).
2002///
2003/// Reject `--flag` args whose bare name is not in the payload's
2004/// `known_flags` allowlist. Returns `Ok(())` when the payload
2005/// declared no allowlist (`known_flags: None`) — the opt-in
2006/// contract defaults to "permissive" so payloads wrapping
2007/// open-ended binaries (stress-ng, fio, schbench) aren't forced
2008/// to enumerate every flag their upstream tool accepts.
2009///
2010/// Recognises two flag shapes: `--foo` (flag-only) and
2011/// `--foo=value` (flag-with-attached-value). Non-flag args
2012/// (positional, `-short`, everything else) are passed through
2013/// without inspection — the allowlist scopes to long flags only.
2014///
2015/// Extracted out of `apply_ops`'s `Op::RunPayload` arm so the
2016/// validation is unit-testable without standing up a full Ctx
2017/// / scenario state. See the caller for how the allowlist is
2018/// threaded through Op::RunPayload execution.
2019fn validate_known_flags(payload: &crate::test_support::Payload, args: &[String]) -> Result<()> {
2020    let Some(allowlist) = payload.known_flags else {
2021        return Ok(());
2022    };
2023    for arg in args {
2024        let Some(flag_body) = arg.strip_prefix("--") else {
2025            continue;
2026        };
2027        // `split('=').next()` is infallible: `str::split` always
2028        // yields at least one element (the full string when no
2029        // separator is present). The prior `unwrap_or("")` fallback
2030        // was dead code — the empty-name branch below never fired
2031        // via this path since `flag_body` had already passed the
2032        // `strip_prefix("--")` filter above (leaving at least one
2033        // character). Kept the `name.is_empty()` guard in place
2034        // only to handle the degenerate `"--"` bare-dashes case,
2035        // which produces `flag_body = ""` → `name = ""`.
2036        let name = flag_body
2037            .split('=')
2038            .next()
2039            .expect("str::split always yields at least one element");
2040        if name.is_empty() {
2041            continue;
2042        }
2043        if !allowlist.contains(&name) {
2044            anyhow::bail!(
2045                "Op::RunPayload: payload '{}' received unknown flag \
2046                 '--{name}' — not in its known_flags allowlist \
2047                 {allowlist:?}. Check the spelling against the \
2048                 payload's declared flags; if '--{name}' is a new \
2049                 legitimate flag, add it to `Payload::known_flags`.",
2050                payload.name,
2051            );
2052        }
2053    }
2054    Ok(())
2055}
2056
2057fn validate_mempolicy_cpuset(
2058    policy: &MemPolicy,
2059    flags: crate::workload::MpolFlags,
2060    cpuset: &BTreeSet<usize>,
2061    ctx: &Ctx,
2062    cgroup_name: &str,
2063) -> Result<()> {
2064    use crate::workload::MpolFlags;
2065
2066    // Reject unknown bits before any other check. The `MpolFlags`
2067    // type is a `u32` bitfield covering three documented bits
2068    // (STATIC_NODES, RELATIVE_NODES, NUMA_BALANCING); any other bit
2069    // set in `flags` is either a user typo (raw-constructing the
2070    // struct with an arbitrary integer) or forward-compat from a
2071    // future kernel flag that this validator hasn't learned yet.
2072    // Either way, surfacing unknown bits here prevents a silent
2073    // semantic mismatch — the kernel would either reject with
2074    // EINVAL or (worse) treat the bit as a flag we don't model.
2075    let known_bits = MpolFlags::STATIC_NODES.bits()
2076        | MpolFlags::RELATIVE_NODES.bits()
2077        | MpolFlags::NUMA_BALANCING.bits();
2078    let unknown_bits = flags.bits() & !known_bits;
2079    if unknown_bits != 0 {
2080        anyhow::bail!(
2081            "cgroup '{}': MpolFlags contains unknown bit(s) {:#x} (known bits: \
2082             STATIC_NODES={:#x}, RELATIVE_NODES={:#x}, NUMA_BALANCING={:#x}); \
2083             refusing to forward to the kernel — update MpolFlags to model the \
2084             new bit before using it, or clear the bit at the call site",
2085            cgroup_name,
2086            unknown_bits,
2087            MpolFlags::STATIC_NODES.bits(),
2088            MpolFlags::RELATIVE_NODES.bits(),
2089            MpolFlags::NUMA_BALANCING.bits(),
2090        );
2091    }
2092
2093    // `STATIC_NODES | RELATIVE_NODES` is a kernel-rejected combination —
2094    // `MPOL_F_STATIC_NODES` and `MPOL_F_RELATIVE_NODES` are mutually
2095    // exclusive (see `include/uapi/linux/mempolicy.h` + the
2096    // `sanitize_mpol_flags` helper in `mm/mempolicy.c`, which bails
2097    // with `EINVAL` if both are set). Fail early here instead of
2098    // letting the syscall return a generic error — the scenario
2099    // caller almost certainly meant one or the other, not both.
2100    if flags.contains(MpolFlags::STATIC_NODES) && flags.contains(MpolFlags::RELATIVE_NODES) {
2101        anyhow::bail!(
2102            "cgroup '{}': MpolFlags::STATIC_NODES and MpolFlags::RELATIVE_NODES are \
2103             mutually exclusive (the kernel will reject the set_mempolicy syscall with \
2104             EINVAL); pick whichever matches the intended semantics — STATIC_NODES \
2105             for absolute node ids that survive cpuset changes, RELATIVE_NODES for \
2106             cpuset-relative indices",
2107            cgroup_name,
2108        );
2109    }
2110
2111    let policy_nodes = policy.node_set();
2112    if policy_nodes.is_empty() {
2113        return Ok(());
2114    }
2115
2116    // `STATIC_NODES`: nodemask is treated as absolute node ids and NOT
2117    // intersected with the cpuset. The cpuset-coverage check below
2118    // does not apply, but we DO need to verify the referenced nodes
2119    // actually exist on the host — a policy pinning node 7 on a
2120    // 2-node host would fail at syscall time; surfacing it here
2121    // names the offender.
2122    if flags.contains(MpolFlags::STATIC_NODES) {
2123        let host_nodes = ctx.topo.numa_node_ids();
2124        let missing: Vec<usize> = policy_nodes
2125            .iter()
2126            .copied()
2127            .filter(|n| !host_nodes.contains(n))
2128            .collect();
2129        if !missing.is_empty() {
2130            anyhow::bail!(
2131                "cgroup '{}': MemPolicy with MpolFlags::STATIC_NODES references \
2132                 NUMA node(s) {:?} that do not exist on this host (host nodes: {:?}); \
2133                 the kernel will reject or silently drop the policy (Preferred can \
2134                 silently fall back to local allocation; Bind/Interleave reject with \
2135                 EINVAL) — fix the MemPolicy or pick a host with the required nodes",
2136                cgroup_name,
2137                missing,
2138                host_nodes,
2139            );
2140        }
2141        return Ok(());
2142    }
2143
2144    // `RELATIVE_NODES`: nodemask is an ordinal into the cpuset's
2145    // allowed nodes, not an absolute node id set. The cpuset-coverage
2146    // check compares absolute ids, so it does not apply here — the
2147    // kernel does the relative-to-absolute remap internally. Trust
2148    // the caller and bypass the coverage bail, same shape as the
2149    // STATIC_NODES early return.
2150    if flags.contains(MpolFlags::RELATIVE_NODES) {
2151        return Ok(());
2152    }
2153
2154    let cpuset_numa = ctx.topo.numa_nodes_for_cpuset(cpuset);
2155    let uncovered: Vec<usize> = policy_nodes
2156        .iter()
2157        .copied()
2158        .filter(|n| !cpuset_numa.contains(n))
2159        .collect();
2160    if !uncovered.is_empty() {
2161        anyhow::bail!(
2162            "cgroup '{}': MemPolicy references NUMA node(s) {:?} \
2163             outside the cpuset's coverage (cpuset covers node(s) \
2164             {:?}) — some or all of the worker's allocations would \
2165             live on NUMA nodes its CPUs cannot reach locally, \
2166             producing cross-socket allocation traffic that is \
2167             almost certainly unintended. Two fixes: \
2168             (a) add .mpol_flags(MpolFlags::STATIC_NODES) to \
2169             declare the cross-node placement intentional (the \
2170             flag survives cpuset rebinds; see MpolFlags doc), or \
2171             (b) widen the cpuset to cover the policy's nodes \
2172             (e.g. CpusetSpec::Numa(N) for each referenced N, or \
2173             a CpusetSpec::Exact set that spans both).",
2174            cgroup_name,
2175            uncovered,
2176            cpuset_numa,
2177        );
2178    }
2179    Ok(())
2180}
2181
2182/// SIGKILL every worker in `handles` without reaping. Used only on
2183/// the scheduler-death paths in [`run_scenario`]: when the scheduler
2184/// under test dies, the kernel disables sched_ext and its workers
2185/// fall back to the builtin scheduler, so a CPU-bound worker pool
2186/// would otherwise make the per-worker `waitpid` reap in the
2187/// subsequent [`collect_step`] / [`collect_backdrop`]
2188/// scheduling-gated (teardown time scaling with worker count).
2189/// Delivering SIGKILL to every worker first lets that reap find
2190/// already-exiting workers. See
2191/// [`crate::workload::WorkloadHandle::sigkill_workers`].
2192fn sigkill_handles(handles: &[(String, WorkloadHandle)]) {
2193    for (_, h) in handles {
2194        h.sigkill_workers();
2195    }
2196}
2197
2198/// Collect step-local worker results and produce an AssertResult.
2199///
2200/// Drains step-local handles + payload handles; backdrop state is
2201/// untouched. Called at every step boundary (success AND error
2202/// paths) as the "Step is fully bounded" teardown. The
2203/// `step_state` goes out of scope at the end of this step's
2204/// iteration, so its `CgroupGroup` drop removes every step-local
2205/// cgroup immediately after `run_scenario` propagates the result
2206/// of this call.
2207///
2208/// Before draining handles, every step-local cgroup is unfrozen
2209/// (`cgroup.freeze` ← 0). An [`Op::FreezeCgroup`] without a paired
2210/// [`Op::UnfreezeCgroup`] would leave step-local tasks frozen at
2211/// step boundary, where the graceful cooperative stop cannot make
2212/// progress: the worker stop is a non-fatal SIGUSR1, and a frozen
2213/// task re-enters the freezer trap on a non-fatal signal
2214/// (`kernel/signal.c` `do_freezer_trap`) rather than running its
2215/// handler — so it never flips its stop flag or reports until
2216/// thawed, and `stop_and_collect` burns its full collection
2217/// deadline before falling back to a sentinel report. (A fatal
2218/// SIGKILL DOES wake and kill a frozen task — it diverges before
2219/// the freezer trap — so worker death and the subsequent rmdir are
2220/// unaffected by the freeze; the unfreeze only restores the
2221/// graceful path's ability to collect real reports.) Failures are
2222/// logged at warn level only — a missing freezer file or a cgroup
2223/// that was already torn down is benign at teardown time, and
2224/// propagating would mask the real workload result.
2225fn collect_step(
2226    step_state: &mut StepState<'_>,
2227    checks: &crate::assert::Assert,
2228    topo: &crate::topology::TestTopology,
2229    cgroups: &dyn crate::cgroup::CgroupOps,
2230    // The 1-indexed phase step_index (BASELINE=0, Step k -> k+1) this
2231    // teardown belongs to, threaded so collect_handles can attach the per-phase
2232    // per_cgroup carrier. `None` for the defensive backdrop-staging collect,
2233    // which has no step attribution.
2234    step_index: Option<u16>,
2235) -> AssertResult {
2236    // Unfreeze every step-local cgroup before the graceful collect.
2237    // A frozen worker re-enters the freezer trap on the cooperative
2238    // (non-fatal) SIGUSR1 stop instead of running its handler, so it
2239    // never reports until thawed. (SIGKILL would still kill it; only
2240    // the graceful report path needs the thaw.)
2241    for name in step_state.cgroups.names() {
2242        if let Err(e) = cgroups.set_freeze(name, false) {
2243            tracing::warn!(
2244                cgroup = %name,
2245                err = %format!("{e:#}"),
2246                "collect_step: pre-teardown unfreeze failed; any frozen workers will yield sentinel reports (still SIGKILL-reaped, no leak)"
2247            );
2248        }
2249    }
2250    // Kill any CgroupDef::workload / Op::RunPayload payload binaries
2251    // still live at step teardown so cgroupfs cleanup does not trip
2252    // EBUSY. Metrics are emitted to the SHM ring by PayloadHandle::kill
2253    // via the `evaluate()` pipeline.
2254    drain_all_payload_handles(&mut step_state.payload_handles);
2255    // Drain any host-mode stuck reports that accumulated during the
2256    // step BEFORE dropping the monitor handle (Drop joins the
2257    // polling thread). Reports get folded into the merged
2258    // [`AssertResult`] below as
2259    // [`crate::assert::DetailKind::WorkerStuck`] failures. `take`
2260    // ensures the handle drops here (joining the thread) so the
2261    // polling thread exits before per-step teardown returns.
2262    let stuck_reports = if let Some(handle) = step_state.stuck_monitor.take() {
2263        let reports = handle.drain();
2264        drop(handle);
2265        reports
2266    } else {
2267        Vec::new()
2268    };
2269    let handles = std::mem::take(&mut step_state.handles);
2270    let mut result = crate::scenario::collect_handles(
2271        handles.into_iter().map(|(name, h)| {
2272            let cpuset = step_state.cpusets.get(&name);
2273            (name, h, cpuset)
2274        }),
2275        checks,
2276        Some(topo),
2277        step_index,
2278    );
2279    for report in stuck_reports {
2280        result.record_fail(crate::assert::AssertDetail::new(
2281            crate::assert::DetailKind::WorkerStuck,
2282            format_stuck_report(&report),
2283        ));
2284    }
2285    result
2286}
2287
2288/// Render a [`crate::scenario::host_stuck::StuckReport`] as a
2289/// human-readable single-string assertion detail. Emits the stuck
2290/// pid + comm, the sample-window summary (first and last values
2291/// for both counters PLUS the computed `last - first` delta — both
2292/// expected to be zero for a true stuck condition but rendered from the
2293/// samples themselves so a predicate-tolerance refactor stays
2294/// observable), and the diagnostic subset (state, wchan, syscall,
2295/// cgroup, host loadavg, optional kernel stack). The diagnostic's
2296/// `status_full` field is intentionally OMITTED — the parsed
2297/// `state` letter carries the actionable signal and the full
2298/// status file is verbose; sidecar consumers keying off
2299/// [`crate::assert::DetailKind::WorkerStuck`] can match on the
2300/// kind discriminator and read the full StuckReport (carries the
2301/// status_full field) without parsing this message.
2302///
2303/// Format is multi-line so the operator can read the trip at a
2304/// glance without parsing structured output.
2305fn format_stuck_report(report: &crate::scenario::host_stuck::StuckReport) -> String {
2306    use std::fmt::Write as _;
2307    let mut s = String::new();
2308    let _ = writeln!(
2309        s,
2310        "worker stuck detected: pid={} comm={:?} (host-mode /proc/<pid>/sched polling)",
2311        report.pid, report.comm,
2312    );
2313    if let (Some(first), Some(last)) = (report.samples.first(), report.samples.last()) {
2314        // Compute deltas from the actual sample values so a
2315        // predicate-tolerance refactor (e.g. "fire on Δ <= N
2316        // switches rather than == 0") stays observable in the
2317        // rendered output. Saturating subtraction handles the
2318        // pathological counter-rollover case without panic.
2319        let nr_delta = last.nr_switches.saturating_sub(first.nr_switches);
2320        let rt_delta = last
2321            .sum_exec_runtime_ns
2322            .saturating_sub(first.sum_exec_runtime_ns);
2323        let _ = writeln!(
2324            s,
2325            "  sample window: nr_switches {} -> {} (delta {nr_delta}), sum_exec_runtime_ns {} -> {} (delta {rt_delta}), {} samples",
2326            first.nr_switches,
2327            last.nr_switches,
2328            first.sum_exec_runtime_ns,
2329            last.sum_exec_runtime_ns,
2330            report.samples.len(),
2331        );
2332    }
2333    let d = &report.diagnostic;
2334    let _ = writeln!(s, "  state: {}", d.state);
2335    let _ = writeln!(s, "  wchan: {}", d.wchan);
2336    let _ = writeln!(s, "  syscall: {}", d.syscall);
2337    let _ = writeln!(s, "  cgroup: {}", d.cgroup);
2338    let _ = writeln!(s, "  host loadavg: {}", d.host_loadavg);
2339    if let Some(stack) = &d.stack {
2340        let _ = writeln!(s, "  kernel stack:\n{stack}");
2341    }
2342    s
2343}
2344
2345/// Collect backdrop (persistent) worker results. Called once at
2346/// scenario end after every Step has torn down. The
2347/// `backdrop_state.cgroups` RAII guard drops persistent cgroups
2348/// when `backdrop_state` itself drops.
2349///
2350/// Mirrors [`collect_step`]'s pre-teardown unfreeze pass over every
2351/// tracked cgroup, for the same reason: a backdrop cgroup left
2352/// frozen at scenario end cannot run the graceful cooperative stop.
2353/// A frozen task re-enters the freezer trap on the non-fatal SIGUSR1
2354/// (`kernel/signal.c` `do_freezer_trap`) instead of running its
2355/// handler, so its workers never report until thawed. The asymmetry
2356/// between step-local and backdrop teardown — only the former
2357/// unfreezing — would surface as backdrop workers yielding sentinel
2358/// reports (after burning the collection deadline) on every scenario
2359/// whose Backdrop froze a cgroup and never unfroze it. Symmetric
2360/// unfreeze brings the backdrop path back in line with the per-step
2361/// path. (As in `collect_step`, a fatal SIGKILL still kills a frozen
2362/// worker, so death and the `BackdropState::cgroups` RAII rmdir are
2363/// unaffected by the freeze.)
2364fn collect_backdrop(
2365    backdrop_state: &mut BackdropState<'_>,
2366    checks: &crate::assert::Assert,
2367    topo: &crate::topology::TestTopology,
2368    cgroups: &dyn crate::cgroup::CgroupOps,
2369) -> AssertResult {
2370    // Unfreeze every backdrop cgroup before the graceful collect.
2371    // Same rationale as `collect_step`: a frozen worker re-enters
2372    // the freezer trap on the cooperative (non-fatal) SIGUSR1 stop
2373    // instead of running its handler, so it never reports until
2374    // thawed. (SIGKILL would still kill it; only the graceful report
2375    // path needs the thaw.)
2376    for name in backdrop_state.cgroups.names() {
2377        if let Err(e) = cgroups.set_freeze(name, false) {
2378            tracing::warn!(
2379                cgroup = %name,
2380                err = %format!("{e:#}"),
2381                "collect_backdrop: pre-teardown unfreeze failed; any frozen workers will yield sentinel reports (still SIGKILL-reaped, no leak)"
2382            );
2383        }
2384    }
2385    drain_all_payload_handles(&mut backdrop_state.payload_handles);
2386    let handles = std::mem::take(&mut backdrop_state.handles);
2387    crate::scenario::collect_handles(
2388        handles.into_iter().map(|(name, h)| {
2389            let cpuset = backdrop_state.cpusets.get(&name);
2390            (name, h, cpuset)
2391        }),
2392        checks,
2393        Some(topo),
2394        // Backdrop spans every phase: the None arm expands each worker's
2395        // per-phase PhaseSlices into one per-epoch PhaseBucket
2396        // (expand_backdrop_phase_buckets) — per-epoch, not single-step,
2397        // attribution.
2398        None,
2399    )
2400}
2401
2402/// Kill every payload handle whose cgroup matches `cgroup` and drop
2403/// the matched entries from `handles`. Runs before the cgroup is
2404/// removed or stopped; failures are logged to stderr but do not
2405/// propagate — the cgroup removal is best-effort already, and the
2406/// payload-kill failure is never the primary error.
2407///
2408/// **Metric emission depends on the explicit `.kill()` call** —
2409/// if a future refactor replaces the `.kill()` below with plain
2410/// `drop(handle)`, the `PayloadHandle::drop` SIGKILLs the child
2411/// but skips the evaluate-and-emit pipeline that records metrics
2412/// to the SHM ring. Test helpers that drain payload handles
2413/// likewise route through `drain_all_payload_handles` for the
2414/// same reason. Preserve `.kill()` on every path that claims to
2415/// drain handles for metric capture.
2416///
2417/// Drop order across matched entries is LIFO (last pushed, first
2418/// dropped) — the loop walks indices from the tail toward index 0
2419/// using `Vec::remove` so newer matched entries' embedded
2420/// `SigchldScope`s restore the SIGCHLD disposition before older
2421/// matches do, matching the save-and-restore chain documented on
2422/// `PayloadHandle` in `payload_run.rs`. `Vec::swap_remove` would
2423/// rotate the tail into the freed slot and break LIFO across
2424/// matches; `Vec::remove` preserves the relative order of the
2425/// remaining (unmatched) survivors. Note: SIGCHLD scope LIFO across
2426/// the FULL vec is structurally unsalvageable in any partial-drain
2427/// helper — unmatched entries that stay alive in `handles` outlive
2428/// their younger matched siblings whose scopes already restored.
2429/// The full-vec LIFO contract holds only when every handle is
2430/// dropped together via [`drain_all_payload_handles`].
2431fn drain_payload_handles_for_cgroup(handles: &mut Vec<PayloadEntry>, cgroup: &str) {
2432    let mut i = handles.len();
2433    while i > 0 {
2434        i -= 1;
2435        if handles[i].cgroup.as_str() == cgroup {
2436            let entry = handles.remove(i);
2437            if let Err(e) = entry.handle.kill() {
2438                eprintln!("ktstr: kill payload in cgroup '{cgroup}': {e:#}");
2439            }
2440        }
2441    }
2442}
2443
2444/// Kill every payload handle regardless of cgroup and clear the
2445/// vector. Called at step-sequence teardown so every handle gets a
2446/// terminal `.kill()` (and therefore a sidecar metric emission) even
2447/// when no explicit `RemoveCgroup`/`StopCgroup` op targeted it.
2448///
2449/// Drop order is LIFO (last pushed, first dropped) — `Vec::pop`
2450/// returns the tail first, so `PayloadHandle::drop` runs in reverse
2451/// creation order. Each handle's embedded `SigchldScope` captured the
2452/// `SIGCHLD` disposition that was live at construction time (the
2453/// previous scope's installed `SIG_DFL`). Restoring in LIFO unwinds
2454/// the save-and-restore chain back to the original disposition; FIFO
2455/// drop (e.g. `Vec::drain(..)`) restores intermediate `SIG_DFL` values
2456/// out of order and leaks `SIG_DFL` past the outermost scope. See the
2457/// DROP-ORDER-CRITICAL note on `PayloadHandle` in `payload_run.rs`.
2458fn drain_all_payload_handles(handles: &mut Vec<PayloadEntry>) {
2459    while let Some(entry) = handles.pop() {
2460        if let Err(e) = entry.handle.kill() {
2461            eprintln!(
2462                "ktstr: teardown kill payload in cgroup {}: {e:#}",
2463                render_cgroup_key(&entry.cgroup),
2464            );
2465        }
2466    }
2467}
2468
2469#[cfg(test)]
2470mod tests;
2471
2472#[cfg(test)]
2473mod workers_pct_construction_tests;
2474
2475#[cfg(test)]
2476mod kernel_op_dispatch_tests;