ktstr/scenario/ops/mod.rs
1//! Composable ops/steps system for dynamic cgroup topology changes.
2//!
3//! [`Op`] is an atomic cgroup operation. [`Step`] sequences ops with a
4//! hold period. [`CgroupDef`] bundles create + cpuset + spawn into a
5//! single declaration. [`execute_steps()`] runs a step sequence with
6//! scheduler liveness checks and stimulus event recording.
7//!
8//! See the [Ops and Steps](https://ktstr.dev/guide/concepts/ops.html)
9//! chapter for a guide.
10//!
11//! # Cgroup tooling at a glance
12//!
13//! ktstr exposes the cgroup v2 surface across two layers — declarative
14//! steady-state via [`CgroupDef`] (set at scenario-setup time, holds
15//! for the cgroup's lifetime) and imperative state-transitions via
16//! [`Op`] (applied mid-step, describe transitions over time):
17//!
18//! | Knob | Layer | API entry | Underlying file | When to use |
19//! |------|-------|-----------|-----------------|-------------|
20//! | CPU affinity | setup | [`CgroupDef::cpuset`] | `cpuset.cpus` | Bind workers to a CPU subset for the whole run. |
21//! | NUMA-mem affinity | setup | [`CgroupDef::cpuset_mems`] | `cpuset.mems` | Constrain allocations to specific NUMA nodes. |
22//! | CPU bandwidth | setup | [`CgroupDef::cpu_quota_pct`] / [`CgroupDef::cpu_quota`] / [`CgroupDef::cpu_unlimited`] | `cpu.max` | Cap CPU time per period (1 CPU at 50% / 2 CPU at 100% / etc). |
23//! | CPU share weight | setup | [`CgroupDef::cpu_weight`] | `cpu.weight` | Bias relative CPU share when siblings contend. |
24//! | Memory ceiling | setup | [`CgroupDef::memory_max`] / [`CgroupDef::memory_unlimited`] | `memory.max` | Hard ceiling — exceeding triggers cgroup OOM. |
25//! | Memory throttle | setup | [`CgroupDef::memory_high`] | `memory.high` | Soft throttle: triggers reclaim, not OOM. |
26//! | Memory protection | setup | [`CgroupDef::memory_low`] | `memory.low` | Soft protection: kernel reclaims from siblings first. |
27//! | Swap cap | setup | [`CgroupDef::memory_swap_max`] / [`CgroupDef::memory_swap_unlimited`] | `memory.swap.max` | Cap how much memory can spill to swap (CONFIG_SWAP=y). |
28//! | IO share | setup | [`CgroupDef::io_weight`] | `io.weight` | Bias relative IO share when siblings contend. |
29//! | Task ceiling | setup | [`CgroupDef::pids_max`] / [`CgroupDef::pids_unlimited`] | `pids.max` | Cap process+thread count — fork/clone returns EAGAIN at limit. |
30//! | Mid-run cpuset rebind | mid-step | [`Op::set_cpuset`] / [`Op::clear_cpuset`] / [`Op::swap_cpusets`] | `cpuset.cpus` | Move cpuset on a live cgroup mid-scenario. |
31//! | Mid-run task migration | mid-step | [`Op::move_all_tasks`] | `cgroup.procs` | Move workers from one cgroup to another. |
32//! | Pause/resume | mid-step | [`Op::freeze_cgroup`] / [`Op::unfreeze_cgroup`] | `cgroup.freeze` | Suspend every task in the cgroup; resume later. |
33//! | Add/remove cgroup | mid-step | [`Op::add_cgroup`] / [`Op::remove_cgroup`] / [`Op::stop_cgroup`] | (cgroupfs mkdir/rmdir) | Spawn / tear down a cgroup mid-scenario. |
34//!
35//! # Worked examples
36//!
37//! * **Static topology** (one cgroup, fixed cpuset, weight-biased
38//! compute): [`CgroupDef`] type-level docs.
39//! * **Suspend/resume** (3-Step idiom — run, freeze, run again):
40//! [`Op::FreezeCgroup`] doc.
41//! * **Memory-cap teardown** (rewind a base CgroupDef's swap cap):
42//! [`CgroupDef::memory_swap_unlimited`] doc.
43//!
44//! # Implementation entry points
45//!
46//! Every knob ends in [`crate::cgroup::CgroupOps`] (production:
47//! [`crate::cgroup::CgroupManager`]; tests: a recording `MockCgroupOps`
48//! double). `apply_setup` runs the [`CgroupDef`] passes; `apply_ops`
49//! dispatches the [`Op`] variants. Both share `ctx.cgroups` so a test
50//! that uses both layers writes through the same RAII teardown
51//! (`crate::scenario::CgroupGroup::Drop`).
52//!
53//! # File layout
54//!
55//! `types` holds the data model: [`Op`], [`CgroupDef`], [`Step`],
56//! [`HoldSpec`], [`Setup`], [`CpusetSpec`], the per-controller limits
57//! structs, and every builder constructor. Re-exported from this module
58//! so external paths remain `crate::scenario::ops::Op` etc. The executor
59//! drives that model against [`crate::cgroup::CgroupOps`] via `apply_setup`
60//! (sibling `setup` module) and `apply_ops` (sibling `dispatch` module),
61//! and exposes the [`execute_steps`] / [`execute_scenario`] family of
62//! public entry points (this file).
63
64mod types;
65pub use types::*;
66
67mod setup;
68pub use setup::PLACEMENT_LOG_PATH;
69use setup::apply_setup;
70
71mod dispatch;
72#[cfg(test)]
73use dispatch::{
74 REPLACE_NOT_TRYING_DEADLINE_S, build_kernel_op_request, dispatch_kernel_op_request,
75 merge_adjacent_cold_writes, staged_scheduler_log_path, wait_for_accessor_publish_or_bail,
76 wait_for_worker_state_not_trying_or_bail, write_entries_from_writes,
77};
78use dispatch::{apply_ops, render_cgroup_key};
79// Re-export the scx-state reader + enum. The guest-init scheduler
80// attach-readiness predicate (rust_init::scheduler::scx_attach_ready) reads
81// scx_state() to confirm SCX_ENABLED before the workload dispatches, and the
82// survives_storm probe's host unit test (`test_support::probe_tests`) drives
83// scheduler liveness on the host (no VM) via the set_test_scx_state seam.
84// `dispatch` is otherwise private to `ops`.
85#[cfg(test)]
86pub(crate) use dispatch::set_test_scx_state;
87pub(crate) use dispatch::{ScxState, scx_down, scx_state};
88
89use std::collections::BTreeSet;
90use std::thread;
91use std::time::Duration;
92
93use anyhow::{Context, Result};
94
95use crate::assert::AssertResult;
96use crate::scenario::backdrop;
97#[cfg(test)]
98use crate::scenario::process_alive;
99use crate::scenario::{CgroupGroup, Ctx};
100use crate::vmm::guest_comms;
101use crate::vmm::wire::StimulusPayload;
102use crate::workload::{MemPolicy, WorkloadHandle};
103
104// ---------------------------------------------------------------------------
105// Step executor
106// ---------------------------------------------------------------------------
107
108/// Persistent scenario-wide state owned by
109/// [`execute_scenario_with`]. Lives for the entire step sequence;
110/// cgroups, workload handles, and payload handles declared by the
111/// [`Backdrop`](backdrop::Backdrop) go here and only tear
112/// down at scenario end (success or Err). See [`StepState`] for
113/// the step-local counterpart.
114struct BackdropState<'a> {
115 /// RAII cgroup guard for persistent cgroups — removes them on drop.
116 cgroups: CgroupGroup<'a>,
117 /// Active workload handles in persistent cgroups, keyed by cgroup name.
118 handles: Vec<(String, WorkloadHandle)>,
119 /// Resolved cpusets per persistent cgroup name.
120 cpusets: std::collections::HashMap<String, BTreeSet<usize>>,
121 /// Active payload-binary handles owned by the backdrop. Drained
122 /// via `.kill()` at scenario teardown so the metric-emission
123 /// pipeline still fires.
124 payload_handles: Vec<PayloadEntry>,
125 /// BPF map fds opened via [`crate::scenario::ops::types::Op::PinBpfMap`].
126 /// Keyed by the map name the caller requested; the
127 /// [`std::os::fd::OwnedFd`] holds an extra refcount on the
128 /// kernel-side `struct bpf_map` so the map survives any
129 /// scheduler-process teardown (including
130 /// [`crate::scenario::ops::types::Op::ReplaceScheduler`]) until
131 /// scenario end. Drops close the fds and release the refcount.
132 pinned_bpf_maps: std::collections::HashMap<String, std::os::fd::OwnedFd>,
133}
134
135impl<'a> BackdropState<'a> {
136 /// Empty backdrop state (no persistent entities), scoped to `ctx.cgroups`.
137 fn empty(ctx: &'a Ctx) -> Self {
138 Self {
139 cgroups: CgroupGroup::new(ctx.cgroups),
140 handles: Vec::new(),
141 cpusets: std::collections::HashMap::new(),
142 payload_handles: Vec::new(),
143 pinned_bpf_maps: std::collections::HashMap::new(),
144 }
145 }
146}
147
148/// Step-local execution state. Fresh per step, torn down at step
149/// boundary: cgroups removed (via RAII drop), workload handles
150/// collected, payload handles killed with metric emission. Any ops
151/// in the step that reference a cgroup name look here first before
152/// falling through to [`BackdropState`].
153struct StepState<'a> {
154 /// RAII cgroup guard — removes step-local cgroups on drop.
155 cgroups: CgroupGroup<'a>,
156 /// Active workload handles keyed by step-local cgroup name.
157 handles: Vec<(String, WorkloadHandle)>,
158 /// Resolved cpusets per step-local cgroup name, for isolation checks.
159 cpusets: std::collections::HashMap<String, BTreeSet<usize>>,
160 /// Active payload-binary handles keyed by cgroup name. Each entry
161 /// came from either a [`CgroupDef::workload`] spawn in
162 /// `apply_setup` or an explicit [`Op::RunPayload`] invocation;
163 /// `source` tags which path spawned it so the duplicate-name
164 /// dedup in `Op::RunPayload` can point at the original site. All
165 /// are killed during step-teardown / cgroup removal so cgroupfs
166 /// cleanup never trips EBUSY on a live process.
167 payload_handles: Vec<PayloadEntry>,
168 /// Host-mode worker stuck monitor, started lazily at the end of
169 /// the first successful [`apply_setup`] when running outside a
170 /// VM (no `is_guest`, no `cargo_test_mode`) and at least one
171 /// worker exists. The handle's [`Drop`] joins the polling
172 /// thread when [`StepState`] drops; [`collect_step`] drains
173 /// any accumulated reports before that drop so they reach the
174 /// scenario's [`AssertResult`]. `None` in every guest-side
175 /// scenario and in `cargo_test_mode` runs — the host-side
176 /// monitor is the only stuck signal available in host-mode,
177 /// where the freeze coordinator / KVM-side stall plumbing is
178 /// not running. See [`crate::scenario::host_stuck`] for the
179 /// signal definition and detection latency contract.
180 stuck_monitor: Option<crate::scenario::host_stuck::StuckMonitorHandle>,
181}
182
183impl<'a> StepState<'a> {
184 /// Empty step state scoped to `ctx.cgroups`.
185 fn empty(ctx: &'a Ctx) -> Self {
186 Self {
187 cgroups: CgroupGroup::new(ctx.cgroups),
188 handles: Vec::new(),
189 cpusets: std::collections::HashMap::new(),
190 payload_handles: Vec::new(),
191 stuck_monitor: None,
192 }
193 }
194}
195
196/// Combined mutable view over step-local and backdrop state.
197///
198/// Every function that touches execution state (apply_setup,
199/// apply_ops, the drain helpers) receives a
200/// `ScenarioState`; lookups prefer step-local, falling through to
201/// backdrop. New state created via ops/setup inside a step writes
202/// to step-local by default — that is the primary mechanism
203/// enforcing per-step bounded lifetime. Setup for the Backdrop
204/// itself (run once before the step loop) writes straight to the
205/// backdrop side via [`ScenarioState::with_target_backdrop`].
206struct ScenarioState<'a, 'b> {
207 step: &'b mut StepState<'a>,
208 backdrop: &'b mut BackdropState<'a>,
209 /// When true, all mutations route to [`Self::backdrop`] instead
210 /// of [`Self::step`]. Set by [`Self::with_target_backdrop`] when
211 /// running the Backdrop's initial `apply_setup` / `apply_ops`
212 /// before the first step.
213 target_backdrop: bool,
214}
215
216impl<'a, 'b> ScenarioState<'a, 'b> {
217 /// Build a combined scenario view. Starts with the step-local
218 /// slot as the mutation target — call [`Self::with_target_backdrop`]
219 /// to flip into backdrop-setup mode for Backdrop's own
220 /// apply_setup / apply_ops pass.
221 fn new(step: &'b mut StepState<'a>, backdrop: &'b mut BackdropState<'a>) -> Self {
222 Self {
223 step,
224 backdrop,
225 target_backdrop: false,
226 }
227 }
228
229 /// Run `f` with writes routed to the backdrop side.
230 fn with_target_backdrop<R>(&mut self, f: impl FnOnce(&mut Self) -> R) -> R {
231 let prev = self.target_backdrop;
232 self.target_backdrop = true;
233 let r = f(self);
234 self.target_backdrop = prev;
235 r
236 }
237
238 /// `cgroups` group that receives newly-created cgroups. Step-local
239 /// by default; backdrop when [`Self::with_target_backdrop`] is active.
240 fn target_cgroups(&mut self) -> &mut CgroupGroup<'a> {
241 if self.target_backdrop {
242 &mut self.backdrop.cgroups
243 } else {
244 &mut self.step.cgroups
245 }
246 }
247
248 /// `handles` vec that receives newly-spawned workload handles.
249 fn target_handles(&mut self) -> &mut Vec<(String, WorkloadHandle)> {
250 if self.target_backdrop {
251 &mut self.backdrop.handles
252 } else {
253 &mut self.step.handles
254 }
255 }
256
257 /// `cpusets` map that receives resolved cpusets for new cgroups.
258 fn target_cpusets(&mut self) -> &mut std::collections::HashMap<String, BTreeSet<usize>> {
259 if self.target_backdrop {
260 &mut self.backdrop.cpusets
261 } else {
262 &mut self.step.cpusets
263 }
264 }
265
266 /// `payload_handles` vec that receives newly-spawned payload handles.
267 fn target_payload_handles(&mut self) -> &mut Vec<PayloadEntry> {
268 if self.target_backdrop {
269 &mut self.backdrop.payload_handles
270 } else {
271 &mut self.step.payload_handles
272 }
273 }
274
275 /// Resolved cpuset for a cgroup name, looked up step-first then backdrop.
276 fn lookup_cpuset(&self, name: &str) -> Option<&BTreeSet<usize>> {
277 self.step
278 .cpusets
279 .get(name)
280 .or_else(|| self.backdrop.cpusets.get(name))
281 }
282
283 /// Returns the live payload handle matching the composite key
284 /// (`payload_name`, `cgroup_key`) from either step-local or
285 /// backdrop state, or `None` when no entry matches. Used for
286 /// the `Op::RunPayload` duplicate guard, which now treats
287 /// "same payload in a different cgroup" as legitimate rather
288 /// than a name collision.
289 fn find_live_payload_with_cgroup(
290 &self,
291 payload_name: &str,
292 cgroup_key: &str,
293 ) -> Option<&PayloadEntry> {
294 let matches =
295 |e: &&PayloadEntry| e.handle.payload_name() == payload_name && e.cgroup == cgroup_key;
296 self.step
297 .payload_handles
298 .iter()
299 .find(matches)
300 .or_else(|| self.backdrop.payload_handles.iter().find(matches))
301 }
302
303 /// Drop a payload handle by composite key (`name`, optional
304 /// `cgroup`). Checks step-local first, then backdrop.
305 ///
306 /// - `cgroup = Some(c)`: exact match on both name and cgroup.
307 /// - `cgroup = None`: if exactly one entry matches `name` across
308 /// both slots, consume it (backward-compat for
309 /// `Op::wait_payload(name)` / `Op::kill_payload(name)` when
310 /// only one copy is live). If two or more match, returns
311 /// `Err(ambiguous_cgroups)` where `ambiguous_cgroups` is the
312 /// list of cgroup keys for the candidates so the caller can
313 /// produce an actionable error.
314 ///
315 /// Returns `Ok(None)` when no entry matches.
316 fn take_payload_by_name(
317 &mut self,
318 name: &str,
319 cgroup: Option<&str>,
320 ) -> std::result::Result<Option<PayloadEntry>, Vec<String>> {
321 if let Some(c) = cgroup {
322 // Composite-key path: exact match on both.
323 if let Some(idx) = self
324 .step
325 .payload_handles
326 .iter()
327 .position(|e| e.handle.payload_name() == name && e.cgroup == c)
328 {
329 return Ok(Some(self.step.payload_handles.swap_remove(idx)));
330 }
331 if let Some(idx) = self
332 .backdrop
333 .payload_handles
334 .iter()
335 .position(|e| e.handle.payload_name() == name && e.cgroup == c)
336 {
337 return Ok(Some(self.backdrop.payload_handles.swap_remove(idx)));
338 }
339 return Ok(None);
340 }
341 // Name-only path: disambiguate across both slots before
342 // consuming, so a mid-test wait on an ambiguous name
343 // surfaces the caller's bug rather than silently waiting
344 // on the first match.
345 let mut step_idx: Option<usize> = None;
346 let mut backdrop_idx: Option<usize> = None;
347 let mut cgroups: Vec<String> = Vec::new();
348 for (i, e) in self.step.payload_handles.iter().enumerate() {
349 if e.handle.payload_name() == name {
350 if step_idx.is_none() {
351 step_idx = Some(i);
352 }
353 cgroups.push(e.cgroup.clone());
354 }
355 }
356 for (i, e) in self.backdrop.payload_handles.iter().enumerate() {
357 if e.handle.payload_name() == name {
358 if backdrop_idx.is_none() && step_idx.is_none() {
359 backdrop_idx = Some(i);
360 }
361 cgroups.push(e.cgroup.clone());
362 }
363 }
364 if cgroups.len() > 1 {
365 return Err(cgroups);
366 }
367 if let Some(i) = step_idx {
368 return Ok(Some(self.step.payload_handles.swap_remove(i)));
369 }
370 if let Some(i) = backdrop_idx {
371 return Ok(Some(self.backdrop.payload_handles.swap_remove(i)));
372 }
373 Ok(None)
374 }
375
376 /// Drain every live payload handle in step + backdrop state by
377 /// calling `.kill()` so the metric-emission pipeline fires. Used
378 /// on error paths in the step loop so mid-scenario failure still
379 /// leaves a usable sidecar.
380 fn drain_all_payloads(&mut self) {
381 drain_all_payload_handles(&mut self.step.payload_handles);
382 drain_all_payload_handles(&mut self.backdrop.payload_handles);
383 }
384
385 /// Kill every payload handle (step-first, then backdrop) whose
386 /// cgroup matches `cgroup`. Called before a cgroup removal so
387 /// cgroupfs cleanup does not trip EBUSY on a live process.
388 fn drain_payloads_for_cgroup(&mut self, cgroup: &str) {
389 drain_payload_handles_for_cgroup(&mut self.step.payload_handles, cgroup);
390 drain_payload_handles_for_cgroup(&mut self.backdrop.payload_handles, cgroup);
391 }
392
393 /// Remove every workload handle whose key matches `cgroup`. The
394 /// handles themselves drop (which SIGKILLs the workers) — this is
395 /// appropriate for `Op::StopCgroup` and `Op::RemoveCgroup`.
396 fn drop_handles_for_cgroup(&mut self, cgroup: &str) {
397 self.step.handles.retain(|(n, _)| n.as_str() != cgroup);
398 self.backdrop.handles.retain(|(n, _)| n.as_str() != cgroup);
399 }
400
401 /// Forget a tracked cpuset (step-first, then backdrop) for a cgroup.
402 fn forget_cpuset(&mut self, cgroup: &str) {
403 self.step.cpusets.remove(cgroup);
404 self.backdrop.cpusets.remove(cgroup);
405 }
406
407 /// Record / overwrite the resolved cpuset for a cgroup. If the
408 /// cgroup is known to step-local state, the step-local entry
409 /// updates; if it's known to backdrop, the backdrop entry
410 /// updates; otherwise the entry goes into the currently-active
411 /// target (step-local, or backdrop inside `with_target_backdrop`).
412 fn record_cpuset(&mut self, cgroup: &str, cpuset: BTreeSet<usize>) {
413 if self.step.cpusets.contains_key(cgroup) {
414 self.step.cpusets.insert(cgroup.to_string(), cpuset);
415 } else if self.backdrop.cpusets.contains_key(cgroup) {
416 self.backdrop.cpusets.insert(cgroup.to_string(), cpuset);
417 } else {
418 self.target_cpusets().insert(cgroup.to_string(), cpuset);
419 }
420 }
421
422 /// Re-key every workload handle from `from` to `to`. When `to`
423 /// names a Backdrop-owned cgroup, step-local handles are also
424 /// transferred into [`Self::backdrop`] so their lifetime extends
425 /// to scenario end instead of dying at step teardown. Backdrop
426 /// handles stay in the backdrop slot regardless of `to`.
427 ///
428 /// Called by `Op::MoveAllTasks` after the kernel-side
429 /// `cgroup.procs` writes succeed so subsequent ops that address
430 /// the moved workers by cgroup name find them under the new key
431 /// and in the correct state slot.
432 fn rename_handles(&mut self, from: &str, to: &str) {
433 let to_is_backdrop = self.cgroup_name_is_backdrop(to);
434 if to_is_backdrop {
435 // Move step-local handles keyed under `from` into the
436 // backdrop slot, re-keyed to `to`. Iterate in reverse so
437 // swap_remove indices stay stable.
438 let mut i = self.step.handles.len();
439 while i > 0 {
440 i -= 1;
441 if self.step.handles[i].0.as_str() == from {
442 let (_, handle) = self.step.handles.swap_remove(i);
443 self.backdrop.handles.push((to.to_string(), handle));
444 }
445 }
446 } else {
447 // Step-local destination: keep ownership, just rename.
448 for (name, _) in &mut self.step.handles {
449 if name.as_str() == from {
450 *name = to.to_string();
451 }
452 }
453 }
454 // Backdrop handles are never demoted to step-local ownership
455 // regardless of destination — a backdrop worker is declared
456 // persistent and stays persistent for the scenario. Rename
457 // in place so subsequent ops still find it under the new key.
458 for (name, _) in &mut self.backdrop.handles {
459 if name.as_str() == from {
460 *name = to.to_string();
461 }
462 }
463 }
464
465 /// Iterate every live workload handle across step + backdrop.
466 /// Used by `Op::MoveAllTasks` / `Op::SetAffinity` which act on
467 /// whichever cgroup owns the handle without caring about which
468 /// state slot it's in.
469 fn all_handles(&self) -> impl Iterator<Item = &(String, WorkloadHandle)> {
470 self.step.handles.iter().chain(self.backdrop.handles.iter())
471 }
472
473 /// True iff a cgroup with the given name is already tracked by
474 /// either step-local or backdrop state. Used to reject duplicate
475 /// names at `apply_setup` time so a user can't accidentally
476 /// shadow a Backdrop cgroup with a step-local [`CgroupDef`].
477 fn cgroup_name_is_tracked(&self, name: &str) -> bool {
478 self.step.cgroups.names().iter().any(|n| n == name)
479 || self.backdrop.cgroups.names().iter().any(|n| n == name)
480 }
481
482 /// True iff a cgroup with the given name is tracked by backdrop
483 /// (persistent) state. Used by `Op::MoveAllTasks` to decide
484 /// handle-ownership transfer direction (step→backdrop transfers
485 /// the handle into the persistent slot; backdrop→step-local is
486 /// rejected because it would orphan workers at step teardown).
487 fn cgroup_name_is_backdrop(&self, name: &str) -> bool {
488 self.backdrop.cgroups.names().iter().any(|n| n == name)
489 }
490}
491
492/// Whether a live payload handle was spawned by an explicit
493/// [`Op::RunPayload`] inside the step or by a
494/// [`CgroupDef::workload`] attachment at `apply_setup`. Held by
495/// every [`PayloadEntry`] so the dedup path in `Op::RunPayload`
496/// can name the original source when rejecting a second spawn of
497/// the same name.
498#[derive(Debug, Clone, Copy, PartialEq, Eq)]
499enum PayloadSource {
500 /// Spawned by `CgroupDef::workload(&payload)` during `apply_setup`.
501 CgroupDefWorkload,
502 /// Spawned by `Op::RunPayload { payload, .. }` inside the step's ops.
503 OpRunPayload,
504}
505
506impl PayloadSource {
507 /// Human-readable tag for error output. Describes the API surface
508 /// that originated the spawn, not the internal dispatch site.
509 fn describe(self) -> &'static str {
510 match self {
511 PayloadSource::CgroupDefWorkload => "CgroupDef::workload",
512 PayloadSource::OpRunPayload => "Op::RunPayload",
513 }
514 }
515}
516
517/// One live payload handle plus the cgroup it runs inside and the
518/// API surface that spawned it. `cgroup` is empty iff
519/// `source == PayloadSource::OpRunPayload` was invoked without a
520/// `cgroup = Some(...)` argument — in which case the payload runs
521/// in whatever cgroup its parent process inherited (no explicit
522/// placement).
523struct PayloadEntry {
524 cgroup: String,
525 source: PayloadSource,
526 handle: crate::scenario::payload_run::PayloadHandle,
527}
528
529/// Map the BPF probe's current scheduler-exit classification onto
530/// the [`crate::assert::DetailKind`] variant the three liveness
531/// emission sites push. Reads [`crate::probe::process::sched_exit_kind`]
532/// which mirrors the probe's `ktstr_err_exit_detected` BSS latch
533/// across threads.
534///
535/// Returns:
536/// - `SchedulerCrashed` when the probe observed a non-clean kernel
537/// exit (any path that latched `ktstr_err_exit_detected`).
538/// - `SchedulerExitedCleanly` when the probe ran but never observed
539/// the latch (clean `SCX_EXIT_NONE` teardown, or the scheduler
540/// exited for a benign reason).
541/// - `SchedulerDiedUnknownReason` when the probe has not classified
542/// yet — typically the probe pipeline never wired for this run
543/// (host-only test, no scheduler attached) or the poll thread has
544/// not completed a first iteration since the prior reset.
545fn sched_died_detail_kind() -> crate::assert::DetailKind {
546 use crate::assert::DetailKind;
547 use crate::probe::process::{SchedExitKind, sched_exit_kind};
548 match sched_exit_kind() {
549 SchedExitKind::Crashed => DetailKind::SchedulerCrashed,
550 SchedExitKind::Clean => DetailKind::SchedulerExitedCleanly,
551 SchedExitKind::Unknown => DetailKind::SchedulerDiedUnknownReason,
552 }
553}
554
555#[cfg(not(test))]
556fn scheduler_process_died() -> Option<bool> {
557 match crate::vmm::rust_init::current_scheduler_liveness() {
558 Ok(Some(alive)) => Some(!alive),
559 Ok(None) => None,
560 // An unusable retained pidfd cannot prove scheduler liveness; fail
561 // closed exactly like the prior process probe.
562 Err(_) => Some(true),
563 }
564}
565
566#[cfg(test)]
567fn scheduler_process_died() -> Option<bool> {
568 crate::vmm::rust_init::sched_pid().map(|pid| !process_alive(pid))
569}
570
571/// Classify scheduler liveness for the guest-side `survives_storm` probe:
572/// `Some(kind)` when a scheduler is still expected to run but has died or gone
573/// down — its retained pidfd is terminal OR the scx state reads
574/// `disabling`/`disabled` ([`dispatch::scx_down`]) — and `None` otherwise.
575///
576/// Mirrors the inter-step / post-loop liveness gate `run_scenario` runs for
577/// `execute_*` scenarios, so a clean `Op::DetachScheduler` — which removes the
578/// owner before this would observe `disabled` — does not trip it. The kind
579/// comes from [`sched_died_detail_kind`]; in the primary test VM the BPF
580/// err-exit latch is unread, so the kind is
581/// [`crate::assert::DetailKind::SchedulerDiedUnknownReason`] (the scx state is
582/// kind-less — a crash and a clean unregister both read `disabling`/`disabled`).
583pub(crate) fn sched_liveness_failure_kind() -> Option<crate::assert::DetailKind> {
584 if scheduler_process_died().is_some_and(|died| died || dispatch::scx_down()) {
585 Some(sched_died_detail_kind())
586 } else {
587 None
588 }
589}
590
591/// Execute a single step with CgroupDefs that hold for the full duration.
592///
593/// Convenience wrapper around [`execute_steps`] for the common pattern
594/// of creating cgroups and running them for [`HoldSpec::FULL`].
595pub fn execute_defs(ctx: &Ctx, defs: Vec<CgroupDef>) -> Result<AssertResult> {
596 execute_steps(ctx, vec![Step::with_defs(defs, HoldSpec::FULL)])
597}
598
599/// Block until the host freeze coordinator has ADOPTED its
600/// kernel-symbol accessor — signalled via the
601/// `SIGNAL_ACCESSOR_READY` wake byte →
602/// `accessor_ready_latch` (set by `hvc0_poll_loop`). A failure dump
603/// captured by a stall AFTER this returns renders real BPF map values
604/// instead of placeholders, because the coordinator's `owned_accessor`
605/// is adopted before the stall fires. Call this in a dump-asserting
606/// scenario before triggering its stall (e.g. before [`execute_steps`]
607/// with a `--stall-after` scheduler).
608///
609/// Guest-only: a no-op on the host (unit tests), where the latch is
610/// never armed. Warn-and-proceed on a 60s timeout — a never-adopted
611/// accessor is a worker failure, and surfacing it as a placeholder dump
612/// is more useful than blocking the test forever (mirrors the
613/// `wait_for_map_write` gate's soft-timeout policy). On that timeout the
614/// "renders real values" guarantee above does NOT hold: the wait returns
615/// without adoption and a subsequent stall may dump placeholders, which
616/// the test's post-VM dump assertions then surface as a failure.
617///
618/// The latch is sticky (level-triggered) and sets once, on the FIRST
619/// adoption. A re-init publish after a scheduler swap
620/// (`Op::ReplaceScheduler`) reuses the same latch, so a later
621/// `await_accessor_ready` returns immediately and does NOT re-synchronise
622/// on the post-swap adoption — today's only caller gates the first stall,
623/// before any swap.
624pub fn await_accessor_ready() {
625 if guest_comms::is_guest() {
626 let latch = crate::vmm::rust_init::accessor_ready_latch();
627 if !latch.wait_timeout(Duration::from_secs(60)) {
628 tracing::warn!(
629 "await_accessor_ready timed out after 60s — host freeze \
630 coordinator did not signal accessor adoption; a dump from a \
631 stall after this point may render placeholder map values"
632 );
633 }
634 }
635}
636
637/// Execute a sequence of steps against the given context.
638///
639/// Convenience wrapper around [`execute_steps_with`] that passes
640/// `None` for checks, falling back to `ctx.assert`. Use
641/// [`execute_steps_with`] when you need to override `ctx.assert`.
642pub fn execute_steps(ctx: &Ctx, steps: Vec<Step>) -> Result<AssertResult> {
643 execute_steps_with(ctx, steps, None)
644}
645
646/// Execute a [`Backdrop`](backdrop::Backdrop) + Steps sequence
647/// against the given context.
648///
649/// The Backdrop declares persistent scenario-wide state
650/// (long-running payloads, cgroups referenced by many Steps) while
651/// Steps express bounded per-phase behavior. The runtime sets up
652/// the Backdrop before the first Step, runs the Step sequence
653/// with per-Step teardown (cgroups removed, workload handles
654/// collected, payload handles killed at step boundary), and tears
655/// the Backdrop down at the end.
656pub fn execute_scenario(
657 ctx: &Ctx,
658 backdrop: backdrop::Backdrop,
659 steps: Vec<Step>,
660) -> Result<AssertResult> {
661 execute_scenario_with(ctx, backdrop, steps, None)
662}
663
664/// [`execute_scenario`] with an explicit
665/// [`Assert`](crate::assert::Assert) override — the Backdrop
666/// equivalent of [`execute_steps_with`].
667pub fn execute_scenario_with(
668 ctx: &Ctx,
669 backdrop: backdrop::Backdrop,
670 steps: Vec<Step>,
671 checks: Option<&crate::assert::Assert>,
672) -> Result<AssertResult> {
673 run_scenario(ctx, backdrop, steps, checks)
674}
675
676/// Execute steps with an explicit [`Assert`](crate::assert::Assert) for
677/// worker checks. When `checks` is `Some`, it overrides `ctx.assert`.
678/// When `None`, uses `ctx.assert` (the merged three-layer config).
679///
680/// Thin wrapper around [`execute_scenario_with`] with an empty
681/// [`Backdrop`](backdrop::Backdrop) — every Step's effects
682/// (cgroups, workloads, payloads) tear down at the step boundary.
683pub fn execute_steps_with(
684 ctx: &Ctx,
685 steps: Vec<Step>,
686 checks: Option<&crate::assert::Assert>,
687) -> Result<AssertResult> {
688 execute_scenario_with(ctx, backdrop::Backdrop::new(), steps, checks)
689}
690
691/// Compute the union of cgroup v2 controllers required by a
692/// Backdrop and Step sequence. Walks every [`CgroupDef`] declaration
693/// and every [`Op`] variant, returning the smallest set of
694/// controllers that must be enabled in `cgroup.subtree_control` for
695/// the scenario's per-knob writes to land.
696///
697/// Mapping:
698/// - [`CgroupDef::cpuset`] / [`CgroupDef::cpuset_mems`] → `Controller::Cpuset`
699/// - [`CgroupDef::cpu`] → `Controller::Cpu`
700/// - [`CgroupDef::memory`] → `Controller::Memory`
701/// - [`CgroupDef::pids`] → `Controller::Pids`
702/// - [`CgroupDef::io`] → `Controller::Io`
703/// - [`Op::SetCpuset`] / [`Op::ClearCpuset`] / [`Op::SwapCpusets`] /
704/// [`Op::SetAffinity`] → `Controller::Cpuset`
705/// - Every other [`Op`] variant ([`Op::FreezeCgroup`],
706/// [`Op::AddCgroup`], [`Op::Spawn`], [`Op::MoveAllTasks`], etc.)
707/// touches cgroup-core knobs (`cgroup.freeze`, `cgroup.procs`,
708/// `mkdir`/`rmdir`) which are ungated by any controller and
709/// contribute nothing to this set.
710///
711/// Returning the SMALLEST set lets a test that intentionally
712/// requires the absence of a controller (e.g. testing behavior on
713/// a kernel without `+cpu`) get an empty subtree_control write.
714fn required_controllers(
715 ctx: &Ctx,
716 backdrop: &backdrop::Backdrop,
717 steps: &[Step],
718) -> BTreeSet<crate::cgroup::Controller> {
719 use crate::cgroup::Controller;
720 fn absorb_def(set: &mut BTreeSet<Controller>, def: &CgroupDef) {
721 if def.cpuset.is_some() || def.cpuset_mems.is_some() {
722 set.insert(Controller::Cpuset);
723 }
724 if def.cpu.is_some() {
725 set.insert(Controller::Cpu);
726 }
727 if def.memory.is_some() {
728 set.insert(Controller::Memory);
729 }
730 if def.io.is_some() {
731 set.insert(Controller::Io);
732 }
733 if def.pids.is_some() {
734 set.insert(Controller::Pids);
735 }
736 }
737 fn absorb_op(set: &mut BTreeSet<Controller>, op: &Op) {
738 if matches!(
739 op,
740 Op::SetCpuset { .. }
741 | Op::ClearCpuset { .. }
742 | Op::SwapCpusets { .. }
743 | Op::SetAffinity { .. }
744 ) {
745 set.insert(Controller::Cpuset);
746 }
747 // AddCgroupDef carries a full CgroupDef whose knobs may
748 // require any of the same controllers absorb_def covers. The
749 // op-applied def goes through apply_setup at op-execute time,
750 // which writes to those controller files; the parent's
751 // subtree_control must already have the controllers enabled
752 // by then, so absorb the def's needs into the pre-scenario
753 // controller setup the same way step-local CgroupDefs do.
754 if let Op::AddCgroupDef { def } = op {
755 absorb_def(set, def);
756 }
757 }
758 let mut set = BTreeSet::new();
759 for def in &backdrop.cgroups {
760 absorb_def(&mut set, def);
761 }
762 for op in &backdrop.ops {
763 absorb_op(&mut set, op);
764 }
765 for step in steps {
766 for def in step.setup.resolve(ctx) {
767 absorb_def(&mut set, &def);
768 }
769 for op in &step.ops {
770 absorb_op(&mut set, op);
771 }
772 }
773 set
774}
775
776/// Validate every step hold spec and reject scheduler-kind Backdrop
777/// payloads up front so failures surface at the declaration, before
778/// any runtime state is created.
779fn validate_scenario_inputs(backdrop: &backdrop::Backdrop, steps: &[Step]) -> Result<()> {
780 // Validate every step's hold spec up front so a typo doesn't
781 // reach `Duration::from_secs_f64(NaN)` / `thread::sleep(ZERO)` /
782 // a no-yield Loop busy-wait after ops have already been applied.
783 for (i, step) in steps.iter().enumerate() {
784 if let Err(reason) = step.hold.validate() {
785 anyhow::bail!("step {i} hold validation: {reason}");
786 }
787 }
788 // Validate Backdrop payloads before creating any runtime state.
789 // Only binary payloads can be spawned by Op::RunPayload, which
790 // is what the Backdrop setup uses under the hood. Reject
791 // scheduler-kind payloads here so the failure surface is the
792 // Backdrop declaration, not a mid-scenario spawn error after
793 // cgroups have already been created.
794 for p in &backdrop.payloads {
795 if p.is_scheduler() {
796 anyhow::bail!(
797 "Backdrop::push_payload received scheduler-kind Payload '{}' — \
798 only PayloadKind::Binary payloads run in the Backdrop; \
799 place scheduler-kind payloads on the #[ktstr_test(scheduler = ...)] \
800 attribute instead",
801 p.name,
802 );
803 }
804 }
805 // Scheduler-kind payloads smuggled via Backdrop::push_op(Op::RunPayload { ... })
806 // would otherwise bypass the check above and only bail deep inside
807 // apply_ops. Reject them here with a Backdrop-specific error so
808 // the failure surface matches the declaration surface.
809 for op in &backdrop.ops {
810 if let Op::RunPayload { payload, .. } = op
811 && payload.is_scheduler()
812 {
813 anyhow::bail!(
814 "Backdrop::push_op(Op::RunPayload) received scheduler-kind Payload '{}' — \
815 only PayloadKind::Binary payloads run in the Backdrop; \
816 place scheduler-kind payloads on the #[ktstr_test(scheduler = ...)] \
817 attribute instead",
818 payload.name,
819 );
820 }
821 }
822 Ok(())
823}
824
825/// Guest-side gate that blocks until the host's queued BPF map writes
826/// land, so the workload phase observes fresh map values.
827fn wait_for_host_map_write(ctx: &Ctx) {
828 // When a host-side BPF map write is configured the test framework
829 // sets `wait_for_map_write=true`; in that case block until the
830 // guest's `hvc0_poll_loop` observes
831 // [`crate::vmm::virtio_console::SIGNAL_BPF_WRITE_DONE`] (pushed by
832 // the host's `bpf-map-write` thread after every queued
833 // `bpf_map_write` lands) and fires the `bpf_map_write_done` latch.
834 // Without this gate the workload phase races against the host's
835 // map writes and may observe a stale BPF map value.
836 //
837 // Guest-only path. On the host (unit tests) the latch is never
838 // armed, so we skip the wait entirely. The 60 s timeout matches
839 // the bpf-map-write thread's combined phase 1 + phase 2 budget
840 // (30 s accessor init + 30 s map discovery in
841 // `freeze_coord::start_bpf_map_write`); a real timeout means the
842 // host failed to resolve a map. The scenario continues anyway
843 // (rather than `bail!`) because the legacy rendezvous also let
844 // the guest proceed under its own timeout, and a bail here would
845 // mask the underlying host-side resolution failure with a
846 // test-side `Err`.
847 if ctx.wait_for_map_write && guest_comms::is_guest() {
848 let latch = crate::vmm::rust_init::bpf_map_write_done_latch();
849 if !latch.wait_timeout(Duration::from_secs(60)) {
850 tracing::warn!(
851 "wait_for_map_write timed out after 60s — host bpf-map-write \
852 thread may have failed to resolve a queued map; proceeding \
853 with the workload regardless"
854 );
855 }
856 }
857}
858
859/// Run persistent Backdrop setup. `Ok(None)` on success; `Ok(Some(r))`
860/// carries the failure AssertResult when setup errors so the caller
861/// returns it directly.
862fn run_backdrop_setup<'a>(
863 ctx: &'a Ctx,
864 backdrop: &backdrop::Backdrop,
865 backdrop_state: &mut BackdropState<'a>,
866 effective_checks: &crate::assert::Assert,
867 result: &AssertResult,
868) -> Result<Option<AssertResult>> {
869 // --- Backdrop setup (persistent) ---
870 // Run before the first Step. Cgroups + payloads declared on
871 // `backdrop` land in `backdrop_state` so they survive every
872 // Step's teardown. On error, drain Backdrop payload handles
873 // (metric emission) and propagate.
874 if backdrop.is_empty() {
875 return Ok(None);
876 }
877 let mut step_staging = StepState::empty(ctx);
878 let mut scratch = ScenarioState::new(&mut step_staging, backdrop_state);
879 let setup_res = scratch.with_target_backdrop(|s| {
880 // Order: cgroups → ops → payloads. CgroupDefs go first so
881 // a later `Op::add_cgroup` / `Op::run_payload_in_cgroup`
882 // can target cgroups that `apply_setup` just created.
883 // Payloads spawn last so `run_payload` resolving a cgroup
884 // placement lands inside a cgroup that either apply pass
885 // already built.
886 if !backdrop.cgroups.is_empty() {
887 apply_setup(ctx, s, &backdrop.cgroups)?;
888 }
889 // Raw ops: typically `Op::AddCgroup` for empty move-target
890 // cgroups (can't be expressed via CgroupDef because
891 // apply_setup forces a worker spawn), or placement-aware
892 // `Op::RunPayload` targeting a just-created backdrop
893 // cgroup.
894 if !backdrop.ops.is_empty() {
895 apply_ops(ctx, s, &backdrop.ops, false)?;
896 }
897 // Shorthand payloads: one Op::RunPayload per entry,
898 // inherited cgroup placement.
899 if !backdrop.payloads.is_empty() {
900 let ops: Vec<Op> = backdrop
901 .payloads
902 .iter()
903 .map(|p| Op::run_payload(p, Vec::<String>::new()))
904 .collect();
905 apply_ops(ctx, s, &ops, false)?;
906 }
907 Ok::<(), anyhow::Error>(())
908 });
909 if let Err(err) = setup_res {
910 // Scheduler crashed during backdrop setup (e.g. a worker
911 // tripped the BPF error before the first Step): SIGKILL the
912 // spawned workers up front so the collect reaps below aren't
913 // scheduling-gated behind fallback workers (see
914 // `sigkill_handles`). Gated on `scx_down` (crash-only) so a
915 // non-crash setup error keeps the graceful collect path.
916 if sched_crashed_unexpectedly() {
917 sigkill_handles(&backdrop_state.handles);
918 sigkill_handles(&step_staging.handles);
919 }
920 // Collect any workers that DID spawn before the failure
921 // so their stats reach the final result instead of being
922 // discarded by `WorkloadHandle::drop` (which SIGKILLs
923 // without gathering scheduler-side data). `collect_*`
924 // drain `payload_handles` internally, so the backdrop-
925 // and step-side payloads still get `.kill()` (SHM metric
926 // emission) on the error path.
927 //
928 // `with_target_backdrop` routes every target writer to
929 // the backdrop slot, so `step_staging` normally holds
930 // nothing — but collect defensively so a partial-failure
931 // path that leaks a non-backdrop write surfaces here
932 // rather than disappearing into `StepState::drop`.
933 let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
934 let staging_result = collect_step(
935 &mut step_staging,
936 effective_checks,
937 ctx.topo,
938 ctx.cgroups,
939 // Defensive backdrop-staging collect: no step attribution.
940 None,
941 );
942 r.merge(staging_result);
943 r.merge(result.clone());
944 // step_staging's CgroupGroup RAII still drops here,
945 // removing any cgroups the failed Backdrop setup routed
946 // into step-local state.
947 r.record_fail(crate::assert::AssertDetail::new(
948 crate::assert::DetailKind::Other,
949 format!("Backdrop setup failed: {err:#}"),
950 ));
951 return Ok(Some(r));
952 }
953 // `step_staging` should not have accumulated anything
954 // because `with_target_backdrop` routed every target writer
955 // to the backdrop side. Collect any stray handles defensively
956 // before dropping so a future refactor that leaks a non-
957 // backdrop write here surfaces as a missed teardown rather
958 // than silently discarded state.
959 drain_all_payload_handles(&mut step_staging.payload_handles);
960 Ok(None)
961}
962
963/// 1-indexed phase number for scenario Step `step_idx` (BASELINE=0,
964/// Step k -> k+1), saturating at `u16::MAX` past the encoding range.
965fn phase_step_index(step_idx: usize) -> u16 {
966 u16::try_from(step_idx)
967 .ok()
968 .and_then(|i| i.checked_add(1))
969 .unwrap_or(u16::MAX)
970}
971
972/// Build the inter-step scheduler-death failure result: collect
973/// backdrop workers, merge the accumulated result, record the
974/// `format_sched_died_after_step` detail.
975fn build_sched_died_after_step(
976 backdrop_state: &mut BackdropState<'_>,
977 result: AssertResult,
978 ctx: &Ctx,
979 effective_checks: &crate::assert::Assert,
980 step_idx: usize,
981 steps_len: usize,
982 scenario_start: std::time::Instant,
983) -> AssertResult {
984 // Collect backdrop-owned workload handles into the
985 // result before reporting the crash so whatever the
986 // persistent workers produced is still assertable.
987 let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
988 r.merge(result);
989 r.record_fail(crate::assert::AssertDetail::new(
990 sched_died_detail_kind(),
991 crate::assert::format_sched_died_after_step(
992 step_idx,
993 steps_len,
994 scenario_start.elapsed().as_secs_f64(),
995 ),
996 ));
997 r
998}
999
1000/// Build the step-error failure result: collect backdrop workers,
1001/// merge the accumulated result, record the `step N failed` detail.
1002fn build_step_failed(
1003 backdrop_state: &mut BackdropState<'_>,
1004 result: AssertResult,
1005 ctx: &Ctx,
1006 effective_checks: &crate::assert::Assert,
1007 step_idx: usize,
1008 err: &anyhow::Error,
1009) -> AssertResult {
1010 // Collect Backdrop-owned workload handles into a fresh
1011 // result first, then merge the accumulated step result
1012 // on top. `collect_backdrop` drains
1013 // `backdrop_state.payload_handles` internally, so the
1014 // backdrop-side payloads still get `.kill()` (metric
1015 // emission) on the error path. Ordering mirrors the
1016 // scheduler-crash path above so detail order is
1017 // consistent across both Ok(failed) returns.
1018 let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1019 r.merge(result);
1020 r.record_fail(crate::assert::AssertDetail::new(
1021 crate::assert::DetailKind::Other,
1022 format!("step {step_idx} failed: {err:#}"),
1023 ));
1024 r
1025}
1026
1027/// Build the sched-died-during-hold failure result: collect backdrop
1028/// workers, merge the accumulated result, record the
1029/// `format_sched_died_during_workload` detail.
1030fn build_sched_died_during_hold(
1031 backdrop_state: &mut BackdropState<'_>,
1032 result: AssertResult,
1033 ctx: &Ctx,
1034 effective_checks: &crate::assert::Assert,
1035 scenario_start: std::time::Instant,
1036) -> AssertResult {
1037 let mut r = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1038 r.merge(result);
1039 r.record_fail(crate::assert::AssertDetail::new(
1040 sched_died_detail_kind(),
1041 crate::assert::format_sched_died_during_workload(scenario_start.elapsed().as_secs_f64()),
1042 ));
1043 r
1044}
1045
1046/// Emit the ScenarioEnd marker, run the final liveness check, tear the
1047/// Backdrop down, and fold in a sched-died detail when the scheduler
1048/// died after the last hold.
1049fn finish_scenario(
1050 ctx: &Ctx,
1051 backdrop_state: &mut BackdropState<'_>,
1052 effective_checks: &crate::assert::Assert,
1053 scenario_start: std::time::Instant,
1054 steps_len: usize,
1055 final_total_iterations: Option<(u64, u64)>,
1056 mut result: AssertResult,
1057) -> AssertResult {
1058 // ScenarioEnd marker. Routes through `send_scenario_end`
1059 // (virtio-console port-1 bulk channel).
1060 // Carries the last cleanly-completed step's COINCIDENT (elapsed_ms,
1061 // iterations) pair so the host can give the last step an
1062 // `iteration_rate` over a well-formed window. The elapsed
1063 // is the end-of-hold timestamp captured WITH the count inside
1064 // run_step — NOT recomputed here, which would include the
1065 // pause+collect teardown wall-time and under-report the rate. When
1066 // no step completed a hold (e.g. every step's scheduler died) the
1067 // pair is `None`: fall back to (now, 0), which the host skips
1068 // (0 baseline / e<=s).
1069 if guest_comms::is_guest() {
1070 let (end_elapsed_ms, end_iterations) = final_total_iterations
1071 .unwrap_or_else(|| (scenario_start.elapsed().as_millis() as u64, 0));
1072 crate::vmm::guest_comms::send_scenario_end(end_elapsed_ms, end_iterations);
1073 }
1074
1075 // Final liveness check. Live `crate::vmm::rust_init::sched_pid()`
1076 // read instead of `ctx.sched_pid` snapshot so a mid-scenario
1077 // Op::ReplaceScheduler swap reflects the new pid here too.
1078 // sched_pid() == None ⇒ no scheduler configured (kernel-default
1079 // path) OR Op::DetachScheduler cleared it; no liveness to
1080 // report on either case.
1081 let sched_dead = scheduler_process_died().is_some_and(|died| died || dispatch::scx_down());
1082
1083 // Scheduler died after the last hold: SIGKILL the backdrop
1084 // workers up front so the teardown reap below isn't
1085 // scheduling-gated behind still-spinning workers that fell back
1086 // to the builtin scheduler (see `sigkill_handles`).
1087 if sched_dead {
1088 sigkill_handles(&backdrop_state.handles);
1089 }
1090
1091 // --- Backdrop teardown ---
1092 let backdrop_result = collect_backdrop(backdrop_state, effective_checks, ctx.topo, ctx.cgroups);
1093 result.merge(backdrop_result);
1094
1095 if sched_dead {
1096 result.record_fail(crate::assert::AssertDetail::new(
1097 sched_died_detail_kind(),
1098 crate::assert::format_sched_died_after_all_steps(
1099 steps_len,
1100 scenario_start.elapsed().as_secs_f64(),
1101 ),
1102 ));
1103 }
1104
1105 result
1106}
1107
1108/// Guest workload-progress publisher. It emits scenario-elapsed time from the
1109/// scenario driver's EXISTING scheduler-liveness wakeups (plus lifecycle
1110/// boundaries), so the host can clock periodic captures off guest progress
1111/// without adding a periodically-runnable guest thread of its own.
1112///
1113/// The driver's hold path already wakes every 100 ms to inspect the BPF
1114/// err-exit latch / sched_ext state. Publishing from those wakes preserves the
1115/// heartbeat's cadence while the capture-active workload is running at no
1116/// additional scheduler wakeup cost. Setup and paused teardown publish only at
1117/// their natural boundaries; there is no useful periodic-capture progress to
1118/// report while the driver is not in a workload hold.
1119#[derive(Clone, Copy)]
1120struct WorkloadHeartbeat {
1121 scenario_start: std::time::Instant,
1122}
1123
1124impl WorkloadHeartbeat {
1125 fn publish(self) {
1126 crate::vmm::guest_comms::send_workload_progress(
1127 self.scenario_start.elapsed().as_millis() as u64
1128 );
1129 }
1130}
1131
1132/// Internal driver: runs Backdrop setup, the Step loop with
1133/// per-Step teardown, and final Backdrop teardown.
1134fn run_scenario(
1135 ctx: &Ctx,
1136 backdrop: backdrop::Backdrop,
1137 steps: Vec<Step>,
1138 checks: Option<&crate::assert::Assert>,
1139) -> Result<AssertResult> {
1140 validate_scenario_inputs(&backdrop, &steps)?;
1141 let effective_checks = checks.unwrap_or(&ctx.assert);
1142
1143 // Enable the controllers this scenario actually needs in
1144 // `cgroup.subtree_control` BEFORE any cgroupfs writes land. The
1145 // union is computed from every CgroupDef and Op declared in the
1146 // backdrop+steps; tests that declare no controller-gated knobs
1147 // get an empty set (parent dir created, no subtree_control walk).
1148 let required = required_controllers(ctx, &backdrop, &steps);
1149 ctx.cgroups
1150 .setup(&required)
1151 .context("enable cgroup controllers in subtree_control")?;
1152
1153 let mut backdrop_state = BackdropState::empty(ctx);
1154 let mut result = AssertResult::pass();
1155
1156 // (scenario-relative elapsed_ms, cumulative worker iteration count)
1157 // at the END of the most-recently completed step, captured
1158 // coincidently inside `run_step` while that step's workers are still
1159 // alive. Carries the LAST step's pair out of the loop for terminal
1160 // boundary telemetry. `None` until the first step completes a hold
1161 // cleanly.
1162 let mut final_total_iterations: Option<(u64, u64)> = None;
1163
1164 let scenario_start = std::time::Instant::now();
1165
1166 // ScenarioStart marker. `is_guest` short-circuits in host
1167 // contexts (unit tests) where the bulk port and SHM ring are
1168 // both absent and `send_scenario_start` would log a no-op warning.
1169 if guest_comms::is_guest() {
1170 crate::vmm::guest_comms::send_scenario_start();
1171 }
1172
1173 // Workload-progress publisher: drives the host's periodic-capture clock
1174 // off guest progress. Only present when this run declares periodic
1175 // captures (`KTSTR_AWAIT_PERIODIC_READY`, set iff num_snapshots > 0).
1176 // Unlike the old detached heartbeat, this is passive state carried by the
1177 // scenario driver; `hold_or_sched_died` publishes from wakeups the driver
1178 // already performs for scheduler liveness.
1179 let workload_heartbeat = (guest_comms::is_guest()
1180 && crate::vmm::rust_init::cmdline_val("KTSTR_AWAIT_PERIODIC_READY").is_some())
1181 .then_some(WorkloadHeartbeat { scenario_start });
1182 if let Some(heartbeat) = workload_heartbeat {
1183 heartbeat.publish();
1184 }
1185
1186 wait_for_host_map_write(ctx);
1187
1188 if let Some(r) = run_backdrop_setup(
1189 ctx,
1190 &backdrop,
1191 &mut backdrop_state,
1192 effective_checks,
1193 &result,
1194 )? {
1195 return Ok(r);
1196 }
1197
1198 // --- Step loop with per-Step teardown ---
1199 for (step_idx, step) in steps.iter().enumerate() {
1200 // Check scheduler liveness between steps (skip before first).
1201 // Live `crate::vmm::rust_init::sched_pid()` read instead of
1202 // `ctx.sched_pid` snapshot so a mid-scenario
1203 // `Op::ReplaceScheduler` swap is reflected — the swap
1204 // dispatcher atomically publishes the replacement's coherent
1205 // process owner, and this check then observes the new pid's
1206 // liveness (not the dead boot pid). `None` means
1207 // either no scheduler was configured at boot or
1208 // `Op::DetachScheduler` cleared the pid; the liveness probe
1209 // cannot meaningfully report on a pid that doesn't exist.
1210 if step_idx > 0 && scheduler_process_died().is_some_and(|died| died || dispatch::scx_down())
1211 {
1212 // Scheduler died between steps: the kernel has disabled
1213 // sched_ext and the backdrop workers have fallen back to
1214 // the builtin scheduler. SIGKILL them up front so the
1215 // collect reap below isn't scheduling-gated behind
1216 // still-spinning workers (see `sigkill_handles`).
1217 sigkill_handles(&backdrop_state.handles);
1218 return Ok(build_sched_died_after_step(
1219 &mut backdrop_state,
1220 result,
1221 ctx,
1222 effective_checks,
1223 step_idx,
1224 steps.len(),
1225 scenario_start,
1226 ));
1227 }
1228
1229 let mut step_state = StepState::empty(ctx);
1230 let mut sched_died_during_hold = false;
1231 // Publish the 1-indexed phase number for this Step so the
1232 // freeze-coordinator periodic-capture path and the on-demand
1233 // Op::CaptureSnapshot / Op::WatchSnapshot apply arms all
1234 // stamp the captures they take with the correct scenario
1235 // phase. The 1-indexed encoding (scenario Step k -> phase
1236 // k + 1) reserves phase 0 for the pre-first-Step BASELINE
1237 // settle window. `Release` pairs with the consumers'
1238 // `Acquire` load so a sample stamped with this value
1239 // happens-after any state the Step has set up before
1240 // calling run_step.
1241 let phase_step_index = phase_step_index(step_idx);
1242 ctx.current_step
1243 .store(phase_step_index, std::sync::atomic::Ordering::Release);
1244 // Broadcast the phase to this handle's backdrop (persistent)
1245 // workers so each drains its per-phase PhaseSlice at this
1246 // boundary. Step-local pools are NOT bumped here — they
1247 // re-spawn per step and get per-phase attribution via
1248 // collect_step's step_index instead.
1249 for (_, h) in &backdrop_state.handles {
1250 h.set_phase_epoch(u32::from(phase_step_index));
1251 }
1252 // Install the assert-side phase guard for the scenario
1253 // driver's thread for the duration of this Step. Every
1254 // AssertDetail / PassDetail / InfoNote constructed under
1255 // the run_step call below auto-stamps its `phase` field
1256 // with "Step[<step_idx>]" via the thread-local snapshot
1257 // in `crate::assert::current_phase_label`. On Drop the
1258 // prior label is restored (BASELINE outside any Step), so
1259 // assertions evaluated post-loop (e.g. at scenario
1260 // teardown) stamp with the right outer scope.
1261 let _phase_guard = crate::assert::PhaseGuard::install_step(step_idx as u16);
1262 let step_res = run_step(
1263 ctx,
1264 step,
1265 step_idx,
1266 &mut step_state,
1267 &mut backdrop_state,
1268 scenario_start,
1269 effective_checks,
1270 &mut sched_died_during_hold,
1271 &mut final_total_iterations,
1272 workload_heartbeat,
1273 );
1274
1275 // Close this step's hold window for backdrop workers: write the
1276 // inter-step gap sentinel so work done between this StepEnd and
1277 // the next StepStart is NOT folded into step `step_idx`'s slice
1278 // (matching the host's hold-only window clamp, and step-local
1279 // workers, which do not exist during the inter-step gap).
1280 for (_, h) in &backdrop_state.handles {
1281 h.set_phase_epoch(u32::MAX);
1282 }
1283
1284 // Scheduler died during this step's hold: the kernel has
1285 // disabled sched_ext and the workers (step and backdrop) have
1286 // fallen back to the builtin scheduler. If CPU-bound they
1287 // would CFS-starve the per-worker reap in the collect calls
1288 // below, so SIGKILL them all up front — the reaps then find
1289 // already-exiting workers (see `sigkill_handles`). Gated on
1290 // the death flag so the normal teardown keeps its graceful
1291 // cooperative-stop + report-collection path.
1292 if sched_died_during_hold || sched_crashed_unexpectedly() {
1293 sigkill_handles(&step_state.handles);
1294 sigkill_handles(&backdrop_state.handles);
1295 }
1296
1297 if guest_comms::is_guest() {
1298 crate::vmm::guest_comms::send_scenario_pause();
1299 }
1300
1301 let step_result = collect_step(
1302 &mut step_state,
1303 effective_checks,
1304 ctx.topo,
1305 ctx.cgroups,
1306 // The SAME 1-indexed value stamped onto this step's StepStart
1307 // frames (build_stimulus), so the guest per_cgroup keys on the
1308 // identical step_index the host rebuilds buckets under.
1309 Some(phase_step_index),
1310 );
1311 result.merge(step_result);
1312
1313 // A step-level error is converted into a failure on the
1314 // accumulated result after teardown has run so every step
1315 // boundary leaves clean state behind even on failure. The
1316 // caller keeps the prior-steps' merged AssertResult plus
1317 // the error context as a detail, instead of an opaque Err
1318 // that discards everything.
1319 if let Err(err) = step_res {
1320 // Scheduler may have crashed between the pre-collect gate
1321 // above and here (e.g. a non-crash step error, then scx went
1322 // down during `collect_step`): SIGKILL the backdrop workers
1323 // up front so this collect isn't scheduling-gated behind the
1324 // fallback pool (see `sigkill_handles`). Crashed-only gate so
1325 // a plain step error with a live scheduler keeps the graceful
1326 // cooperative-stop path.
1327 if sched_crashed_unexpectedly() {
1328 sigkill_handles(&backdrop_state.handles);
1329 }
1330 return Ok(build_step_failed(
1331 &mut backdrop_state,
1332 result,
1333 ctx,
1334 effective_checks,
1335 step_idx,
1336 &err,
1337 ));
1338 }
1339
1340 // Scheduler exited during the step's hold-period sleep —
1341 // [`run_step`] cut the hold short and stamped
1342 // `sched_died_during_hold`. Emit the in-step
1343 // sched-died message before continuing to the next step
1344 // boundary; otherwise the post-loop probe would fire after
1345 // the full scenario duration and stamp a misleading elapsed
1346 // time. Same Backdrop-then-step merge order as the
1347 // inter-step path above so detail ordering stays consistent.
1348 if sched_died_during_hold {
1349 return Ok(build_sched_died_during_hold(
1350 &mut backdrop_state,
1351 result,
1352 ctx,
1353 effective_checks,
1354 scenario_start,
1355 ));
1356 }
1357 }
1358
1359 Ok(finish_scenario(
1360 ctx,
1361 &mut backdrop_state,
1362 effective_checks,
1363 scenario_start,
1364 steps.len(),
1365 final_total_iterations,
1366 result,
1367 ))
1368}
1369
1370/// Sleep up to `dur`, returning early if `sched_pid` exits.
1371///
1372/// Returns `true` the first time the scheduler is observed dead,
1373/// `false` if the full duration elapsed with no death observed.
1374/// When `sched_pid` is `None` (kernel-default scheduling, no
1375/// scheduler process to monitor), behaves exactly like
1376/// [`thread::sleep`] and always returns `false`.
1377///
1378/// Implementation duplicates the pidfd retained by the coherent scheduler
1379/// owner and waits with `epoll_wait`, so the waiter is kernel-blocked on the
1380/// exact process identity until either the scheduler exits
1381/// (pidfd becomes readable) or the per-step hold elapses. This
1382/// drops crash-detection latency from one poll-tick (the previous
1383/// 100 ms cadence) to ~0: the kernel wakes the epoll waiter as
1384/// soon as the task transitions to EXIT_ZOMBIE. Mirrors
1385/// [`crate::scenario::payload_run`]'s `wait_with_deadline` shape.
1386/// Minimum kernel: Linux 5.3.
1387///
1388/// Deadline honoring: the `epoll_wait` timeout is re-derived from
1389/// `saturating_duration_since` each iteration so `EINTR` restarts
1390/// narrow the remaining window rather than extending it.
1391///
1392/// Failure handling: if the owner disappears between the caller's snapshot and
1393/// the pidfd duplication, the exact scheduler is already terminal and the
1394/// function returns `true`. Descriptor duplication, epoll creation/registration,
1395/// timeout conversion, and wait failures panic with an operator-actionable
1396/// message. Polling fallbacks were removed per the project-wide "no polling
1397/// fallbacks for evented paths" rule: these failures indicate a catastrophic
1398/// environment defect rather than a recoverable transient. A loud panic
1399/// surfaces the defect immediately; the prior silent sleep+probe fallback
1400/// masked it as test flakiness. Numeric `pidfd_open` exists only in the
1401/// host-unit-test seam below, whose synthetic scheduler PID has no guest owner.
1402///
1403/// Scheduling jitter under load can leave the actual elapsed time
1404/// modestly above `dur`.
1405/// Loud panic on env- or code-defect failures inside [`hold_or_sched_died`].
1406/// Centralised so every site emits the same module-qualified prefix
1407/// `ktstr::scenario::hold_or_sched_died` — operator can grep that exact
1408/// string to land at the panic source. `op` names the failed primitive,
1409/// `pid` carries the in-scope pid for cross-reference with /proc, `err`
1410/// renders the underlying errno or nix error, `advice` is the one-line
1411/// remediation classified by failure class (env vs framework code defect).
1412#[cold]
1413#[track_caller]
1414fn panic_evented_hold_defect(
1415 op: &str,
1416 pid: libc::pid_t,
1417 err: impl std::fmt::Display,
1418 advice: &str,
1419) -> ! {
1420 panic!("ktstr::scenario::hold_or_sched_died: {op} failed (pid={pid}): {err} — {advice}");
1421}
1422
1423/// True when a scheduler is still expected running (`sched_pid` set) AND
1424/// sched_ext has gone down (`disabling`/`disabled`) — an unexpected
1425/// crash/unregister of the scheduler under test, detected via the
1426/// probe-independent [`dispatch::scx_down`] sysfs read. Unlike the BPF
1427/// err-exit latch (whose host mirror is only populated in the
1428/// auto-repro/dump VM, never the primary VM that runs the test), this is
1429/// live in the PRIMARY VM. The `sched_pid` guard keeps a clean
1430/// [`Op::DetachScheduler`] — which clears the pid before the state
1431/// settles to `disabled` — off the SIGKILL fast-path. Used by the
1432/// teardown gates that don't already hold the scheduler pid in scope.
1433fn sched_crashed_unexpectedly() -> bool {
1434 crate::vmm::rust_init::sched_pid().is_some() && dispatch::scx_down()
1435}
1436
1437fn hold_or_sched_died(
1438 dur: Duration,
1439 sched_pid: Option<libc::pid_t>,
1440 workload_heartbeat: Option<WorkloadHeartbeat>,
1441) -> bool {
1442 use crate::probe::process::{SchedExitKind, sched_exit_kind};
1443 use nix::sys::epoll::{Epoll, EpollCreateFlags, EpollEvent, EpollFlags, EpollTimeout};
1444 use std::os::fd::AsFd;
1445 #[cfg(test)]
1446 use std::os::fd::{FromRawFd, OwnedFd};
1447
1448 // The BPF err-exit latch (read via `sched_exit_kind`) flips at the
1449 // error-class sched_ext exit — i.e. at the crash, before the crashed
1450 // scheduler PROCESS exits (a scheduler like scx_lavd observes the
1451 // disable through its ~1s userspace poll loop, then flushes its dump
1452 // and exits, so its process exit trails the crash by ~that long). Poll
1453 // it ALONGSIDE the pidfd so a crash aborts the hold at ~crash time, not
1454 // at the process exit. The pidfd remains the backstop for
1455 // clean/non-error process exits. 100ms is negligible vs the crash + the
1456 // scheduler's exit latency.
1457 const ERR_EXIT_POLL: Duration = Duration::from_millis(100);
1458 let crashed = || matches!(sched_exit_kind(), SchedExitKind::Crashed);
1459
1460 // Clone the pidfd retained by the coherent scheduler owner while its lock
1461 // is held. Reopening `sched_pid` after the lock is released would recreate
1462 // the numeric-PID reuse race that the owner record exists to eliminate.
1463 let exact_scheduler = match crate::vmm::rust_init::clone_current_scheduler_pidfd() {
1464 Ok(identity) => identity,
1465 Err(error) => panic_evented_hold_defect(
1466 "dup(current scheduler pidfd)",
1467 sched_pid.unwrap_or(-1),
1468 error,
1469 "the owner retains a live pidfd; dup failure indicates fd exhaustion",
1470 ),
1471 };
1472 #[cfg(test)]
1473 let exact_scheduler = match (exact_scheduler, sched_pid) {
1474 (Some(identity), _) => Some(identity),
1475 (None, Some(pid)) => {
1476 // Host unit tests inject a synthetic pid without installing the
1477 // guest-only process owner. Keep that seam test-only.
1478 let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0u32) as libc::c_int };
1479 if raw < 0 {
1480 None
1481 } else {
1482 Some((pid, unsafe { OwnedFd::from_raw_fd(raw) }))
1483 }
1484 }
1485 (None, None) => None,
1486 };
1487 if sched_pid.is_some() && exact_scheduler.is_none() {
1488 // The caller observed an owner immediately before this clone and a
1489 // concurrent orderly transition removed it. Treat that exact owner as
1490 // terminal rather than opening whichever task now owns the old number.
1491 return true;
1492 }
1493
1494 if dur.is_zero() {
1495 if let Some(heartbeat) = workload_heartbeat {
1496 heartbeat.publish();
1497 }
1498 return crashed()
1499 || (exact_scheduler.is_some() && dispatch::scx_down())
1500 || exact_scheduler.as_ref().is_some_and(|(_, pidfd)| {
1501 !matches!(crate::vmm::rust_init::pidfd_is_alive(pidfd), Ok(true))
1502 });
1503 }
1504 let deadline = std::time::Instant::now() + dur;
1505 let Some((pid, pidfd)) = exact_scheduler else {
1506 // No scheduler pid (host-only run, or the pid was not recorded):
1507 // no pidfd to wait on, but the err-exit latch still fires on a
1508 // crash — poll it in short intervals instead of sleeping blind
1509 // for the whole window.
1510 loop {
1511 if let Some(heartbeat) = workload_heartbeat {
1512 heartbeat.publish();
1513 }
1514 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1515 if remaining.is_zero() {
1516 return crashed();
1517 }
1518 if crashed() {
1519 return true;
1520 }
1521 thread::sleep(remaining.min(ERR_EXIT_POLL));
1522 }
1523 };
1524
1525 // epoll setup. EPOLL_CLOEXEC matches `wait_with_deadline` to
1526 // avoid leaking the epoll fd into any post-fork descendant.
1527 let epoll = match Epoll::new(EpollCreateFlags::EPOLL_CLOEXEC) {
1528 Ok(e) => e,
1529 Err(e) => {
1530 let fd = std::os::fd::AsRawFd::as_raw_fd(&pidfd);
1531 panic_evented_hold_defect(
1532 "epoll_create1(EPOLL_CLOEXEC)",
1533 pid,
1534 format_args!("{e} (pidfd={fd})"),
1535 "epoll has been universally available since 2.6; failure = \
1536 env defect — check ulimit -n and CONFIG_EPOLL",
1537 );
1538 }
1539 };
1540 // `data` field is unused — we only ever watch one fd. The add()
1541 // syscall still needs an `EpollEvent` with populated events.
1542 let event = EpollEvent::new(EpollFlags::EPOLLIN, 0);
1543 if let Err(e) = epoll.add(pidfd.as_fd(), event) {
1544 panic_evented_hold_defect(
1545 "epoll_ctl(ADD)",
1546 pid,
1547 e,
1548 "epoll_ctl(ADD) on a freshly-opened pidfd should never fail \
1549 in a healthy kernel; documented errors (EBADF/EEXIST/ENOMEM) \
1550 are env defects — likely fd exhaustion or memory pressure",
1551 );
1552 }
1553
1554 let mut events = [EpollEvent::empty()];
1555 loop {
1556 if let Some(heartbeat) = workload_heartbeat {
1557 heartbeat.publish();
1558 }
1559 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1560 if remaining.is_zero() {
1561 // Hold elapsed without a wakeup. Re-probe ALL signals to
1562 // catch a race where the pid exited, the err-exit latch
1563 // flipped, or sched_ext went down (sysfs) between the last
1564 // `epoll_wait` return and the deadline check (e.g. during
1565 // EINTR re-entry). `pid` is `Some` here, so the
1566 // scheduler-expected guard for `scx_down` is satisfied.
1567 return crashed()
1568 || dispatch::scx_down()
1569 || !matches!(crate::vmm::rust_init::pidfd_is_alive(&pidfd), Ok(true));
1570 }
1571
1572 // Abort at the crash, not at the lingering process exit. Two
1573 // probe-independent signals fire well before a crashed
1574 // scheduler's process exits: the BPF err-exit latch (live only
1575 // in the auto-repro/dump VM) and `/sys/kernel/sched_ext/state`
1576 // going `disabling`/`disabled` (live in the PRIMARY VM, where the
1577 // latch mirror is never populated — this is what makes a primary
1578 // VM crash abort the hold at crash time rather than ~1s later at
1579 // process exit). `pid` is `Some` here, so the
1580 // scheduler-expected guard for `scx_down` holds.
1581 if crashed() || dispatch::scx_down() {
1582 return true;
1583 }
1584
1585 // Cap the epoll timeout at the latch poll interval so the latch
1586 // is re-checked every `ERR_EXIT_POLL`, not only when the pidfd
1587 // becomes readable. `PollTimeout` (aliased as `EpollTimeout`)
1588 // stores the value as `i32`; single-pass clamp via
1589 // `u128 → i32::MAX` so a large remainder saturates at the max
1590 // accepted value instead of overflowing through the u32.
1591 let poll = remaining.min(ERR_EXIT_POLL);
1592 let ms_i32 = poll.as_millis().min(i32::MAX as u128) as i32;
1593 let timeout_param = match EpollTimeout::try_from(ms_i32) {
1594 Ok(t) => t,
1595 Err(e) => {
1596 // ms_i32 was clamped to the i32 range above so
1597 // EpollTimeout::try_from (which accepts i32) cannot
1598 // overflow. Reaching this arm means the EpollTimeout API
1599 // changed shape — code defect, not env transient.
1600 panic_evented_hold_defect(
1601 "EpollTimeout::try_from",
1602 pid,
1603 format_args!("{e} (input={ms_i32})"),
1604 "input was pre-clamped to fit i32; failure indicates an \
1605 upstream nix EpollTimeout API change requiring code update",
1606 );
1607 }
1608 };
1609
1610 match epoll.wait(&mut events, timeout_param) {
1611 Ok(0) => {
1612 // Poll-interval timeout, no pidfd event. Loop back to
1613 // re-check the err-exit latch and the deadline at the
1614 // top of the loop.
1615 }
1616 Ok(_) => {
1617 // pidfd became readable — task transitioned to
1618 // EXIT_ZOMBIE. Scheduler is dead.
1619 return true;
1620 }
1621 Err(nix::errno::Errno::EINTR) => {
1622 // Signal interrupted the wait; loop and re-compute
1623 // the remaining window.
1624 }
1625 Err(e) => {
1626 panic_evented_hold_defect(
1627 "epoll_wait",
1628 pid,
1629 e,
1630 "epoll_wait on a freshly-created epoll with a single \
1631 valid pidfd cannot legitimately fail outside EINTR; \
1632 documented errors (EBADF/EFAULT/EINVAL) are framework- \
1633 internal memory-safety defects — investigate concurrent \
1634 fd mutation or stack-frame corruption",
1635 );
1636 }
1637 }
1638 }
1639}
1640
1641/// Run a single step's setup + ops + hold against step-local state.
1642///
1643/// On error, the caller is expected to invoke `collect_step` for
1644/// per-step teardown (which runs regardless) and then propagate.
1645///
1646/// `sched_died_during_hold` is set to `true` when the hold-period
1647/// liveness poll observes the scheduler process exiting; the caller
1648/// uses this to emit [`crate::assert::format_sched_died_during_workload`]
1649/// instead of waiting for the post-loop probe to fire (which would
1650/// stamp the message with the full scenario duration even though
1651/// the death happened mid-step).
1652#[allow(clippy::too_many_arguments)]
1653fn run_step<'a>(
1654 ctx: &Ctx,
1655 step: &Step,
1656 step_idx: usize,
1657 step_state: &mut StepState<'a>,
1658 backdrop_state: &mut BackdropState<'a>,
1659 scenario_start: std::time::Instant,
1660 _effective_checks: &crate::assert::Assert,
1661 sched_died_during_hold: &mut bool,
1662 // Set to (scenario-relative elapsed_ms, cumulative worker iteration
1663 // count) sampled coincidently at this step's clean completion
1664 // Left unchanged on the sched-died early-return paths so
1665 // a failed step does not overwrite a prior step's good value with a
1666 // misleading post-death count.
1667 final_total_iterations: &mut Option<(u64, u64)>,
1668 workload_heartbeat: Option<WorkloadHeartbeat>,
1669) -> Result<()> {
1670 let mut scenario = ScenarioState::new(step_state, backdrop_state);
1671
1672 // Any `?` out of apply_ops / apply_setup would bypass the
1673 // per-step teardown ordering; `drain_on_err!` kills payload
1674 // handles across step + backdrop (metric-emitting) before
1675 // propagating so a mid-scenario spawn failure still leaves a
1676 // usable sidecar.
1677 macro_rules! drain_on_err {
1678 ($scenario:expr, $e:expr) => {
1679 match $e {
1680 Ok(v) => v,
1681 Err(err) => {
1682 $scenario.drain_all_payloads();
1683 return Err(err);
1684 }
1685 }
1686 };
1687 }
1688
1689 match step.hold {
1690 HoldSpec::Loop { interval } => {
1691 // Setup runs once before the loop.
1692 if !step.setup.is_empty() {
1693 let defs = step.setup.resolve(ctx);
1694 drain_on_err!(scenario, apply_setup(ctx, &mut scenario, &defs));
1695 }
1696 // Emit the Loop step's start stimulus frame + resume, like
1697 // the non-Loop arm below. A Loop step is a phase like any
1698 // other: without its own start frame the synthesized
1699 // scenario-end terminal would pair against the
1700 // PRIOR step's frame and misattribute the entire Loop
1701 // window's throughput to the wrong step. The resume
1702 // re-enables periodic captures for the loop body and matches
1703 // the prior step's `run_steps` pause (same pause/resume
1704 // pairing the non-Loop arm uses; the first-step unmatched
1705 // resume is already the existing pattern).
1706 if guest_comms::is_guest() {
1707 let payload = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1708 crate::vmm::guest_comms::send_stimulus(zerocopy::IntoBytes::as_bytes(&payload));
1709 crate::vmm::guest_comms::send_scenario_resume();
1710 }
1711 // Loop mode: apply ops repeatedly at interval until
1712 // the remaining scenario time is exhausted, or the
1713 // scheduler process exits — whichever fires first.
1714 let deadline = scenario_start + ctx.duration;
1715 while std::time::Instant::now() < deadline {
1716 drain_on_err!(scenario, apply_ops(ctx, &mut scenario, &step.ops, true));
1717 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
1718 // Live `sched_pid()` read so a mid-loop
1719 // Op::ReplaceScheduler swap is watched at the NEW
1720 // pid, not the stale boot snapshot in ctx.
1721 if hold_or_sched_died(
1722 remaining.min(interval),
1723 crate::vmm::rust_init::sched_pid(),
1724 workload_heartbeat,
1725 ) {
1726 *sched_died_during_hold = true;
1727 return Ok(());
1728 }
1729 }
1730 }
1731 _ => {
1732 // Ops first (e.g. parent cgroup creation), then
1733 // CgroupDef setup (children with workers).
1734 //
1735 // Footgun: a workload-producing `CgroupDef` in
1736 // `step.setup` is invisible to `step.ops` operating on
1737 // it, because step.ops runs BEFORE apply_setup creates
1738 // the cgroup AND registers the WorkloadHandle in
1739 // `state.all_handles()`. The failure mode is per-Op:
1740 // `Op::MoveAllTasks` against the absent source filters
1741 // `state.all_handles()` by name, finds zero matches,
1742 // iterates an empty `pid_batches`, and exits with no
1743 // work done — silent (no error surfaced to the
1744 // operator); `Op::CaptureCgroupProcs` reading the
1745 // missing `cgroup.procs` via `read_procs` returns the
1746 // ENOENT `with_context` and propagates `?`, bailing the
1747 // ops phase — loud (the test crashes with the cgroup
1748 // name in the error). Tests that need a freshly-spawned
1749 // worker pool to feed step ops must declare the
1750 // producing `CgroupDef` in the Backdrop (which runs
1751 // before any Step) when `step.setup` is `Setup::Defs(_)`.
1752 // When `step.setup` is `Setup::Factory(_)` (runtime-
1753 // generated defs that can't be hoisted at edit time),
1754 // the only viable remediation is restructuring into
1755 // multiple Steps — producer factory in Step N's setup,
1756 // consumer ops in Step N+1.
1757 drain_on_err!(scenario, apply_ops(ctx, &mut scenario, &step.ops, false));
1758 if !step.setup.is_empty() {
1759 let defs = step.setup.resolve(ctx);
1760 drain_on_err!(scenario, apply_setup(ctx, &mut scenario, &defs));
1761 }
1762
1763 // Write stimulus event after applying ops. Routes through
1764 // `crate::vmm::guest_comms::send_stimulus` (virtio-console
1765 // port-1 bulk channel). `is_guest` keeps the
1766 // `build_stimulus` walk off the host where the write would
1767 // no-op.
1768 if guest_comms::is_guest() {
1769 let payload = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1770 crate::vmm::guest_comms::send_stimulus(zerocopy::IntoBytes::as_bytes(&payload));
1771 }
1772
1773 if guest_comms::is_guest() {
1774 crate::vmm::guest_comms::send_scenario_resume();
1775 }
1776 let hold_dur = match step.hold {
1777 HoldSpec::Frac(f) => Duration::from_secs_f64(ctx.duration.as_secs_f64() * f),
1778 HoldSpec::Fixed(d) => d,
1779 HoldSpec::Loop { .. } => unreachable!(),
1780 };
1781 let remaining = (scenario_start + ctx.duration)
1782 .saturating_duration_since(std::time::Instant::now());
1783 let hold_dur = hold_dur.min(remaining);
1784 // Live `sched_pid()` read — matches the loop arm above
1785 // so the hold watches the post-Op::ReplaceScheduler
1786 // pid, not the stale boot snapshot.
1787 if hold_or_sched_died(
1788 hold_dur,
1789 crate::vmm::rust_init::sched_pid(),
1790 workload_heartbeat,
1791 ) {
1792 *sched_died_during_hold = true;
1793 return Ok(());
1794 }
1795 }
1796 }
1797
1798 // Capture the cumulative iteration count AND its scenario-relative
1799 // timestamp at this step's end, AT THE SAME INSTANT, while the
1800 // step's workers are still alive — `collect_step` tears them down
1801 // right after `run_step` returns, so this is the last moment a
1802 // step-inclusive sum is observable. `run_steps` forwards the LAST
1803 // step's (elapsed_ms, iterations) pair in the widened `ScenarioEnd`
1804 // frame. Sample both HERE so the terminal telemetry describes one
1805 // coincident endpoint rather than mixing a pre-teardown count with a
1806 // post-teardown clock. Only reached on a clean hold; gated on `is_guest`
1807 // so the host-side scenario walk does not sum a handle set that never ran.
1808 if guest_comms::is_guest() {
1809 let end_elapsed_ms = scenario_start.elapsed().as_millis() as u64;
1810 let end_iterations: u64 = scenario
1811 .all_handles()
1812 .flat_map(|(_, h)| h.snapshot_iterations())
1813 .sum();
1814 *final_total_iterations = Some((end_elapsed_ms, end_iterations));
1815
1816 // Emit a per-step StepEnd frame carrying this step's coincident
1817 // end-of-hold (elapsed_ms, total_iterations) and the SAME
1818 // 1-indexed step_index as its StepStart. This preserves raw step-local
1819 // boundary telemetry; the canonical iteration rate comes from the
1820 // CPU-time carrier. build_stimulus supplies the op/cgroup/worker fields
1821 // + the step index; override its recomputed elapsed/iterations with the
1822 // values captured above so the StepEnd pair stays coincident with
1823 // `final_total_iterations` (no pause/teardown wall-time creep).
1824 let mut step_end = build_stimulus(&scenario_start, step_idx, &step.ops, &scenario);
1825 step_end.elapsed_ms = u32::try_from(end_elapsed_ms).unwrap_or(u32::MAX);
1826 step_end.total_iterations = end_iterations;
1827 guest_comms::send_step_end(zerocopy::IntoBytes::as_bytes(&step_end));
1828 }
1829
1830 Ok(())
1831}
1832
1833/// Build a StimulusPayload from the current scenario state (step + backdrop).
1834///
1835/// # step_idx u16 saturation
1836///
1837/// `step_idx` is a `usize` on the caller side but the wire
1838/// `StimulusPayload.step_index` is a `u16` — the slot is sized for
1839/// realistic scenarios (≤ 65 536 distinct indices, `0..=u16::MAX`).
1840/// Any `step_idx` > `u16::MAX as usize` is clamped to `u16::MAX` by
1841/// `to_u16` below, with a `tracing::warn!` that names the overflow.
1842/// Downstream consumers of the StepStart wire frame therefore see
1843/// every step past index `u16::MAX` collapsed onto the same
1844/// `step_index` value (`u16::MAX`) — the ordering is preserved for
1845/// the first 65 536 steps (indices `0..=u16::MAX`), but labels
1846/// saturate and become ambiguous once the scenario crosses the
1847/// boundary. Scenarios that need to distinguish individual steps
1848/// past `u16::MAX` must widen the wire schema field; the
1849/// saturating-clip preserves visible wake ordering at the cost of
1850/// individuality in the deep tail.
1851fn build_stimulus(
1852 scenario_start: &std::time::Instant,
1853 step_idx: usize,
1854 ops: &[Op],
1855 state: &ScenarioState<'_, '_>,
1856) -> StimulusPayload {
1857 let mut op_kinds: u32 = 0;
1858 for op in ops {
1859 op_kinds |= 1 << op.discriminant();
1860 }
1861
1862 let total_iterations: u64 = state
1863 .all_handles()
1864 .flat_map(|(_, h)| h.snapshot_iterations())
1865 .sum();
1866
1867 let cgroup_count = state.step.cgroups.names().len() + state.backdrop.cgroups.names().len();
1868 let worker_count = state.step.handles.len() + state.backdrop.handles.len();
1869
1870 // Saturate narrowing conversions for the wire schema: the
1871 // StimulusPayload fields are sized for realistic scenarios
1872 // (u32 ms, u16 counts) but `as u32` / `as u16` silently
1873 // wrap on overflow, poisoning downstream consumers. Log the
1874 // overflow so the operator sees which field exceeded its
1875 // bound and substitute MAX — clipped-high is a safer wire
1876 // value than silently wrapping to a small number.
1877 let to_u32 = |field: &str, v: u128| -> u32 {
1878 u32::try_from(v).unwrap_or_else(|_| {
1879 tracing::warn!(
1880 field,
1881 value = %v,
1882 "StimulusPayload field overflowed u32; saturating to u32::MAX",
1883 );
1884 u32::MAX
1885 })
1886 };
1887 let to_u16 = |field: &str, v: usize| -> u16 {
1888 u16::try_from(v).unwrap_or_else(|_| {
1889 tracing::warn!(
1890 field,
1891 value = v,
1892 "StimulusPayload field overflowed u16; saturating to u16::MAX",
1893 );
1894 u16::MAX
1895 })
1896 };
1897
1898 // Encode the 1-indexed phase number per the framework's
1899 // phase convention -- the BASELINE (pre-first-Step) window owns
1900 // 0, scenario Step k publishes k + 1. Saturate at u16::MAX
1901 // (rather than wrap) so a pathological 65k-step scenario still
1902 // produces a clipped-high value the host parser can recognise
1903 // instead of silently rolling over.
1904 let phase_step_index: u16 = u16::try_from(step_idx)
1905 .ok()
1906 .and_then(|i| i.checked_add(1))
1907 .unwrap_or_else(|| {
1908 tracing::warn!(
1909 field = "step_index",
1910 value = step_idx,
1911 "StimulusPayload step_index overflowed u16 after 1-indexed encoding; saturating to u16::MAX",
1912 );
1913 u16::MAX
1914 });
1915 StimulusPayload {
1916 elapsed_ms: to_u32("elapsed_ms", scenario_start.elapsed().as_millis()),
1917 step_index: phase_step_index,
1918 op_count: to_u16("op_count", ops.len()),
1919 op_kinds,
1920 cgroup_count: to_u16("cgroup_count", cgroup_count),
1921 worker_count: to_u16("worker_count", worker_count),
1922 total_iterations,
1923 }
1924}
1925
1926/// Validate that a MemPolicy's node set is consistent with the
1927/// cgroup's scenario intent — the cpuset the cgroup runs in and
1928/// the host topology.
1929///
1930/// # Empty-nodemask early return
1931///
1932/// Policies with no nodemask — [`MemPolicy::Default`] and
1933/// [`MemPolicy::Local`] — carry no node IDs to validate against,
1934/// so this function returns `Ok(())` unconditionally for them
1935/// (after the unknown-bit and mutual-exclusion flag guards run).
1936/// Every other variant — any variant carrying a nodemask,
1937/// currently [`MemPolicy::Bind`], [`MemPolicy::Preferred`],
1938/// [`MemPolicy::PreferredMany`], [`MemPolicy::Interleave`], and
1939/// [`MemPolicy::WeightedInterleave`] — reaches the cpuset /
1940/// host-topology coverage logic below.
1941///
1942/// # Why this is a scenario-intent check, not a kernel guard
1943///
1944/// ktstr writes `cpuset.cpus` on each cgroup but never writes
1945/// `cpuset.mems`, so `cpuset.mems` keeps its inherited default —
1946/// the permissive "all nodes" set in every ktstr deployment
1947/// shape (PID 1 inside the guest VM, cgroup root on the host).
1948/// The kernel's `set_mempolicy(2)` path always runs the policy's
1949/// nodemask through `mpol_set_nodemask` in `mm/mempolicy.c`, which
1950/// intersects it with the caller's `mems_allowed` before it is
1951/// stored on the task; because ktstr never narrows `mems_allowed`,
1952/// that intersection is an identity operation under ktstr's
1953/// deployment — the stored nodemask equals the one the caller
1954/// supplied, and the kernel never rejects or silently trims the
1955/// policy the way it would if `mems_allowed` were disjoint from
1956/// the requested set. Rejection of a mismatched policy is
1957/// therefore validator-only: if this function does not bail, the
1958/// policy lands on the syscall unchanged and `run_steps` commits
1959/// to running the worker with a misconfigured allocation target.
1960///
1961/// What the validator catches is a **scenario-design mismatch**:
1962/// you pinned CPUs on NUMA node X (via `CpusetSpec::Numa(X)`) but
1963/// asked the mempolicy to bind/prefer/interleave a disjoint node Y,
1964/// meaning the worker's compute is local to node X while its
1965/// allocations live on node Y — producing cross-socket traffic
1966/// that the test author almost certainly did not intend. Surface
1967/// the mismatch here before `run_steps` commits to the policy.
1968///
1969/// `MpolFlags::STATIC_NODES` is the rebind-behavior flag. Two
1970/// kernel sites encode the semantics: `mpol_set_nodemask` in
1971/// `mm/mempolicy.c` consumes the flag during policy creation (it
1972/// determines whether the supplied nodemask is stored absolute or
1973/// remapped against the caller's cpuset at install time), and
1974/// `mpol_rebind_policy` (same file) branches on the flag when the
1975/// cpuset's `mems_allowed` changes after the policy was installed
1976/// — with `STATIC_NODES` set, the stored nodemask is unchanged;
1977/// without it, the kernel remaps the nodemask against the new
1978/// `mems_allowed`. Since ktstr never rebinds `cpuset.mems` mid-run,
1979/// only the install-time semantics applies, and the flag is
1980/// effectively a cross-node-intent declaration for the validator's
1981/// purposes — a sign the author knows the intent is "allocations on
1982/// a node outside the CPU-affinity cpuset" and has opted in to
1983/// that shape.
1984///
1985/// # Flag-specific handling (in order of evaluation)
1986///
1987/// - `STATIC_NODES | RELATIVE_NODES` both set → bail: the kernel
1988/// rejects this combination with `EINVAL`; surfacing it here
1989/// names the offender before the syscall.
1990/// - `STATIC_NODES` only → the caller has declared intentional
1991/// cross-node placement. Skip the cpuset-intent check, but each
1992/// referenced node must exist on the host topology or the
1993/// kernel will reject the policy. Verify existence; bail with
1994/// the missing nodes if any.
1995/// - `RELATIVE_NODES` only → the nodemask is an ordinal into the
1996/// cpuset's allowed-nodes set. Cpuset coverage does not apply in
1997/// absolute-id terms, so bypass.
1998/// - No relevant flag set → enforce cpuset-intent coverage:
1999/// every policy node must appear in the cpuset's covered NUMA
2000/// nodes. Bail naming the uncovered nodes AND both escape
2001/// hatches (STATIC_NODES opt-in; widening the cpuset).
2002///
2003/// Reject `--flag` args whose bare name is not in the payload's
2004/// `known_flags` allowlist. Returns `Ok(())` when the payload
2005/// declared no allowlist (`known_flags: None`) — the opt-in
2006/// contract defaults to "permissive" so payloads wrapping
2007/// open-ended binaries (stress-ng, fio, schbench) aren't forced
2008/// to enumerate every flag their upstream tool accepts.
2009///
2010/// Recognises two flag shapes: `--foo` (flag-only) and
2011/// `--foo=value` (flag-with-attached-value). Non-flag args
2012/// (positional, `-short`, everything else) are passed through
2013/// without inspection — the allowlist scopes to long flags only.
2014///
2015/// Extracted out of `apply_ops`'s `Op::RunPayload` arm so the
2016/// validation is unit-testable without standing up a full Ctx
2017/// / scenario state. See the caller for how the allowlist is
2018/// threaded through Op::RunPayload execution.
2019fn validate_known_flags(payload: &crate::test_support::Payload, args: &[String]) -> Result<()> {
2020 let Some(allowlist) = payload.known_flags else {
2021 return Ok(());
2022 };
2023 for arg in args {
2024 let Some(flag_body) = arg.strip_prefix("--") else {
2025 continue;
2026 };
2027 // `split('=').next()` is infallible: `str::split` always
2028 // yields at least one element (the full string when no
2029 // separator is present). The prior `unwrap_or("")` fallback
2030 // was dead code — the empty-name branch below never fired
2031 // via this path since `flag_body` had already passed the
2032 // `strip_prefix("--")` filter above (leaving at least one
2033 // character). Kept the `name.is_empty()` guard in place
2034 // only to handle the degenerate `"--"` bare-dashes case,
2035 // which produces `flag_body = ""` → `name = ""`.
2036 let name = flag_body
2037 .split('=')
2038 .next()
2039 .expect("str::split always yields at least one element");
2040 if name.is_empty() {
2041 continue;
2042 }
2043 if !allowlist.contains(&name) {
2044 anyhow::bail!(
2045 "Op::RunPayload: payload '{}' received unknown flag \
2046 '--{name}' — not in its known_flags allowlist \
2047 {allowlist:?}. Check the spelling against the \
2048 payload's declared flags; if '--{name}' is a new \
2049 legitimate flag, add it to `Payload::known_flags`.",
2050 payload.name,
2051 );
2052 }
2053 }
2054 Ok(())
2055}
2056
2057fn validate_mempolicy_cpuset(
2058 policy: &MemPolicy,
2059 flags: crate::workload::MpolFlags,
2060 cpuset: &BTreeSet<usize>,
2061 ctx: &Ctx,
2062 cgroup_name: &str,
2063) -> Result<()> {
2064 use crate::workload::MpolFlags;
2065
2066 // Reject unknown bits before any other check. The `MpolFlags`
2067 // type is a `u32` bitfield covering three documented bits
2068 // (STATIC_NODES, RELATIVE_NODES, NUMA_BALANCING); any other bit
2069 // set in `flags` is either a user typo (raw-constructing the
2070 // struct with an arbitrary integer) or forward-compat from a
2071 // future kernel flag that this validator hasn't learned yet.
2072 // Either way, surfacing unknown bits here prevents a silent
2073 // semantic mismatch — the kernel would either reject with
2074 // EINVAL or (worse) treat the bit as a flag we don't model.
2075 let known_bits = MpolFlags::STATIC_NODES.bits()
2076 | MpolFlags::RELATIVE_NODES.bits()
2077 | MpolFlags::NUMA_BALANCING.bits();
2078 let unknown_bits = flags.bits() & !known_bits;
2079 if unknown_bits != 0 {
2080 anyhow::bail!(
2081 "cgroup '{}': MpolFlags contains unknown bit(s) {:#x} (known bits: \
2082 STATIC_NODES={:#x}, RELATIVE_NODES={:#x}, NUMA_BALANCING={:#x}); \
2083 refusing to forward to the kernel — update MpolFlags to model the \
2084 new bit before using it, or clear the bit at the call site",
2085 cgroup_name,
2086 unknown_bits,
2087 MpolFlags::STATIC_NODES.bits(),
2088 MpolFlags::RELATIVE_NODES.bits(),
2089 MpolFlags::NUMA_BALANCING.bits(),
2090 );
2091 }
2092
2093 // `STATIC_NODES | RELATIVE_NODES` is a kernel-rejected combination —
2094 // `MPOL_F_STATIC_NODES` and `MPOL_F_RELATIVE_NODES` are mutually
2095 // exclusive (see `include/uapi/linux/mempolicy.h` + the
2096 // `sanitize_mpol_flags` helper in `mm/mempolicy.c`, which bails
2097 // with `EINVAL` if both are set). Fail early here instead of
2098 // letting the syscall return a generic error — the scenario
2099 // caller almost certainly meant one or the other, not both.
2100 if flags.contains(MpolFlags::STATIC_NODES) && flags.contains(MpolFlags::RELATIVE_NODES) {
2101 anyhow::bail!(
2102 "cgroup '{}': MpolFlags::STATIC_NODES and MpolFlags::RELATIVE_NODES are \
2103 mutually exclusive (the kernel will reject the set_mempolicy syscall with \
2104 EINVAL); pick whichever matches the intended semantics — STATIC_NODES \
2105 for absolute node ids that survive cpuset changes, RELATIVE_NODES for \
2106 cpuset-relative indices",
2107 cgroup_name,
2108 );
2109 }
2110
2111 let policy_nodes = policy.node_set();
2112 if policy_nodes.is_empty() {
2113 return Ok(());
2114 }
2115
2116 // `STATIC_NODES`: nodemask is treated as absolute node ids and NOT
2117 // intersected with the cpuset. The cpuset-coverage check below
2118 // does not apply, but we DO need to verify the referenced nodes
2119 // actually exist on the host — a policy pinning node 7 on a
2120 // 2-node host would fail at syscall time; surfacing it here
2121 // names the offender.
2122 if flags.contains(MpolFlags::STATIC_NODES) {
2123 let host_nodes = ctx.topo.numa_node_ids();
2124 let missing: Vec<usize> = policy_nodes
2125 .iter()
2126 .copied()
2127 .filter(|n| !host_nodes.contains(n))
2128 .collect();
2129 if !missing.is_empty() {
2130 anyhow::bail!(
2131 "cgroup '{}': MemPolicy with MpolFlags::STATIC_NODES references \
2132 NUMA node(s) {:?} that do not exist on this host (host nodes: {:?}); \
2133 the kernel will reject or silently drop the policy (Preferred can \
2134 silently fall back to local allocation; Bind/Interleave reject with \
2135 EINVAL) — fix the MemPolicy or pick a host with the required nodes",
2136 cgroup_name,
2137 missing,
2138 host_nodes,
2139 );
2140 }
2141 return Ok(());
2142 }
2143
2144 // `RELATIVE_NODES`: nodemask is an ordinal into the cpuset's
2145 // allowed nodes, not an absolute node id set. The cpuset-coverage
2146 // check compares absolute ids, so it does not apply here — the
2147 // kernel does the relative-to-absolute remap internally. Trust
2148 // the caller and bypass the coverage bail, same shape as the
2149 // STATIC_NODES early return.
2150 if flags.contains(MpolFlags::RELATIVE_NODES) {
2151 return Ok(());
2152 }
2153
2154 let cpuset_numa = ctx.topo.numa_nodes_for_cpuset(cpuset);
2155 let uncovered: Vec<usize> = policy_nodes
2156 .iter()
2157 .copied()
2158 .filter(|n| !cpuset_numa.contains(n))
2159 .collect();
2160 if !uncovered.is_empty() {
2161 anyhow::bail!(
2162 "cgroup '{}': MemPolicy references NUMA node(s) {:?} \
2163 outside the cpuset's coverage (cpuset covers node(s) \
2164 {:?}) — some or all of the worker's allocations would \
2165 live on NUMA nodes its CPUs cannot reach locally, \
2166 producing cross-socket allocation traffic that is \
2167 almost certainly unintended. Two fixes: \
2168 (a) add .mpol_flags(MpolFlags::STATIC_NODES) to \
2169 declare the cross-node placement intentional (the \
2170 flag survives cpuset rebinds; see MpolFlags doc), or \
2171 (b) widen the cpuset to cover the policy's nodes \
2172 (e.g. CpusetSpec::Numa(N) for each referenced N, or \
2173 a CpusetSpec::Exact set that spans both).",
2174 cgroup_name,
2175 uncovered,
2176 cpuset_numa,
2177 );
2178 }
2179 Ok(())
2180}
2181
2182/// SIGKILL every worker in `handles` without reaping. Used only on
2183/// the scheduler-death paths in [`run_scenario`]: when the scheduler
2184/// under test dies, the kernel disables sched_ext and its workers
2185/// fall back to the builtin scheduler, so a CPU-bound worker pool
2186/// would otherwise make the per-worker `waitpid` reap in the
2187/// subsequent [`collect_step`] / [`collect_backdrop`]
2188/// scheduling-gated (teardown time scaling with worker count).
2189/// Delivering SIGKILL to every worker first lets that reap find
2190/// already-exiting workers. See
2191/// [`crate::workload::WorkloadHandle::sigkill_workers`].
2192fn sigkill_handles(handles: &[(String, WorkloadHandle)]) {
2193 for (_, h) in handles {
2194 h.sigkill_workers();
2195 }
2196}
2197
2198/// Collect step-local worker results and produce an AssertResult.
2199///
2200/// Drains step-local handles + payload handles; backdrop state is
2201/// untouched. Called at every step boundary (success AND error
2202/// paths) as the "Step is fully bounded" teardown. The
2203/// `step_state` goes out of scope at the end of this step's
2204/// iteration, so its `CgroupGroup` drop removes every step-local
2205/// cgroup immediately after `run_scenario` propagates the result
2206/// of this call.
2207///
2208/// Before draining handles, every step-local cgroup is unfrozen
2209/// (`cgroup.freeze` ← 0). An [`Op::FreezeCgroup`] without a paired
2210/// [`Op::UnfreezeCgroup`] would leave step-local tasks frozen at
2211/// step boundary, where the graceful cooperative stop cannot make
2212/// progress: the worker stop is a non-fatal SIGUSR1, and a frozen
2213/// task re-enters the freezer trap on a non-fatal signal
2214/// (`kernel/signal.c` `do_freezer_trap`) rather than running its
2215/// handler — so it never flips its stop flag or reports until
2216/// thawed, and `stop_and_collect` burns its full collection
2217/// deadline before falling back to a sentinel report. (A fatal
2218/// SIGKILL DOES wake and kill a frozen task — it diverges before
2219/// the freezer trap — so worker death and the subsequent rmdir are
2220/// unaffected by the freeze; the unfreeze only restores the
2221/// graceful path's ability to collect real reports.) Failures are
2222/// logged at warn level only — a missing freezer file or a cgroup
2223/// that was already torn down is benign at teardown time, and
2224/// propagating would mask the real workload result.
2225fn collect_step(
2226 step_state: &mut StepState<'_>,
2227 checks: &crate::assert::Assert,
2228 topo: &crate::topology::TestTopology,
2229 cgroups: &dyn crate::cgroup::CgroupOps,
2230 // The 1-indexed phase step_index (BASELINE=0, Step k -> k+1) this
2231 // teardown belongs to, threaded so collect_handles can attach the per-phase
2232 // per_cgroup carrier. `None` for the defensive backdrop-staging collect,
2233 // which has no step attribution.
2234 step_index: Option<u16>,
2235) -> AssertResult {
2236 // Unfreeze every step-local cgroup before the graceful collect.
2237 // A frozen worker re-enters the freezer trap on the cooperative
2238 // (non-fatal) SIGUSR1 stop instead of running its handler, so it
2239 // never reports until thawed. (SIGKILL would still kill it; only
2240 // the graceful report path needs the thaw.)
2241 for name in step_state.cgroups.names() {
2242 if let Err(e) = cgroups.set_freeze(name, false) {
2243 tracing::warn!(
2244 cgroup = %name,
2245 err = %format!("{e:#}"),
2246 "collect_step: pre-teardown unfreeze failed; any frozen workers will yield sentinel reports (still SIGKILL-reaped, no leak)"
2247 );
2248 }
2249 }
2250 // Kill any CgroupDef::workload / Op::RunPayload payload binaries
2251 // still live at step teardown so cgroupfs cleanup does not trip
2252 // EBUSY. Metrics are emitted to the SHM ring by PayloadHandle::kill
2253 // via the `evaluate()` pipeline.
2254 drain_all_payload_handles(&mut step_state.payload_handles);
2255 // Drain any host-mode stuck reports that accumulated during the
2256 // step BEFORE dropping the monitor handle (Drop joins the
2257 // polling thread). Reports get folded into the merged
2258 // [`AssertResult`] below as
2259 // [`crate::assert::DetailKind::WorkerStuck`] failures. `take`
2260 // ensures the handle drops here (joining the thread) so the
2261 // polling thread exits before per-step teardown returns.
2262 let stuck_reports = if let Some(handle) = step_state.stuck_monitor.take() {
2263 let reports = handle.drain();
2264 drop(handle);
2265 reports
2266 } else {
2267 Vec::new()
2268 };
2269 let handles = std::mem::take(&mut step_state.handles);
2270 let mut result = crate::scenario::collect_handles(
2271 handles.into_iter().map(|(name, h)| {
2272 let cpuset = step_state.cpusets.get(&name);
2273 (name, h, cpuset)
2274 }),
2275 checks,
2276 Some(topo),
2277 step_index,
2278 );
2279 for report in stuck_reports {
2280 result.record_fail(crate::assert::AssertDetail::new(
2281 crate::assert::DetailKind::WorkerStuck,
2282 format_stuck_report(&report),
2283 ));
2284 }
2285 result
2286}
2287
2288/// Render a [`crate::scenario::host_stuck::StuckReport`] as a
2289/// human-readable single-string assertion detail. Emits the stuck
2290/// pid + comm, the sample-window summary (first and last values
2291/// for both counters PLUS the computed `last - first` delta — both
2292/// expected to be zero for a true stuck condition but rendered from the
2293/// samples themselves so a predicate-tolerance refactor stays
2294/// observable), and the diagnostic subset (state, wchan, syscall,
2295/// cgroup, host loadavg, optional kernel stack). The diagnostic's
2296/// `status_full` field is intentionally OMITTED — the parsed
2297/// `state` letter carries the actionable signal and the full
2298/// status file is verbose; sidecar consumers keying off
2299/// [`crate::assert::DetailKind::WorkerStuck`] can match on the
2300/// kind discriminator and read the full StuckReport (carries the
2301/// status_full field) without parsing this message.
2302///
2303/// Format is multi-line so the operator can read the trip at a
2304/// glance without parsing structured output.
2305fn format_stuck_report(report: &crate::scenario::host_stuck::StuckReport) -> String {
2306 use std::fmt::Write as _;
2307 let mut s = String::new();
2308 let _ = writeln!(
2309 s,
2310 "worker stuck detected: pid={} comm={:?} (host-mode /proc/<pid>/sched polling)",
2311 report.pid, report.comm,
2312 );
2313 if let (Some(first), Some(last)) = (report.samples.first(), report.samples.last()) {
2314 // Compute deltas from the actual sample values so a
2315 // predicate-tolerance refactor (e.g. "fire on Δ <= N
2316 // switches rather than == 0") stays observable in the
2317 // rendered output. Saturating subtraction handles the
2318 // pathological counter-rollover case without panic.
2319 let nr_delta = last.nr_switches.saturating_sub(first.nr_switches);
2320 let rt_delta = last
2321 .sum_exec_runtime_ns
2322 .saturating_sub(first.sum_exec_runtime_ns);
2323 let _ = writeln!(
2324 s,
2325 " sample window: nr_switches {} -> {} (delta {nr_delta}), sum_exec_runtime_ns {} -> {} (delta {rt_delta}), {} samples",
2326 first.nr_switches,
2327 last.nr_switches,
2328 first.sum_exec_runtime_ns,
2329 last.sum_exec_runtime_ns,
2330 report.samples.len(),
2331 );
2332 }
2333 let d = &report.diagnostic;
2334 let _ = writeln!(s, " state: {}", d.state);
2335 let _ = writeln!(s, " wchan: {}", d.wchan);
2336 let _ = writeln!(s, " syscall: {}", d.syscall);
2337 let _ = writeln!(s, " cgroup: {}", d.cgroup);
2338 let _ = writeln!(s, " host loadavg: {}", d.host_loadavg);
2339 if let Some(stack) = &d.stack {
2340 let _ = writeln!(s, " kernel stack:\n{stack}");
2341 }
2342 s
2343}
2344
2345/// Collect backdrop (persistent) worker results. Called once at
2346/// scenario end after every Step has torn down. The
2347/// `backdrop_state.cgroups` RAII guard drops persistent cgroups
2348/// when `backdrop_state` itself drops.
2349///
2350/// Mirrors [`collect_step`]'s pre-teardown unfreeze pass over every
2351/// tracked cgroup, for the same reason: a backdrop cgroup left
2352/// frozen at scenario end cannot run the graceful cooperative stop.
2353/// A frozen task re-enters the freezer trap on the non-fatal SIGUSR1
2354/// (`kernel/signal.c` `do_freezer_trap`) instead of running its
2355/// handler, so its workers never report until thawed. The asymmetry
2356/// between step-local and backdrop teardown — only the former
2357/// unfreezing — would surface as backdrop workers yielding sentinel
2358/// reports (after burning the collection deadline) on every scenario
2359/// whose Backdrop froze a cgroup and never unfroze it. Symmetric
2360/// unfreeze brings the backdrop path back in line with the per-step
2361/// path. (As in `collect_step`, a fatal SIGKILL still kills a frozen
2362/// worker, so death and the `BackdropState::cgroups` RAII rmdir are
2363/// unaffected by the freeze.)
2364fn collect_backdrop(
2365 backdrop_state: &mut BackdropState<'_>,
2366 checks: &crate::assert::Assert,
2367 topo: &crate::topology::TestTopology,
2368 cgroups: &dyn crate::cgroup::CgroupOps,
2369) -> AssertResult {
2370 // Unfreeze every backdrop cgroup before the graceful collect.
2371 // Same rationale as `collect_step`: a frozen worker re-enters
2372 // the freezer trap on the cooperative (non-fatal) SIGUSR1 stop
2373 // instead of running its handler, so it never reports until
2374 // thawed. (SIGKILL would still kill it; only the graceful report
2375 // path needs the thaw.)
2376 for name in backdrop_state.cgroups.names() {
2377 if let Err(e) = cgroups.set_freeze(name, false) {
2378 tracing::warn!(
2379 cgroup = %name,
2380 err = %format!("{e:#}"),
2381 "collect_backdrop: pre-teardown unfreeze failed; any frozen workers will yield sentinel reports (still SIGKILL-reaped, no leak)"
2382 );
2383 }
2384 }
2385 drain_all_payload_handles(&mut backdrop_state.payload_handles);
2386 let handles = std::mem::take(&mut backdrop_state.handles);
2387 crate::scenario::collect_handles(
2388 handles.into_iter().map(|(name, h)| {
2389 let cpuset = backdrop_state.cpusets.get(&name);
2390 (name, h, cpuset)
2391 }),
2392 checks,
2393 Some(topo),
2394 // Backdrop spans every phase: the None arm expands each worker's
2395 // per-phase PhaseSlices into one per-epoch PhaseBucket
2396 // (expand_backdrop_phase_buckets) — per-epoch, not single-step,
2397 // attribution.
2398 None,
2399 )
2400}
2401
2402/// Kill every payload handle whose cgroup matches `cgroup` and drop
2403/// the matched entries from `handles`. Runs before the cgroup is
2404/// removed or stopped; failures are logged to stderr but do not
2405/// propagate — the cgroup removal is best-effort already, and the
2406/// payload-kill failure is never the primary error.
2407///
2408/// **Metric emission depends on the explicit `.kill()` call** —
2409/// if a future refactor replaces the `.kill()` below with plain
2410/// `drop(handle)`, the `PayloadHandle::drop` SIGKILLs the child
2411/// but skips the evaluate-and-emit pipeline that records metrics
2412/// to the SHM ring. Test helpers that drain payload handles
2413/// likewise route through `drain_all_payload_handles` for the
2414/// same reason. Preserve `.kill()` on every path that claims to
2415/// drain handles for metric capture.
2416///
2417/// Drop order across matched entries is LIFO (last pushed, first
2418/// dropped) — the loop walks indices from the tail toward index 0
2419/// using `Vec::remove` so newer matched entries' embedded
2420/// `SigchldScope`s restore the SIGCHLD disposition before older
2421/// matches do, matching the save-and-restore chain documented on
2422/// `PayloadHandle` in `payload_run.rs`. `Vec::swap_remove` would
2423/// rotate the tail into the freed slot and break LIFO across
2424/// matches; `Vec::remove` preserves the relative order of the
2425/// remaining (unmatched) survivors. Note: SIGCHLD scope LIFO across
2426/// the FULL vec is structurally unsalvageable in any partial-drain
2427/// helper — unmatched entries that stay alive in `handles` outlive
2428/// their younger matched siblings whose scopes already restored.
2429/// The full-vec LIFO contract holds only when every handle is
2430/// dropped together via [`drain_all_payload_handles`].
2431fn drain_payload_handles_for_cgroup(handles: &mut Vec<PayloadEntry>, cgroup: &str) {
2432 let mut i = handles.len();
2433 while i > 0 {
2434 i -= 1;
2435 if handles[i].cgroup.as_str() == cgroup {
2436 let entry = handles.remove(i);
2437 if let Err(e) = entry.handle.kill() {
2438 eprintln!("ktstr: kill payload in cgroup '{cgroup}': {e:#}");
2439 }
2440 }
2441 }
2442}
2443
2444/// Kill every payload handle regardless of cgroup and clear the
2445/// vector. Called at step-sequence teardown so every handle gets a
2446/// terminal `.kill()` (and therefore a sidecar metric emission) even
2447/// when no explicit `RemoveCgroup`/`StopCgroup` op targeted it.
2448///
2449/// Drop order is LIFO (last pushed, first dropped) — `Vec::pop`
2450/// returns the tail first, so `PayloadHandle::drop` runs in reverse
2451/// creation order. Each handle's embedded `SigchldScope` captured the
2452/// `SIGCHLD` disposition that was live at construction time (the
2453/// previous scope's installed `SIG_DFL`). Restoring in LIFO unwinds
2454/// the save-and-restore chain back to the original disposition; FIFO
2455/// drop (e.g. `Vec::drain(..)`) restores intermediate `SIG_DFL` values
2456/// out of order and leaks `SIG_DFL` past the outermost scope. See the
2457/// DROP-ORDER-CRITICAL note on `PayloadHandle` in `payload_run.rs`.
2458fn drain_all_payload_handles(handles: &mut Vec<PayloadEntry>) {
2459 while let Some(entry) = handles.pop() {
2460 if let Err(e) = entry.handle.kill() {
2461 eprintln!(
2462 "ktstr: teardown kill payload in cgroup {}: {e:#}",
2463 render_cgroup_key(&entry.cgroup),
2464 );
2465 }
2466 }
2467}
2468
2469#[cfg(test)]
2470mod tests;
2471
2472#[cfg(test)]
2473mod workers_pct_construction_tests;
2474
2475#[cfg(test)]
2476mod kernel_op_dispatch_tests;