ktstr/workload/spawn/mod.rs
1//! Spawn pipeline: `WorkloadHandle`, `SpawnGuard`, `GroupParams`,
2//! `ThreadWorker`, the report shapes (`WorkerReport`, `WorkerExitInfo`,
3//! `Migration`), and the helpers that thread workers through fork or
4//! `std::thread::spawn`. Split out of `workload/mod.rs` to separate
5//! the production code path from its co-located tests. Tests are
6//! co-located with the production code in topic-grouped sibling
7//! files (`tests_lifecycle`, `tests_grandchild`, `tests_composed`,
8//! ...) that import shared fixtures from `testing.rs` via
9//! `use super::testing::*;`.
10
11use anyhow::{Context, Result};
12use std::collections::{BTreeMap, BTreeSet};
13use std::io::{Read, Write};
14use std::os::unix::io::FromRawFd;
15use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
16use std::time::{Duration, Instant};
17
18use super::affinity::{AffinityIntent, ResolvedAffinity, resolve_affinity, set_thread_affinity};
19use super::config::{CloneMode, MemPolicy, MpolFlags, SchedPolicy, WorkSpec, WorkloadConfig};
20use super::types::*;
21use super::worker::worker_main;
22
23mod cleanup;
24use cleanup::{close_fds_silently, kill_and_killpg};
25
26pub(super) static STOP: AtomicBool = AtomicBool::new(false);
27
28/// A single CPU migration event observed by a worker.
29///
30/// No `Default` impl — a default-constructed Migration would be
31/// `{at_ns: 0, from_cpu: 0, to_cpu: 0}` (a "migration from CPU 0 to
32/// CPU 0 at time 0" is a contradiction: a migration where source ==
33/// dest is not a real migration). Downstream analysis (NUMA migration
34/// counts, scheduler-balance ratios) that assumes `from_cpu !=
35/// to_cpu` would misread default values as real migrations. Construct
36/// every Migration explicitly via [`Migration::new`].
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
38pub struct Migration {
39 /// Nanoseconds since worker start.
40 pub at_ns: u64,
41 /// CPU before migration.
42 pub from_cpu: usize,
43 /// CPU after migration.
44 pub to_cpu: usize,
45}
46
47impl Migration {
48 /// Const constructor for a Migration event. Pair-symmetric with
49 /// the struct fields; lets `Migration::new(ns, from, to)` replace
50 /// `Migration { at_ns: ns, from_cpu: from, to_cpu: to }` in
51 /// hand-built test fixtures.
52 pub const fn new(at_ns: u64, from_cpu: usize, to_cpu: usize) -> Self {
53 Self {
54 at_ns,
55 from_cpu,
56 to_cpu,
57 }
58 }
59}
60
61/// Build a nodemask bitmask and maxnode value for `set_mempolicy(2)`
62/// and `mbind(2)`.
63///
64/// Returns `(nodemask_vec, maxnode)`. The nodemask is a bitmask of
65/// `c_ulong` words where bit N corresponds to NUMA node N. `maxnode`
66/// must be `max_node + 2` because the kernel's `get_nodes()` does
67/// `--maxnode` before reading the bitmask.
68pub fn build_nodemask(nodes: &BTreeSet<usize>) -> (Vec<libc::c_ulong>, libc::c_ulong) {
69 if nodes.is_empty() {
70 return (vec![], 0);
71 }
72 let max_node = nodes.iter().copied().max().unwrap_or(0);
73 let mask_bits = max_node + 2;
74 let bits_per_word = std::mem::size_of::<libc::c_ulong>() * 8;
75 let mask_words = mask_bits.div_ceil(bits_per_word);
76 let mut nodemask = vec![0 as libc::c_ulong; mask_words];
77 for &node in nodes {
78 nodemask[node / bits_per_word] |= 1 << (node % bits_per_word);
79 }
80 (nodemask, mask_bits as libc::c_ulong)
81}
82
83const MPOL_PREFERRED_MANY: i32 = 5;
84const MPOL_WEIGHTED_INTERLEAVE: i32 = 6;
85
86/// Worker-side `futex_wait` timeout for STOP-signal polling across
87/// every blocking workload primitive (FutexPingPong, FutexFanOut,
88/// FanOutCompute, MutexContention). Workers block inside the
89/// per-variant futex with this timespec; on wake (or timeout) they
90/// re-check [`STOP`] and either continue working or exit cleanly.
91/// At 100ms the worst-case shutdown latency a `stop_and_collect`
92/// caller must budget for is ~100ms above the flush/IO cost; see
93/// [`WorkloadHandle::stop_and_collect`]'s "Shutdown latency"
94/// paragraph for the caller-facing contract.
95pub(super) const WORKER_STOP_POLL_NS: libc::c_long = 100_000_000;
96
97/// Packaged [`libc::timespec`] for every worker-side `futex_wait`
98/// across the blocking workload primitives. Duplicating the struct
99/// literal per call site drifted the `tv_nsec` field between variants
100/// during earlier edits; a single const keeps the shutdown-latency
101/// budget documented on [`WORKER_STOP_POLL_NS`] authoritative.
102pub(super) const FUTEX_WAIT_TIMEOUT: libc::timespec = libc::timespec {
103 tv_sec: 0,
104 tv_nsec: WORKER_STOP_POLL_NS,
105};
106
107/// Post-wake spin count used by the fan-out messenger variants
108/// ([`WorkType::FutexFanOut`] and [`WorkType::FanOutCompute`]) AFTER
109/// each broadcast wake. Gives receivers a short uncontended window
110/// to run to their reservoir-push before the next wake cycle
111/// arrives. Threaded through `spin_burst` rather than a raw
112/// `std::hint::spin_loop` so the messenger also contributes to
113/// `work_units` — matching FanOutCompute's existing pattern so
114/// both variants' messengers report comparable throughput to
115/// downstream assertions.
116pub(super) const FAN_OUT_POST_WAKE_SPIN_ITERS: u64 = 256;
117
118/// Outcome of [`apply_mempolicy_with_flags`].
119///
120/// Makes each branch of the apply path observable to tests and any
121/// future caller that wants to react to the result (the production
122/// worker-setup caller in `worker::worker_main` discards it — applying
123/// the policy is best-effort there, matching the `setpriority` /
124/// `set_sched_policy` soft-fail idiom). The variants partition the
125/// function's exits exactly:
126///
127/// - [`SkippedDefault`](MempolicyOutcome::SkippedDefault): `policy ==
128/// MemPolicy::Default` — no syscall (inherit the parent's policy).
129/// - [`SkippedEmpty`](MempolicyOutcome::SkippedEmpty): a
130/// node-set-bearing variant (`Bind` / `Interleave` /
131/// `PreferredMany` / `WeightedInterleave`) carried an empty set — no
132/// `set_mempolicy(2)` is issued. Issuing one with an empty mask
133/// would be a different (and wrong) request to the kernel, so the
134/// skip is the contract, not merely "did not crash".
135/// - [`Applied`](MempolicyOutcome::Applied): `set_mempolicy(2)`
136/// returned 0.
137/// - [`Failed`](MempolicyOutcome::Failed): `set_mempolicy(2)` returned
138/// non-zero; the errno is logged to stderr and carried here.
139#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(super) enum MempolicyOutcome {
141 /// `MemPolicy::Default` — no syscall.
142 SkippedDefault,
143 /// A node-set variant with an empty set — no syscall.
144 SkippedEmpty,
145 /// `set_mempolicy(2)` succeeded.
146 Applied,
147 /// `set_mempolicy(2)` failed; carries the errno.
148 Failed(i32),
149}
150
151/// Call `set_mempolicy(2)` for the current process with mode flags.
152///
153/// No-op for `MemPolicy::Default` (returns
154/// [`MempolicyOutcome::SkippedDefault`]) and for a node-set variant
155/// with an empty set (returns [`MempolicyOutcome::SkippedEmpty`]).
156/// Logs a warning on syscall failure and returns
157/// [`MempolicyOutcome::Failed`]; returns [`MempolicyOutcome::Applied`]
158/// on success.
159pub(super) fn apply_mempolicy_with_flags(policy: &MemPolicy, flags: MpolFlags) -> MempolicyOutcome {
160 let (mode, node_set): (i32, BTreeSet<usize>) = match policy {
161 MemPolicy::Default => return MempolicyOutcome::SkippedDefault,
162 MemPolicy::Bind(nodes) => (libc::MPOL_BIND, nodes.clone()),
163 MemPolicy::Preferred(node) => (libc::MPOL_PREFERRED, [*node].into_iter().collect()),
164 MemPolicy::Interleave(nodes) => (libc::MPOL_INTERLEAVE, nodes.clone()),
165 MemPolicy::PreferredMany(nodes) => (MPOL_PREFERRED_MANY, nodes.clone()),
166 MemPolicy::WeightedInterleave(nodes) => (MPOL_WEIGHTED_INTERLEAVE, nodes.clone()),
167 MemPolicy::Local => {
168 let rc = unsafe {
169 libc::syscall(
170 libc::SYS_set_mempolicy,
171 libc::MPOL_LOCAL | flags.bits() as i32,
172 std::ptr::null::<libc::c_ulong>(),
173 0 as libc::c_ulong,
174 )
175 };
176 if rc != 0 {
177 let err = std::io::Error::last_os_error();
178 eprintln!("ktstr: set_mempolicy(MPOL_LOCAL) failed: {err}");
179 return MempolicyOutcome::Failed(err.raw_os_error().unwrap_or(0));
180 }
181 return MempolicyOutcome::Applied;
182 }
183 };
184 if node_set.is_empty() {
185 eprintln!("ktstr: set_mempolicy: empty node set, skipping");
186 return MempolicyOutcome::SkippedEmpty;
187 }
188 let (mask, maxnode) = build_nodemask(&node_set);
189 let effective_mode = mode | flags.bits() as i32;
190 let rc = unsafe {
191 libc::syscall(
192 libc::SYS_set_mempolicy,
193 effective_mode,
194 mask.as_ptr(),
195 maxnode,
196 )
197 };
198 if rc != 0 {
199 let err = std::io::Error::last_os_error();
200 eprintln!("ktstr: set_mempolicy(mode={mode}, nodes={node_set:?}) failed: {err}");
201 return MempolicyOutcome::Failed(err.raw_os_error().unwrap_or(0));
202 }
203 MempolicyOutcome::Applied
204}
205
206/// Terminate the calling forked-child worker with success status (code 0).
207///
208/// Uses the `_exit(2)` syscall via `libc::_exit` directly rather than [`std::process::exit`] or
209/// returning from `main` so that:
210/// 1. Rust drop glue does NOT run on inherited parent state (open file
211/// descriptors, mmaps, atexit handlers, thread-local destructors).
212/// Forked children share address space with the parent at fork
213/// time; running destructors on the child side would also tear down
214/// parent-owned resources via the shared FD table / OS handles.
215/// 2. `exit_group(2)` semantics (which `_exit` invokes) tear down only
216/// the calling tgid. Forked children have their own tgid distinct
217/// from the test-runner's, so the parent test runner survives.
218///
219/// Code 0 signals to the parent's collect_results path that the worker
220/// completed the work-loop normally — the WorkerReport (if any was
221/// written) is valid and should be merged into the test verdict.
222#[inline]
223fn exit_child_success() -> ! {
224 // SAFETY: `libc::_exit` is `unsafe` because it bypasses Rust drop
225 // execution + C stdlib atexit. That bypass IS the contract here
226 // for forked children: running destructors on shared inherited
227 // state corrupts parent-owned resources. exit_group(2) tear-down
228 // of the child's tgid is sound — the parent is in a different
229 // tgid.
230 unsafe { libc::_exit(0) }
231}
232
233/// Terminate the calling forked-child worker with error status (code 1).
234///
235/// Same `_exit`-vs-Rust-exit rationale as [`exit_child_success`]. Code 1
236/// signals to the parent's collect_results path that the worker failed
237/// before completing the work-loop normally — the parent treats the
238/// WorkerReport as missing or invalid and surfaces the failure to the
239/// test verdict.
240///
241/// Use this on any error path inside the forked-child closure: poll
242/// timeout, syscall failure, panic via `catch_unwind`, write-report
243/// failure, orphan detection.
244#[inline]
245fn exit_child_error() -> ! {
246 // SAFETY: see [`exit_child_success`] — identical contract.
247 unsafe { libc::_exit(1) }
248}
249
250/// Apply `nice` to the calling worker via `setpriority(2)`.
251///
252/// Always invokes `setpriority(PRIO_PROCESS, 0, nice)` — including
253/// for `nice == 0`, which writes 0 explicitly rather than
254/// inheriting. The "skip the syscall, inherit the parent's nice"
255/// state lives one layer up: callers gate the call on
256/// [`WorkSpec::nice`](crate::workload::WorkSpec::nice) /
257/// [`WorkloadConfig::nice`](crate::workload::WorkloadConfig::nice)
258/// being `Some(_)` and pass through the inner value (so `Some(0)`
259/// becomes a real `setpriority(0)` and `None` skips the call
260/// entirely). The kernel clamps `niceval` to
261/// `[MIN_NICE, MAX_NICE]` (-20..19) inside `setpriority`, so any
262/// out-of-range input is normalised by the syscall itself rather
263/// than rejected.
264///
265/// Failures are logged once via stderr and do not abort the
266/// worker — matches the [`apply_mempolicy_with_flags`] /
267/// [`set_thread_affinity`] / `set_sched_policy` error idiom in
268/// `worker_main`. The expected failure mode is `EACCES` from
269/// `set_one_prio` → `can_nice` when an unprivileged worker tries
270/// to lower nice (negative niceval) without `CAP_SYS_NICE`.
271pub(super) fn apply_nice(nice: i32) {
272 let rc = unsafe { libc::setpriority(libc::PRIO_PROCESS, 0, nice) };
273 if rc != 0 {
274 warn_setpriority_failed_once();
275 }
276}
277
278/// Print a single `setpriority` failure warning for the lifetime
279/// of the process. Same rationale as
280/// `warn_schedstat_unavailable_once`: dozens of workers will fail
281/// once each on an unprivileged host that requested negative nice,
282/// and a per-worker line floods the test log.
283pub(super) fn warn_setpriority_failed_once() {
284 static WARNED: std::sync::Once = std::sync::Once::new();
285 WARNED.call_once(|| {
286 let errno = std::io::Error::last_os_error();
287 eprintln!(
288 "workload: setpriority(PRIO_PROCESS) failed: {errno}; nice value not applied (CAP_SYS_NICE may be required for negative nice)"
289 );
290 });
291}
292
293/// Telemetry collected from a worker process after it stops.
294///
295/// Normal reports: each field is populated by the worker itself
296/// (inside the VM) and serialized via a pipe to the parent process.
297/// Sentinel reports: sentinel reports synthesized by
298/// [`WorkloadHandle::stop_and_collect`] on worker-exit carry
299/// parent-populated `exit_info` with the remaining fields at their
300/// [`Default`] values (the worker never emitted on the pipe, so
301/// the parent is the sole source of truth for the surfaced
302/// outcome).
303///
304/// # Default trade-off
305///
306/// [`Default`] produces a zero/empty report. The trade-off:
307///
308/// - **Pro:** sentinel/test code can spread `..WorkerReport::default()`
309/// so adding a field does not require touching every sentinel site.
310/// - **Con:** zero-valued fields are valid report outputs (e.g. a
311/// worker that never blocked has `wake_latencies_ns: vec![]`), so
312/// a missing field cannot be distinguished from a real-zero field at
313/// the reader. Consumers that need "was this field actually set"
314/// must track presence out-of-band (e.g. whether the work type
315/// populates the field per `wake_latencies_ns`'s doc).
316///
317/// Decision: keep the `Default` impl. Sentinel ergonomics outweigh
318/// the distinguishability cost — every real consumer already knows
319/// which fields a given `WorkType` populates, and the alternative
320/// (removing `Default` and hand-listing every field at sentinel
321/// sites) introduces a worse drift problem that silently skips new
322/// telemetry instead of reporting it as zero.
323///
324/// Callers building a sentinel report should spread
325/// `..WorkerReport::default()` rather than listing every field by hand
326/// -- the sentinel drifts silently when a field is added.
327#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, crate::Claim)]
328pub struct WorkerReport {
329 /// Kernel TID from `gettid(2)`. For [`CloneMode::Fork`] each
330 /// worker is its own thread-group leader so `gettid() == getpid()
331 /// == tgid`; the report's tid is interchangeable with the
332 /// worker's pid in libc / cgroup APIs. For [`CloneMode::Thread`]
333 /// every worker shares the parent's tgid and `gettid()` is the
334 /// only identifier that discriminates per-task identity, so the
335 /// report's tid is what feeds `sched_setaffinity(tid, ...)` and
336 /// `cgroup.threads` writes (NOT `cgroup.procs` — see the warning
337 /// on [`WorkloadHandle::worker_pids`]). Stored as `pid_t` (i32)
338 /// to match the kernel's native type and avoid the silent
339 /// u32→i32 sign-cast wraparound at libc boundaries
340 /// (kill/waitpid/Pid::from_raw).
341 pub tid: i32,
342 /// Cumulative work iterations (incremented by `spin_burst` or I/O loops).
343 /// Read by the fairness/no-progress gate (`assert_not_stuck` /
344 /// `min_work_units`) and `assert_throughput_parity`; NOT summed into
345 /// `CgroupStats::total_iterations`, which reads [`iterations`](Self::iterations).
346 /// A `Custom` worker that wants throughput assertions must also populate
347 /// [`iterations`](Self::iterations).
348 pub work_units: u64,
349 /// Thread CPU time from `CLOCK_THREAD_CPUTIME_ID` (ns).
350 pub cpu_time_ns: u64,
351 /// Wall-clock time from worker-start to stop flag (ns).
352 /// Measured from the worker's first `Instant::now()` in
353 /// `worker_main` (immediately after the start handshake) to the
354 /// outer-loop exit (when the per-worker `stop` flag is observed
355 /// `true`); covers both Fork-mode workers (signal-driven flag)
356 /// and Thread-mode workers (parent-driven flag).
357 pub wall_time_ns: u64,
358 /// `wall_time_ns - cpu_time_ns`: total off-CPU time (ns).
359 ///
360 /// Includes all time the worker was not executing on a CPU: runnable
361 /// queue wait, voluntary sleep, I/O wait, futex wait, etc.
362 pub off_cpu_ns: u64,
363 /// Number of observed CPU migrations (checked every 1024 work units).
364 pub migration_count: u64,
365 /// Set of all CPUs this worker ran on.
366 pub cpus_used: BTreeSet<usize>,
367 /// Ordered list of CPU migration events with timestamps.
368 pub migrations: Vec<Migration>,
369 /// Longest worker CPU-TIME gap observed between 1024-work-unit
370 /// checkpoints (ms) — the `CLOCK_THREAD_CPUTIME_ID` delta the worker
371 /// consumed without crossing a progress checkpoint. This is the Stuck
372 /// gate's measure (`max_gap_ms` thresholds compare against it): a high
373 /// value means compute-without-progress (livelock, a pathologically
374 /// expensive iteration), which is real at ANY host dilation — CPU time
375 /// excludes host preemption and guest run-queue wait by construction.
376 /// A BLOCKED or starved-runnable worker accrues no CPU here; detecting
377 /// blocked-stuck is the cell watchdog's domain. Granularity: multiples
378 /// of the checkpoint interval (1024 work units, plus one extra sample
379 /// at each phase transition).
380 pub max_gap_ms: u64,
381 /// Longest WALL-CLOCK gap between checkpoints (ms) — the historical
382 /// pre-CPU-conversion gate measure, kept verbatim (including its
383 /// blocking-site baseline resets) as EVIDENCE: under host dilation or
384 /// guest starvation wall-without-CPU shows up here while
385 /// [`max_gap_ms`](Self::max_gap_ms) stays low. Independent maximum —
386 /// its peak may come from a different checkpoint than the CPU peak.
387 /// `#[serde(default)]`: sidecar JSON written before this field existed
388 /// reads 0 (guest→host postcard is same-binary, always in sync).
389 #[serde(default)]
390 pub max_gap_wall_ms: u64,
391 /// CPU where the longest (CPU-time) gap happened.
392 pub max_gap_cpu: usize,
393 /// When the longest (CPU-time) gap happened (wall ms from start).
394 pub max_gap_at_ms: u64,
395 /// Per-wakeup latency samples (ns). Measures off-CPU time
396 /// between the call that blocks (any blocking primitive — pipe
397 /// `read`, futex wait, `poll`, `sched_yield`, `nanosleep`, etc.)
398 /// and the wakeup that resumes execution; not a yield-specific
399 /// measure.
400 /// Populated for blocking work types: Bursty, PipeIo, FutexPingPong,
401 /// FutexFanOut, FanOutCompute, CacheYield, CachePipe, IoSyncWrite,
402 /// IoRandRead, IoConvoy, NiceSweep,
403 /// AffinityChurn, PolicyChurn, MutexContention, ForkExit (parent's
404 /// waitpid wait), Sequence with Sleep/Yield/Io phases.
405 ///
406 /// Distinct from [`iteration_costs_ns`](Self::iteration_costs_ns):
407 /// this field measures the OFF-CPU gap between blocks (scheduler
408 /// wake latency); `iteration_costs_ns` measures the wall-clock
409 /// duration of a single compute iteration. The three pure-compute
410 /// variants that populate `iteration_costs_ns` —
411 /// [`WorkType::AluHot`], [`WorkType::SmtSiblingSpin`], and
412 /// [`WorkType::IpcVariance`] — never block and report
413 /// `wake_latencies_ns: vec![]`. Other compute variants
414 /// (e.g. SpinWait, YieldHeavy, Mixed) populate neither
415 /// reservoir.
416 pub wake_latencies_ns: Vec<u64>,
417 /// Total number of wake-latency observations the worker
418 /// recorded, INCLUDING any that were dropped by the reservoir
419 /// sampler. `wake_latencies_ns` is reservoir-clamped to at
420 /// most `MAX_WAKE_SAMPLES` (100_000) entries; on a long run
421 /// that accumulates more than that many wake events, the
422 /// vector stays at its cap while this counter keeps climbing.
423 /// Host-side consumers that want to report "total wakeups
424 /// observed" (vs. "entries in the sample") read this field;
425 /// percentile / CV computations read `wake_latencies_ns`.
426 pub wake_sample_total: u64,
427 /// Per-iteration wall-clock duration of one compute iteration (ns),
428 /// including any scheduler preemption. Measured via
429 /// `Instant::now()` (CLOCK_MONOTONIC), so a sample includes any
430 /// off-CPU time the kernel inserted mid-iteration. The variance
431 /// across iterations is the load-bearing scheduler signal —
432 /// preemption inflates samples and that inflation is the
433 /// observable.
434 ///
435 /// Reservoir-sampled at the same cap (`MAX_WAKE_SAMPLES` =
436 /// 100_000) as [`wake_latencies_ns`](Self::wake_latencies_ns),
437 /// using the same Algorithm-R sampler.
438 ///
439 /// Populated for pure compute work types where the worker
440 /// never blocks: [`WorkType::AluHot`], [`WorkType::SmtSiblingSpin`],
441 /// and [`WorkType::IpcVariance`]. Each sample is the elapsed
442 /// time from the start to the end of one outer-loop iteration's
443 /// compute burst.
444 ///
445 /// Distinct from [`wake_latencies_ns`](Self::wake_latencies_ns):
446 /// the wake-latency reservoir captures off-CPU time (futex /
447 /// pipe / nanosleep wakeups); this reservoir captures the
448 /// wall-clock duration of one compute iteration (which
449 /// includes any scheduler preemption inside the iteration).
450 /// The two are NOT comparable across variants — a
451 /// scheduler-A/B test that wants iteration cost for a compute
452 /// variant reads this field; a test that wants wake latency
453 /// for a blocking variant reads `wake_latencies_ns`.
454 pub iteration_costs_ns: Vec<u64>,
455 /// Total number of iteration-cost observations the worker
456 /// recorded, INCLUDING any that were dropped by the reservoir
457 /// sampler. Mirrors [`wake_sample_total`](Self::wake_sample_total)
458 /// but for [`iteration_costs_ns`](Self::iteration_costs_ns):
459 /// host-side consumers that want "total compute iterations
460 /// observed" read this field; distribution computations read
461 /// `iteration_costs_ns` directly.
462 pub iteration_cost_sample_total: u64,
463 /// Per-timer-cycle latency samples (ns) for
464 /// [`crate::workload::WorkType::TimerLatency`]: the observed
465 /// `clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME)` wake time minus the
466 /// absolute deadline, floored at 0. Reservoir-clamped to at most
467 /// `MAX_WAKE_SAMPLES`, distinct from `wake_latencies_ns` so cyclictest-style
468 /// timer latency does not blur with the blocking variants' wake latency.
469 /// `vec![]` for every non-`TimerLatency` variant.
470 pub timer_latencies_ns: Vec<u64>,
471 /// Total timer-cycle observations, INCLUDING any the reservoir dropped —
472 /// the true population for unbiased cross-phase weighting. Mirrors
473 /// [`wake_sample_total`](Self::wake_sample_total) for `timer_latencies_ns`.
474 pub timer_sample_total: u64,
475 /// Outer-loop iteration count. What `CgroupStats::total_iterations` sums
476 /// and what the derived `iteration_rate`, `iterations_per_worker`, and
477 /// `migration_ratio` divide by; NOT read by
478 /// the zero-work-units gate, which reads [`work_units`](Self::work_units). A
479 /// `Custom` worker that wants the starvation / `min_work_units` gate
480 /// honored must also populate [`work_units`](Self::work_units).
481 pub iterations: u64,
482 /// Delta of /proc/self/schedstat field 2 (run_delay) over the work loop.
483 pub schedstat_run_delay_ns: u64,
484 /// Delta of /proc/self/schedstat field 3 (pcount — number of
485 /// times the task was scheduled in over the work loop). This is
486 /// NOT a context-switch count; `/proc/<pid>/status`'s
487 /// `voluntary_ctxt_switches` / `nonvoluntary_ctxt_switches` are
488 /// the true context-switch counters and are not read here.
489 pub schedstat_run_count: u64,
490 /// Delta of /proc/self/schedstat field 1 (cpu_time) over the work loop.
491 pub schedstat_cpu_time_ns: u64,
492 /// `true` when the worker reached its natural end — either the
493 /// outer work loop observed STOP and exited cleanly, or a
494 /// custom-closure payload returned from its `run` function. A
495 /// sentinel report synthesised by
496 /// [`WorkloadHandle::stop_and_collect`]'s decode-failure
497 /// fallback (see `exit_info` below) carries `false`. Lets downstream
498 /// consumers distinguish "worker ran to completion and
499 /// observed zero iterations" (`completed: true, iterations: 0`
500 /// — legitimate for pathologically short test windows) from
501 /// "worker died / timed out before recording anything"
502 /// (`completed: false, iterations: 0` — the sentinel shape).
503 pub completed: bool,
504 /// Per-NUMA-node page counts from `/proc/self/numa_maps` after workload.
505 /// Keyed by node ID. Empty when numa_maps is unavailable. numa_maps reports
506 /// the per-node RESIDENT pages of the calling task's mm. For
507 /// [`CloneMode::Fork`] workers (the scenario-engine default) each worker has
508 /// a disjoint mm, so SUMming across workers is the true cgroup page total;
509 /// for [`CloneMode::Thread`] siblings share one mm and each reports the SAME
510 /// residency, so a SUM counts shared pages once per thread. Consumers
511 /// (cgroup_stats / phase_cgroup_stats) SUM
512 /// this — correct for the Fork default; the Thread-mode caveat is inherited
513 /// identically by both reducers (no per-phase divergence).
514 pub numa_pages: BTreeMap<usize, u64>,
515 /// Delta of `/proc/vmstat` `numa_pages_migrated` over the work loop. This is
516 /// the SYSTEM-WIDE vmstat `NUMA_PAGE_MIGRATE` vm_event (summed across all
517 /// CPUs), NOT the per-task `task_struct` field of the same name surfaced in
518 /// `/proc/PID/sched`. Because every worker reads the same system-wide
519 /// counter, consumers fold it as MAX across workers (a SUM would inflate it
520 /// by the worker count) — see `PhaseCgroupStats::cross_node_migrated`. The vm_event is bumped only by
521 /// NUMA balancing (the source of NUMA page migrations), so the delta is 0 on
522 /// kernels/configs without it (a measurement-availability caveat, not a wrong
523 /// value).
524 pub vmstat_numa_pages_migrated: u64,
525 /// Diagnostic attached only to sentinel reports — populated when
526 /// `stop_and_collect` synthesized the entry because no (or
527 /// unparseable) postcard payload came back on the report pipe.
528 /// `None` on every real worker-produced report. Lets operators
529 /// distinguish the four failure shapes that all collapse to
530 /// "empty pipe + no report":
531 ///
532 /// - [`WorkerExitInfo::Exited`] with a non-zero code: worker
533 /// reached `_exit(code)` without writing the report —
534 /// typically the `catch_unwind` Err arm in the worker-child
535 /// closure (panic under `panic = "unwind"`) or the 30s
536 /// poll-start timeout's early `_exit(1)`.
537 /// - [`WorkerExitInfo::Signaled`]: worker was killed — SIGABRT
538 /// under `panic = "abort"`, SIGKILL from the still-alive
539 /// escalation in `stop_and_collect`, or an external signal
540 /// (OOM killer, operator SIGKILL).
541 /// - [`WorkerExitInfo::TimedOut`]: worker never exited within the
542 /// 5s collection deadline and the WNOHANG reap observed
543 /// `StillAlive` — escalated via SIGKILL + `waitpid(None)`.
544 /// - [`WorkerExitInfo::WaitFailed`]: `waitpid` itself returned an
545 /// error (ECHILD / EINTR). Typically a plumbing bug — the child
546 /// was reaped by an external signal handler, a double-reap
547 /// regression, or the pid was recycled.
548 ///
549 /// No `skip_serializing_if`: postcard is a positional, schemaless
550 /// format — every Serialize call must emit every field in the
551 /// same order or the decoder reads the next field's bytes off
552 /// the wire (silent data corruption). The Option<…> tag itself
553 /// (one byte) is the only overhead on the live-worker path.
554 pub exit_info: Option<WorkerExitInfo>,
555 /// `true` when this worker served as the messenger for a
556 /// wake-fanout work type ([`WorkType::FutexFanOut`] or
557 /// [`WorkType::FanOutCompute`]) — the single writer that
558 /// advances the shared generation and issues `futex_wake` for
559 /// its group. `false` for receivers and for every non-fanout
560 /// work type.
561 ///
562 /// Populated from the `is_messenger` flag on the
563 /// `futex: Option<(*mut u32, bool)>` parameter threaded into
564 /// `worker_main`. A sentinel report synthesized by the
565 /// decode-failure fallback in
566 /// [`WorkloadHandle::stop_and_collect`] carries `false` via
567 /// [`Default`], matching its `completed: false` shape.
568 ///
569 /// Enables per-worker latency-participation assertions in
570 /// tests — a receiver worker produces `wake_latencies_ns`
571 /// entries while its messenger pair records wake-side work but
572 /// no wake latency. Without this field, tests had to
573 /// cross-reference per-group indexing or guess from the empty
574 /// vector — ambiguous on groups where the messenger legitimately
575 /// exits before producing a report.
576 pub is_messenger: bool,
577 /// Index of the worker group this report belongs to.
578 ///
579 /// `0` denotes the primary group described by
580 /// [`WorkloadConfig`]'s top-level `work_type` / `num_workers` /
581 /// `affinity` / `sched_policy` fields. `1..=N` denotes
582 /// composed groups in the order they appear in
583 /// [`WorkloadConfig::composed`]. Reports collected by
584 /// [`WorkloadHandle::stop_and_collect`] are tagged with the
585 /// `group_idx` of the spawning [`WorkSpec`] (or `0` for the
586 /// primary), so per-group filtering in test assertions can
587 /// cleanly partition the vector.
588 ///
589 /// Sentinel reports (synthesized on missing or undecodable
590 /// payload / panic / timeout) carry the `group_idx` of the
591 /// worker whose pid the sentinel replaces, so a "this composed
592 /// group failed" assertion still works on an outright crash.
593 pub group_idx: usize,
594 /// Rendered error from the worker's `set_thread_affinity`
595 /// call, or `None` when affinity setup succeeded (or the
596 /// worker had no affinity to apply). Populated by
597 /// `worker_main` when the pre-loop
598 /// `set_thread_affinity(tid, cpus)` returns `Err` — the
599 /// worker continues with the inherited (or kernel-default)
600 /// cpumask so the test still produces an observable outcome,
601 /// but the failure is now surfaced in the report instead of
602 /// being silently dropped via `let _ = …`. The expected
603 /// failure shape is EINVAL from a requested cpu that is
604 /// outside the cpuset cgroup's `cpus.allowed` mask or the
605 /// kernel's online mask; EPERM is reachable when a more
606 /// privileged tracer set the worker's cpus_allowed and a
607 /// container policy denies further widening. Sentinel
608 /// reports synthesised by
609 /// [`WorkloadHandle::stop_and_collect`] leave this field at
610 /// its default `None` — a worker that died before
611 /// `worker_main` ran has no affinity-error observation.
612 ///
613 /// No `skip_serializing_if`: postcard is positional and
614 /// schemaless, so every Serialize call must emit every field
615 /// in the same order — skipping a field shifts the decoder
616 /// onto the next field's bytes (silent corruption). The
617 /// Option<…> tag (one byte) is the only overhead on the
618 /// success path.
619 pub affinity_error: Option<String>,
620 /// `set_sched_policy` failure text (`{e:#}`), or `None` when the
621 /// per-worker scheduling-policy set succeeded (or the policy was the
622 /// `Normal` no-op). Load-bearing for the verifier dispatch probe: a
623 /// probe worker configured `SchedPolicy::Ext` whose
624 /// `sched_setattr(SCHED_EXT)` was rejected (e.g. the scheduler set
625 /// `scx.disallow` on it) stays SCHED_OTHER, so its `iterations`
626 /// progress does NOT prove the BPF scheduler dispatched it —
627 /// `run_and_confirm_dispatch` excludes any worker with a
628 /// `sched_policy_error` from the dispatch proof. Sentinel reports
629 /// synthesised by [`WorkloadHandle::stop_and_collect`] leave this
630 /// `None` (no policy set was attempted). Same positional-serde
631 /// reasoning as [`Self::affinity_error`]: no `skip_serializing_if`.
632 pub sched_policy_error: Option<String>,
633 /// Per-phase telemetry slices for a backdrop (persistent) worker
634 /// that spanned scenario steps. EMPTY for step-local workers and
635 /// for a backdrop worker that remained in the pre-step baseline or
636 /// inter-step sentinel epoch. A worker drains its open measured
637 /// epoch either when the parent-driven `phase_epoch` changes or at
638 /// shutdown, including when a delayed worker first runs after the
639 /// parent has already published a measured epoch. Each slice carries
640 /// the per-phase subset of the whole-run telemetry above, scoped to
641 /// one phase's hold window; the host expands these into per-epoch
642 /// `PhaseBucket` entries — the per-phase attribution a backdrop
643 /// worker otherwise lacks, since it is collected once with
644 /// `step_index = None`.
645 ///
646 /// Appended LAST so the positional postcard decode order of every
647 /// prior field is unchanged. `#[claim(skip)]`: there is no
648 /// test-author claim surface for the raw wire slices — assertions
649 /// run against the host-expanded `PhaseCgroupStats` carriers, which
650 /// carry their own `#[derive(Claim)]`.
651 #[claim(skip)]
652 pub phase_slices: Vec<PhaseSlice>,
653 /// Whole-run taobench COUNTER aggregate — `Some` only for a Taobench worker,
654 /// `None` otherwise. Shipped so the host can derive run-level qps/hit Rate keys
655 /// (`taobench_*_ops_per_cpu_sec_whole` / `taobench_hit_fraction` /
656 /// `taobench_command_hit_rate`) for `--noise-adjust` spread analysis. The
657 /// per-phase `PhaseSlice::taobench` carriers feed the per-phase metrics + the
658 /// serve-latency distribution; this whole-run carrier holds COUNTERS only
659 /// ([`TaobenchStats`](crate::workload::taobench::run::TaobenchStats)) — the
660 /// serve histogram is per-phase data, and the per-phase `elapsed_ns` is
661 /// MAX-merged across concurrent threads so summing phase windows is the wrong
662 /// qps denominator, hence the engine's authoritative whole-run counter
663 /// aggregate is shipped directly. Appended LAST (after `phase_slices`) to keep
664 /// the positional postcard decode order of every prior field unchanged. `pub`
665 /// (every `WorkerReport` field is `pub`, so external custom workers can
666 /// struct-literal construct it via [`crate::workload::WorkType::custom`]; a
667 /// non-taobench worker sets `None`). `#[claim(skip)]`: framework wire plumbing,
668 /// not a test-author claim surface; assertions run against the host-derived
669 /// run-level `taobench_*` Rate metrics.
670 #[claim(skip)]
671 pub taobench_whole: Option<crate::workload::taobench::run::TaobenchStats>,
672}
673
674/// Per-phase telemetry for one backdrop worker over one scenario
675/// step's HOLD window `[StepStart(k), StepEnd(k))`. A backdrop worker
676/// spans every phase, so it accumulates a fresh slice between each
677/// parent-driven `phase_epoch` transition and finalizes it when the
678/// epoch moves (drain-on-change). Carries the per-phase subset of
679/// [`WorkerReport`]'s whole-run telemetry that has a host-side
680/// per-cgroup carrier (`PhaseCgroupStats`), so the host can pool slices
681/// across workers into per-epoch `PhaseBucket`s. Counter fields are
682/// per-phase deltas (`end_snap − start_snap`, re-baselined at each
683/// boundary); `cpus_used` and `numa_pages` are per-phase observations
684/// (gauges). Excludes `iteration_costs_ns` — that reservoir has no
685/// per-cgroup carrier at any level (it feeds only the run-level
686/// benchmark gate).
687#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
688pub struct PhaseSlice {
689 /// The 1-indexed phase this slice covers (scenario Step k → k+1;
690 /// `0` = BASELINE, `u32::MAX` = inter-step gap). The host maps this
691 /// to a `PhaseBucket.step_index`; gap- and baseline-tagged slices
692 /// have no host bucket and are discarded.
693 pub phase_epoch: u32,
694 /// CPUs this worker ran on DURING this phase.
695 pub cpus_used: BTreeSet<usize>,
696 /// Per-wakeup latency samples (ns) recorded during this phase,
697 /// reservoir-clamped to at most `MAX_PHASE_WAKE_SAMPLES` — a
698 /// smaller cap than the whole-run `MAX_WAKE_SAMPLES` so a worker
699 /// spanning many phases stays within the report pipe. The host
700 /// re-caps the merged per-cgroup pool at the larger
701 /// `MAX_WAKE_SAMPLES`.
702 pub wake_latencies_ns: Vec<u64>,
703 /// Total wake observations during this phase, INCLUDING any the
704 /// reservoir dropped — the true population for unbiased
705 /// cross-phase weighting.
706 pub wake_sample_total: u64,
707 /// Per-timer-cycle latency samples (ns) recorded during this phase by a
708 /// [`crate::workload::WorkType::TimerLatency`] worker, reservoir-clamped to
709 /// `MAX_PHASE_WAKE_SAMPLES` (like `wake_latencies_ns`). Distinct carrier so
710 /// timer latency does not blur with wake latency.
711 pub timer_latencies_ns: Vec<u64>,
712 /// Total timer-cycle observations during this phase, INCLUDING any the
713 /// reservoir dropped — the true population for unbiased cross-phase
714 /// weighting. Mirrors `wake_sample_total` for `timer_latencies_ns`.
715 pub timer_sample_total: u64,
716 /// `/proc/self/schedstat` run_delay (field 2) delta over this phase (ns).
717 pub run_delay_ns: u64,
718 /// Off-CPU time during this phase (ns): `wall_ns − cpu_time` over
719 /// the phase. Paired with `wall_ns` so the host derives the
720 /// off-CPU fraction and skips the not-measured `wall_ns == 0` case
721 /// rather than dividing by zero.
722 pub off_cpu_ns: u64,
723 /// Wall-clock duration of this phase as the worker observed it
724 /// (ns) — the denominator for the off-CPU fraction. `0` for a
725 /// phase the worker never ran in (not-measured).
726 pub wall_ns: u64,
727 /// CPU migrations observed during this phase.
728 pub migration_count: u64,
729 /// Outer-loop iterations during this phase.
730 pub iterations: u64,
731 /// `/proc/self/schedstat` cpu_time (field 1, sum_exec_runtime)
732 /// delta over this phase (ns).
733 pub schedstat_cpu_time_ns: u64,
734 /// Per-NUMA-node resident page counts observed at this phase's end
735 /// (`/proc/self/numa_maps`). A gauge, not a delta.
736 pub numa_pages: BTreeMap<usize, u64>,
737 /// `/proc/vmstat numa_pages_migrated` delta over this phase
738 /// (system-wide counter; host folds as MAX across workers).
739 pub vmstat_numa_pages_migrated: u64,
740 /// Longest checkpoint-to-checkpoint wall gap during this phase (ms).
741 pub max_gap_ms: u64,
742 /// CPU where this phase's longest gap happened — coupled with
743 /// `max_gap_ms` as an argmax pair (a bare max would desync the gap
744 /// from its CPU).
745 pub max_gap_cpu: usize,
746 /// Per-phase schbench engine metrics, present ONLY for a
747 /// `WorkType::Schbench` backdrop worker (`None` for every other work
748 /// type — the generic drain leaves it `None`). Carries the phase's merged
749 /// wakeup/request histograms + run-delay raw pairs; the host re-pools these
750 /// across workers ([`crate::workload::schbench::run::SchbenchPhaseStats`])
751 /// and derives per-phase percentiles into `PhaseBucket.metrics`.
752 /// `pub(crate)`: an internal carrier (test authors read `PhaseBucket`, not
753 /// `PhaseSlice`) whose element type is `pub(crate)`. Integer-only, so it
754 /// preserves `PhaseSlice`'s `Eq`.
755 pub(crate) schbench: Option<crate::workload::schbench::run::SchbenchPhaseStats>,
756 /// Per-phase taobench engine metrics, present ONLY for a
757 /// `WorkType::Taobench` backdrop worker (`None` for every other work type —
758 /// the generic drain leaves it `None`). Carries the phase's request- and
759 /// response-time op counters plus the open-loop serve-latency histogram
760 /// ([`crate::workload::taobench::run::TaobenchPhaseStats`]); the host derives
761 /// per-phase qps / hit-ratio / serve-latency percentiles into
762 /// `PhaseBucket.metrics`. `pub(crate)`: an internal carrier (test authors read
763 /// `PhaseBucket`, not `PhaseSlice`). `PhaseSlice`'s `Eq` is preserved because
764 /// `PlatStats` is `Eq` (the schbench carrier above already holds histograms),
765 /// not because the field is integer-only.
766 pub(crate) taobench: Option<crate::workload::taobench::run::TaobenchPhaseStats>,
767}
768
769/// Reason a sentinel [`WorkerReport`] was synthesized — attached to
770/// the report's `exit_info` field so operators can triage a missing
771/// or undecodable postcard payload without cross-referencing
772/// parent-side logs.
773///
774/// Invariant: every variant carries the `waitpid`-derived status for
775/// the worker PID as of the end of `stop_and_collect`. Ordered from
776/// most-informative (exit code) to least (plumbing failure).
777/// No `Default` impl — every variant carries observed-outcome state.
778/// A default-constructed value would have to pick one variant (TimedOut
779/// was the obvious candidate) but that risks silent-pass: a test
780/// fixture using `..Default::default()` on a struct containing
781/// `WorkerExitInfo` gets "worker never exited within the deadline,"
782/// which an operator triaging the failure would chase for minutes
783/// before realizing the value came from a missing field. Construct
784/// every WorkerExitInfo explicitly via the variant the test scenario
785/// expects (e.g. `WorkerExitInfo::Exited(0)` for a clean success).
786#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
787pub enum WorkerExitInfo {
788 /// `WIFEXITED=true` with the given exit code. Non-zero under
789 /// `panic = "unwind"` means catch_unwind caught a panic in the
790 /// worker-child closure and `_exit(1)` fired, or the 30s
791 /// parent-ready poll timed out. Zero means the worker ran to
792 /// completion but failed to write / serialize the report — a
793 /// postcard encode or pipe-write failure that didn't panic.
794 Exited(i32),
795 /// `WIFSIGNALED=true` with the given signal number. Under
796 /// `panic = "abort"` a worker panic raises SIGABRT (signal 6);
797 /// other values indicate external kill, OOM killer, or the
798 /// still-alive-escalation SIGKILL (signal 9) from this function.
799 Signaled(i32),
800 /// Worker was still running after the 5s shared collection
801 /// deadline; escalated via SIGKILL + blocking `waitpid`. The
802 /// child's final status is not retained — the reap happened past
803 /// the point where operator diagnostics would differ between a
804 /// clean timeout and a signal storm.
805 TimedOut,
806 /// `waitpid` itself returned `Err` — typically ECHILD (child
807 /// already reaped by an external signal handler or a double-reap
808 /// regression) or EINTR. Message is the rendered `errno` string.
809 WaitFailed(String),
810 /// Thread-mode worker panicked. `JoinHandle::join()` returned
811 /// `Err`; the inner payload is downcast to a `&str` / `String`
812 /// (the canonical `panic!` payload shapes) and recorded here so
813 /// the operator can triage without scraping the test log. This
814 /// variant is exclusive to [`CloneMode::Thread`] — fork workers
815 /// surface panics via `Exited(1)` or `Signaled(SIGABRT)`
816 /// depending on the panic strategy.
817 Panicked(String),
818}
819
820/// Pure mapping from a `waitpid` outcome to the diagnostic
821/// [`WorkerExitInfo`] attached to a sentinel [`WorkerReport`].
822///
823/// Split out of [`WorkloadHandle::stop_and_collect`] so the four
824/// shapes each resolve to a `WorkerExitInfo` variant without pulling
825/// in the full collection loop's state (pipe drain, SIGKILL
826/// escalation, pid lifetime). Pure input → output means the variant
827/// matrix is directly testable without spawning children.
828///
829/// Shape → variant:
830/// - `Ok(Exited(_, code))` → [`WorkerExitInfo::Exited`]
831/// - `Ok(Signaled(_, sig, _))` → [`WorkerExitInfo::Signaled`]
832/// - `Ok(StillAlive)` → [`WorkerExitInfo::TimedOut`]
833/// - `Ok(_ exotic)` → [`WorkerExitInfo::TimedOut`] (Stopped /
834/// PtraceEvent / PtraceSyscall / Continued; not reachable for a
835/// plain forked worker with no ptrace parent, but collapsed rather
836/// than silently dropped so coverage stays exhaustive)
837/// - `Err(errno)` → [`WorkerExitInfo::WaitFailed`]
838pub(super) fn classify_wait_outcome(
839 source: Result<nix::sys::wait::WaitStatus, nix::errno::Errno>,
840) -> WorkerExitInfo {
841 match source {
842 Ok(nix::sys::wait::WaitStatus::Exited(_, code)) => WorkerExitInfo::Exited(code),
843 Ok(nix::sys::wait::WaitStatus::Signaled(_, sig, _)) => WorkerExitInfo::Signaled(sig as i32),
844 Ok(nix::sys::wait::WaitStatus::StillAlive) => WorkerExitInfo::TimedOut,
845 Ok(_) => WorkerExitInfo::TimedOut,
846 Err(e) => WorkerExitInfo::WaitFailed(e.to_string()),
847 }
848}
849
850/// Extract a human-readable panic payload from a
851/// [`std::thread::Result`] `Err` value. The two canonical shapes
852/// are `&'static str` (`panic!("literal")`) and `String`
853/// (`panic!("{x}")` post-formatting); anything else falls back to
854/// a fixed sentinel.
855///
856/// Pure mapping (no IO, no allocation past `String::clone`) so the
857/// stop_and_collect path can call it on every joined-and-panicked
858/// thread without performance cliffs.
859pub(super) fn extract_panic_payload(payload: Box<dyn std::any::Any + Send>) -> String {
860 if let Some(s) = payload.downcast_ref::<&'static str>() {
861 (*s).to_string()
862 } else if let Some(s) = payload.downcast_ref::<String>() {
863 s.clone()
864 } else {
865 "<non-string panic payload>".to_string()
866 }
867}
868
869/// Wall-clock time budget for joining a thread-mode worker after
870/// its per-task `stop` has been flipped. Mirrors the fork-mode
871/// `stop_and_collect` 5s shared deadline so neither dispatch path
872/// can serially exhaust the test runtime by hanging on a single
873/// stuck worker. The 100ms `FUTEX_WAIT_TIMEOUT` inside
874/// `worker_main`'s blocking primitives means a well-behaved worker
875/// observes `stop=true` within 100ms of the parent's flip; the 5s
876/// budget covers IO drain, scheduling delays under contention, and
877/// post-loop cleanup (NUMA stat reads, schedstat snapshots).
878pub(super) const THREAD_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
879
880/// Block until `join` reports finished or `timeout` elapses.
881/// Returns `Some(thread_result)` on successful join, `None` on
882/// timeout.
883///
884/// Implementation: wait on `exit_evt` (the worker's "I'm about to
885/// return" eventfd, bumped from a Drop guard inside the thread
886/// closure) via `epoll_wait` with a `timerfd` for the safety
887/// deadline. A spurious wake (e.g. EINTR or a stale eventfd-counter
888/// drain) loops back into the wait without orphaning the worker —
889/// the timerfd carries the absolute deadline.
890///
891/// Std lacks a native timed-join API; an alternative side-thread
892/// "joiner + channel" pattern would orphan the joiner on timeout
893/// (joining is non-cancellable in std), which keeps the thread
894/// alive past `WorkloadHandle::drop` and prevents process exit.
895/// The eventfd path replaces the previous 10ms sleep-poll loop
896/// without that orphan cost.
897pub(super) fn join_thread_with_timeout(
898 join: std::thread::JoinHandle<WorkerReport>,
899 exit_evt: &vmm_sys_util::eventfd::EventFd,
900 timeout: Duration,
901) -> Option<std::thread::Result<WorkerReport>> {
902 use std::os::unix::io::AsRawFd;
903 use vmm_sys_util::epoll::{ControlOperation, Epoll, EpollEvent, EventSet};
904 use vmm_sys_util::timerfd::TimerFd;
905
906 if join.is_finished() {
907 return Some(join.join());
908 }
909
910 let epoll = match Epoll::new() {
911 Ok(e) => e,
912 Err(e) => {
913 tracing::warn!(%e, "join_thread_with_timeout: epoll_create1 failed");
914 return None;
915 }
916 };
917 if let Err(e) = epoll.ctl(
918 ControlOperation::Add,
919 exit_evt.as_raw_fd(),
920 EpollEvent::new(EventSet::IN, 0),
921 ) {
922 tracing::warn!(%e, "join_thread_with_timeout: add exit_evt to epoll");
923 return None;
924 }
925 let mut timer = match TimerFd::new() {
926 Ok(t) => t,
927 Err(e) => {
928 tracing::warn!(%e, "join_thread_with_timeout: timerfd_create failed");
929 return None;
930 }
931 };
932 if let Err(e) = timer.reset(timeout, None) {
933 tracing::warn!(%e, "join_thread_with_timeout: timerfd_settime failed");
934 return None;
935 }
936 if let Err(e) = epoll.ctl(
937 ControlOperation::Add,
938 timer.as_raw_fd(),
939 EpollEvent::new(EventSet::IN, 1),
940 ) {
941 tracing::warn!(%e, "join_thread_with_timeout: add timerfd to epoll");
942 return None;
943 }
944
945 let deadline = Instant::now() + timeout;
946 let mut events = [EpollEvent::default(); 2];
947 loop {
948 if join.is_finished() {
949 return Some(join.join());
950 }
951 let remaining = deadline.saturating_duration_since(Instant::now());
952 if remaining.is_zero() {
953 return None;
954 }
955 match epoll.wait(-1, &mut events) {
956 Ok(_) => {}
957 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
958 Err(e) => {
959 tracing::warn!(%e, "join_thread_with_timeout: epoll_wait failed");
960 return None;
961 }
962 }
963 }
964}
965
966/// `worker_main`'s loop-check predicate: returns `true` when the
967/// worker should stop iterating. Reads BOTH the per-worker `stop`
968/// flag and the global [`STOP`] flag; either set request causes
969/// exit.
970///
971/// Why both:
972/// - Per-worker `stop` is what `WorkloadHandle::stop_and_collect`
973/// flips for graceful shutdown. For Fork mode the per-worker
974/// `stop` IS the global [`STOP`] (the SIGUSR1 handler flips it).
975/// For Thread mode each worker has its own `Arc<AtomicBool>`
976/// passed via `&AtomicBool`.
977/// - The global [`STOP`] is what the SIGUSR1 handler sets. For
978/// Fork mode the worker's per-process [`STOP`] is the same
979/// AtomicBool the handler writes. For Thread mode every thread
980/// shares the parent process's address space, so a SIGUSR1
981/// delivered to the parent (e.g. Ctrl-C / a test harness signal)
982/// flips the shared global [`STOP`] but NOT the per-worker
983/// `stop` Arcs. Without this disjunction, Thread workers would
984/// silently keep running through a parent-level shutdown
985/// request.
986///
987/// `#[inline]` because the call site is two atomic loads + an OR.
988/// Relaxed ordering on both reads matches every existing site —
989/// no cross-field happens-before edge to establish.
990#[inline]
991pub(super) fn stop_requested(stop: &AtomicBool) -> bool {
992 stop.load(Ordering::Relaxed) || STOP.load(Ordering::Relaxed)
993}
994
995/// Per-thread worker state for [`CloneMode::Thread`] dispatch.
996///
997/// Thread workers cannot be reaped via `waitpid` (they share a tgid
998/// with the parent), so the lifecycle uses Rust's [`std::thread`]
999/// primitives instead of pid-based syscalls:
1000///
1001/// - `tid` is published by the worker thread post-spawn via
1002/// `gettid()` so the parent can address the kernel task for
1003/// `sched_setaffinity(tid, ...)` and report it from
1004/// [`WorkloadHandle::worker_pids`]. `Arc<AtomicI32>` because the
1005/// thread closure owns the publisher and the parent reads it
1006/// without joining.
1007/// - `stop` replaces the global [`STOP`] signal-flag for thread
1008/// mode: the parent flips it from
1009/// [`WorkloadHandle::stop_and_collect`], the worker observes it
1010/// inside `worker_main`'s `stop.load(Relaxed)` checks. SIGUSR1 is
1011/// process-wide and useless for per-thread stop control.
1012/// - `start_tx` is the rendezvous channel: the parent calls
1013/// `send(())` from [`WorkloadHandle::start`]; the thread blocks
1014/// in `recv()` until then. `Option` so `start` can take it and
1015/// drop it (idempotent re-call is a no-op when `None`).
1016/// - `join` holds the [`std::thread::JoinHandle`] returned by
1017/// `thread::spawn`; `stop_and_collect` joins each handle to
1018/// retrieve the [`WorkerReport`]. `Option` so `stop_and_collect`
1019/// can take ownership and `Drop` does not double-join.
1020pub(super) struct ThreadWorker {
1021 tid: std::sync::Arc<std::sync::atomic::AtomicI32>,
1022 stop: std::sync::Arc<AtomicBool>,
1023 pub(super) start_tx: Option<std::sync::mpsc::SyncSender<()>>,
1024 join: Option<std::thread::JoinHandle<WorkerReport>>,
1025 /// Eventfd bumped by the worker thread's `WorkerExitSignal` Drop
1026 /// guard before the thread returns from its closure. Lets
1027 /// [`join_thread_with_timeout`] block in `epoll_wait` instead of
1028 /// sleep-polling [`std::thread::JoinHandle::is_finished`]. Counter
1029 /// mode (not semaphore) — the value never matters; only the edge
1030 /// from 0 to non-zero does. The Arc is cloned into the closure
1031 /// for the Drop guard; the parent retains the original here.
1032 exit_evt: std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
1033}
1034
1035/// Defense-in-depth Drop for [`ThreadWorker`]. Rust's
1036/// [`std::thread::JoinHandle`] does NOT join its thread on drop —
1037/// it detaches, and the thread continues running until completion.
1038/// `WorkloadHandle::drop`, `WorkloadHandle::stop_and_collect`, and
1039/// `SpawnGuard::drop` already explicitly `take()` the JoinHandle and
1040/// route it through [`join_thread_with_timeout`]; this impl exists
1041/// for the case where some future refactor lets a `ThreadWorker`
1042/// fall out of scope without going through one of those paths.
1043///
1044/// Behavior: if `join` is still `Some` when this Drop fires, flip
1045/// `stop` (so the worker exits cleanly), drop `start_tx` (in case
1046/// the worker is still parked on `recv()`), and join with the
1047/// shared 5s budget. Errors / timeouts are swallowed because Drop
1048/// has nothing to assert against; the upstream paths produce the
1049/// auditable diagnostics.
1050impl Drop for ThreadWorker {
1051 fn drop(&mut self) {
1052 if let Some(j) = self.join.take() {
1053 self.stop.store(true, Ordering::Relaxed);
1054 self.start_tx.take();
1055 let _ = join_thread_with_timeout(j, &self.exit_evt, THREAD_JOIN_TIMEOUT);
1056 }
1057 }
1058}
1059
1060/// Per-fork-child report-decoding shape and bookkeeping recorded
1061/// alongside each entry in [`SpawnGuard::children`] and
1062/// [`WorkloadHandle::children`].
1063///
1064/// Two variants:
1065///
1066/// - [`ForkedChildKind::Worker`]: the conventional fork-mode worker
1067/// (one process per worker, single postcard `WorkerReport`).
1068/// Carries the worker's `group_idx` so a sentinel report on a
1069/// missing payload can be tagged correctly.
1070/// - [`ForkedChildKind::PcommContainer`]: a single thread-group
1071/// leader that owns `num_workers` worker threads across one or
1072/// more logical groups. The leader sets its own `comm` to `pcomm`
1073/// via `prctl(PR_SET_NAME)` before spawning the threads, so every
1074/// worker thread's `task->group_leader->comm` is `pcomm` exactly
1075/// (the [`WorkSpec::pcomm`] builder rejects > 15 bytes —
1076/// `TASK_COMM_LEN - 1` — so the framework never feeds the kernel
1077/// a name `__set_task_comm` would truncate). Reports are a single
1078/// `serde_json` `Vec<WorkerReport>` (one entry per worker thread;
1079/// per-thread `group_idx` lives inside each `WorkerReport`).
1080///
1081/// Note the per-variant wire-format split: `Worker` uses postcard and
1082/// `PcommContainer` uses serde_json. The encodings are dispatched by
1083/// `ForkedChildKind` at decode time on the parent.
1084#[derive(Clone, Debug)]
1085pub(super) enum ForkedChildKind {
1086 /// Conventional fork-mode worker (one process per worker).
1087 /// Report wire format: bare `b'r'` ready byte followed by a
1088 /// `postcard::to_stdvec(&WorkerReport)` document.
1089 Worker { group_idx: usize },
1090 /// pcomm thread-group leader hosting worker threads across one
1091 /// or more logical groups. `groups` records the per-group
1092 /// `(group_idx, num_workers)` layout in the order threads were
1093 /// spawned: `groups[0]` contributes the first
1094 /// `groups[0].1` worker reports, `groups[1]` the next
1095 /// `groups[1].1`, and so on. Total expected reports =
1096 /// `groups.iter().map(|(_, n)| n).sum()`. The layout drives
1097 /// sentinel distribution: when the JSON payload is missing or
1098 /// short, the parent emits sentinels tagged with the right
1099 /// per-group `group_idx` so per-group filters partition
1100 /// correctly.
1101 ///
1102 /// Report wire format: bare `b'r'` ready byte followed by a
1103 /// `serde_json::to_vec(&Vec<WorkerReport>)` document, one entry
1104 /// per worker thread (in `(group_idx, within-group index)`
1105 /// traversal order matching the `groups` layout).
1106 PcommContainer { groups: Vec<(usize, usize)> },
1107}
1108
1109/// Bookkeeping for a single forked-child process owned by the spawn
1110/// pipeline. Replaces the prior `(pid, report_fd, start_fd)` tuple so
1111/// per-child decoding metadata ([`ForkedChildKind`]) lives alongside
1112/// the pipe fds and pid.
1113#[derive(Clone, Debug)]
1114pub(super) struct ForkedChild {
1115 /// Forked child's pid (== tgid for both `Worker` and
1116 /// `PcommContainer`: the worker is its own thread-group leader,
1117 /// the container is the thread-group leader of its inner threads).
1118 pub pid: libc::pid_t,
1119 /// Parent-side read end of the report pipe. The child writes one
1120 /// `b'r'` ready byte followed by either a postcard-encoded single
1121 /// `WorkerReport` (Worker) or a `serde_json` `Vec<WorkerReport>`
1122 /// (PcommContainer) per the [`ForkedChildKind`] tag.
1123 pub report_fd: std::os::unix::io::RawFd,
1124 /// Parent-side write end of the start pipe. Closed by
1125 /// [`crate::workload::WorkloadHandle::start`] after writing the
1126 /// start byte; set to `-1` thereafter.
1127 pub start_fd: std::os::unix::io::RawFd,
1128 /// Per-child decoding shape. See [`ForkedChildKind`].
1129 pub kind: ForkedChildKind,
1130}
1131
1132/// Handle to spawned worker tasks. Workers block until
1133/// [`start()`](Self::start) is called.
1134///
1135/// The [`CloneMode`] in the [`WorkloadConfig`] selects how each
1136/// worker is created. Within one [`WorkloadHandle`] every worker
1137/// uses the same mode, so exactly one of `children` or `threads`
1138/// is populated; the other is empty. This avoids per-worker mode
1139/// dispatch on the hot path and keeps each vec's per-mode
1140/// invariants (pid-based vs JoinHandle-based reaping) cohesive.
1141///
1142/// - [`CloneMode::Fork`] populates `children` — separate process
1143/// per worker, reaped via `waitpid`, signaled via SIGUSR1.
1144/// - [`CloneMode::Thread`] populates `threads` — separate kernel
1145/// task in the parent's thread group via [`std::thread::spawn`],
1146/// joined via `JoinHandle`. Workers share the parent's tgid;
1147/// per-worker cgroup placement requires `cgroup.threads`
1148/// (cgroup v2 thread mode), which ktstr scenarios do not
1149/// currently configure — Thread-mode workers inherit the
1150/// parent's cgroup.
1151#[must_use = "dropping a WorkloadHandle immediately tears down all worker tasks"]
1152pub struct WorkloadHandle {
1153 /// Forked-child processes owned by this handle. Each entry is a
1154 /// [`ForkedChild`] carrying pid, parent-side pipe fds, and a
1155 /// [`ForkedChildKind`] tag that drives report decoding (single
1156 /// postcard `WorkerReport` for `Worker`; `serde_json`
1157 /// `Vec<WorkerReport>` for `PcommContainer`). Empty when every
1158 /// group used [`CloneMode::Thread`] without `pcomm`.
1159 children: Vec<ForkedChild>,
1160 /// Thread-mode workers. Empty when `clone_mode` is not
1161 /// [`CloneMode::Thread`].
1162 threads: Vec<ThreadWorker>,
1163 started: bool,
1164 /// Shared mmap regions for futex-based work types (one per worker group). Unmapped on drop.
1165 futex_ptrs: Vec<*mut u32>,
1166 /// Per-region byte length, parallel to `futex_ptrs`. Each
1167 /// region was sized at spawn time to its source group's
1168 /// natural width (4 for FutexPingPong / FutexFanOut /
1169 /// MutexContention / etc., 16 for FanOutCompute, 32 + Q*8 for
1170 /// ProducerConsumerImbalance — see [`futex_region_size_for`]).
1171 /// `futex_ptrs[i]` and `futex_region_sizes[i]` describe the
1172 /// same region; both are consumed pairwise on `Drop` so each
1173 /// `munmap` call receives the matching length.
1174 futex_region_sizes: Vec<usize>,
1175 /// MAP_SHARED region of per-worker iteration counters. Workers
1176 /// atomically store their iteration count; parent reads via
1177 /// `snapshot_iterations()`. Pointer to the first element; length
1178 /// is the active worker collection's len. Typed as
1179 /// `*mut AtomicU64` rather than `*mut u64` so the 8-byte
1180 /// alignment guarantee (inherited from the page-aligned
1181 /// iter_counters mmap site in `WorkloadHandle::spawn`) and the
1182 /// atomic-only-access invariant are encoded in the type system
1183 /// instead of prose. `AtomicU64` is layout-compatible with `u64`:
1184 /// `std::mem::size_of::<AtomicU64>() == std::mem::align_of::<AtomicU64>() == 8`
1185 /// on every supported target, so casting the `*mut c_void`
1186 /// returned by `mmap` to `*mut AtomicU64` is sound.
1187 iter_counters: *mut AtomicU64,
1188 /// Number of AtomicU64 slots in iter_counters (== num_workers at spawn time).
1189 iter_counter_len: usize,
1190 /// MAP_SHARED single-word phase-epoch region. The scenario engine
1191 /// stores the current 1-indexed phase here (0 = BASELINE, Step k →
1192 /// k + 1, `u32::MAX` = inter-step gap); every backdrop worker of
1193 /// this handle reads the SAME pointer (not a per-worker slot — the
1194 /// epoch is a broadcast value, all the handle's workers transition
1195 /// phases together) to attribute its per-phase `PhaseSlice`s. A
1196 /// single `AtomicU32`, not the per-worker `AtomicU64` layout of
1197 /// `iter_counters`, because one shared word suffices. Null until
1198 /// the spawn-site mmap installs it.
1199 phase_epoch: *mut AtomicU32,
1200 /// Byte length of the `phase_epoch` mmap, for `munmap` on drop
1201 /// (`size_of::<AtomicU32>()`; the kernel rounds the mapping up to a
1202 /// page).
1203 phase_epoch_bytes: usize,
1204 /// Inter-worker paired pipes `(ab, ba)` for PipeIo / CachePipe.
1205 /// Transferred from [`SpawnGuard`] on success; closed by
1206 /// [`WorkloadHandle::drop`] AFTER worker shutdown so Thread-mode
1207 /// workers (which share the parent's fd table) can finish their
1208 /// pipe ops before the close. Under Fork mode each child holds
1209 /// its own fd-table copy via `fork()`, so the parent's late
1210 /// close is a no-op for the children. Empty when `work_type`
1211 /// is neither PipeIo nor CachePipe.
1212 pipe_pairs: Vec<([i32; 2], [i32; 2])>,
1213 /// Per-chain pipe rings for `WakeChain { wake: WakeMechanism::Pipe }`.
1214 /// Outer Vec is one entry per chain (= `num_workers / depth`);
1215 /// inner Vec is `depth` pipes per chain. Same ownership rule as
1216 /// `pipe_pairs`: transferred from [`SpawnGuard`] on success,
1217 /// closed by [`WorkloadHandle::drop`] AFTER worker shutdown so
1218 /// Thread-mode chain workers don't observe `EBADF` mid-run.
1219 /// Empty when `work_type` is not `WakeChain { wake: Pipe }`.
1220 chain_pipes: Vec<Vec<[i32; 2]>>,
1221 /// Shared first-iteration signal eventfd. `Some` only when the
1222 /// workload set [`WorkloadConfig::signal_first_iteration`] AND the
1223 /// dispatch is fork with at least one worker; each fork worker
1224 /// `write(2)`s a `1` after its first counted iteration and the parent
1225 /// blocks on this fd in
1226 /// [`Self::wait_first_iteration_all`]. The [`std::os::fd::OwnedFd`]
1227 /// closes the parent's copy on `Drop`; fork children hold their own
1228 /// fd-table copies (the fd is `EFD_CLOEXEC`, harmless — fork workers
1229 /// never `exec`). `None` for every other workload — the polled
1230 /// `snapshot_iterations` path is unchanged.
1231 first_iter_signal: Option<std::os::fd::OwnedFd>,
1232}
1233
1234/// Per-variant byte length for the MAP_SHARED futex region.
1235///
1236/// Each WorkType that needs a shared region has a fixed natural
1237/// size:
1238///
1239/// - [`WorkType::FanOutCompute`] needs 16 bytes — futex `u32` at
1240/// offset 0, wake-timestamp `u64` at offset 8.
1241/// - [`WorkType::ProducerConsumerImbalance`] needs a ring buffer:
1242/// reserve-head `u64` @ 0, tail `u64` @ 8, producer-wake `u32`
1243/// @ 16, consumer-wake `u32` @ 20, publish-head `u64` @ 24,
1244/// then `Q` × `u64` ring slots starting at offset 32. Total
1245/// bytes = `32 + Q*8`. The two head counters split the MPMC
1246/// protocol: producers CAS-advance the reserve head to claim a
1247/// unique slot index, write the slot, then in slot-index FIFO
1248/// order release-store the publish head; consumers
1249/// acquire-load the publish head so a slot is visible to a
1250/// reader only after its producer's data write
1251/// synchronizes-with the consumer through the publish-head
1252/// release. `queue_depth_target` is `u64` to match the variant;
1253/// an `as usize` truncation on a 32-bit host could silently
1254/// produce a sub-page region with a malformed queue, so the
1255/// conversion is clamped at `usize::MAX/8 - 4` to keep the
1256/// layout well-defined (one fewer slot than before to make
1257/// room for `pub_head`). Realistic configs use Q in the
1258/// hundreds-to-thousands; the clamp only triggers on a
1259/// degenerate input that itself fails admission control
1260/// elsewhere (the queue is far larger than RAM).
1261/// - [`WorkType::SignalStorm`] needs 8 bytes — two `u32` tid
1262/// slots (worker 0's tid @ offset 0, worker 1's @ offset 4).
1263/// - Everything else: `u32` (4 bytes).
1264///
1265/// Returning the same byte count for every WorkType variant lets
1266/// the caller mmap exactly what's needed for THIS group rather
1267/// than the MAX across all groups, so a small-variant group
1268/// composed alongside a large `ProducerConsumerImbalance` no
1269/// longer pays the large group's per-region overhead.
1270pub(super) fn futex_region_size_for(work_type: &WorkType) -> usize {
1271 match work_type {
1272 WorkType::FanOutCompute { .. } => 16,
1273 WorkType::ProducerConsumerImbalance {
1274 queue_depth_target, ..
1275 } => {
1276 let q = std::cmp::min(*queue_depth_target as usize, usize::MAX / 8 - 4);
1277 32 + q * 8
1278 }
1279 // SignalStorm exchanges two u32 tid slots through the region
1280 // (worker 0's tid @ offset 0, worker 1's @ offset 4), so it
1281 // needs 8 bytes, not the default single u32.
1282 WorkType::SignalStorm { .. } => 2 * std::mem::size_of::<u32>(),
1283 _ => std::mem::size_of::<u32>(),
1284 }
1285}
1286
1287/// Block on a counter-mode eventfd `raw` until its accumulated count
1288/// reaches `need`, or `deadline` passes. Returns `true` when the count is
1289/// reached, `false` on timeout / unexpected error. Extracted from
1290/// [`WorkloadHandle::wait_first_iteration_all`] so the poll/accumulate
1291/// logic is unit-testable against a raw eventfd without spawning workers.
1292///
1293/// Each `read(2)` on a counter-mode eventfd returns the current counter
1294/// and resets it to 0, so the value is accumulated across reads. `need ==
1295/// 0` returns `true` immediately (vacuous). EINTR (poll or read) and a
1296/// spurious EFD_NONBLOCK EAGAIN re-poll against the same deadline; any
1297/// other error returns `false`.
1298fn poll_eventfd_reaches(raw: i32, deadline: Instant, need: u64) -> bool {
1299 if need == 0 {
1300 return true;
1301 }
1302 let mut acc: u64 = 0;
1303 loop {
1304 let now = Instant::now();
1305 if now >= deadline {
1306 return false;
1307 }
1308 let remaining = deadline - now;
1309 // poll(2) timeout in ms: truncating `as_millis` under-waits by
1310 // <1ms (the outer deadline re-check bounds the total), and the
1311 // `.max(1)` avoids a busy-spin on a sub-ms remainder.
1312 let timeout_ms = remaining.as_millis().min(i32::MAX as u128).max(1) as i32;
1313 let mut pfd = libc::pollfd {
1314 fd: raw,
1315 events: libc::POLLIN,
1316 revents: 0,
1317 };
1318 let ret = unsafe { libc::poll(&mut pfd, 1, timeout_ms) };
1319 if ret < 0 {
1320 let err = std::io::Error::last_os_error();
1321 // EINTR (e.g. SIGCHLD coalescing under the probe's forked
1322 // workers): re-poll against the same deadline.
1323 if err.raw_os_error() == Some(libc::EINTR) {
1324 continue;
1325 }
1326 return false;
1327 }
1328 if ret == 0 {
1329 // Timed out this slice; the loop top re-checks the deadline.
1330 continue;
1331 }
1332 if pfd.revents & libc::POLLIN != 0 {
1333 // Counter-mode read: returns the summed counter and resets it
1334 // to 0, so accumulate across reads.
1335 let mut buf: u64 = 0;
1336 let r = unsafe {
1337 libc::read(
1338 raw,
1339 &mut buf as *mut u64 as *mut libc::c_void,
1340 std::mem::size_of::<u64>(),
1341 )
1342 };
1343 if r == std::mem::size_of::<u64>() as isize {
1344 acc = acc.saturating_add(buf);
1345 if acc >= need {
1346 return true;
1347 }
1348 } else if r < 0 {
1349 let err = std::io::Error::last_os_error();
1350 // EAGAIN: EFD_NONBLOCK spurious wake with no pending count;
1351 // EINTR: signal mid-read. Either way re-poll.
1352 if matches!(err.raw_os_error(), Some(libc::EAGAIN) | Some(libc::EINTR)) {
1353 continue;
1354 }
1355 return false;
1356 }
1357 }
1358 }
1359}
1360
1361/// Scope guard that owns every resource acquired during
1362/// [`WorkloadHandle::spawn`]'s partial setup. If `spawn` returns
1363/// early (via `?` or `bail!`), the guard's `Drop` kills and reaps any
1364/// already-forked children, closes every open pipe fd, and munmaps
1365/// every shared region — so a mid-setup failure never leaks fds,
1366/// zombie processes, or anonymous-shared pages.
1367///
1368/// On success, [`SpawnGuard::into_handle`] moves every live
1369/// resource — children/threads, futex regions, iter-counter
1370/// region, AND `pipe_pairs` / `chain_pipes` — into the returned
1371/// [`WorkloadHandle`]. The guard's subsequent `Drop` runs against
1372/// empty Vecs/null pointers and is a no-op on the success path.
1373/// On the early-bail path (an `?` inside `WorkloadHandle::spawn`)
1374/// the guard still owns whatever it allocated and `Drop` cleans
1375/// it all up — fds, processes, threads, mmaps. Pipe fds are
1376/// closed by the handle (not the guard) because Thread-mode
1377/// workers share the parent's fd table; closing the fds before
1378/// worker shutdown would surface as `EBADF` on every pipe op a
1379/// thread runs after spawn returns.
1380pub(super) struct SpawnGuard {
1381 /// Inter-worker paired pipes `(ab, ba)` for PipeIo/CachePipe.
1382 /// Transferred to [`WorkloadHandle`] on success; closed by the
1383 /// guard only on the early-bail path. Under Fork mode each
1384 /// child holds its own fd-table copy via `fork()`; under
1385 /// Thread mode every worker thread shares these fds with the
1386 /// parent.
1387 pipe_pairs: Vec<([i32; 2], [i32; 2])>,
1388 /// Per-chain pipe rings for `WakeChain { wake: WakeMechanism::Pipe }`. Outer
1389 /// Vec is one entry per chain (= `num_workers / depth`); inner
1390 /// Vec is `depth` pipes per chain. Pipe `i` connects stage `i`
1391 /// (writer) to stage `(i + 1) % depth` (reader). Same ownership
1392 /// shape as `pipe_pairs`: transferred to the handle on success,
1393 /// closed by the guard only on the early-bail path.
1394 chain_pipes: Vec<Vec<[i32; 2]>>,
1395 /// Shared-memory futex regions (transferred to handle on success).
1396 futex_ptrs: Vec<*mut u32>,
1397 /// Per-region byte length, parallel to `futex_ptrs`. Each
1398 /// region is sized to its source group's natural width
1399 /// (4 / 16 / 32+Q*8 — see [`futex_region_size_for`]) and
1400 /// recorded here at `spawn_group` time so munmap on Drop
1401 /// can call `libc::munmap(ptr, len)` with the matching length
1402 /// even when groups with different natural sizes co-exist.
1403 /// `futex_ptrs[i]` and `futex_region_sizes[i]` describe the
1404 /// same region.
1405 futex_region_sizes: Vec<usize>,
1406 /// Per-worker iteration counter region (transferred on success).
1407 /// Typed matches the handle field; see `WorkloadHandle::iter_counters`.
1408 iter_counters: *mut AtomicU64,
1409 iter_counter_bytes: usize,
1410 /// Per-handle phase-epoch region (transferred to handle on
1411 /// success). Single `AtomicU32` word shared by all the handle's
1412 /// workers; see `WorkloadHandle::phase_epoch`.
1413 phase_epoch: *mut AtomicU32,
1414 phase_epoch_bytes: usize,
1415 /// Already-forked children (either conventional workers or
1416 /// pcomm containers) with their parent-side pipe fds and
1417 /// per-child decoding shape (transferred to handle on success).
1418 children: Vec<ForkedChild>,
1419 /// Already-spawned thread workers (transferred on success).
1420 /// Cleanup on early-exit flips each `stop` and joins each
1421 /// thread, since threads share the parent's address space and
1422 /// must be drained cooperatively (no `kill` equivalent).
1423 threads: Vec<ThreadWorker>,
1424 /// First-iteration signal eventfd (transferred to the handle on
1425 /// success; see [`WorkloadHandle::first_iter_signal`]). The
1426 /// [`std::os::fd::OwnedFd`] closes on the guard's `Drop`, so the
1427 /// early-bail path releases the fd exactly like the iter_counters
1428 /// mmap it is allocated beside — no explicit cleanup needed.
1429 first_iter_signal: Option<std::os::fd::OwnedFd>,
1430}
1431
1432impl SpawnGuard {
1433 fn new() -> Self {
1434 Self {
1435 pipe_pairs: Vec::new(),
1436 chain_pipes: Vec::new(),
1437 futex_ptrs: Vec::new(),
1438 futex_region_sizes: Vec::new(),
1439 iter_counters: std::ptr::null_mut(),
1440 iter_counter_bytes: 0,
1441 phase_epoch: std::ptr::null_mut(),
1442 phase_epoch_bytes: 0,
1443 children: Vec::new(),
1444 threads: Vec::new(),
1445 first_iter_signal: None,
1446 }
1447 }
1448
1449 /// Transfer live resources into a [`WorkloadHandle`]. Leaves the
1450 /// guard's `children`, `threads`, `futex_ptrs`,
1451 /// `futex_region_sizes`, `iter_counters`, `pipe_pairs`, and
1452 /// `chain_pipes` empty, so the guard's subsequent `Drop` is a
1453 /// no-op on the success path. The handle is now the sole owner
1454 /// of every resource — its own `Drop` closes the pipe fds
1455 /// AFTER worker shutdown completes, which is the ordering
1456 /// Thread mode requires (workers share the parent's fd table;
1457 /// closing pre-shutdown would surface as `EBADF` on every
1458 /// worker's pipe op). Fork mode is unaffected either way: each
1459 /// child holds its own fd-table copy via `fork()`, so the
1460 /// parent's close timing is invisible to the child.
1461 fn into_handle(mut self) -> WorkloadHandle {
1462 let children = std::mem::take(&mut self.children);
1463 let threads = std::mem::take(&mut self.threads);
1464 let futex_ptrs = std::mem::take(&mut self.futex_ptrs);
1465 let futex_region_sizes = std::mem::take(&mut self.futex_region_sizes);
1466 let iter_counters = std::mem::replace(&mut self.iter_counters, std::ptr::null_mut());
1467 let iter_counter_bytes = std::mem::replace(&mut self.iter_counter_bytes, 0);
1468 let iter_counter_len = iter_counter_bytes / std::mem::size_of::<AtomicU64>();
1469 let phase_epoch = std::mem::replace(&mut self.phase_epoch, std::ptr::null_mut());
1470 let phase_epoch_bytes = std::mem::replace(&mut self.phase_epoch_bytes, 0);
1471 let pipe_pairs = std::mem::take(&mut self.pipe_pairs);
1472 let chain_pipes = std::mem::take(&mut self.chain_pipes);
1473 let first_iter_signal = self.first_iter_signal.take();
1474 WorkloadHandle {
1475 children,
1476 threads,
1477 started: false,
1478 futex_ptrs,
1479 futex_region_sizes,
1480 iter_counters,
1481 iter_counter_len,
1482 phase_epoch,
1483 phase_epoch_bytes,
1484 pipe_pairs,
1485 chain_pipes,
1486 first_iter_signal,
1487 }
1488 }
1489}
1490
1491impl Drop for SpawnGuard {
1492 fn drop(&mut self) {
1493 // Kill and reap any already-forked children first, so their
1494 // pipe ends are not left blocked when we close the parent
1495 // side. `nix` wrappers replace the raw libc calls — kill
1496 // returns `Result<()>` (we swallow ECHILD/ESRCH in the
1497 // already-exited case), waitpid returns `Result<WaitStatus>`
1498 // (we discard the status in the cleanup path), close returns
1499 // `Result<()>` (we swallow EBADF for fds an earlier arm may
1500 // have already closed).
1501 for child in &self.children {
1502 let npid = nix::unistd::Pid::from_raw(child.pid);
1503 let _ = nix::sys::signal::kill(npid, nix::sys::signal::Signal::SIGKILL);
1504 let _ = nix::sys::wait::waitpid(npid, None);
1505 }
1506 // Close each child's parent-side report/start fds.
1507 for child in &self.children {
1508 close_fds_silently(&[child.report_fd, child.start_fd]);
1509 }
1510 // Stop and join any partially-spawned threads. Threads
1511 // share our address space, so `kill` does not reach them
1512 // and the only safe teardown is "flip stop, drop the start
1513 // channel (in case worker is still parked on `recv`), then
1514 // join". Dropping `start_tx` causes `recv` on the worker
1515 // side to return `Err(Disconnected)`, unblocking a thread
1516 // that has not yet been signaled. After both signals
1517 // (stop=true and start_tx dropped), `worker_main`'s outer
1518 // loop exits at the next `stop.load(Relaxed)` check (max
1519 // ~100ms latency from the `FUTEX_WAIT_TIMEOUT` poll
1520 // cadence) and the thread completes. `join` returns the
1521 // partial `WorkerReport` (or `Err` on panic, which we
1522 // swallow because mid-spawn cleanup has nothing to assert).
1523 for tw in &mut self.threads {
1524 tw.stop.store(true, Ordering::Relaxed);
1525 // Drop start_tx FIRST so a worker still parked on
1526 // recv() unblocks via Disconnected.
1527 tw.start_tx.take();
1528 if let Some(j) = tw.join.take() {
1529 // SpawnGuard cleanup uses the same `THREAD_JOIN_TIMEOUT`
1530 // budget as `stop_and_collect` and `WorkloadHandle::drop`
1531 // so a stuck worker can't pin mid-spawn error recovery.
1532 // Errors (panic / timeout) are silently dropped — the
1533 // mid-spawn path has nothing to assert against beyond
1534 // not leaking, and the spawn-side bail message has
1535 // already named the failure mode that triggered cleanup.
1536 let _ = join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT);
1537 }
1538 }
1539 // Early-bail pipe close. On the success path, into_handle
1540 // moved both `pipe_pairs` and `chain_pipes` into the handle,
1541 // so these Vecs are empty here and these loops iterate
1542 // nothing. On the early-bail path the guard still owns the
1543 // partially-allocated pipes and must close them now — the
1544 // child arm of each fork already closed any inherited
1545 // copies it held, and Thread-mode early-bail joined any
1546 // partially-spawned threads above before this loop runs.
1547 for (ab, ba) in &self.pipe_pairs {
1548 close_fds_silently(&[ab[0], ab[1], ba[0], ba[1]]);
1549 }
1550 for chain in &self.chain_pipes {
1551 for pipe in chain {
1552 close_fds_silently(&[pipe[0], pipe[1]]);
1553 }
1554 }
1555 // Munmap shared regions. `futex_ptrs[i]` and
1556 // `futex_region_sizes[i]` describe the same region, so each
1557 // munmap receives the exact length used for the matching
1558 // mmap. The two vectors are appended in lockstep inside
1559 // `spawn_group`, so they have identical lengths in every
1560 // observable state.
1561 for (&ptr, &size) in self.futex_ptrs.iter().zip(self.futex_region_sizes.iter()) {
1562 unsafe {
1563 libc::munmap(ptr as *mut libc::c_void, size);
1564 }
1565 }
1566 if !self.iter_counters.is_null() && self.iter_counter_bytes > 0 {
1567 unsafe {
1568 libc::munmap(
1569 self.iter_counters as *mut libc::c_void,
1570 self.iter_counter_bytes,
1571 );
1572 }
1573 }
1574 if !self.phase_epoch.is_null() && self.phase_epoch_bytes > 0 {
1575 unsafe {
1576 libc::munmap(
1577 self.phase_epoch as *mut libc::c_void,
1578 self.phase_epoch_bytes,
1579 );
1580 }
1581 }
1582 }
1583}
1584
1585// SAFETY: futex_ptrs, iter_counters, and phase_epoch are MAP_SHARED anonymous pages
1586// created before fork, so every forked child inherits a pointer copy
1587// of the same underlying kernel object. Children read/write their own
1588// futex word — via `std::ptr::read_volatile`/`write_volatile` for
1589// most WorkType variants, or via `AtomicU32`/`AtomicU64` references
1590// re-derived from the raw pointer for FanOutCompute, which needs
1591// release-acquire ordering to publish `wake_ns` alongside the
1592// generation advance — and atomically store into their dedicated
1593// iter_counters slot (via a shared `&AtomicU64` reference derived
1594// from the `*mut AtomicU64` region pointer); the parent reads
1595// all slots via `snapshot_iterations` and is the sole process that
1596// munmaps the region, on WorkloadHandle::drop after every child has
1597// been reaped.
1598//
1599// Per-mode aliasing rationale:
1600//
1601// - Fork mode: each forked child constructs its own process-local
1602// `&AtomicU32`/`&AtomicU64` shared reference into the MAP_SHARED
1603// page from the inherited raw pointer. No reference value ever
1604// crosses a process boundary — each process synthesises its own
1605// reference from the same underlying kernel object. Interior
1606// mutation through a shared atomic reference is permitted by
1607// Rust's aliasing model because `AtomicU32`/`AtomicU64` wrap an
1608// `UnsafeCell`; the post-fork alias relation is therefore not an
1609// aliasing-rule violation.
1610//
1611// - Thread mode: under [`CloneMode::Thread`] every worker thread
1612// shares the parent process's single address space — the same
1613// raw `*mut AtomicU32`/`*mut AtomicU64` pointer is dereferenced
1614// from multiple threads concurrently, and the resulting
1615// `&AtomicU32`/`&AtomicU64` shared references coexist for
1616// overlapping lifetimes. This is sound for the same reason
1617// `Arc<AtomicU64>` is sound: atomic types' `UnsafeCell`-wrapped
1618// storage permits concurrent shared-reference access by design,
1619// and the underlying load/store instructions are by construction
1620// non-tearing on every supported target. No `&mut` reference is
1621// ever materialised; every access is via the atomic API. The
1622// MAP_SHARED region is allocated once before any worker spawns
1623// and `munmap`ped after every worker has been joined, so the
1624// underlying kernel object outlives every alias.
1625//
1626// `phase_epoch` follows the same MAP_SHARED aliasing rules with the
1627// roles reversed: the parent (scenario engine) is the sole WRITER of
1628// the single `AtomicU32` word (Release stores at each step boundary)
1629// and the backdrop workers are READERS (Relaxed loads). It is
1630// telemetry — no happens-before edge is required, matching the
1631// `iter_counters` Relaxed-store / Relaxed-read precedent.
1632unsafe impl Send for WorkloadHandle {}
1633unsafe impl Sync for WorkloadHandle {}
1634
1635/// Pointer-sized addresses passed across a thread-spawn boundary.
1636///
1637/// Rust's auto-`Send` inference on closures conservatively treats
1638/// `*mut T` as `!Send` even inside a wrapper struct destructured in
1639/// the closure body — the destructured field type leaks into the
1640/// closure's auto-trait check. The simplest workaround is to round-
1641/// trip the pointers through `usize` (Send + Copy) and re-cast on
1642/// the receiver side. Soundness is identical: thread-mode workers
1643/// share the parent's address space, so the addresses retain
1644/// meaning across the thread boundary, and the underlying
1645/// MAP_SHARED regions are owned by the guard / handle for the full
1646/// duration of every worker.
1647///
1648/// `SendFutexPtr` carries a (futex_address, pos) tuple wrapped in
1649/// `Option`; `None` is the "no futex required" case for work types
1650/// that don't need shared memory. `SendIterSlotPtr` carries a single
1651/// address (zero ⇒ no iter_slot publish).
1652#[derive(Clone, Copy)]
1653pub(super) struct SendFutexPtr(Option<(usize, usize)>);
1654
1655#[derive(Clone, Copy)]
1656pub(super) struct SendIterSlotPtr(usize);
1657
1658impl SendFutexPtr {
1659 fn new(p: Option<(*mut u32, usize)>) -> Self {
1660 SendFutexPtr(p.map(|(ptr, pos)| (ptr as usize, pos)))
1661 }
1662
1663 /// Re-cast back into the `*mut u32` + `pos` tuple `worker_main`
1664 /// expects.
1665 fn into_raw(self) -> Option<(*mut u32, usize)> {
1666 self.0.map(|(addr, pos)| (addr as *mut u32, pos))
1667 }
1668}
1669
1670impl SendIterSlotPtr {
1671 fn new(p: *mut AtomicU64) -> Self {
1672 SendIterSlotPtr(p as usize)
1673 }
1674
1675 fn into_raw(self) -> *mut AtomicU64 {
1676 self.0 as *mut AtomicU64
1677 }
1678}
1679
1680/// Send-safe carrier for the single shared `phase_epoch` word (the SAME
1681/// pointer for every worker in a group), mirroring [`SendIterSlotPtr`].
1682/// Round-trips the address through `usize` so a worker closure's capture
1683/// set stays `Send` (no raw-pointer field in the closure type).
1684#[derive(Clone, Copy)]
1685pub(super) struct SendPhaseEpochPtr(usize);
1686
1687impl SendPhaseEpochPtr {
1688 fn new(p: *mut AtomicU32) -> Self {
1689 SendPhaseEpochPtr(p as usize)
1690 }
1691
1692 fn into_raw(self) -> *mut AtomicU32 {
1693 self.0 as *mut AtomicU32
1694 }
1695}
1696
1697/// Per-group resolved view of [`WorkloadConfig`] used by the
1698/// spawn pipeline.
1699///
1700/// [`WorkloadHandle::spawn`] iterates one `GroupParams` per group
1701/// it spawns: the primary group (`group_idx == 0`) is built from
1702/// the top-level [`WorkloadConfig`] fields via
1703/// [`Self::primary`], and each composed [`WorkSpec`] entry is
1704/// resolved into its own `GroupParams` (with `group_idx ==
1705/// 1..=N`) via [`Self::from_composed`]. Both paths funnel through
1706/// [`Self::from_work_spec`] for the actual field copy.
1707///
1708/// `GroupParams` is the post-resolution shape — `num_workers` is a
1709/// concrete `usize` (not the `Option<usize>` that [`WorkSpec`]
1710/// carries), `affinity` is a concrete [`ResolvedAffinity`] (not
1711/// the [`AffinityIntent`] that [`WorkSpec`] carries). The spawn
1712/// pipeline operates on `GroupParams` exclusively so it never has
1713/// to deal with the unresolved intent/optional shapes that the
1714/// user-facing types expose.
1715///
1716/// `clone_mode` is shared across every group — the top-level
1717/// [`WorkloadConfig::clone_mode`] selects fork vs thread dispatch
1718/// for the entire workload, and [`WorkSpec`] carries no
1719/// `clone_mode` field of its own (composed entries inherit the
1720/// parent's mode; the [`SpawnGuard`]'s lifecycle assumes a single
1721/// dispatch path).
1722#[derive(Clone)]
1723pub(super) struct GroupParams {
1724 work_type: WorkType,
1725 /// Park each worker after `n` counted iterations rather than
1726 /// running to stop. Only the primary group can carry a non-`None`
1727 /// value ([`WorkloadConfig::park_after_iterations`] has no
1728 /// [`WorkSpec`] counterpart), so composed groups always inherit
1729 /// `None` through [`Self::from_work_spec`].
1730 park_after: Option<u64>,
1731 /// Signal the handle's first-iteration eventfd after each fork
1732 /// worker's first counted iteration. Like `park_after`, only the
1733 /// primary group can carry `true`
1734 /// ([`WorkloadConfig::signal_first_iteration`] has no [`WorkSpec`]
1735 /// counterpart), so composed groups always inherit `false` through
1736 /// [`Self::from_work_spec`].
1737 signal_first_iteration: bool,
1738 sched_policy: SchedPolicy,
1739 mem_policy: MemPolicy,
1740 mpol_flags: MpolFlags,
1741 nice: Option<i32>,
1742 comm: Option<String>,
1743 uid: Option<u32>,
1744 gid: Option<u32>,
1745 numa_node: Option<u32>,
1746 affinity: ResolvedAffinity,
1747 num_workers: usize,
1748 group_idx: usize,
1749}
1750
1751impl GroupParams {
1752 /// Extract a [`GroupParams`] from a [`WorkSpec`] given the
1753 /// resolved sibling values. This is the single field-extraction
1754 /// site — both [`Self::primary`] and [`Self::from_composed`]
1755 /// funnel through here, so the field-by-field copy lives in one
1756 /// place.
1757 ///
1758 /// The caller is responsible for resolving the
1759 /// [`WorkSpec::num_workers`] `Option<usize>` to a concrete
1760 /// `usize` and the [`WorkSpec::affinity`] [`AffinityIntent`] to
1761 /// a concrete [`ResolvedAffinity`]. The remaining fields
1762 /// (`work_type`, `sched_policy`, `mem_policy`, `mpol_flags`,
1763 /// `nice`) are copied verbatim — they need no resolution
1764 /// because both [`WorkSpec`] and [`GroupParams`] carry them in
1765 /// their final runtime form (`nice` round-trips as
1766 /// `Option<i32>` so `None` continues to mean "skip the
1767 /// `setpriority(2)` call" at the spawn-time gate).
1768 fn from_work_spec(
1769 spec: &WorkSpec,
1770 group_idx: usize,
1771 resolved_affinity: ResolvedAffinity,
1772 resolved_num_workers: usize,
1773 ) -> Self {
1774 Self {
1775 work_type: spec.work_type.clone(),
1776 // WorkSpec has no park-after knob; `primary` overrides this
1777 // from `WorkloadConfig::park_after_iterations`, composed
1778 // groups keep `None`.
1779 park_after: None,
1780 // Same shape as `park_after`: no WorkSpec counterpart, so
1781 // `primary` overrides from `WorkloadConfig::signal_first_iteration`
1782 // and composed groups keep `false`.
1783 signal_first_iteration: false,
1784 sched_policy: spec.sched_policy,
1785 mem_policy: spec.mem_policy.clone(),
1786 mpol_flags: spec.mpol_flags,
1787 nice: spec.nice,
1788 comm: spec.comm.as_ref().map(|c| c.to_string()),
1789 uid: spec.uid,
1790 gid: spec.gid,
1791 numa_node: spec.numa_node,
1792 affinity: resolved_affinity,
1793 num_workers: resolved_num_workers,
1794 group_idx,
1795 }
1796 }
1797
1798 /// Resolve an [`AffinityIntent`] to a [`ResolvedAffinity`] under
1799 /// the spawn-time gate: only `Inherit`, `Exact`, and
1800 /// `RandomSubset` carry enough information to resolve without
1801 /// scenario context (the caller supplies the `from` pool for
1802 /// `RandomSubset`, so per-worker sampling stays self-contained).
1803 /// Topology-aware variants (`SingleCpu`, `LlcAligned`,
1804 /// `CrossCgroup`, `SmtSiblingPair`) require a
1805 /// [`crate::topology::TestTopology`] / cpuset state that
1806 /// [`WorkloadHandle::spawn`] does not have, so they bail with an
1807 /// actionable diagnostic.
1808 ///
1809 /// `site` names the location of the affinity field for the bail
1810 /// message — `"WorkloadConfig::affinity"` for the primary group,
1811 /// `"composed[N].affinity"` for entries inside `composed`. Pinned
1812 /// across both call sites so the gate matches exactly and a
1813 /// future variant addition is rejected uniformly.
1814 pub(super) fn resolve_spawn_affinity(
1815 intent: &AffinityIntent,
1816 site: &str,
1817 ) -> Result<ResolvedAffinity> {
1818 match intent {
1819 AffinityIntent::Inherit => Ok(ResolvedAffinity::None),
1820 AffinityIntent::Exact(cpus) => {
1821 if cpus.is_empty() {
1822 anyhow::bail!(
1823 "{site} = AffinityIntent::Exact with empty CPU set \
1824 would produce EINVAL from sched_setaffinity; \
1825 use AffinityIntent::Inherit for no affinity \
1826 constraint",
1827 );
1828 }
1829 Ok(ResolvedAffinity::Fixed(cpus.clone()))
1830 }
1831 AffinityIntent::RandomSubset { from, count } => {
1832 if from.is_empty() {
1833 anyhow::bail!(
1834 "{site} = AffinityIntent::RandomSubset with empty \
1835 pool; use AffinityIntent::Inherit for no affinity \
1836 constraint",
1837 );
1838 }
1839 if *count == 0 {
1840 anyhow::bail!(
1841 "{site} = AffinityIntent::RandomSubset with \
1842 count=0; use AffinityIntent::Inherit for no \
1843 affinity constraint",
1844 );
1845 }
1846 Ok(ResolvedAffinity::Random {
1847 from: from.clone(),
1848 count: *count,
1849 })
1850 }
1851 AffinityIntent::SingleCpu
1852 | AffinityIntent::LlcAligned
1853 | AffinityIntent::CrossCgroup
1854 | AffinityIntent::SmtSiblingPair => {
1855 anyhow::bail!(
1856 "{site} = {:?} requires scenario context; use \
1857 AffinityIntent::Exact(set), \
1858 AffinityIntent::RandomSubset {{ from, count }}, \
1859 or AffinityIntent::Inherit when spawning directly \
1860 via WorkloadHandle::spawn. Topology-aware variants \
1861 resolve automatically inside #[ktstr_test] \
1862 scenarios.",
1863 intent,
1864 );
1865 }
1866 }
1867 }
1868
1869 /// Build the primary group's parameters from the top-level
1870 /// [`WorkloadConfig`] fields. `group_idx` is fixed to `0`.
1871 ///
1872 /// Synthesises a [`WorkSpec`] view of the top-level config
1873 /// fields and funnels through [`Self::from_work_spec`] so the
1874 /// field-by-field copy lives in exactly one place. The
1875 /// synthesised spec mirrors the resolved sibling values
1876 /// (`num_workers: Some(n)`, `affinity: Inherit`) — the spawn
1877 /// pipeline never reads it.
1878 ///
1879 /// `WorkloadConfig::affinity` is an [`AffinityIntent`]
1880 /// (type-unified with [`WorkSpec::affinity`]); resolution to
1881 /// [`ResolvedAffinity`] runs through
1882 /// [`Self::resolve_spawn_affinity`] under the same gate as
1883 /// [`Self::from_composed`]. Topology-aware variants
1884 /// (`SingleCpu`, `LlcAligned`, `CrossCgroup`, `SmtSiblingPair`)
1885 /// require scenario context; the scenario engine pre-resolves
1886 /// them via
1887 /// `crate::scenario::intent_for_spawn` (which round-trips
1888 /// `RandomSubset` verbatim and flattens topology-aware variants
1889 /// to `Exact`) before building [`WorkloadConfig`], so the gate
1890 /// only ever sees `Inherit`, `Exact`, or `RandomSubset` from
1891 /// this path.
1892 fn primary(config: &WorkloadConfig) -> Result<Self> {
1893 let resolved_affinity =
1894 Self::resolve_spawn_affinity(&config.affinity, "WorkloadConfig::affinity")?;
1895 let spec = WorkSpec {
1896 work_type: config.work_type.clone(),
1897 sched_policy: config.sched_policy,
1898 num_workers: Some(config.num_workers),
1899 affinity: AffinityIntent::Inherit,
1900 mem_policy: config.mem_policy.clone(),
1901 mpol_flags: config.mpol_flags,
1902 nice: config.nice,
1903 comm: config.comm.clone(),
1904 uid: config.uid,
1905 gid: config.gid,
1906 numa_node: config.numa_node,
1907 pcomm: None,
1908 workers_pct: None,
1909 };
1910 let mut params = Self::from_work_spec(&spec, 0, resolved_affinity, config.num_workers);
1911 // The park-after knob lives on WorkloadConfig, not the
1912 // synthesised WorkSpec above, so carry it in after the copy.
1913 params.park_after = config.park_after_iterations;
1914 // Same for the first-iteration signal knob.
1915 params.signal_first_iteration = config.signal_first_iteration;
1916 Ok(params)
1917 }
1918
1919 /// Resolve a composed [`WorkSpec`] into per-group parameters,
1920 /// applying the spawn-time rules documented on
1921 /// [`WorkloadConfig::composed`]:
1922 ///
1923 /// - `num_workers` must be `Some(n)`; the `None` default
1924 /// resolved by the scenario engine via
1925 /// `Ctx::workers_per_cgroup` is unreachable here. A `None`
1926 /// value is rejected with an actionable diagnostic.
1927 /// - `affinity` resolution runs through
1928 /// [`Self::resolve_spawn_affinity`] —
1929 /// [`AffinityIntent::Inherit`] (mapped to
1930 /// [`ResolvedAffinity::None`]),
1931 /// [`AffinityIntent::Exact`] (mapped to
1932 /// [`ResolvedAffinity::Fixed`]), and
1933 /// [`AffinityIntent::RandomSubset`] (mapped to
1934 /// [`ResolvedAffinity::Random`]) are accepted; topology-aware
1935 /// variants are rejected.
1936 ///
1937 /// Composed entries inherit the parent
1938 /// [`WorkloadConfig::clone_mode`]; [`WorkSpec`] has no
1939 /// `clone_mode` field of its own.
1940 fn from_composed(spec: &WorkSpec, group_idx: usize) -> Result<Self> {
1941 if spec.pcomm.is_some() {
1942 anyhow::bail!(
1943 "composed[{}].pcomm: pcomm via WorkloadHandle::spawn is not supported; \
1944 use WorkloadHandle::spawn_pcomm_cgroup or CgroupDef (apply_setup) — \
1945 spawn always forks one process per worker and never coalesces into \
1946 a thread-group leader",
1947 group_idx - 1,
1948 );
1949 }
1950 let num_workers = spec.num_workers.ok_or_else(|| {
1951 anyhow::anyhow!(
1952 "composed[{}].num_workers must be set explicitly at spawn time \
1953 (the Some/None resolution via Ctx::workers_per_cgroup is only \
1954 available through the scenario engine; \
1955 WorkloadHandle::spawn requires a concrete count)",
1956 group_idx - 1,
1957 )
1958 })?;
1959 let site = format!("composed[{}].affinity", group_idx - 1);
1960 let affinity = Self::resolve_spawn_affinity(&spec.affinity, &site)?;
1961 Ok(Self::from_work_spec(spec, group_idx, affinity, num_workers))
1962 }
1963}
1964
1965/// Shared per-group admission rules common to every spawn entry
1966/// point. Validates only the rules that are dispatch-agnostic —
1967/// `worker_group_size` divisibility, `chain_pipe_depth >= 2`,
1968/// `IdleChurn` zero-duration rejections, `IpcVariance` zero-knob
1969/// rejections. Dispatch-specific compatibility checks (CloneMode
1970/// vs WorkType, pcomm vs WorkType) stay at each entry point's
1971/// admission block since their reasoning differs per dispatch.
1972///
1973/// Called from [`WorkloadHandle::spawn`] (per-group inside
1974/// `groups`) and [`WorkloadHandle::spawn_pcomm_cgroup`] (per-group
1975/// inside its own `groups`). Centralises the rules so a future
1976/// addition (e.g. a new `WorkType` zero-rejection) lives in one
1977/// place.
1978pub(super) fn validate_workload_admission(group: &GroupParams) -> Result<()> {
1979 if let Some(group_size) = group.work_type.worker_group_size()
1980 && (group.num_workers == 0 || !group.num_workers.is_multiple_of(group_size))
1981 {
1982 return Err(WorkTypeValidationError::NonDivisibleWorkerCount {
1983 name: group.work_type.name().to_string(),
1984 group_idx: group.group_idx,
1985 group_size,
1986 num_workers: group.num_workers,
1987 }
1988 .into());
1989 }
1990 if let Some(depth) = group.work_type.chain_pipe_depth()
1991 && depth < 2
1992 {
1993 return Err(WorkTypeValidationError::InsufficientWakeChainDepth {
1994 depth,
1995 group_idx: group.group_idx,
1996 }
1997 .into());
1998 }
1999 if let WorkType::IdleChurn {
2000 burst_duration,
2001 sleep_duration,
2002 ..
2003 } = group.work_type
2004 {
2005 if burst_duration.is_zero() {
2006 return Err(WorkTypeValidationError::ZeroBurstDuration {
2007 group_idx: group.group_idx,
2008 }
2009 .into());
2010 }
2011 if sleep_duration.is_zero() {
2012 return Err(WorkTypeValidationError::ZeroSleepDuration {
2013 group_idx: group.group_idx,
2014 }
2015 .into());
2016 }
2017 }
2018 if let WorkType::IpcVariance {
2019 hot_iters,
2020 cold_iters,
2021 period_iters,
2022 } = group.work_type
2023 {
2024 if hot_iters == 0 {
2025 return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2026 field: "hot_iters",
2027 group_idx: group.group_idx,
2028 }
2029 .into());
2030 }
2031 if cold_iters == 0 {
2032 return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2033 field: "cold_iters",
2034 group_idx: group.group_idx,
2035 }
2036 .into());
2037 }
2038 if period_iters == 0 {
2039 return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2040 field: "period_iters",
2041 group_idx: group.group_idx,
2042 }
2043 .into());
2044 }
2045 }
2046 if let WorkType::TimerLatency { interval_us } = group.work_type
2047 && interval_us == 0
2048 {
2049 return Err(WorkTypeValidationError::ZeroTimerInterval {
2050 group_idx: group.group_idx,
2051 }
2052 .into());
2053 }
2054 if let WorkType::NetTraffic { frame_bytes, .. } = group.work_type
2055 && !(60..=1514).contains(&frame_bytes)
2056 {
2057 return Err(WorkTypeValidationError::NetTrafficFrameBytes {
2058 frame_bytes,
2059 group_idx: group.group_idx,
2060 }
2061 .into());
2062 }
2063 if let WorkType::IrqWake { frame_bytes, .. } = group.work_type
2064 && !(60..=1514).contains(&frame_bytes)
2065 {
2066 return Err(WorkTypeValidationError::IrqWakeFrameBytes {
2067 frame_bytes,
2068 group_idx: group.group_idx,
2069 }
2070 .into());
2071 }
2072 Ok(())
2073}
2074
2075/// Spawn a single thread-mode worker via [`std::thread::Builder`].
2076///
2077/// The thread closure runs `worker_main` directly with the same
2078/// per-worker arguments the fork dispatch passes, except `stop` is
2079/// a per-worker `Arc<AtomicBool>` instead of the global [`STOP`].
2080/// Start rendezvous uses an `mpsc::sync_channel(0)` because every
2081/// worker needs to block until the parent calls
2082/// [`WorkloadHandle::start`]; the parent then sends `()` to each
2083/// worker's `start_tx` to unblock them in order.
2084///
2085/// Also uses a capacity-1 `mpsc::sync_channel(1)` tid-publish
2086/// channel so this function blocks (up to 2 s) until the worker
2087/// thread reaches the `gettid()` publish point. That makes
2088/// [`WorkloadHandle::worker_pids`] safe to call immediately after
2089/// [`WorkloadHandle::spawn`] returns without first calling
2090/// [`WorkloadHandle::start`] — the publish happens BEFORE the
2091/// start handshake, so post-spawn tid reads always observe the
2092/// gettid() value (or spawn would have bailed with a "did not
2093/// publish gettid() within 2 s" error).
2094///
2095/// SIGUSR1 is process-wide and useless for per-thread stop control,
2096/// so this path does not install a signal handler. The parent flips
2097/// `stop` directly from [`WorkloadHandle::stop_and_collect`].
2098#[allow(clippy::too_many_arguments)]
2099pub(super) fn spawn_thread_worker(
2100 guard: &mut SpawnGuard,
2101 group: &GroupParams,
2102 affinity: Option<BTreeSet<usize>>,
2103 worker_pipe_fds: Option<(i32, i32)>,
2104 worker_futex: Option<(*mut u32, usize)>,
2105 iter_slot: *mut AtomicU64,
2106 phase_epoch: *mut AtomicU32,
2107 // First-iteration signal fd (`-1` = absent). Always `-1` on this
2108 // thread path: the eventfd is allocated only for fork dispatch, so a
2109 // thread-mode worker never signals (a plain `i32` copy, captured by
2110 // the closure below).
2111 first_iter_signal_fd: i32,
2112) -> Result<()> {
2113 use std::sync::Arc;
2114 use std::sync::atomic::AtomicI32;
2115 use std::sync::mpsc;
2116
2117 // SyncSender(0) — bounded rendezvous channel. The thread blocks
2118 // in `recv()` until the parent sends `()`; if the parent drops
2119 // the sender first (mid-spawn cleanup or early bail), `recv()`
2120 // returns `Err(Disconnected)` and the closure exits cleanly.
2121 let (start_tx, start_rx) = mpsc::sync_channel::<()>(0);
2122 // tid-publish handshake. Capacity 1 so the worker's send is
2123 // non-blocking; the parent's `recv_timeout` below blocks until
2124 // the worker reaches the publish point. Without this, the
2125 // post-spawn `WorkloadHandle::worker_pids()` race-reads the
2126 // initial 0 sentinel because the worker thread hasn't been
2127 // scheduled to the `tid_thread.store(my_tid, Release)` site yet
2128 // — surfaced as a spurious `Thread SpinWait worker reported
2129 // non-positive tid=0` test failure in
2130 // tests/worker_thread_integration.rs.
2131 let (tid_pub_tx, tid_pub_rx) = mpsc::sync_channel::<()>(1);
2132 let stop = Arc::new(AtomicBool::new(false));
2133 let tid = Arc::new(AtomicI32::new(0));
2134 // Per-worker exit eventfd: bumped by a Drop guard inside the
2135 // closure right before the thread returns its `WorkerReport`. The
2136 // parent's `join_thread_with_timeout` blocks in `epoll_wait` on
2137 // this fd instead of sleep-polling `is_finished`. Created with
2138 // `EFD_NONBLOCK` so the Drop-time `write` cannot block; counter
2139 // mode so a missed read just accumulates without losing the edge.
2140 let exit_evt = Arc::new(
2141 vmm_sys_util::eventfd::EventFd::new(libc::EFD_NONBLOCK)
2142 .context("create thread-worker exit eventfd")?,
2143 );
2144
2145 // Clone Arcs for the closure. The thread takes ownership of the
2146 // closure-side handles; the parent retains the originals via
2147 // ThreadWorker for stop signaling and tid reading.
2148 let stop_thread = Arc::clone(&stop);
2149 let tid_thread = Arc::clone(&tid);
2150 let exit_evt_thread = Arc::clone(&exit_evt);
2151 let work_type = group.work_type.clone();
2152 let park_after = group.park_after;
2153 let sched_policy = group.sched_policy;
2154 let mem_policy = group.mem_policy.clone();
2155 let mpol_flags = group.mpol_flags;
2156 let nice = group.nice;
2157 let comm = group.comm.clone();
2158 let uid = group.uid;
2159 let gid = group.gid;
2160 let numa_node = group.numa_node;
2161 let group_idx = group.group_idx;
2162 let num_workers = group.num_workers;
2163
2164 // The closure must be `Send` to cross the thread boundary.
2165 // `worker_pipe_fds` is `Option<(i32, i32)>` (Copy + Send), but
2166 // `worker_futex` and `iter_slot` are raw pointers and not
2167 // `Send` by default. The module-level `SendFutexPtr` and
2168 // `SendIterSlotPtr` newtypes round-trip the addresses through
2169 // `usize` so the closure's capture set is genuinely Send (no
2170 // raw-pointer field appears in the closure type).
2171 let futex_send = SendFutexPtr::new(worker_futex);
2172 let iter_slot_send = SendIterSlotPtr::new(iter_slot);
2173 let phase_epoch_send = SendPhaseEpochPtr::new(phase_epoch);
2174
2175 let join = std::thread::Builder::new()
2176 .name(format!("ktstr-worker-g{group_idx}-{}", guard.threads.len()))
2177 .spawn(move || {
2178 // Drop guard: signal the exit eventfd as the closure
2179 // unwinds, regardless of whether `worker_main` returned
2180 // normally or panicked. The parent's
2181 // `join_thread_with_timeout` blocks in `epoll_wait` on
2182 // this fd; a panic that bypassed the explicit signal
2183 // would otherwise leave the parent waiting until the
2184 // safety timerfd fires. Drop runs even under unwinding,
2185 // so this guard captures both the normal and panic
2186 // paths.
2187 struct WorkerExitSignal(std::sync::Arc<vmm_sys_util::eventfd::EventFd>);
2188 impl Drop for WorkerExitSignal {
2189 fn drop(&mut self) {
2190 let _ = self.0.write(1);
2191 }
2192 }
2193 let _exit_signal = WorkerExitSignal(exit_evt_thread);
2194
2195 // Publish gettid() so the parent can address this task
2196 // for sched_setaffinity and report it from worker_pids.
2197 // gettid() is the kernel TID; getpid() would return the
2198 // shared tgid, which collides across threads.
2199 let my_tid: libc::pid_t = unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t };
2200 // Release pairs with Acquire on the parent's
2201 // `tid.load()` sites so any reader observing a non-zero
2202 // tid also sees the worker's post-start state. Cheap on
2203 // every supported target (release-store on the Arc's
2204 // underlying AtomicI32 is a single instruction).
2205 tid_thread.store(my_tid, Ordering::Release);
2206 // Notify the parent that tid is published. Capacity-1
2207 // channel means this send is non-blocking; the parent's
2208 // `recv_timeout` below blocks until this fires so
2209 // post-spawn callers see populated tids. Drop the
2210 // sender after send so any future replays are no-ops.
2211 let _ = tid_pub_tx.send(());
2212 drop(tid_pub_tx);
2213
2214 // Block on start rendezvous. `Err(_)` means the parent
2215 // dropped start_tx before sending — return a sentinel
2216 // WorkerReport without doing any work.
2217 if start_rx.recv().is_err() {
2218 return WorkerReport {
2219 tid: my_tid,
2220 completed: false,
2221 group_idx,
2222 ..WorkerReport::default()
2223 };
2224 }
2225
2226 // Re-cast usize addresses back into raw pointers for
2227 // worker_main. SAFETY: the ownership and lifetime
2228 // arguments documented on `SendFutexPtr` /
2229 // `SendIterSlotPtr` ensure these pointers are still
2230 // live when worker_main dereferences them.
2231 let futex = futex_send.into_raw();
2232 let slot = iter_slot_send.into_raw();
2233 let epoch = phase_epoch_send.into_raw();
2234
2235 worker_main(
2236 affinity,
2237 work_type,
2238 sched_policy,
2239 mem_policy,
2240 mpol_flags,
2241 nice,
2242 comm.as_deref(),
2243 uid,
2244 gid,
2245 numa_node,
2246 worker_pipe_fds,
2247 futex,
2248 park_after,
2249 first_iter_signal_fd,
2250 slot,
2251 epoch,
2252 &stop_thread,
2253 group_idx,
2254 )
2255 })
2256 .with_context(|| {
2257 format!(
2258 "thread::spawn for worker {}/{} (group {}) failed",
2259 guard.threads.len() + 1,
2260 num_workers,
2261 group_idx,
2262 )
2263 })?;
2264
2265 // Block until the worker reaches the gettid() publish point.
2266 // 2 s deadline is generous: a fresh std::thread on a healthy
2267 // host reaches the closure body within microseconds; under
2268 // VM/CI pressure ~ms. Anything past 2 s indicates the worker
2269 // thread failed to schedule at all — surface as a typed error
2270 // rather than letting a downstream `worker_pids()` reader race-
2271 // read the initial 0 sentinel and tip over a tid-check assertion.
2272 match tid_pub_rx.recv_timeout(std::time::Duration::from_secs(2)) {
2273 Ok(()) => {}
2274 Err(mpsc::RecvTimeoutError::Timeout) => {
2275 anyhow::bail!(
2276 "spawn_thread_worker: worker {} (group {}) did not publish gettid() \
2277 within 2 s of thread::spawn — the worker thread failed to schedule. \
2278 Likely cause: host fd / thread-stack exhaustion, or the std runtime \
2279 thread pool is wedged. tid stays at 0; subsequent worker_pids() \
2280 reads would surface the sentinel.",
2281 guard.threads.len() + 1,
2282 group_idx,
2283 );
2284 }
2285 Err(mpsc::RecvTimeoutError::Disconnected) => {
2286 anyhow::bail!(
2287 "spawn_thread_worker: worker {} (group {}) closed tid-publish channel \
2288 without sending — the closure panicked before reaching the gettid() \
2289 publish point. Check the thread name's panic output for the cause.",
2290 guard.threads.len() + 1,
2291 group_idx,
2292 );
2293 }
2294 }
2295
2296 guard.threads.push(ThreadWorker {
2297 tid,
2298 stop,
2299 start_tx: Some(start_tx),
2300 join: Some(join),
2301 exit_evt,
2302 });
2303 Ok(())
2304}
2305
2306/// Per-group resource indices into the [`SpawnGuard`]'s flat
2307/// vectors used by [`spawn_pcomm_container`] to compute per-thread
2308/// pipe / chain / futex / iter-slot addresses inside the forked
2309/// container. Built by [`WorkloadHandle::spawn_pcomm_cgroup`] in
2310/// lockstep with the per-group resource allocation pass so the
2311/// container's threads can address their own group's resources from
2312/// the inherited shared regions.
2313#[derive(Clone, Copy, Debug)]
2314pub(super) struct PcommGroupResources {
2315 /// Offset into the iter_counters MAP_SHARED region for this
2316 /// group's first worker. Subsequent workers occupy
2317 /// `iter_offset + i` for `i in 0..num_workers`.
2318 pub iter_offset: usize,
2319 /// Index of this group's first pipe pair in `guard.pipe_pairs`
2320 /// (only meaningful when `needs_pipes` is true).
2321 pub pipe_pair_base: usize,
2322 /// Index of this group's first chain pipe in `guard.chain_pipes`
2323 /// (only meaningful when `chain_depth` is `Some(_)`).
2324 pub chain_pipes_base: usize,
2325 /// Index of this group's first futex region in `guard.futex_ptrs`
2326 /// (only meaningful when `needs_futex` is true).
2327 pub futex_ptrs_base: usize,
2328 /// True when the group's `WorkType` requires inter-worker pipe
2329 /// pairs (PipeIo / CachePipe).
2330 pub needs_pipes: bool,
2331 /// `Some(depth)` when the group's `WorkType` requires a chain
2332 /// pipe ring (WakeChain { wake: Pipe }); `None` otherwise.
2333 pub chain_depth: Option<usize>,
2334 /// True when the group's `WorkType` requires a MAP_SHARED futex
2335 /// region (FutexPingPong / FutexFanOut / FanOutCompute /
2336 /// MutexContention / etc.).
2337 pub needs_futex: bool,
2338 /// Worker-group-size for per-worker `pos` computation inside
2339 /// the futex region (e.g. 2 for FutexPingPong, the messenger /
2340 /// receiver split for FutexFanOut). Defaulted to 2 when the
2341 /// `WorkType` does not declare a group_size.
2342 pub futex_group_size: usize,
2343}
2344
2345/// Fork one thread-group leader hosting every worker thread for the
2346/// supplied groups: fork the leader, set its `comm` to `pcomm` via
2347/// `prctl(PR_SET_NAME)`, then spawn `groups[k].num_workers` worker
2348/// threads per group inside it. The [`WorkSpec::pcomm`] builder
2349/// rejects > 15 bytes (TASK_COMM_LEN-1), so the framework never
2350/// feeds the kernel a name `__set_task_comm` would truncate. Every
2351/// worker thread shares the leader's tgid, so
2352/// `task->group_leader->comm == pcomm` byte-for-byte. Models real
2353/// workloads like `chrome` (pcomm) hosting `ThreadPoolForeg` and
2354/// `GPU Process` (per-thread comm via [`WorkSpec::comm`]) or `java`
2355/// (pcomm) hosting `GC Thread` and `C2 CompilerThre`.
2356///
2357/// `groups` carries per-group [`GroupParams`] and `resources` carries
2358/// the parallel [`PcommGroupResources`] pre-built by
2359/// [`WorkloadHandle::spawn_pcomm_cgroup`] from the surrounding
2360/// per-group resource-allocation pass; the two slices have identical
2361/// length and `resources[k]` describes the indices owned by
2362/// `groups[k]`.
2363///
2364/// `container_uid` / `container_gid` are applied via `setresuid` /
2365/// `setresgid` once on the leader before spawning threads — the
2366/// leader takes the configured process credentials, and each worker
2367/// thread additionally re-applies its own merged credentials inside
2368/// `worker_main` at thread creation time (idempotent when the merge
2369/// produced the same value, which is the common case for a
2370/// CgroupDef-level default flowing through `merged_works`).
2371///
2372/// # Wire format
2373///
2374/// The parent collects via the same report pipe used by conventional
2375/// fork workers:
2376///
2377/// 1. After the start handshake, the leader writes one `b'r'` ready
2378/// byte to the report pipe after all threads are spawned, a
2379/// stronger guarantee than the conventional fork worker's
2380/// pre-loop byte.
2381/// 2. After all worker threads have joined, the leader serializes
2382/// `Vec<WorkerReport>` (one entry per worker thread, in
2383/// `(group_idx, within-group order)` traversal order) via
2384/// `serde_json::to_vec` and writes the bytes to the report pipe
2385/// before `_exit(0)`. The parent decodes via
2386/// `serde_json::from_slice::<Vec<WorkerReport>>`. The report pipe
2387/// is sized at 8 MiB; parent `read_to_end` drains concurrently
2388/// with the leader's `write_all` so payloads exceeding the pipe
2389/// size still drain correctly.
2390///
2391/// On a decode-failure or short payload, the parent emits one
2392/// sentinel report per expected worker so per-group filtering and
2393/// `assert_not_stuck` see the correct cardinality.
2394///
2395/// # Lifecycle
2396///
2397/// - `PR_SET_PDEATHSIG(SIGKILL)`: when the parent dies, the kernel
2398/// sends SIGKILL to the LEADER thread (the calling task at fork
2399/// time). SIGKILL on any thread is fatal-to-tgid: the kernel
2400/// routes it through `complete_signal` → `do_group_exit` →
2401/// `zap_other_threads`, which sets `SIGNAL_GROUP_EXIT` on every
2402/// sibling thread and walks the thread list signalling each. So
2403/// the cascade to worker threads happens via
2404/// `zap_other_threads`, NOT via per-task PDEATHSIG inheritance
2405/// (PDEATHSIG itself is per-task and is cleared on clone). Net:
2406/// parent death → leader's SIGKILL → all worker threads die,
2407/// so the container cannot outlive the harness.
2408/// - `setpgid(0, 0)`: the leader becomes its own process group
2409/// leader so the parent's stop/kill `killpg` reaches any
2410/// descendants the workers spawn.
2411/// - SIGUSR1 handler installed pre-thread-spawn: any
2412/// `kill(leader_pid, SIGUSR1)` from the parent flips the leader's
2413/// STOP, which every worker thread observes via
2414/// `stop_requested(&STOP)` on its next loop check.
2415/// - `prctl(PR_SET_NAME, pcomm)`: sets the leader's `task->comm`,
2416/// which becomes `task->group_leader->comm` for every spawned
2417/// thread byte-for-byte (the [`WorkSpec::pcomm`] builder rejects
2418/// names longer than 15 bytes — `TASK_COMM_LEN - 1` — so the
2419/// framework never feeds the kernel a name `__set_task_comm`
2420/// would truncate).
2421/// - `setresuid(container_uid, ...)` / `setresgid(container_gid,
2422/// ...)` (if specified) apply once before the start-byte poll.
2423/// - Threads spawn AFTER the start byte arrives, so `start()`'s
2424/// serial start-byte writes still gate work entry (the parent
2425/// moves the leader to its cgroup before sending start).
2426#[allow(clippy::too_many_arguments)]
2427pub(super) fn spawn_pcomm_container(
2428 guard: &mut SpawnGuard,
2429 pcomm: &str,
2430 container_uid: Option<u32>,
2431 container_gid: Option<u32>,
2432 groups: &[GroupParams],
2433 resources: &[PcommGroupResources],
2434) -> Result<()> {
2435 debug_assert_eq!(
2436 groups.len(),
2437 resources.len(),
2438 "spawn_pcomm_container: groups / resources must have the same length",
2439 );
2440
2441 // Allocate the container's report and start pipes. Parent holds
2442 // report_fds[0] (read end) and start_fds[1] (write end); the
2443 // container holds the inverse ends post-fork. O_CLOEXEC matches
2444 // the defense-in-depth posture used by every other pipe in the
2445 // spawn pipeline — defends against future exec paths that could
2446 // otherwise leak the fd into a helper.
2447 let mut report_fds = [0i32; 2];
2448 if unsafe { libc::pipe2(report_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
2449 anyhow::bail!(
2450 "pcomm container (pcomm={pcomm:?}): report pipe2 failed: {}",
2451 std::io::Error::last_os_error(),
2452 );
2453 }
2454 // Grow the report pipe to 8 MiB so the container's `write_all` of
2455 // the JSON-encoded `Vec<WorkerReport>` issues without blocking
2456 // for moderate worker counts. For larger payloads the parent's
2457 // `read_to_end` drains concurrently with the container's
2458 // `write_all`, so a pipe overflow degrades to extra wakeups
2459 // rather than truncation. Best-effort failure (older kernel /
2460 // EPERM) leaves the default size in place — large payloads
2461 // still drain via the concurrent flow but with more wake cycles.
2462 const REPORT_PIPE_SIZE: libc::c_int = 8 * 1024 * 1024;
2463 let prev_size = unsafe { libc::fcntl(report_fds[1], libc::F_SETPIPE_SZ, REPORT_PIPE_SIZE) };
2464 if prev_size < 0 {
2465 let err = std::io::Error::last_os_error();
2466 tracing::warn!(
2467 pcomm,
2468 requested_size = REPORT_PIPE_SIZE,
2469 %err,
2470 "F_SETPIPE_SZ on pcomm container report pipe failed; falling back \
2471 to default pipe capacity",
2472 );
2473 }
2474 let mut start_fds = [0i32; 2];
2475 if unsafe { libc::pipe2(start_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
2476 unsafe {
2477 libc::close(report_fds[0]);
2478 libc::close(report_fds[1]);
2479 }
2480 anyhow::bail!(
2481 "pcomm container (pcomm={pcomm:?}): start pipe2 failed: {}",
2482 std::io::Error::last_os_error(),
2483 );
2484 }
2485
2486 // Block SIGUSR1 across fork so the container inherits a blocked
2487 // mask; the post-fork install + restore pattern matches the
2488 // conventional fork worker (see `spawn_group`'s fork dispatch
2489 // for the full rationale).
2490 let mut old_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
2491 let mut block_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
2492 let psm_block_rc = unsafe {
2493 libc::sigemptyset(&mut block_mask);
2494 libc::sigaddset(&mut block_mask, libc::SIGUSR1);
2495 libc::pthread_sigmask(libc::SIG_BLOCK, &block_mask, &mut old_mask)
2496 };
2497 if psm_block_rc != 0 {
2498 tracing::warn!(
2499 rc = psm_block_rc,
2500 pcomm,
2501 "pthread_sigmask(SIG_BLOCK, SIGUSR1) failed pre-fork for pcomm container; \
2502 container inherits unblocked SIGUSR1 and may terminate on default \
2503 action before installing handler",
2504 );
2505 }
2506
2507 // Sum of all groups' workers — matches `WorkerReport` cardinality
2508 // the parent expects.
2509 let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
2510
2511 let pid = unsafe { libc::fork() };
2512 match pid {
2513 -1 => {
2514 let psm_restore_rc = unsafe {
2515 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
2516 };
2517 if psm_restore_rc != 0 {
2518 tracing::warn!(
2519 rc = psm_restore_rc,
2520 "pthread_sigmask(SIG_SETMASK) failed restoring mask after pcomm \
2521 container fork failure; SIGUSR1 may remain blocked in this thread",
2522 );
2523 }
2524 unsafe {
2525 libc::close(report_fds[0]);
2526 libc::close(report_fds[1]);
2527 libc::close(start_fds[0]);
2528 libc::close(start_fds[1]);
2529 }
2530 anyhow::bail!(
2531 "pcomm container (pcomm={pcomm:?}): fork failed: {}",
2532 std::io::Error::last_os_error(),
2533 );
2534 }
2535 0 => {
2536 // Container child. Mirror the conventional fork worker's
2537 // post-fork init: PDEATHSIG, getppid orphan check,
2538 // setpgid, SIGUSR1 handler install + mask restore. This
2539 // is the exact same defensive sequence; the only
2540 // structural difference is what happens AFTER the
2541 // start-byte handshake (thread spawn instead of inline
2542 // worker_main).
2543 unsafe {
2544 libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL);
2545 }
2546 if std::env::var_os(crate::KTSTR_GUEST_INIT_ENV).is_none()
2547 && unsafe { libc::getppid() } == 1
2548 {
2549 exit_child_success();
2550 }
2551 unsafe {
2552 libc::setpgid(0, 0);
2553 }
2554 STOP.store(false, Ordering::Relaxed);
2555 let sig_prev = unsafe {
2556 libc::signal(
2557 libc::SIGUSR1,
2558 sigusr1_handler as *const () as libc::sighandler_t,
2559 )
2560 };
2561 if sig_prev == libc::SIG_ERR {
2562 let errno = std::io::Error::last_os_error();
2563 eprintln!(
2564 "ktstr: signal(SIGUSR1) install failed in pcomm container: {errno}; \
2565 graceful stop unavailable, killpg escalation will reap"
2566 );
2567 }
2568 let psm_unblock_rc = unsafe {
2569 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
2570 };
2571 if psm_unblock_rc != 0 {
2572 eprintln!(
2573 "ktstr: pthread_sigmask(SIG_SETMASK) unblock failed in pcomm \
2574 container: rc={psm_unblock_rc}; SIGUSR1 stays blocked, killpg \
2575 escalation will reap"
2576 );
2577 }
2578 // Close the parent's ends. Keep report_fds[1] (the
2579 // container's write end) and start_fds[0] (the
2580 // container's read end).
2581 unsafe {
2582 libc::close(report_fds[0]);
2583 libc::close(start_fds[1]);
2584 }
2585
2586 // Inherited-fd sweep. The container forked from the
2587 // harness inherits every fd in the parent's table —
2588 // including pipe fds from OTHER live `WorkloadHandle`s
2589 // and any harness-only fds (tracing subscribers etc.).
2590 // Close everything not in the keep-set:
2591 // - 0 / 1 / 2 (stdio).
2592 // - `report_fds[1]` (this container's write end).
2593 // - `start_fds[0]` (this container's read end).
2594 // - This spawn's `pipe_pairs` (per-pair inter-worker
2595 // fds) and `chain_pipes` (per-chain ring fds) —
2596 // worker threads will use them once they start.
2597 //
2598 // MAP_SHARED regions (futex / iter_counters) are mmap
2599 // memory inherited via CLONE_VM, not fd entries; nothing
2600 // to keep on that axis. The sweep walks
2601 // `/proc/self/fd` once and uses raw `libc::close` on
2602 // each non-kept fd; failure to close (EBADF) is benign
2603 // — the entry is gone by the time we get to it. Heap
2604 // allocation here is acceptable post-fork (the
2605 // conventional fork worker also heap-allocates in
2606 // `worker_main`).
2607 let mut keep_fds: std::collections::BTreeSet<i32> = std::collections::BTreeSet::new();
2608 keep_fds.insert(0);
2609 keep_fds.insert(1);
2610 keep_fds.insert(2);
2611 keep_fds.insert(report_fds[1]);
2612 keep_fds.insert(start_fds[0]);
2613 for (ab, ba) in &guard.pipe_pairs {
2614 keep_fds.insert(ab[0]);
2615 keep_fds.insert(ab[1]);
2616 keep_fds.insert(ba[0]);
2617 keep_fds.insert(ba[1]);
2618 }
2619 for chain in &guard.chain_pipes {
2620 for pipe in chain {
2621 keep_fds.insert(pipe[0]);
2622 keep_fds.insert(pipe[1]);
2623 }
2624 }
2625 if let Ok(entries) = std::fs::read_dir("/proc/self/fd") {
2626 let mut to_close: Vec<i32> = Vec::new();
2627 for entry in entries.flatten() {
2628 if let Some(name) = entry.file_name().to_str()
2629 && let Ok(fd) = name.parse::<i32>()
2630 && !keep_fds.contains(&fd)
2631 {
2632 to_close.push(fd);
2633 }
2634 }
2635 for fd in to_close {
2636 unsafe {
2637 libc::close(fd);
2638 }
2639 }
2640 } else {
2641 eprintln!(
2642 "ktstr: pcomm container (pcomm={pcomm:?}): /proc/self/fd \
2643 read_dir failed; inherited-fd sweep skipped",
2644 );
2645 }
2646
2647 // prctl(PR_SET_NAME, pcomm). Setting it on the container's
2648 // tgid leader makes `task->group_leader->comm == pcomm`
2649 // for every spawned thread (kernel/sys.c::prctl_set_name).
2650 // The WorkSpec::pcomm builder rejects empty strings,
2651 // interior NULs, and names > 15 bytes (TASK_COMM_LEN - 1)
2652 // at declaration time, so neither CString construction
2653 // nor `__set_task_comm`'s `min(strlen, TASK_COMM_LEN-1)`
2654 // truncation can fire for any value that arrives here.
2655 // `unwrap_or_default` is a defensive fall-through to an
2656 // empty CString — the kernel accepts it; the leader's
2657 // comm is set to the empty string and the failure is
2658 // surfaced to stderr.
2659 let c_pcomm = std::ffi::CString::new(pcomm).unwrap_or_default();
2660 let prctl_rc = unsafe { libc::prctl(libc::PR_SET_NAME, c_pcomm.as_ptr()) };
2661 if prctl_rc != 0 {
2662 let errno = std::io::Error::last_os_error();
2663 eprintln!(
2664 "ktstr: prctl(PR_SET_NAME, {pcomm:?}) failed on pcomm container: {errno}",
2665 );
2666 }
2667
2668 // Apply container-level credentials. Order matters:
2669 // setresgid first (while still uid=0 we can change the
2670 // primary group), setresuid second (after the uid drop
2671 // the process loses CAP_SETGID and a later gid change
2672 // would EPERM). Failures are surfaced to stderr but do
2673 // not halt the container — the threads inside still
2674 // run with the inherited credentials, and worker_main's
2675 // own setresuid / setresgid calls (which run after this)
2676 // act as a second-line defense for any per-thread merge
2677 // that produced a different value than the container
2678 // default.
2679 if let Some(gid) = container_gid {
2680 let rc = unsafe { libc::setresgid(gid, gid, gid) };
2681 if rc != 0 {
2682 let errno = std::io::Error::last_os_error();
2683 eprintln!("ktstr: setresgid({gid}) failed on pcomm container: {errno}");
2684 }
2685 }
2686 if let Some(uid) = container_uid {
2687 let rc = unsafe { libc::setresuid(uid, uid, uid) };
2688 if rc != 0 {
2689 let errno = std::io::Error::last_os_error();
2690 eprintln!("ktstr: setresuid({uid}) failed on pcomm container: {errno}");
2691 }
2692 }
2693
2694 // Wait for the parent's start byte (poll with 30s
2695 // timeout — same budget as the conventional fork
2696 // worker). Match the conventional worker's behaviour:
2697 // on `poll <= 0` call exit_child_error so the parent's
2698 // collect path observes a missing report and emits
2699 // sentinels.
2700 let mut pfd = libc::pollfd {
2701 fd: start_fds[0],
2702 events: libc::POLLIN,
2703 revents: 0,
2704 };
2705 let ret = unsafe { libc::poll(&mut pfd, 1, 30_000) };
2706 if ret <= 0 {
2707 exit_child_error();
2708 }
2709 let mut buf = [0u8; 1];
2710 {
2711 let mut f = unsafe { std::fs::File::from_raw_fd(start_fds[0]) };
2712 let _ = f.read_exact(&mut buf);
2713 drop(f);
2714 }
2715
2716 // Unlike fork workers, the pcomm container spawns threads after
2717 // the start byte. A latched STOP from a SIGUSR1 race during mask
2718 // restore would produce N zero-work threads. Reset so threads
2719 // get a fair start.
2720 STOP.store(false, Ordering::Relaxed);
2721
2722 // Spawn worker threads for every group. Each thread
2723 // computes its own affinity / pipe-fd / futex /
2724 // iter_slot from its (group_idx, within-group index)
2725 // pair, matching the conventional fork loop's per-worker
2726 // computation but inside the container's address space
2727 // (so threads share the parent-allocated MAP_SHARED
2728 // regions and inter-worker pipes inherited via fork).
2729 // Each spawned thread gets its own `EventFd` cloned into
2730 // the closure. A `WorkerExitSignal` Drop guard inside the
2731 // closure writes 1 to that fd as the closure unwinds —
2732 // covering both the normal-return and panic paths. The
2733 // container's join phase below `epoll_wait`s on every
2734 // eventfd, so a thread's exit edge is delivered without
2735 // any polling sleep. This mirrors `join_thread_with_timeout`'s
2736 // pattern but without a timer fd: the parent's
2737 // `stop_and_collect` SIGKILL is the only timeout authority,
2738 // per the user's "event-driven only, no sleeps or
2739 // timeouts" rule.
2740 let mut joins: Vec<(
2741 std::thread::JoinHandle<WorkerReport>,
2742 std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
2743 )> = Vec::with_capacity(total_workers);
2744 // Snapshot the guard's resource bookkeeping we need
2745 // inside each thread closure. The guard is borrowed
2746 // mutably by the surrounding `spawn_pcomm_cgroup` and
2747 // dropped on the parent side after the function returns;
2748 // after fork the container has its own copy of every
2749 // field, and these snapshots are clones owned by the
2750 // container's process so the closures can capture them
2751 // without aliasing the guard.
2752 let pipe_pairs_snapshot: Vec<([i32; 2], [i32; 2])> = guard.pipe_pairs.clone();
2753 let chain_pipes_snapshot: Vec<Vec<[i32; 2]>> = guard.chain_pipes.clone();
2754 let futex_ptrs_snapshot: Vec<*mut u32> = guard.futex_ptrs.clone();
2755 let iter_counters_base = guard.iter_counters;
2756 let phase_epoch_base = guard.phase_epoch;
2757
2758 for (group, res) in groups.iter().zip(resources.iter()) {
2759 let num_workers = group.num_workers;
2760 for i in 0..num_workers {
2761 // Per-thread affinity resolution. Random-subset
2762 // affinity samples a fresh subset per thread, so
2763 // each call produces an independent BTreeSet.
2764 // Fixed affinity returns the same BTreeSet for
2765 // every thread.
2766 //
2767 // resolve_affinity bails on caller bugs (count==0,
2768 // empty pool). In the pcomm container's forked-
2769 // child context we cannot propagate Err up; the
2770 // convention is eprintln + [`exit_child_error`]
2771 // (the named helper used by every forked-child
2772 // failure site in this file — start-byte
2773 // poll-timeout bail, EventFd::new bail,
2774 // thread::Builder::spawn bail) so the parent
2775 // collects a missing-report rather than silently
2776 // spawning workers without the affinity the test
2777 // author requested. The prior degrade-to-None
2778 // behaviour was a silent-drop bug we close here.
2779 let affinity = match resolve_affinity(&group.affinity) {
2780 Ok(a) => a,
2781 Err(e) => {
2782 eprintln!(
2783 "ktstr: pcomm container (group {}): resolve_affinity \
2784 for thread {i}/{num_workers} failed: {e:#}. \
2785 Aborting container — refusing to spawn workers \
2786 without the requested affinity (no silent drops).",
2787 group.group_idx,
2788 );
2789 // exit_group(2) tears down every sibling
2790 // worker thread spawned in earlier loop
2791 // iterations along with this one — the
2792 // intended fail-stop behaviour (parent
2793 // observes a missing report and surfaces
2794 // a sentinel).
2795 exit_child_error();
2796 }
2797 };
2798
2799 let worker_pipe_fds: Option<(i32, i32)> = if res.needs_pipes {
2800 let pair_idx = res.pipe_pair_base + i / 2;
2801 let (ab, ba) = &pipe_pairs_snapshot[pair_idx];
2802 if i % 2 == 0 {
2803 Some((ba[0], ab[1]))
2804 } else {
2805 Some((ab[0], ba[1]))
2806 }
2807 } else if let Some(depth) = res.chain_depth
2808 && depth > 0
2809 {
2810 let chain_idx = res.chain_pipes_base + i / depth;
2811 let stage = i % depth;
2812 let prev_stage = (stage + depth - 1) % depth;
2813 let chain = &chain_pipes_snapshot[chain_idx];
2814 Some((chain[prev_stage][0], chain[stage][1]))
2815 } else {
2816 None
2817 };
2818
2819 let worker_futex: Option<(*mut u32, usize)> = if res.needs_futex {
2820 let futex_group_idx = res.futex_ptrs_base + i / res.futex_group_size;
2821 let pos = i % res.futex_group_size;
2822 Some((futex_ptrs_snapshot[futex_group_idx], pos))
2823 } else {
2824 None
2825 };
2826
2827 let iter_slot: *mut AtomicU64 = if !iter_counters_base.is_null() {
2828 unsafe { iter_counters_base.add(res.iter_offset + i) }
2829 } else {
2830 std::ptr::null_mut()
2831 };
2832
2833 // Round raw pointers through Send-newtypes so
2834 // the closure's auto-Send check passes (same
2835 // wrapper pattern `spawn_thread_worker` uses).
2836 let futex_send = SendFutexPtr::new(worker_futex);
2837 let iter_slot_send = SendIterSlotPtr::new(iter_slot);
2838 let phase_epoch_send = SendPhaseEpochPtr::new(phase_epoch_base);
2839
2840 let work_type = group.work_type.clone();
2841 let park_after = group.park_after;
2842 let sched_policy = group.sched_policy;
2843 let mem_policy = group.mem_policy.clone();
2844 let mpol_flags = group.mpol_flags;
2845 let nice = group.nice;
2846 let comm = group.comm.clone();
2847 let uid = group.uid;
2848 let gid = group.gid;
2849 let numa_node = group.numa_node;
2850 let group_idx = group.group_idx;
2851
2852 // Per-thread exit eventfd. Cloned into the
2853 // closure as `Arc<EventFd>`; the closure-local
2854 // `WorkerExitSignal` Drop guard writes 1 to the
2855 // fd on every exit path (normal return AND
2856 // panic). The container's join phase below
2857 // `epoll_wait`s on every eventfd to deliver the
2858 // exit edge without polling. EFD_NONBLOCK keeps
2859 // the Drop-time `write` from blocking — counter
2860 // mode means the writer just bumps the counter
2861 // without losing the edge.
2862 let exit_evt = match vmm_sys_util::eventfd::EventFd::new(libc::EFD_NONBLOCK) {
2863 Ok(efd) => std::sync::Arc::new(efd),
2864 Err(e) => {
2865 // Hard fail: continuing with fewer
2866 // threads silently breaks the
2867 // workload's worker count and the
2868 // parent's report count. exit_child_error
2869 // so the parent's sentinel path observes
2870 // a missing payload and emits one
2871 // sentinel per expected report. Reuses
2872 // the same fail-stop contract as the
2873 // start-byte poll timeout (`ret <=
2874 // 0`).
2875 eprintln!(
2876 "ktstr: pcomm container (group {}): EventFd::new for \
2877 worker {}/{num_workers} failed: {e}; aborting container",
2878 group.group_idx,
2879 i + 1,
2880 );
2881 exit_child_error();
2882 }
2883 };
2884 let exit_evt_thread = std::sync::Arc::clone(&exit_evt);
2885
2886 let join = std::thread::Builder::new()
2887 .name(format!("ktstr-pcomm-g{group_idx}-{i}"))
2888 .spawn(move || {
2889 // Drop guard: bumps the eventfd as the
2890 // closure unwinds. The write is
2891 // non-blocking (EFD_NONBLOCK) and the
2892 // counter accumulates, so a missed
2893 // read just queues the next read's
2894 // value — no edge loss. Drop runs
2895 // under both normal return and
2896 // unwinding (panic), so the container's
2897 // join phase observes EVERY thread's
2898 // exit, including panicked ones.
2899 struct WorkerExitSignal(std::sync::Arc<vmm_sys_util::eventfd::EventFd>);
2900 impl Drop for WorkerExitSignal {
2901 fn drop(&mut self) {
2902 let _ = self.0.write(1);
2903 }
2904 }
2905 let _exit_signal = WorkerExitSignal(exit_evt_thread);
2906
2907 let futex = futex_send.into_raw();
2908 let slot = iter_slot_send.into_raw();
2909 let epoch = phase_epoch_send.into_raw();
2910 worker_main(
2911 affinity,
2912 work_type,
2913 sched_policy,
2914 mem_policy,
2915 mpol_flags,
2916 nice,
2917 comm.as_deref(),
2918 uid,
2919 gid,
2920 numa_node,
2921 worker_pipe_fds,
2922 futex,
2923 park_after,
2924 // pcomm containers run threads inside a
2925 // forked container; the first-iteration
2926 // eventfd is allocated only on the plain
2927 // fork-dispatch path, so pcomm workers never
2928 // signal.
2929 -1,
2930 slot,
2931 epoch,
2932 &STOP,
2933 group_idx,
2934 )
2935 });
2936 match join {
2937 Ok(j) => joins.push((j, exit_evt)),
2938 Err(e) => {
2939 // Hard fail: see EventFd path above.
2940 // A partially-spawned container leaves
2941 // the parent guessing about which
2942 // reports are real vs missing;
2943 // exit_child_error collapses to the
2944 // parent's sentinel path with
2945 // deterministic cardinality.
2946 eprintln!(
2947 "ktstr: pcomm container (group {}): thread::spawn for \
2948 worker {}/{num_workers} failed: {e}; aborting container",
2949 group.group_idx,
2950 i + 1,
2951 );
2952 exit_child_error();
2953 }
2954 }
2955 }
2956 }
2957
2958 // Publish the ready byte AFTER every worker thread is
2959 // alive. The parent's barrier polls every report fd for
2960 // POLLIN with a bounded deadline; the byte's correct
2961 // semantic is "the container has spawned every worker
2962 // thread and they are now running" — earlier (pre-spawn)
2963 // is observable to the parent before threads exist, which
2964 // races with `set_affinity(idx)` / `worker_pids()` calls
2965 // the harness may issue between the parent's barrier wake
2966 // and the work loop. Each thread starts work immediately
2967 // upon spawn (no per-thread handshake gate) since the
2968 // CONTAINER's start-byte poll above is the workload's
2969 // single start gate; once the container observes the
2970 // start byte, every thread it spawns is by construction
2971 // intended to run. Done as a raw `libc::write` to avoid
2972 // taking ownership of `report_fds[1]` (we still need it
2973 // for the JSON write below).
2974 let ready_byte: u8 = b'r';
2975 unsafe {
2976 libc::write(
2977 report_fds[1],
2978 &ready_byte as *const u8 as *const libc::c_void,
2979 1,
2980 );
2981 }
2982
2983 // Join every thread via `epoll_wait` on per-thread exit
2984 // eventfds. Each thread's `WorkerExitSignal` Drop guard
2985 // writes 1 to its fd as the closure unwinds (normal
2986 // return AND panic), so the container's `epoll_wait`
2987 // observes EVERY thread's exit edge without polling.
2988 // No timeout: per the user's "event-driven only, no
2989 // sleeps or timeouts" rule, the container blocks in
2990 // `epoll_wait(-1)` until each fd fires. The parent's
2991 // `stop_and_collect` 5s collect deadline + SIGKILL is
2992 // the only timeout authority; if a thread hangs (no
2993 // Drop guard fires — no realistic path to that under
2994 // the closures we spawn), the parent SIGKILLs the
2995 // container, the kernel tears down every thread, and
2996 // the container exits via the signal.
2997 //
2998 // Setup-failure fallback: if `Epoll::new` or any
2999 // `epoll.ctl(Add)` fails, fall back to blocking
3000 // `JoinHandle::join` per thread — that itself is
3001 // event-driven from the kernel's perspective (the
3002 // syscall blocks on the thread's exit signal, no
3003 // userspace polling). The fallback path is rare
3004 // (epoll_create1 failure means out of fds or out of
3005 // memory, both of which are pre-existing failure modes
3006 // the conventional fork worker also has).
3007 use std::os::unix::io::AsRawFd;
3008 use vmm_sys_util::epoll::{ControlOperation, Epoll, EpollEvent, EventSet};
3009 let mut indexed_joins: Vec<(
3010 usize,
3011 std::thread::JoinHandle<WorkerReport>,
3012 std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
3013 )> = joins
3014 .into_iter()
3015 .enumerate()
3016 .map(|(i, (j, evt))| (i, j, evt))
3017 .collect();
3018 let mut reports: Vec<WorkerReport> = Vec::with_capacity(indexed_joins.len());
3019
3020 let epoll_setup: Option<Epoll> = match Epoll::new() {
3021 Ok(ep) => {
3022 let mut ok = true;
3023 for (idx, _, evt) in &indexed_joins {
3024 if let Err(e) = ep.ctl(
3025 ControlOperation::Add,
3026 evt.as_raw_fd(),
3027 EpollEvent::new(EventSet::IN, *idx as u64),
3028 ) {
3029 eprintln!(
3030 "ktstr: pcomm container (pcomm={pcomm:?}): epoll.ctl(Add) \
3031 for thread {idx} failed: {e}; falling back to blocking \
3032 join per thread",
3033 );
3034 ok = false;
3035 break;
3036 }
3037 }
3038 if ok { Some(ep) } else { None }
3039 }
3040 Err(e) => {
3041 eprintln!(
3042 "ktstr: pcomm container (pcomm={pcomm:?}): Epoll::new failed: {e}; \
3043 falling back to blocking join per thread",
3044 );
3045 None
3046 }
3047 };
3048
3049 if let Some(epoll) = epoll_setup {
3050 // Event-driven join loop. `epoll_wait(-1, …)` blocks
3051 // until at least one eventfd fires, which only
3052 // happens when a thread's `WorkerExitSignal` Drop
3053 // guard runs. The wake delivers `events[k].data()`
3054 // as the index of the finished thread; we
3055 // deregister, locate the entry, and `join()` it
3056 // (the `join()` call may briefly wait for the OS
3057 // thread to fully exit after the Drop guard, but
3058 // that wait is itself event-driven inside the
3059 // kernel — no userspace polling).
3060 let mut events_buf: Vec<EpollEvent> =
3061 vec![EpollEvent::default(); indexed_joins.len().max(1)];
3062 while !indexed_joins.is_empty() {
3063 let n = match epoll.wait(-1, &mut events_buf) {
3064 Ok(n) => n,
3065 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
3066 Err(e) => {
3067 eprintln!(
3068 "ktstr: pcomm container (pcomm={pcomm:?}): epoll.wait \
3069 failed: {e}; falling back to blocking join for the \
3070 remaining {} thread(s)",
3071 indexed_joins.len(),
3072 );
3073 break;
3074 }
3075 };
3076 for ev in &events_buf[..n] {
3077 let target_idx = ev.data() as usize;
3078 let pos = match indexed_joins
3079 .iter()
3080 .position(|(idx, _, _)| *idx == target_idx)
3081 {
3082 Some(p) => p,
3083 None => continue,
3084 };
3085 let (idx, j, evt) = indexed_joins.swap_remove(pos);
3086 if let Err(e) = epoll.ctl(
3087 ControlOperation::Delete,
3088 evt.as_raw_fd(),
3089 EpollEvent::default(),
3090 ) {
3091 eprintln!(
3092 "ktstr: pcomm container (pcomm={pcomm:?}): epoll.ctl(Del) \
3093 for thread {idx} failed: {e}",
3094 );
3095 }
3096 let _ = evt.read();
3097 match j.join() {
3098 Ok(r) => reports.push(r),
3099 Err(payload) => {
3100 let msg = extract_panic_payload(payload);
3101 eprintln!(
3102 "ktstr: pcomm container (pcomm={pcomm:?}): thread {idx} \
3103 panicked: {msg}",
3104 );
3105 reports.push(WorkerReport {
3106 completed: false,
3107 exit_info: Some(WorkerExitInfo::Panicked(msg)),
3108 ..WorkerReport::default()
3109 });
3110 }
3111 }
3112 }
3113 }
3114 }
3115 // Either the epoll path completed (drained
3116 // `indexed_joins`) or it bailed mid-flight; either way,
3117 // any remaining handles are joined by the kernel-blocking
3118 // `JoinHandle::join` below. `join` is event-driven inside
3119 // the kernel (futex on the thread's TID); no userspace
3120 // polling.
3121 for (idx, j, _evt) in indexed_joins {
3122 match j.join() {
3123 Ok(r) => reports.push(r),
3124 Err(payload) => {
3125 let msg = extract_panic_payload(payload);
3126 eprintln!(
3127 "ktstr: pcomm container (pcomm={pcomm:?}): thread {idx} \
3128 panicked: {msg}",
3129 );
3130 reports.push(WorkerReport {
3131 completed: false,
3132 exit_info: Some(WorkerExitInfo::Panicked(msg)),
3133 ..WorkerReport::default()
3134 });
3135 }
3136 }
3137 }
3138
3139 // Encode and write the report stream as a single
3140 // `Vec<WorkerReport>` JSON document via `serde_json`.
3141 // serde_json for the pcomm `Vec<WorkerReport>`; the
3142 // fork-mode workers use postcard for the single
3143 // `WorkerReport`. The pcomm container is a
3144 // fork-mode child and its payload sits on the same
3145 // per-child pipe used by every other forked worker.
3146 // The parent decodes via
3147 // `serde_json::from_slice::<Vec<WorkerReport>>`. Encode
3148 // failures fall through to exit_child_success — the
3149 // parent's sentinel path handles missing / truncated
3150 // payloads.
3151 // tracing is unsafe in a forked child (the parent's
3152 // subscriber may hold a lock); use eprintln + raw fd.
3153 let bytes = match serde_json::to_vec(&reports) {
3154 Ok(v) => v,
3155 Err(e) => {
3156 eprintln!(
3157 "ktstr: pcomm container (pcomm={pcomm:?}): serde_json encode \
3158 of {} reports failed: {e}",
3159 reports.len(),
3160 );
3161 Vec::new()
3162 }
3163 };
3164 {
3165 let mut f = unsafe { std::fs::File::from_raw_fd(report_fds[1]) };
3166 if let Err(e) = f.write_all(&bytes) {
3167 eprintln!(
3168 "ktstr: pcomm container (pcomm={pcomm:?}): report write_all \
3169 of {} bytes failed: {e}",
3170 bytes.len(),
3171 );
3172 }
3173 drop(f);
3174 }
3175 exit_child_success();
3176 }
3177 child_pid => {
3178 // Parent. Restore signal mask, close the wrong ends,
3179 // record the container in `guard.children` so its
3180 // lifecycle (kill on bail / collect on stop) is shared
3181 // with conventional workers.
3182 let psm_parent_restore_rc = unsafe {
3183 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
3184 };
3185 if psm_parent_restore_rc != 0 {
3186 tracing::warn!(
3187 rc = psm_parent_restore_rc,
3188 "pthread_sigmask(SIG_SETMASK) failed restoring mask in parent \
3189 post-pcomm-fork; SIGUSR1 stays blocked in this thread for \
3190 the lifetime of the workload",
3191 );
3192 }
3193 unsafe {
3194 libc::close(report_fds[1]);
3195 libc::close(start_fds[0]);
3196 }
3197 // Per-group layout for sentinel distribution. Each
3198 // entry is `(group_idx, num_workers)`; total report
3199 // count is the sum of `num_workers` across the layout
3200 // (== `total_workers` invariant). When the parent
3201 // emits sentinels for a missing JSON payload, the
3202 // layout drives per-group_idx tagging so per-group
3203 // filters partition correctly even on a failure.
3204 let group_layout: Vec<(usize, usize)> = groups
3205 .iter()
3206 .map(|g| (g.group_idx, g.num_workers))
3207 .collect();
3208 debug_assert_eq!(
3209 group_layout.iter().map(|(_, n)| n).sum::<usize>(),
3210 total_workers,
3211 "spawn_pcomm_container: group_layout total must match total_workers",
3212 );
3213 guard.children.push(ForkedChild {
3214 pid: child_pid,
3215 report_fd: report_fds[0],
3216 start_fd: start_fds[1],
3217 kind: ForkedChildKind::PcommContainer {
3218 groups: group_layout,
3219 },
3220 });
3221 Ok(())
3222 }
3223 }
3224}
3225
3226/// Internal dispatch shape resolved from
3227/// [`WorkloadConfig::clone_mode`] inside [`WorkloadHandle::spawn`].
3228pub(super) enum Dispatch {
3229 Fork,
3230 Thread,
3231}
3232
3233impl WorkloadHandle {
3234 /// Fork one thread-group leader hosting every worker in `works`
3235 /// as worker threads inside a single forked process. Used by
3236 /// `apply_setup` when a `CgroupDef` declares `pcomm` — every
3237 /// `WorkSpec` in the same `CgroupDef` is coalesced into one
3238 /// thread-group leader whose `task->comm` carries `pcomm`
3239 /// exactly (the [`WorkSpec::pcomm`] builder rejects > 15 bytes
3240 /// — `TASK_COMM_LEN - 1` — so the framework never feeds the
3241 /// kernel a name `__set_task_comm` would truncate). Every
3242 /// spawned thread reads its `task->group_leader->comm` as
3243 /// `pcomm` for the leader's lifetime.
3244 ///
3245 /// `works` must already be fully resolved: each entry's
3246 /// `num_workers` must be `Some(_)` and `affinity` must be
3247 /// non-topology-aware (Inherit / Exact / RandomSubset).
3248 /// `apply_setup` runs the standard scenario-engine resolution
3249 /// (`resolve_num_workers` + `intent_for_spawn`) before calling
3250 /// in.
3251 ///
3252 /// `container_uid` / `container_gid` apply to the leader's
3253 /// process credentials via `setresuid` / `setresgid` once,
3254 /// inside the forked leader, before threads spawn. Each worker
3255 /// thread additionally re-applies its merged uid/gid inside
3256 /// `worker_main` at thread creation time; for the common case
3257 /// of a single CgroupDef-level default flowing through
3258 /// `merged_works`, the per-thread call is idempotent.
3259 ///
3260 /// On `works.is_empty()` the function returns a handle with no
3261 /// children — no fork, no resource allocation. Same for the
3262 /// empty-thread case across ALL groups (every `num_workers ==
3263 /// 0`): the leader is skipped because there is no work to host.
3264 /// This matches the `pcomm_zero_workers_no_container_spawn`
3265 /// contract.
3266 ///
3267 /// Group-level admission rejects WorkType variants that conflict
3268 /// with the threaded shape (every worker shares the leader's
3269 /// tgid):
3270 ///
3271 /// - [`WorkType::ForkExit`]: a fork from a thread of a
3272 /// multi-threaded process inherits all locks held by other
3273 /// threads at fork time, which the child cannot release;
3274 /// safe-but-degraded use cases would still need
3275 /// `CloneMode::Fork` for clean lock-free child state.
3276 /// - [`WorkType::CgroupChurn`]: writing the worker tid to
3277 /// `cgroup.procs` migrates the entire leader tgid (every
3278 /// sibling thread) — the test loses control of its own
3279 /// cgroup placement.
3280 pub fn spawn_pcomm_cgroup(
3281 pcomm: &str,
3282 container_uid: Option<u32>,
3283 container_gid: Option<u32>,
3284 works: &[WorkSpec],
3285 ) -> Result<Self> {
3286 // Empty pcomm input is treated as "no pcomm" by the
3287 // apply_setup caller; reject here as a safety belt — every
3288 // production caller has already filtered empty strings.
3289 if pcomm.is_empty() {
3290 anyhow::bail!(
3291 "spawn_pcomm_cgroup: pcomm must be a non-empty string; \
3292 the caller (apply_setup) treats `Some(\"\")` as \
3293 `None` and falls through to the conventional fork \
3294 spawn — empty here is a programmer error.",
3295 );
3296 }
3297
3298 // Structural mem_policy validation per-spec — same gate as
3299 // [`WorkloadHandle::spawn`]'s [`WorkloadConfig::validate`],
3300 // applied at the second public spawn entry point. Closes the
3301 // bypass that would otherwise allow invalid mem_policy via
3302 // this entry to reach `apply_mempolicy_with_flags`'s silent-
3303 // skip arm. The `exit_group(2)` thread-mode unsoundness
3304 // documented on [`WorkloadConfig::validate`] does not apply
3305 // here (pcomm-container workers run inside a forked container
3306 // with a new tgid distinct from the test runner's — the
3307 // container forks once via `libc::fork`, then workers are
3308 // spawned as threads sharing the container's tgid; a
3309 // hypothetical inner `_exit(1)` would only tear down the
3310 // container, not the test runner), but the validation
3311 // symmetry between the two pub entries prevents future
3312 // refactors from inadvertently re-opening the bypass.
3313 for (i, spec) in works.iter().enumerate() {
3314 spec.mem_policy.validate().map_err(|e| {
3315 anyhow::anyhow!("WorkloadHandle::spawn_pcomm_cgroup: works[{i}].mem_policy: {e}",)
3316 })?;
3317 }
3318
3319 // Build a `GroupParams` per `WorkSpec` using the same
3320 // resolver `WorkloadHandle::spawn` uses for composed
3321 // entries. group_idx is the entry's position in `works` (0-
3322 // based); the parent has no notion of "primary" inside a
3323 // pcomm container — every entry is a peer.
3324 let mut groups: Vec<GroupParams> = Vec::with_capacity(works.len());
3325 for (i, spec) in works.iter().enumerate() {
3326 let num_workers = spec.num_workers.ok_or_else(|| {
3327 anyhow::anyhow!(
3328 "spawn_pcomm_cgroup: works[{i}].num_workers must be set explicitly \
3329 (apply_setup runs resolve_num_workers before calling in)",
3330 )
3331 })?;
3332 let site = format!("works[{i}].affinity");
3333 let affinity = GroupParams::resolve_spawn_affinity(&spec.affinity, &site)?;
3334 groups.push(GroupParams::from_work_spec(spec, i, affinity, num_workers));
3335 }
3336
3337 // Per-group admission. Mirrors `WorkloadHandle::spawn` for
3338 // shared rules (worker_group_size divisibility,
3339 // chain-depth-< 2, IdleChurn / IpcVariance zero rejection)
3340 // and adds pcomm-specific rejections (ForkExit,
3341 // CgroupChurn) since the container's threaded shape —
3342 // every worker is a thread of one forked tgid sharing one
3343 // fd table — introduces hazards those variants cannot
3344 // tolerate.
3345 for group in &groups {
3346 if matches!(group.work_type, WorkType::ForkExit) {
3347 anyhow::bail!(
3348 "WorkSpec::pcomm is incompatible with WorkType::ForkExit \
3349 (works[{}]): a fork from a thread of a multi-threaded \
3350 container inherits all locks held by sibling threads at \
3351 fork time, producing undefined behaviour for any libc \
3352 primitive in the child. Drop pcomm or pick a different \
3353 work type.",
3354 group.group_idx,
3355 );
3356 }
3357 if matches!(group.work_type, WorkType::CgroupChurn { .. }) {
3358 anyhow::bail!(
3359 "WorkSpec::pcomm is incompatible with WorkType::CgroupChurn \
3360 (works[{}]): CgroupChurn writes the worker tid to \
3361 `cgroup.procs`, which the kernel resolves to the whole \
3362 tgid and migrates the entire pcomm container (every \
3363 sibling thread). Drop pcomm or pick a different work type.",
3364 group.group_idx,
3365 );
3366 }
3367 validate_workload_admission(group)?;
3368 }
3369
3370 // No-work shortcut: every group has zero workers (or the
3371 // works list itself is empty). Fork would produce a
3372 // container that blocks on the start byte forever and
3373 // outlives the test handle; skip it. The handle still
3374 // has a sensible Drop (empty children/threads/regions).
3375 let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
3376 if total_workers == 0 {
3377 return Ok(SpawnGuard::new().into_handle());
3378 }
3379
3380 // All failable acquisitions route through `guard`. If any
3381 // `?` returns early, the guard's Drop SIGKILLs+reaps any
3382 // already-forked container, closes open pipe fds, and
3383 // munmaps the shared regions — so no leak on a mid-spawn
3384 // error path.
3385 let mut guard = SpawnGuard::new();
3386
3387 // Per-worker iteration counter region (MAP_SHARED). Sized
3388 // for ALL groups' workers laid out contiguously; matches
3389 // the `WorkloadHandle::spawn` allocation but with a
3390 // pcomm-specific diagnostic prefix.
3391 let size = total_workers * std::mem::size_of::<AtomicU64>();
3392 let ptr = unsafe {
3393 libc::mmap(
3394 std::ptr::null_mut(),
3395 size,
3396 libc::PROT_READ | libc::PROT_WRITE,
3397 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3398 -1,
3399 0,
3400 )
3401 };
3402 if ptr == libc::MAP_FAILED {
3403 let errno = std::io::Error::last_os_error();
3404 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3405 anyhow::bail!(
3406 "mmap(MAP_SHARED|MAP_ANONYMOUS, {size} bytes) for the \
3407 per-worker iter_counters region failed: {errno}{hint}; \
3408 this region holds one AtomicU64 per worker thread \
3409 ({total_workers} thread(s) inside the pcomm={pcomm:?} \
3410 container) so the parent can snapshot iteration \
3411 counts via `snapshot_iterations()`.",
3412 );
3413 }
3414 guard.iter_counters = ptr as *mut AtomicU64;
3415 guard.iter_counter_bytes = size;
3416
3417 // Single shared phase-epoch word (MAP_SHARED, zero-init to
3418 // 0 = BASELINE) — same role as in `WorkloadHandle::spawn`: the
3419 // scenario engine bumps it per step boundary and backdrop
3420 // workers read it for per-phase attribution.
3421 let epoch_size = std::mem::size_of::<AtomicU32>();
3422 let epoch_ptr = unsafe {
3423 libc::mmap(
3424 std::ptr::null_mut(),
3425 epoch_size,
3426 libc::PROT_READ | libc::PROT_WRITE,
3427 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3428 -1,
3429 0,
3430 )
3431 };
3432 if epoch_ptr == libc::MAP_FAILED {
3433 let errno = std::io::Error::last_os_error();
3434 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3435 anyhow::bail!(
3436 "mmap(MAP_SHARED|MAP_ANONYMOUS, {epoch_size} bytes) for the \
3437 phase-epoch word (pcomm={pcomm:?}) failed: {errno}{hint}; \
3438 this single AtomicU32 broadcasts the current phase to \
3439 backdrop workers for per-phase attribution.",
3440 );
3441 }
3442 guard.phase_epoch = epoch_ptr as *mut AtomicU32;
3443 guard.phase_epoch_bytes = epoch_size;
3444
3445 // Per-group resource allocation (pipe pairs, chain pipes,
3446 // futex regions). Same shape as `WorkloadHandle::spawn_group`'s
3447 // prologue but inlined here so the per-group bookkeeping
3448 // (`PcommGroupResources`) is built directly for
3449 // `spawn_pcomm_container`. `iter_offset` runs across groups
3450 // and tracks each group's first iter_slot.
3451 let mut resources: Vec<PcommGroupResources> = Vec::with_capacity(groups.len());
3452 let mut iter_offset: usize = 0;
3453 for group in &groups {
3454 let needs_pipes = matches!(
3455 group.work_type,
3456 WorkType::PipeIo { .. } | WorkType::CachePipe { .. }
3457 );
3458 let chain_depth = group.work_type.chain_pipe_depth();
3459 let needs_futex = group.work_type.needs_shared_mem();
3460 let pipe_pair_base = guard.pipe_pairs.len();
3461 let chain_pipes_base = guard.chain_pipes.len();
3462 let futex_ptrs_base = guard.futex_ptrs.len();
3463 let futex_region_size = futex_region_size_for(&group.work_type);
3464 let futex_group_size = group.work_type.worker_group_size().unwrap_or(2);
3465
3466 if needs_pipes {
3467 for _ in 0..group.num_workers / 2 {
3468 let mut ab = [0i32; 2];
3469 if unsafe { libc::pipe2(ab.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3470 anyhow::bail!(
3471 "pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3472 group.group_idx,
3473 std::io::Error::last_os_error(),
3474 );
3475 }
3476 let mut ba = [0i32; 2];
3477 if unsafe { libc::pipe2(ba.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3478 unsafe {
3479 libc::close(ab[0]);
3480 libc::close(ab[1]);
3481 }
3482 anyhow::bail!(
3483 "pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3484 group.group_idx,
3485 std::io::Error::last_os_error(),
3486 );
3487 }
3488 guard.pipe_pairs.push((ab, ba));
3489 }
3490 }
3491
3492 if let Some(depth) = chain_depth
3493 && depth > 0
3494 && group.num_workers >= depth
3495 {
3496 let chains = group.num_workers / depth;
3497 for _ in 0..chains {
3498 let mut chain: Vec<[i32; 2]> = Vec::with_capacity(depth);
3499 let mut alloc_ok = true;
3500 for _ in 0..depth {
3501 let mut p = [0i32; 2];
3502 if unsafe { libc::pipe2(p.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3503 alloc_ok = false;
3504 break;
3505 }
3506 chain.push(p);
3507 }
3508 if !alloc_ok {
3509 for p in &chain {
3510 unsafe {
3511 libc::close(p[0]);
3512 libc::close(p[1]);
3513 }
3514 }
3515 anyhow::bail!(
3516 "WakeChain pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3517 group.group_idx,
3518 std::io::Error::last_os_error(),
3519 );
3520 }
3521 guard.chain_pipes.push(chain);
3522 }
3523 }
3524
3525 if needs_futex {
3526 for _ in 0..group.num_workers / futex_group_size {
3527 let region = unsafe {
3528 libc::mmap(
3529 std::ptr::null_mut(),
3530 futex_region_size,
3531 libc::PROT_READ | libc::PROT_WRITE,
3532 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3533 -1,
3534 0,
3535 )
3536 };
3537 if region == libc::MAP_FAILED {
3538 let errno = std::io::Error::last_os_error();
3539 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3540 anyhow::bail!(
3541 "mmap(MAP_SHARED|MAP_ANONYMOUS, {futex_region_size} bytes) \
3542 for a futex shared-memory region (pcomm={pcomm:?}, group {}) \
3543 failed: {errno}{hint}",
3544 group.group_idx,
3545 );
3546 }
3547 unsafe {
3548 std::ptr::write_bytes(region as *mut u8, 0, futex_region_size);
3549 }
3550 guard.futex_ptrs.push(region as *mut u32);
3551 guard.futex_region_sizes.push(futex_region_size);
3552 }
3553 }
3554
3555 resources.push(PcommGroupResources {
3556 iter_offset,
3557 pipe_pair_base,
3558 chain_pipes_base,
3559 futex_ptrs_base,
3560 needs_pipes,
3561 chain_depth,
3562 needs_futex,
3563 futex_group_size,
3564 });
3565 iter_offset += group.num_workers;
3566 }
3567
3568 // Fork the container and spawn its threads. On success the
3569 // container is registered in `guard.children`; on failure the
3570 // guard's Drop reaps it.
3571 spawn_pcomm_container(
3572 &mut guard,
3573 pcomm,
3574 container_uid,
3575 container_gid,
3576 &groups,
3577 &resources,
3578 )?;
3579
3580 Ok(guard.into_handle())
3581 }
3582
3583 /// Spawn worker tasks. Workers block until
3584 /// [`start()`](Self::start) is called, allowing the caller to
3585 /// move fork-mode workers into cgroups first. The worker creation
3586 /// primitive (`fork` or `std::thread::spawn`) is selected by
3587 /// [`WorkloadConfig::clone_mode`].
3588 pub fn spawn(config: &WorkloadConfig) -> Result<Self> {
3589 // Reject invalid mem_policy before any worker context exists;
3590 // see [`WorkloadConfig::validate`] for the silent-skip
3591 // rationale and why a worker-side `_exit(1)` fix is unsound.
3592 config.validate()?;
3593
3594 let dispatch = match &config.clone_mode {
3595 CloneMode::Fork => Dispatch::Fork,
3596 CloneMode::Thread => Dispatch::Thread,
3597 };
3598
3599 // Build the per-group params list: primary first
3600 // (group_idx == 0), then composed[k] resolved into
3601 // group_idx == k+1. The resolver enforces the "spawn-time
3602 // resolution rules" documented on
3603 // [`WorkloadConfig::composed`] (Q1: num_workers must be
3604 // explicit; Q2: only Inherit/Exact affinity reachable from
3605 // spawn() — topology-aware variants need the scenario
3606 // engine).
3607 //
3608 // Every group inherits the parent
3609 // [`WorkloadConfig::clone_mode`]: SpawnGuard's lifecycle
3610 // assumes a single dispatch path (every guard.children
3611 // entry is a fork-mode child reaped via waitpid; every
3612 // guard.threads entry is a thread-mode worker joined via
3613 // JoinHandle). Mixing modes inside one guard would route
3614 // teardown through the wrong code path, so [`WorkSpec`]
3615 // carries no `clone_mode` field — it is a workload-wide
3616 // property fixed by [`WorkloadConfig::clone_mode`].
3617 let mut groups: Vec<GroupParams> = Vec::with_capacity(1 + config.composed.len());
3618 groups.push(GroupParams::primary(config)?);
3619 for (i, spec) in config.composed.iter().enumerate() {
3620 groups.push(GroupParams::from_composed(spec, i + 1)?);
3621 }
3622
3623 // Per-group admission. Each group's work_type is checked
3624 // independently — a malformed composed entry bails the
3625 // whole workload before any resources are acquired.
3626 for group in &groups {
3627 // Thread mode + ForkExit is incompatible. ForkExit's worker
3628 // body calls `libc::fork()` from inside `worker_main` to
3629 // exercise the fork -> exit -> wait lifecycle. Under
3630 // [`CloneMode::Thread`] the worker is one thread of the
3631 // harness's multi-threaded tgid, and a `fork()` from a thread
3632 // of a multi-threaded process duplicates ONLY the calling
3633 // thread: any lock another harness thread holds at fork time
3634 // (glibc malloc arena, internal mutexes) stays locked forever
3635 // in the child, which has no thread left to release it. The
3636 // child here only `_exit`s (async-signal-safe), so it survives
3637 // this in practice — but the hazard is real and the
3638 // fork -> exit primitive is faithfully exercised only when each
3639 // worker is its own process. (The child does NOT share the
3640 // harness tgid: `fork()` omits CLONE_THREAD, so copy_process
3641 // gives it group_leader=self / tgid=pid, a fresh singleton
3642 // group. Its `libc::_exit(0)` issues exit_group(2) ->
3643 // `do_group_exit` -> `zap_other_threads`, but a singleton tgid
3644 // has no other threads for `zap_other_threads` to SIGKILL, so
3645 // the teardown ends only the child; the harness tgid is
3646 // untouched.) Reject at spawn with an actionable diagnostic;
3647 // CloneMode::Fork gives each worker its own tgid and a
3648 // lock-clean address space.
3649 if matches!(dispatch, Dispatch::Thread) && matches!(group.work_type, WorkType::ForkExit)
3650 {
3651 anyhow::bail!(
3652 "CloneMode::Thread is incompatible with WorkType::ForkExit \
3653 (group {}) — ForkExit forks inside the worker, and a fork \
3654 from a thread of the multi-threaded harness inherits \
3655 sibling-held locks the child cannot release; only a \
3656 separate process faithfully exercises the fork/exit \
3657 lifecycle. Use CloneMode::Fork for ForkExit workloads.",
3658 group.group_idx,
3659 );
3660 }
3661 // Fork mode + EpollStorm is incompatible. EpollStorm
3662 // creates an eventfd + epoll fd inside worker pos 0 and
3663 // publishes their integer fd numbers through the per-group
3664 // shared mmap region (`efd_slot` / `epfd_slot`); siblings
3665 // load those numbers and operate on them as if they
3666 // referred to the same kernel objects. Under
3667 // [`CloneMode::Fork`] each forked child holds its own copy
3668 // of the parent's fd table at fork time, but the eventfd
3669 // and epoll fd are created AFTER the fork on worker pos 0
3670 // — so sibling children's fd tables never contain those
3671 // descriptors. The integer numbers they read from the
3672 // shared region either resolve to unrelated fds the child
3673 // happened to have at the same slot or fail with EBADF.
3674 // The fd table is genuinely shared only under
3675 // [`CloneMode::Thread`]. Reject at spawn time with an
3676 // actionable diagnostic; CloneMode::Thread is the correct
3677 // choice for EpollStorm.
3678 if matches!(dispatch, Dispatch::Fork)
3679 && matches!(group.work_type, WorkType::EpollStorm { .. })
3680 {
3681 anyhow::bail!(
3682 "CloneMode::Fork is incompatible with WorkType::EpollStorm \
3683 (group {}) — EpollStorm publishes eventfd/epoll fd numbers \
3684 through a shared mmap region for siblings to consume, but \
3685 forked children hold independent fd tables that never \
3686 contain those post-fork descriptors. Use CloneMode::Thread \
3687 for EpollStorm workloads.",
3688 group.group_idx,
3689 );
3690 }
3691 // Thread mode + CgroupChurn is incompatible. CgroupChurn's
3692 // worker body writes its own tid (`SYS_gettid`) to a
3693 // sibling cgroup's `cgroup.procs` file. The kernel's
3694 // `__cgroup_procs_write` resolves the tid to its
3695 // task_struct, then migrates the *entire* thread group
3696 // leader's task and every member of its tgid to the
3697 // target cgroup (see `cgroup_attach_task` /
3698 // `cgroup_migrate` in kernel/cgroup/cgroup.c — procs-file
3699 // semantics are tgid-wide, contrast `cgroup.threads`
3700 // which is per-thread under the threaded controller).
3701 // Under [`CloneMode::Thread`] every worker is a member of
3702 // the test harness's tgid, so the first CgroupChurn write
3703 // migrates the harness itself and every sibling worker
3704 // thread out from under the host. Reject at spawn time
3705 // with an actionable diagnostic; CloneMode::Fork gives
3706 // each worker its own tgid so a procs-file write moves
3707 // only that worker.
3708 if matches!(dispatch, Dispatch::Thread)
3709 && matches!(group.work_type, WorkType::CgroupChurn { .. })
3710 {
3711 anyhow::bail!(
3712 "CloneMode::Thread is incompatible with WorkType::CgroupChurn \
3713 (group {}) — CgroupChurn writes the worker tid to \
3714 `cgroup.procs`, which the kernel resolves to the whole tgid \
3715 and migrates every sibling thread (including the harness) \
3716 to the target cgroup. Use CloneMode::Fork for CgroupChurn \
3717 workloads so each worker is a separate tgid.",
3718 group.group_idx,
3719 );
3720 }
3721 // Thread mode + CgroupAttachStorm is incompatible because the
3722 // worker installs `SIGCHLD = SIG_IGN` at entry (under
3723 // ReapMode::SigIgn) to auto-reap its forked children. Under
3724 // [`CloneMode::Thread`] the worker is a pthread of the harness
3725 // thread group and shares the harness `sighand` (CLONE_THREAD
3726 // implies CLONE_SIGHAND), so that install changes SIGCHLD
3727 // disposition for the WHOLE harness process — corrupting the
3728 // harness's own child reaping — not just this worker. (fork()
3729 // from a thread of the multithreaded harness is also fragile;
3730 // the forked child is safe here only because it does nothing
3731 // but `_exit`.) Under [`CloneMode::Fork`] each worker is its
3732 // own process with a private sighand, so the SIG_IGN install
3733 // and the forked-child lifecycle stay confined to that worker.
3734 // Note: the storm writes the forked CHILD's pid (its own tgid)
3735 // to cgroup.procs, so — unlike CgroupChurn's own-tid write —
3736 // the migration never moves the harness; the hazard is the
3737 // shared-sighand SIG_IGN install, not tgid migration. Reject at
3738 // spawn time with an actionable diagnostic.
3739 if matches!(dispatch, Dispatch::Thread)
3740 && matches!(group.work_type, WorkType::CgroupAttachStorm { .. })
3741 {
3742 anyhow::bail!(
3743 "CloneMode::Thread is incompatible with WorkType::CgroupAttachStorm \
3744 (group {}) — the worker installs SIGCHLD=SIG_IGN to auto-reap its \
3745 forked children, but a thread-group worker shares the harness \
3746 sighand, so that install corrupts the harness's own child reaping. \
3747 Use CloneMode::Fork for CgroupAttachStorm workloads so each worker \
3748 has its own process and private sighand.",
3749 group.group_idx,
3750 );
3751 }
3752 // Dispatch-agnostic admission rules
3753 // (worker_group_size divisibility, chain depth >= 2,
3754 // IdleChurn / IpcVariance zero-rejections) live in
3755 // [`validate_workload_admission`] and are shared with
3756 // [`Self::spawn_pcomm_cgroup`]. The
3757 // CloneMode-vs-WorkType compat checks above stay
3758 // dispatch-specific because each dispatch path gates
3759 // independently: Thread+ForkExit (here) and pcomm+ForkExit
3760 // ([`Self::spawn_pcomm_cgroup`]) reject for the SAME reason
3761 // (a fork from a thread of a multi-threaded process inherits
3762 // sibling-held locks the child cannot release) via separate
3763 // per-path gates; Fork+EpollStorm has no pcomm equivalent.
3764 validate_workload_admission(group)?;
3765 }
3766
3767 // futex region sizing is per-group, not MAX'd across all
3768 // groups. Each group's futex region has its own natural
3769 // size determined by [`futex_region_size_for`] (FanOutCompute
3770 // = 16, ProducerConsumerImbalance = 32 + Q*8, everything
3771 // else = 4). Storing the size alongside each pointer in
3772 // `SpawnGuard::futex_region_sizes` lets Drop munmap each
3773 // region with its own length, so a small-variant group
3774 // composed alongside a large ProducerConsumerImbalance group
3775 // is no longer inflated to the large size.
3776 //
3777 // The kernel rounds munmap length up to PAGE_SIZE, so the
3778 // per-region waste for sub-page allocations is bounded at
3779 // one page; the previous MAX-across-groups approach could
3780 // waste many pages per small-variant group when paired with
3781 // a large queue_depth_target.
3782
3783 // All failable acquisitions in this function route through
3784 // `guard`. If any `?`/`bail!` returns early, the guard's Drop
3785 // SIGKILLs+reaps forked children, closes open pipe fds, and
3786 // munmaps the shared regions — so no leak on a mid-spawn
3787 // error path.
3788 let mut guard = SpawnGuard::new();
3789
3790 // Per-worker iteration counter region (MAP_SHARED). Sized
3791 // for ALL groups' workers laid out contiguously: primary
3792 // group occupies slots `[0, primary.num_workers)`, composed
3793 // group `k` occupies slots starting at the running offset
3794 // tracked by `iter_offset` in the per-group spawn loop
3795 // below. Each worker atomically stores its iteration count
3796 // to its assigned slot; the parent reads all slots via
3797 // `snapshot_iterations()`. The mmap base is page-aligned
3798 // (kernel guarantee), so casting to `*mut AtomicU64` is
3799 // sound: page alignment (≥ 4096) ≥ AtomicU64 alignment (8),
3800 // and the region size is an exact multiple of
3801 // `size_of::<AtomicU64>()` (== 8). Each `.add(i)` moves by
3802 // `i * 8` bytes, preserving the 8-byte alignment invariant.
3803 // No non-atomic access to the region exists anywhere in the
3804 // crate, so the atomic-only aliasing rule (workers + parent
3805 // share `&AtomicU64` references derived from the raw
3806 // pointer) holds.
3807 let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
3808 if total_workers > 0 {
3809 let size = total_workers * std::mem::size_of::<AtomicU64>();
3810 let ptr = unsafe {
3811 libc::mmap(
3812 std::ptr::null_mut(),
3813 size,
3814 libc::PROT_READ | libc::PROT_WRITE,
3815 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3816 -1,
3817 0,
3818 )
3819 };
3820 if ptr == libc::MAP_FAILED {
3821 let errno = std::io::Error::last_os_error();
3822 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3823 anyhow::bail!(
3824 "mmap(MAP_SHARED|MAP_ANONYMOUS, {size} bytes) for the \
3825 per-worker iter_counters region failed: {errno}{hint}; \
3826 this region holds one AtomicU64 per worker \
3827 ({total_workers} slots across {} group(s)) so the parent \
3828 can snapshot iteration counts via \
3829 `snapshot_iterations()`. Remediation: reduce num_workers \
3830 (each worker consumes 8 bytes of this region, rounded up \
3831 to a page) or raise `vm.max_map_count` / the memory \
3832 cgroup limit.",
3833 groups.len(),
3834 );
3835 }
3836 guard.iter_counters = ptr as *mut AtomicU64;
3837 guard.iter_counter_bytes = size;
3838 // Single shared phase-epoch word (MAP_SHARED, MAP_ANONYMOUS
3839 // zero-inits it to 0 = BASELINE). The scenario engine bumps
3840 // it at each step boundary; backdrop workers read it to
3841 // attribute per-phase telemetry. Allocated for every handle
3842 // — a step-local pool whose epoch is never bumped simply
3843 // observes no change and emits no PhaseSlices.
3844 let epoch_size = std::mem::size_of::<AtomicU32>();
3845 let epoch_ptr = unsafe {
3846 libc::mmap(
3847 std::ptr::null_mut(),
3848 epoch_size,
3849 libc::PROT_READ | libc::PROT_WRITE,
3850 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3851 -1,
3852 0,
3853 )
3854 };
3855 if epoch_ptr == libc::MAP_FAILED {
3856 let errno = std::io::Error::last_os_error();
3857 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3858 anyhow::bail!(
3859 "mmap(MAP_SHARED|MAP_ANONYMOUS, {epoch_size} bytes) for the \
3860 phase-epoch word failed: {errno}{hint}; this single \
3861 AtomicU32 lets the scenario engine broadcast the current \
3862 phase to backdrop workers for per-phase attribution.",
3863 );
3864 }
3865 guard.phase_epoch = epoch_ptr as *mut AtomicU32;
3866 guard.phase_epoch_bytes = epoch_size;
3867 }
3868
3869 // First-iteration signal eventfd (C6). Allocated BEFORE the fork
3870 // loop — like the iter_counters mmap above — so every forked child
3871 // inherits its fd-table copy. Only when the workload opted in AND
3872 // the dispatch is fork with ≥1 worker: thread / pcomm dispatch pass
3873 // a `-1` fd to `worker_main` and never signal, and a zero-worker
3874 // spawn has nothing to wait on. Counter mode (`0` initial value)
3875 // so the parent's `wait_first_iteration_all` reads the SUM of the
3876 // per-worker `1`s. EFD_NONBLOCK keeps the worker's post-first-iter
3877 // `write` non-blocking; EFD_CLOEXEC is safe because fork workers
3878 // never `exec` (a stale fd across an exec would only leak, not
3879 // misbehave). The `OwnedFd` lives on the guard (closed on early
3880 // bail) and transfers to the handle on success.
3881 if config.signal_first_iteration && matches!(dispatch, Dispatch::Fork) && total_workers > 0
3882 {
3883 let raw = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
3884 if raw < 0 {
3885 let errno = std::io::Error::last_os_error();
3886 anyhow::bail!(
3887 "eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC) for the first-iteration \
3888 signal failed: {errno}; this fd lets the verifier dispatch \
3889 probe block on the first-iteration edge instead of polling \
3890 snapshot_iterations. Remediation: raise the process fd limit \
3891 (RLIMIT_NOFILE).",
3892 );
3893 }
3894 // SAFETY: `raw` is a fresh, exclusively-owned eventfd from the
3895 // successful `eventfd(2)` above; wrapping it in OwnedFd takes
3896 // sole ownership so it is closed exactly once on Drop.
3897 guard.first_iter_signal = Some(unsafe { std::os::fd::OwnedFd::from_raw_fd(raw) });
3898 }
3899
3900 // Spawn each group in declaration order. `iter_offset`
3901 // tracks the running offset into the iter_counters mmap
3902 // (slot allocation per the layout commented above). Each
3903 // group's pipes / chain_pipes / futex_ptrs are appended to
3904 // the guard's flat vectors; we record per-group base
3905 // offsets so the per-worker fork loop can compute
3906 // global-vector indices from per-group worker indices and
3907 // the close-other-fds child path can iterate the full
3908 // guard while still identifying its own group's resources.
3909 let mut iter_offset: usize = 0;
3910 for group in &groups {
3911 Self::spawn_group(&mut guard, group, &dispatch, iter_offset)?;
3912 iter_offset += group.num_workers;
3913 }
3914
3915 // Success: transfer every live resource (children, threads,
3916 // futex_ptrs, iter_counters, pipe_pairs, chain_pipes) into
3917 // the handle. The guard's subsequent Drop sees empty Vecs
3918 // and a null iter_counters pointer — it is a no-op on this
3919 // path. Pipe fds are closed by `WorkloadHandle::drop` AFTER
3920 // worker shutdown so Thread-mode workers (which share the
3921 // parent's fd table) finish their pipe ops before the close.
3922 Ok(guard.into_handle())
3923 }
3924
3925 /// Spawn a single worker group's resources and per-worker
3926 /// tasks, appending each into the shared [`SpawnGuard`].
3927 ///
3928 /// Each group records its own base offsets into the guard's
3929 /// flat vectors at entry time, then uses those offsets when
3930 /// computing per-worker `pair_idx` / `chain_idx` /
3931 /// `futex_group_idx`. The fork-child close-other-fds block
3932 /// iterates the FULL guard so it sweeps fds belonging to
3933 /// other groups too — without that sweep, a composed-group
3934 /// worker would inherit (and never close) every primary-group
3935 /// pipe fd.
3936 ///
3937 /// Resource ownership is uniform across groups: every
3938 /// allocated pipe / mmap region lives in the guard's flat
3939 /// vectors and is freed by `SpawnGuard::Drop` on early-bail or
3940 /// transferred to [`WorkloadHandle`] on success via
3941 /// [`SpawnGuard::into_handle`].
3942 #[allow(clippy::too_many_arguments)]
3943 fn spawn_group(
3944 guard: &mut SpawnGuard,
3945 group: &GroupParams,
3946 dispatch: &Dispatch,
3947 iter_offset: usize,
3948 ) -> Result<()> {
3949 let needs_pipes = matches!(
3950 group.work_type,
3951 WorkType::PipeIo { .. } | WorkType::CachePipe { .. }
3952 );
3953 let chain_depth = group.work_type.chain_pipe_depth();
3954 let needs_futex = group.work_type.needs_shared_mem();
3955
3956 // Record the bases into the guard's flat vectors BEFORE
3957 // appending this group's allocations. The base values
3958 // identify "where this group's resources start" — the
3959 // per-worker fork loop combines `pipe_pair_base + i / 2`
3960 // (and analogous for chain_idx / futex_group_idx) to
3961 // address its own resources without colliding with another
3962 // group's range.
3963 let pipe_pair_base = guard.pipe_pairs.len();
3964 let chain_pipes_base = guard.chain_pipes.len();
3965 let futex_ptrs_base = guard.futex_ptrs.len();
3966 // Per-group natural size for the futex MAP_SHARED region —
3967 // each region in this group's range gets exactly this many
3968 // bytes (rather than the previous global MAX across every
3969 // group). See `futex_region_size_for` for the per-variant
3970 // sizing rules.
3971 let futex_region_size = futex_region_size_for(&group.work_type);
3972
3973 // For paired work types, create one pipe per worker pair before forking.
3974 // pipe_pairs[pair_idx] = (read_fd, write_fd) for the A->B direction,
3975 // and a second pipe for B->A. Use `pipe2(O_CLOEXEC)` instead
3976 // of bare `pipe(2)`: O_CLOEXEC is the correct default for
3977 // any kernel fd in long-running processes — fds without
3978 // O_CLOEXEC silently leak into any exec path.
3979 if needs_pipes {
3980 for _ in 0..group.num_workers / 2 {
3981 let mut ab = [0i32; 2]; // A writes, B reads
3982 if unsafe { libc::pipe2(ab.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3983 anyhow::bail!("pipe2 failed: {}", std::io::Error::last_os_error());
3984 }
3985 let mut ba = [0i32; 2]; // B writes, A reads
3986 if unsafe { libc::pipe2(ba.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3987 // Close the ab half we just created: it is not
3988 // yet owned by the guard, so its Drop won't
3989 // otherwise reach it.
3990 unsafe {
3991 libc::close(ab[0]);
3992 libc::close(ab[1]);
3993 }
3994 anyhow::bail!("pipe2 failed: {}", std::io::Error::last_os_error());
3995 }
3996 guard.pipe_pairs.push((ab, ba));
3997 }
3998 }
3999
4000 // For WakeChain { wake: WakeMechanism::Pipe }, allocate `depth` pipes per
4001 // chain (one pipe per stage). Pipe `i` connects stage `i`
4002 // (writer) to stage `(i + 1) % depth` (reader). On any
4003 // `pipe2()` failure mid-allocation, close the half-built
4004 // chain's pipes before bailing — the chain is not yet
4005 // pushed onto `guard.chain_pipes`, so its Drop won't
4006 // otherwise reach those fds. `O_CLOEXEC` matches the
4007 // defense-in-depth posture documented above on the
4008 // pipe-pair allocation.
4009 if let Some(depth) = chain_depth
4010 && depth > 0
4011 && group.num_workers >= depth
4012 {
4013 let chains = group.num_workers / depth;
4014 for _ in 0..chains {
4015 let mut chain: Vec<[i32; 2]> = Vec::with_capacity(depth);
4016 let mut alloc_ok = true;
4017 for _ in 0..depth {
4018 let mut p = [0i32; 2];
4019 if unsafe { libc::pipe2(p.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4020 alloc_ok = false;
4021 break;
4022 }
4023 chain.push(p);
4024 }
4025 if !alloc_ok {
4026 for p in &chain {
4027 unsafe {
4028 libc::close(p[0]);
4029 libc::close(p[1]);
4030 }
4031 }
4032 anyhow::bail!(
4033 "WakeChain pipe2 allocation failed: {}",
4034 std::io::Error::last_os_error()
4035 );
4036 }
4037 guard.chain_pipes.push(chain);
4038 }
4039 }
4040
4041 // For FutexPingPong/FutexFanOut/FanOutCompute/MutexContention, allocate
4042 // one shared region per worker group via MAP_SHARED|MAP_ANONYMOUS
4043 // so all members of the fork see the same physical page.
4044 // Each region is sized exactly to the variant's natural
4045 // need (see [`futex_region_size_for`]) — the per-region
4046 // size is recorded in `guard.futex_region_sizes` parallel
4047 // to `guard.futex_ptrs`, so munmap on Drop receives the
4048 // correct length even when groups with different natural
4049 // sizes co-exist in the same workload.
4050 let futex_group_size = group.work_type.worker_group_size().unwrap_or(2);
4051 if needs_futex {
4052 for _ in 0..group.num_workers / futex_group_size {
4053 let ptr = unsafe {
4054 libc::mmap(
4055 std::ptr::null_mut(),
4056 futex_region_size,
4057 libc::PROT_READ | libc::PROT_WRITE,
4058 libc::MAP_SHARED | libc::MAP_ANONYMOUS,
4059 -1,
4060 0,
4061 )
4062 };
4063 if ptr == libc::MAP_FAILED {
4064 let errno = std::io::Error::last_os_error();
4065 let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
4066 anyhow::bail!(
4067 "mmap(MAP_SHARED|MAP_ANONYMOUS, {futex_region_size} bytes) \
4068 for a futex shared-memory region failed: {errno}{hint}; \
4069 this region backs the {:?} worker-group's (group {}) \
4070 inter-process futex word and is allocated \
4071 before fork so every child inherits the same \
4072 mapping. Remediation: reduce num_workers (each \
4073 futex group consumes one shared page) or raise \
4074 `vm.max_map_count` / the memory cgroup limit.",
4075 group.work_type.name(),
4076 group.group_idx,
4077 );
4078 }
4079 unsafe { std::ptr::write_bytes(ptr as *mut u8, 0, futex_region_size) };
4080 guard.futex_ptrs.push(ptr as *mut u32);
4081 guard.futex_region_sizes.push(futex_region_size);
4082 }
4083 }
4084
4085 for i in 0..group.num_workers {
4086 let affinity = resolve_affinity(&group.affinity)?;
4087
4088 // Determine pipe fds for this worker.
4089 //
4090 // Three shapes use the same `Option<(read_fd, write_fd)>`
4091 // parameter:
4092 // - PipeIo / CachePipe (paired): worker A reads `ba[0]`,
4093 // writes `ab[1]`; worker B reads `ab[0]`, writes
4094 // `ba[1]`.
4095 // - WakeChain { wake: WakeMechanism::Pipe } (chain ring): stage `s`
4096 // reads from pipe `(s + depth - 1) % depth` (its
4097 // predecessor's write end's matching read end) and
4098 // writes to pipe `s` (its own pipe's write end, which
4099 // stage `s + 1` reads from).
4100 // - Everything else: `None`.
4101 //
4102 // Indices are computed in the GLOBAL `guard.pipe_pairs`
4103 // / `guard.chain_pipes` space by adding the per-group
4104 // base recorded at the top of `spawn_group`. A composed
4105 // group's pipe-pair-base, for example, equals the sum
4106 // of every prior group's pipe-pair count, so its first
4107 // worker pair is allocated immediately after the
4108 // primary's last entry — no collision, no aliasing.
4109 let worker_pipe_fds: Option<(i32, i32)> = if needs_pipes {
4110 let pair_idx = pipe_pair_base + i / 2;
4111 let (ref ab, ref ba) = guard.pipe_pairs[pair_idx];
4112 if i % 2 == 0 {
4113 // Worker A: writes to ab[1], reads from ba[0]
4114 Some((ba[0], ab[1]))
4115 } else {
4116 // Worker B: writes to ba[1], reads from ab[0]
4117 Some((ab[0], ba[1]))
4118 }
4119 } else if let Some(depth) = chain_depth
4120 && depth > 0
4121 {
4122 let chain_idx = chain_pipes_base + i / depth;
4123 let stage = i % depth;
4124 let prev_stage = (stage + depth - 1) % depth;
4125 let chain = &guard.chain_pipes[chain_idx];
4126 // Read end of predecessor's pipe; write end of own
4127 // pipe. The kernel pipe pair is `[read_end,
4128 // write_end]` per `libc::pipe`'s manpage.
4129 Some((chain[prev_stage][0], chain[stage][1]))
4130 } else {
4131 None
4132 };
4133
4134 // Futex pointer for this worker. The `pos` is the
4135 // worker's index inside its futex group: `pos == 0`
4136 // is the group's "first" worker (the role that varies
4137 // per-variant — pair-A for FutexPingPong, messenger for
4138 // FutexFanOut/FanOutCompute, waker for ThunderingHerd/
4139 // AsymmetricWaker, chain-head for WakeChain). Variants
4140 // that need finer-grained per-worker positioning
4141 // (PriorityInversion's 3 tiers, ProducerConsumerImbalance's
4142 // producer/consumer split, RtStarvation's RT/CFS split,
4143 // WakeChain's stage index) consume `pos` directly.
4144 let worker_futex: Option<(*mut u32, usize)> = if needs_futex {
4145 let futex_group_idx = futex_ptrs_base + i / futex_group_size;
4146 let pos = i % futex_group_size;
4147 Some((guard.futex_ptrs[futex_group_idx], pos))
4148 } else {
4149 None
4150 };
4151
4152 // Shared iteration counter slot for this worker. The
4153 // group-local index `i` is added to the spawn-time
4154 // `iter_offset` so each group's slot range is disjoint
4155 // from every other group's.
4156 let iter_slot: *mut AtomicU64 = if !guard.iter_counters.is_null() {
4157 unsafe { guard.iter_counters.add(iter_offset + i) }
4158 } else {
4159 std::ptr::null_mut()
4160 };
4161 // Single shared phase-epoch word — the SAME pointer for every
4162 // worker (no per-worker offset). Non-null for every handle; the
4163 // scenario bumps it only for backdrop handles, so step-local
4164 // workers observe no change and emit no slices.
4165 let phase_epoch_ptr: *mut AtomicU32 = guard.phase_epoch;
4166
4167 // First-iteration signal fd for this worker (C6). `-1` (absent)
4168 // unless the group opted in AND the handle allocated the
4169 // eventfd — which happens only for fork dispatch (thread /
4170 // pcomm groups leave `guard.first_iter_signal` None, so this
4171 // resolves to `-1` and the worker never signals). A plain `i32`
4172 // copy, so it crosses the fork/thread closure boundary without
4173 // a Send newtype.
4174 let first_iter_signal_fd: i32 = if group.signal_first_iteration {
4175 use std::os::unix::io::AsRawFd;
4176 guard
4177 .first_iter_signal
4178 .as_ref()
4179 .map(|f| f.as_raw_fd())
4180 .unwrap_or(-1)
4181 } else {
4182 -1
4183 };
4184
4185 // Per-mode dispatch. Thread-mode workers do not need
4186 // pipes — the rendezvous and report channels are
4187 // in-process Rust primitives (`mpsc::sync_channel(0)` +
4188 // `JoinHandle`). Fork mode uses the pipe-based
4189 // scaffolding below.
4190 match dispatch {
4191 Dispatch::Thread => {
4192 spawn_thread_worker(
4193 guard,
4194 group,
4195 affinity,
4196 worker_pipe_fds,
4197 worker_futex,
4198 iter_slot,
4199 phase_epoch_ptr,
4200 first_iter_signal_fd,
4201 )?;
4202 continue;
4203 }
4204 Dispatch::Fork => {
4205 // fall through to the pipe-based dispatch below
4206 }
4207 }
4208
4209 // Create pipe for report and a second pipe for "start" signal.
4210 // Local cleanup on second-pipe failure: the guard has no
4211 // per-worker tracking of half-allocated pipes, so the first
4212 // half closes here before the bail. `O_CLOEXEC` matches
4213 // the defense-in-depth posture above on the inter-worker
4214 // pipe pairs and chain pipes.
4215 let mut report_fds = [0i32; 2];
4216 if unsafe { libc::pipe2(report_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4217 anyhow::bail!(
4218 "worker {}/{} (group {}): report pipe2 failed: {}",
4219 i + 1,
4220 group.num_workers,
4221 group.group_idx,
4222 std::io::Error::last_os_error(),
4223 );
4224 }
4225 // Grow the report pipe so a worker that produces a worst-
4226 // case report (two `Vec<u64>` reservoirs capped at
4227 // `MAX_WAKE_SAMPLES` (100_000) entries each, plus
4228 // smaller fields) finishes `write_all` without blocking
4229 // for the parent to drain. The default pipe capacity on
4230 // Linux is 16 pages (64 KiB on 4 KiB pages, 256 KiB on
4231 // 16 KiB pages); a postcard-encoded `WorkerReport` with
4232 // both reservoirs full is at most ~1.8 MiB (u64 varint
4233 // is up to 9 bytes per entry, 200_000 entries total).
4234 // Without this grow, the worker blocks inside
4235 // `f.write_all(&bytes)` waiting for the parent's
4236 // `read_to_end`. The parent reads workers serially with
4237 // a shared 5 s deadline (`stop_and_collect`); if the
4238 // first worker's drain consumes the budget, every
4239 // subsequent worker's `poll` budget is 0, the parent
4240 // closes the read fd without reading, and the child's
4241 // blocked write returns EPIPE — the report is lost and
4242 // the failure surfaces as a sentinel `WorkerReport`.
4243 // Sizing the pipe to 8 MiB lets the child write the
4244 // full payload in one non-blocking burst, exit, and
4245 // close its write end; when the parent later polls the
4246 // read fd, all data is already buffered in the kernel
4247 // and `read_to_end` returns immediately with EOF.
4248 //
4249 // F_SETPIPE_SZ rounds the requested size up to the next
4250 // power of two; unprivileged callers are clamped to
4251 // `/proc/sys/fs/pipe-max-size` (typically 1 MiB).
4252 // ktstr always runs as root inside the guest, so the
4253 // grow succeeds. A best-effort failure (older kernel
4254 // without F_SETPIPE_SZ, or an unexpected EPERM) leaves
4255 // the default size in place — workers with small
4256 // reports still complete; workers with large reports
4257 // fall back to the historical blocking-write behaviour.
4258 // Logging the failure surfaces the regression without
4259 // failing the whole spawn.
4260 const REPORT_PIPE_SIZE: libc::c_int = 8 * 1024 * 1024;
4261 // SAFETY: `report_fds[1]` is the freshly opened write
4262 // end returned by `pipe2` above; F_SETPIPE_SZ accepts
4263 // any pipe fd (read or write end — both refer to the
4264 // same kernel `struct pipe_inode_info`).
4265 let prev_size =
4266 unsafe { libc::fcntl(report_fds[1], libc::F_SETPIPE_SZ, REPORT_PIPE_SIZE) };
4267 if prev_size < 0 {
4268 let err = std::io::Error::last_os_error();
4269 tracing::warn!(
4270 worker = i + 1,
4271 num_workers = group.num_workers,
4272 group_idx = group.group_idx,
4273 requested_size = REPORT_PIPE_SIZE,
4274 %err,
4275 "F_SETPIPE_SZ on report pipe failed; falling back to default \
4276 pipe capacity. Workers producing >64 KiB reports may block \
4277 in write_all and have their reports truncated by the \
4278 parent's deadline-driven fd close."
4279 );
4280 }
4281 let mut start_fds = [0i32; 2];
4282 if unsafe { libc::pipe2(start_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4283 unsafe {
4284 libc::close(report_fds[0]);
4285 libc::close(report_fds[1]);
4286 }
4287 anyhow::bail!(
4288 "worker {}/{} (group {}): start pipe2 failed: {}",
4289 i + 1,
4290 group.num_workers,
4291 group.group_idx,
4292 std::io::Error::last_os_error(),
4293 );
4294 }
4295
4296 // Block SIGUSR1 across fork so the child inherits a
4297 // SIGUSR1-blocked mask. Without this, there is a
4298 // window between `fork()` returning in the child and
4299 // the child installing `sigusr1_handler` below where
4300 // SIGUSR1's default disposition (terminate) is in
4301 // effect. A `stop_and_collect` call that races against
4302 // a worker still in mid-init — e.g. if a sibling
4303 // worker spawned just before this one already pushed
4304 // its (pid, ...) into `guard.children` and a parent
4305 // thread initiated stop signaling — could deliver
4306 // SIGUSR1 to this child before its handler is
4307 // installed and terminate it via the default action.
4308 // The block + post-install unblock pattern queues any
4309 // SIGUSR1 sent in the window so the handler observes
4310 // it on the unblock; the worker then sets `STOP` and
4311 // exits gracefully.
4312 //
4313 // pthread_sigmask is the multi-threaded equivalent of
4314 // sigprocmask: ktstr's parent process is genuinely
4315 // multi-threaded (test runner threads, tracing
4316 // threads), and `sigprocmask` semantics are unspecified
4317 // in MT programs per signal-safety(7) /
4318 // pthread_sigmask(3) — pthread_sigmask is the correct
4319 // primitive to use here. The fork()-inherited mask is
4320 // the calling thread's mask at fork time, exactly what
4321 // we want.
4322 //
4323 // Old mask is captured so the parent path can restore
4324 // its prior signal mask after fork — the block must
4325 // be transient on the parent side; the parent's
4326 // SIGUSR1 stays under whatever disposition the host
4327 // application configured (typically the default;
4328 // ktstr does not handle SIGUSR1 in the parent
4329 // process).
4330 let mut old_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
4331 let mut block_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
4332 // pthread_sigmask returns 0 on success, a positive errno
4333 // on failure (per POSIX — does NOT set errno). A non-zero
4334 // return here means the SIGUSR1 mask did not actually get
4335 // installed: the child would inherit an unblocked SIGUSR1
4336 // and the post-fork race window between fork() and the
4337 // child's signal() install would terminate the worker on
4338 // its default action. Log so the failure surfaces.
4339 let psm_block_rc = unsafe {
4340 libc::sigemptyset(&mut block_mask);
4341 libc::sigaddset(&mut block_mask, libc::SIGUSR1);
4342 libc::pthread_sigmask(libc::SIG_BLOCK, &block_mask, &mut old_mask)
4343 };
4344 if psm_block_rc != 0 {
4345 tracing::warn!(
4346 rc = psm_block_rc,
4347 "pthread_sigmask(SIG_BLOCK, SIGUSR1) failed pre-fork; child inherits unblocked SIGUSR1 and may terminate on default action before installing handler"
4348 );
4349 }
4350
4351 let pid = unsafe { libc::fork() };
4352 match pid {
4353 -1 => {
4354 // Fork failed: restore the parent's previous
4355 // signal mask before bailing so this failure
4356 // doesn't leave SIGUSR1 blocked in the calling
4357 // thread for the rest of the process lifetime.
4358 let psm_restore_rc = unsafe {
4359 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4360 };
4361 if psm_restore_rc != 0 {
4362 tracing::warn!(
4363 rc = psm_restore_rc,
4364 "pthread_sigmask(SIG_SETMASK) failed restoring mask after fork failure; SIGUSR1 may remain blocked in this thread"
4365 );
4366 }
4367 // Close both fresh pipes so they don't leak
4368 // before the guard reaps the already-forked
4369 // siblings.
4370 unsafe {
4371 libc::close(report_fds[0]);
4372 libc::close(report_fds[1]);
4373 libc::close(start_fds[0]);
4374 libc::close(start_fds[1]);
4375 }
4376 anyhow::bail!(
4377 "worker {}/{} (group {}): fork failed: {}",
4378 i + 1,
4379 group.num_workers,
4380 group.group_idx,
4381 std::io::Error::last_os_error(),
4382 );
4383 }
4384 0 => {
4385 // Child: set parent-death signal BEFORE any other
4386 // post-fork setup so the kernel SIGKILLs this worker
4387 // immediately if the parent dies during the remaining
4388 // init (close fd loops, signal handler install, start-
4389 // pipe wait, worker_main). Without PR_SET_PDEATHSIG,
4390 // a parent crash between fork and start leaves workers
4391 // reparented to init and spinning indefinitely —
4392 // they'd outlive the test run, consume the cgroup's
4393 // CPU, and block the next scenario's cgroup teardown
4394 // with EBUSY. SIGKILL is the only safe choice: it
4395 // cannot be masked and runs before any of this child's
4396 // destructors execute (good — those destructors still
4397 // reference the parent's guard). prctl is NOT listed
4398 // as async-signal-safe by signal-safety(7); safe to
4399 // call here because this is a single-threaded
4400 // post-fork child before any signal handlers are
4401 // installed, so no interleaving can observe partial
4402 // state.
4403 unsafe {
4404 libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL);
4405 }
4406 // Fork-race close: if the parent died between fork()
4407 // return and the prctl above, this child was already
4408 // reparented (typically to pid 1) before PDEATHSIG
4409 // was armed — the death signal is keyed on the CURRENT
4410 // parent, not the parent-at-fork-time, so the signal
4411 // will never fire. getppid() == 1 means we are already
4412 // orphaned; exit now instead of running the full
4413 // worker loop only to leak into init. Using `_exit`
4414 // (async-signal-safe) rather than `exit` so Rust
4415 // destructors that reference the parent's now-dead
4416 // guard don't run on the fork stack.
4417 //
4418 // PR_SET_CHILD_SUBREAPER exception: when an ancestor
4419 // of the ktstr process has called
4420 // `prctl(PR_SET_CHILD_SUBREAPER, 1)` (systemd user
4421 // scopes, some container runtimes, certain CI
4422 // harnesses), orphaned descendants reparent to the
4423 // nearest live subreaper rather than pid 1. In that
4424 // case `getppid() == 1` is false after an orphan-
4425 // race even though the original parent is dead —
4426 // the check is a best-effort fast-path for the
4427 // common "pid 1 catches orphans" case and does NOT
4428 // fire under subreaper ancestry. PDEATHSIG still
4429 // fires correctly in that scenario (the signal is
4430 // triggered when the CURRENT parent dies, and the
4431 // subreaper then inherits us), so the guard is a
4432 // narrowing of the leak window, not an elimination.
4433 //
4434 // Guest-init exception: inside a ktstr guest VM the
4435 // test driver IS pid 1 (it runs as /init), so every
4436 // worker forked by a scenario legitimately has
4437 // `getppid() == 1` even though the parent is alive
4438 // and well. Firing the orphan guard there would kill
4439 // every worker on startup and produce sentinel
4440 // "0 cpus, 0 iterations" reports. `ktstr_guest_init`
4441 // sets `KTSTR_GUEST_INIT=1` before dispatch; that
4442 // variable is inherited by every descendant process,
4443 // so its presence is a reliable signal that pid 1 is
4444 // the legitimate parent. Host-side workloads leave
4445 // the variable unset and retain the orphan detection.
4446 if std::env::var_os(crate::KTSTR_GUEST_INIT_ENV).is_none()
4447 && unsafe { libc::getppid() } == 1
4448 {
4449 exit_child_success();
4450 }
4451 // Make this worker its own process-group leader so
4452 // any descendants it spawns inherit `pgid == worker_pid`.
4453 // `stop_and_collect` / Drop issue `killpg(worker_pid,
4454 // SIGKILL)` alongside the direct `kill` — without a
4455 // private pgid, descendants forked by a
4456 // [`WorkType::Custom`] body (or any future workload
4457 // that spawns helpers) stay in the parent Rust
4458 // process's pgid, survive the worker's SIGKILL, and
4459 // orphan onto init. PR_SET_PDEATHSIG handles the
4460 // "parent crashes" case but is per-task and cleared
4461 // on fork, so grandchildren don't inherit it — the
4462 // pgid route is the only safe reach for them when
4463 // teardown is explicit. Failure is silently ignored:
4464 // the only reachable failure mode for setpgid(0, 0)
4465 // in a just-forked child is EPERM from an earlier
4466 // setsid (we never call it), so a return of -1 here
4467 // means the kernel invariant changed and the reach
4468 // degrades to "leader only" — same as the pre-batch
4469 // behavior. Async-signal-safe per signal-safety(7).
4470 unsafe {
4471 libc::setpgid(0, 0);
4472 }
4473 // Install signal handler BEFORE unblocking
4474 // SIGUSR1: the parent blocked SIGUSR1 across
4475 // fork so this child inherits a blocked mask;
4476 // any SIGUSR1 sent by `stop_and_collect` while
4477 // the mask is in effect queues until we
4478 // unblock. Once the handler is registered we
4479 // restore the original (pre-block) mask via
4480 // `pthread_sigmask(SIG_SETMASK, …)`, which
4481 // delivers any pending SIGUSR1 directly into
4482 // `sigusr1_handler` and sets `STOP` cleanly —
4483 // the worker proceeds into the start-wait
4484 // poll() with handler armed and any pending
4485 // signal already consumed. Without this two-
4486 // step, SIGUSR1 would fire under its default
4487 // disposition (terminate) during the window
4488 // between `fork()` and `signal()`.
4489 STOP.store(false, Ordering::Relaxed);
4490 // libc::signal returns SIG_ERR (cast as
4491 // sighandler_t) on failure with errno set.
4492 // pthread_sigmask returns 0 on success, errno
4493 // value on failure. Both failures here are
4494 // visible defects: signal() fail means SIGUSR1
4495 // keeps default disposition (terminate) and
4496 // stop_and_collect cannot stop the worker
4497 // gracefully — it falls through to the
4498 // killpg/SIGKILL escalation; pthread_sigmask()
4499 // fail means the SIGUSR1 we blocked across
4500 // fork stays blocked, so the parent's stop
4501 // SIGUSR1 queues forever in the child and the
4502 // child runs the full work loop until the
4503 // killpg escalation.
4504 //
4505 // tracing is unsafe in a post-fork child
4506 // before _exit (the parent's tracing
4507 // subscriber may be locked elsewhere); use
4508 // eprintln! which is fork-safe per
4509 // signal-safety(7) (write(2)).
4510 let sig_prev = unsafe {
4511 libc::signal(
4512 libc::SIGUSR1,
4513 sigusr1_handler as *const () as libc::sighandler_t,
4514 )
4515 };
4516 if sig_prev == libc::SIG_ERR {
4517 let errno = std::io::Error::last_os_error();
4518 eprintln!(
4519 "ktstr: signal(SIGUSR1) install failed in worker child: {errno}; graceful stop unavailable, killpg escalation will reap"
4520 );
4521 }
4522 let psm_unblock_rc = unsafe {
4523 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4524 };
4525 if psm_unblock_rc != 0 {
4526 eprintln!(
4527 "ktstr: pthread_sigmask(SIG_SETMASK) unblock failed in worker child: rc={psm_unblock_rc}; SIGUSR1 stays blocked, killpg escalation will reap"
4528 );
4529 }
4530 // Close unused pipe ends
4531 unsafe {
4532 libc::close(report_fds[0]);
4533 libc::close(start_fds[1]);
4534 }
4535 // Close pipe ends belonging to other workers
4536 // in this pair, AND every pipe fd that belongs
4537 // to any other pair anywhere in the workload —
4538 // including pairs owned by other groups, since
4539 // every pre-fork allocation lives in
4540 // `guard.pipe_pairs` regardless of which group
4541 // declared it. The fork inherits the parent's
4542 // entire fd table; without this sweep, a
4543 // composed-group worker would hold open every
4544 // primary-group pipe fd for its lifetime,
4545 // producing fd leaks and (for chain-shaped
4546 // workloads) keeping reader-side blocks live
4547 // when the writer-side closes.
4548 if needs_pipes {
4549 let pair_idx = pipe_pair_base + i / 2;
4550 let (ref ab, ref ba) = guard.pipe_pairs[pair_idx];
4551 if i % 2 == 0 {
4552 // Worker A keeps ba[0] (read) and ab[1] (write).
4553 // Close ab[0] and ba[1].
4554 unsafe {
4555 libc::close(ab[0]);
4556 libc::close(ba[1]);
4557 }
4558 } else {
4559 // Worker B keeps ab[0] (read) and ba[1] (write).
4560 // Close ab[1] and ba[0].
4561 unsafe {
4562 libc::close(ab[1]);
4563 libc::close(ba[0]);
4564 }
4565 }
4566 // Close all pipe fds from other pairs (any group).
4567 for (j, (ab2, ba2)) in guard.pipe_pairs.iter().enumerate() {
4568 if j != pair_idx {
4569 unsafe {
4570 libc::close(ab2[0]);
4571 libc::close(ab2[1]);
4572 libc::close(ba2[0]);
4573 libc::close(ba2[1]);
4574 }
4575 }
4576 }
4577 } else {
4578 // Worker doesn't own any pipe pair, but
4579 // other groups' pipe pairs are still in the
4580 // child's fd table — close them all.
4581 for (ab2, ba2) in guard.pipe_pairs.iter() {
4582 unsafe {
4583 libc::close(ab2[0]);
4584 libc::close(ab2[1]);
4585 libc::close(ba2[0]);
4586 libc::close(ba2[1]);
4587 }
4588 }
4589 }
4590 if let Some(depth) = chain_depth
4591 && depth > 0
4592 {
4593 let chain_idx = chain_pipes_base + i / depth;
4594 let stage = i % depth;
4595 let prev_stage = (stage + depth - 1) % depth;
4596 // Close every fd in the chain that this
4597 // stage does not own. Owned fds (kept open):
4598 // - chain[prev_stage][0]: read end of the
4599 // pipe predecessor writes to.
4600 // - chain[stage][1]: write end of the
4601 // pipe successor reads from.
4602 // Everything else is the inverse end of an
4603 // owned pipe or fully unrelated.
4604 for (s, pipe) in guard.chain_pipes[chain_idx].iter().enumerate() {
4605 // Always close the write end of the
4606 // predecessor's pipe (we only need its
4607 // read end).
4608 if s == prev_stage {
4609 unsafe {
4610 libc::close(pipe[1]);
4611 }
4612 // Always close the read end of our own
4613 // pipe (we only need its write end).
4614 } else if s == stage {
4615 unsafe {
4616 libc::close(pipe[0]);
4617 }
4618 // Pipes belonging to neither this stage
4619 // nor its predecessor: close both ends.
4620 } else {
4621 unsafe {
4622 libc::close(pipe[0]);
4623 libc::close(pipe[1]);
4624 }
4625 }
4626 }
4627 // Close every fd from other chains (any group).
4628 for (cj, chain) in guard.chain_pipes.iter().enumerate() {
4629 if cj != chain_idx {
4630 for pipe in chain {
4631 unsafe {
4632 libc::close(pipe[0]);
4633 libc::close(pipe[1]);
4634 }
4635 }
4636 }
4637 }
4638 } else {
4639 // This group has no chain pipes, but other
4640 // groups may. Close every chain-pipe fd
4641 // inherited via fork — leaving a primary
4642 // group's chain pipe open in a composed
4643 // worker would prevent the chain from ever
4644 // observing EOF on its read ends.
4645 for chain in guard.chain_pipes.iter() {
4646 for pipe in chain {
4647 unsafe {
4648 libc::close(pipe[0]);
4649 libc::close(pipe[1]);
4650 }
4651 }
4652 }
4653 }
4654 // Layered defense against child-side unwinding
4655 // reaching the forked-from-parent drops:
4656 //
4657 // 1. No-op panic hook — the default hook prints a
4658 // multi-line backtrace to stderr, which is a
4659 // shared fd with the parent post-fork. A panic
4660 // in the child would interleave garbled output
4661 // with the parent's tracing log and confuse
4662 // downstream parsers. Install a silent hook
4663 // before catch_unwind ONLY under
4664 // `panic = "unwind"` — under
4665 // `panic = "abort"` (release / nextest
4666 // profile) the silent hook would suppress the
4667 // panic message before SIGABRT fires, leaving
4668 // operators with zero diagnostic from a
4669 // crashed worker. Letting the default hook
4670 // write a backtrace to stderr is the lesser
4671 // evil under abort: the message may interleave
4672 // with parent tracing output but at least
4673 // surfaces the panic site. catch_unwind itself
4674 // is a no-op under abort (the closure is
4675 // invoked normally and the abort terminates
4676 // the child before any Err return), so gating
4677 // only the hook installation — not the
4678 // catch_unwind call — keeps the unwind-build
4679 // fast-path unchanged while restoring
4680 // release-build observability.
4681 //
4682 // 2. `_exit(0|1)` after the closure (success or
4683 // catch_unwind Err) — the child never returns
4684 // to a frame whose `SpawnGuard` Drop could
4685 // run. `_exit(2)` bypasses Rust's stack-unwind
4686 // drops and the static-destructor table both,
4687 // so the parent-owned `SpawnGuard` whose
4688 // storage was duplicated by `fork()` cannot
4689 // SIGKILL its siblings (fratricide) from the
4690 // child path.
4691 //
4692 // 3. `panic::catch_unwind` — catches any panic
4693 // before it escapes this arm. Belt-and-braces
4694 // against (a) additional Drops on this
4695 // frame's stack (e.g. future refactors that
4696 // add more RAII) and (b) alloc/OOM panics
4697 // during worker_main / postcard encode.
4698 //
4699 // Caveat: catch_unwind is a no-op under
4700 // `panic = "abort"`, which ktstr's Cargo.toml
4701 // DOES set in `[profile.release]`. In release
4702 // builds a panic inside this closure aborts
4703 // the child immediately (SIGABRT); the
4704 // `catch_unwind` call compiles but never
4705 // returns `Err`, and neither the
4706 // `f.write_all(&bytes)` nor the `_exit(1)`
4707 // below runs on the panic path. The parent's
4708 // `stop_and_collect` therefore observes a
4709 // missing WorkerReport and fills in a
4710 // sentinel — that sentinel fallback IS the
4711 // release-build correctness mechanism. Under
4712 // abort the silent hook is NOT installed (see
4713 // item 1) so the default panic handler writes
4714 // the panic location and message to stderr
4715 // just before SIGABRT, giving operators the
4716 // diagnostic they would otherwise have lost.
4717 // Defense (2) still applies unchanged: abort
4718 // skips Drops (matching the `_exit` path's
4719 // no-Drop guarantee). Dev/test builds (cargo
4720 // test, cargo nextest run — dev profile
4721 // inherits default unwind semantics) still
4722 // get a real `catch_unwind` Err → `_exit(1)`
4723 // fast-path with a silent hook so the
4724 // backtrace doesn't pollute test output.
4725 //
4726 // Global-state safety under unwind, scoped
4727 // to `worker_main`'s reachable code path —
4728 // the `fork()` child's observable set. Two
4729 // items: `STOP: AtomicBool` and
4730 // `STATIC_HOST_INFO: OnceLock<_>`. Neither
4731 // of them carries a Drop whose body touches
4732 // the inherited MAP_SHARED regions or the
4733 // parent-owned pipe fds. Under a
4734 // hypothetical unwind that escaped
4735 // `catch_unwind` (a double-panic that
4736 // bypasses the landing pad), the only
4737 // fork-child Drops that actually matter are
4738 // the parent-owned `SpawnGuard` (covered by
4739 // the `_exit` no-Drop guarantee above) and
4740 // the child-local `wake_latencies_ns` /
4741 // `migrations` `Vec<T>` (per-process heap,
4742 // no cross-process impact). `STATIC_HOST_INFO`'s
4743 // inner Drop frees a handful of
4744 // `Option<String>`s and is safe on either
4745 // side of fork. Crate-wide statics outside
4746 // this set (fetch, probe, vmm, …) are out
4747 // of scope — this audit pins only what the
4748 // fork-child can reach from `worker_main`.
4749 //
4750 // 4. `_exit(1)` on catch_unwind Err, `_exit(0)`
4751 // on Ok — bypasses Rust's global static
4752 // destructors that a plain `return` would
4753 // run.
4754 //
4755 // `AssertUnwindSafe` is justified: the child
4756 // unconditionally _exits after this block, so no
4757 // post-unwind invariant can be observed.
4758 #[cfg(panic = "unwind")]
4759 {
4760 let _ = std::panic::take_hook();
4761 std::panic::set_hook(Box::new(|_| {}));
4762 }
4763 let child_result =
4764 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4765 // Wait for parent to move us to cgroup before starting work.
4766 // Use poll() with a 30s timeout — signal-safe after fork,
4767 // prevents hanging forever if the parent stalls.
4768 let mut pfd = libc::pollfd {
4769 fd: start_fds[0],
4770 events: libc::POLLIN,
4771 revents: 0,
4772 };
4773 let ret = unsafe { libc::poll(&mut pfd, 1, 30_000) };
4774 if ret <= 0 {
4775 exit_child_error();
4776 }
4777 let mut buf = [0u8; 1];
4778 let mut f = unsafe { std::fs::File::from_raw_fd(start_fds[0]) };
4779 let _ = f.read_exact(&mut buf);
4780 drop(f);
4781 // Do NOT reset STOP here. STOP is initialized to
4782 // false at the pre-handshake site above (after the
4783 // signal-handler install) and a SIGUSR1 delivered
4784 // between the mask unblock and this point reflects
4785 // a legitimate parent-issued stop request — the
4786 // explicit-start path in `stop_and_collect` skips
4787 // the readiness barrier and races
4788 // `kill(pid, SIGUSR1)` against the worker's start-
4789 // byte read here, so any STOP=true observed at
4790 // this point must propagate into the work loop.
4791 // Clobbering it would lose the stop and produce a
4792 // worker that loops until the killpg/SIGKILL
4793 // escalation.
4794 // Publish a single "ready" byte on the report pipe
4795 // BEFORE entering `worker_main`. The parent's
4796 // auto-start barrier in `stop_and_collect` polls
4797 // every worker's report fd for POLLIN with a
4798 // bounded deadline and consumes this byte as the
4799 // explicit signal that the worker has finished
4800 // its post-fork init (cgroup placement, SIGUSR1
4801 // unblock) and is about to enter the
4802 // work loop. Replaces the prior 500 ms blind
4803 // sleep — under host CPU contention, real worker
4804 // start latency can exceed that sleep, dropping
4805 // a worker into its loop AFTER `stop_and_collect`
4806 // has already signalled stop, which surfaces as
4807 // a starvation false-positive.
4808 //
4809 // Done as a raw `libc::write` (not `File::write`)
4810 // to keep the report-fd's ownership inside its
4811 // existing `std::fs::File::from_raw_fd` block
4812 // below — wrapping it here would close the fd
4813 // when the local goes out of scope, before the
4814 // post-`worker_main` write_all path. A 1-byte
4815 // write to a freshly-grown 8 MiB pipe with no
4816 // reader contention completes synchronously
4817 // without blocking; signal-safe after fork.
4818 //
4819 // The byte is `b'r'`. The parent's collect path
4820 // strips a leading `b'r'` before
4821 // `postcard::from_bytes` so the
4822 // explicit-start call site (which skips the
4823 // barrier) still parses correctly. A zero return or short write
4824 // is treated as best-effort: if the kernel
4825 // refuses the write (EFAULT cannot occur with a
4826 // stack pointer; EBADF cannot occur on a fresh
4827 // fd; EPIPE only fires if the reader closed,
4828 // which the parent does not do mid-spawn), the
4829 // parent's barrier-deadline path falls back to
4830 // the same "skip this worker" branch as a
4831 // genuinely-dead worker.
4832 let ready_byte: u8 = b'r';
4833 unsafe {
4834 libc::write(
4835 report_fds[1],
4836 &ready_byte as *const u8 as *const libc::c_void,
4837 1,
4838 );
4839 }
4840 // Now run. Fork-mode workers thread the global
4841 // STOP through `worker_main` — the SIGUSR1 handler
4842 // is process-wide, so flipping `STOP` from
4843 // `sigusr1_handler` is what reaches the loop's
4844 // `stop.load(Relaxed)` checks.
4845 let report = worker_main(
4846 affinity,
4847 group.work_type.clone(),
4848 group.sched_policy,
4849 group.mem_policy.clone(),
4850 group.mpol_flags,
4851 group.nice,
4852 group.comm.as_deref(),
4853 group.uid,
4854 group.gid,
4855 group.numa_node,
4856 worker_pipe_fds,
4857 worker_futex,
4858 group.park_after,
4859 // Fork worker: the real signal fd (or `-1`
4860 // when the group did not opt in). The child
4861 // holds its own fd-table copy of the parent's
4862 // eventfd, inherited across `fork()`.
4863 first_iter_signal_fd,
4864 iter_slot,
4865 phase_epoch_ptr,
4866 &STOP,
4867 group.group_idx,
4868 );
4869 let bytes = postcard::to_stdvec(&report).unwrap_or_default();
4870 let mut f = unsafe { std::fs::File::from_raw_fd(report_fds[1]) };
4871 if let Err(e) = f.write_all(&bytes) {
4872 // tracing is unsafe in a post-fork child
4873 // (the global subscriber may be holding a
4874 // lock taken in another thread of the
4875 // parent process before fork); eprintln
4876 // goes straight to stderr without locking
4877 // a tracing subscriber. The parent
4878 // observes a partial / empty pipe payload
4879 // and emits a sentinel WorkerReport with
4880 // exit_info populated, but without this
4881 // log line the underlying I/O error
4882 // (EPIPE if the parent closed the read
4883 // end early, EFBIG / ENOSPC on a backing-
4884 // file pipe, EIO on a transient kernel
4885 // failure) is invisible.
4886 eprintln!("ktstr: worker report write_all failed: {e}");
4887 }
4888 drop(f);
4889 }));
4890 if child_result.is_ok() {
4891 exit_child_success();
4892 } else {
4893 exit_child_error();
4894 }
4895 }
4896 child_pid => {
4897 // Parent: restore the pre-fork signal mask
4898 // (the block was transient — only the child
4899 // inherits the blocked SIGUSR1, and that
4900 // child unblocks immediately after installing
4901 // its handler). Without this restore, SIGUSR1
4902 // would stay blocked in the calling thread for
4903 // the lifetime of the workload, swallowing any
4904 // signal directed at the parent process.
4905 let psm_parent_restore_rc = unsafe {
4906 libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4907 };
4908 if psm_parent_restore_rc != 0 {
4909 tracing::warn!(
4910 rc = psm_parent_restore_rc,
4911 "pthread_sigmask(SIG_SETMASK) failed restoring mask in parent post-fork; SIGUSR1 stays blocked in this thread for the lifetime of the workload"
4912 );
4913 }
4914 // Close unused pipe ends.
4915 unsafe {
4916 libc::close(report_fds[1]);
4917 libc::close(start_fds[0]);
4918 }
4919 // child_pid is positive by the -1 arm above, so
4920 // fits in pid_t directly — store as pid_t so
4921 // every downstream libc call avoids the u32→i32
4922 // sign-cast wraparound bug.
4923 guard.children.push(ForkedChild {
4924 pid: child_pid,
4925 report_fd: report_fds[0],
4926 start_fd: start_fds[1],
4927 kind: ForkedChildKind::Worker {
4928 group_idx: group.group_idx,
4929 },
4930 });
4931 }
4932 }
4933 }
4934
4935 Ok(())
4936 }
4937
4938 /// Kernel TIDs of all worker tasks, in spawn order.
4939 ///
4940 /// Returned as `libc::pid_t` — the kernel's native type — so
4941 /// callers feed them directly into `kill`, `waitpid`,
4942 /// `Pid::from_raw`, and `sched_setaffinity` writes without any
4943 /// sign-cast at the libc boundary.
4944 ///
4945 /// # WARNING — `cgroup.procs` for `CloneMode::Thread`
4946 ///
4947 /// **For `CloneMode::Thread`, passing these TIDs to a
4948 /// `cgroup.procs` write migrates the ENTIRE test-runner process
4949 /// into that cgroup**: cgroup.procs writes are tgid-scoped, and
4950 /// every Thread worker shares the test runner's tgid. The first
4951 /// such write moves the test harness, every parent thread, and
4952 /// every sibling worker into the destination cgroup; subsequent
4953 /// writes are no-ops because they all point at the same tgid.
4954 /// Use cgroup v2 threaded-mode cgroups with `cgroup.threads`
4955 /// for per-thread placement. `CloneMode::Fork` is the right
4956 /// choice when each worker needs its own cgroup.
4957 ///
4958 /// # Per-mode interpretation
4959 ///
4960 /// - [`CloneMode::Fork`]: each entry is the worker's pid
4961 /// (== tgid == kernel tid because the worker is its own
4962 /// thread-group leader). Safe to feed into `cgroup.procs`.
4963 /// - [`CloneMode::Thread`]: each entry is the worker's
4964 /// `gettid()` value — distinct kernel tasks inside the
4965 /// parent's tgid. Safe for `sched_setaffinity(tid, ...)`;
4966 /// safe for `cgroup.threads` writes under a threaded-mode
4967 /// cgroup; **not** safe for `cgroup.procs` (see warning above).
4968 ///
4969 /// # Thread tid publish ordering
4970 ///
4971 /// Thread workers publish their `gettid()` via an
4972 /// `Arc<AtomicI32>` BEFORE the start handshake (the publish is
4973 /// the first thing the worker closure does, before blocking on
4974 /// `start_rx`). `spawn_thread_worker` blocks on a paired
4975 /// rendezvous channel until the worker reaches the publish
4976 /// point, so by the time [`WorkloadHandle::spawn`] returns,
4977 /// every thread worker's `tid` is non-zero (or spawn would have
4978 /// bailed with a "failed to publish gettid() within 2 s"
4979 /// error). Post-spawn `worker_pids()` is safe to call without
4980 /// first calling [`Self::start`]. The publish uses `Release`;
4981 /// this reader uses `Acquire`, pairing release-acquire so that
4982 /// any reader observing a non-zero tid is also guaranteed to
4983 /// observe the worker's post-publish state.
4984 ///
4985 /// # `pcomm` containers
4986 ///
4987 /// Groups configured with [`WorkSpec::pcomm`] yield a single
4988 /// container process whose tgid leader pid is what the parent
4989 /// holds; the per-thread `gettid()` values of the workers running
4990 /// inside the container are not exported across the process
4991 /// boundary. Each pcomm group contributes ONE entry to the
4992 /// returned vector — the container pid — regardless of how many
4993 /// thread workers it hosts. This pid is correct for `cgroup.procs`
4994 /// migration (the container's whole tgid moves) but is NOT
4995 /// suitable as a target for per-thread `sched_setaffinity` —
4996 /// see [`Self::set_affinity`] for the matching error path.
4997 ///
4998 /// Order: forked children (conventional workers + pcomm
4999 /// containers, in spawn order) followed by Thread-mode workers
5000 /// (in spawn order). A workload that mixes pcomm groups with
5001 /// non-pcomm Thread-mode groups produces both populated
5002 /// collections; a workload using only one dispatch path produces
5003 /// only one populated collection.
5004 pub fn worker_pids(&self) -> Vec<libc::pid_t> {
5005 let mut out = Vec::with_capacity(self.children.len() + self.threads.len());
5006 out.extend(self.children.iter().map(|c| c.pid));
5007 out.extend(self.threads.iter().map(|tw| tw.tid.load(Ordering::Acquire)));
5008 out
5009 }
5010
5011 /// Worker pids suitable for `cgroup.procs` migration.
5012 ///
5013 /// `cgroup.procs` is **tgid-scoped** in the kernel: writing a
5014 /// tid migrates the entire thread group containing that tid
5015 /// (`kernel/cgroup/cgroup.c::__cgroup_procs_write` resolves the
5016 /// passed pid to its leader via `find_lock_task_mm` /
5017 /// `cgroup_procs_write_start`). Under [`CloneMode::Thread`]
5018 /// every worker shares the test harness's tgid, so feeding
5019 /// [`Self::worker_pids`] to `cgroup.procs` would migrate the
5020 /// harness itself — catastrophic.
5021 ///
5022 /// Returns the per-worker pids when the spawn used
5023 /// [`CloneMode::Fork`] (each worker has its own tgid). Bails
5024 /// for [`CloneMode::Thread`] with an actionable diagnostic
5025 /// pointing at `cgroup.threads` (the thread-scoped sibling) as
5026 /// the right migration sink for thread workers.
5027 ///
5028 /// Callers that integrate with `cgroup.procs` writes — e.g.
5029 /// [`crate::cgroup::CgroupManager::move_tasks`] — should call
5030 /// this in place of [`Self::worker_pids`] so a misconfigured
5031 /// Thread-mode test fails at the migration step rather than
5032 /// silently moving the harness into the per-test cgroup.
5033 pub fn worker_pids_for_cgroup_procs(&self) -> Result<Vec<libc::pid_t>> {
5034 if !self.threads.is_empty() {
5035 anyhow::bail!(
5036 "WorkloadHandle::worker_pids_for_cgroup_procs: workers were \
5037 spawned with CloneMode::Thread; their pids share the test \
5038 harness's tgid and a `cgroup.procs` write would migrate the \
5039 harness. Use `cgroup.threads` (thread-scoped) for Thread-mode \
5040 workers, or switch to CloneMode::Fork."
5041 );
5042 }
5043 Ok(self.worker_pids())
5044 }
5045
5046 /// Signal all workers to start working (after they've been
5047 /// placed in cgroups, if applicable).
5048 ///
5049 /// Idempotent — subsequent calls after the first are no-ops.
5050 pub fn start(&mut self) {
5051 if self.started {
5052 return;
5053 }
5054 self.started = true;
5055 // Fork-mode: write a byte to each child's start pipe (covers
5056 // both conventional workers and pcomm containers — both wait
5057 // on the same start-byte handshake before entering their work
5058 // path).
5059 for child in &mut self.children {
5060 unsafe {
5061 libc::write(child.start_fd, b"s".as_ptr() as *const _, 1);
5062 libc::close(child.start_fd);
5063 }
5064 child.start_fd = -1;
5065 }
5066 // Thread-mode: send `()` on each worker's start_tx. The
5067 // SyncSender(0) rendezvous means each send blocks until the
5068 // worker calls recv(); if the worker has been joined or has
5069 // panicked before reaching recv, send returns Err which we
5070 // swallow (the join in stop_and_collect surfaces the real
5071 // exit). Take ownership so a future start() call (illegal
5072 // by the idempotence guard above) can't re-send.
5073 for tw in &mut self.threads {
5074 if let Some(tx) = tw.start_tx.take() {
5075 let _ = tx.send(());
5076 }
5077 }
5078 }
5079
5080 /// Set CPU affinity for worker at `idx`.
5081 ///
5082 /// For [`CloneMode::Fork`] the per-worker pid addresses a
5083 /// distinct kernel task. For [`CloneMode::Thread`] the worker's
5084 /// `gettid()` is what `sched_setaffinity(tid, ...)` accepts;
5085 /// this method reads the tid from the worker's
5086 /// `Arc<AtomicI32>` (with `Acquire` ordering, paired with the
5087 /// `Release` publish on the worker thread). Returns an error
5088 /// if the thread has not yet published its tid — call
5089 /// [`start()`](Self::start) first so the worker reaches its
5090 /// `gettid()` publish before reading.
5091 ///
5092 /// Bails for `ForkedChildKind::PcommContainer` entries: the
5093 /// container's tgid leader pid is the only kernel handle the
5094 /// parent holds for that group, but `sched_setaffinity` against
5095 /// that pid pins only the leader thread (a placeholder thread
5096 /// that never enters the work loop), not the worker threads
5097 /// running the workload. The container's worker tids are not
5098 /// published across the process boundary. Bake the affinity into
5099 /// [`WorkSpec::affinity`] at spawn time instead — `worker_main`
5100 /// applies it per-thread inside the container.
5101 ///
5102 /// Index space: `[0, children.len())` addresses forked children
5103 /// (conventional workers and pcomm containers), and
5104 /// `[children.len(), children.len() + threads.len())` addresses
5105 /// Thread-mode workers, matching the ordering of
5106 /// [`Self::worker_pids`].
5107 pub fn set_affinity(&self, idx: usize, cpus: &BTreeSet<usize>) -> Result<()> {
5108 let pid = if idx < self.children.len() {
5109 let child = &self.children[idx];
5110 if let ForkedChildKind::PcommContainer { groups } = &child.kind {
5111 let total: usize = groups.iter().map(|(_, n)| n).sum();
5112 anyhow::bail!(
5113 "set_affinity: child {idx} is a pcomm container \
5114 hosting {total} thread workers; per-thread \
5115 tids are not exported across the process boundary. \
5116 Set affinity via WorkSpec::affinity at spawn time \
5117 so worker_main applies it inside the container."
5118 );
5119 }
5120 child.pid
5121 } else {
5122 let thread_idx = idx - self.children.len();
5123 let tid = self.threads[thread_idx].tid.load(Ordering::Acquire);
5124 if tid == 0 {
5125 anyhow::bail!(
5126 "set_affinity: thread worker {thread_idx} has not yet \
5127 published gettid() (call start() first)"
5128 );
5129 }
5130 tid
5131 };
5132 set_thread_affinity(pid, cpus)
5133 }
5134
5135 /// Read all workers' current iteration counts from shared memory.
5136 ///
5137 /// Each element is the monotonically increasing iteration count for
5138 /// that worker, read with Relaxed ordering. Returns an empty vec
5139 /// if no workers were spawned.
5140 ///
5141 /// # Ordering rationale — why Relaxed is sound
5142 ///
5143 /// Every producer (the worker-side store at the
5144 /// `worker_main` publish sites) writes its slot with Relaxed
5145 /// ordering, and this reader loads with Relaxed too. No
5146 /// happens-before edge is needed because no host-side consumer
5147 /// pairs the iteration count with OTHER shared state: the
5148 /// parent samples these counters to answer "is this worker
5149 /// still making progress?" and feeds deltas into gap
5150 /// detection, not into any data-dependent follow-up read from
5151 /// a different shared memory location. A stale value on one
5152 /// sample is self-correcting — the next snapshot picks up the
5153 /// newer count without any cross-field invariant to break.
5154 ///
5155 /// The per-slot single-producer / multi-sampler shape is
5156 /// inherently non-tearing on every supported target
5157 /// (AtomicU64 is architecture-primitive on x86_64 and aarch64
5158 /// LSE with 8-byte alignment enforced by the type). The only
5159 /// question is ordering, and the audit above concludes Relaxed
5160 /// is load-bearingly correct — promoting either side to
5161 /// Acquire/Release would add a barrier with no corresponding
5162 /// paired operation to synchronise with.
5163 pub fn snapshot_iterations(&self) -> Vec<u64> {
5164 if self.iter_counters.is_null() || self.iter_counter_len == 0 {
5165 return Vec::new();
5166 }
5167 (0..self.iter_counter_len)
5168 .map(|i| {
5169 // SAFETY: alignment + atomic-only-access invariant
5170 // established at the iter_counters mmap site in
5171 // `WorkloadHandle::spawn` and carried by the
5172 // `*mut AtomicU64` type. Relaxed ordering: see the
5173 // rationale in the outer doc comment.
5174 unsafe { &*self.iter_counters.add(i) }.load(Ordering::Relaxed)
5175 })
5176 .collect()
5177 }
5178
5179 /// Block until at least `n` workers have signalled their first counted
5180 /// iteration, or `deadline` passes. Returns `true` on the former,
5181 /// `false` on timeout. The evented counterpart to polling
5182 /// [`Self::snapshot_iterations`]: workers `write(2)` a `1` into the
5183 /// shared first-iteration eventfd (see
5184 /// [`WorkloadConfig::signal_first_iteration`]) and this drains the
5185 /// counter — which sums those `1`s — accumulating across reads until it
5186 /// reaches `n`.
5187 ///
5188 /// Caller contract: only meaningful when the workload set
5189 /// `signal_first_iteration` (and thus spawned fork workers with the
5190 /// eventfd). Without the eventfd there is nothing to wait on, so this
5191 /// returns `false` immediately (a `debug_assert` catches a misuse in
5192 /// debug builds). `n == 0` returns `true` vacuously.
5193 pub(crate) fn wait_first_iteration_all(&self, deadline: Instant, n: usize) -> bool {
5194 let Some(signal_fd) = self.first_iter_signal.as_ref() else {
5195 debug_assert!(
5196 false,
5197 "wait_first_iteration_all called on a handle with no first-iteration \
5198 eventfd (workload did not set signal_first_iteration, or dispatch \
5199 was not fork)",
5200 );
5201 return false;
5202 };
5203 use std::os::unix::io::AsRawFd;
5204 poll_eventfd_reaches(signal_fd.as_raw_fd(), deadline, n as u64)
5205 }
5206
5207 /// Broadcast the current scenario phase to this handle's workers.
5208 /// The scenario engine calls this at each step boundary — the
5209 /// 1-indexed phase (Step k → k + 1) at StepStart, `u32::MAX` (the
5210 /// inter-step gap sentinel) at StepEnd — so a backdrop worker
5211 /// drains a per-phase `PhaseSlice` on each transition. No-op when
5212 /// the handle has no `phase_epoch` region. Release store pairs with
5213 /// the worker's Relaxed load; it is telemetry, so no happens-before
5214 /// edge is required (see the `unsafe impl Send for WorkloadHandle`
5215 /// SAFETY note), but Release keeps the store promptly visible.
5216 pub(crate) fn set_phase_epoch(&self, epoch: u32) {
5217 if self.phase_epoch.is_null() {
5218 return;
5219 }
5220 // SAFETY: alignment + atomic-only-access invariant established
5221 // at the phase_epoch mmap site in `WorkloadHandle::spawn` /
5222 // `spawn_pcomm_cgroup` and carried by the `*mut AtomicU32`
5223 // type. The parent is the sole writer; workers are readers.
5224 unsafe { &*self.phase_epoch }.store(epoch, Ordering::Release);
5225 }
5226
5227 /// Stop all workers, collect their reports, and wait for exit.
5228 ///
5229 /// Auto-starts workers if [`start()`](Self::start) was not called,
5230 /// then waits on an event-driven barrier — each fork worker
5231 /// writes a single `b'r'` byte to its report pipe immediately
5232 /// after the start handshake completes, and the parent polls
5233 /// every report fd for `POLLIN` with a 5 s deadline. The
5234 /// barrier wakes the moment the slowest worker finishes its
5235 /// post-fork init, replacing the prior unconditional 500 ms
5236 /// sleep that under-waited under host CPU contention and
5237 /// over-waited on idle hosts. Thread-mode workers are pre-
5238 /// synchronised by `start()`'s `mpsc::sync_channel(0)` rendezvous,
5239 /// so the barrier is a no-op when no fork children were spawned.
5240 /// Consumes `self` -- workers cannot be restarted.
5241 ///
5242 /// Workers that fail to produce a report (died, timed out, or wrote
5243 /// corrupt data) get a zeroed-out sentinel report with `work_units: 0`.
5244 /// This ensures `assert_not_stuck` catches dead workers as starvation
5245 /// failures.
5246 ///
5247 /// # Shutdown latency
5248 ///
5249 /// Workers spend their steady-state time blocked inside a
5250 /// `futex_wait` with timeout `WORKER_STOP_POLL_NS` (~100 ms).
5251 /// The "stop signal" is a per-mode flag the worker checks on
5252 /// every futex-wait wake; the wake interval bounds shutdown
5253 /// latency.
5254 ///
5255 /// _Fork mode_ — `stop_and_collect` sends SIGUSR1 to each
5256 /// worker pid; the per-process `sigusr1_handler` flips the
5257 /// global `STOP` in that worker's CoW address space, and the
5258 /// worker observes it on the NEXT futex wake (partner-writes
5259 /// or the 100 ms timeout, whichever comes first). The signal
5260 /// handler is process-wide and reaches one worker per kill().
5261 ///
5262 /// _Thread mode_ — `stop_and_collect` calls
5263 /// `worker.stop.store(true, Relaxed)` directly on each
5264 /// worker's `Arc<AtomicBool>`. SIGUSR1 is process-wide and
5265 /// useless for per-thread stop control, so no signal is sent;
5266 /// the worker observes the flag flip on its next futex-wait
5267 /// wake at the same 100 ms cadence.
5268 ///
5269 /// Callers that budget a graceful-shutdown window should
5270 /// allow at least one `WORKER_STOP_POLL_NS` tick (~100 ms)
5271 /// between flag flip and final collect, over and above any
5272 /// report-flush / IO latency. Tighter windows can race the
5273 /// worker's pre-stop iteration and surface as a missing
5274 /// report, which is then mapped to the sentinel path above.
5275 ///
5276 /// # Exit-shape invariance
5277 ///
5278 /// Collection discriminates purely on the presence and validity of
5279 /// the worker's pipe-delivered postcard payload — **not** on
5280 /// `waitpid` exit status. Under `panic = "unwind"` (dev/test
5281 /// profile) the worker's
5282 /// `catch_unwind` arm calls `_exit(1)` so the parent sees
5283 /// `WIFEXITED=true`, `WEXITSTATUS=1`; under `panic = "abort"`
5284 /// (release profile) the worker aborts with `SIGABRT` so the parent
5285 /// sees `WIFEXITED=false`, `WTERMSIG=6`. Either way, a panicking
5286 /// worker never finishes `f.write_all(&bytes)` on the report pipe,
5287 /// so the collect poll + bounded read hands back an empty (or
5288 /// truncated) buffer, `postcard::from_bytes` fails, and the
5289 /// sentinel path fires. Partial writes from a panic between successful
5290 /// `write_all` and `_exit(0)` are not reachable — the write is the
5291 /// last non-trivial statement inside the catch_unwind closure.
5292 /// The `waitpid` call later in this function exists solely for
5293 /// reaping zombies; its return value feeds only the "still alive
5294 /// → SIGKILL escalate" branch and is never mapped to report
5295 /// state (the sentinel path DOES now read it to populate
5296 /// [`WorkerExitInfo`] on the attached diagnostic, but the
5297 /// correctness discrimination — sentinel vs real report — still
5298 /// happens purely on pipe payload presence).
5299 pub fn stop_and_collect(mut self) -> Vec<WorkerReport> {
5300 // Auto-start if not explicitly started (workers in parent cgroup)
5301 let was_started = self.started;
5302 self.start();
5303
5304 // Event-driven worker-started barrier for the auto-start path.
5305 //
5306 // ORDERING CONSTRAINT: this drain MUST run BEFORE the SIGUSR1
5307 // stop signals below. A fork worker signalled before it enters
5308 // its measurement loop reports 0 iterations and looks starved
5309 // when it was merely slow to start; waiting for each worker's
5310 // post-handshake ready byte guarantees stop lands after the
5311 // worker is counting. `drain_ready_bytes` documents the poll
5312 // mechanics and the POLLHUP-drop semantics.
5313 //
5314 // Thread-mode workers need no barrier here: `start()` above sent
5315 // on a `mpsc::sync_channel(0)` SyncSender(0) rendezvous, which
5316 // blocks the parent until the worker's matching `recv()` returns
5317 // — by the time `start()` returns every thread worker has
5318 // crossed its start handshake. Only the fork-mode pipe-based
5319 // start signal is fire-and-forget, so the barrier is gated on
5320 // `!self.children.is_empty()`.
5321 //
5322 // Budget: the 5 s deadline mirrors the collect deadline below;
5323 // worst case (no worker ever writes its byte) bounds the wait at
5324 // the same 5 s the legacy sleep + collect path budgeted for the
5325 // entire stop_and_collect.
5326 if !was_started && !self.children.is_empty() {
5327 let pending: Vec<i32> = self.children.iter().map(|c| c.report_fd).collect();
5328 drain_ready_bytes(pending, Instant::now() + Duration::from_secs(5));
5329 }
5330
5331 let mut reports = Vec::new();
5332 let children = std::mem::take(&mut self.children);
5333 let threads = std::mem::take(&mut self.threads);
5334
5335 // NOTE: the explicit-start (was_started) ready-byte drain used
5336 // to sit here, ahead of the stop signals, as a blocking 1-byte
5337 // read per child. That wedged teardown whenever a worker the
5338 // scheduler under test never ran had not yet written its byte
5339 // (guest PID-1 blocked forever → VM watchdog). It now runs
5340 // AFTER the signals below, via the bounded `drain_ready_bytes`.
5341
5342 // Signal all fork-mode children to stop via SIGUSR1; the
5343 // signal handler flips that child's process-local STOP, which
5344 // worker_main's `stop_requested` checks read. For pcomm
5345 // containers the SIGUSR1 flips the container's STOP, which
5346 // every thread inside the container observes via `&STOP`
5347 // passed to `worker_main` — one signal stops the whole
5348 // group cleanly. `pid` is `libc::pid_t`, so it flows to
5349 // `Pid::from_raw` without the u32→i32 sign-cast wraparound
5350 // that produced `kill(-1, ...)` session-wide reaps when the
5351 // old u32 pid exceeded i32::MAX.
5352 for child in &children {
5353 let _ = nix::sys::signal::kill(
5354 nix::unistd::Pid::from_raw(child.pid),
5355 nix::sys::signal::Signal::SIGUSR1,
5356 );
5357 }
5358 // Signal all thread-mode workers by flipping each worker's
5359 // per-task `stop`. SIGUSR1 is process-wide and useless for
5360 // per-thread stop; the Arc<AtomicBool> threaded through
5361 // worker_main is the only path that reaches an individual
5362 // thread without affecting siblings.
5363 for tw in &threads {
5364 tw.stop.store(true, Ordering::Relaxed);
5365 }
5366
5367 // Explicit-start (was_started) ready-byte drain. Signal-first
5368 // (above) so healthy workers begin tearing down during any drain
5369 // wait, and the drain is bounded (`drain_ready_bytes`, same 5 s
5370 // as the barrier) so a worker the scheduler under test never ran
5371 // — and thus never wrote its 'r' — cannot wedge teardown on a
5372 // blocking read. The stale byte must still be consumed: the
5373 // collect poll below routes on POLLIN, so a byte left in the
5374 // pipe would send a report-less worker down the payload-read
5375 // branch instead of the sentinel path.
5376 if was_started {
5377 let pending: Vec<i32> = children.iter().map(|c| c.report_fd).collect();
5378 drain_ready_bytes(pending, Instant::now() + Duration::from_secs(5));
5379 }
5380
5381 // Collect reports with a shared 5s deadline across all workers.
5382 // Each worker gets the remaining budget, so no-progress workers
5383 // (e.g. under degrade mode) don't serially exhaust the VM
5384 // timeout.
5385 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
5386
5387 // Bound on the post-SIGKILL worker reap (below). A worker normally
5388 // exits <<1s after SIGKILL (post-crash bypass keeps it
5389 // CFS-schedulable). The rare wedge: a sched_setaffinity flipper
5390 // stuck uninterruptible in `affine_move_task`'s `wait_for_completion`
5391 // — a migration that can't finish until the target becomes
5392 // migratable, which during the crash/bypass window can take seconds
5393 // — cannot take its pending SIGKILL until that syscall returns, so a
5394 // blocking `waitpid` would stall collect for that whole window.
5395 // The deadline is SHARED across workers: every such wedge waits out
5396 // the SAME crash/bypass window, so once one times out the rest get
5397 // the remaining (~0) budget and give up immediately rather than
5398 // summing to N×timeout. Mirrors the scheduler-side
5399 // `reap_child_bounded` (rust_init/scheduler.rs); on timeout the zombie is left
5400 // for VM reboot to reap (SIGKILL cannot make it exit faster — see
5401 // that fn's doc).
5402 const WORKER_REAP_TIMEOUT: Duration = Duration::from_secs(3);
5403 let mut worker_reap_deadline: Option<Instant> = None;
5404 // Evented, bounded reap of a SIGKILLed worker `pid`: wait up to
5405 // `timeout` for it to become a zombie (`pidfd_open` + `poll`),
5406 // then reap non-blocking. Returns false on timeout (worker still
5407 // uninterruptibly wedged — the rare case `WORKER_REAP_TIMEOUT`
5408 // documents; left for VM reboot).
5409 fn reap_pid_bounded(pid: libc::pid_t, timeout: Duration) -> bool {
5410 // Non-blocking reap of `pid` (observed StillAlive just above,
5411 // so this succeeds only once it has become a zombie).
5412 let reap = || {
5413 matches!(
5414 nix::sys::wait::waitpid(
5415 nix::unistd::Pid::from_raw(pid),
5416 Some(nix::sys::wait::WaitPidFlag::WNOHANG),
5417 ),
5418 Ok(nix::sys::wait::WaitStatus::Exited(..))
5419 | Ok(nix::sys::wait::WaitStatus::Signaled(..))
5420 )
5421 };
5422 match crate::sync::pidfd_poll_exited(pid, timeout) {
5423 // Readable => zombie => the WNOHANG reap is non-blocking.
5424 crate::sync::PidfdWait::Exited => {
5425 let _ = reap();
5426 true
5427 }
5428 // Still uninterruptibly wedged (see `WORKER_REAP_TIMEOUT`):
5429 // leave the zombie-to-be for VM reboot (SIGKILL already
5430 // pending).
5431 crate::sync::PidfdWait::TimedOut => false,
5432 // pidfd_open failed (ESRCH/gone): one non-blocking reap.
5433 crate::sync::PidfdWait::NoPidfd => reap(),
5434 }
5435 }
5436
5437 for child in children {
5438 let mut buf = Vec::new();
5439 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
5440 let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5441 let mut poll_ready = false;
5442 if ms > 0 {
5443 let mut pfd = libc::pollfd {
5444 fd: child.report_fd,
5445 events: libc::POLLIN,
5446 revents: 0,
5447 };
5448 let ready = unsafe { libc::poll(&mut pfd, 1, ms) };
5449 poll_ready = ready > 0;
5450 }
5451
5452 let npid = nix::unistd::Pid::from_raw(child.pid);
5453 if !poll_ready {
5454 // Deadline expired without POLLIN — the child produced
5455 // no report, so `buf` stays empty (→ sentinel path).
5456 // Kill first, then close, so no writer is left blocked
5457 // on a pipe the parent has stopped draining (the child
5458 // may have written only the 'r' ready byte but no
5459 // payload).
5460 kill_and_killpg(npid, nix::sys::signal::Signal::SIGKILL);
5461 let _ = nix::unistd::close(child.report_fd);
5462 } else {
5463 // Child responded (POLLIN) — read the payload bounded by
5464 // the SHARED collect `deadline`, then close and kill. A
5465 // worker that wrote its 'r' (or a partial payload) and
5466 // then wedged before EOF would otherwise hold a blocking
5467 // `read_to_end` hostage; polling to the deadline and
5468 // giving up yields a short/empty buf, so the postcard /
5469 // serde decode fails and the existing sentinel path
5470 // fires. Preserve read→close→kill order (the old
5471 // File-drop close is now an explicit `close`).
5472 let mut chunk = [0u8; 64 * 1024];
5473 loop {
5474 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
5475 // ms == 0 is fine: a zero-timeout poll still drains
5476 // data already buffered in the pipe without
5477 // blocking, which keeps the healthy at-deadline-edge
5478 // case intact (the child exited; payload + EOF are
5479 // buffered and read out in one pass here).
5480 let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5481 let mut pfd = libc::pollfd {
5482 fd: child.report_fd,
5483 events: libc::POLLIN,
5484 revents: 0,
5485 };
5486 // SAFETY: single owned pollfd; nfds == 1; ms >= 0.
5487 let ret = unsafe { libc::poll(&mut pfd, 1, ms) };
5488 if ret < 0 {
5489 if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted
5490 {
5491 continue;
5492 }
5493 break;
5494 }
5495 if ret == 0 {
5496 // Timeout — deadline reached with no more data.
5497 break;
5498 }
5499 // SAFETY: `chunk` is a valid owned buffer of
5500 // `chunk.len()` bytes; `report_fd` is the read end
5501 // the parent owns until the close below.
5502 let n = unsafe {
5503 libc::read(
5504 child.report_fd,
5505 chunk.as_mut_ptr() as *mut libc::c_void,
5506 chunk.len(),
5507 )
5508 };
5509 if n > 0 {
5510 buf.extend_from_slice(&chunk[..n as usize]);
5511 continue;
5512 }
5513 if n == 0 {
5514 // EOF — writer closed; payload complete.
5515 break;
5516 }
5517 // n < 0: retry on EINTR, otherwise give up.
5518 if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted {
5519 continue;
5520 }
5521 break;
5522 }
5523 let _ = nix::unistd::close(child.report_fd);
5524 kill_and_killpg(npid, nix::sys::signal::Signal::SIGKILL);
5525 }
5526 let waited = nix::sys::wait::waitpid(npid, Some(nix::sys::wait::WaitPidFlag::WNOHANG));
5527 let still_running = matches!(waited, Ok(nix::sys::wait::WaitStatus::StillAlive));
5528 let exit_info_source: Result<nix::sys::wait::WaitStatus, nix::errno::Errno> =
5529 if still_running {
5530 // Bounded reap (see `reap_pid_bounded` + the shared
5531 // `worker_reap_deadline` above): replaces a blocking
5532 // `waitpid(npid, None)` that stalled on a worker
5533 // uninterruptibly wedged in an affine_move_task
5534 // migration (seconds; see `WORKER_REAP_TIMEOUT`).
5535 // exit_info stays `StillAlive` either way — the prior
5536 // blocking reap also discarded the status here.
5537 let rd = *worker_reap_deadline
5538 .get_or_insert_with(|| Instant::now() + WORKER_REAP_TIMEOUT);
5539 let remaining = rd.saturating_duration_since(Instant::now());
5540 reap_pid_bounded(child.pid, remaining);
5541 Ok(nix::sys::wait::WaitStatus::StillAlive)
5542 } else {
5543 waited
5544 };
5545
5546 // Strip a leading `b'r'` ready byte if it is still present.
5547 // Children write this byte unconditionally on every success
5548 // path right after the start handshake; both collect paths
5549 // drain it up front now (`drain_ready_bytes`, called from
5550 // the auto-start barrier and the explicit-start drain). This
5551 // strip is the defensive fallback for the worker that wrote
5552 // its byte only after the bounded drain gave up at its
5553 // deadline — the byte then sits in the pipe ahead of the
5554 // payload here. Strip exactly 1 byte if present so the
5555 // per-kind decoder sees a clean payload either way.
5556 let report_slice: &[u8] = if buf.first() == Some(&b'r') {
5557 &buf[1..]
5558 } else {
5559 &buf[..]
5560 };
5561
5562 // Per-kind decoding. `Worker` carries a single postcard
5563 // `WorkerReport`. `PcommContainer` carries
5564 // a `serde_json` `Vec<WorkerReport>` — serde_json for
5565 // the pcomm container; fork-mode workers use postcard
5566 // for the single `WorkerReport`. Both
5567 // payloads ride the EOF-terminated pipe with no length
5568 // prefix; the parent's `read_to_end` provides framing.
5569 // On a decode failure, emit one sentinel per expected
5570 // report so downstream consumers (per-group filtering,
5571 // assert_not_stuck) see the correct cardinality.
5572 match child.kind {
5573 ForkedChildKind::Worker { group_idx } => {
5574 let decoded: Result<WorkerReport, _> = postcard::from_bytes(report_slice);
5575 if let Ok(report) = decoded {
5576 reports.push(report);
5577 } else {
5578 let exit_info = classify_wait_outcome(exit_info_source);
5579 eprintln!(
5580 "ktstr: worker pid={} returned no report ({} bytes read, exit={exit_info:?})",
5581 child.pid,
5582 buf.len(),
5583 );
5584 reports.push(WorkerReport {
5585 tid: child.pid,
5586 group_idx,
5587 exit_info: Some(exit_info),
5588 ..WorkerReport::default()
5589 });
5590 }
5591 }
5592 ForkedChildKind::PcommContainer { groups } => {
5593 let total_workers: usize = groups.iter().map(|(_, n)| n).sum();
5594 let decoded: Result<Vec<WorkerReport>, _> =
5595 serde_json::from_slice(report_slice);
5596 match decoded {
5597 Ok(mut decoded_reports) if decoded_reports.len() == total_workers => {
5598 reports.append(&mut decoded_reports);
5599 }
5600 Ok(mut decoded_reports) => {
5601 // Cardinality mismatch — surface the
5602 // partial set we did get and pad with
5603 // sentinels so the total report count
5604 // still equals `total_workers`. A short
5605 // payload typically signals the
5606 // thread-group leader died mid-encode
5607 // (panic in `serde_json::to_vec`, OOM
5608 // during Vec growth, or a write_all
5609 // truncated by SIGKILL escalation).
5610 // Sentinels are tagged with the right
5611 // per-group `group_idx` using the
5612 // `groups` layout: groups[0] consumed
5613 // the first groups[0].1 slots,
5614 // groups[1] the next groups[1].1, etc.
5615 // We emit sentinels for the trailing
5616 // missing slots, computing each
5617 // sentinel's group_idx by walking the
5618 // layout from `got` forward.
5619 let exit_info = classify_wait_outcome(exit_info_source);
5620 eprintln!(
5621 "ktstr: pcomm thread-group leader pid={} returned {} of {} reports ({} bytes read, exit={exit_info:?})",
5622 child.pid,
5623 decoded_reports.len(),
5624 total_workers,
5625 buf.len(),
5626 );
5627 // Surplus reports (decoded > total_workers)
5628 // must not leak into the parent's report
5629 // stream — downstream cardinality assertions
5630 // (per-group filtering, assert_not_stuck)
5631 // assume exactly `total_workers` entries.
5632 // Truncate to the layout's total before the
5633 // `got..total_workers` loop runs (which is
5634 // a no-op for the surplus case after this).
5635 decoded_reports.truncate(total_workers);
5636 let got = decoded_reports.len();
5637 for r in decoded_reports {
5638 reports.push(r);
5639 }
5640 // Compute group_idx for each missing
5641 // slot by walking the layout. `slot` is
5642 // a 0-based global index from `got` to
5643 // `total_workers - 1`; we accumulate
5644 // per-group counts to find which group
5645 // owns each slot.
5646 for slot in got..total_workers {
5647 let mut acc = 0usize;
5648 let mut g_idx = 0usize;
5649 for &(gi, n) in &groups {
5650 if slot < acc + n {
5651 g_idx = gi;
5652 break;
5653 }
5654 acc += n;
5655 }
5656 reports.push(WorkerReport {
5657 tid: child.pid,
5658 group_idx: g_idx,
5659 exit_info: Some(exit_info.clone()),
5660 ..WorkerReport::default()
5661 });
5662 }
5663 }
5664 Err(_) => {
5665 let exit_info = classify_wait_outcome(exit_info_source);
5666 eprintln!(
5667 "ktstr: pcomm thread-group leader pid={} returned no decodable report ({} bytes read, exit={exit_info:?})",
5668 child.pid,
5669 buf.len(),
5670 );
5671 // Total decode failure — emit one
5672 // sentinel per slot, tagging each with
5673 // the correct group_idx.
5674 for &(gi, n) in &groups {
5675 for _ in 0..n {
5676 reports.push(WorkerReport {
5677 tid: child.pid,
5678 group_idx: gi,
5679 exit_info: Some(exit_info.clone()),
5680 ..WorkerReport::default()
5681 });
5682 }
5683 }
5684 }
5685 }
5686 }
5687 }
5688 }
5689
5690 // Thread-mode collection: join each worker's JoinHandle
5691 // (with the [`THREAD_JOIN_TIMEOUT`] budget) and adopt the
5692 // returned [`WorkerReport`]. Per-worker `stop` was flipped
5693 // above; the worker observes it in worker_main's
5694 // `stop.load(Relaxed)` checks (max ~100ms latency from the
5695 // FUTEX_WAIT_TIMEOUT poll cadence). Three outcomes:
5696 //
5697 // 1. Ok(report): join returned the worker's WorkerReport.
5698 // Push as-is.
5699 // 2. Err(payload): the thread panicked. Build a sentinel
5700 // report and attach
5701 // `exit_info: Some(WorkerExitInfo::Panicked(msg))`
5702 // where `msg` comes from `extract_panic_payload`.
5703 // 3. Timeout (5s elapsed without is_finished): emit a
5704 // tracing::warn and push a sentinel with
5705 // `exit_info: Some(WorkerExitInfo::TimedOut)` —
5706 // `worker_main` should have observed the per-worker
5707 // `stop` within 100ms, so a 5s no-show signals a
5708 // genuinely stuck worker (deadlock, infinite spin,
5709 // blocking syscall the runtime can't interrupt).
5710 // stop_and_collect does NOT process::exit on timeout —
5711 // the orphan thread keeps running until the test
5712 // harness exits, but any subsequent worker uses a
5713 // fresh per-worker `stop` so the orphan can't pollute
5714 // later runs.
5715 for mut tw in threads {
5716 // Drop start_tx (idempotent — `start()` may have already
5717 // taken it). If start() ran first, `start_tx` is
5718 // already `None` and the take is a no-op; if the caller
5719 // skipped start() entirely, dropping start_tx here
5720 // signals the worker via `Disconnected` so it exits
5721 // cleanly without the rendezvous send.
5722 tw.start_tx.take();
5723 let tid = tw.tid.load(Ordering::Acquire);
5724 if let Some(j) = tw.join.take() {
5725 match join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT) {
5726 Some(Ok(report)) => reports.push(report),
5727 Some(Err(payload)) => {
5728 let msg = extract_panic_payload(payload);
5729 eprintln!("ktstr: thread worker tid={tid} panicked: {msg}");
5730 reports.push(WorkerReport {
5731 tid,
5732 completed: false,
5733 exit_info: Some(WorkerExitInfo::Panicked(msg)),
5734 ..WorkerReport::default()
5735 });
5736 }
5737 None => {
5738 tracing::warn!(
5739 tid,
5740 timeout_secs = THREAD_JOIN_TIMEOUT.as_secs(),
5741 "thread worker did not join within timeout — leaking the \
5742 thread; sentinel report attached with TimedOut exit_info"
5743 );
5744 reports.push(WorkerReport {
5745 tid,
5746 completed: false,
5747 exit_info: Some(WorkerExitInfo::TimedOut),
5748 ..WorkerReport::default()
5749 });
5750 }
5751 }
5752 }
5753 }
5754
5755 reports
5756 }
5757
5758 /// Deliver the terminal stop to every live worker without
5759 /// reaping. Signal-only: NO `waitpid`, NO fd close, NO consume of
5760 /// `self`, no blocking of any kind. Reaping, the fd/region
5761 /// cleanup, and (for `stop_and_collect`) report decoding all
5762 /// belong to the subsequent [`Self::stop_and_collect`] or
5763 /// [`Drop`] — doing any of them here would double-reap. This only
5764 /// pre-delivers the terminal signal so that reap finds workers
5765 /// already exiting.
5766 ///
5767 /// # Why (fork mode — the case this exists for)
5768 ///
5769 /// When the sched_ext scheduler under test errors out, the kernel
5770 /// disables sched_ext and its tasks fall back to the builtin
5771 /// scheduler, where CPU-bound workers keep spinning under
5772 /// contention. The normal reap in `stop_and_collect` SIGKILLs one
5773 /// worker then *blocks* in `waitpid` for it while the rest have
5774 /// had only the cooperative SIGUSR1 and keep spinning, so each
5775 /// dying worker must win a CPU slice against every still-runnable
5776 /// sibling before it can run its exit path — the serial reap
5777 /// becomes scheduling-gated and teardown time scales with worker
5778 /// count. `kill_and_killpg(SIGKILL)` here is delivered, not
5779 /// cooperative: once every worker carries a pending SIGKILL none
5780 /// re-enters userspace, so no worker competes for the CPU while
5781 /// the reap drains them, and each killed worker's report-pipe
5782 /// write end closes so the reap's `poll` returns `POLLHUP`
5783 /// immediately instead of waiting out its deadline. One
5784 /// `kill_and_killpg` per `ForkedChild` covers both a plain worker
5785 /// and a pcomm container: SIGKILL on any thread is fatal to the
5786 /// whole thread group, and `killpg` additionally reaches
5787 /// descendants — every forked child is its own process-group
5788 /// leader via `setpgid(0, 0)`, so pgid == child pid (mirrors the
5789 /// `killpg` in [`Drop`]).
5790 ///
5791 /// # Thread mode
5792 ///
5793 /// Thread-mode workers share the harness tgid and cannot be
5794 /// signalled individually, so there is NO SIGKILL fast path for
5795 /// them. This flips each worker's cooperative `stop` flag — the
5796 /// same store `stop_and_collect` and [`Drop`] already do, just
5797 /// earlier — and the worker still only observes it on its next
5798 /// futex-wait wake (`WORKER_STOP_POLL_NS`, ~100 ms). It is
5799 /// therefore NOT a teardown speedup for thread mode; it is kept
5800 /// for symmetry and is cheap and harmless (an idempotent relaxed
5801 /// store). The win is fork-mode only, which is where the
5802 /// SIGKILL-vs-cooperative distinction exists.
5803 ///
5804 /// # Frozen cgroups
5805 ///
5806 /// A cgroup-frozen worker is not a special case here. The freezer
5807 /// parks a userspace task in `TASK_INTERRUPTIBLE` (the jobctl
5808 /// freezer trap), so a fatal signal wakes it and it exits on its
5809 /// next `get_signal` — SIGKILL kills a frozen worker with no
5810 /// unfreeze required. (`collect_step` / `collect_backdrop` still
5811 /// unfreeze each cgroup as their first teardown step for reasons of
5812 /// their own; this method does not depend on that.)
5813 pub(crate) fn sigkill_workers(&self) {
5814 for child in &self.children {
5815 kill_and_killpg(
5816 nix::unistd::Pid::from_raw(child.pid),
5817 nix::sys::signal::Signal::SIGKILL,
5818 );
5819 }
5820 for tw in &self.threads {
5821 tw.stop.store(true, Ordering::Relaxed);
5822 }
5823 }
5824}
5825
5826/// Bounded drain of the per-worker "ready byte". Each fork worker
5827/// writes a single `b'r'` byte to its report pipe immediately after
5828/// the start-pipe handshake completes (see the matching write inside
5829/// `worker_main`'s catch_unwind closure). This polls every still-
5830/// pending fd for `POLLIN` with the remaining budget until `deadline`,
5831/// consuming exactly one byte per worker, and returns once every fd
5832/// has produced its byte / hung up or the deadline expires.
5833///
5834/// The raw `libc::read` does NOT take ownership of any fd — the caller
5835/// still owns them via the `children` Vec and reads the payload tail on
5836/// its own deadline.
5837///
5838/// Per-fd outcomes:
5839/// - `POLLIN`, `read` == 1: ready byte consumed; drop from pending.
5840/// - `POLLIN`, `read` == 0: EOF — writer closed without sending the
5841/// byte (worker died pre-write). Drop; the collect path reads 0
5842/// bytes and the sentinel-report branch fires.
5843/// - `POLLIN`, `read` < 0, EINTR: keep pending, re-poll next cycle.
5844/// - `POLLIN`, `read` < 0, other: warn and drop (treat as ready).
5845/// - `POLLHUP` / `POLLERR` / `POLLNVAL`: worker closed the write end
5846/// without sending the byte (panic during post-fork init, or
5847/// kernel-killed before the write). Drop; sentinel path fires.
5848/// - EINTR on `poll`: re-poll with the remaining budget.
5849/// - hard `poll` failure (EFAULT impossible with an owned Vec;
5850/// ENOMEM extreme): warn and bail; the collect path's per-fd poll
5851/// handles each worker individually.
5852///
5853/// Bounding is the point: a worker the scheduler under test never ran
5854/// never writes its byte, so a blocking read here would wedge teardown
5855/// (guest PID-1 blocks forever → host watchdog). The deadline lets a
5856/// never-scheduled worker fall through to the collect sentinel path
5857/// instead.
5858fn drain_ready_bytes(mut pending: Vec<i32>, deadline: Instant) {
5859 while !pending.is_empty() {
5860 let remaining = deadline.saturating_duration_since(Instant::now());
5861 if remaining.is_zero() {
5862 // Deadline expired with workers still pending. Stop
5863 // waiting; the collect path treats any worker whose pipe
5864 // never produced data as a sentinel report.
5865 break;
5866 }
5867 let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5868 let mut pfds: Vec<libc::pollfd> = pending
5869 .iter()
5870 .map(|&fd| libc::pollfd {
5871 fd,
5872 events: libc::POLLIN,
5873 revents: 0,
5874 })
5875 .collect();
5876 // SAFETY: `pfds` is a non-empty owned Vec; nfds matches its
5877 // length; `ms >= 0` since the remaining-zero branch bails
5878 // first. Return codes are interpreted below.
5879 let ret = unsafe { libc::poll(pfds.as_mut_ptr(), pfds.len() as libc::nfds_t, ms) };
5880 if ret < 0 {
5881 let err = std::io::Error::last_os_error();
5882 if err.kind() == std::io::ErrorKind::Interrupted {
5883 // EINTR — re-poll with the remaining budget. The
5884 // deadline guard bounds total wait regardless of EINTR
5885 // frequency.
5886 continue;
5887 }
5888 tracing::warn!(
5889 %err,
5890 pending = pending.len(),
5891 "drain_ready_bytes: poll failed; falling through to per-worker collect"
5892 );
5893 break;
5894 }
5895 // ret == 0 means the per-iteration timeout fired without any fd
5896 // ready — measured against `remaining`, so the next iteration's
5897 // saturating_duration_since is zero and the top-of-loop guard
5898 // exits. Don't break here: a future cycle stays useful if the
5899 // clock jumped backward, at the cost of one bail iteration.
5900 if ret > 0 {
5901 pending.retain(|&fd| {
5902 // Find this fd's pollfd entry. Linear scan is fine:
5903 // typical worker counts are <100 and a HashMap costs
5904 // more in setup than the scan saves.
5905 let pfd = pfds.iter().find(|p| p.fd == fd);
5906 let revents = pfd.map(|p| p.revents).unwrap_or(0);
5907 if revents & libc::POLLIN != 0 {
5908 let mut byte: u8 = 0;
5909 // SAFETY: `&mut byte` is a valid 1-byte buffer; `fd`
5910 // is the report read end the caller owns until
5911 // collect drains it. 0 / -1 returns handled below.
5912 let n = unsafe { libc::read(fd, &mut byte as *mut u8 as *mut libc::c_void, 1) };
5913 if n >= 0 {
5914 // n == 1: byte consumed. n == 0: EOF pre-write.
5915 // Either way drop from pending.
5916 return false;
5917 }
5918 let err = std::io::Error::last_os_error();
5919 if err.kind() == std::io::ErrorKind::Interrupted {
5920 return true;
5921 }
5922 tracing::warn!(
5923 %err,
5924 fd,
5925 "drain_ready_bytes: byte read failed; treating worker as ready"
5926 );
5927 return false;
5928 }
5929 if revents & (libc::POLLHUP | libc::POLLERR | libc::POLLNVAL) != 0 {
5930 // Worker closed the write end without sending the
5931 // byte. Drop; the sentinel-report branch fires.
5932 return false;
5933 }
5934 // No POLLIN, no hangup — keep waiting.
5935 true
5936 });
5937 }
5938 }
5939}
5940
5941impl Drop for WorkloadHandle {
5942 fn drop(&mut self) {
5943 use nix::sys::signal::{Signal, kill};
5944 use nix::sys::wait::waitpid;
5945 use nix::unistd::{Pid, close};
5946
5947 // Forked children (conventional workers AND pcomm
5948 // containers). `pid` is `libc::pid_t` — stored as i32 so
5949 // `Pid::from_raw` receives the kernel's native representation
5950 // directly, not the sign-cast of a u32 that could alias
5951 // negative values (including -1, i.e. every process in the
5952 // session).
5953 for child in &self.children {
5954 let pid = child.pid;
5955 let nix_pid = Pid::from_raw(pid);
5956 // killpg first: reach descendants the child may have
5957 // forked (Custom workloads, ForkExit caught mid-fork).
5958 // pgid == child pid because every forked child (worker
5959 // or pcomm container) calls `setpgid(0, 0)` at fork time.
5960 // ESRCH (group gone / no members) is expected and not a
5961 // warning-worthy failure; swallow it to keep the log
5962 // clean when the common no-descendants case drops.
5963 if let Err(e) = nix::sys::signal::killpg(nix_pid, Signal::SIGKILL)
5964 && e != nix::errno::Errno::ESRCH
5965 {
5966 tracing::warn!(pid, %e, "killpg failed in WorkloadHandle::drop");
5967 }
5968 if let Err(e) = kill(nix_pid, Signal::SIGKILL) {
5969 tracing::warn!(pid, %e, "kill failed in WorkloadHandle::drop");
5970 }
5971 if let Err(e) = waitpid(nix_pid, None) {
5972 tracing::warn!(pid, %e, "waitpid failed in WorkloadHandle::drop");
5973 }
5974 for fd in [child.report_fd, child.start_fd] {
5975 if fd >= 0
5976 && let Err(e) = close(fd)
5977 {
5978 tracing::warn!(fd, %e, "close failed in WorkloadHandle::drop");
5979 }
5980 }
5981 }
5982 // Thread-mode workers: flip stop, drop start_tx (in case
5983 // worker hasn't yet recv'd), join with the same 5s budget
5984 // `stop_and_collect` uses. Threads share the parent's
5985 // address space — there is no `kill` equivalent and no
5986 // MAP_SHARED ownership to give back. Drop still applies
5987 // the timeout so a stuck worker doesn't pin
5988 // `WorkloadHandle::drop` indefinitely; on timeout we log
5989 // the leak via `tracing::warn!` and proceed.
5990 let threads = std::mem::take(&mut self.threads);
5991 for mut tw in threads {
5992 tw.stop.store(true, Ordering::Relaxed);
5993 tw.start_tx.take();
5994 if let Some(j) = tw.join.take() {
5995 let tid = tw.tid.load(Ordering::Acquire);
5996 match join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT) {
5997 Some(Ok(_)) => {}
5998 Some(Err(e)) => {
5999 let payload = extract_panic_payload(e);
6000 tracing::warn!(
6001 tid,
6002 payload,
6003 "thread worker panicked in WorkloadHandle::drop"
6004 );
6005 }
6006 None => {
6007 tracing::warn!(
6008 tid,
6009 timeout_secs = THREAD_JOIN_TIMEOUT.as_secs(),
6010 "thread worker failed to join within timeout in \
6011 WorkloadHandle::drop — leaking the thread"
6012 );
6013 }
6014 }
6015 }
6016 }
6017 // Close inter-worker pipe pairs and chain pipes AFTER worker
6018 // shutdown. Ordering matters for Thread mode: every worker
6019 // thread shares the parent's fd table, so closing a pipe fd
6020 // before its using thread joins would surface to that thread
6021 // as `EBADF` on the next read/write/poll syscall. The
6022 // children-reap loop above and the threads-join loop above
6023 // both block until their worker is reaped or joined; only
6024 // then do these closes run, which is when the workers are
6025 // guaranteed to no longer touch their fds. Fork mode is
6026 // unaffected either way: each child held its own fd-table
6027 // copy via `fork()`, so this close is a no-op for the
6028 // child's view (its own copy was closed by the post-fork
6029 // close-other-fds block in spawn_group).
6030 //
6031 // Errors from `close` are logged via `tracing::warn!` rather
6032 // than swallowed — `EBADF` here would indicate a double-close
6033 // (an aliased ownership bug) and is more diagnostic than the
6034 // SpawnGuard early-bail path's silent close. SpawnGuard's
6035 // Drop swallows EBADF deliberately because mid-spawn the
6036 // guard may share fd ownership with already-closed
6037 // half-allocated state; the handle on the other hand has
6038 // sole ownership at this point.
6039 for (ab, ba) in &self.pipe_pairs {
6040 for fd in [ab[0], ab[1], ba[0], ba[1]] {
6041 if let Err(e) = close(fd) {
6042 tracing::warn!(fd, %e, "close failed for pipe_pair fd in WorkloadHandle::drop");
6043 }
6044 }
6045 }
6046 for chain in &self.chain_pipes {
6047 for pipe in chain {
6048 for fd in [pipe[0], pipe[1]] {
6049 if let Err(e) = close(fd) {
6050 tracing::warn!(fd, %e, "close failed for chain_pipe fd in WorkloadHandle::drop");
6051 }
6052 }
6053 }
6054 }
6055 for (&ptr, &size) in self.futex_ptrs.iter().zip(self.futex_region_sizes.iter()) {
6056 unsafe {
6057 libc::munmap(ptr as *mut libc::c_void, size);
6058 }
6059 }
6060 if !self.iter_counters.is_null() && self.iter_counter_len > 0 {
6061 unsafe {
6062 libc::munmap(
6063 self.iter_counters as *mut libc::c_void,
6064 self.iter_counter_len * std::mem::size_of::<u64>(),
6065 );
6066 }
6067 }
6068 if !self.phase_epoch.is_null() && self.phase_epoch_bytes > 0 {
6069 unsafe {
6070 libc::munmap(
6071 self.phase_epoch as *mut libc::c_void,
6072 self.phase_epoch_bytes,
6073 );
6074 }
6075 }
6076 }
6077}
6078
6079/// SIGUSR1 handler installed in the fork-mode child post-fork. Flips
6080/// the per-process global [`STOP`] so `worker_main`'s outer loop
6081/// exits at the next `stop_requested` check.
6082pub(super) extern "C" fn sigusr1_handler(_: libc::c_int) {
6083 STOP.store(true, Ordering::Relaxed);
6084}
6085
6086/// Render an actionable hint for a failed
6087/// `mmap(MAP_SHARED | MAP_ANONYMOUS)` call based on the observed
6088/// `errno`. Shared between the futex-region mmap and the
6089/// iter_counters mmap in [`WorkloadHandle::spawn`] so the two
6090/// sites emit identical hint text per errno — a drift would mean
6091/// two related failures produce inconsistent remediation advice.
6092///
6093/// Takes `Option<i32>` (the output of `std::io::Error::raw_os_error`)
6094/// so an unrecognised errno folds cleanly through the `_ => ""`
6095/// arm without forcing callers to `unwrap`.
6096///
6097/// The leading space on every non-empty arm lets callers format
6098/// as `"...failed: {errno}{hint};"` without having to add a
6099/// conditional separator — an empty hint disappears cleanly.
6100pub(super) fn mmap_shared_anon_errno_hint(errno: Option<i32>) -> &'static str {
6101 match errno {
6102 Some(libc::ENOMEM) => {
6103 " (ENOMEM: host is out of memory \
6104 or /proc/sys/vm/max_map_count is too low — \
6105 check `sysctl vm.max_map_count` and `free -h`)"
6106 }
6107 Some(libc::EPERM) => {
6108 " (EPERM: MAP_SHARED|MAP_ANONYMOUS \
6109 rejected by the kernel — check memory cgroup \
6110 limits and container seccomp policy)"
6111 }
6112 Some(libc::EINVAL) => {
6113 " (EINVAL: invalid length or \
6114 flag combination — verify num_workers > 0 so the \
6115 region size is non-zero, and that the total size \
6116 does not overflow usize)"
6117 }
6118 _ => "",
6119 }
6120}
6121
6122#[cfg(test)]
6123mod testing;
6124#[cfg(test)]
6125mod tests_composed;
6126#[cfg(test)]
6127mod tests_fan_out;
6128#[cfg(test)]
6129mod tests_futex;
6130#[cfg(test)]
6131mod tests_grandchild;
6132#[cfg(test)]
6133mod tests_idle_churn;
6134#[cfg(test)]
6135mod tests_integration;
6136#[cfg(test)]
6137mod tests_lifecycle;
6138#[cfg(test)]
6139mod tests_mempolicy;
6140#[cfg(test)]
6141mod tests_misc;
6142#[cfg(test)]
6143mod tests_pcomm;
6144#[cfg(test)]
6145mod tests_sched_policy;
6146#[cfg(test)]
6147mod tests_spawn_guard;
6148#[cfg(test)]
6149mod tests_thread_mode;
6150#[cfg(test)]
6151mod tests_wake_chain;
6152
6153/// Direct tests for the bounded ready-byte drain that fix 1a extracted
6154/// from the auto-start barrier. A full fake-child `stop_and_collect`
6155/// test would need a hand-rolled fork worker that never writes its
6156/// byte; exercising `drain_ready_bytes` against raw pipe fds covers the
6157/// three outcomes that matter — never-writes (deadline bound), one byte
6158/// consumed exactly, and POLLHUP drop — without that scaffolding.
6159#[cfg(test)]
6160mod tests_drain_bound {
6161 use super::drain_ready_bytes;
6162 use std::time::{Duration, Instant};
6163
6164 // Blocking pipe; returns (read_fd, write_fd). Both fds must be
6165 // closed by the caller.
6166 fn make_pipe() -> (i32, i32) {
6167 let mut fds = [0i32; 2];
6168 // SAFETY: `fds` is a valid 2-element buffer.
6169 let rc = unsafe { libc::pipe(fds.as_mut_ptr()) };
6170 assert_eq!(rc, 0, "pipe() failed: {}", std::io::Error::last_os_error());
6171 (fds[0], fds[1])
6172 }
6173
6174 // A worker the scheduler never ran leaves its write end open with
6175 // no data: the drain must block until the deadline (never wedge on
6176 // the blocking read the old code used) and then return.
6177 #[test]
6178 fn writer_never_writes_returns_at_deadline() {
6179 let (r, w) = make_pipe();
6180 let budget = Duration::from_millis(300);
6181 let start = Instant::now();
6182 drain_ready_bytes(vec![r], Instant::now() + budget);
6183 let elapsed = start.elapsed();
6184 assert!(
6185 elapsed >= Duration::from_millis(250),
6186 "returned before the deadline: {elapsed:?}"
6187 );
6188 assert!(
6189 elapsed < Duration::from_secs(3),
6190 "exceeded the deadline bound: {elapsed:?}"
6191 );
6192 // SAFETY: both fds are still open and owned here.
6193 unsafe {
6194 libc::close(r);
6195 libc::close(w);
6196 }
6197 }
6198
6199 // The ready byte is consumed exactly once: a following payload byte
6200 // must remain in the pipe for the collect read.
6201 #[test]
6202 fn ready_byte_consumed_exactly_one() {
6203 let (r, w) = make_pipe();
6204 let payload = b"rX";
6205 // SAFETY: valid buffer/len; `w` is open.
6206 let n = unsafe { libc::write(w, payload.as_ptr() as *const libc::c_void, payload.len()) };
6207 assert_eq!(n, 2, "short write");
6208 let start = Instant::now();
6209 drain_ready_bytes(vec![r], Instant::now() + Duration::from_secs(5));
6210 assert!(
6211 start.elapsed() < Duration::from_secs(1),
6212 "did not return promptly after the byte arrived: {:?}",
6213 start.elapsed()
6214 );
6215 let mut buf = [0u8; 8];
6216 // SAFETY: valid buffer/len; `r` is open.
6217 let got = unsafe { libc::read(r, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) };
6218 assert_eq!(got, 1, "exactly one byte should remain after the drain");
6219 assert_eq!(buf[0], b'X', "the wrong byte was consumed");
6220 // SAFETY: both fds still open and owned.
6221 unsafe {
6222 libc::close(r);
6223 libc::close(w);
6224 }
6225 }
6226
6227 // Writer closes without sending the byte: POLLHUP must drop the fd
6228 // immediately rather than waiting out the deadline.
6229 #[test]
6230 fn closed_writer_dropped_via_pollhup() {
6231 let (r, w) = make_pipe();
6232 // SAFETY: `w` is open; closing it makes `r` report POLLHUP.
6233 unsafe {
6234 libc::close(w);
6235 }
6236 let start = Instant::now();
6237 drain_ready_bytes(vec![r], Instant::now() + Duration::from_secs(5));
6238 assert!(
6239 start.elapsed() < Duration::from_secs(1),
6240 "did not drop on POLLHUP: {:?}",
6241 start.elapsed()
6242 );
6243 // SAFETY: `r` still open and owned.
6244 unsafe {
6245 libc::close(r);
6246 }
6247 }
6248}
6249
6250/// Direct tests for `poll_eventfd_reaches` (the evented first-iteration
6251/// wait's core) against a raw counter-mode eventfd — no worker spawn
6252/// needed. Covers the two contract halves the verifier probe relies on:
6253/// fewer than `n` signals times out at the deadline; exactly `n` (summed
6254/// across writes) wakes early.
6255#[cfg(test)]
6256mod tests_poll_eventfd {
6257 use super::poll_eventfd_reaches;
6258 use std::time::{Duration, Instant};
6259
6260 // Counter-mode eventfd (EFD_NONBLOCK so a drained read cannot block).
6261 fn make_eventfd() -> i32 {
6262 let fd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
6263 assert!(
6264 fd >= 0,
6265 "eventfd() failed: {}",
6266 std::io::Error::last_os_error()
6267 );
6268 fd
6269 }
6270
6271 fn signal(fd: i32, times: u64) {
6272 for _ in 0..times {
6273 let one: u64 = 1;
6274 let n = unsafe { libc::write(fd, &one as *const u64 as *const libc::c_void, 8) };
6275 assert_eq!(
6276 n,
6277 8,
6278 "eventfd write short: {}",
6279 std::io::Error::last_os_error()
6280 );
6281 }
6282 }
6283
6284 /// Fewer than `n` signals: the wait must time out at the deadline and
6285 /// return `false` (a never-run worker leaves the count short).
6286 #[test]
6287 fn under_count_times_out_false() {
6288 let fd = make_eventfd();
6289 signal(fd, 2); // k = 2 < n = 4
6290 let start = Instant::now();
6291 let ok = poll_eventfd_reaches(fd, Instant::now() + Duration::from_millis(200), 4);
6292 let elapsed = start.elapsed();
6293 assert!(!ok, "under-count must not report all-signalled");
6294 assert!(
6295 elapsed >= Duration::from_millis(150),
6296 "returned before the deadline: {elapsed:?}",
6297 );
6298 assert!(
6299 elapsed < Duration::from_secs(3),
6300 "exceeded the deadline bound: {elapsed:?}"
6301 );
6302 unsafe {
6303 libc::close(fd);
6304 }
6305 }
6306
6307 /// Exactly `n` signals (accumulated in the counter): the wait wakes
6308 /// early with `true`, well before a generous deadline.
6309 #[test]
6310 fn full_count_returns_true_early() {
6311 let fd = make_eventfd();
6312 signal(fd, 4); // n = 4
6313 let start = Instant::now();
6314 let ok = poll_eventfd_reaches(fd, Instant::now() + Duration::from_secs(5), 4);
6315 assert!(ok, "n signals must report all-signalled");
6316 assert!(
6317 start.elapsed() < Duration::from_secs(1),
6318 "did not wake promptly on the count: {:?}",
6319 start.elapsed(),
6320 );
6321 unsafe {
6322 libc::close(fd);
6323 }
6324 }
6325
6326 /// The count sums across SEPARATE reads: signals that arrive after the
6327 /// first drained read still accumulate toward `n`.
6328 #[test]
6329 fn count_accumulates_across_reads() {
6330 let fd = make_eventfd();
6331 signal(fd, 3);
6332 // First: 3 < 4, short deadline → false, draining the counter to 0.
6333 assert!(!poll_eventfd_reaches(
6334 fd,
6335 Instant::now() + Duration::from_millis(120),
6336 4
6337 ));
6338 // A single further signal added to the fresh (drained) counter is
6339 // only 1, so a second call still needs the FULL 4 — the accumulator
6340 // is per-call, matching one probe run. Confirm 4 fresh signals wake.
6341 signal(fd, 4);
6342 assert!(poll_eventfd_reaches(
6343 fd,
6344 Instant::now() + Duration::from_secs(5),
6345 4
6346 ));
6347 unsafe {
6348 libc::close(fd);
6349 }
6350 }
6351
6352 /// `need == 0` (no workers) returns `true` vacuously without polling.
6353 #[test]
6354 fn zero_need_is_vacuously_true() {
6355 let fd = make_eventfd();
6356 assert!(poll_eventfd_reaches(fd, Instant::now(), 0));
6357 unsafe {
6358 libc::close(fd);
6359 }
6360 }
6361}